diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..33fa04ed --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +version: 2 +updates: + - package-ecosystem: cargo + directory: / + schedule: + interval: daily + allow: + - dependency-name: "cellule-*" + groups: + cellule: + patterns: + - "cellule-*" + open-pull-requests-limit: 1 diff --git a/.github/workflows/qualify-release.yml b/.github/workflows/qualify-release.yml index 3236233d..0fa5dfeb 100644 --- a/.github/workflows/qualify-release.yml +++ b/.github/workflows/qualify-release.yml @@ -20,6 +20,8 @@ jobs: timeout-minutes: 60 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - name: Check Cellule release alignment + run: python3 scripts/check_cellule.py - name: Initialize qualification evidence before tool setup run: | mkdir qualification @@ -29,7 +31,7 @@ jobs: run: | rustup component add clippy rustfmt sudo apt-get update - sudo apt-get install -y git-lfs openssh-client + sudo apt-get install -y git-lfs openssh-client openssl aws --version git lfs install - name: Bind exact source and dependency inputs @@ -45,23 +47,25 @@ jobs: metadata = json.loads(Path('qualification/metadata.json').read_text()) manifest = tomllib.loads(Path('crates/canopy-server/Cargo.toml').read_text()) - revisions = {value['rev'] for value in manifest['dependencies'].values() - if isinstance(value, dict) and value.get('git') == 'https://github.com/crabbuild/cellule.git'} - assert len(revisions) == 1 - revision = revisions.pop() + pins = {value['version'] for name, value in manifest['dependencies'].items() + if name.startswith('cellule-')} + assert len(pins) == 1 and next(iter(pins)).startswith('=') + version = pins.pop()[1:] packages = {package['name']: package['source'] for package in metadata['packages'] if package['name'].startswith('cellule-')} assert set(packages) == {'cellule-app', 'cellule-host', 'cellule-ltx', 'cellule-runtime', 'cellule-store', 'cellule-types'} - assert all(source == f'git+https://github.com/crabbuild/cellule.git?rev={revision}#{revision}' + assert all(source == 'registry+https://github.com/rust-lang/crates.io-index' for source in packages.values()) + assert all(package['version'] == version for package in metadata['packages'] + if package['name'].startswith('cellule-')) files = subprocess.check_output(['git', 'ls-files', 'crates', 'scripts', 'Cargo.toml', 'Cargo.lock', '.github/workflows'], text=True).splitlines() image = json.loads(subprocess.check_output(['docker', 'image', 'inspect', 'rustfs/rustfs:1.0.0-beta.8-glibc'], text=True))[0] result = { 'head': subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip(), - 'cellule_revision': revision, + 'cellule_version': version, 'packages': packages, 'source_sha256': {name: hashlib.sha256(Path(name).read_bytes()).hexdigest() for name in files}, 'rustc': subprocess.check_output(['rustc', '-Vv'], text=True), @@ -80,11 +84,9 @@ jobs: cargo fmt --all -- --check cargo clippy --release --workspace --all-targets --locked -- -D warnings 2>&1 | tee qualification/release-clippy.log - name: Test release workspace - run: cargo test --release --workspace --locked -- --test-threads=4 2>&1 | tee qualification/release-workspace.log + run: cargo test --release --workspace --locked -- --test-threads=1 2>&1 | tee qualification/release-workspace.log - name: Check Python qualification harness run: python3 -B -m unittest discover -s scripts -p 'test_*.py' 2>&1 | tee qualification/python-harness.log - - name: Qualify release Git compatibility against RustFS - run: python3 scripts/qualify_size.py --provider-only --release 2>&1 | tee qualification/release-rustfs.log - name: Build and retain release executable run: | cargo build --release --locked --bin canopy 2>&1 | tee qualification/release-build.log @@ -98,13 +100,15 @@ jobs: for name, digest in inputs['source_sha256'].items()) binary = Path('target/release/canopy') digest = hashlib.sha256(binary.read_bytes()).hexdigest() - result = {'head': inputs['head'], 'cellule_revision': inputs['cellule_revision'], + result = {'head': inputs['head'], 'cellule_version': inputs['cellule_version'], 'binary_sha256': digest, 'binary_size_bytes': binary.stat().st_size, 'bound_source_unchanged': True, 'scope': inputs['scope']} Path('qualification/release-output.json').write_text(json.dumps(result, indent=2) + '\n') PY sha256sum target/release/canopy > qualification/canopy.sha256 tar -czf qualification/canopy-release-linux.tar.gz -C target/release canopy + - name: Qualify release Git compatibility and process recovery against RustFS + run: python3 scripts/qualify_size.py --provider-only --release --process-binary target/release/canopy 2>&1 | tee qualification/release-rustfs.log - name: Retain qualification evidence including failed attempts if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 9396a66b..e42b53ce 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -15,11 +15,13 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Check Cellule release alignment + run: python3 scripts/check_cellule.py - name: Install tools run: | rustup component add clippy rustfmt sudo apt-get update - sudo apt-get install -y git-lfs openssh-client + sudo apt-get install -y git-lfs openssh-client openssl git lfs install - name: Report tool versions run: | @@ -32,8 +34,8 @@ jobs: - name: Check lints run: cargo clippy --workspace --all-targets --locked -- -D warnings - name: Test - run: cargo test --workspace --locked - - name: Qualify Git compatibility against RustFS - run: python3 scripts/qualify_size.py --provider-only + run: cargo test --workspace --locked -- --test-threads=1 - name: Build server run: cargo build --locked --bin canopy + - name: Qualify Git compatibility and process recovery against RustFS + run: python3 scripts/qualify_size.py --provider-only --process-binary target/debug/canopy diff --git a/.gitignore b/.gitignore index 55897546..2707bbc3 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ /deploy/config.json /deploy/.proof-*/ + +__pycache__/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..62a2fed3 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,19 @@ +# Cellule integration + +- Keep Canopy on the latest published Cellule release. Update all direct + `cellule-*` exact version pins and `Cargo.lock` together, then adapt callers + to the current APIs and run the verification gates. +- Support only the compiled release and its current storage formats. Breaking + Cellule or Canopy schema changes use a fresh storage prefix. Do not add old + readers, retained module code, migration chains, compatibility adapters, or + mixed-version operation to preserve an earlier deployment. +- Keep release, identity, format, and fencing validation. Rejection of older + state is intentional. Same-release crash recovery, retries, backup/restore, + and standard Git/Git LFS protocol support remain required. +- Cellule's single schema-initialization `MigrationDescriptor` is required for + fresh Cells; it is not an upgrade path. Keep command codec identities aligned + with the current handlers; do not reset version numbers to hide a change. +- Run `python3 scripts/check_cellule.py`, formatting, Clippy, ordinary tests, + and the isolated provider/process qualification for dependency changes. Record + any qualification that was not run; retain historical evidence with its + original dependency version. diff --git a/Cargo.lock b/Cargo.lock index 3f73fc48..e12621df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -450,8 +450,9 @@ dependencies = [ [[package]] name = "cellule-app" -version = "0.1.0" -source = "git+https://github.com/crabbuild/cellule.git?rev=161067f5a21703b3e257024bcb64e565fd9657b4#161067f5a21703b3e257024bcb64e565fd9657b4" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b211ee2e9719b8c75906f0218d5621827526a67a959dd99fc54ad0e1a969c52e" dependencies = [ "blake3", "cellule-runtime", @@ -459,9 +460,11 @@ dependencies = [ [[package]] name = "cellule-host" -version = "0.1.0" -source = "git+https://github.com/crabbuild/cellule.git?rev=161067f5a21703b3e257024bcb64e565fd9657b4#161067f5a21703b3e257024bcb64e565fd9657b4" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c372232c1e30332d7468003d9efed396fbb67d4c79f2bef8d1efecb12ae1debf" dependencies = [ + "blake3", "cellule-app", "cellule-runtime", "futures-util", @@ -473,8 +476,9 @@ dependencies = [ [[package]] name = "cellule-ltx" -version = "0.1.0" -source = "git+https://github.com/crabbuild/cellule.git?rev=161067f5a21703b3e257024bcb64e565fd9657b4#161067f5a21703b3e257024bcb64e565fd9657b4" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59a5bcca2da813d30d048651212ab76bf678e7ad001a70df3b2997e0661179e7" dependencies = [ "async-trait", "blake3", @@ -495,8 +499,9 @@ dependencies = [ [[package]] name = "cellule-runtime" -version = "0.1.0" -source = "git+https://github.com/crabbuild/cellule.git?rev=161067f5a21703b3e257024bcb64e565fd9657b4#161067f5a21703b3e257024bcb64e565fd9657b4" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb3bc9e5375b4668b01454a73e7ca5ec8ac7efd4d8fd23e1b7064418466fba4d" dependencies = [ "blake3", "bytes", @@ -521,8 +526,9 @@ dependencies = [ [[package]] name = "cellule-store" -version = "0.1.0" -source = "git+https://github.com/crabbuild/cellule.git?rev=161067f5a21703b3e257024bcb64e565fd9657b4#161067f5a21703b3e257024bcb64e565fd9657b4" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29273d95baf35553f8512ccc0b72b88ad8f0c447de5c142c1e83d2048688a07f" dependencies = [ "async-trait", "blake3", @@ -544,8 +550,9 @@ dependencies = [ [[package]] name = "cellule-types" -version = "0.1.0" -source = "git+https://github.com/crabbuild/cellule.git?rev=161067f5a21703b3e257024bcb64e565fd9657b4#161067f5a21703b3e257024bcb64e565fd9657b4" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff52f9c6b2e11fb9065d77dced00b9b680c7c35617051b9a6b3b91dca6c8a996" dependencies = [ "schemars", "serde", diff --git a/README.md b/README.md index 2ab98bf7..00fdd4c3 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ This page keeps the project overview, setup path, and first repository workflow. | Browser | Embedded repository, issue, pull-request and account views | [Repository browser](docs/user-guide.md#repository-browser) | These are implementation capabilities, not a production capacity guarantee. -The [roadmap](ROADMAP.md) lists remaining work such as migrations, full fault +The [roadmap](ROADMAP.md) lists remaining work such as release validation, full fault qualification, collection, provider qualification, observability and a repeatable deployment. The [delivery plan](docs/delivery-plan.md) records release gates; the [performance plan](docs/performance-plan.md) separates measured results from @@ -49,8 +49,9 @@ before relying on a feature in a new environment. The **Directory Cell** maps an owner and repository name to a stable repository UUID and owns accounts. Each UUID identifies a **Repository Cell** containing -authoritative refs, Git metadata, ACLs and collaboration records. Large Git -blobs and LFS bodies are immutable external objects referenced by that Cell. +authoritative catalog/ref roots, ACLs and collaboration records. Native Git +packs, indexes, catalog metadata and LFS bodies are immutable external objects +referenced by the committed state. Native Git handles wire protocols using a **disposable cache**: after local disk loss, Canopy restores published Cell state from the object store and rebuilds the cache. See the [persisted contracts](docs/contracts.md) for publication and @@ -92,7 +93,8 @@ sequenceDiagram This is a development path for one node with an existing S3-compatible object store. The [bounded Linux deployment](deploy/README.md) describes the container profile and its resource limits. Use a **new storage prefix** for this build; -there is no migration from earlier preview layouts yet. +breaking schema or format changes require a fresh prefix. This build supports +only its compiled release and current storage formats. 1. Install Rust 1.97 or newer and Git with `http-backend`, `merge-tree --write-tree` (`-z --name-only --no-messages`) and `commit-tree`. diff --git a/ROADMAP.md b/ROADMAP.md index 37b0b2c8..aa740a0a 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,6 +1,6 @@ # Canopy roadmap -Canopy's Git-hosting core works, but the service is not yet ready for an unattended team deployment. This roadmap tracks the remaining work from the current `main` baseline (`ab43e71`, 2026-09-27). It is a planning checklist, not a claim that every capability mentioned below is absent. See [delivery gates](docs/delivery-plan.md) for the detailed acceptance record, [Git compatibility](docs/git-compatibility.md) for protocol coverage, and [performance evidence](docs/performance-plan.md) for measured limits. +Canopy implements the Git-hosting core, but the current native-storage build still needs end-to-end qualification before an unattended team deployment. This is a planning checklist, not a claim that every capability mentioned below is absent. See the [current storage qualification status](docs/benchmarks/2026-10-14-storage-optimization.md), [delivery gates](docs/delivery-plan.md), [Git compatibility](docs/git-compatibility.md), and [performance evidence](docs/performance-plan.md). ## How to use this roadmap @@ -13,14 +13,13 @@ Canopy's Git-hosting core works, but the service is not yet ready for an unatten This is the first intended production-use milestone. The existing bounded Compose profile is a starting point, not a completed install or capacity claim. -### R01. Safe upgrades and schema migration +### R01. Current-release deployment and format validation -- [ ] Version persistent Canopy and runtime formats; add a fenced, repeatable upgrade command and migration path for a populated deployment. -- [ ] Require a verified backup and compatible release/configuration before migration; document failed-upgrade recovery and rollback limits. -- [ ] Upgrade a prior release containing Git objects, LFS, ACLs, issues, checks, and pull requests; interrupt and retry each migration phase, then clone and inspect the restored state. -- [ ] Define whether mixed-version nodes are supported. Until proven, enforce a fully drained, single-version upgrade. +- [ ] Qualify release, identity, storage-format, schema and fencing rejection against the exact release candidate. +- [ ] Qualify fresh initialization and same-release crash recovery, retries, backup and restore with populated Git, LFS and collaboration state. +- [ ] Document a fully drained deployment using the latest published Cellule release and a fresh storage prefix for breaking Cellule or Canopy format changes. -**Current state:** schema changes still require a fresh storage prefix; maintenance admission and drain exist, but there is no upgrade path. +**Current state:** only the compiled release and its current formats are supported. Fresh-prefix deployment for breaking changes is intentional. Old readers, retained module code, migration chains, compatibility adapters and mixed-version operation are outside the supported contract. Cellule's single schema-initialization descriptor remains required for fresh Cells. ### R02. Backup, restore, and owner-recovery fault qualification @@ -37,7 +36,7 @@ This is the first intended production-use milestone. The existing bounded Compos - [ ] Add a fenced collector for abandoned uploads, old outcomes/candidates, and unreachable Git/LFS objects, with an inspectable dry run and bounded work. - [ ] Prove with concurrent reads, backup, publication, owner loss, and injected collector failures that required bytes are never removed; collect abandoned bytes only after their grace period. -**Current state:** no production collector; abandoned staging and outcomes can accumulate. Collection must remain conservative until its root proof passes. +**Current state:** `canopy gc` uses Cellule's retention-aware collector under a verified maintenance drain, backup pins, grace/deletion bounds and a durable worker claim. It collects superseded Cellule state. The [October 15 qualification](docs/benchmarks/2026-10-15-native-storage.md) records three full repositories before and after collection and fresh restore. Product-owned pack/catalog/LFS artifacts and abandoned staging still need their own root and retention policy; R03 remains open. ### R04. Trustworthy CI and release artifacts @@ -46,7 +45,7 @@ This is the first intended production-use milestone. The existing bounded Compos - [ ] Add required failure-path and provider gates to branch protection, then publish versioned, digest-identified images and a repeatable release checklist. - [ ] Record toolchain, Git, Git LFS, Cellule revision, image digest, provider, and test outcomes for each release candidate. -**Current state:** `.github/workflows/verify.yml` invokes the full size gate on `ubuntu-latest`; a previous hosted run exhausted disk. +**Current state:** standard CI runs formatting, Clippy, ordinary tests, Python harness checks and provider/process probes. The separate release workflow repeats correctness checks in release mode with provider/process probes. Both use `--provider-only`; the full size gate still needs a runner with sufficient capacity and recorded evidence for the exact release candidate. ### R05. Production object-store qualification @@ -99,7 +98,7 @@ This is the first intended production-use milestone. The existing bounded Compos ### Private team release exit checks - [ ] R01–R10 acceptance proof is recorded for the exact release candidate and selected object store. -- [ ] A clean install, upgrade of a populated prior release, backup, source-prefix loss, and restore all pass the operator runbook. +- [ ] A clean install, current-format rejection checks, same-release backup, source-prefix loss and restore all pass the operator runbook. Breaking formats use a fresh prefix. - [ ] Stock Git and Git LFS work after recovery; permissions, issues, reviews, checks, and pull requests retain their state. - [ ] The release notes state measured limits for active repositories, concurrent transfers, local disk, memory, restore time, and object-store cost. Unmeasured capacity is not advertised. diff --git a/crates/canopy-object-storage/src/artifact/tests.rs b/crates/canopy-object-storage/src/artifact/tests.rs index 3fe8ba9f..8d4ef713 100644 --- a/crates/canopy-object-storage/src/artifact/tests.rs +++ b/crates/canopy-object-storage/src/artifact/tests.rs @@ -11,6 +11,32 @@ fn key(body: &[u8]) -> ArtifactKey { } } +#[tokio::test] +async fn predecessor_manifest_is_rejected_even_with_a_matching_descriptor_digest() -> Result { + let store: Arc = Arc::new(InMemory::new()); + let artifacts = ArtifactStore::new(Arc::clone(&store), [1; 16]); + let body = b"current immutable artifact"; + let key = key(body); + let mut descriptor = artifacts + .put( + key, + body.len() as u64, + key.binding_digest, + &mut body.as_slice(), + ) + .await?; + let path = artifacts.path(key, descriptor.digest)?; + let mut manifest = store.get(&path).await?.bytes().await?.to_vec(); + manifest[..8].copy_from_slice(b"CANOPY01"); + descriptor.manifest_digest = *blake3::hash(&manifest).as_bytes(); + store.put(&path, manifest.into()).await?; + assert!(matches!( + artifacts.ranges_owned(key, descriptor, Arc::new(())).await, + Err(ArtifactError::Corrupt) + )); + Ok(()) +} + #[tokio::test] async fn selected_parts_authenticate_without_fetching_unrequested_bytes() -> Result { let store: Arc = Arc::new(InMemory::new()); diff --git a/crates/canopy-object-storage/src/blob/mod.rs b/crates/canopy-object-storage/src/blob/mod.rs deleted file mode 100644 index 640520d8..00000000 --- a/crates/canopy-object-storage/src/blob/mod.rs +++ /dev/null @@ -1,167 +0,0 @@ -//! Immutable external bytes for large Git blobs. - -use std::{sync::Arc, time::Duration}; - -use bytes::Bytes; -use canopy_git_format::{ObjectFormat, ObjectHasher, ObjectId, ObjectKind}; -use object_store::{ObjectStore, path::Path}; -use sha2::{Digest, Sha256}; -use tokio::io::{AsyncRead, AsyncReadExt}; - -mod read; -pub use read::LargeBlobRead; - -const CHUNK_BYTES: usize = crate::external::PART_BYTES; -const IO_TIMEOUT: Duration = Duration::from_secs(120); - -#[derive(Clone, Copy, Debug)] -pub struct LargeBlobReference { - pub oid: ObjectId, - pub size: u64, - pub blake3: [u8; 32], - pub sha256: [u8; 32], -} - -#[derive(Debug, thiserror::Error)] -pub enum LargeBlobError { - #[error("large blob store failed")] - Store(#[from] object_store::Error), - #[error("large blob input failed")] - Io(#[from] std::io::Error), - #[error("large blob worker failed")] - Task(#[from] tokio::task::JoinError), - #[error("large blob transfer timed out")] - Timeout, - #[error("large blob bytes disagree with their SQLite reference")] - Corrupt, -} - -struct Hashes { - oid: ObjectHasher, - sha256: Sha256, - blake3: blake3::Hasher, -} - -impl Hashes { - fn new(format: ObjectFormat, size: u64) -> Self { - let oid = ObjectHasher::new(format, ObjectKind::Blob, size); - Self { - oid, - sha256: Sha256::new(), - blake3: blake3::Hasher::new(), - } - } - - async fn update(mut self, bytes: Bytes) -> Result<(Self, Bytes), LargeBlobError> { - Ok(tokio::task::spawn_blocking(move || { - self.oid.update(&bytes); - self.sha256.update(&bytes); - self.blake3.update(&bytes); - (self, bytes) - }) - .await?) - } - - fn finish(self, size: u64) -> LargeBlobReference { - LargeBlobReference { - oid: self.oid.finalize(), - size, - sha256: self.sha256.finalize().into(), - blake3: *self.blake3.finalize().as_bytes(), - } - } -} - -/// Repository-scoped content-addressed body store. -pub struct LargeBlobStore { - store: Arc, - repository_id: [u8; 16], -} - -impl LargeBlobStore { - pub fn new(store: Arc, repository_id: [u8; 16]) -> Self { - Self { - store, - repository_id, - } - } - - /// Streams exactly `size` bytes, verifying the Git OID before immutable publication. - /// The caller retains this future through cleanup; abrupt process loss may leave staging. - pub async fn put( - &self, - oid: ObjectId, - size: u64, - input: &mut (impl AsyncRead + Unpin), - ) -> Result { - let stage = Path::from(format!( - "repos/{}/git-blob-staging/{}", - hex::encode(self.repository_id), - uuid::Uuid::new_v4() - )); - let mut upload = crate::external::Upload::new(self.store.clone(), stage).await?; - let result = async { - let mut hashes = Hashes::new(oid.format(), size); - let mut remaining = size; - loop { - let length = remaining.min(CHUNK_BYTES as u64) as usize; - let mut bytes = vec![0; length]; - tokio::time::timeout(IO_TIMEOUT, input.read_exact(&mut bytes)) - .await - .map_err(|_| LargeBlobError::Timeout)??; - let (updated, bytes) = hashes.update(Bytes::from(bytes)).await?; - hashes = updated; - upload.write(bytes).await?; - remaining -= length as u64; - if remaining == 0 { - break; - } - } - let reference = hashes.finish(size); - if reference.oid != oid { - return Err(LargeBlobError::Corrupt); - } - upload - .publish(&blob_path(self.repository_id, &reference.sha256), size) - .await?; - self.verify(&reference).await?; - Ok(reference) - } - .await; - let cleanup = upload.cleanup().await; - if result.is_err() - && let Err(error) = &cleanup - { - tracing::warn!(error = %error, "Git blob staging cleanup failed"); - } - let reference = result?; - cleanup?; - Ok(reference) - } - - /// Opens a bounded reader that validates all three hashes before returning its final range. - pub async fn read( - &self, - reference: &LargeBlobReference, - ) -> Result { - LargeBlobRead::open(self.store.clone(), self.repository_id, *reference).await - } - - /// Checks an immutable object's bytes without materializing the full body. - pub async fn verify(&self, reference: &LargeBlobReference) -> Result<(), LargeBlobError> { - let mut read = self.read(reference).await?; - while read.next().await?.is_some() {} - Ok(()) - } -} - -pub fn blob_path(repository_id: [u8; 16], sha256: &[u8; 32]) -> Path { - Path::from(format!( - "repos/{}/git-blobs/{}", - hex::encode(repository_id), - hex::encode(sha256) - )) -} - -#[cfg(test)] -mod tests; diff --git a/crates/canopy-object-storage/src/blob/read.rs b/crates/canopy-object-storage/src/blob/read.rs deleted file mode 100644 index 4b1f2460..00000000 --- a/crates/canopy-object-storage/src/blob/read.rs +++ /dev/null @@ -1,82 +0,0 @@ -use super::*; - -/// Sequential bounded ranges from one immutable Git blob. -pub struct LargeBlobRead { - store: Arc, - path: Path, - manifest: crate::external::Manifest, - reference: LargeBlobReference, - offset: u64, - hashes: Option, -} - -impl LargeBlobRead { - pub(super) async fn open( - store: Arc, - repository_id: [u8; 16], - reference: LargeBlobReference, - ) -> Result { - let path = blob_path(repository_id, &reference.sha256); - let manifest = crate::external::open(store.as_ref(), &path, reference.size).await?; - let read = Self { - store, - path, - manifest, - reference, - offset: 0, - hashes: Some(Hashes::new(reference.oid.format(), reference.size)), - }; - if reference.size == 0 { - read.check(Hashes::new(reference.oid.format(), 0))?; - } - Ok(read) - } - - pub fn reference(&self) -> LargeBlobReference { - self.reference - } - - /// Returns at most 8 MiB; an error or canceled read invalidates the reader. - pub async fn next(&mut self) -> Result, LargeBlobError> { - if self.offset == self.reference.size { - return Ok(None); - } - // Taking the verifier poisons cancellation/error paths. A partially read - // range cannot be silently retried with a truncated checksum history. - let hashes = self.hashes.take().ok_or(LargeBlobError::Corrupt)?; - let bytes = tokio::time::timeout(IO_TIMEOUT, self.range()) - .await - .map_err(|_| LargeBlobError::Timeout)??; - let (hashes, bytes) = hashes.update(bytes).await?; - let end = self.offset + bytes.len() as u64; - if end == self.reference.size { - self.check(hashes)?; - } else { - self.hashes = Some(hashes); - } - self.offset = end; - Ok(Some(bytes)) - } - - async fn range(&self) -> Result { - Ok(crate::external::read( - self.store.as_ref(), - &self.path, - &self.manifest, - self.reference.size, - self.offset, - ) - .await?) - } - - fn check(&self, hashes: Hashes) -> Result<(), LargeBlobError> { - let actual = hashes.finish(self.reference.size); - if actual.oid != self.reference.oid - || actual.sha256 != self.reference.sha256 - || actual.blake3 != self.reference.blake3 - { - return Err(LargeBlobError::Corrupt); - } - Ok(()) - } -} diff --git a/crates/canopy-object-storage/src/blob/tests.rs b/crates/canopy-object-storage/src/blob/tests.rs deleted file mode 100644 index 1517e773..00000000 --- a/crates/canopy-object-storage/src/blob/tests.rs +++ /dev/null @@ -1,165 +0,0 @@ -use super::*; -use canopy_git_format::{ObjectFormat, ObjectId, ObjectKind, object_id}; -use object_store::{ObjectStoreExt, memory::InMemory}; -use std::future::poll_fn; - -type Result = std::result::Result>; - -#[tokio::test] -async fn streamed_objects_verify_all_hashes_without_consuming_the_next_frame() -> Result { - let store: Arc = Arc::new(InMemory::new()); - let blobs = LargeBlobStore::new(store.clone(), [1; 16]); - for (format, size) in [ObjectFormat::Sha1, ObjectFormat::Sha256] - .into_iter() - .flat_map(|format| [0, 1, CHUNK_BYTES, CHUNK_BYTES + 1].map(move |size| (format, size))) - { - let body = vec![21; size]; - let mut input = body.clone(); - input.extend_from_slice(b"\nnext frame"); - let mut input = input.as_slice(); - let oid = object_id(format, ObjectKind::Blob, &body); - let reference = blobs.put(oid, size as u64, &mut input).await?; - assert_eq!(input, b"\nnext frame"); - let replay = blobs.put(oid, size as u64, &mut body.as_slice()).await?; - assert_eq!(reference.sha256, replay.sha256); - let mut reader = blobs.read(&reference).await?; - let mut bytes_read = 0; - while let Some(bytes) = reader.next().await? { - assert!(bytes.len() <= CHUNK_BYTES); - assert_eq!(bytes.as_ref(), &body[bytes_read..bytes_read + bytes.len()]); - bytes_read += bytes.len(); - } - assert_eq!(bytes_read, size); - } - let mut objects = store.list(None); - while let Some(object) = poll_fn(|cx| objects.as_mut().poll_next(cx)).await { - assert!(!object?.location.as_ref().contains("staging")); - } - Ok(()) -} - -#[tokio::test] -async fn bad_input_does_not_publish_and_existing_corruption_is_not_overwritten() -> Result { - let store: Arc = Arc::new(InMemory::new()); - let blobs = LargeBlobStore::new(store.clone(), [2; 16]); - let body = b"immutable blob"; - let oid = object_id(ObjectFormat::Sha1, ObjectKind::Blob, body); - assert!(matches!( - blobs - .put( - ObjectId::Sha1([0; 20]), - body.len() as u64, - &mut body.as_slice() - ) - .await, - Err(LargeBlobError::Corrupt) - )); - assert!( - blobs - .put(oid, body.len() as u64 + 1, &mut body.as_slice()) - .await - .is_err() - ); - let listing = store.list_with_delimiter(None).await?; - assert!(listing.objects.is_empty() && listing.common_prefixes.is_empty()); - let reference = blobs - .put(oid, body.len() as u64, &mut body.as_slice()) - .await?; - let path = crate::external::part(&blob_path([2; 16], &reference.sha256), 0); - store - .put(&path, Bytes::from(vec![0; body.len()]).into()) - .await?; - let mut reader = blobs.read(&reference).await?; - assert!(matches!(reader.next().await, Err(LargeBlobError::Corrupt))); - assert!(matches!(reader.next().await, Err(LargeBlobError::Corrupt))); - assert!(matches!( - blobs - .put(oid, body.len() as u64, &mut body.as_slice()) - .await, - Err(LargeBlobError::Corrupt) - )); - assert_eq!( - store.get(&path).await?.bytes().await?.as_ref(), - vec![0; body.len()] - ); - Ok(()) -} - -#[tokio::test] -async fn metadata_hash_corruption_withholds_the_final_range() -> Result { - let store: Arc = Arc::new(InMemory::new()); - let blobs = LargeBlobStore::new(store.clone(), [3; 16]); - let body = vec![42; CHUNK_BYTES + 17]; - let reference = blobs - .put( - object_id(ObjectFormat::Sha1, ObjectKind::Blob, &body), - body.len() as u64, - &mut body.as_slice(), - ) - .await?; - crate::external::copy_parts( - store.as_ref(), - &blob_path([3; 16], &reference.sha256), - &blob_path([3; 16], &[0; 32]), - reference.size, - ) - .await?; - store - .copy( - &blob_path([3; 16], &reference.sha256), - &blob_path([3; 16], &[0; 32]), - ) - .await?; - for bad in [ - LargeBlobReference { - sha256: [0; 32], - ..reference - }, - LargeBlobReference { - oid: ObjectId::Sha1([0; 20]), - ..reference - }, - LargeBlobReference { - blake3: [0; 32], - ..reference - }, - ] { - let mut reader = blobs.read(&bad).await?; - assert_eq!( - reader.next().await?.ok_or("first range")?.len(), - CHUNK_BYTES - ); - assert!(matches!(reader.next().await, Err(LargeBlobError::Corrupt))); - } - Ok(()) -} - -#[tokio::test] -async fn replacement_between_ranges_invalidates_the_reader() -> Result { - let store: Arc = Arc::new(InMemory::new()); - let blobs = LargeBlobStore::new(store.clone(), [4; 16]); - let body = vec![42; CHUNK_BYTES + 17]; - let reference = blobs - .put( - object_id(ObjectFormat::Sha1, ObjectKind::Blob, &body), - body.len() as u64, - &mut body.as_slice(), - ) - .await?; - let mut reader = blobs.read(&reference).await?; - reader.next().await?.ok_or("first range")?; - store - .put( - &blob_path([4; 16], &reference.sha256), - Bytes::from(vec![21; body.len()]).into(), - ) - .await?; - assert!(matches!( - reader.next().await, - Err(LargeBlobError::Store( - object_store::Error::Precondition { .. } - )) - )); - assert!(matches!(reader.next().await, Err(LargeBlobError::Corrupt))); - Ok(()) -} diff --git a/crates/canopy-object-storage/src/external.rs b/crates/canopy-object-storage/src/external.rs index 07f06679..9523fc5c 100644 --- a/crates/canopy-object-storage/src/external.rs +++ b/crates/canopy-object-storage/src/external.rs @@ -1,4 +1,4 @@ -//! Immutable, fixed-size parts for Git blob and LFS bodies. +//! Immutable, verified parts for native artifacts and LFS bodies. use bytes::{Bytes, BytesMut}; use object_store::{ @@ -11,7 +11,6 @@ use std::{future::Future, sync::Arc, time::Duration}; pub const PART_BYTES: usize = 8 * 1024 * 1024; pub const MAX_PARTS: u64 = 65_536; pub const MAX_ARTIFACT_BYTES: u64 = PART_BYTES as u64 * MAX_PARTS; -const MAGIC: &[u8; 8] = b"CANOPY01"; const HASHED_MAGIC: &[u8; 8] = b"CANOPY02"; pub struct Manifest { @@ -107,12 +106,6 @@ impl Upload { Ok(()) } - pub async fn publish(&mut self, path: &Path, size: u64) -> object_store::Result<()> { - let mut manifest = MAGIC.to_vec(); - manifest.extend_from_slice(&size.to_le_bytes()); - self.publish_manifest(path, size, manifest).await - } - pub async fn publish_hashed( &mut self, path: &Path, @@ -161,19 +154,6 @@ impl Upload { Ok(root) } - async fn publish_manifest( - &mut self, - path: &Path, - size: u64, - manifest: Vec, - ) -> object_store::Result<()> { - if self.active.is_some() || self.parts != size.div_ceil(PART_BYTES as u64).max(1) { - return Err(invalid()); - } - copy_parts(self.store.as_ref(), &self.stage, path, size).await?; - self.create_manifest(path, manifest).await - } - async fn create_manifest( &mut self, path: &Path, @@ -271,21 +251,6 @@ async fn bounded( .await } -pub async fn open( - store: &dyn ObjectStore, - path: &Path, - size: u64, -) -> object_store::Result { - let (meta, bytes) = bounded(store, path, GetOptions::default(), 16).await?; - if &bytes[..8] != MAGIC || bytes[8..] != size.to_le_bytes() { - return Err(invalid()); - } - if size == 0 { - bounded(store, &part(path, 0), GetOptions::default(), 0).await?; - } - Ok(Manifest { meta, bytes }) -} - pub async fn open_hashed( store: &dyn ObjectStore, path: &Path, diff --git a/crates/canopy-object-storage/src/lib.rs b/crates/canopy-object-storage/src/lib.rs index 279aeb66..d8293de4 100644 --- a/crates/canopy-object-storage/src/lib.rs +++ b/crates/canopy-object-storage/src/lib.rs @@ -1,7 +1,4 @@ -//! Immutable object parts and content-addressed large Git blobs. +//! Immutable native artifacts and object parts for Git/LFS. pub mod artifact; -pub mod blob; pub mod external; - -pub use blob::{LargeBlobError, LargeBlobRead, LargeBlobReference, LargeBlobStore}; diff --git a/crates/canopy-server/Cargo.toml b/crates/canopy-server/Cargo.toml index 6fb4d336..f34ad938 100644 --- a/crates/canopy-server/Cargo.toml +++ b/crates/canopy-server/Cargo.toml @@ -2,6 +2,7 @@ name = "canopy-server" version = "0.1.0" edition = "2024" +rust-version = "1.97" license = "Apache-2.0" [[bin]] @@ -16,11 +17,11 @@ axum = "0.8.9" base64 = "0.22" blake3 = "1.8" bytes = "1.11" -cellule-app = { git = "https://github.com/crabbuild/cellule.git", rev = "161067f5a21703b3e257024bcb64e565fd9657b4" } -cellule-host = { git = "https://github.com/crabbuild/cellule.git", rev = "161067f5a21703b3e257024bcb64e565fd9657b4" } -cellule-ltx = { git = "https://github.com/crabbuild/cellule.git", rev = "161067f5a21703b3e257024bcb64e565fd9657b4", features = ["replica"] } -cellule-runtime = { git = "https://github.com/crabbuild/cellule.git", rev = "161067f5a21703b3e257024bcb64e565fd9657b4" } -cellule-store = { git = "https://github.com/crabbuild/cellule.git", rev = "161067f5a21703b3e257024bcb64e565fd9657b4" } +cellule-app = { version = "=0.2.0" } +cellule-host = { version = "=0.2.0" } +cellule-ltx = { version = "=0.2.0", features = ["replica"] } +cellule-runtime = { version = "=0.2.0" } +cellule-store = { version = "=0.2.0" } ed25519-dalek = "2" flate2 = "1.1" futures-core = "0.3" diff --git a/crates/canopy-server/src/ancestry.rs b/crates/canopy-server/src/ancestry.rs deleted file mode 100644 index 6166de44..00000000 --- a/crates/canopy-server/src/ancestry.rs +++ /dev/null @@ -1,201 +0,0 @@ -//! Bounded certificates over parent links extracted from verified commit objects. - -use crate::{RepositoryCell, RepositoryModule, server::mutation_identity}; -use cellule_runtime::{ - CellModule, Command, Error, InvocationError, codec::BoundedDecoder, codec::BoundedEncoder, - codec::CodecError, codec::WireValue, primitives::sql::SqlBatch, primitives::sql::SqlStatement, - primitives::sql::SqlValue, registry::CommandContext, registry::CommandResult, -}; -use std::collections::{HashMap, hash_map::Entry}; - -type Oid = crate::ObjectId; -const PAGE: usize = 128; - -pub(crate) struct AncestryProof { - ancestor: Oid, - steps: Vec<(Oid, Oid)>, -} - -impl WireValue for AncestryProof { - fn encode(&self, out: &mut BoundedEncoder) -> Result<(), CodecError> { - if self.steps.is_empty() || self.steps.len() > PAGE { - return Err(CodecError::Invalid("ancestry proof size")); - } - out.write_bytes(&self.ancestor)?; - out.write_count(self.steps.len())?; - for (child, parent) in &self.steps { - out.write_bytes(child)?; - out.write_bytes(parent)?; - } - Ok(()) - } - fn decode(input: &mut BoundedDecoder<'_>) -> Result { - let ancestor = oid(input)?; - let count = input.read_count()?; - if count == 0 || count > PAGE { - return Err(CodecError::Invalid("ancestry proof size")); - } - let mut steps = Vec::with_capacity(count); - for _ in 0..count { - steps.push((oid(input)?, oid(input)?)); - } - Ok(Self { ancestor, steps }) - } -} -fn oid(input: &mut BoundedDecoder<'_>) -> Result { - input - .read_bytes()? - .try_into() - .map_err(|_| CodecError::Invalid("ancestry OID width")) -} - -pub(crate) struct CertifyAncestry; -impl Command for CertifyAncestry { - const MODULE: &'static str = RepositoryModule::NAME; - const ID: u32 = 7; - const CODEC_VERSION: u32 = 2; - type Input = AncestryProof; - type Output = bool; - fn execute( - context: &mut CommandContext<'_, '_>, - proof: AncestryProof, - ) -> cellule_runtime::Result> { - for (child, parent) in proof.steps { - let result = context.sql(&SqlBatch { statements: vec![SqlStatement { - sql: "SELECT EXISTS (SELECT 1 FROM commit_parents WHERE child = ?1 AND parent = ?2) AND (?2 = ?3 OR EXISTS (SELECT 1 FROM commit_ancestry WHERE ancestor = ?3 AND descendant = ?2))".into(), - parameters: vec![SqlValue::Blob(child.to_vec()), SqlValue::Blob(parent.to_vec()), SqlValue::Blob(proof.ancestor.to_vec())], - }] })?; - if !matches!( - result - .first() - .and_then(|set| set.rows.first()) - .map(Vec::as_slice), - Some([SqlValue::Integer(1)]) - ) { - return Ok(CommandResult::Rejected(false)); - } - context.sql(&SqlBatch { statements: vec![SqlStatement { - sql: "INSERT INTO commit_ancestry (ancestor, descendant) VALUES (?1, ?2) ON CONFLICT DO NOTHING".into(), - parameters: vec![SqlValue::Blob(proof.ancestor.to_vec()), SqlValue::Blob(child.to_vec())], - }] })?; - } - Ok(CommandResult::Success(true)) - } -} - -impl RepositoryCell { - pub(crate) async fn prepare_ancestry( - &self, - ancestor: Oid, - descendant: Oid, - ) -> Result<(), InvocationError> { - self.ancestry_path(ancestor, descendant) - .await - .map_err(|source| { - InvocationError::NotStarted(Error::Facility { - name: "commit ancestry preparation", - source, - }) - }) - } - - async fn ancestry_path( - &self, - ancestor: Oid, - descendant: Oid, - ) -> Result<(), Box> { - if ancestor == descendant { - return Ok(()); - } - let known = self - .sql - .query( - None, - SqlBatch { - statements: vec![SqlStatement { - sql: - "SELECT 1 FROM commit_ancestry WHERE ancestor = ?1 AND descendant = ?2" - .into(), - parameters: vec![ - SqlValue::Blob(ancestor.to_vec()), - SqlValue::Blob(descendant.to_vec()), - ], - }], - }, - ) - .await?; - if known.output.first().is_some_and(|set| !set.rows.is_empty()) { - return Ok(()); - } - let mut previous: HashMap> = HashMap::from([(descendant, None)]); - let mut pending = vec![(descendant, None::)]; - let mut edges = 0; - while let Some((child, after)) = pending.pop() { - let result = self.sql.query(None, SqlBatch { statements: vec![SqlStatement { - sql: format!("SELECT parent, EXISTS (SELECT 1 FROM commit_ancestry WHERE ancestor = ?3 AND descendant = p.parent) FROM commit_parents p WHERE child = ?1 AND parent > ?2 ORDER BY parent LIMIT {PAGE}"), - parameters: vec![SqlValue::Blob(child.to_vec()), SqlValue::Blob(after.map_or_else(Vec::new, |oid| oid.to_vec())), SqlValue::Blob(ancestor.to_vec())], - }] }).await?; - let rows = &result - .output - .first() - .ok_or(Error::Command("missing commit parents"))? - .rows; - edges += rows.len(); - if edges > 250_000 { - return Err(Error::Command("commit ancestry traversal limit").into()); - } - let mut parents = Vec::with_capacity(rows.len()); - for row in rows { - let [SqlValue::Blob(parent), SqlValue::Integer(known)] = row.as_slice() else { - return Err(Error::Command("invalid commit parent").into()); - }; - let parent: Oid = parent - .as_slice() - .try_into() - .map_err(|_| Error::Command("invalid parent OID"))?; - parents.push(parent); - let Entry::Vacant(entry) = previous.entry(parent) else { - continue; - }; - entry.insert(Some(child)); - if previous.len() > 100_000 { - return Err(Error::Command("commit ancestry traversal limit").into()); - } - if parent == ancestor || *known == 1 { - let mut steps = Vec::new(); - let mut parent = parent; - while let Some(Some(child)) = previous.get(&parent).copied() { - steps.push((child, parent)); - parent = child; - } - // Each bounded command proves its chosen links itself. Partial - // preparation retains only immutable facts, never ref authority. - for steps in steps.chunks(PAGE) { - self.application - .command::( - &self.target, - mutation_identity()?, - AncestryProof { - ancestor, - steps: steps.to_vec(), - }, - ) - .await?; - } - return Ok(()); - } - } - if parents.len() == PAGE { - pending.push((child, parents.last().copied())); - } - for parent in parents.into_iter().rev() { - if previous.get(&parent) == Some(&Some(child)) { - pending.push((parent, None)); - } - } - } - // A non-ancestor has no positive certificate. Final publication decides - // against the current rule; this search never certifies a negative result. - Ok(()) - } -} diff --git a/crates/canopy-server/src/branch_rules/mod.rs b/crates/canopy-server/src/branch_rules/mod.rs index b4a21490..a6d8bd3c 100644 --- a/crates/canopy-server/src/branch_rules/mod.rs +++ b/crates/canopy-server/src/branch_rules/mod.rs @@ -5,7 +5,7 @@ use crate::ReadIdentity; pub(crate) mod command; use crate::{ - PushPlan, RefUpdate, RepositoryCell, + RefUpdate, RepositoryCell, default_branch::valid_default_branch, directory::{TokenScope, validate_component}, }; @@ -197,52 +197,6 @@ impl RepositoryCell { .collect::>>() .map_err(InvocationError::NotStarted) } - - pub(crate) async fn prepare_branch_proofs( - &self, - plan: &PushPlan, - ) -> Result<(), InvocationError> { - if plan.updates.is_empty() || plan.updates.len() > crate::refs::MAX_UPDATES { - return Ok(()); - } - let role = self - .access_level(&plan.actor, None) - .await - .map_err(|source| { - InvocationError::NotStarted(Error::Facility { - name: "branch proof authorization", - source: Box::new(source), - }) - })?; - if !role.output.is_some_and(|role| role >= TokenScope::Write) { - return Ok(()); - } - for updates in plan.updates.chunks(128) { - let policies = self.branch_policies(updates).await.map_err(|source| { - InvocationError::NotStarted(Error::Facility { - name: "branch proof policy", - source: Box::new(source), - }) - })?; - for (update, policy) in updates.iter().zip(policies) { - let (Some(old), Some(new)) = ( - update.expected.as_ref().and_then(|old| old.oid), - update.new_oid, - ) else { - continue; - }; - if policy.is_some_and(|policy| { - !policy.require_pull_request - && policy.fast_forward_only - && policy.checks_pass - && !policy.ancestry - }) { - self.prepare_ancestry(old, new).await?; - } - } - } - Ok(()) - } } pub(crate) struct Policy { @@ -271,53 +225,19 @@ impl Policy { } } -pub(crate) fn policies_allow( - context: &cellule_runtime::registry::CommandContext<'_, '_>, - plan: &PushPlan, - merge: Option<&crate::pulls::merge::ReviewedMerge>, -) -> cellule_runtime::Result { - for updates in plan.updates.chunks(128) { - let result = context.sql(&SqlBatch { - statements: updates.iter().map(policy_statement).collect(), - })?; - for (update, set) in updates.iter().zip(&result) { - if decode_policy(std::slice::from_ref(set))?.is_some_and(|policy| { - !policy.allows_ref(update, true) - || (policy.require_pull_request - && !merge.is_some_and(|merge| merge.authorizes(update))) - }) { - return Ok(false); - } - } - } - Ok(true) -} - -fn policy_statement(update: &RefUpdate) -> SqlStatement { - policy_statement_for(update, None) -} /// The final publisher supplies only MAC-verified catalog ancestry evidence. /// Current check-context versions/reporters and branch rules stay in this SQL. pub(crate) fn policy_statement_with_ancestry(update: &RefUpdate, ancestry: bool) -> SqlStatement { - policy_statement_for(update, Some(ancestry)) -} -fn policy_statement_for(update: &RefUpdate, ancestry: Option) -> SqlStatement { let old = update.expected.as_ref().and_then(|old| old.oid); - let ancestry_sql = if ancestry.is_some() { - "?4" - } else { - "(?2 IS NULL OR coalesce(?2 = ?3, 0) OR EXISTS (SELECT 1 FROM commit_ancestry WHERE ancestor = ?2 AND descendant = ?3))" - }; - let mut parameters = vec![ + let ancestry_sql = "?4"; + let parameters = vec![ SqlValue::Text(update.name.clone()), old.map_or(SqlValue::Null, |oid| SqlValue::Blob(oid.to_vec())), update .new_oid .map_or(SqlValue::Null, |oid| SqlValue::Blob(oid.to_vec())), + SqlValue::Integer(i64::from(ancestry)), ]; - if let Some(value) = ancestry { - parameters.push(SqlValue::Integer(i64::from(value))); - } SqlStatement { sql: format!( "SELECT b.deny_deletions, b.fast_forward, NOT EXISTS (SELECT 1 FROM branch_required_checks q LEFT JOIN check_contexts c ON c.name = q.context LEFT JOIN check_runs r ON r.number = (SELECT number FROM check_runs WHERE oid = ?3 AND context = q.context AND context_version = c.version ORDER BY number DESC LIMIT 1) WHERE q.reference = b.reference AND (c.enabled IS NOT 1 OR r.state IS NOT 'success' OR r.reporter IS NOT c.reporter)), {ancestry_sql}, b.require_pull_request FROM branch_rules b WHERE b.reference = ?1 AND b.enabled = 1" diff --git a/crates/canopy-server/src/deployment/backup/mod.rs b/crates/canopy-server/src/deployment/backup/mod.rs index 76dd6ace..689d7ca3 100644 --- a/crates/canopy-server/src/deployment/backup/mod.rs +++ b/crates/canopy-server/src/deployment/backup/mod.rs @@ -3,7 +3,6 @@ use super::*; use crate::{ REPOSITORIES, REPOSITORY_DATABASE_LIMIT_BYTES, - blob::LargeBlobError, lfs::LfsError, server::{ServerError, unix_now_ms}, }; @@ -33,8 +32,6 @@ pub enum BackupError { Store(#[from] StorageError), #[error("backup external part copy failed")] External(#[from] object_store::Error), - #[error("backup Git blob verification failed")] - Blob(#[from] LargeBlobError), #[error("backup LFS verification failed")] Lfs(#[from] LfsError), #[error("backup SQLite read failed")] diff --git a/crates/canopy-server/src/deployment/maintenance.rs b/crates/canopy-server/src/deployment/maintenance.rs new file mode 100644 index 00000000..9b72c183 --- /dev/null +++ b/crates/canopy-server/src/deployment/maintenance.rs @@ -0,0 +1,165 @@ +//! Persistent exclusion for offline administration, including interrupted sweeps. + +use super::*; +use bytes::Bytes; +use cellule_store::{ETag, StorageError}; +use serde::Deserialize; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(super) enum Action { + Collect, + Recover, + End, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Worker { + node: String, + key: String, + workspace: String, +} + +impl Worker { + pub(super) fn new(config: &WorkerConfig, workspace: &std::path::Path) -> Result { + Ok(Self { + node: hex::encode(config.node.as_bytes()), + key: hex::encode(config.signing_key.verifying_key().as_bytes()), + workspace: workspace + .to_str() + .ok_or(Error::Retention("maintenance workspace is not UTF-8"))? + .into(), + }) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "state", rename_all = "snake_case", deny_unknown_fields)] +enum State { + Idle, + Active { + operation: String, + action: Action, + worker: Option, + run: String, + }, +} + +pub(super) struct Claim { + path: Path, + state: State, + token: ETag, +} + +fn path(deployment: &Deployment) -> Path { + deployment.prefix.clone().join("canopy-maintenance-v1.json") +} + +async fn load(deployment: &Deployment) -> Result> { + match deployment + .layout + .store() + .get_with_etag_bounded(&path(deployment), 4096) + .await + { + Ok((body, token)) => Ok(Some((serde_json::from_slice(&body)?, token))), + Err(StorageError::NotFound { .. }) => Ok(None), + Err(error) => Err(error.into()), + } +} + +pub(super) async fn require_idle(deployment: &Deployment) -> Result<()> { + if load(deployment) + .await? + .is_some_and(|(state, _)| state != State::Idle) + { + return Err(Error::Retention( + "an offline maintenance worker has not settled", + )); + } + Ok(()) +} + +/// Worker adoption is permitted only while holding the original local workspace +/// lock. There is deliberately no timeout takeover of a collector: a paused +/// delete must never overlap reopened admission, even after a node lease expires. +pub(super) async fn acquire( + deployment: &Deployment, + operation: RequestId, + action: Action, + worker: Option, +) -> Result { + let operation = hex::encode(operation.as_bytes()); + let previous = load(deployment).await?; + if let Some(( + State::Active { + operation: old_operation, + action: old_action, + worker: old_worker, + .. + }, + _, + )) = &previous + && (*old_operation != operation || *old_action != action || *old_worker != worker) + { + return Err(Error::Retention( + "another offline maintenance worker owns the deployment", + )); + } + // End is an idempotent release CAS. Collect/recover callers prove the + // old process closed by acquiring its same canonical workspace first. + let state = State::Active { + operation, + action, + worker, + run: uuid::Uuid::new_v4().to_string(), + }; + let bytes = serde_json::to_vec(&state)?; + if bytes.len() > 4096 { + return Err(Error::Retention("maintenance claim exceeds size limit")); + } + let path = path(deployment); + let token = match previous { + Some((_, token)) => { + deployment + .layout + .store() + .update(&path, Bytes::from(bytes), token) + .await? + } + None => { + deployment + .layout + .store() + .create_strict_with_etag(&path, Bytes::from(bytes)) + .await? + } + }; + Ok(Claim { path, state, token }) +} + +impl Claim { + pub(super) async fn require_current(&self, deployment: &Deployment) -> Result<()> { + if !load(deployment) + .await? + .is_some_and(|(state, token)| state == self.state && token == self.token) + { + return Err(Error::Retention("maintenance worker claim changed")); + } + Ok(()) + } + + pub(super) async fn finish(self, deployment: &Deployment) -> Result<()> { + deployment + .layout + .store() + .update( + &self.path, + Bytes::from(serde_json::to_vec(&State::Idle)?), + self.token, + ) + .await?; + Ok(()) + } +} diff --git a/crates/canopy-server/src/deployment/mod.rs b/crates/canopy-server/src/deployment/mod.rs index ec5282a8..9487190b 100644 --- a/crates/canopy-server/src/deployment/mod.rs +++ b/crates/canopy-server/src/deployment/mod.rs @@ -15,12 +15,15 @@ use serde::Serialize; mod backup; pub use backup::{BackupError, BackupReport}; +mod maintenance; mod recovery; +mod retention; mod root; pub use recovery::WorkerConfig; +pub use retention::GarbageCollectionReport; /// Incompatible repository deployment and local cache format. -pub const STORAGE_FORMAT: &str = "canopy-pack-v1"; +pub const STORAGE_FORMAT: &str = "canopy-pack-v2"; /// Read-only admission before local reclamation, probes or Cell activation. pub(crate) async fn validate_service_root(store: &Store, prefix: &Path) -> Result<()> { @@ -154,6 +157,7 @@ impl Deployment { /// Rejects nodes when identity, selected release, image or phase differs. pub async fn require_ready(&self) -> Result<()> { + maintenance::require_idle(self).await?; self.require_service_root().await?; self.identities.layout(self.identity).await?; let record = self.record().await?; @@ -201,6 +205,7 @@ impl Deployment { /// Closes fleet admission under an explicit, retryable maintenance operation. pub async fn begin_maintenance(&self, operation: RequestId) -> Result { + maintenance::require_idle(self).await?; self.require_service_root().await?; self.identities.layout(self.identity).await?; let record = self.record().await?; @@ -271,6 +276,17 @@ impl Deployment { &self, operation: RequestId, now_ms: i64, + ) -> Result { + let claim = maintenance::acquire(self, operation, maintenance::Action::End, None).await?; + let result = self.end_maintenance_inner(operation, now_ms).await; + claim.finish(self).await?; + result + } + + async fn end_maintenance_inner( + &self, + operation: RequestId, + now_ms: i64, ) -> Result { self.require_service_root().await?; self.identities.layout(self.identity).await?; diff --git a/crates/canopy-server/src/deployment/recovery.rs b/crates/canopy-server/src/deployment/recovery.rs index 8300b9c9..cc2caa5a 100644 --- a/crates/canopy-server/src/deployment/recovery.rs +++ b/crates/canopy-server/src/deployment/recovery.rs @@ -57,9 +57,13 @@ impl Deployment { config: WorkerConfig, ) -> std::result::Result<(), ServerError> { self.require_maintenance(operation).await?; - let data_dir = config.data_dir; + let data_dir = config.data_dir.clone(); let local = tokio::task::spawn_blocking(move || workspace::Workspace::open(&data_dir)).await??; + let worker = maintenance::Worker::new(&config, local.path())?; + let claim = + maintenance::acquire(self, operation, maintenance::Action::Recover, Some(worker)) + .await?; crate::server::storage::probe( self.layout.store(), &self.layout.application_prefix().join("canopy-probe"), @@ -172,16 +176,19 @@ impl Deployment { } else { return Err(Error::Release("unknown maintenance Cell namespace").into()); }; - // Existing catalog entries pin their original contract. - // Retained executable code is valid for restoration; - // requiring target code would strand predecessor Cells. - // Acquisition separately validates persisted Control. - if !self.registry.supports_cell( - entry.namespace(), - entry.role(), - entry.initial_code(), - entry.initial_schema(), - ) { + let module_code = self + .registry + .module_code(module) + .ok_or(Error::Release("maintenance module is not compiled"))?; + let schema_version = if entry.namespace() == REPOSITORIES { + crate::REPOSITORY_SCHEMA_VERSION + } else { + 1 + }; + if entry.role() != cellule_runtime::cell::catalog::CatalogRole::Sql + || entry.initial_code() != module_code + || entry.initial_schema() != schema_version + { return Err( Error::Release("maintenance Cell descriptor differs").into() ); @@ -228,6 +235,7 @@ impl Deployment { self.nodes .withdraw(&*observed.lock().await, unix_now_ms()?) .await?; + claim.finish(self).await?; result } } diff --git a/crates/canopy-server/src/deployment/retention.rs b/crates/canopy-server/src/deployment/retention.rs new file mode 100644 index 00000000..9cdff7be --- /dev/null +++ b/crates/canopy-server/src/deployment/retention.rs @@ -0,0 +1,108 @@ +//! Product maintenance drain around Cellule's authoritative object collector. + +use super::*; +use crate::{ + REPOSITORY_DATABASE_LIMIT_BYTES, + server::{ServerError, unix_now_ms, workspace}, +}; +use cellule_ltx::{DiskBudget, Host}; +use cellule_runtime::recovery::retention::{CellGarbageCollector, GarbageCollectionPolicy}; + +/// Counts from a verified live-root/backup-pin mark and bounded immutable sweep. +#[derive(Debug, Serialize)] +pub struct GarbageCollectionReport { + pub listed_objects: u64, + pub immutable_candidates: u64, + pub reachable_objects: u64, + pub retained_objects: u64, + pub grace_objects: u64, + pub eligible_objects: u64, + pub deleted_objects: u64, + pub current_controls: u64, + pub retained_pins: u64, + pub complete: bool, +} + +impl Deployment { + /// Reclaims old immutable Cell objects after every server, reader, backup + /// worker and recovery owner has drained. Leaves the release in maintenance. + /// Same-workspace retries resume a killed worker; dropping the caller cannot + /// release exclusion before the sweep finishes. Git packs/LFS are separate + /// product objects and are not deleted by the Cellule collector. + pub async fn collect_garbage( + &self, + operation: RequestId, + config: WorkerConfig, + grace_ms: u64, + max_deletes: u64, + ) -> std::result::Result { + let deployment = self.clone(); + tokio::spawn(async move { + deployment + .collect(operation, config, grace_ms, max_deletes) + .await + }) + .await? + } + + async fn collect( + &self, + operation: RequestId, + config: WorkerConfig, + grace_ms: u64, + max_deletes: u64, + ) -> std::result::Result { + self.require_maintenance(operation).await?; + // Validate policy before claiming any persistent exclusion. + GarbageCollectionPolicy::new(unix_now_ms()?, grace_ms, max_deletes)?; + let data_dir = config.data_dir.clone(); + let local = + tokio::task::spawn_blocking(move || workspace::Workspace::open(&data_dir)).await??; + let worker = maintenance::Worker::new(&config, local.path())?; + let claim = + maintenance::acquire(self, operation, maintenance::Action::Collect, Some(worker)) + .await?; + let result = async { + self.require_maintenance(operation).await?; + if !self.status(unix_now_ms()?).await?.drained { + return Err( + Error::Retention("deployment readers and writers have not drained").into(), + ); + } + claim.require_current(self).await?; + let host = Host::default() + .with_local_disk_budget(DiskBudget::new(config.local_disk_limit_bytes)); + let collector = CellGarbageCollector::new( + self.layout.clone(), + self.identity, + crate::replica_limits(REPOSITORY_DATABASE_LIMIT_BYTES, 64 * 1024 * 1024), + host, + )?; + let report = collector + .collect( + &self.record().await?, + local.path(), + GarbageCollectionPolicy::new(unix_now_ms()?, grace_ms, max_deletes)?, + ) + .await?; + claim.require_current(self).await?; + Ok::<_, ServerError>(GarbageCollectionReport { + listed_objects: report.listed_objects(), + immutable_candidates: report.immutable_candidates(), + reachable_objects: report.reachable_objects(), + retained_objects: report.retained_objects(), + grace_objects: report.grace_objects(), + eligible_objects: report.eligible_objects(), + deleted_objects: report.deleted_objects(), + current_controls: report.current_controls(), + retained_pins: report.retained_pins(), + complete: report.complete(), + }) + } + .await; + // The collector has joined all work on return. Errors retain the release + // fence but allow a fresh bounded pass, including partial provider failure. + claim.finish(self).await?; + result + } +} diff --git a/crates/canopy-server/src/deployment/tests.rs b/crates/canopy-server/src/deployment/tests.rs index 17b60658..fafab3ee 100644 --- a/crates/canopy-server/src/deployment/tests.rs +++ b/crates/canopy-server/src/deployment/tests.rs @@ -8,7 +8,7 @@ use object_store::{ObjectStoreExt, memory::InMemory}; type TestResult = std::result::Result<(), Box>; -mod retained_maintenance; +mod catalog_contracts; #[tokio::test] async fn root_reservation_limit_includes_the_format_envelope_before_any_write() -> TestResult { @@ -42,7 +42,8 @@ async fn old_or_unknown_root_format_cannot_initialize_identity_or_release() -> T br#"{"kind":"service"}"#.as_slice(), br#"{"purpose":{"kind":"service"}}"#, br#"{"format":"future-format","purpose":{"kind":"service"}}"#, - br#"{"format":"canopy-pack-v1","purpose":{"kind":"service"},"extra":true}"#, + br#"{"format":"canopy-pack-v1","purpose":{"kind":"service"}}"#, + br#"{"format":"canopy-pack-v2","purpose":{"kind":"service"},"extra":true}"#, ] { let deployment = fixture()?; let path = deployment.prefix.clone().join("canopy-root-v1.json"); @@ -78,7 +79,7 @@ async fn packed_root_envelope_reuses_purpose_and_is_rejected_by_old_decoder() -> assert_eq!( serde_json::from_slice::(&bytes)?, serde_json::json!({ - "format": "canopy-pack-v1", "purpose": { "kind": "service" } + "format": STORAGE_FORMAT, "purpose": { "kind": "service" } }) ); // The old decoder is exactly the existing tagged RootPurpose type. @@ -237,7 +238,12 @@ async fn existing_catalog_without_release_is_not_silently_adopted() -> TestResul .module_code(RepositoryModule::NAME) .ok_or("missing code")?; CellCatalog::new(deployment.layout.clone(), deployment.identity.tenant()) - .provision(CatalogEntry::new(&target, CatalogRole::Sql, code, 1)?) + .provision(CatalogEntry::new( + &target, + CatalogRole::Sql, + code, + crate::REPOSITORY_SCHEMA_VERSION, + )?) .await?; assert!(deployment.initialize().await.is_err()); assert!(deployment.releases.load().await?.is_none()); @@ -313,6 +319,166 @@ fn recovery_config(data_dir: std::path::PathBuf) -> WorkerConfig { } } +#[tokio::test(flavor = "multi_thread")] +async fn collection_deletes_only_old_unreachable_cell_objects_and_stays_in_maintenance() +-> TestResult { + let deployment = fixture()?; + deployment.initialize().await?; + let files = tempfile::TempDir::new()?; + let operation = RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()); + let orphan = deployment.layout.incarnation_object_path( + &[11; 32], + &[12; 16], + &[13; 32], + cellule_ltx::CellObjectKind::Directory, + ); + let product = deployment + .prefix + .clone() + .join("repos/fixture/git-packs/unreferenced.pack"); + for path in [&orphan, &product] { + deployment + .layout + .store() + .create_strict(path, bytes::Bytes::from_static(b"unreferenced")) + .await?; + } + assert!( + deployment + .collect_garbage( + operation, + recovery_config(files.path().join("before")), + 1, + 1 + ) + .await + .is_err() + ); + deployment.begin_maintenance(operation).await?; + // A large grace window retains the candidate, and the compiled descriptor + // remains marked even when its modification time is beyond that boundary. + let report = deployment + .collect_garbage( + operation, + recovery_config(files.path().join("grace")), + 60_000, + 1, + ) + .await?; + assert_eq!(report.deleted_objects, 0); + assert!(report.grace_objects >= 1); + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + let report = deployment + .collect_garbage( + operation, + recovery_config(files.path().join("collect")), + 1, + 1, + ) + .await?; + assert_eq!(report.deleted_objects, 1); + assert!(report.complete); + assert!(matches!( + deployment.layout.store().get_with_etag(&orphan).await, + Err(cellule_store::StorageError::NotFound { .. }) + )); + assert!( + deployment + .layout + .store() + .get_with_etag(&product) + .await + .is_ok() + ); + assert!( + deployment + .status(crate::server::unix_now_ms()?) + .await? + .drained + ); + assert!(deployment.require_ready().await.is_err()); + deployment + .end_maintenance(operation, crate::server::unix_now_ms()?) + .await?; + deployment.require_ready().await?; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn collection_rejects_even_expired_advertisements_before_deleting() -> TestResult { + let deployment = fixture()?; + deployment.initialize().await?; + advertise_owner(&deployment, true).await?; + let operation = RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()); + deployment.begin_maintenance(operation).await?; + let files = tempfile::TempDir::new()?; + assert!( + deployment + .collect_garbage(operation, recovery_config(files.path().to_path_buf()), 1, 1) + .await + .is_err() + ); + maintenance::require_idle(&deployment).await?; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn interrupted_collection_blocks_reopening_until_same_workspace_worker_resumes() -> TestResult +{ + let deployment = fixture()?; + deployment.initialize().await?; + let operation = RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()); + deployment.begin_maintenance(operation).await?; + let files = tempfile::TempDir::new()?; + let config = recovery_config(files.path().to_path_buf()); + let local = crate::server::workspace::Workspace::open(files.path())?; + let worker = maintenance::Worker::new(&config, local.path())?; + let claim = maintenance::acquire( + &deployment, + operation, + maintenance::Action::Collect, + Some(worker), + ) + .await?; + assert!( + deployment + .end_maintenance(operation, crate::server::unix_now_ms()?) + .await + .is_err() + ); + assert!( + deployment + .recover_maintenance(operation, recovery_config(files.path().join("other"))) + .await + .is_err() + ); + // The same worker cannot adopt its claim while its original workspace is held. + let identical = || WorkerConfig { + node: config.node, + signing_key: config.signing_key.clone(), + endpoint: config.endpoint.clone(), + data_dir: config.data_dir.clone(), + local_disk_limit_bytes: config.local_disk_limit_bytes, + }; + assert!( + deployment + .collect_garbage(operation, identical(), 1, 1) + .await + .is_err() + ); + drop(local); + let report = deployment + .collect_garbage(operation, identical(), 1, 1) + .await?; + assert_eq!(report.deleted_objects, 0); + assert!(claim.require_current(&deployment).await.is_err()); + maintenance::require_idle(&deployment).await?; + deployment + .end_maintenance(operation, crate::server::unix_now_ms()?) + .await?; + Ok(()) +} + async fn advertise_owner(deployment: &Deployment, expired: bool) -> TestResult { use cellule_runtime::{ SessionId, identity::NodeId, node::NodeAdvertisement, node::NodeCapacity, @@ -362,7 +528,12 @@ async fn maintenance_recovery_initializes_abandoned_catalog_and_unpublished_owne uuid::Uuid::new_v4().into_bytes(), )?; let proof = catalog - .provision(CatalogEntry::new(&target, CatalogRole::Sql, code, 1)?) + .provision(CatalogEntry::new( + &target, + CatalogRole::Sql, + code, + crate::REPOSITORY_SCHEMA_VERSION, + )?) .await?; if unpublished { authority @@ -472,7 +643,12 @@ async fn maintenance_recovery_rejects_missing_published_root() -> TestResult { .module_code(RepositoryModule::NAME) .ok_or("missing code")?; let proof = CellCatalog::new(deployment.layout.clone(), deployment.identity.tenant()) - .provision(CatalogEntry::new(&target, CatalogRole::Sql, code, 1)?) + .provision(CatalogEntry::new( + &target, + CatalogRole::Sql, + code, + crate::REPOSITORY_SCHEMA_VERSION, + )?) .await?; let authority = CellAuthority::new(deployment.layout.clone()); let initial = authority diff --git a/crates/canopy-server/src/deployment/tests/catalog_contracts.rs b/crates/canopy-server/src/deployment/tests/catalog_contracts.rs new file mode 100644 index 00000000..96e0ff0b --- /dev/null +++ b/crates/canopy-server/src/deployment/tests/catalog_contracts.rs @@ -0,0 +1,70 @@ +//! Maintenance refuses noncurrent catalog contracts before allocating Control. +use super::*; +use crate::directory; +use cellule_runtime::{CellTarget, NamespaceId}; + +fn deployment_code() -> std::result::Result> { + fixture()? + .registry + .module_code(crate::directory::DirectoryModule::NAME) + .ok_or_else(|| "directory code missing".into()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn maintenance_rejects_unknown_catalog_contracts_without_initializing_control() -> TestResult +{ + let code = deployment_code()?; + for (namespace, role, code, schema) in [ + ( + directory::DIRECTORY, + CatalogRole::Sql, + Digest::from_bytes([99; 32]), + 1, + ), + (directory::DIRECTORY, CatalogRole::Sql, code, 2), + (directory::DIRECTORY, CatalogRole::Kv, code, 1), + (NamespaceId::from_bytes([99; 16]), CatalogRole::Sql, code, 1), + ] { + let deployment = fixture()?; + deployment.initialize().await?; + let files = tempfile::TempDir::new()?; + let target = CellTarget::new( + deployment.identity.tenant(), + deployment.identity.application(), + namespace, + &[0, 0, 0, 0], + )?; + let entry = CatalogEntry::new(&target, role, code, schema)?; + let catalog = CellCatalog::new(deployment.layout.clone(), deployment.identity.tenant()); + catalog.provision(entry.clone()).await?; + let authority = CellAuthority::new(deployment.layout.clone()); + let operation = RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()); + deployment.begin_maintenance(operation).await?; + assert!( + deployment + .recover_maintenance(operation, recovery_config(files.path().join("reject"))) + .await + .is_err() + ); + assert!(authority.load(target.cell_id()).await?.is_none()); + assert_eq!( + catalog + .lookup(target.cell_id()) + .await? + .ok_or("catalog missing")? + .entry(), + &entry + ); + assert_eq!( + deployment.record().await?.state(), + ReleaseState::Maintenance + ); + assert!( + !deployment + .status(crate::server::unix_now_ms()?) + .await? + .drained + ); + } + Ok(()) +} diff --git a/crates/canopy-server/src/deployment/tests/retained_maintenance.rs b/crates/canopy-server/src/deployment/tests/retained_maintenance.rs deleted file mode 100644 index a7bec549..00000000 --- a/crates/canopy-server/src/deployment/tests/retained_maintenance.rs +++ /dev/null @@ -1,350 +0,0 @@ -//! Owned fixtures exercise the real maintenance worker, not only registry admission. -//! They do not claim old-executable or retained RustFS corpus recovery. - -use super::*; -use crate::directory::{self, DirectoryModule}; -use cellule_ltx::{CellReplica, DiskBudget, Host}; -use cellule_runtime::{ - CellRuntime, CellTarget, NamespaceId, NodeLeaseGuard, SessionId, cell::worker::SqlWorkerPool, - control::Owner, identity::IncarnationId, -}; - -fn retained_code() -> std::result::Result> { - let previous = - include_bytes!("../../../tests/directory_cell/fixtures/c51-selected-release.json") - .trim_ascii_end(); - assert_eq!( - blake3::hash(previous).to_hex().as_str(), - "e31bf1a951e2fa19d91e9f964b2ddeade1a81b05a20ad628362819a1487c16b1" - ); - let descriptor: serde_json::Value = serde_json::from_slice(previous)?; - let module = descriptor["modules"] - .as_array() - .ok_or("modules missing")? - .iter() - .find(|module| module["name"] == DirectoryModule::NAME) - .ok_or("Directory missing")?; - Ok(Digest::from_bytes( - hex::decode(module["code"].as_str().ok_or("code missing")?)? - .try_into() - .map_err(|_| "invalid retained code")?, - )) -} - -fn runtime(session: SessionId) -> cellule_runtime::Result { - CellRuntime::new_with_replica_host( - SqlWorkerPool::new(1, 1)?, - 64 * 1024 * 1024, - session, - Host::default().with_local_disk_budget(DiskBudget::new(256 * 1024 * 1024)), - ) -} - -#[tokio::test(flavor = "multi_thread")] -async fn maintenance_recovers_unpublished_retained_directory() -> TestResult { - let deployment = fixture()?; - deployment.initialize().await?; - advertise_owner(&deployment, true).await?; - let files = tempfile::TempDir::new()?; - let target = directory::directory_target( - deployment.identity.tenant(), - deployment.identity.application(), - )?; - let entry = CatalogEntry::new(&target, CatalogRole::Sql, retained_code()?, 1)?; - let catalog = CellCatalog::new(deployment.layout.clone(), deployment.identity.tenant()); - let proof = catalog.provision(entry.clone()).await?; - let authority = CellAuthority::new(deployment.layout.clone()); - authority - .create_initial( - &proof, - IncarnationId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - Owner { - session: SessionId::from_bytes([6; 16]), - endpoint: "https://fixture.example.invalid".into(), - }, - ) - .await?; - let operation = RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()); - deployment.begin_maintenance(operation).await?; - deployment - .recover_maintenance(operation, recovery_config(files.path().join("recover"))) - .await?; - let after = authority - .load(target.cell_id()) - .await? - .ok_or("Cell missing")?; - assert_eq!(after.value().state, ControlState::Idle); - assert!(after.value().owner.is_none()); - assert!(after.value().root.is_some()); - assert_eq!(after.value().code, entry.initial_code()); - assert_eq!(after.value().schema, 1); - assert_eq!( - catalog - .lookup(target.cell_id()) - .await? - .ok_or("catalog missing")? - .entry(), - &entry - ); - deployment - .end_maintenance(operation, crate::server::unix_now_ms()?) - .await?; - Ok(()) -} - -#[tokio::test(flavor = "multi_thread")] -async fn maintenance_recovers_published_retained_directory_without_changing_data() -> TestResult { - recover_published_directory(false).await -} - -#[tokio::test(flavor = "multi_thread")] -async fn maintenance_recovers_retained_directory_after_same_descriptor_image_selection() --> TestResult { - recover_published_directory(true).await -} - -async fn recover_published_directory(change_image: bool) -> TestResult { - let deployment = fixture()?; - deployment.initialize().await?; - let files = tempfile::TempDir::new()?; - let target = directory::directory_target( - deployment.identity.tenant(), - deployment.identity.application(), - )?; - let code = retained_code()?; - assert!( - deployment - .registry - .supports_cell(directory::DIRECTORY, CatalogRole::Sql, code, 1) - ); - assert!( - !deployment - .registry - .is_current_cell(directory::DIRECTORY, CatalogRole::Sql, code, 1) - ); - let entry = CatalogEntry::new(&target, CatalogRole::Sql, code, 1)?; - let catalog = CellCatalog::new(deployment.layout.clone(), deployment.identity.tenant()); - let proof = catalog.provision(entry.clone()).await?; - let authority = CellAuthority::new(deployment.layout.clone()); - let session = SessionId::from_bytes([6; 16]); - let owner = Owner { - session, - endpoint: "https://fixture.example.invalid".into(), - }; - let initial = authority - .create_initial( - &proof, - IncarnationId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - owner, - ) - .await?; - let now = crate::server::unix_now_ms()?; - let guard = NodeLeaseGuard::new(now, now + crate::server::LEASE_MS)?; - let old_runtime = CellRuntime::new_with_replica_host_requiring_node_lease( - SqlWorkerPool::new(1, 1)?, - 64 * 1024 * 1024, - session, - Host::default().with_local_disk_budget(DiskBudget::new(256 * 1024 * 1024)), - )?; - old_runtime.install_node_lease(guard.clone())?; - let limits = crate::replica_limits(crate::REPOSITORY_DATABASE_LIMIT_BYTES, 16 * 1024 * 1024); - old_runtime.bootstrap( - proof.clone(), - CellReplica::new(deployment.layout.clone(), *target.cell_id().as_bytes(), *initial.value().incarnation.as_bytes(), limits)?, - authority.clone(), initial, files.path().join("old.sqlite"), - |transaction| { - transaction.execute_batch(directory::SCHEMA)?; - transaction.execute_batch("CREATE TABLE maintenance_marker(value TEXT NOT NULL); INSERT INTO maintenance_marker VALUES ('retained bytes');")?; - Ok(()) - }, - ).await?; - let before = authority - .load(target.cell_id()) - .await? - .ok_or("published Cell missing")?; - assert_eq!(before.value().state, ControlState::Serving); - assert!(before.value().root.is_some()); - // Fence only this owned runtime, then close its SQL workers. No raw Control writes. - guard.fence(); - assert!(old_runtime.shutdown().await.is_err()); - assert_eq!( - authority - .load(target.cell_id()) - .await? - .ok_or("Cell lost")? - .value(), - before.value() - ); - advertise_owner(&deployment, !change_image).await?; - let operation = RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()); - let deployment = if change_image { - // Model a closed, canonically retired old-image owner, as in the retained - // corpus. Synthetic image digests here do not qualify physical binaries. - let now = crate::server::unix_now_ms()?; - let advertisement = deployment - .nodes - .load(session, now) - .await? - .ok_or("owner missing")?; - deployment.nodes.withdraw(&advertisement, now).await?; - assert!(deployment.nodes.is_retired(session).await?); - let bridge = Deployment::new( - deployment.layout.store().clone(), - deployment.prefix.clone(), - deployment.identity, - deployment.nodes.fleet(), - Digest::from_bytes([55; 32]), - Arc::clone(&deployment.registry), - )?; - assert!(bridge.require_ready().await.is_err()); - let old = deployment.record().await?; - let prepared = deployment - .releases - .prepare( - deployment.registry.release_bytes(), - deployment.registry.release_digest(), - old.revision(), - &bridge.image, - operation, - ) - .await?; - assert_eq!(prepared.current(), old.current()); - assert_eq!(prepared.desired(), old.current()); - assert_eq!(prepared.desired_image(), bridge.image); - assert!(deployment.begin_maintenance(operation).await.is_err()); - assert!( - deployment - .recover_maintenance(operation, recovery_config(files.path().join("wrong-image"))) - .await - .is_err() - ); - assert!(!files.path().join("wrong-image").exists()); - bridge - } else { - deployment - }; - deployment.begin_maintenance(operation).await?; - deployment - .recover_maintenance(operation, recovery_config(files.path().join("recover"))) - .await?; - let after = authority - .load(target.cell_id()) - .await? - .ok_or("recovered Cell missing")?; - assert_eq!(after.value().state, ControlState::Idle); - assert!(after.value().owner.is_none()); - assert_eq!(after.value().root, before.value().root); - assert_eq!(after.value().code, code); - assert_eq!(after.value().schema, 1); - assert_eq!( - catalog - .lookup(target.cell_id()) - .await? - .ok_or("catalog missing")? - .entry(), - &entry - ); - assert!( - deployment - .status(crate::server::unix_now_ms()?) - .await? - .drained - ); - deployment - .end_maintenance(operation, crate::server::unix_now_ms()?) - .await?; - let reader = runtime(SessionId::from_bytes([31; 16]))?; - let read = reader - .acquire_idle_restored( - proof, - CellReplica::new( - deployment.layout.clone(), - *target.cell_id().as_bytes(), - *after.value().incarnation.as_bytes(), - limits, - )?, - authority, - after, - files.path().join("read.sqlite"), - Owner { - session: SessionId::from_bytes([31; 16]), - endpoint: "https://reader.example.invalid".into(), - }, - ) - .await?; - let bytes = read - .query(0, 64, |connection| { - let value: String = - connection - .query_row("SELECT value FROM maintenance_marker", [], |row| row.get(0))?; - Ok(value.into_bytes()) - }) - .await; - reader.shutdown().await?; - assert_eq!(bytes?, b"retained bytes"); - Ok(()) -} - -#[tokio::test(flavor = "multi_thread")] -async fn maintenance_rejects_unknown_catalog_contracts_without_initializing_control() -> TestResult -{ - let retained = retained_code()?; - for (namespace, role, code, schema) in [ - ( - directory::DIRECTORY, - CatalogRole::Sql, - Digest::from_bytes([99; 32]), - 1, - ), - (directory::DIRECTORY, CatalogRole::Sql, retained, 2), - (directory::DIRECTORY, CatalogRole::Kv, retained, 1), - ( - NamespaceId::from_bytes([99; 16]), - CatalogRole::Sql, - retained, - 1, - ), - ] { - let deployment = fixture()?; - deployment.initialize().await?; - let files = tempfile::TempDir::new()?; - let target = CellTarget::new( - deployment.identity.tenant(), - deployment.identity.application(), - namespace, - &[0, 0, 0, 0], - )?; - let entry = CatalogEntry::new(&target, role, code, schema)?; - let catalog = CellCatalog::new(deployment.layout.clone(), deployment.identity.tenant()); - catalog.provision(entry.clone()).await?; - let authority = CellAuthority::new(deployment.layout.clone()); - let operation = RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()); - deployment.begin_maintenance(operation).await?; - assert!( - deployment - .recover_maintenance(operation, recovery_config(files.path().join("reject"))) - .await - .is_err() - ); - assert!(authority.load(target.cell_id()).await?.is_none()); - assert_eq!( - catalog - .lookup(target.cell_id()) - .await? - .ok_or("catalog missing")? - .entry(), - &entry - ); - assert_eq!( - deployment.record().await?.state(), - ReleaseState::Maintenance - ); - assert!( - !deployment - .status(crate::server::unix_now_ms()?) - .await? - .drained - ); - } - Ok(()) -} diff --git a/crates/canopy-server/src/directory/mod.rs b/crates/canopy-server/src/directory/mod.rs index 3e7933b8..6369ca0b 100644 --- a/crates/canopy-server/src/directory/mod.rs +++ b/crates/canopy-server/src/directory/mod.rs @@ -18,15 +18,10 @@ use cellule_app::{ApplicationHandle, CellType}; use cellule_runtime::{ ApplicationId, CellModule, CellTarget, Committed, Digest, Error, InvocationError, MigrationDescriptor, ModuleDescriptor, MutationIdentity, NamespaceDescriptor, NamespaceId, - Observed, Receipt, RegistryBuilder, SqlCell, SqlModule, TenantId, - cell::catalog::CatalogRole, - partition_for_shard, - primitives::sql::SqlBatch, - primitives::sql::SqlResultSet, - primitives::sql::SqlStatement, - primitives::sql::SqlValue, - primitives::sql::register_sql, - registry::{OperationDescriptor, RetainedCodeDescriptor}, + Observed, Receipt, RegistryBuilder, SqlCell, SqlModule, TenantId, cell::catalog::CatalogRole, + partition_for_shard, primitives::sql::SqlBatch, primitives::sql::SqlResultSet, + primitives::sql::SqlStatement, primitives::sql::SqlValue, primitives::sql::register_sql, + registry::OperationDescriptor, }; use crate::{CanopyApplication, ReadIdentity, validate_repository_id}; @@ -42,19 +37,6 @@ const QUERIES: [OperationDescriptor; 3] = [ timed_sql::AUTHENTICATE_OPERATION, ]; -// The selected c51 release used the same schema and credential contracts, -// before the separately bounded authentication query was added. Keep that -// exact code executable while persisted Cells roll to the new descriptor. -const RETAINED_CODES: [RetainedCodeDescriptor; 1] = [RetainedCodeDescriptor { - code: Digest::from_bytes([ - 0xf7, 0x25, 0x4e, 0xda, 0x9d, 0x5d, 0x33, 0x95, 0x66, 0xf4, 0x54, 0x57, 0x50, 0x26, 0x18, - 0xad, 0x13, 0xcb, 0xbf, 0x6e, 0x5a, 0x74, 0x59, 0x5f, 0x5b, 0x3c, 0xe4, 0x66, 0x53, 0xea, - 0x12, 0xf1, - ]), - schema_min: 1, - schema_max: 1, -}]; - const fn operation(id: u32) -> OperationDescriptor { OperationDescriptor { id, @@ -94,7 +76,7 @@ impl CellModule for DirectoryModule { source.update(SCHEMA.as_bytes()); Digest::from_bytes(*source.finalize().as_bytes()) }, - retained_codes: &RETAINED_CODES, + retained_codes: &[], schema_min: 1, schema_max: 1, migrations: MIGRATIONS.get_or_init(|| { diff --git a/crates/canopy-server/src/git_cache/maintenance.rs b/crates/canopy-server/src/git_cache/maintenance.rs index 4b2e3683..3345f2ce 100644 --- a/crates/canopy-server/src/git_cache/maintenance.rs +++ b/crates/canopy-server/src/git_cache/maintenance.rs @@ -71,18 +71,6 @@ fn index_order(path: &Path, format: crate::ObjectFormat) -> io::Result bool { - self.durable_packs - .read() - .expect("durable inventory poisoned") - .contains(&sha) - } - pub(crate) fn mark_durable_pack(&self, sha: [u8; 32]) { - self.durable_packs - .write() - .expect("durable inventory poisoned") - .insert(sha); - } #[cfg(test)] pub(crate) async fn pack_sources( self: &Arc, @@ -112,147 +100,6 @@ impl GitCache { }) .await? } - pub(crate) async fn install_pack( - self: &Arc, - hash: crate::ObjectId, - pack: LargeBlobRead, - index: LargeBlobRead, - ) -> Result<(), CacheError> { - if hash.format() != self.object_format { - return Err(io::Error::other("pack format mismatch").into()); - } - self.write_artifact( - format!("objects/pack/pack-{}.pack", hex::encode(hash)), - pack, - ) - .await?; - self.write_artifact( - format!("objects/pack/pack-{}.idx", hex::encode(hash)), - index, - ) - .await?; - let cache = Arc::clone(self); - tokio::task::spawn_blocking(move || { - let path = cache - .git_dir() - .join(format!("objects/pack/pack-{}.idx", hex::encode(hash))); - let index = crate::git_format::pack_index::PackIndex::open(&path, cache.object_format)?; - if index.pack_checksum() != hash { - return Err(io::Error::other("index/pack binding mismatch").into()); - } - cache.register_checked_index(index)?; - cache - .pack_files - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - cache - .write_generation - .fetch_add(1, std::sync::atomic::Ordering::SeqCst); - Ok(()) - }) - .await? - } - async fn write_artifact( - self: &Arc, - relative: String, - mut reader: LargeBlobRead, - ) -> Result<(), CacheError> { - let destination = self.git_dir().join(relative); - // Retry a canceled pair without downloading/charging a second full pack. - // File names alone (possibly SHA-1) are insufficient: verify all hashes. - let expected = reader.reference(); - let existing = destination.clone(); - if tokio::task::spawn_blocking(move || -> io::Result { - use sha2::Digest; - use std::io::Read; - let mut file = match File::open(existing) { - Ok(file) => file, - Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), - Err(error) => return Err(error), - }; - if file.metadata()?.len() != expected.size { - return Err(io::Error::other("cached artifact size mismatch")); - } - let mut oid = crate::git_format::ObjectHasher::new( - expected.oid.format(), - ObjectKind::Blob, - expected.size, - ); - let mut sha = sha2::Sha256::new(); - let mut blake = blake3::Hasher::new(); - let mut buffer = vec![0; 8 << 20]; - loop { - let n = file.read(&mut buffer)?; - if n == 0 { - break; - } - oid.update(&buffer[..n]); - sha.update(&buffer[..n]); - blake.update(&buffer[..n]); - } - if oid.finalize() != expected.oid - || <[u8; 32]>::from(sha.finalize()) != expected.sha256 - || *blake.finalize().as_bytes() != expected.blake3 - { - return Err(io::Error::other("cached artifact digest mismatch")); - } - Ok(true) - }) - .await?? - { - return Ok(()); - } - let (file, temporary) = tempfile::NamedTempFile::new_in( - destination - .parent() - .ok_or_else(|| io::Error::other("missing pack directory"))?, - )? - .into_parts(); - let mut writer = CacheWriter { - file, - cache: Arc::clone(self), - }; - while let Some(bytes) = reader.next().await? { - writer = tokio::task::spawn_blocking(move || { - writer.write_all(&bytes)?; - Ok::<_, io::Error>(writer) - }) - .await??; - } - tokio::task::spawn_blocking(move || { - writer.flush()?; - writer.file.sync_all()?; - drop(writer); - if destination.exists() { - // Compare complete bytes, not merely Git's SHA-1 filename. - use std::io::Read; - let mut old = File::open(&destination)?; - let mut new = File::open(&temporary)?; - let mut a = vec![0; 1 << 20]; - let mut b = vec![0; 1 << 20]; - loop { - let n = old.read(&mut a)?; - let m = new.read(&mut b)?; - if n != m || a[..n] != b[..m] { - return Err(io::Error::other( - "cached pack differs from authoritative bytes", - )); - } - if n == 0 { - break; - } - } - return Ok(()); - } - temporary - .persist_noclobber(destination) - .map_err(|error| error.error)?; - Ok(()) - }) - .await??; - Ok(()) - } - /// Reuse only packs whose *every* object was verified and durably recorded - /// during this ingestion. Extra/unverified objects disable this optimization. #[cfg(test)] pub(crate) async fn retain_verified_packs( self: &Arc, @@ -306,9 +153,6 @@ impl GitCache { cache .write_generation .fetch_add(1, std::sync::atomic::Ordering::SeqCst); - cache - .pack_files - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); cache.register_index( &cache.git_dir().join("objects/pack").join( path.file_name() @@ -354,11 +198,6 @@ impl GitCache { next.reservation()? .try_grow(reserve) .map_err(io::Error::other)?; - let durable = self - .durable_packs - .read() - .map_err(|_| io::Error::other("durable inventory poisoned"))? - .clone(); let maintenance = self .native .for_class(crate::native_resources::NativeClass::Maintenance); @@ -502,12 +341,6 @@ impl GitCache { .and_then(|v| v); next.reconcile().await?; result.map_err(worker_error)?; - next.pack_files - .store(1, std::sync::atomic::Ordering::Relaxed); - *next - .durable_packs - .write() - .map_err(|_| io::Error::other("durable inventory poisoned"))? = durable; Ok(next) } } diff --git a/crates/canopy-server/src/git_cache/mod.rs b/crates/canopy-server/src/git_cache/mod.rs index d4badbbe..1018bea4 100644 --- a/crates/canopy-server/src/git_cache/mod.rs +++ b/crates/canopy-server/src/git_cache/mod.rs @@ -1,7 +1,6 @@ //! Disposable Git files charged to the node's shared disk budget. use std::{ - collections::HashSet, fs::{self, File, OpenOptions}, io::{self, BufWriter, Write}, path::{Path, PathBuf}, @@ -14,14 +13,15 @@ use flate2::{Compression, write::ZlibEncoder}; use std::collections::BTreeMap; use tokio::sync::Mutex; -use crate::{ - blob::{LargeBlobError, LargeBlobRead}, - refs::valid_ref_name, -}; +use crate::refs::valid_ref_name; +#[cfg(test)] +use crate::ObjectKind; +use crate::RefExpectation; #[cfg(test)] use crate::object_id; -use crate::{ObjectKind, RefExpectation}; +#[cfg(test)] +use std::collections::HashSet; pub(crate) const CACHE_PREFIX: &str = "canopy-git-"; @@ -38,8 +38,6 @@ pub enum CacheError { Io(#[from] io::Error), #[error("Git cache disk admission failed")] Budget(#[from] LtxError), - #[error("Git blob hydration failed")] - Blob(#[from] LargeBlobError), #[error("Git cache worker failed")] Task(#[from] tokio::task::JoinError), } @@ -74,11 +72,9 @@ pub(crate) struct GitCache { // fetches share a completed loose object without serializing all objects. object_writes: OnceLock<[Arc>; 64]>, packed: RwLock>, - durable_packs: RwLock>, pub(crate) selection: Mutex<()>, #[cfg(test)] pub(crate) loose_objects: std::sync::atomic::AtomicU64, - pub(crate) pack_files: std::sync::atomic::AtomicU64, pub(crate) write_generation: std::sync::atomic::AtomicU64, } @@ -142,11 +138,9 @@ impl GitCache { objects, object_writes: OnceLock::new(), packed: RwLock::new(Vec::new()), - durable_packs: RwLock::new(HashSet::new()), selection: Mutex::new(()), #[cfg(test)] loose_objects: std::sync::atomic::AtomicU64::new(0), - pack_files: std::sync::atomic::AtomicU64::new(0), write_generation: std::sync::atomic::AtomicU64::new(0), }); for directory in ["objects/info", "objects/pack", "refs/heads", "refs/tags", "hooks"] { @@ -354,71 +348,6 @@ impl GitCache { .await? } - #[cfg(test)] - pub(crate) async fn store_blob( - self: &Arc, - reader: LargeBlobRead, - ) -> Result<(), CacheError> { - self.store_blob_reader(BlobReader::External(reader)).await - } - #[cfg(test)] - pub(crate) async fn store_native_blob( - self: &Arc, - reader: crate::pack_store::NativePackedRead, - ) -> Result<(), CacheError> { - self.store_blob_reader(BlobReader::Packed(reader)).await - } - #[cfg(test)] - async fn store_blob_reader(self: &Arc, mut reader: BlobReader) -> Result<(), CacheError> { - let (oid, size) = reader.metadata(); - let write = self.object_write_lock(oid).lock_owned().await; - let cache = Arc::clone(self); - let pending = tokio::task::spawn_blocking(move || { - if oid.format() != cache.object_format { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Git object format mismatch", - ) - .into()); - } - if cache.object_present(oid)? { - return Ok::<_, CacheError>(None); - } - let (mut encoder, temporary, destination) = cache.object_writer(oid)?; - encoder.write_all(format!("blob {}\0", size).as_bytes())?; - Ok::<_, CacheError>(Some((encoder, temporary, destination))) - }) - .await??; - let Some((mut encoder, temporary, destination)) = pending else { - return Ok(()); - }; - while let Some(bytes) = reader.next().await? { - // The writer owns the cache reservation until each compression worker - // exits, including when its async waiter is canceled. - encoder = tokio::task::spawn_blocking(move || { - encoder.write_all(&bytes)?; - Ok::<_, CacheError>(encoder) - }) - .await??; - } - let cache = Arc::clone(self); - tokio::task::spawn_blocking(move || { - let _write = write; - drop(encoder.finish()?); - temporary - .persist_noclobber(destination) - .map_err(|error| error.error)?; - cache - .loose_objects - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - cache - .write_generation - .fetch_add(1, std::sync::atomic::Ordering::SeqCst); - Ok::<_, CacheError>(()) - }) - .await? - } - #[cfg(test)] pub(crate) async fn store_refs( self: &Arc, @@ -495,6 +424,7 @@ impl GitCache { )?) } + #[cfg(test)] fn register_checked_index( &self, index: crate::git_format::pack_index::PackIndex, @@ -605,27 +535,3 @@ fn tree_bytes(path: &Path) -> io::Result { mod tests; mod maintenance; - -#[cfg(test)] -enum BlobReader { - External(LargeBlobRead), - Packed(crate::pack_store::NativePackedRead), -} -#[cfg(test)] -impl BlobReader { - fn metadata(&self) -> (crate::ObjectId, u64) { - match self { - Self::External(read) => (read.reference().oid, read.reference().size), - Self::Packed(read) => (read.oid, read.size), - } - } - async fn next(&mut self) -> Result, CacheError> { - match self { - Self::External(read) => Ok(read.next().await?), - Self::Packed(read) => read - .next() - .await - .map_err(|error| io::Error::other(error).into()), - } - } -} diff --git a/crates/canopy-server/src/git_cache/tests.rs b/crates/canopy-server/src/git_cache/tests.rs index 47118eb4..ba3a0b44 100644 --- a/crates/canopy-server/src/git_cache/tests.rs +++ b/crates/canopy-server/src/git_cache/tests.rs @@ -39,6 +39,11 @@ async fn streamed_native_extraction_publishes_only_a_verified_complete_object() source .store_object(expected.oid, expected.kind, body.clone()) .await?; + let foreign_body = b"an unrelated object must remain private"; + let foreign = object_id(format, ObjectKind::Blob, foreign_body); + source + .store_object(foreign, ObjectKind::Blob, foreign_body.to_vec()) + .await?; let mut wrong = expected; wrong.digest[0] ^= 1; assert!( @@ -52,6 +57,7 @@ async fn streamed_native_extraction_publishes_only_a_verified_complete_object() .copy_native_owned(source.git_dir(), expected, source.clone()) .await?; assert!(target.object_present(expected.oid)?); + assert!(!target.object_present(foreign)?); let mut objects = GitObjects::batch_owned(&target.git_dir(), &native, target.clone())?; assert_eq!(objects.read_verified(expected, body.len()).await?, body); objects.finish().await?; @@ -533,213 +539,3 @@ async fn failed_cleanup_does_not_release_disk_admission() -> Result<(), Box Result<(), Box> { - let root = tempfile::TempDir::new()?; - let budget = DiskBudget::new(1 << 20); - let cache = GitCache::create( - root.path().into(), - budget.clone(), - "refs/heads/main", - crate::ObjectFormat::Sha1, - crate::native_resources::NativeResources::default() - .scope(crate::native_resources::NativeClass::Foreground), - ) - .await?; - let blobs = - crate::blob::LargeBlobStore::new(Arc::new(object_store::memory::InMemory::new()), [7; 16]); - let body = vec![17; 9 * 1024 * 1024]; - let reference = blobs - .put( - object_id(crate::ObjectFormat::Sha1, ObjectKind::Blob, &body), - body.len() as u64, - &mut body.as_slice(), - ) - .await?; - cache.store_blob(blobs.read(&reference).await?).await?; - let output = tokio::process::Command::new("git") - .arg("--git-dir") - .arg(cache.git_dir()) - .args(["cat-file", "blob", &hex::encode(reference.oid)]) - .output() - .await?; - assert!(output.status.success()); - assert_eq!(output.stdout, body); - assert_eq!(budget.used(), tree_bytes(cache.root())?); - drop(cache); - assert_eq!(budget.used(), 0); - - let cache = GitCache::create( - root.path().into(), - budget.clone(), - "refs/heads/main", - crate::ObjectFormat::Sha1, - crate::native_resources::NativeResources::default() - .scope(crate::native_resources::NativeClass::Foreground), - ) - .await?; - let occupied = budget.try_reserve(budget.capacity() - budget.used())?; - assert!( - cache - .store_blob(blobs.read(&reference).await?) - .await - .is_err_and(|error| error.is_admission()) - ); - drop(cache); - assert_eq!(budget.used(), occupied.bytes()); - Ok(()) -} - -#[tokio::test] -async fn corrupt_stream_never_installs_a_reusable_object() -> Result<(), Box> -{ - use object_store::ObjectStoreExt; - let root = tempfile::TempDir::new()?; - let budget = DiskBudget::new(1 << 20); - let cache = GitCache::create( - root.path().into(), - budget.clone(), - "refs/heads/main", - crate::ObjectFormat::Sha1, - crate::native_resources::NativeResources::default() - .scope(crate::native_resources::NativeClass::Foreground), - ) - .await?; - let store = Arc::new(object_store::memory::InMemory::new()); - let blobs = crate::blob::LargeBlobStore::new(store.clone(), [9; 16]); - let mut body = vec![23; 9 * 1024 * 1024]; - let reference = blobs - .put( - object_id(crate::ObjectFormat::Sha1, ObjectKind::Blob, &body), - body.len() as u64, - &mut body.as_slice(), - ) - .await?; - // The first range can be written before the final range detects corruption. - // Only the test-owned blob key is changed, then restored before retry. - let path = crate::external::part(&crate::blob::blob_path([9; 16], &reference.sha256), 1); - let metadata = store.head(&path).await?; - let last = body.last_mut().ok_or("empty fixture")?; - *last ^= 1; - store - .put(&metadata.location, body[8 * 1024 * 1024..].to_vec().into()) - .await?; - assert!( - cache - .store_blob(blobs.read(&reference).await?) - .await - .is_err() - ); - assert_eq!( - cache.missing_objects(vec![reference.oid]).await?, - vec![reference.oid] - ); - *body.last_mut().ok_or("empty fixture")? ^= 1; - store - .put(&metadata.location, body[8 * 1024 * 1024..].to_vec().into()) - .await?; - cache.store_blob(blobs.read(&reference).await?).await?; - assert!(cache.missing_objects(vec![reference.oid]).await?.is_empty()); - drop(cache); - assert_eq!(budget.used(), 0); - Ok(()) -} - -#[tokio::test] -async fn incomplete_pack_extracts_verified_large_blobs_without_admitting_foreign_members() --> Result<(), Box> { - let root = tempfile::TempDir::new()?; - let budget = DiskBudget::new(256 << 20); - let source = GitCache::create( - root.path().into(), - budget.clone(), - "refs/heads/main", - crate::ObjectFormat::Sha1, - crate::native_resources::NativeResources::default() - .scope(crate::native_resources::NativeClass::Foreground), - ) - .await?; - let body = vec![b'x'; 2 << 20]; - let oid = object_id(crate::ObjectFormat::Sha1, ObjectKind::Blob, &body); - let digest = *blake3::hash(&body).as_bytes(); - source - .store_object(oid, ObjectKind::Blob, body.clone()) - .await?; - let foreign = object_id(crate::ObjectFormat::Sha1, ObjectKind::Blob, b"foreign"); - source - .store_object(foreign, ObjectKind::Blob, b"foreign".to_vec()) - .await?; - let packed = source.repacked(root.path().into(), budget.clone()).await?; - let (hash, pack, index, ids) = packed.pack_sources().await?.pop().unwrap(); - assert_eq!( - ids.iter().copied().collect::>(), - [oid, foreign].into_iter().collect() - ); - let store = Arc::new(object_store::memory::InMemory::new()); - let reader = crate::pack_store::PackReader::new( - store, - [4; 16], - root.path().into(), - budget.clone(), - crate::ObjectFormat::Sha1, - source.native.clone(), - ); - let record = crate::pack_store::PackRecord { - hash, - pack: reader.upload(pack).await?, - index: reader.upload(index).await?, - approved: false, - }; - let target = GitCache::create( - root.path().into(), - budget.clone(), - "refs/heads/main", - crate::ObjectFormat::Sha1, - crate::native_resources::NativeResources::default() - .scope(crate::native_resources::NativeClass::Foreground), - ) - .await?; - target - .store_native_blob( - reader - .native_reader(record.clone(), oid, body.len() as u64, digest) - .await?, - ) - .await?; - assert!(target.missing_objects(vec![oid]).await?.is_empty()); - assert_eq!(target.missing_objects(vec![foreign]).await?, vec![foreign]); - let output = crate::native_git::command(&target.git_dir())? - .args(["cat-file", "blob", &hex::encode(oid)]) - .output() - .await?; - assert!(output.status.success()); - assert_eq!(output.stdout, body); - assert!( - reader - .cache() - .await? - .missing_objects(vec![oid, foreign]) - .await? - .len() - == 2 - ); - let invalid = GitCache::create( - root.path().into(), - budget.clone(), - "refs/heads/main", - crate::ObjectFormat::Sha1, - crate::native_resources::NativeResources::default() - .scope(crate::native_resources::NativeClass::Foreground), - ) - .await?; - assert!( - invalid - .store_native_blob(reader.native_reader(record, oid, 2 << 20, [0; 32]).await?) - .await - .is_err() - ); - assert_eq!(invalid.missing_objects(vec![oid]).await?, vec![oid]); - Ok(()) -} diff --git a/crates/canopy-server/src/git_gateway/candidates/produce.rs b/crates/canopy-server/src/git_gateway/candidates/produce.rs index b2d85902..1c234406 100644 --- a/crates/canopy-server/src/git_gateway/candidates/produce.rs +++ b/crates/canopy-server/src/git_gateway/candidates/produce.rs @@ -231,6 +231,7 @@ impl GitGateway { gateway.disk_budget.clone(), base, MetadataLimits::default(), + crate::packs::closure::default_limits(), ) .await .map_err(work)?; diff --git a/crates/canopy-server/src/git_gateway/head.rs b/crates/canopy-server/src/git_gateway/head.rs index 30af755d..0653e7de 100644 --- a/crates/canopy-server/src/git_gateway/head.rs +++ b/crates/canopy-server/src/git_gateway/head.rs @@ -119,6 +119,7 @@ impl GitGateway { gateway.disk_budget.clone(), base, MetadataLimits::default(), + crate::packs::closure::default_limits(), ) .await .map_err(work)? diff --git a/crates/canopy-server/src/git_gateway/merge.rs b/crates/canopy-server/src/git_gateway/merge.rs index 8b642680..2d77b0ec 100644 --- a/crates/canopy-server/src/git_gateway/merge.rs +++ b/crates/canopy-server/src/git_gateway/merge.rs @@ -144,6 +144,7 @@ impl GitGateway { gateway.disk_budget.clone(), base, MetadataLimits::default(), + crate::packs::closure::default_limits(), ) .await .map_err(work)? diff --git a/crates/canopy-server/src/git_gateway/mod.rs b/crates/canopy-server/src/git_gateway/mod.rs index 5069b8ac..34d0e4e2 100644 --- a/crates/canopy-server/src/git_gateway/mod.rs +++ b/crates/canopy-server/src/git_gateway/mod.rs @@ -20,7 +20,6 @@ use tokio::sync::{Mutex, OnceCell}; use crate::{ PushPlan, RefExpectation, RefUpdate, RepositoryCell, - blob::LargeBlobError, directory::TokenScope, git_cache::CacheError, git_http::{GitHttpBackend, GitHttpError, GitHttpRequest, GitHttpResponse}, @@ -53,8 +52,6 @@ pub enum GatewayError { Input(#[from] InputError), #[error("repository Cell operation failed")] Cell(#[source] CellError), - #[error("large Git blob store failed")] - Blob(#[from] LargeBlobError), #[error("cache I/O failed")] Io(#[from] std::io::Error), #[error("Git cache command failed: {0}")] @@ -88,7 +85,6 @@ pub struct GitGateway { repository: Arc, signer_directory: Option>, certificate_seed: Arc>, - _pack_reader: Arc, lfs: Arc, artifacts: Arc, scratch_root: PathBuf, @@ -110,30 +106,11 @@ impl GitGateway { Arc::clone(&blob_store), repository.repository_id(), )); - // A reader belongs to this gateway's workspace and disk admission. - // Another gateway may use a different root/budget for the same Cell. - let pack_reader = Arc::new(crate::pack_store::PackReader::new( - Arc::clone(&blob_store), - repository.repository_id(), - scratch_root.clone(), - disk_budget.clone(), - repository.object_format(), - native.clone(), - )); - { - let mut readers = repository - .pack_readers - .lock() - .expect("packed reader registry poisoned"); - readers.retain(|reader| reader.strong_count() > 0); - readers.push(Arc::downgrade(&pack_reader)); - } let lfs = Arc::new(LfsService::new(Arc::clone(&repository), blob_store)); Self { repository, signer_directory: None, certificate_seed: Arc::new(OnceCell::new()), - _pack_reader: pack_reader, lfs, artifacts, scratch_root, diff --git a/crates/canopy-server/src/git_gateway/push/native.rs b/crates/canopy-server/src/git_gateway/push/native.rs index 9e4095f1..7febece7 100644 --- a/crates/canopy-server/src/git_gateway/push/native.rs +++ b/crates/canopy-server/src/git_gateway/push/native.rs @@ -244,6 +244,7 @@ impl GitGateway { gateway.disk_budget.clone(), base, MetadataLimits::default(), + crate::packs::closure::default_limits(), ) .await .map_err(input)?; diff --git a/crates/canopy-server/src/git_objects/tests.rs b/crates/canopy-server/src/git_objects/tests.rs index 52b85028..e1ea1515 100644 --- a/crates/canopy-server/src/git_objects/tests.rs +++ b/crates/canopy-server/src/git_objects/tests.rs @@ -225,21 +225,17 @@ async fn batch_reads_large_blob_across_pipe_buffers() -> TestResult { )?; assert_eq!(objects.next().await?, Some(oid)); let mut object = objects.read(oid).await?; - let store = crate::blob::LargeBlobStore::new( - std::sync::Arc::new(object_store::memory::InMemory::new()), - [1; 16], - ); - let reference = store.put(oid, object.size, &mut object.reader).await?; + let mut buffer = vec![0; 64 << 10]; + let mut offset = 0; + while offset < body.len() { + let length = buffer.len().min(body.len() - offset); + object.reader.read_exact(&mut buffer[..length]).await?; + assert_eq!(&buffer[..length], &body[offset..offset + length]); + offset += length; + } object.finish().await?; assert_eq!(objects.next().await?, None); objects.finish().await?; - let mut reader = store.read(&reference).await?; - let mut offset = 0; - while let Some(bytes) = reader.next().await? { - assert_eq!(bytes.as_ref(), &body[offset..offset + bytes.len()]); - offset += bytes.len(); - } - assert_eq!(offset, body.len()); struct BlobSink; impl EdgeSink for BlobSink { async fn append( diff --git a/crates/canopy-server/src/graph/mod.rs b/crates/canopy-server/src/graph/mod.rs index be80433e..56347f0d 100644 --- a/crates/canopy-server/src/graph/mod.rs +++ b/crates/canopy-server/src/graph/mod.rs @@ -1,279 +1,13 @@ -//! Bounded object certificates and constant-per-ref graph checks at publication. +//! Typed structural Git parsing for native catalog verification. +use crate::ObjectKind; -use std::collections::BTreeMap; - -use crate::{ObjectKind, PushPlan, RepositoryModule, object_batch::VERIFY_BATCH_BYTES, object_id}; -use cellule_runtime::{ - CellModule, Command, Error, codec::BoundedDecoder, codec::BoundedEncoder, codec::CodecError, - codec::WireValue, primitives::sql::SqlBatch, primitives::sql::SqlResultSet, - primitives::sql::SqlStatement, primitives::sql::SqlValue, registry::CommandContext, - registry::CommandResult, -}; - -mod preparation; pub(crate) mod stream; - +#[cfg(test)] type Oid = crate::ObjectId; +#[cfg(test)] type Edge = (Oid, Option); -const MAX_CERTIFICATES: usize = 512; - -#[derive(Default)] -pub(crate) struct CertificateBatch(Vec); - -impl WireValue for CertificateBatch { - fn encode(&self, encoder: &mut BoundedEncoder) -> Result<(), CodecError> { - if !(1..=MAX_CERTIFICATES).contains(&self.0.len()) { - return Err(CodecError::Invalid("certificate batch count")); - } - encoder.write_count(self.0.len())?; - for oid in &self.0 { - encoder.write_bytes(oid)?; - } - Ok(()) - } - fn decode(decoder: &mut BoundedDecoder<'_>) -> Result { - let count = decoder.read_count()?; - if !(1..=MAX_CERTIFICATES).contains(&count) { - return Err(CodecError::Invalid("certificate batch count")); - } - let mut batch = Self(Vec::with_capacity(count)); - for _ in 0..count { - batch.0.push( - decoder - .read_bytes()? - .try_into() - .map_err(|_| CodecError::Invalid("certificate OID width"))?, - ); - } - Ok(batch) - } -} - -#[derive(Clone)] -struct Status { - kind: ObjectKind, - bytes: u64, - certified: bool, -} - -fn status_query(oids: &[Oid]) -> SqlBatch { - let placeholders = vec!["?"; oids.len()].join(","); - SqlBatch { - statements: vec![SqlStatement { - sql: format!( - "SELECT o.oid, o.kind, CASE WHEN o.storage IN ('external', 'packed') THEN 0 ELSE o.size END, c.oid FROM objects o LEFT JOIN object_closure c ON c.oid = o.oid WHERE o.oid IN ({placeholders})" - ), - parameters: oids - .iter() - .map(|oid| SqlValue::Blob(oid.to_vec())) - .collect(), - }], - } -} - -fn statuses(results: &[SqlResultSet]) -> cellule_runtime::Result> { - let mut states = BTreeMap::new(); - for row in &results - .first() - .ok_or(Error::Command("missing graph result"))? - .rows - { - let [ - SqlValue::Blob(oid), - SqlValue::Text(kind), - SqlValue::Integer(bytes), - certificate, - ] = row.as_slice() - else { - return Err(Error::Command("invalid graph status")); - }; - let kind = parse_kind(kind.as_bytes()).ok_or(Error::Command("invalid graph kind"))?; - let bytes = u64::try_from(*bytes).map_err(|_| Error::Command("invalid graph size"))?; - states.insert( - oid.as_slice() - .try_into() - .map_err(|_| Error::Command("invalid graph OID"))?, - Status { - kind, - bytes, - certified: matches!(certificate, SqlValue::Blob(_)), - }, - ); - } - Ok(states) -} - -pub(crate) struct CertifyObjects; -impl Command for CertifyObjects { - const MODULE: &'static str = RepositoryModule::NAME; - const ID: u32 = 6; - const CODEC_VERSION: u32 = 3; - type Input = CertificateBatch; - type Output = bool; - - fn execute( - context: &mut CommandContext<'_, '_>, - batch: CertificateBatch, - ) -> cellule_runtime::Result> { - let mut verified = 0; - for oid in batch.0 { - let states = statuses(&context.sql(&status_query(&[oid]))?)?; - let Some(state) = states.get(&oid) else { - return Ok(CommandResult::Rejected(false)); - }; - if state.certified { - continue; - } - if verified != 0 && state.bytes > VERIFY_BATCH_BYTES.saturating_sub(verified) { - return Ok(CommandResult::Rejected(false)); - } - verified += state.bytes; - let Some(edges) = object_edges(context, oid, state.kind)? else { - return Ok(CommandResult::Rejected(false)); - }; - for edges in edges.chunks(MAX_CERTIFICATES) { - let oids: Vec<_> = edges.iter().map(|(oid, _)| *oid).collect(); - let states = statuses(&context.sql(&status_query(&oids))?)?; - if edges.iter().any(|(oid, kind)| { - !states.get(oid).is_some_and(|state| { - state.certified && kind.is_none_or(|kind| kind == state.kind) - }) - }) { - return Ok(CommandResult::Rejected(false)); - } - } - // Only certified, typed edges enter the reachability index. Keeping - // this with closure publication prevents lazy fetch from trusting - // staged or malformed object graphs. - for edges in edges.chunks(crate::object_batch::MAX_OBJECTS) { - context.sql(&SqlBatch { - statements: edges - .iter() - .map(|(child, _)| SqlStatement { - sql: "INSERT INTO object_edges (parent, child) VALUES (?1, ?2)".into(), - parameters: vec![ - SqlValue::Blob(oid.to_vec()), - SqlValue::Blob(child.to_vec()), - ], - }) - .collect(), - })?; - } - if state.kind == ObjectKind::Commit { - let parents: Vec<_> = edges - .iter() - .filter(|(_, kind)| *kind == Some(ObjectKind::Commit)) - .map(|(parent, _)| *parent) - .collect(); - for parents in parents.chunks(crate::object_batch::MAX_OBJECTS) { - context.sql(&SqlBatch { statements: parents.iter().map(|parent| SqlStatement { - sql: "INSERT INTO commit_parents (child, parent) VALUES (?1, ?2) ON CONFLICT DO NOTHING".into(), - parameters: vec![SqlValue::Blob(oid.to_vec()), SqlValue::Blob(parent.to_vec())], - }).collect() })?; - } - } - // These proofs depend only on immutable objects. A future collector must - // invalidate certificates before removing any reachable object or chunk. - context.sql(&SqlBatch { - statements: vec![SqlStatement { - sql: "INSERT INTO object_closure (oid) VALUES (?1)".into(), - parameters: vec![SqlValue::Blob(oid.to_vec())], - }], - })?; - } - Ok(CommandResult::Success(true)) - } -} - -pub(crate) fn certified_roots( - context: &CommandContext<'_, '_>, - plan: &PushPlan, -) -> cellule_runtime::Result { - for updates in plan.updates.chunks(MAX_CERTIFICATES) { - let oids: Vec<_> = updates.iter().filter_map(|update| update.new_oid).collect(); - if oids.is_empty() { - continue; - } - let states = statuses(&context.sql(&status_query(&oids))?)?; - if !updates.iter().all(|update| { - update.new_oid.is_none_or(|oid| { - states.get(&oid).is_some_and(|state| { - state.certified - && (!update.name.starts_with("refs/heads/") - || state.kind == ObjectKind::Commit) - }) - }) - }) { - return Ok(false); - } - } - Ok(true) -} - -fn object_edges( - context: &CommandContext<'_, '_>, - oid: Oid, - kind: ObjectKind, -) -> cellule_runtime::Result>> { - let result = context.sql(&SqlBatch { - statements: vec![SqlStatement { - sql: "SELECT storage, body, digest, size, chunk_id FROM objects WHERE oid = ?1".into(), - parameters: vec![SqlValue::Blob(oid.to_vec())], - }], - })?; - let Some( - [ - SqlValue::Text(storage), - body, - SqlValue::Blob(digest), - SqlValue::Integer(size), - upload, - ], - ) = result - .first() - .and_then(|set| set.rows.first()) - .map(Vec::as_slice) - else { - return Err(Error::Command("invalid stored graph object")); - }; - let edges = match (storage.as_str(), body, upload) { - ("inline", SqlValue::Blob(body), SqlValue::Null) => { - if object_id(oid.format(), kind, body) != oid - || blake3::hash(body).as_bytes() != digest.as_slice() - { - return Err(Error::Command("corrupt stored graph object")); - } - let Some(edges) = edges(oid.format(), kind, body) else { - return Ok(None); - }; - edges - } - // The immutable upload is verified before its SQLite record is published. - // Blobs have no outgoing Git edges; no network I/O belongs in this transaction. - ("external" | "packed", SqlValue::Null, SqlValue::Null) if kind == ObjectKind::Blob => { - Vec::new() - } - ("chunked", SqlValue::Null, SqlValue::Blob(upload)) => { - let invalid = || Error::Command("invalid stored graph object chunks"); - let body = crate::object_chunks::body( - context, - oid, - kind, - upload.as_slice().try_into().map_err(|_| invalid())?, - u64::try_from(*size).map_err(|_| invalid())?, - digest.as_slice().try_into().map_err(|_| invalid())?, - )? - .ok_or_else(invalid)?; - let Some(edges) = edges(oid.format(), kind, &body) else { - return Ok(None); - }; - edges - } - _ => return Err(Error::Command("invalid stored graph object")), - }; - Ok(Some(edges)) -} +#[cfg(test)] pub(crate) fn edges( format: crate::ObjectFormat, kind: ObjectKind, @@ -304,6 +38,7 @@ pub(crate) fn edges( Some(edges) } +#[cfg(test)] pub(crate) fn tag_edge(mut body: &[u8]) -> Option { let target = hex_oid(line(&mut body)?.strip_prefix(b"object ")?)?; let kind = parse_kind(line(&mut body)?.strip_prefix(b"type ")?)?; @@ -311,6 +46,7 @@ pub(crate) fn tag_edge(mut body: &[u8]) -> Option { Some((target, Some(kind))) } +#[cfg(test)] fn line<'a>(body: &mut &'a [u8]) -> Option<&'a [u8]> { let newline = body.iter().position(|byte| *byte == b'\n')?; let line = &body[..newline]; @@ -318,6 +54,7 @@ fn line<'a>(body: &mut &'a [u8]) -> Option<&'a [u8]> { Some(line) } +#[cfg(test)] fn tree_edges(format: crate::ObjectFormat, mut body: &[u8]) -> Option> { let mut edges = Vec::new(); while !body.is_empty() { @@ -363,6 +100,7 @@ fn tree_edges(format: crate::ObjectFormat, mut body: &[u8]) -> Option> Some(edges) } +#[cfg(test)] fn hex_oid(text: &[u8]) -> Option { let oid = Oid::from_hex(text).ok()?; (!oid.is_zero()).then_some(oid) diff --git a/crates/canopy-server/src/graph/preparation.rs b/crates/canopy-server/src/graph/preparation.rs deleted file mode 100644 index 0d1d02d0..00000000 --- a/crates/canopy-server/src/graph/preparation.rs +++ /dev/null @@ -1,232 +0,0 @@ -use super::*; -use crate::{ - RepositoryCell, - directory::{TokenScope, validate_component}, -}; -use cellule_runtime::{InvocationError, MutationIdentity, identity::RequestId}; -use std::{ - collections::{BTreeSet, HashMap, HashSet}, - time::{SystemTime, UNIX_EPOCH}, -}; - -#[derive(Debug, thiserror::Error)] -enum PrepareError { - #[error("graph query failed")] - Query(#[from] InvocationError>), - #[error("graph certificate publication failed")] - Certificate(#[from] InvocationError), - #[error("graph data is invalid")] - Invalid(#[from] Error), - #[error("graph worker failed")] - Worker(#[from] tokio::task::JoinError), - #[error("graph clock failed")] - Clock(#[from] std::time::SystemTimeError), - #[error("graph clock is outside the supported range")] - ClockRange(#[from] std::num::TryFromIntError), -} - -enum Visit { - Enter(Edge), - Leave(Oid, ObjectKind, u64), -} - -impl RepositoryCell { - pub(crate) async fn prepare_graph(&self, plan: &PushPlan) -> Result<(), InvocationError> { - self.prepare_graph_inner(plan).await.map_err(|error| { - InvocationError::NotStarted(Error::Facility { - name: "Git graph preparation", - source: Box::new(error), - }) - }) - } - - async fn prepare_graph_inner(&self, plan: &PushPlan) -> Result<(), PrepareError> { - // The authoritative ACL/version checks remain in finalization. Avoid doing - // graph work for invalid or unauthorized plans before that transaction. - if plan.updates.is_empty() - || plan.updates.len() > crate::refs::MAX_UPDATES - || validate_component(&plan.actor).is_err() - { - return Ok(()); - } - if !self - .access_level(&plan.actor, None) - .await? - .output - .is_some_and(|level| level >= TokenScope::Write) - { - return Ok(()); - } - let started = std::time::Instant::now(); - // One preparer per process bounds aggregate memory while retaining no - // arbitrary repository/object-count ceiling. Durable certificates still - // recheck typed dependencies inside each bounded Cell transaction. - static PREPARERS: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(1); - let _preparer = PREPARERS - .acquire() - .await - .map_err(|_| Error::Command("graph admission closed"))?; - let mut nodes = HashMap::new(); - let mut ready = HashMap::new(); - let mut discovered = HashSet::new(); - let mut frontier: BTreeSet<_> = plan - .updates - .iter() - .filter_map(|update| update.new_oid) - .collect(); - while !frontier.is_empty() { - let page: Vec<_> = frontier.iter().take(MAX_CERTIFICATES).copied().collect(); - for oid in &page { - frontier.remove(oid); - discovered.insert(*oid); - } - let result = self.sql.query(None, status_query(&page)).await?; - let states = statuses(&result.output)?; - let mut structure = BTreeSet::new(); - for oid in &page { - let Some(state) = states.get(oid) else { - return Ok(()); - }; - if state.certified { - ready.insert(*oid, state.kind); - continue; - } - nodes.insert(*oid, (state.kind, state.bytes, Some(Vec::new()))); - if state.kind != ObjectKind::Blob { - structure.insert(*oid); - } - } - while !structure.is_empty() { - let ids: Vec<_> = structure - .iter() - .take(crate::object_batch::MAX_OBJECTS) - .copied() - .collect(); - let records = self.selected_objects(&ids).await?; - if records.is_empty() { - return Err(Error::Command("empty graph body page").into()); - } - for record in records { - structure.remove(&record.oid); - let oid = record.oid; - let kind = record.kind; - let body = match record.storage { - crate::ObjectStorage::Inline(body) => body, - crate::ObjectStorage::Chunked { .. } => { - self.object(oid, None) - .await? - .output - .ok_or(Error::Command("missing graph body"))? - .1 - } - _ => return Err(Error::Command("invalid structural storage").into()), - }; - let parsed = - tokio::task::spawn_blocking(move || edges(oid.format(), kind, &body)) - .await?; - if let Some(edges) = &parsed { - frontier.extend( - edges - .iter() - .map(|(oid, _)| *oid) - .filter(|oid| !discovered.contains(oid)), - ); - } - nodes - .get_mut(&oid) - .ok_or(Error::Command("missing graph node"))? - .2 = parsed; - } - } - } - drop(discovered); - let loaded = nodes.len(); - let mut pending: Vec<_> = plan - .updates - .iter() - .filter_map(|update| { - update.new_oid.map(|oid| { - Visit::Enter(( - oid, - update - .name - .starts_with("refs/heads/") - .then_some(ObjectKind::Commit), - )) - }) - }) - .collect(); - let mut visiting = HashSet::new(); - let mut batch = CertificateBatch::default(); - let mut bytes = 0; - while let Some(visit) = pending.pop() { - match visit { - Visit::Enter((oid, expected)) => { - if let Some(kind) = ready.get(&oid) { - if expected.is_some_and(|expected| expected != *kind) { - return Ok(()); - } - continue; - } - // Dependencies outside the captured uncertified inventory - // must already be certified. The command checks this, and - // final publication always checks the roots again. - let Some((kind, weight, children)) = nodes.remove(&oid) else { - if visiting.contains(&oid) { - return Ok(()); - } - continue; - }; - if expected.is_some_and(|expected| expected != kind) || !visiting.insert(oid) { - return Ok(()); - } - let Some(children) = children else { - return Ok(()); - }; - pending.push(Visit::Leave(oid, kind, weight)); - for edge in children.into_iter().rev() { - pending.push(Visit::Enter(edge)); - } - } - Visit::Leave(oid, kind, weight) => { - if !batch.0.is_empty() && weight > VERIFY_BATCH_BYTES.saturating_sub(bytes) { - self.certify_batch(std::mem::take(&mut batch)).await?; - bytes = 0; - } - batch.0.push(oid); - bytes += weight; - ready.insert(oid, kind); - visiting.remove(&oid); - if batch.0.len() == MAX_CERTIFICATES || bytes >= VERIFY_BATCH_BYTES { - self.certify_batch(std::mem::take(&mut batch)).await?; - bytes = 0; - } - } - } - } - if !batch.0.is_empty() { - self.certify_batch(batch).await?; - } - tracing::info!( - objects = loaded, - elapsed_seconds = started.elapsed().as_secs_f64(), - "prepared Git graph certificates" - ); - Ok(()) - } - - async fn certify_batch(&self, batch: CertificateBatch) -> Result<(), PrepareError> { - let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; - let identity = MutationIdentity { - request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - issued_at_ms: now, - expires_at_ms: now - .checked_add(60_000) - .ok_or(Error::Command("graph clock overflow"))?, - }; - self.application - .command::(&self.target, identity, batch) - .await?; - Ok(()) - } -} diff --git a/crates/canopy-server/src/graph/tests.rs b/crates/canopy-server/src/graph/tests.rs index e4b054bb..4dc948b1 100644 --- a/crates/canopy-server/src/graph/tests.rs +++ b/crates/canopy-server/src/graph/tests.rs @@ -1,29 +1,5 @@ use super::*; -#[test] -fn certificate_codec_rejects_empty_oversized_and_malformed_batches() { - for count in [0, MAX_CERTIFICATES + 1] { - let mut encoder = BoundedEncoder::new(1 << 20).unwrap(); - encoder.write_count(count).unwrap(); - let input = encoder.finish(); - assert!( - CertificateBatch::decode(&mut BoundedDecoder::new(&input, 1 << 20).unwrap()).is_err() - ); - } - let mut encoder = BoundedEncoder::new(1 << 20).unwrap(); - encoder.write_count(1).unwrap(); - encoder.write_bytes(&[0; 19]).unwrap(); - let input = encoder.finish(); - assert!(CertificateBatch::decode(&mut BoundedDecoder::new(&input, 1 << 20).unwrap()).is_err()); - let batch = CertificateBatch(vec![crate::ObjectId::Sha1([1; 20]); MAX_CERTIFICATES]); - let mut encoder = BoundedEncoder::new(1 << 20).unwrap(); - batch.encode(&mut encoder).unwrap(); - let input = encoder.finish(); - let mut decoder = BoundedDecoder::new(&input, 1 << 20).unwrap(); - assert_eq!(CertificateBatch::decode(&mut decoder).unwrap().0, batch.0); - decoder.finish().unwrap(); -} - #[test] fn repeated_edges_share_one_proof_but_different_types_do_not() { let mut body = Vec::new(); diff --git a/crates/canopy-server/src/lib.rs b/crates/canopy-server/src/lib.rs index 78ec22a5..1fb2a8ff 100644 --- a/crates/canopy-server/src/lib.rs +++ b/crates/canopy-server/src/lib.rs @@ -4,11 +4,10 @@ use std::sync::OnceLock; use cellule_app::{ApplicationBuilder, ApplicationHandle, CellApplication, CellType}; use cellule_runtime::{ - ApplicationId, BuildDescriptor, CellModule, CellTarget, Committed, Digest, Error, - MigrationDescriptor, ModuleDescriptor, NamespaceDescriptor, NamespaceId, Observed, Receipt, - RegistryBuilder, SqlCell, SqlModule, TenantId, cell::catalog::CatalogRole, - primitives::sql::SqlBatch, primitives::sql::SqlStatement, primitives::sql::SqlValue, - primitives::sql::register_sql, registry::OperationDescriptor, + ApplicationId, BuildDescriptor, CellModule, CellTarget, Digest, Error, MigrationDescriptor, + ModuleDescriptor, NamespaceDescriptor, NamespaceId, RegistryBuilder, SqlCell, SqlModule, + TenantId, cell::catalog::CatalogRole, primitives::sql::register_sql, + registry::OperationDescriptor, }; mod access; @@ -16,7 +15,6 @@ mod admission; pub use admission::AdmissionPermit; mod visibility; pub use visibility::{RepositoryVisibility, Visibility}; -mod ancestry; pub mod branch_rules; pub mod checks; mod default_branch; @@ -34,22 +32,9 @@ mod git_read; mod graph; pub mod http; pub mod issues; -pub mod blob { - //! Verified, immutable Git blob bodies stored outside the Repository Cell. - - #[cfg(test)] - pub(crate) use canopy_object_storage::blob::blob_path; - pub use canopy_object_storage::blob::{ - LargeBlobError, LargeBlobRead, LargeBlobReference, LargeBlobStore, - }; -} pub mod lfs; mod native_git; pub mod native_resources; -mod object_batch; -mod object_chunks; -mod object_reads; -mod pack_store; pub mod packs; pub mod pulls; mod push; @@ -62,12 +47,11 @@ mod web; pub use access::{COLLABORATOR_PAGE_SIZE, Collaborator, ReadIdentity}; pub use default_branch::DefaultBranch; -pub use object_batch::ObjectBatch; -pub use object_chunks::ObjectStageError; pub use push::{PushCertificateReceipt, PushError, PushReceipt, VerifiedPushCertificate}; -pub use refs::{FinalizePush, PushPlan, RefExpectation, RefPage, RefReadError, RefUpdate}; +pub use refs::{PushPlan, RefExpectation, RefPage, RefUpdate}; pub const REPOSITORIES: NamespaceId = NamespaceId::from_bytes([71; 16]); +pub const REPOSITORY_SCHEMA_VERSION: u32 = 4; pub const INLINE_OBJECT_LIMIT: usize = 768 * 1024; // SQLite's on-disk page count and maximum page size bound representable databases. // This is a format boundary, not a repository quota. @@ -98,8 +82,8 @@ const fn operation_with_codec(id: u32, codec_version: u32) -> OperationDescripto OperationDescriptor { id, codec_version, - schema_min: 1, - schema_max: 1, + schema_min: REPOSITORY_SCHEMA_VERSION, + schema_max: REPOSITORY_SCHEMA_VERSION, input_limit: 1 << 20, output_limit: 1 << 20, } @@ -127,33 +111,6 @@ pub(crate) fn validate_repository_id(repository: [u8; 16]) -> cellule_runtime::R Ok(()) } -/// Byte location for a verified Git object record. -pub enum ObjectStorage { - Inline(Vec), - Chunked { - upload: [u8; 16], - size: u64, - blake3: [u8; 32], - }, - Packed { - size: u64, - blake3: [u8; 32], - pack: [u8; 32], - }, - External { - size: u64, - blake3: [u8; 32], - sha256: [u8; 32], - }, -} - -/// Immutable Git object returned by one bounded Repository Cell read. -pub struct StoredObject { - pub oid: crate::ObjectId, - pub kind: ObjectKind, - pub storage: ObjectStorage, -} - pub struct RepositoryModule; impl SqlModule for RepositoryModule { @@ -174,6 +131,9 @@ impl CellModule for RepositoryModule { let mut source = blake3::Hasher::new(); source.update(include_bytes!("lib.rs")); source.update(include_bytes!("deployment/mod.rs")); + source.update(include_bytes!("deployment/maintenance.rs")); + source.update(include_bytes!("deployment/retention.rs")); + source.update(include_bytes!("deployment/recovery.rs")); source.update(include_bytes!("deployment/root.rs")); source.update(include_bytes!("deployment/backup/bodies.rs")); source.update(include_bytes!("deployment/backup/native.rs")); @@ -182,6 +142,7 @@ impl CellModule for RepositoryModule { source.update(include_bytes!("packs/publication/backup.rs")); source.update(include_bytes!("packs/publication/recovery/backup.rs")); source.update(include_bytes!("server/mod.rs")); + source.update(include_bytes!("server/catalog_admission.rs")); source.update(include_bytes!("server/lifecycle.rs")); source.update(include_bytes!("admission.rs")); source.update(include_bytes!("server/workspace/mod.rs")); @@ -192,14 +153,8 @@ impl CellModule for RepositoryModule { source.update(include_bytes!("refs.rs")); source.update(include_bytes!("default_branch.rs")); source.update(include_bytes!("graph/mod.rs")); - source.update(include_bytes!("ancestry.rs")); source.update(include_bytes!("branch_rules/mod.rs")); source.update(include_bytes!("branch_rules/command.rs")); - source.update(include_bytes!("graph/preparation.rs")); - source.update(include_bytes!("object_batch/mod.rs")); - source.update(include_bytes!("object_chunks/mod.rs")); - source.update(include_bytes!("object_reads/mod.rs")); - source.update(include_bytes!("pack_store.rs")); source.update(include_bytes!("git_objects/mod.rs")); source.update(include_bytes!("git_cache/artifacts.rs")); source.update(include_bytes!("git_cache/verified.rs")); @@ -416,9 +371,6 @@ impl CellModule for RepositoryModule { source.update(include_bytes!( "../../canopy-object-storage/src/artifact.rs" )); - source.update(include_bytes!( - "../../canopy-object-storage/src/blob/mod.rs" - )); source.update(include_bytes!( "../../canopy-object-storage/src/external.rs" )); @@ -449,11 +401,11 @@ impl CellModule for RepositoryModule { Digest::from_bytes(*source.finalize().as_bytes()) }, retained_codes: &[], - schema_min: 1, - schema_max: 1, + schema_min: REPOSITORY_SCHEMA_VERSION, + schema_max: REPOSITORY_SCHEMA_VERSION, migrations: MIGRATIONS.get_or_init(|| { [MigrationDescriptor { - version: 1, + version: REPOSITORY_SCHEMA_VERSION, sql: SCHEMA, digest: Digest::from_bytes(*blake3::hash(SCHEMA.as_bytes()).as_bytes()), }] @@ -501,6 +453,7 @@ fn repository_cell_type() -> cellule_runtime::Result { 1, )? .with_entity_partitions()? + .with_schema_range(REPOSITORY_SCHEMA_VERSION, REPOSITORY_SCHEMA_VERSION)? .with_limits(REPOSITORY_DATABASE_LIMIT_BYTES, 64 * 1024 * 1024) } @@ -511,9 +464,6 @@ pub struct RepositoryCell { sql: SqlCell, application: ApplicationHandle, target: CellTarget, - // Gateways own the cache lifetime. Sharing the reader through a weak - // reference must not retain its original disk budget after gateway eviction. - pack_readers: std::sync::Mutex>>, serving: std::sync::Mutex>>, staging: std::sync::Mutex>>, } @@ -542,7 +492,6 @@ impl RepositoryCell { sql: application.sql::(target.clone())?, application: application.clone(), target, - pack_readers: std::sync::Mutex::new(Vec::new()), serving: std::sync::Mutex::new(None), staging: std::sync::Mutex::new(None), }) @@ -604,176 +553,6 @@ impl RepositoryCell { }; pool.snapshot(actor).await } - - /// Prepares bounded graph certificates, then publishes one all-or-none ref plan. - pub async fn finalize_push( - &self, - identity: cellule_runtime::MutationIdentity, - plan: PushPlan, - ) -> std::result::Result, cellule_runtime::InvocationError> { - // Keep each transport-heavy phase in its own allocation. Embedding all - // three futures multiplies stack copies when debug callers poll a push. - Box::pin(self.prepare_graph(&plan)).await?; - Box::pin(self.prepare_branch_proofs(&plan)).await?; - Box::pin( - self.application - .command::(&self.target, identity, plan), - ) - .await - } - - pub async fn object( - &self, - oid: crate::ObjectId, - minimum: Option, - ) -> std::result::Result< - Observed)>>, - cellule_runtime::InvocationError>, - > { - let result = self - .sql - .query( - minimum, - SqlBatch { - statements: vec![SqlStatement { - sql: - "SELECT kind, body, digest, size, chunk_id, storage, external_sha256 FROM objects WHERE oid = ?1" - .into(), - parameters: vec![SqlValue::Blob(oid.to_vec())], - }], - }, - ) - .await?; - let Some(row) = result.output.first().and_then(|set| set.rows.first()) else { - return Ok(Observed { - output: None, - receipt: result.receipt, - }); - }; - let [ - SqlValue::Text(kind), - body, - SqlValue::Blob(digest), - SqlValue::Integer(size), - upload, - SqlValue::Text(storage), - locator, - ] = row.as_slice() - else { - return Err(cellule_runtime::InvocationError::NotStarted( - Error::Command("invalid stored object row"), - )); - }; - let kind = match kind.as_str() { - "blob" => ObjectKind::Blob, - "tree" => ObjectKind::Tree, - "commit" => ObjectKind::Commit, - "tag" => ObjectKind::Tag, - _ => { - return Err(cellule_runtime::InvocationError::NotStarted( - Error::Command("invalid stored object kind"), - )); - } - }; - if storage == "packed" { - let SqlValue::Blob(pack) = locator else { - return Err(cellule_runtime::InvocationError::NotStarted( - Error::Command("invalid pack locator"), - )); - }; - let pack: [u8; 32] = pack.as_slice().try_into().map_err(|_| { - cellule_runtime::InvocationError::NotStarted(Error::Command("invalid pack locator")) - })?; - if *size < 0 || *size > INLINE_OBJECT_LIMIT as i64 { - return Err(cellule_runtime::InvocationError::NotStarted( - Error::Command("object exceeds bounded body reader"), - )); - } - let record = self.pack_record(pack).await?; - let reader = self - .pack_readers - .lock() - .map_err(|_| { - cellule_runtime::InvocationError::NotStarted(Error::Command( - "packed reader registry poisoned", - )) - })? - .iter() - .find_map(std::sync::Weak::upgrade) - .ok_or(cellule_runtime::InvocationError::NotStarted( - Error::Command("packed reader unavailable"), - ))?; - let body = reader - .read_blob( - record, - oid, - *size as u64, - digest.as_slice().try_into().map_err(|_| { - cellule_runtime::InvocationError::NotStarted(Error::Command( - "invalid packed digest", - )) - })?, - ) - .await - .map_err(|error| { - cellule_runtime::InvocationError::NotStarted(Error::Facility { - name: "packed Git body", - source: Box::new(error), - }) - })?; - if kind != ObjectKind::Blob - || body.len() as i64 != *size - || object_id(oid.format(), kind, &body) != oid - || blake3::hash(&body).as_bytes() != digest.as_slice() - { - return Err(cellule_runtime::InvocationError::NotStarted( - Error::Command("corrupt packed Git body"), - )); - } - return Ok(Observed { - output: Some((kind, body)), - receipt: result.receipt, - }); - } - let body = match (body, upload) { - (SqlValue::Blob(bytes), SqlValue::Null) - if usize::try_from(*size).ok() == Some(bytes.len()) => - { - bytes.clone() - } - (SqlValue::Null, SqlValue::Blob(upload)) => { - let invalid = || { - cellule_runtime::InvocationError::NotStarted(Error::Command( - "invalid object chunk reference", - )) - }; - self.chunked_body( - oid, - kind, - upload.as_slice().try_into().map_err(|_| invalid())?, - u64::try_from(*size).map_err(|_| invalid())?, - digest.as_slice().try_into().map_err(|_| invalid())?, - ) - .await? - } - _ => { - return Err(cellule_runtime::InvocationError::NotStarted( - Error::Command("invalid stored object body"), - )); - } - }; - if object_id(oid.format(), kind, &body) != oid - || blake3::hash(&body).as_bytes() != digest.as_slice() - { - return Err(cellule_runtime::InvocationError::NotStarted( - Error::Command("corrupt stored object"), - )); - } - Ok(Observed { - output: Some((kind, body)), - receipt: result.receipt, - }) - } } /// Build evidence for the statically linked Canopy Cell application. diff --git a/crates/canopy-server/src/main.rs b/crates/canopy-server/src/main.rs index ecb4ca84..2211fe69 100644 --- a/crates/canopy-server/src/main.rs +++ b/crates/canopy-server/src/main.rs @@ -47,7 +47,7 @@ struct FileSshConfig { #[derive(Debug, Error)] enum StartupError { #[error( - "usage: canopy | canopy maintenance status | canopy maintenance begin|recover|end | canopy backup create|verify | canopy backup restore " + "usage: canopy | canopy maintenance status | canopy maintenance begin|recover|end | canopy gc | canopy backup create|verify | canopy backup restore " )] Usage, #[error("SSH host key is invalid")] @@ -118,6 +118,37 @@ async fn run() -> Result<(), StartupError> { let Some(first) = args.next() else { return Err(StartupError::Usage); }; + if first == "gc" { + let path = args.next().ok_or(StartupError::Usage)?; + let operation = args.next().ok_or(StartupError::Usage)?; + let grace = args.next().ok_or(StartupError::Usage)?; + let limit = args.next().ok_or(StartupError::Usage)?; + if args.next().is_some() { + return Err(StartupError::Usage); + } + let file: FileConfig = serde_json::from_slice(&std::fs::read(path)?)?; + let operation = RequestId::from_bytes( + Uuid::parse_str(operation.to_str().ok_or(StartupError::Usage)?)?.into_bytes(), + ); + let number = |value: &std::ffi::OsStr| -> Result { + value + .to_str() + .ok_or(StartupError::Usage)? + .parse() + .map_err(|_| StartupError::Usage) + }; + let deployment = deployment(&file)?; + let report = deployment + .collect_garbage( + operation, + worker_config(file)?, + number(&grace)?, + number(&limit)?, + ) + .await?; + println!("{}", serde_json::to_string(&report)?); + return Ok(()); + } if first == "backup" { let path = args.next().ok_or(StartupError::Usage)?; let action = args.next().ok_or(StartupError::Usage)?; diff --git a/crates/canopy-server/src/object_batch/mod.rs b/crates/canopy-server/src/object_batch/mod.rs deleted file mode 100644 index f698ad30..00000000 --- a/crates/canopy-server/src/object_batch/mod.rs +++ /dev/null @@ -1,401 +0,0 @@ -//! Bounded, atomic publication of verified repository object records. - -use std::collections::BTreeSet; - -use cellule_runtime::{ - CellModule, Command, Committed, Error, InvocationError, MutationIdentity, Observed, - codec::BoundedDecoder, codec::BoundedEncoder, codec::CodecError, codec::WireValue, - primitives::sql::SqlBatch, primitives::sql::SqlResultSet, primitives::sql::SqlStatement, - primitives::sql::SqlValue, registry::CommandContext, registry::CommandResult, -}; - -use crate::{ - INLINE_OBJECT_LIMIT, ObjectKind, ObjectStorage, RepositoryCell, RepositoryModule, StoredObject, - object_id, -}; - -pub(crate) const MAX_OBJECTS: usize = 128; -// Publication amortizes durable commits independently of bounded read pages. -pub(crate) const MAX_BATCH_OBJECTS: usize = 2048; -pub(crate) const VERIFY_BATCH_BYTES: u64 = 64 * 1024 * 1024; -#[cfg(test)] -pub(crate) const INPUT_LIMIT: u32 = 4 * 1024 * 1024; -// Leave room for record metadata inside Cellule's bounded command wire format. -const INLINE_BATCH_BYTES: usize = 3 * 1024 * 1024; - -/// Batches small objects together and verifies oversized SQLite objects individually. -#[derive(Default)] -pub struct ObjectBatch { - objects: Vec, - inline_bytes: usize, - verified_bytes: u64, -} - -impl ObjectBatch { - /// Adds a record, returning it unchanged if the batch's count or byte budget is full. - pub fn try_push(&mut self, object: StoredObject) -> Result<(), StoredObject> { - let bytes = match &object.storage { - ObjectStorage::Inline(body) => body.len(), - ObjectStorage::Packed { .. } - | ObjectStorage::External { .. } - | ObjectStorage::Chunked { .. } => 0, - }; - let verified = match &object.storage { - ObjectStorage::Inline(body) => body.len() as u64, - ObjectStorage::Chunked { size, .. } => *size, - ObjectStorage::External { .. } | ObjectStorage::Packed { .. } => 0, - }; - if self.objects.len() == MAX_BATCH_OBJECTS - || bytes > INLINE_OBJECT_LIMIT - || bytes > INLINE_BATCH_BYTES - self.inline_bytes - || i64::try_from(verified).is_err() - || (!self.objects.is_empty() - && verified > VERIFY_BATCH_BYTES.saturating_sub(self.verified_bytes)) - { - return Err(object); - } - self.inline_bytes += bytes; - self.verified_bytes += verified; - self.objects.push(object); - Ok(()) - } - - #[must_use] - pub fn is_empty(&self) -> bool { - self.objects.is_empty() - } -} - -impl WireValue for ObjectBatch { - fn encode(&self, encoder: &mut BoundedEncoder) -> Result<(), CodecError> { - if self.is_empty() { - return Err(CodecError::Invalid("object batch is empty")); - } - encoder.write_count(self.objects.len())?; - for object in &self.objects { - encoder.write_bytes(&object.oid)?; - encoder.write_u8(match object.kind { - ObjectKind::Blob => 0, - ObjectKind::Tree => 1, - ObjectKind::Commit => 2, - ObjectKind::Tag => 3, - })?; - match &object.storage { - ObjectStorage::Inline(body) => { - encoder.write_u8(0)?; - encoder.write_bytes(body)?; - } - ObjectStorage::Chunked { - upload, - size, - blake3, - } => { - encoder.write_u8(2)?; - encoder.write_bytes(upload)?; - encoder.write_u64(*size)?; - encoder.write_bytes(blake3)?; - } - ObjectStorage::Packed { size, blake3, pack } => { - encoder.write_u8(3)?; - encoder.write_u64(*size)?; - encoder.write_bytes(blake3)?; - encoder.write_bytes(pack)?; - } - ObjectStorage::External { - size, - blake3, - sha256, - } => { - encoder.write_u8(1)?; - encoder.write_u64(*size)?; - encoder.write_bytes(blake3)?; - encoder.write_bytes(sha256)?; - } - } - } - Ok(()) - } - - fn decode(decoder: &mut BoundedDecoder<'_>) -> Result { - let count = decoder.read_count()?; - if !(1..=MAX_BATCH_OBJECTS).contains(&count) { - return Err(CodecError::Invalid("object count is outside batch bounds")); - } - let mut batch = Self::default(); - for _ in 0..count { - let oid = crate::ObjectId::try_from(decoder.read_bytes()?) - .map_err(|_| CodecError::Invalid("invalid Git object ID"))?; - let kind = match decoder.read_u8()? { - 0 => ObjectKind::Blob, - 1 => ObjectKind::Tree, - 2 => ObjectKind::Commit, - 3 => ObjectKind::Tag, - _ => return Err(CodecError::Invalid("invalid Git object kind")), - }; - let storage = match decoder.read_u8()? { - 0 => { - let body = decoder.read_bytes()?; - if body.len() > INLINE_OBJECT_LIMIT - || body.len() > INLINE_BATCH_BYTES - batch.inline_bytes - { - return Err(CodecError::Invalid( - "object batch exceeds inline byte budget", - )); - } - ObjectStorage::Inline(body.to_vec()) - } - 1 => ObjectStorage::External { - size: decoder.read_u64()?, - blake3: fixed(decoder)?, - sha256: fixed(decoder)?, - }, - 3 => ObjectStorage::Packed { - size: decoder.read_u64()?, - blake3: fixed(decoder)?, - pack: fixed(decoder)?, - }, - 2 => ObjectStorage::Chunked { - upload: fixed(decoder)?, - size: decoder.read_u64()?, - blake3: fixed(decoder)?, - }, - _ => return Err(CodecError::Invalid("invalid Git object storage")), - }; - batch - .try_push(StoredObject { oid, kind, storage }) - .map_err(|_| CodecError::Invalid("object batch exceeds bounds"))?; - } - Ok(batch) - } -} - -fn fixed(decoder: &mut BoundedDecoder<'_>) -> Result<[u8; N], CodecError> { - decoder - .read_bytes()? - .try_into() - .map_err(|_| CodecError::Invalid("invalid object identity width")) -} - -pub(crate) struct PutObjects; - -impl Command for PutObjects { - const MODULE: &'static str = RepositoryModule::NAME; - const ID: u32 = 5; - const CODEC_VERSION: u32 = 5; - type Input = ObjectBatch; - type Output = (); - - fn execute( - context: &mut CommandContext<'_, '_>, - batch: Self::Input, - ) -> cellule_runtime::Result> { - let identity = context.sql(&SqlBatch { - statements: vec![SqlStatement { - sql: "SELECT object_format FROM repository_identity WHERE singleton = 1".into(), - parameters: Vec::new(), - }], - })?; - let Some([SqlValue::Text(format)]) = identity - .first() - .and_then(|set| set.rows.first()) - .map(Vec::as_slice) - else { - return Err(Error::Command("repository identity is absent")); - }; - let format = crate::ObjectFormat::parse(format) - .ok_or(Error::Command("invalid repository object format"))?; - if batch - .objects - .iter() - .any(|object| object.oid.format() != format) - { - return Ok(CommandResult::Rejected(())); - } - for object in batch.objects { - let (size, digest, storage, body, sha256, chunk_id) = match object.storage { - ObjectStorage::Inline(body) => { - if object_id(object.oid.format(), object.kind, &body) != object.oid { - return Ok(CommandResult::Rejected(())); - } - ( - body.len() as i64, - blake3::hash(&body).as_bytes().to_vec(), - "inline", - SqlValue::Blob(body), - SqlValue::Null, - SqlValue::Null, - ) - } - ObjectStorage::Packed { size, blake3, pack } => { - if object.kind != ObjectKind::Blob || i64::try_from(size).is_err() { - return Ok(CommandResult::Rejected(())); - } - let result = context.sql(&SqlBatch { - statements: vec![SqlStatement { - sql: "SELECT 1 FROM git_packs WHERE sha256 = ?1".into(), - parameters: vec![SqlValue::Blob(pack.to_vec())], - }], - })?; - if result.first().is_none_or(|set| set.rows.is_empty()) { - return Ok(CommandResult::Rejected(())); - } - ( - size as i64, - blake3.to_vec(), - "packed", - SqlValue::Null, - SqlValue::Blob(pack.to_vec()), - SqlValue::Null, - ) - } - ObjectStorage::External { - size, - blake3, - sha256, - } => { - if object.kind != ObjectKind::Blob || i64::try_from(size).is_err() { - return Ok(CommandResult::Rejected(())); - } - ( - size as i64, - blake3.to_vec(), - "external", - SqlValue::Null, - SqlValue::Blob(sha256.to_vec()), - SqlValue::Null, - ) - } - ObjectStorage::Chunked { - upload, - size, - blake3, - } => { - if crate::object_chunks::body( - context, - object.oid, - object.kind, - upload, - size, - blake3, - )? - .is_none() - { - return Ok(CommandResult::Rejected(())); - } - ( - size as i64, - blake3.to_vec(), - "chunked", - SqlValue::Null, - SqlValue::Null, - SqlValue::Blob(upload.to_vec()), - ) - } - }; - let expected = vec![ - SqlValue::Text(object.kind.git_name().into()), - SqlValue::Integer(size), - SqlValue::Blob(digest), - SqlValue::Text(storage.into()), - body, - sha256, - ]; - let mut parameters = vec![SqlValue::Blob(object.oid.to_vec())]; - parameters.extend(expected.iter().cloned()); - parameters.push(chunk_id); - let result = context.sql(&SqlBatch { statements: vec![ - SqlStatement { - sql: "INSERT INTO objects (oid, kind, size, digest, storage, body, external_sha256, chunk_id) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) ON CONFLICT(oid) DO NOTHING".into(), - parameters, - }, - SqlStatement { - sql: "SELECT kind, size, digest, storage, body, external_sha256 FROM objects WHERE oid = ?1".into(), - parameters: vec![SqlValue::Blob(object.oid.to_vec())], - }, - ]})?; - // A collision or corrupt existing record rejects the entire application - // savepoint. No earlier record in this batch may survive that rejection. - if result.get(1).and_then(|set| set.rows.first()) != Some(&expected) { - return Ok(CommandResult::Rejected(())); - } - // Verified blobs are leaves. Certify them in this same savepoint, - // avoiding another full read/hash of every historical blob later. - if object.kind == ObjectKind::Blob { - context.sql(&SqlBatch { - statements: vec![SqlStatement { - sql: "INSERT INTO object_closure (oid) VALUES (?1) ON CONFLICT DO NOTHING" - .into(), - parameters: vec![SqlValue::Blob(object.oid.to_vec())], - }], - })?; - } - } - Ok(CommandResult::Success(())) - } -} - -impl RepositoryCell { - /// Publishes a bounded object batch atomically; external bytes must already be verified and uploaded. - pub async fn put_objects( - &self, - identity: MutationIdentity, - batch: ObjectBatch, - ) -> Result, InvocationError<()>> { - self.application - .command::(&self.target, identity, batch) - .await - } - - /// Returns recorded IDs from a nonempty candidate list of at most 128 objects. - pub async fn existing_objects( - &self, - candidates: &[crate::ObjectId], - ) -> Result>, InvocationError>> { - if !(1..=MAX_OBJECTS).contains(&candidates.len()) { - return Err(InvocationError::NotStarted(Error::Command( - "object lookup count is outside bounds", - ))); - } - let placeholders = vec!["?"; candidates.len()].join(","); - let result = self - .sql - .query( - None, - SqlBatch { - statements: vec![SqlStatement { - sql: format!("SELECT oid FROM objects WHERE oid IN ({placeholders})"), - parameters: candidates - .iter() - .map(|oid| SqlValue::Blob(oid.to_vec())) - .collect(), - }], - }, - ) - .await?; - let rows = &result - .output - .first() - .ok_or(InvocationError::NotStarted(Error::Command( - "missing object lookup result", - )))? - .rows; - let mut objects = BTreeSet::new(); - for row in rows { - let [SqlValue::Blob(oid)] = row.as_slice() else { - return Err(InvocationError::NotStarted(Error::Command( - "invalid object lookup row", - ))); - }; - objects.insert(oid.as_slice().try_into().map_err(|_| { - InvocationError::NotStarted(Error::Command("invalid stored Git OID")) - })?); - } - Ok(Observed { - output: objects, - receipt: result.receipt, - }) - } -} - -#[cfg(test)] -mod tests; diff --git a/crates/canopy-server/src/object_batch/tests.rs b/crates/canopy-server/src/object_batch/tests.rs deleted file mode 100644 index 6b5bf95e..00000000 --- a/crates/canopy-server/src/object_batch/tests.rs +++ /dev/null @@ -1,169 +0,0 @@ -use super::*; - -fn inline(size: usize) -> StoredObject { - StoredObject { - oid: crate::ObjectId::Sha1([0; 20]), - kind: ObjectKind::Blob, - storage: ObjectStorage::Inline(vec![11; size]), - } -} - -#[test] -fn maximum_batch_round_trips_below_the_operation_wire_limit() { - let mut batch = ObjectBatch::default(); - for _ in 0..MAX_BATCH_OBJECTS - 1 { - assert!( - batch - .try_push(StoredObject { - oid: crate::ObjectId::Sha1([1; 20]), - kind: ObjectKind::Blob, - storage: ObjectStorage::External { - size: 1_000_000, - blake3: [2; 32], - sha256: [3; 32] - }, - }) - .is_ok() - ); - } - assert!(batch.try_push(inline(INLINE_OBJECT_LIMIT)).is_ok()); - let mut encoder = BoundedEncoder::new(INPUT_LIMIT).unwrap(); - batch.encode(&mut encoder).unwrap(); - let encoded = encoder.finish(); - assert!(encoded.len() < INPUT_LIMIT as usize); - let mut decoder = BoundedDecoder::new(&encoded, INPUT_LIMIT).unwrap(); - let decoded = ObjectBatch::decode(&mut decoder).unwrap(); - decoder.finish().unwrap(); - let mut encoder = BoundedEncoder::new(INPUT_LIMIT).unwrap(); - decoded.encode(&mut encoder).unwrap(); - assert_eq!(encoder.finish(), encoded); -} - -#[test] -fn a_full_batch_returns_the_unconsumed_record() { - let mut batch = ObjectBatch::default(); - assert!(batch.try_push(inline(INLINE_OBJECT_LIMIT + 1)).is_err()); - for _ in 0..INLINE_BATCH_BYTES / INLINE_OBJECT_LIMIT { - assert!(batch.try_push(inline(INLINE_OBJECT_LIMIT)).is_ok()); - } - let leftover = match batch.try_push(inline(1)) { - Err(object) => object, - Ok(()) => panic!("byte budget must reject the record"), - }; - let mut next = ObjectBatch::default(); - assert!(next.try_push(leftover).is_ok()); - for _ in 1..MAX_BATCH_OBJECTS { - assert!(next.try_push(inline(0)).is_ok()); - } - assert!(next.try_push(inline(0)).is_err()); -} - -#[test] -fn decoding_enforces_aggregate_bytes_and_count_before_building_a_batch() { - for count in [0, MAX_BATCH_OBJECTS + 1] { - let mut encoder = BoundedEncoder::new(INPUT_LIMIT).unwrap(); - encoder.write_count(count).unwrap(); - let bytes = encoder.finish(); - let mut decoder = BoundedDecoder::new(&bytes, INPUT_LIMIT).unwrap(); - assert!(ObjectBatch::decode(&mut decoder).is_err()); - } - let mut encoder = BoundedEncoder::new(INPUT_LIMIT).unwrap(); - encoder.write_count(5).unwrap(); - for size in [ - INLINE_OBJECT_LIMIT, - INLINE_OBJECT_LIMIT, - INLINE_OBJECT_LIMIT, - INLINE_OBJECT_LIMIT, - 1, - ] { - encoder.write_bytes(&[0; 20]).unwrap(); - encoder.write_u8(0).unwrap(); // Blob. - encoder.write_u8(0).unwrap(); // Inline. - encoder.write_bytes(&vec![5; size]).unwrap(); - } - let bytes = encoder.finish(); - let mut decoder = BoundedDecoder::new(&bytes, INPUT_LIMIT).unwrap(); - assert!(ObjectBatch::decode(&mut decoder).is_err()); - let mut encoder = BoundedEncoder::new(INPUT_LIMIT).unwrap(); - encoder.write_count(1).unwrap(); - encoder.write_bytes(&[0; 20]).unwrap(); - encoder.write_u8(0).unwrap(); - encoder.write_u8(0).unwrap(); - encoder - .write_bytes(&vec![5; INLINE_OBJECT_LIMIT + 1]) - .unwrap(); - let bytes = encoder.finish(); - let mut decoder = BoundedDecoder::new(&bytes, INPUT_LIMIT).unwrap(); - assert!(ObjectBatch::decode(&mut decoder).is_err()); - assert!( - ObjectBatch::default() - .encode(&mut BoundedEncoder::new(1 << 20).unwrap()) - .is_err() - ); -} - -#[test] -fn chunk_references_bound_total_verification_bytes_and_round_trip() { - let chunked = |size| StoredObject { - oid: crate::ObjectId::Sha1([1; 20]), - kind: ObjectKind::Commit, - storage: ObjectStorage::Chunked { - upload: [2; 16], - size, - blake3: [3; 32], - }, - }; - let mut batch = ObjectBatch::default(); - assert!(batch.try_push(chunked(VERIFY_BATCH_BYTES)).is_ok()); - assert!(batch.try_push(inline(1)).is_err()); - let mut encoded = BoundedEncoder::new(1 << 20).unwrap(); - batch.encode(&mut encoded).unwrap(); - let encoded = encoded.finish(); - let mut decoder = BoundedDecoder::new(&encoded, 1 << 20).unwrap(); - let mut decoded = ObjectBatch::decode(&mut decoder).unwrap(); - decoder.finish().unwrap(); - assert!(decoded.try_push(chunked(1)).is_err()); - let mut malicious = BoundedEncoder::new(1 << 20).unwrap(); - malicious.write_count(2).unwrap(); - for size in [VERIFY_BATCH_BYTES, 1] { - malicious.write_bytes(&[1; 20]).unwrap(); - malicious.write_u8(2).unwrap(); - malicious.write_u8(2).unwrap(); - malicious.write_bytes(&[2; 16]).unwrap(); - malicious.write_u64(size).unwrap(); - malicious.write_bytes(&[3; 32]).unwrap(); - } - let malicious = malicious.finish(); - assert!(ObjectBatch::decode(&mut BoundedDecoder::new(&malicious, 1 << 20).unwrap()).is_err()); - let mut oversized = ObjectBatch::default(); - assert!(oversized.try_push(chunked(u64::MAX)).is_err()); -} - -#[test] -fn packed_metadata_batch_preserves_immutable_locator_without_blob_expansion() { - let mut batch = ObjectBatch::default(); - for _ in 0..MAX_BATCH_OBJECTS { - batch - .try_push(StoredObject { - oid: crate::ObjectId::Sha1([1; 20]), - kind: ObjectKind::Blob, - storage: ObjectStorage::Packed { - size: INLINE_OBJECT_LIMIT as u64, - blake3: [2; 32], - pack: [3; 32], - }, - }) - .ok() - .unwrap(); - } - let mut encoder = BoundedEncoder::new(INPUT_LIMIT).unwrap(); - batch.encode(&mut encoder).unwrap(); - let bytes = encoder.finish(); - assert!(bytes.len() < 256 * 1024); - let mut decoder = BoundedDecoder::new(&bytes, INPUT_LIMIT).unwrap(); - let decoded = ObjectBatch::decode(&mut decoder).unwrap(); - decoder.finish().unwrap(); - let mut encoder = BoundedEncoder::new(INPUT_LIMIT).unwrap(); - decoded.encode(&mut encoder).unwrap(); - assert_eq!(encoder.finish(), bytes); -} diff --git a/crates/canopy-server/src/object_chunks/mod.rs b/crates/canopy-server/src/object_chunks/mod.rs deleted file mode 100644 index aacd4fb9..00000000 --- a/crates/canopy-server/src/object_chunks/mod.rs +++ /dev/null @@ -1,230 +0,0 @@ -//! Staged SQLite chunks, verified before an immutable object record is published. - -use cellule_runtime::{ - Error, InvocationError, MutationIdentity, identity::RequestId, primitives::sql::SqlBatch, - primitives::sql::SqlResultSet, primitives::sql::SqlStatement, primitives::sql::SqlValue, - registry::CommandContext, -}; - -use crate::{ - INLINE_OBJECT_LIMIT, ObjectKind, ObjectStorage, RepositoryCell, StoredObject, object_id, -}; - -pub(crate) const CHUNK_BYTES: usize = 512 * 1024; - -#[derive(Debug, thiserror::Error)] -pub enum ObjectStageError { - #[error("only non-blob objects above the inline limit can be staged")] - Invalid, - #[error("object chunk staging failed")] - Cell(#[from] InvocationError>), -} - -impl RepositoryCell { - /// Stages a large non-blob object in SQLite; put_objects must verify and publish the returned record. - pub async fn stage_object( - &self, - identity: MutationIdentity, - kind: ObjectKind, - body: &[u8], - ) -> Result { - if kind == ObjectKind::Blob - || body.len() <= INLINE_OBJECT_LIMIT - || i64::try_from(body.len()).is_err() - { - return Err(ObjectStageError::Invalid); - } - let upload = *identity.request_id.as_bytes(); - let mut statements = vec![SqlStatement { - sql: "INSERT INTO object_uploads (id) VALUES (?1)".into(), - parameters: vec![SqlValue::Blob(upload.to_vec())], - }]; - for (part, bytes) in body.chunks(CHUNK_BYTES).enumerate() { - statements.push(SqlStatement { - sql: "INSERT INTO object_chunks (upload_id, part, body) VALUES (?1, ?2, ?3)".into(), - parameters: vec![ - SqlValue::Blob(upload.to_vec()), - SqlValue::Integer(part as i64), - SqlValue::Blob(bytes.to_vec()), - ], - }); - let mut hash = blake3::Hasher::new(); - hash.update(b"canopy-object-chunk-v1"); - hash.update(&upload); - hash.update(&(part as u64).to_le_bytes()); - let mut request_id = [0; 16]; - request_id.copy_from_slice(&hash.finalize().as_bytes()[..16]); - self.sql - .batch( - MutationIdentity { - request_id: RequestId::from_bytes(request_id), - issued_at_ms: identity.issued_at_ms, - expires_at_ms: identity.expires_at_ms, - }, - SqlBatch { - statements: std::mem::take(&mut statements), - }, - ) - .await?; - } - Ok(StoredObject { - oid: object_id(self.object_format(), kind, body), - kind, - storage: ObjectStorage::Chunked { - upload, - size: body.len() as u64, - blake3: *blake3::hash(body).as_bytes(), - }, - }) - } - - pub(crate) async fn chunked_body( - &self, - oid: crate::ObjectId, - kind: ObjectKind, - upload: [u8; 16], - size: u64, - digest: [u8; 32], - ) -> Result, InvocationError>> { - let invalid = - || InvocationError::NotStarted(Error::Command("incomplete or corrupt object chunks")); - let mut chunks = Chunks::new(oid, kind, upload, size, digest).ok_or_else(invalid)?; - while !chunks.complete() { - let result = self.sql.query(None, chunks.query()).await?; - if !chunks.append(&result.output) { - return Err(invalid()); - } - } - chunks.finish().ok_or_else(invalid) - } -} - -/// The caller's command transaction binds verification and publication atomically. -pub(crate) fn body( - context: &CommandContext<'_, '_>, - oid: crate::ObjectId, - kind: ObjectKind, - upload: [u8; 16], - size: u64, - digest: [u8; 32], -) -> cellule_runtime::Result>> { - let Some(mut chunks) = Chunks::new(oid, kind, upload, size, digest) else { - return Ok(None); - }; - while !chunks.complete() { - let result = context.sql(&chunks.snapshot_query())?; - if !chunks.append_snapshot(&result) { - return Ok(None); - } - } - Ok(chunks.finish()) -} - -struct Chunks { - oid: crate::ObjectId, - kind: ObjectKind, - upload: [u8; 16], - size: usize, - digest: [u8; 32], - body: Vec, - part: usize, -} - -impl Chunks { - fn new( - oid: crate::ObjectId, - kind: ObjectKind, - upload: [u8; 16], - size: u64, - digest: [u8; 32], - ) -> Option { - let size = usize::try_from(size).ok()?; - if kind == ObjectKind::Blob || size <= INLINE_OBJECT_LIMIT || i64::try_from(size).is_err() { - return None; - } - Some(Self { - oid, - kind, - upload, - size, - digest, - body: Vec::new(), - part: 0, - }) - } - - fn query(&self) -> SqlBatch { - SqlBatch { statements: vec![SqlStatement { - // Count also rejects extra chunks; each result carries at most one 512 KiB body. - sql: "SELECT body, (SELECT COUNT(*) FROM object_chunks WHERE upload_id = ?1) FROM object_chunks WHERE upload_id = ?1 AND part = ?2".into(), - parameters: vec![SqlValue::Blob(self.upload.to_vec()), SqlValue::Integer(self.part as i64)], - }] } - } - - fn snapshot_query(&self) -> SqlBatch { - if self.part == 0 { - return self.query(); - } - // The command transaction keeps the count checked by the first read - // stable. Recounting every part scans this upload quadratically. - SqlBatch { - statements: vec![SqlStatement { - sql: "SELECT body FROM object_chunks WHERE upload_id = ?1 AND part = ?2".into(), - parameters: vec![ - SqlValue::Blob(self.upload.to_vec()), - SqlValue::Integer(self.part as i64), - ], - }], - } - } - - fn append_snapshot(&mut self, results: &[SqlResultSet]) -> bool { - if self.part == 0 { - return self.append(results); - } - let Some([SqlValue::Blob(bytes)]) = results - .first() - .and_then(|set| set.rows.first()) - .map(Vec::as_slice) - else { - return false; - }; - self.append_bytes(bytes) - } - - fn append(&mut self, results: &[SqlResultSet]) -> bool { - let Some([SqlValue::Blob(bytes), SqlValue::Integer(count)]) = results - .first() - .and_then(|set| set.rows.first()) - .map(Vec::as_slice) - else { - return false; - }; - if *count != self.size.div_ceil(CHUNK_BYTES) as i64 { - return false; - } - self.append_bytes(bytes) - } - - fn append_bytes(&mut self, bytes: &[u8]) -> bool { - if bytes.len() != CHUNK_BYTES.min(self.size - self.body.len()) { - return false; - } - self.body.extend_from_slice(bytes); - self.part += 1; - true - } - - fn complete(&self) -> bool { - self.body.len() == self.size - } - - fn finish(self) -> Option> { - (object_id(self.oid.format(), self.kind, &self.body) == self.oid - && blake3::hash(&self.body).as_bytes() == &self.digest) - .then_some(self.body) - } -} - -#[cfg(test)] -mod tests; diff --git a/crates/canopy-server/src/object_chunks/tests.rs b/crates/canopy-server/src/object_chunks/tests.rs deleted file mode 100644 index f5a449b1..00000000 --- a/crates/canopy-server/src/object_chunks/tests.rs +++ /dev/null @@ -1,98 +0,0 @@ -use super::*; - -fn results(row: Vec) -> [SqlResultSet; 1] { - [SqlResultSet { - columns: Vec::new(), - rows: vec![row], - rows_affected: 0, - }] -} - -#[test] -fn snapshot_verification_counts_the_upload_only_once() { - let bytes = vec![17; CHUNK_BYTES * 3 + 1]; - let oid = object_id(crate::ObjectFormat::Sha1, ObjectKind::Commit, &bytes); - let mut chunks = Chunks::new( - oid, - ObjectKind::Commit, - [7; 16], - bytes.len() as u64, - *blake3::hash(&bytes).as_bytes(), - ) - .unwrap(); - for (part, body) in bytes.chunks(CHUNK_BYTES).enumerate() { - let query = chunks.snapshot_query(); - assert_eq!(query.statements[0].sql.contains("COUNT(*)"), part == 0); - // Separate async reads have no shared snapshot: keep their count check. - assert!(chunks.query().statements[0].sql.contains("COUNT(*)")); - let mut row = vec![SqlValue::Blob(body.to_vec())]; - if part == 0 { - row.push(SqlValue::Integer(bytes.len().div_ceil(CHUNK_BYTES) as i64)); - } - assert!(chunks.append_snapshot(&results(row))); - } - assert!(chunks.complete()); - assert_eq!(chunks.finish(), Some(bytes)); -} - -#[test] -fn snapshot_verification_rejects_extra_missing_and_malformed_chunks() { - let bytes = vec![17; INLINE_OBJECT_LIMIT + 1]; - let oid = object_id(crate::ObjectFormat::Sha1, ObjectKind::Commit, &bytes); - let mut chunks = Chunks::new( - oid, - ObjectKind::Commit, - [7; 16], - bytes.len() as u64, - *blake3::hash(&bytes).as_bytes(), - ) - .unwrap(); - let first = bytes[..CHUNK_BYTES].to_vec(); - assert!(!chunks.append_snapshot(&results(vec![ - SqlValue::Blob(first.clone()), - SqlValue::Integer(3) - ]))); - assert_eq!(chunks.part, 0); - assert!(chunks.append_snapshot(&results(vec![SqlValue::Blob(first), SqlValue::Integer(2)]))); - assert!(!chunks.append_snapshot(&[])); - assert!(!chunks.append_snapshot(&results(vec![SqlValue::Blob(vec![17, 17])]))); - assert!(!chunks.append_snapshot(&results(vec![SqlValue::Integer(17)]))); - assert_eq!(chunks.part, 1); - assert!(chunks.append_snapshot(&results(vec![SqlValue::Blob( - bytes[CHUNK_BYTES..].to_vec() - )]))); - assert_eq!(chunks.finish(), Some(bytes)); -} - -#[test] -fn snapshot_verification_keeps_both_digest_checks() { - let bytes = vec![17; INLINE_OBJECT_LIMIT + 1]; - for format in [crate::ObjectFormat::Sha1, crate::ObjectFormat::Sha256] { - for corrupt_oid in [false, true] { - let oid = object_id( - format, - ObjectKind::Commit, - if corrupt_oid { - b"wrong identity" - } else { - &bytes - }, - ); - let mut digest = *blake3::hash(&bytes).as_bytes(); - if !corrupt_oid { - digest[0] ^= 1; - } - let mut chunks = - Chunks::new(oid, ObjectKind::Commit, [7; 16], bytes.len() as u64, digest).unwrap(); - assert!(chunks.append_snapshot(&results(vec![ - SqlValue::Blob(bytes[..CHUNK_BYTES].to_vec()), - SqlValue::Integer(2) - ]))); - assert!(chunks.append_snapshot(&results(vec![SqlValue::Blob( - bytes[CHUNK_BYTES..].to_vec() - )]))); - assert!(chunks.complete()); - assert!(chunks.finish().is_none()); - } - } -} diff --git a/crates/canopy-server/src/object_reads/mod.rs b/crates/canopy-server/src/object_reads/mod.rs deleted file mode 100644 index 5562563d..00000000 --- a/crates/canopy-server/src/object_reads/mod.rs +++ /dev/null @@ -1,242 +0,0 @@ -//! Bounded immutable object pages for cold cache hydration. - -use cellule_runtime::{ - Error, InvocationError, Observed, primitives::sql::SqlBatch, primitives::sql::SqlResultSet, - primitives::sql::SqlStatement, primitives::sql::SqlValue, -}; - -use crate::{ - INLINE_OBJECT_LIMIT, ObjectKind, ObjectStorage, RepositoryCell, StoredObject, - object_batch::MAX_OBJECTS, object_id, -}; - -pub(crate) struct ObjectHeaders { - pub(crate) objects: Observed>, -} - -impl RepositoryCell { - /// Reads at most 128 objects and 768 KiB of inline bodies, in OID order. - /// - /// Continue after the last OID until an empty page, including after short - /// pages. Objects are immutable; a future collector must fence this read. - pub async fn object_page( - &self, - after: Option, - ) -> Result>, InvocationError>> { - let headers = self.read_object_headers(None, SqlStatement { - sql: "SELECT sequence, oid, CASE WHEN storage = 'inline' THEN size ELSE 0 END FROM objects WHERE oid > ?1 ORDER BY oid LIMIT ?2".into(), - parameters: vec![SqlValue::Blob(after.map_or_else(Vec::new, |oid| oid.to_vec())), SqlValue::Integer(MAX_OBJECTS as i64)], - }).await?; - self.object_records(headers.objects).await - } - - pub(crate) async fn selected_objects( - &self, - ids: &[crate::ObjectId], - ) -> Result, InvocationError>> { - if ids.is_empty() || ids.len() > MAX_OBJECTS { - return Err(InvocationError::NotStarted(Error::Command( - "invalid object selection", - ))); - } - let placeholders = vec!["?"; ids.len()].join(","); - let headers = self.read_object_headers(None, SqlStatement { - sql: format!("SELECT sequence, oid, CASE WHEN storage = 'inline' THEN size ELSE 0 END FROM objects WHERE oid IN ({placeholders}) ORDER BY oid"), - parameters: ids.iter().map(|oid| SqlValue::Blob(oid.to_vec())).collect(), - }).await?; - Ok(self.object_records(headers.objects).await?.output) - } - - async fn read_object_headers( - &self, - minimum: Option, - statement: SqlStatement, - ) -> Result>> { - let headers = self - .sql - .query( - minimum, - SqlBatch { - statements: vec![statement], - }, - ) - .await?; - let rows = headers - .output - .first() - .ok_or_else(|| InvocationError::NotStarted(Error::Command("missing object headers")))?; - let (ids, _) = decode_headers(&rows.rows).map_err(InvocationError::NotStarted)?; - Ok(ObjectHeaders { - objects: Observed { - output: ids, - receipt: headers.receipt, - }, - }) - } - - pub(crate) async fn object_records( - &self, - headers: Observed>, - ) -> Result>, InvocationError>> { - // Callers may remove already cached IDs, but never add IDs: the header - // query's size bound and receipt protect this body read's wire ceiling. - let mut ids = headers.output; - ids.sort_unstable(); - if ids.is_empty() { - return Ok(Observed { - output: Vec::new(), - receipt: headers.receipt, - }); - } - let placeholders = vec!["?"; ids.len()].join(","); - // Immutable records bind this second read to the selected headers. The - // payload bound leaves room for record metadata under Cellule's 1 MiB cap. - let result = self.sql.query(Some(headers.receipt), SqlBatch { - statements: vec![SqlStatement { - sql: format!("SELECT oid, kind, size, digest, storage, body, external_sha256, chunk_id FROM objects WHERE oid IN ({placeholders}) ORDER BY oid"), - parameters: ids.iter().map(|oid| SqlValue::Blob(oid.to_vec())).collect(), - }], - }).await?; - let rows = result - .output - .into_iter() - .next() - .ok_or_else(|| InvocationError::NotStarted(Error::Command("missing object page")))? - .rows; - // Hash a whole bounded page off the async executor. Move SQL bodies into - // the result so page verification does not duplicate their payloads. - let output = tokio::task::spawn_blocking(move || { - let objects = rows - .into_iter() - .map(decode_object) - .collect::>>()?; - if objects.iter().map(|object| object.oid).ne(ids) { - return Err(Error::Command("objects changed during page read")); - } - Ok(objects) - }) - .await - .map_err(|error| { - InvocationError::NotStarted(Error::Facility { - name: "object page verification", - source: Box::new(error), - }) - })? - .map_err(InvocationError::NotStarted)?; - Ok(Observed { - output, - receipt: result.receipt, - }) - } -} - -fn decode_headers(rows: &[Vec]) -> cellule_runtime::Result<(Vec, i64)> { - let mut ids = Vec::new(); - let mut bytes = 0; - let mut through = 0; - for row in rows { - let [ - SqlValue::Integer(sequence), - SqlValue::Blob(oid), - SqlValue::Integer(size), - ] = row.as_slice() - else { - return Err(Error::Command("invalid object header")); - }; - if *sequence <= 0 { - return Err(Error::Command("invalid object sequence")); - } - let oid = oid - .as_slice() - .try_into() - .map_err(|_| Error::Command("invalid stored object ID"))?; - let size = usize::try_from(*size) - .ok() - .filter(|size| *size <= INLINE_OBJECT_LIMIT) - .ok_or(Error::Command("invalid inline object size"))?; - if size > INLINE_OBJECT_LIMIT - bytes { - break; - } - bytes += size; - ids.push(oid); - through = *sequence; - } - Ok((ids, through)) -} - -fn decode_object(row: Vec) -> cellule_runtime::Result { - let row: [SqlValue; 8] = row - .try_into() - .map_err(|_| Error::Command("invalid stored object row"))?; - let [ - SqlValue::Blob(oid), - SqlValue::Text(kind), - SqlValue::Integer(size), - SqlValue::Blob(digest), - SqlValue::Text(storage), - body, - external_sha256, - chunk_id, - ] = row - else { - return Err(Error::Command("invalid stored object row")); - }; - let oid: crate::ObjectId = oid - .try_into() - .map_err(|_| Error::Command("invalid stored object ID"))?; - let digest: [u8; 32] = digest - .try_into() - .map_err(|_| Error::Command("invalid object digest"))?; - let kind = match kind.as_str() { - "blob" => ObjectKind::Blob, - "tree" => ObjectKind::Tree, - "commit" => ObjectKind::Commit, - "tag" => ObjectKind::Tag, - _ => return Err(Error::Command("invalid stored object kind")), - }; - let storage = match (storage.as_str(), body, external_sha256, chunk_id) { - ("inline", SqlValue::Blob(body), SqlValue::Null, SqlValue::Null) - if usize::try_from(size).ok() == Some(body.len()) - && body.len() <= INLINE_OBJECT_LIMIT - && object_id(oid.format(), kind, &body) == oid - && blake3::hash(&body).as_bytes() == &digest => - { - ObjectStorage::Inline(body) - } - ("packed", SqlValue::Null, SqlValue::Blob(pack), SqlValue::Null) - if kind == ObjectKind::Blob && size >= 0 => - { - ObjectStorage::Packed { - size: size as u64, - blake3: digest, - pack: pack - .try_into() - .map_err(|_| Error::Command("invalid pack locator"))?, - } - } - ("external", SqlValue::Null, SqlValue::Blob(sha256), SqlValue::Null) - if kind == ObjectKind::Blob && size >= 0 => - { - ObjectStorage::External { - size: size as u64, - blake3: digest, - sha256: sha256 - .try_into() - .map_err(|_| Error::Command("invalid SHA-256 digest"))?, - } - } - ("chunked", SqlValue::Null, SqlValue::Null, SqlValue::Blob(upload)) - if kind != ObjectKind::Blob && size > INLINE_OBJECT_LIMIT as i64 => - { - ObjectStorage::Chunked { - upload: upload - .try_into() - .map_err(|_| Error::Command("invalid object chunk reference"))?, - size: size as u64, - blake3: digest, - } - } - _ => return Err(Error::Command("corrupt stored object")), - }; - Ok(StoredObject { oid, kind, storage }) -} diff --git a/crates/canopy-server/src/pack_store.rs b/crates/canopy-server/src/pack_store.rs deleted file mode 100644 index 1cab3aa1..00000000 --- a/crates/canopy-server/src/pack_store.rs +++ /dev/null @@ -1,337 +0,0 @@ -//! Immutable pack artifacts; SQLite retains canonical identities and closure. -use crate::{ - ObjectId, ObjectKind, RepositoryCell, - blob::{LargeBlobReference, LargeBlobStore}, - git_cache::GitCache, - git_gateway::GatewayError, - git_http::GitProcess, -}; -use cellule_ltx::DiskBudget; -use cellule_runtime::{ - Error, InvocationError, - primitives::sql::{SqlBatch, SqlResultSet, SqlStatement, SqlValue}, -}; -use object_store::ObjectStore; -use std::{path::PathBuf, sync::Arc}; -use tokio::sync::Mutex; - -#[derive(Clone)] -pub(crate) struct PackRecord { - pub hash: ObjectId, - pub pack: LargeBlobReference, - pub index: LargeBlobReference, - pub approved: bool, -} - -pub(crate) struct PackReader { - store: LargeBlobStore, - root: PathBuf, - budget: DiskBudget, - format: crate::ObjectFormat, - native: crate::native_resources::NativeScope, - cache: Mutex>>, - installation: Mutex<()>, - private: Mutex)>>, -} -impl PackReader { - pub(crate) fn new( - store: Arc, - repository: [u8; 16], - root: PathBuf, - budget: DiskBudget, - format: crate::ObjectFormat, - native: crate::native_resources::NativeScope, - ) -> Self { - Self { - store: LargeBlobStore::new(store, repository), - root, - budget, - format, - native, - cache: Mutex::new(None), - installation: Mutex::new(()), - private: Mutex::new(None), - } - } - pub(crate) async fn cache(&self) -> Result, GatewayError> { - let mut cache = self.cache.lock().await; - if cache.is_none() { - *cache = Some( - GitCache::create( - self.root.clone(), - self.budget.clone(), - "refs/heads/main", - self.format, - self.native.clone(), - ) - .await?, - ); - } - Ok(Arc::clone( - cache.as_ref().ok_or(GatewayError::MalformedCache)?, - )) - } - pub(crate) async fn install( - &self, - cache: &Arc, - record: &PackRecord, - ) -> Result<(), GatewayError> { - let _installation = self.installation.lock().await; - if cache.has_durable_pack(record.pack.sha256) { - return Ok(()); - } - cache - .install_pack( - record.hash, - self.store.read(&record.pack).await?, - self.store.read(&record.index).await?, - ) - .await?; - if record.approved { - cache.mark_durable_pack(record.pack.sha256); - } - Ok(()) - } - pub(crate) async fn native_reader( - &self, - record: PackRecord, - oid: ObjectId, - size: u64, - digest: [u8; 32], - ) -> Result { - // Incomplete archives stay in one bounded private cache. Their foreign - // members cannot enter a transport cache; each extracted body is checked. - let cache = if record.approved { - self.cache().await? - } else { - let mut private = self.private.lock().await; - if private - .as_ref() - .is_none_or(|(sha, _)| *sha != record.pack.sha256) - { - *private = Some(( - record.pack.sha256, - GitCache::create( - self.root.clone(), - self.budget.clone(), - "refs/heads/main", - self.format, - self.native.clone(), - ) - .await?, - )); - } - Arc::clone(&private.as_ref().ok_or(GatewayError::MalformedCache)?.1) - }; - self.install(&cache, &record).await?; - if !record.approved { - cache.mark_durable_pack(record.pack.sha256); - } - let mut command = crate::native_git::command(&cache.git_dir())?; - command - .args(["cat-file", "blob", &hex::encode(oid)]) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::null()); - let mut process = GitProcess::spawn( - command, - Arc::clone(&cache), - cache - .native - .try_admit(crate::native_resources::NativeWork::Read)?, - )?; - let output = process - .child - .stdout - .take() - .ok_or(GatewayError::MalformedCache)?; - Ok(NativePackedRead { - process, - output, - oid, - #[cfg(test)] - size, - remaining: size, - expected: digest, - canonical: Some(crate::git_format::ObjectHasher::new( - oid.format(), - ObjectKind::Blob, - size, - )), - digest: blake3::Hasher::new(), - finished: false, - }) - } - pub(crate) async fn read_blob( - &self, - record: PackRecord, - oid: ObjectId, - size: u64, - digest: [u8; 32], - ) -> Result, GatewayError> { - if size > crate::INLINE_OBJECT_LIMIT as u64 { - return Err(GatewayError::MalformedCache); - } - let mut reader = self.native_reader(record, oid, size, digest).await?; - let mut body = Vec::with_capacity(size as usize); - while let Some(bytes) = reader.next().await? { - body.extend_from_slice(&bytes); - } - Ok(body) - } - #[cfg(test)] - pub(crate) async fn upload(&self, path: PathBuf) -> Result { - let hash_path = path.clone(); - let (oid, size) = tokio::task::spawn_blocking(move || { - use std::io::Read; - let mut file = std::fs::File::open(hash_path)?; - let size = file.metadata()?.len(); - let mut hash = crate::git_format::ObjectHasher::new( - crate::ObjectFormat::Sha256, - ObjectKind::Blob, - size, - ); - let mut buffer = vec![0; 8 << 20]; - loop { - let count = file.read(&mut buffer)?; - if count == 0 { - break; - } - hash.update(&buffer[..count]); - } - Ok::<_, std::io::Error>((hash.finalize(), size)) - }) - .await??; - Ok(self - .store - .put(oid, size, &mut tokio::fs::File::open(path).await?) - .await?) - } -} - -/// Constant-memory extraction used only when an incomplete pack cannot be -/// admitted as a whole. Verification precedes the final returned body range. -pub(crate) struct NativePackedRead { - process: GitProcess>, - output: tokio::process::ChildStdout, - pub(crate) oid: ObjectId, - #[cfg(test)] - pub(crate) size: u64, - remaining: u64, - expected: [u8; 32], - canonical: Option, - digest: blake3::Hasher, - finished: bool, -} -impl NativePackedRead { - pub(crate) async fn next(&mut self) -> Result, GatewayError> { - use tokio::io::AsyncReadExt; - if self.finished { - return Ok(None); - } - let mut bytes = vec![0; self.remaining.min(8 << 20) as usize]; - tokio::time::timeout( - std::time::Duration::from_secs(120), - self.output.read_exact(&mut bytes), - ) - .await - .map_err(|_| crate::git_http::GitHttpError::Timeout)??; - self.remaining -= bytes.len() as u64; - self.canonical - .as_mut() - .ok_or(GatewayError::MalformedCache)? - .update(&bytes); - self.digest.update(&bytes); - if self.remaining == 0 { - let mut trailing = [0]; - let eof = tokio::time::timeout( - std::time::Duration::from_secs(120), - self.output.read(&mut trailing), - ) - .await - .map_err(|_| crate::git_http::GitHttpError::Timeout)??; - if eof != 0 - || self - .canonical - .take() - .ok_or(GatewayError::MalformedCache)? - .finalize() - != self.oid - || self.digest.finalize().as_bytes() != &self.expected - { - return Err(GatewayError::MalformedCache); - } - let status = - tokio::time::timeout(std::time::Duration::from_secs(120), self.process.wait()) - .await - .map_err(|_| crate::git_http::GitHttpError::Timeout)??; - if !status.success() { - return Err(GatewayError::MalformedCache); - } - self.finished = true; - } - Ok(Some(bytes.into())) - } -} - -const COLUMNS: &str = "sha256, pack_hash, pack_oid, pack_size, pack_digest, index_oid, index_size, index_digest, index_sha256, approved, covered_through"; -type ReadError = InvocationError>; -fn invalid() -> ReadError { - InvocationError::NotStarted(Error::Command("invalid durable pack record")) -} -fn decode(row: &[SqlValue]) -> Result { - let [ - SqlValue::Blob(sha), - SqlValue::Blob(hash), - SqlValue::Blob(pack_oid), - SqlValue::Integer(pack_size), - SqlValue::Blob(pack_digest), - SqlValue::Blob(index_oid), - SqlValue::Integer(index_size), - SqlValue::Blob(index_digest), - SqlValue::Blob(index_sha), - SqlValue::Integer(approved), - SqlValue::Integer(_covered_through), - ] = row - else { - return Err(invalid()); - }; - Ok(PackRecord { - hash: hash.as_slice().try_into().map_err(|_| invalid())?, - pack: LargeBlobReference { - oid: pack_oid.as_slice().try_into().map_err(|_| invalid())?, - size: (*pack_size).try_into().map_err(|_| invalid())?, - blake3: pack_digest.as_slice().try_into().map_err(|_| invalid())?, - sha256: sha.as_slice().try_into().map_err(|_| invalid())?, - }, - index: LargeBlobReference { - oid: index_oid.as_slice().try_into().map_err(|_| invalid())?, - size: (*index_size).try_into().map_err(|_| invalid())?, - blake3: index_digest.as_slice().try_into().map_err(|_| invalid())?, - sha256: index_sha.as_slice().try_into().map_err(|_| invalid())?, - }, - approved: *approved == 1, - }) -} -impl RepositoryCell { - pub(crate) async fn pack_record(&self, sha: [u8; 32]) -> Result { - let result = self - .sql - .query( - None, - SqlBatch { - statements: vec![SqlStatement { - sql: format!("SELECT {COLUMNS} FROM git_packs WHERE sha256 = ?1"), - parameters: vec![SqlValue::Blob(sha.to_vec())], - }], - }, - ) - .await?; - decode( - result - .output - .first() - .and_then(|set| set.rows.first()) - .ok_or_else(invalid)?, - ) - } -} diff --git a/crates/canopy-server/src/packs/catalog/files.rs b/crates/canopy-server/src/packs/catalog/files.rs index 47f464fc..bfd42195 100644 --- a/crates/canopy-server/src/packs/catalog/files.rs +++ b/crates/canopy-server/src/packs/catalog/files.rs @@ -30,7 +30,9 @@ impl Default for CatalogFileLimits { fn default() -> Self { Self { open_files: 32, - cached_files: 16, + // A medium full-history import can have 17 metadata shards plus + // directory files. Leave eight of the 32 slots for borrowed work. + cached_files: 24, metadata: MetadataLimits::default(), } } diff --git a/crates/canopy-server/src/packs/catalog/graph_spool.rs b/crates/canopy-server/src/packs/catalog/graph_spool.rs index 4e91a3fd..9dd1269b 100644 --- a/crates/canopy-server/src/packs/catalog/graph_spool.rs +++ b/crates/canopy-server/src/packs/catalog/graph_spool.rs @@ -40,7 +40,7 @@ impl GraphSpool { db.pragma_update(None, "cache_size", -(i64::from(cache_kib)))?; growth::configure(&db, &mut file)?; growth::transaction(&mut db, &mut file, maximum, |tx| { - tx.execute_batch("CREATE TABLE nodes(oid BLOB PRIMARY KEY,kind TEXT,done INTEGER NOT NULL DEFAULT 0 CHECK(done IN (0,1))) WITHOUT ROWID; CREATE INDEX pending ON nodes(done,oid); CREATE TABLE packs(checksum BLOB PRIMARY KEY,pack BLOB NOT NULL,idx BLOB NOT NULL,pack_bytes INTEGER NOT NULL,idx_bytes INTEGER NOT NULL,objects INTEGER NOT NULL) WITHOUT ROWID;")?; + tx.execute_batch("CREATE TABLE nodes(oid BLOB PRIMARY KEY,kind TEXT,done INTEGER NOT NULL DEFAULT 0 CHECK(done IN (0,1))) WITHOUT ROWID; CREATE INDEX pending ON nodes(done,oid); CREATE TABLE packs(checksum BLOB PRIMARY KEY,pack BLOB NOT NULL,idx BLOB NOT NULL,pack_bytes INTEGER NOT NULL,idx_bytes INTEGER NOT NULL,objects INTEGER NOT NULL,complete INTEGER NOT NULL DEFAULT 0 CHECK(complete IN (0,1))) WITHOUT ROWID;")?; Ok::<_, MetadataError>(()) })?; Ok(Self { @@ -172,7 +172,7 @@ impl GraphSpool { ) -> Result<(), MetadataError> { growth::transaction(&mut self.db, &mut self.file, self.maximum, |tx| { tx.execute( - "INSERT INTO packs VALUES(?1,?2,?3,?4,?5,?6)", + "INSERT INTO packs VALUES(?1,?2,?3,?4,?5,?6,0)", params![ p.git_checksum.as_ref(), p.pack.digest.as_slice(), @@ -185,6 +185,39 @@ impl GraphSpool { Ok::<_, MetadataError>(()) }) } + /// Logical source discovery does not mean its bytes were installed. Bind + /// physical completion to the exact verified pack/index identity separately. + pub(in crate::packs) fn pack_complete( + &self, + p: NativePackDescriptor, + ) -> Result { + if !self.pack_seen(p)? { + return Ok(false); + } + Ok(self.db.query_row( + "SELECT complete FROM packs WHERE checksum=?1", + [p.git_checksum.as_ref()], + |r| r.get(0), + )?) + } + pub(in crate::packs) fn mark_pack_complete( + &mut self, + p: NativePackDescriptor, + ) -> Result<(), MetadataError> { + if !self.pack_seen(p)? { + self.imported(p)?; + } + growth::transaction(&mut self.db, &mut self.file, self.maximum, |tx| { + if tx.execute( + "UPDATE packs SET complete=1 WHERE checksum=?1", + [p.git_checksum.as_ref()], + )? != 1 + { + return Err(MetadataError::Integrity); + } + Ok(()) + }) + } pub(in crate::packs) fn contains(&self, ids: &[ObjectId]) -> Result, MetadataError> { let mut q = self .db diff --git a/crates/canopy-server/src/packs/catalog/graph_spool/tests.rs b/crates/canopy-server/src/packs/catalog/graph_spool/tests.rs index bbe5e6e4..43c9adca 100644 --- a/crates/canopy-server/src/packs/catalog/graph_spool/tests.rs +++ b/crates/canopy-server/src/packs/catalog/graph_spool/tests.rs @@ -167,11 +167,16 @@ fn physical_input_dedup_ignores_namespace_but_rejects_conflicting_bindings() -> }, }; assert!(!s.pack_seen(p)?); + assert!(!s.pack_complete(p)?); s.imported(p)?; + assert!(!s.pack_complete(p)?); + s.mark_pack_complete(p)?; + assert!(s.pack_complete(p)?); let mut other = p; other.operation = [7; 16]; other.pack.manifest_digest = [8; 32]; assert!(s.pack_seen(other)?); + assert!(s.pack_complete(other)?); for mutation in 0..5 { let mut conflict = other; match mutation { @@ -185,7 +190,12 @@ fn physical_input_dedup_ignores_namespace_but_rejects_conflicting_bindings() -> s.pack_seen(conflict), Err(MetadataError::IdentityConflict) )); + assert!(matches!( + s.pack_complete(conflict), + Err(MetadataError::IdentityConflict) + )); } assert_eq!(s.counts()?, (0, 1)); + assert_eq!(s.contains(&[id(ObjectFormat::Sha1, 1)])?, [false]); Ok(()) } diff --git a/crates/canopy-server/src/packs/catalog/tests/files.rs b/crates/canopy-server/src/packs/catalog/tests/files.rs index b5df889a..d6df291c 100644 --- a/crates/canopy-server/src/packs/catalog/tests/files.rs +++ b/crates/canopy-server/src/packs/catalog/tests/files.rs @@ -50,6 +50,59 @@ async fn source(prepared: &Prepared) -> Result { ) } +#[tokio::test] +async fn default_cache_reuses_medium_repository_metadata_working_set() -> Result { + for format in [ObjectFormat::Sha1, ObjectFormat::Sha256] { + let prepared = prepared(format).await?; + let stored_run = run(&prepared).await?; + let objects: Vec<_> = prepared.fixture.objects.values().cloned().collect(); + let (&oid, (expected, _)) = prepared.fixture.objects.iter().next().ok_or("object")?; + let mut segments = Vec::new(); + // Seventeen distinct authenticated metadata files and one directory + // match the working-set shape that exceeded the former cache capacity. + for number in 80..97 { + let mut identity = prepared.fixture.identity; + identity.operation = [number; 16]; + let mut writer = builder(&prepared.fixture, DiskBudget::new(128 << 20), identity)?; + fill(&mut writer, &objects)?; + segments.push( + Arc::new(writer.seal(&prepared.fixture.index)?) + .upload(&prepared.store) + .await?, + ); + } + for (profile, expected_downloads) in [ + ( + CatalogFileLimits { + cached_files: 16, + ..CatalogFileLimits::default() + }, + 36, + ), + (CatalogFileLimits::default(), 18), + ] { + let budget = DiskBudget::new(128 << 20); + let loader = files(&prepared, budget.clone(), profile)?; + for pass in 0..2 { + drop(RunLoader::load(&loader, stored_run).await?); + for &stored in &segments { + let metadata = SourceLoader::load(&loader, stored).await?; + assert_eq!(metadata.header(oid)?.ok_or("header")?.object, *expected); + } + assert_eq!( + loader.stats()?.downloaded_files, + if pass == 0 { 18 } else { expected_downloads } + ); + assert!(loader.stats()?.open_files <= profile.open_files); + assert!(loader.stats()?.cached_files <= profile.cached_files); + } + drop(loader); + assert_eq!(budget.used(), 0); + } + } + Ok(()) +} + #[tokio::test] async fn worker_files_coalesce_native_lookups_and_reuse_cache_across_roots() -> Result { for format in [ObjectFormat::Sha1, ObjectFormat::Sha256] { diff --git a/crates/canopy-server/src/packs/closure/mod.rs b/crates/canopy-server/src/packs/closure/mod.rs index 1399199b..edae4f1e 100644 --- a/crates/canopy-server/src/packs/closure/mod.rs +++ b/crates/canopy-server/src/packs/closure/mod.rs @@ -34,6 +34,16 @@ use spool::Spool; pub use verifier::ClosureVerifier; pub use witness::ClosureWitness; +/// Aggregate incoming graphs contain the edges of every metadata shard. Keep +/// their disposable spool ceiling separate from immutable shard/read limits. +/// Growth still requires admission for database, journal and SQLite overhead. +pub fn default_limits() -> MetadataLimits { + MetadataLimits { + max_file_bytes: 1 << 30, + ..MetadataLimits::default() + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct ClosureBase { pub catalog: StoredCatalog, diff --git a/crates/canopy-server/src/packs/publication/prepare.rs b/crates/canopy-server/src/packs/publication/prepare.rs index 52ad8951..798ef97e 100644 --- a/crates/canopy-server/src/packs/publication/prepare.rs +++ b/crates/canopy-server/src/packs/publication/prepare.rs @@ -178,6 +178,11 @@ pub struct CatalogPreparation { input_checkpoint_digest: Option<[u8; 32]>, failed: bool, } +struct ConstructionLimits { + input: MetadataLimits, + closure: MetadataLimits, + output: MetadataLimits, +} impl CatalogPreparation { pub async fn new( root: &Path, @@ -206,16 +211,30 @@ impl CatalogPreparation { limits: MetadataLimits, output_limits: MetadataLimits, ) -> Result { - Self::new_inner(root, budget, base, limits, output_limits, None).await + Self::new_inner( + root, + budget, + base, + ConstructionLimits { + input: limits, + closure: limits, + output: output_limits, + }, + None, + ) + .await } /// Production construction retains the admitted worker through every /// detached assembler job, while the finished private proof owns no worker. + /// The aggregate graph has an explicit ceiling independent of input shards + /// and output runs; all three still share the caller's admitted disk budget. pub async fn new_staged( context: &StagingContext, root: &Path, budget: DiskBudget, base: Arc, limits: MetadataLimits, + closure_limits: MetadataLimits, ) -> Result { context.ensure_live().map_err(PhysicalError::from)?; if context.token().map_err(PhysicalError::from)? != base.context_token() @@ -227,10 +246,13 @@ impl CatalogPreparation { root, budget, base, - limits, - MetadataLimits { - max_file_bytes: limits.max_file_bytes.min(RUN_TARGET_BYTES), - ..limits + ConstructionLimits { + input: limits, + closure: closure_limits, + output: MetadataLimits { + max_file_bytes: limits.max_file_bytes.min(RUN_TARGET_BYTES), + ..limits + }, }, Some(context.clone()), ) @@ -240,10 +262,14 @@ impl CatalogPreparation { root: &Path, budget: DiskBudget, base: Arc, - limits: MetadataLimits, - output_limits: MetadataLimits, + limits: ConstructionLimits, staging: Option, ) -> Result { + let ConstructionLimits { + input: limits, + closure: closure_limits, + output: output_limits, + } = limits; DirectoryPartitioner::validate_limits(output_limits)?; let (lease, deadline) = base.live_lease()?; let indexes = base.indexes(); @@ -269,7 +295,7 @@ impl CatalogPreparation { Arc::clone(&workspace), budget.clone(), context, - limits, + closure_limits, activity.clone(), ) .await?; @@ -455,14 +481,14 @@ impl CatalogPreparation { &mut self, input: StagedNativeMetadata, ) -> Result<(), CatalogPreparationError> { - let deadline = self.start()?; + self.start()?; let context = self .staging .clone() .ok_or(CatalogPreparationError::Integrity)?; // The complete operation is poisoned on cancellation, including a // failed/absent provider artifact or an incomplete descriptor replay. - let result = timeout_at(deadline, async { + let work = async { let native = input.witness.native(); self.failed = false; self.begin_retained_pack(input.witness).await?; @@ -495,9 +521,14 @@ impl CatalogPreparation { context.ensure_live().map_err(PhysicalError::from)?; self.failed = false; self.finish_pack().await - }) - .await - .map_err(|_| PreparationBaseError::Inactive)?; + }; + // A full pack can span several renewed leases. Observe the shared + // custody clock and permanent fence instead of the first shard's + // deadline; the fixed bound residence ceiling still limits this work. + let result = tokio::select! { + result = work => result, + _ = context.fenced() => Err(PreparationBaseError::Inactive.into()), + }; if result.is_err() { self.failed = true; } @@ -527,9 +558,19 @@ impl CatalogPreparation { if self.active.is_some() { return Err(CatalogPreparationError::Integrity); } - timeout_at(deadline, self.finish_inner()) - .await - .map_err(|_| PreparationBaseError::Inactive)? + if let Some(context) = self.staging.clone() { + // Closure certification and immutable output construction can span + // renewed leases. Observe shared custody and its residence ceiling + // rather than the deadline captured at the start of this phase. + tokio::select! { + result = self.finish_inner() => result, + _ = context.fenced() => Err(PreparationBaseError::Inactive.into()), + } + } else { + timeout_at(deadline, self.finish_inner()) + .await + .map_err(|_| PreparationBaseError::Inactive)? + } } async fn finish_inner(mut self) -> Result { let context = self.base.context(); diff --git a/crates/canopy-server/src/packs/publication/registry.rs b/crates/canopy-server/src/packs/publication/registry.rs index 26505db1..500401e7 100644 --- a/crates/canopy-server/src/packs/publication/registry.rs +++ b/crates/canopy-server/src/packs/publication/registry.rs @@ -6,8 +6,8 @@ const fn command(input_limit: u32, output_limit: u32) -> OperationDe OperationDescriptor { id: C::ID, codec_version: C::CODEC_VERSION, - schema_min: 1, - schema_max: 1, + schema_min: crate::REPOSITORY_SCHEMA_VERSION, + schema_max: crate::REPOSITORY_SCHEMA_VERSION, input_limit, output_limit, } @@ -16,8 +16,8 @@ const fn query(input_limit: u32, output_limit: u32) -> OperationDescri OperationDescriptor { id: Q::ID, codec_version: Q::CODEC_VERSION, - schema_min: 1, - schema_max: 1, + schema_min: crate::REPOSITORY_SCHEMA_VERSION, + schema_max: crate::REPOSITORY_SCHEMA_VERSION, input_limit, output_limit, } diff --git a/crates/canopy-server/src/packs/publication/schema.sql b/crates/canopy-server/src/packs/publication/schema.sql index c10abbc8..0e94699e 100644 --- a/crates/canopy-server/src/packs/publication/schema.sql +++ b/crates/canopy-server/src/packs/publication/schema.sql @@ -1,4 +1,4 @@ --- Fresh canopy-pack-v1 repository schema. No legacy Git bodies or per-object placement. +-- Fresh canopy-pack-v2 repository schema. No legacy Git bodies or per-object placement. -- Product OID membership and ancestry decisions require certified catalog facts. CREATE TABLE refs ( name TEXT PRIMARY KEY, diff --git a/crates/canopy-server/src/packs/publication/serving/session/workspace.rs b/crates/canopy-server/src/packs/publication/serving/session/workspace.rs index a1f45ea9..a379385f 100644 --- a/crates/canopy-server/src/packs/publication/serving/session/workspace.rs +++ b/crates/canopy-server/src/packs/publication/serving/session/workspace.rs @@ -301,6 +301,10 @@ impl ServingPin { } observation.refresh(&inner, &actor).await?; job(&spool, owner.clone(), move |s| s.imported(source)).await?; + if materialize { + job(&spool, owner.clone(), move |s| s.mark_pack_complete(source)) + .await?; + } stats.packs = stats .packs .checked_add(1) diff --git a/crates/canopy-server/src/packs/publication/serving/session/workspace/prepare.rs b/crates/canopy-server/src/packs/publication/serving/session/workspace/prepare.rs index 89473ba8..ae363d0c 100644 --- a/crates/canopy-server/src/packs/publication/serving/session/workspace/prepare.rs +++ b/crates/canopy-server/src/packs/publication/serving/session/workspace/prepare.rs @@ -218,6 +218,51 @@ impl NativeWorkspace { } } } + if filter.is_none() && !needs_blob_sizes { + // Wants were checked against the complete certified ref + // closure above. Reuse authenticated immutable packs for + // an unfiltered fetch; physical presence grants no wants. + // SSH v2 can prepare several groups on the same cache. + // Keep completion on that cache's shared spool and + // serialize installation through its existing lock. + let _selection = core.cache.selection.lock().await; + observation.refresh(&inner, &actor).await?; + let mut sources = reader.source_changes(None, None)?; + loop { + observation.refresh(&inner, &actor).await?; + let page = sources.page(128, 64 << 10).await?; + if page.is_empty() { + break; + } + for record in page { + observation.refresh(&inner, &actor).await?; + let native = record.native(); + native.validate(inner.context.repository(), inner.lease.format)?; + if !job(&core.spool, owner.clone(), move |s| { + s.pack_complete(native) + }) + .await? + { + inner + .context + .files + .install_pack_workspace( + core.cache.clone(), + native, + owner.clone(), + ) + .await?; + observation.refresh(&inner, &actor).await?; + job(&core.spool, owner.clone(), move |s| { + s.mark_pack_complete(native) + }) + .await?; + } + } + } + inner.observe(actor).await?; + return Ok(()); + } loop { observation.refresh(&inner, &actor).await?; let pending = job(&spool, owner.clone(), |s| s.pending()).await?; diff --git a/crates/canopy-server/src/packs/publication/staging_service.rs b/crates/canopy-server/src/packs/publication/staging_service.rs index 041e09a5..a5ca65e4 100644 --- a/crates/canopy-server/src/packs/publication/staging_service.rs +++ b/crates/canopy-server/src/packs/publication/staging_service.rs @@ -1518,6 +1518,19 @@ impl StagingTicket { Ok(ceiling) } #[cfg(test)] + pub(super) fn shorten_bound_lease_for_test( + &self, + remaining: Duration, + ) -> Result { + let mut local = self.job.local.lock().expect("staging local"); + let session = local.bound.as_ref().ok_or(StagingError::NotReady)?; + let deadline = (Instant::now() + remaining).min(session.live_lease()?.1); + *session.deadline.lock().expect("bound deadline") = deadline; + local.deadline = deadline; + self.job.changed.notify_one(); + Ok(deadline) + } + #[cfg(test)] pub(super) fn expire_bound_for_test(&self) -> Result { let mut local = self.job.local.lock().expect("staging local"); let session = local.bound.as_ref().ok_or(StagingError::NotReady)?; @@ -1599,7 +1612,7 @@ impl StagingContext { Ok(()) } } - async fn fenced(&self) { + pub(super) async fn fenced(&self) { let mut status = self.job.status.subscribe(); loop { let (deadline, session) = { diff --git a/crates/canopy-server/src/packs/publication/tests.rs b/crates/canopy-server/src/packs/publication/tests.rs index 5aeca18b..657a5bde 100644 --- a/crates/canopy-server/src/packs/publication/tests.rs +++ b/crates/canopy-server/src/packs/publication/tests.rs @@ -66,8 +66,8 @@ impl CellModule for Module { let descriptor = |id| cellule_runtime::registry::OperationDescriptor { id, codec_version: 1, - schema_min: 1, - schema_max: 1, + schema_min: crate::REPOSITORY_SCHEMA_VERSION, + schema_max: crate::REPOSITORY_SCHEMA_VERSION, input_limit: 4096, output_limit: 4096, }; @@ -106,10 +106,10 @@ impl CellModule for Module { name: Self::NAME, source_digest: Digest::from_bytes([11; 32]), retained_codes: &[], - schema_min: 1, - schema_max: 1, + schema_min: crate::REPOSITORY_SCHEMA_VERSION, + schema_max: crate::REPOSITORY_SCHEMA_VERSION, migrations: Box::leak(Box::new([MigrationDescriptor { - version: 1, + version: crate::REPOSITORY_SCHEMA_VERSION, sql: SCHEMA, digest: Digest::from_bytes(*blake3::hash(SCHEMA.as_bytes()).as_bytes()), }])), @@ -200,7 +200,7 @@ impl Fixture { &target, CatalogRole::Sql, registry.module_code("repository").ok_or("code")?, - 1, + crate::REPOSITORY_SCHEMA_VERSION, )?) .await?; let authority = CellAuthority::new(layout.clone()); diff --git a/crates/canopy-server/src/packs/publication/tests/candidate_publication.rs b/crates/canopy-server/src/packs/publication/tests/candidate_publication.rs index 77134e4e..775cff64 100644 --- a/crates/canopy-server/src/packs/publication/tests/candidate_publication.rs +++ b/crates/canopy-server/src/packs/publication/tests/candidate_publication.rs @@ -78,7 +78,9 @@ async fn command( async fn native_candidate_ready_is_atomic_retained_and_recovers_original_receipt_after_body_retirement() -> Result { for format in [ObjectFormat::Sha1, ObjectFormat::Sha256] { - let (f, graph, merge) = native_merge::initial(format, false).await?; + // Keep the stock-Git fixture construction off this recovery test's + // future frame; it independently exercises preparation and publication. + let (f, graph, merge) = Box::pin(native_merge::initial(format, false)).await?; let (base, _, _) = opened(&f, *uuid::Uuid::new_v4().as_bytes(), graph.store.clone()).await?; let mut native = prepared_for_store( diff --git a/crates/canopy-server/src/packs/publication/tests/native_capture.rs b/crates/canopy-server/src/packs/publication/tests/native_capture.rs index 39b41a83..79888cef 100644 --- a/crates/canopy-server/src/packs/publication/tests/native_capture.rs +++ b/crates/canopy-server/src/packs/publication/tests/native_capture.rs @@ -444,6 +444,46 @@ enum MetadataFault { MissingArtifact, } +#[derive(Clone, Copy, PartialEq, Eq)] +enum RenewalPhase { + Replay, + Finish, +} + +#[tokio::test] +async fn native_receive_metadata_finish_survives_original_deadline_after_custody_renewal() -> Result +{ + for format in [ObjectFormat::Sha1, ObjectFormat::Sha256] { + let qualification = Box::pin(native_receive_metadata_case( + format, + false, + CompletionMode::Success, + None, + Some(RenewalPhase::Finish), + )) + .await?; + assert!(qualification.is_none()); + } + Ok(()) +} + +#[tokio::test] +async fn native_receive_metadata_replay_survives_original_deadline_after_custody_renewal() -> Result +{ + for format in [ObjectFormat::Sha1, ObjectFormat::Sha256] { + let qualification = Box::pin(native_receive_metadata_case( + format, + false, + CompletionMode::Success, + None, + Some(RenewalPhase::Replay), + )) + .await?; + assert!(qualification.is_none()); + } + Ok(()) +} + #[tokio::test] async fn native_receive_metadata_refuses_capacity_corrupt_replay_and_missing_shards_without_publication() -> Result { @@ -453,13 +493,15 @@ async fn native_receive_metadata_refuses_capacity_corrupt_replay_and_missing_sha MetadataFault::TruncatedReplay, MetadataFault::MissingArtifact, ] { - Box::pin(native_receive_metadata_case( + let qualification = Box::pin(native_receive_metadata_case( format, true, CompletionMode::Success, Some(fault), + None, )) .await?; + assert!(qualification.is_none()); } } Ok(()) @@ -472,15 +514,25 @@ async fn publication_state(fixture: &Fixture) -> Result> { Ok([state.0.to_be_bytes(), state.1.to_be_bytes(), state.2.to_be_bytes()].concat()) }).await?) } +type RootedQualification = std::pin::Pin>>; + async fn native_receive_case(format: ObjectFormat, rooted: bool, mode: CompletionMode) -> Result { - Box::pin(native_receive_metadata_case(format, rooted, mode, None)).await + let qualification = Box::pin(native_receive_metadata_case( + format, rooted, mode, None, None, + )) + .await?; + if let Some(qualification) = qualification { + qualification.await?; + } + Ok(()) } async fn native_receive_metadata_case( format: ObjectFormat, rooted: bool, mode: CompletionMode, metadata_fault: Option, -) -> Result { + renewal: Option, +) -> Result> { let ref_free = match mode { CompletionMode::RefFree { kind, .. } => Some(kind), CompletionMode::Durable { .. } @@ -490,7 +542,16 @@ async fn native_receive_metadata_case( }; let fixture = Fixture::new(format).await?; let provider = Arc::new(InMemory::new()); - let store = Arc::new(ArtifactStore::new(provider.clone(), fixture.repository)); + let throttled = Arc::new(object_store::throttle::ThrottledStore::new( + provider.clone(), + object_store::throttle::ThrottleConfig::default(), + )); + let object_store: Arc = if renewal.is_some() { + throttled.clone() + } else { + provider.clone() + }; + let store = Arc::new(ArtifactStore::new(object_store, fixture.repository)); if rooted { let (base, _, _) = opened(&fixture, [159; 16], store.clone()).await?; let root = tempfile::TempDir::new()?; @@ -846,7 +907,7 @@ async fn native_receive_metadata_case( .await?; } cleaned(work_root.path(), &disk).await?; - return Ok(()); + return Ok(None); } if let CompletionMode::RefFree { kind, @@ -876,7 +937,7 @@ async fn native_receive_metadata_case( .await?; cleaned(work_root.path(), &disk).await?; fixture.runtime.shutdown().await?; - return Ok(()); + return Ok(None); } let before_publication = publication_state(&fixture).await?; let physical_root = Arc::new(tempfile::TempDir::new()?); @@ -931,7 +992,7 @@ async fn native_receive_metadata_case( assert_eq!(publication_state(&fixture).await?, before_publication); cleaned(physical_root.path(), &physical_disk).await?; fixture.runtime.shutdown().await?; - return Ok(()); + return Ok(None); } let staged = staged.map_err(|error| error.to_string())?; assert_eq!(staged.shard_count(), input.object_count); @@ -977,6 +1038,10 @@ async fn native_receive_metadata_case( } let producer_root = Arc::clone(&physical_root); let producer_disk = physical_disk.clone(); + let renewal_ticket = ticket.clone(); + let slow_store = throttled.clone(); + let (started, entered) = tokio::sync::oneshot::channel(); + let mut started = Some(started); let work = ticket.spawn_bound(move |_, context| async move { let result = async { let mut builder = CatalogPreparation::new_staged( @@ -985,8 +1050,20 @@ async fn native_receive_metadata_case( producer_disk, base, limits(), + limits(), ) .await?; + if renewal == Some(RenewalPhase::Replay) { + // Force the complete multi-shard replay across an old local + // deadline while a real registered renewal refreshes custody. + slow_store.config_mut(|config| { + config.wait_get_per_byte = Duration::from_micros(10); + }); + let deadline = renewal_ticket + .shorten_bound_lease_for_test(Duration::from_secs(1)) + .map_err(crate::packs::verification::PhysicalError::from)?; + let _ = started.take().expect("one renewal phase").send(deadline); + } let added = builder.add_staged_pack(staged).await; if metadata_fault.is_some() { let error = added.expect_err("invalid staged metadata accepted"); @@ -997,12 +1074,39 @@ async fn native_receive_metadata_case( return Err(error); } added?; + if renewal == Some(RenewalPhase::Finish) { + // Delay the first immutable output upload beyond the old + // deadline while the same bound session receives a renewal. + slow_store.config_mut(|config| { + config.wait_put_per_call = Duration::from_millis(1100); + }); + let deadline = renewal_ticket + .shorten_bound_lease_for_test(Duration::from_secs(1)) + .map_err(crate::packs::verification::PhysicalError::from)?; + let _ = started.take().expect("one renewal phase").send(deadline); + } builder.finish().await } .await; result.map_err(|error| StagingError::Input(Box::new(error))) })?; + let original_deadline = if renewal.is_some() { + let deadline = entered.await?; + // Let the selected assembly phase capture its deadline before renewal. + tokio::time::sleep(Duration::from_millis(50)).await; + ticket.renew_for_test(); + timeout(Duration::from_secs(1), async { + while ticket.bound_renewal().is_none() { + tokio::task::yield_now().await; + } + }) + .await?; + Some(deadline) + } else { + None + }; let prepared = work.wait().await; + throttled.config_mut(|config| *config = object_store::throttle::ThrottleConfig::default()); if metadata_fault.is_some() { assert!(prepared.is_err()); assert!(coordinator.close_and_drain().await.is_empty()); @@ -1010,9 +1114,13 @@ async fn native_receive_metadata_case( assert_eq!(publication_state(&fixture).await?, before_publication); cleaned(physical_root.path(), &physical_disk).await?; fixture.runtime.shutdown().await?; - return Ok(()); + return Ok(None); + } + let prepared = prepared.map_err(|error| format!("{error:?}"))?; + if let Some(deadline) = original_deadline { + assert!(tokio::time::Instant::now() > deadline); + assert!(ticket.bound_session().is_ok()); } - let prepared = prepared.map_err(|error| error.to_string())?; if rooted { if matches!( mode, @@ -1026,38 +1134,42 @@ async fn native_receive_metadata_case( | CompletionMode::PolicyDispatch { .. } | CompletionMode::Dispatch { .. } ) { - let result = qualify_rooted( - super::root_dispatch::Context { - fixture: &fixture, - prepared: Arc::new(prepared), - store: &store, - staging: &coordinator, - ticket: &ticket, - root: physical_root.path(), - budget: physical_disk.clone(), - request: recovered, - }, - mode, - ) - .await; - if mode == CompletionMode::MandatoryRegistration { - assert!(coordinator.close_and_drain().await.is_empty()); - fixture.runtime.shutdown().await?; - } - result?; - if matches!( - mode, - CompletionMode::PolicyRefusalDispatch { .. } - | CompletionMode::PolicyDispatch { .. } - | CompletionMode::Dispatch { .. } - ) { - cleaned(physical_root.path(), &physical_disk).await?; - fixture.runtime.shutdown().await?; - } else { - cleaned(work_root.path(), &disk).await?; - cleaned(physical_root.path(), &physical_disk).await?; - } - return Ok(()); + // Finish polling the large native setup before polling recovery. + // The continuation owns the same fixture, custody and scratch pins. + return Ok(Some(Box::pin(async move { + let result = qualify_rooted( + super::root_dispatch::Context { + fixture: &fixture, + prepared: Arc::new(prepared), + store: &store, + staging: &coordinator, + ticket: &ticket, + root: physical_root.path(), + budget: physical_disk.clone(), + request: recovered, + }, + mode, + ) + .await; + if mode == CompletionMode::MandatoryRegistration { + assert!(coordinator.close_and_drain().await.is_empty()); + fixture.runtime.shutdown().await?; + } + result?; + if matches!( + mode, + CompletionMode::PolicyRefusalDispatch { .. } + | CompletionMode::PolicyDispatch { .. } + | CompletionMode::Dispatch { .. } + ) { + cleaned(physical_root.path(), &physical_disk).await?; + fixture.runtime.shutdown().await?; + } else { + cleaned(work_root.path(), &disk).await?; + cleaned(physical_root.path(), &physical_disk).await?; + } + Ok(()) + }))); } let (fixture, replay) = super::root_completion::spawn_qualify( fixture, @@ -1073,7 +1185,7 @@ async fn native_receive_metadata_case( cleaned(physical_root.path(), &physical_disk).await?; assert!(coordinator.close_and_drain().await.is_empty()); super::root_completion::restored(&fixture, &store, replay).await?; - return Ok(()); + return Ok(None); } let expected = recovered.response.clone(); let producer_root = Arc::clone(&physical_root); @@ -1202,7 +1314,7 @@ async fn native_receive_metadata_case( Ok(Vec::new()) }).await?; fixture.runtime.shutdown().await?; - Ok(()) + Ok(None) } #[tokio::test] diff --git a/crates/canopy-server/src/packs/publication/tests/native_head.rs b/crates/canopy-server/src/packs/publication/tests/native_head.rs index 042bdeb5..e8d5ddc8 100644 --- a/crates/canopy-server/src/packs/publication/tests/native_head.rs +++ b/crates/canopy-server/src/packs/publication/tests/native_head.rs @@ -73,6 +73,16 @@ async fn recover( async fn native_head_reuses_ref_tree_and_preserves_original_receipt_after_typed_retirement() -> Result { for format in [ObjectFormat::Sha1, ObjectFormat::Sha256] { + retired_head(format).await?; + } + Ok(()) +} + +// Construct the complete scenario outside the harness's polling stack frame. +fn retired_head( + format: ObjectFormat, +) -> std::pin::Pin>> { + Box::pin(async move { let (f, graph, _) = initial(format, false).await?; let (prepared, root, budget) = preparation(&f, &graph).await?; let check = prepared.base.capability().2.clone(); @@ -219,8 +229,8 @@ async fn native_head_reuses_ref_tree_and_preserves_original_receipt_after_typed_ runtime.shutdown().await?; drop(graph.prepared); cleaned(graph.root.path(), &graph.budget).await?; - } - Ok(()) + Ok(()) + }) } #[tokio::test] async fn native_head_denials_recheck_current_authority_generation_and_native_branch_existence() diff --git a/crates/canopy-server/src/packs/publication/tests/native_merge/retirement.rs b/crates/canopy-server/src/packs/publication/tests/native_merge/retirement.rs index 6a3a3b87..8b1c9049 100644 --- a/crates/canopy-server/src/packs/publication/tests/native_merge/retirement.rs +++ b/crates/canopy-server/src/packs/publication/tests/native_merge/retirement.rs @@ -40,6 +40,17 @@ async fn retained_pin(f: &Fixture, check: &LeaseCheck, expected: u64) -> Result async fn native_merge_applied_attempt_releases_pin_without_losing_original_uuid_or_receipt() -> Result { for format in [ObjectFormat::Sha1, ObjectFormat::Sha256] { + applied_attempt(format).await?; + } + Ok(()) +} + +// Construct the scenario in a separate call so the harness and polling frame +// hold its heap pointer instead of the large recovery future on a 2 MiB stack. +fn applied_attempt( + format: ObjectFormat, +) -> std::pin::Pin>> { + Box::pin(async move { let (f, graph, request) = initial(format, false).await?; let (prepared, root, budget) = preparation(&f, &graph).await?; let check = prepared.base.capability().2.clone(); @@ -108,13 +119,22 @@ async fn native_merge_applied_attempt_releases_pin_without_losing_original_uuid_ drop(graph.prepared); cleaned(graph.root.path(), &graph.budget).await?; f.runtime.shutdown().await?; - } - Ok(()) + Ok(()) + }) } #[tokio::test] async fn negative_merge_archive_keeps_its_denial_after_same_uuid_succeeds() -> Result { for format in [ObjectFormat::Sha1, ObjectFormat::Sha256] { + negative_archive(format).await?; + } + Ok(()) +} + +fn negative_archive( + format: ObjectFormat, +) -> std::pin::Pin>> { + Box::pin(async move { let (f, graph, request) = initial(format, false).await?; let (prepared, root, budget) = preparation(&f, &graph).await?; let check = prepared.base.capability().2.clone(); @@ -173,8 +193,8 @@ async fn negative_merge_archive_keeps_its_denial_after_same_uuid_succeeds() -> R drop(graph.prepared); cleaned(graph.root.path(), &graph.budget).await?; f.runtime.shutdown().await?; - } - Ok(()) + Ok(()) + }) } #[tokio::test] diff --git a/crates/canopy-server/src/packs/publication/tests/serving/workspace.rs b/crates/canopy-server/src/packs/publication/tests/serving/workspace.rs index 4103fbe0..4ab5c2c0 100644 --- a/crates/canopy-server/src/packs/publication/tests/serving/workspace.rs +++ b/crates/canopy-server/src/packs/publication/tests/serving/workspace.rs @@ -546,6 +546,39 @@ async fn reachability_stops_at_live_refs_without_scanning_other_history() -> Res String::from_utf8(refs)?, "refs/heads/main\nrefs/heads/side\n" ); + workspace + .prepare_fetch(vec![fixture.main], None, false) + .await?; + workspace + .prepare_fetch(vec![fixture.side], None, false) + .await?; + let (first, second) = tokio::join!( + workspace.prepare_fetch(vec![fixture.main], None, false), + workspace.prepare_fetch(vec![fixture.side], None, false), + ); + first?; + second?; + // The pack also physically contains this unrelated tag. Installing it + // must neither expand want authority nor create per-object loose files. + crate::packs::metadata::tests::git( + &workspace.git_dir(), + &["cat-file", "-e", &hex::encode(fixture.tag)], + None, + ) + .await?; + let main = hex::encode(fixture.main); + assert!( + !workspace + .git_dir() + .join("objects") + .join(&main[..2]) + .join(&main[2..]) + .exists() + ); + assert!(matches!( + GitGateway::validate_wants(&workspace, &BTreeSet::from([fixture.tag])).await, + Err(GatewayError::UnreachableWant) + )); let empty = RefStateSnapshotRoot::upload( &store, operation(122), diff --git a/crates/canopy-server/src/pulls/candidates/mod.rs b/crates/canopy-server/src/pulls/candidates/mod.rs index a43167d3..99c2d631 100644 --- a/crates/canopy-server/src/pulls/candidates/mod.rs +++ b/crates/canopy-server/src/pulls/candidates/mod.rs @@ -6,7 +6,6 @@ pub(crate) mod command; pub(crate) mod rebase; use super::merge::{MergeStrategy, oid, policy_state, policy_statement}; use super::*; -use crate::ObjectKind; use cellule_runtime::{ codec::BoundedDecoder, codec::BoundedEncoder, codec::CodecError, codec::WireValue, registry::CommandContext, @@ -276,59 +275,6 @@ pub(crate) fn commit_body(candidate: &MergeCandidate, tree_oid: &str) -> Vec format!("tree {tree_oid}\n{parents}author {actor} <{actor}@users.canopy.invalid> {time} +0000\ncommitter {actor} <{actor}@users.canopy.invalid> {time} +0000\n\n{message}{}",if message.ends_with('\n') {""} else {"\n"}).into_bytes() } -pub(crate) fn publication_oid( - context: &CommandContext<'_, '_>, - number: i64, - request: &merge::MergeRequest, -) -> cellule_runtime::Result> { - let Some(id) = request.candidate_id.as_deref() else { - return Ok(None); - }; - let Some((_, candidate)) = decode(&context.sql(&query(id)?)?)? else { - return Ok(None); - }; - if candidate.number != number - || candidate.request.revision != request.revision - || candidate.request.strategy != request.strategy - { - return Ok(None); - } - let CandidateResult::Ready { - oid: commit, - tree_oid, - } = &candidate.result - else { - return Ok(None); - }; - if !certified(context, &candidate, commit, tree_oid)? { - return Ok(None); - } - Ok(Some(oid(commit)?)) -} -fn certified( - context: &CommandContext<'_, '_>, - candidate: &MergeCandidate, - commit: &str, - tree: &str, -) -> cellule_runtime::Result { - if candidate.request.strategy == MergeStrategy::Rebase { - return rebase::certified(context, candidate, commit, tree); - } - // The trusted native worker owns tree-merging semantics. The transaction - // verifies exact parent order, author, timestamp, message and graph closure. - let body = commit_body(candidate, tree); - let commit = oid(commit)?; - if crate::object_id(commit.format(), ObjectKind::Commit, &body) != commit { - return Ok(false); - } - let rows=context.sql(&SqlBatch {statements:vec![SqlStatement { - sql:"SELECT o.body FROM objects o JOIN object_closure c ON c.oid = o.oid WHERE o.oid = ?1 AND o.kind = 'commit' AND o.storage = 'inline'".into(),parameters:vec![SqlValue::Blob(commit.to_vec())], - }]})?; - Ok( - matches!(rows.first().and_then(|set|set.rows.first()).map(Vec::as_slice),Some([SqlValue::Blob(stored)]) if *stored == body), - ) -} - pub(crate) fn intent_binding(candidate: &MergeCandidate) -> cellule_runtime::Result> { let request = serde_json::to_string(&candidate.request) .map_err(|_| Error::Command("candidate intent encoding"))?; diff --git a/crates/canopy-server/src/pulls/candidates/rebase.rs b/crates/canopy-server/src/pulls/candidates/rebase.rs index 988aff89..cecd653a 100644 --- a/crates/canopy-server/src/pulls/candidates/rebase.rs +++ b/crates/canopy-server/src/pulls/candidates/rebase.rs @@ -6,7 +6,6 @@ pub(crate) const MAX_COMMITS: usize = 128; pub(crate) const MAX_COMMIT_BYTES: usize = 64 * 1024; pub(crate) struct Commit<'a> { - pub(crate) tree: &'a str, pub(crate) parent: &'a str, headers: Vec<&'a [u8]>, message: &'a [u8], @@ -39,7 +38,6 @@ impl<'a> Commit<'a> { field(b"author ")?; field(b"committer ")?; Some(Self { - tree, parent, headers, message: &body[split + 2..], @@ -80,79 +78,3 @@ impl<'a> Commit<'a> { body } } - -pub(super) fn certified( - context: &CommandContext<'_, '_>, - candidate: &MergeCandidate, - tip: &str, - tree: &str, -) -> cellule_runtime::Result { - let mut source = candidate.request.revision.source_oid.clone(); - let mut current = tip.to_owned(); - let base = &candidate.request.revision.base_oid; - for index in 0..MAX_COMMITS { - let Some(original) = body(context, &source)? else { - return Ok(false); - }; - let Some(rewritten) = body(context, ¤t)? else { - return Ok(false); - }; - let Some(original_commit) = Commit::parse(&original) else { - return Ok(false); - }; - let Some(rewritten_commit) = Commit::parse(&rewritten) else { - return Ok(false); - }; - if (index == 0 && rewritten_commit.tree != tree) - || original_commit.rewrite(candidate, rewritten_commit.tree, rewritten_commit.parent) - != rewritten - { - return Ok(false); - } - // Native Git owns replay selection and merge semantics. The transaction - // binds every rewritten commit to the contiguous original chain and a - // certified common ancestor, so a worker cannot omit an intermediate commit. - source = original_commit.parent.to_owned(); - current = rewritten_commit.parent.to_owned(); - if ¤t == base { - if &source == base { - return Ok(true); - } - let rows = context.sql(&SqlBatch { - statements: vec![SqlStatement { - sql: "SELECT 1 FROM commit_ancestry WHERE ancestor = ?1 AND descendant = ?2" - .into(), - parameters: vec![ - SqlValue::Blob(oid(&source)?.to_vec()), - SqlValue::Blob(oid(base)?.to_vec()), - ], - }], - })?; - return Ok(rows.first().is_some_and(|set| !set.rows.is_empty())); - } - } - Ok(false) -} - -fn body( - context: &CommandContext<'_, '_>, - commit: &str, -) -> cellule_runtime::Result>> { - let rows = context.sql(&SqlBatch { statements: vec![SqlStatement { - sql: "SELECT o.body FROM objects o JOIN object_closure c ON c.oid = o.oid WHERE o.oid = ?1 AND o.kind = 'commit' AND o.storage = 'inline' AND o.size <= ?2".into(), - parameters: vec![SqlValue::Blob(oid(commit)?.to_vec()), SqlValue::Integer(MAX_COMMIT_BYTES as i64)], - }] })?; - match rows - .first() - .and_then(|set| set.rows.first()) - .map(Vec::as_slice) - { - Some([SqlValue::Blob(bytes)]) - if crate::object_id(oid(commit)?.format(), ObjectKind::Commit, bytes) - == oid(commit)? => - { - Ok(Some(bytes.clone())) - } - _ => Ok(None), - } -} diff --git a/crates/canopy-server/src/pulls/merge/command.rs b/crates/canopy-server/src/pulls/merge/command.rs index 706ff12b..5074fe1c 100644 --- a/crates/canopy-server/src/pulls/merge/command.rs +++ b/crates/canopy-server/src/pulls/merge/command.rs @@ -1,11 +1,6 @@ use super::*; -use crate::{ - PushPlan, RepositoryModule, - access::{access_statement, decode_access}, -}; use cellule_runtime::{ - CellModule, Command, codec::BoundedDecoder, codec::BoundedEncoder, codec::CodecError, - codec::WireValue, registry::CommandContext, registry::CommandResult, + codec::BoundedDecoder, codec::BoundedEncoder, codec::CodecError, codec::WireValue, }; #[derive(Clone, Debug, Deserialize, Serialize)] @@ -57,128 +52,6 @@ impl WireValue for MergeOutcome { .map_err(|_| CodecError::Invalid("merge output JSON")) } } -pub(crate) struct MergePull; -impl Command for MergePull { - const MODULE: &'static str = RepositoryModule::NAME; - const ID: u32 = 9; - const CODEC_VERSION: u32 = 4; - type Input = MergeInput; - type Output = MergeOutcome; - fn execute( - context: &mut CommandContext<'_, '_>, - input: MergeInput, - ) -> cellule_runtime::Result> { - let role = decode_access(&context.sql(&SqlBatch { - statements: vec![access_statement(&input.actor)], - })?)?; - let rejected = |value| Ok(CommandResult::Rejected(value)); - let Some(role) = role else { - return rejected(MergeOutcome::NotFound); - }; - if role < TokenScope::Write { - return rejected(MergeOutcome::Forbidden); - } - let id = uuid::Uuid::parse_str(&input.request.id) - .map_err(|_| Error::Command("invalid merge UUID"))?; - // Bind actor, parent and full intent. Command timestamps are not part of - // application retry identity, so a lost reply can use a fresh command. - let binding = request_binding(&input); - let previous = context.sql(&SqlBatch { - statements: vec![SqlStatement { - sql: "SELECT binding, id, pull_number, oid, merged_ms, pull_version, source_oid, source_version, base_oid, base_version FROM pull_merges WHERE id = ?1" - .into(), - parameters: vec![SqlValue::Blob(id.as_bytes().to_vec())], - }], - })?; - if let Some(row) = previous.first().and_then(|set| set.rows.first()) { - let Some(SqlValue::Blob(stored)) = row.first() else { - return Err(Error::Command("invalid merge binding")); - }; - if *stored != binding { - return rejected(MergeOutcome::Conflict); - } - return Ok(CommandResult::Success(MergeOutcome::Applied { - merge: record(&row[1..])?, - })); - } - let Some(state) = policy_state(&context.sql(&SqlBatch { - statements: vec![policy_statement(&input.actor, input.number)], - })?)? - else { - return rejected(MergeOutcome::NotFound); - }; - if !state.policy.ready || state.policy.revision.as_ref() != Some(&input.request.revision) { - return rejected(MergeOutcome::Conflict); - } - if !state.policy.reviews_satisfied { - return rejected(MergeOutcome::ReviewsRequired); - } - let base = oid(&input.request.revision.base_oid)?; - let source = match input.request.strategy { - MergeStrategy::FastForward => oid(&input.request.revision.source_oid)?, - MergeStrategy::MergeCommit | MergeStrategy::Squash | MergeStrategy::Rebase => { - match super::super::candidates::publication_oid( - context, - input.number, - &input.request, - )? { - Some(oid) => oid, - None => return rejected(MergeOutcome::Conflict), - } - } - }; - let ancestry = context.sql(&SqlBatch { - statements: vec![SqlStatement { - sql: "SELECT 1 FROM commit_ancestry WHERE ancestor = ?1 AND descendant = ?2".into(), - parameters: vec![ - SqlValue::Blob(base.to_vec()), - SqlValue::Blob(source.to_vec()), - ], - }], - })?; - if ancestry.first().is_none_or(|set| set.rows.is_empty()) { - return rejected(MergeOutcome::NotFastForward); - } - let update = RefUpdate { - name: state.base, - expected: Some(RefExpectation { - oid: Some(base), - version: input.request.revision.base_version, - }), - new_oid: Some(source), - }; - let reviewed = ReviewedMerge { - update: update.clone(), - }; - let plan = PushPlan { - actor: input.actor, - updates: vec![update], - }; - // All writes below share this command transaction. Current checks and - // every normal ref invariant still pass through the canonical publisher. - if !crate::refs::apply_refs(context, &plan, Some(&reviewed))? { - return rejected(MergeOutcome::BranchPolicy); - } - let result=context.sql(&SqlBatch {statements:vec![SqlStatement { - sql:"UPDATE pull_requests SET state = 'merged', version = version + 1, updated_ms = max(updated_ms, ?2) WHERE number = ?1 AND state = 'open'".into(),parameters:vec![SqlValue::Integer(input.number),SqlValue::Integer(input.issued_at_ms)], - },SqlStatement { - sql:"INSERT INTO pull_merges (id, binding, pull_number, oid, merged_ms, pull_version, source_oid, source_version, base_oid, base_version) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)".into(),parameters:vec![SqlValue::Blob(id.as_bytes().to_vec()),SqlValue::Blob(binding),SqlValue::Integer(input.number),SqlValue::Blob(source.to_vec()),SqlValue::Integer(input.issued_at_ms),SqlValue::Integer(input.request.revision.pull_version),SqlValue::Blob(oid(&input.request.revision.source_oid)?.to_vec()),SqlValue::Integer(input.request.revision.source_version),SqlValue::Blob(base.to_vec()),SqlValue::Integer(input.request.revision.base_version)], - }]})?; - if result.first().is_none_or(|set| set.rows_affected != 1) { - return Err(Error::Command("merged pull was not updated")); - } - Ok(CommandResult::Success(MergeOutcome::Applied { - merge: MergeRecord { - id: input.request.id, - number: input.number, - oid: hex::encode(source), - merged_at_ms: input.issued_at_ms, - revision: input.request.revision, - }, - })) - } -} - pub(crate) fn request_binding(input: &MergeInput) -> Vec { super::super::mutations::binding(&[ &input.actor, diff --git a/crates/canopy-server/src/pulls/merge/mod.rs b/crates/canopy-server/src/pulls/merge/mod.rs index daf982a9..cab4c015 100644 --- a/crates/canopy-server/src/pulls/merge/mod.rs +++ b/crates/canopy-server/src/pulls/merge/mod.rs @@ -4,7 +4,7 @@ use crate::ReadIdentity; pub(crate) mod command; use super::*; -use crate::{RefExpectation, RefUpdate, directory::TokenScope}; +use crate::{RefExpectation, RefUpdate}; #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] #[serde(rename_all = "snake_case")] @@ -129,7 +129,6 @@ pub(crate) fn reviewed_native_update( pub(super) struct ReviewState { pub(super) policy: ReviewPolicy, base: String, - writable: bool, } pub(crate) fn valid_request(request: &MergeRequest) -> bool { @@ -187,81 +186,6 @@ impl RepositoryCell { receipt: result.receipt, }) } - /// Prepares ancestry facts and atomically publishes an exact reviewed merge. - /// - /// Current write authority, reviews, checks and ref versions are rechecked at - /// publication. Exact request retries preserve the original merge record. - pub async fn merge_pull( - &self, - identity: MutationIdentity, - actor: &str, - number: i64, - request: MergeRequest, - ) -> Result, InvocationError> { - if !valid_request(&request) || number < 1 || validate_component(actor).is_err() { - return Err(InvocationError::NotStarted(Error::Command( - "invalid merge request", - ))); - } - let state = self - .pull_review_state(ReadIdentity::Account(actor), number) - .await - .map_err(preparation)? - .output; - if state.is_some_and(|state| { - state.writable - && state.policy.ready - && state.policy.reviews_satisfied - && state.policy.revision.as_ref() == Some(&request.revision) - }) { - let base = oid(&request.revision.base_oid).map_err(InvocationError::NotStarted)?; - let source = match &request.candidate_id { - None => { - Some(oid(&request.revision.source_oid).map_err(InvocationError::NotStarted)?) - } - Some(id) => self - .merge_candidate(actor, number, id) - .await - .map_err(preparation)? - .output - .filter(|candidate| { - candidate.request.revision == request.revision - && candidate.request.strategy == request.strategy - }) - .and_then(|candidate| match candidate.result { - super::candidates::CandidateResult::Ready { oid: commit, .. } => { - parse_oid(&commit).and_then(|bytes| bytes.try_into().ok()) - } - _ => None, - }), - }; - if let Some(source) = source { - self.prepare_ancestry(base, source) - .await - .map_err(preparation)?; - } - } - self.application - .command::( - &self.target, - identity, - command::MergeInput { - actor: actor.into(), - number, - request, - issued_at_ms: identity.issued_at_ms, - }, - ) - .await - } -} -fn preparation( - error: impl std::error::Error + Send + Sync + 'static, -) -> InvocationError { - InvocationError::NotStarted(Error::Facility { - name: "merge preparation", - source: Box::new(error), - }) } pub(crate) fn oid(text: &str) -> cellule_runtime::Result { parse_oid(text) @@ -304,7 +228,7 @@ pub(super) fn policy_state(sets: &[SqlResultSet]) -> cellule_runtime::Result cellule_runtime::Result>), -} - /// Expected ref version and optional tip; a missing tip is a retained deletion. #[derive(Clone, Debug, PartialEq, Eq)] pub struct RefExpectation { @@ -56,152 +46,6 @@ pub struct PushPlan { pub updates: Vec, } -impl RepositoryCell { - /// Reads a live or deleted ref and its version; None means the name has never existed. - pub async fn ref_state( - &self, - name: &str, - minimum: Option, - ) -> std::result::Result>, InvocationError>> - { - let result = self - .sql - .query( - minimum, - SqlBatch { - statements: vec![SqlStatement { - sql: "SELECT oid, version FROM refs WHERE name = ?1".into(), - parameters: vec![SqlValue::Text(name.into())], - }], - }, - ) - .await?; - let state = result - .output - .first() - .and_then(|set| set.rows.first()) - .map(|row| decode_ref_row(row)) - .transpose() - .map_err(InvocationError::NotStarted)?; - Ok(Observed { - output: state, - receipt: result.receipt, - }) - } - - /// Reads a byte-bounded page of at most 256 refs; continuations require the first page's generation. - /// - /// A changed generation rejects the page, requiring a new scan from the start. - pub async fn refs_page( - &self, - after: &str, - generation: Option, - ) -> Result, RefReadError> { - if !after.is_empty() && generation.is_none() { - return Err(InvocationError::NotStarted(Error::Command( - "ref cursor requires a generation", - )) - .into()); - } - let result = self - .sql - .query( - None, - SqlBatch { - statements: vec![SqlStatement { - // One SQLite statement binds the generation even to an empty - // final page. Separate observations could miss a concurrent push. - sql: r#" - WITH candidates AS MATERIALIZED ( - SELECT name, oid, version FROM refs - WHERE name > ?1 ORDER BY name LIMIT ?2 - ), ranked AS ( - SELECT name, oid, version, - ROW_NUMBER() OVER (ORDER BY name) AS position, - SUM(LENGTH(CAST(name AS BLOB)) + 64) - OVER (ORDER BY name) AS bytes - FROM candidates - ), page AS MATERIALIZED ( - SELECT name, oid, version FROM ranked - WHERE bytes <= ?3 OR position = 1 - ) - SELECT g.generation, g.default_branch, NULL, NULL, NULL, - EXISTS(SELECT 1 FROM refs WHERE name > - COALESCE((SELECT MAX(name) FROM page), ?1)) - FROM ref_generation g WHERE g.singleton = 1 - UNION ALL - SELECT NULL, NULL, name, oid, version, NULL FROM page - ORDER BY name - "# - .into(), - parameters: vec![ - SqlValue::Text(after.into()), - SqlValue::Integer(REF_PAGE_SIZE as i64), - SqlValue::Integer(REF_PAGE_NAME_BYTES), - ], - }], - }, - ) - .await?; - let rows = result - .output - .first() - .ok_or_else(|| InvocationError::NotStarted(Error::Command("missing refs page")))?; - let row = rows - .rows - .first() - .ok_or_else(|| InvocationError::NotStarted(Error::Command("missing ref generation")))?; - let head = crate::default_branch::decode_head(row).map_err(InvocationError::NotStarted)?; - let has_more = match row.as_slice() { - [ - _, - _, - SqlValue::Null, - SqlValue::Null, - SqlValue::Null, - SqlValue::Integer(more), - ] => *more != 0, - _ => { - return Err( - InvocationError::NotStarted(Error::Command("invalid ref page head")).into(), - ); - } - }; - let current = head.generation; - if generation.is_some_and(|expected| expected != current) { - return Err(RefReadError::Changed); - } - let mut refs = Vec::with_capacity(rows.rows.len().saturating_sub(1)); - for row in rows.rows.iter().skip(1) { - let [ - SqlValue::Null, - SqlValue::Null, - SqlValue::Text(name), - oid, - version, - SqlValue::Null, - ] = row.as_slice() - else { - return Err( - InvocationError::NotStarted(Error::Command("invalid stored ref row")).into(), - ); - }; - let state = decode_ref_row(&[oid.clone(), version.clone()]) - .map_err(InvocationError::NotStarted)?; - refs.push((name.clone(), state)); - } - Ok(Observed { - output: RefPage { - generation: current, - default_branch: head.reference, - refs, - has_more, - }, - receipt: result.receipt, - }) - } -} - impl WireValue for PushPlan { fn encode(&self, encoder: &mut BoundedEncoder) -> Result<(), CodecError> { self.encode_prefix(encoder)?; @@ -318,47 +162,6 @@ fn read_oid(decoder: &mut BoundedDecoder<'_>) -> Result, - plan: Self::Input, - ) -> cellule_runtime::Result> { - Ok(if apply_refs(context, &plan, None)? { - CommandResult::Success(true) - } else { - CommandResult::Rejected(false) - }) - } -} - -// A false decision must precede every write: HTTP completion records its -// rejection report in the same transaction without relying on rollback. -pub(crate) fn apply_refs( - context: &mut CommandContext<'_, '_>, - plan: &PushPlan, - merge: Option<&crate::pulls::merge::ReviewedMerge>, -) -> cellule_runtime::Result { - let Some(validated) = validate_refs(context, plan)? else { - return Ok(false); - }; - if !crate::graph::certified_roots(context, plan)? - || !crate::branch_rules::policies_allow(context, plan, merge)? - { - return Ok(false); - } - validated.apply(context)?; - Ok(true) -} - /// Only ref/ACL validation constructs this result. Catalog membership and /// current branch/merge policy must also pass before consuming it. It borrows /// the immutable plan and cannot be reused by another admitted command. diff --git a/crates/canopy-server/src/repository_http/mod.rs b/crates/canopy-server/src/repository_http/mod.rs index 6c7acdc2..77fb257e 100644 --- a/crates/canopy-server/src/repository_http/mod.rs +++ b/crates/canopy-server/src/repository_http/mod.rs @@ -458,6 +458,23 @@ async fn list_repositories( State(state): State>, Query(query): Query, headers: axum::http::HeaderMap, +) -> Response { + let mut response = repository_list_page(state, query, headers).await; + if response.status() == StatusCode::SERVICE_UNAVAILABLE { + // Every unavailable page follows the same bounded GET retry contract, + // including readiness loss and errors reached while moving cold Cells. + response.headers_mut().insert( + header::RETRY_AFTER, + axum::http::HeaderValue::from_static("1"), + ); + } + response +} + +async fn repository_list_page( + state: Arc, + query: ListRepositoriesQuery, + headers: axum::http::HeaderMap, ) -> Response { if !(state.manager.ready)() { return plain(StatusCode::SERVICE_UNAVAILABLE, "Canopy node is not ready"); @@ -491,19 +508,12 @@ async fn list_repositories( }), ), Ok(_) => plain(StatusCode::SERVICE_UNAVAILABLE, "Canopy node is not ready"), - Err(ServerError::Runtime(cellule_runtime::Error::Capacity(_))) => { - let mut response = plain( - StatusCode::SERVICE_UNAVAILABLE, - "Repository listing capacity is full; retry the request", - ); - response.headers_mut().insert( - header::RETRY_AFTER, - axum::http::HeaderValue::from_static("1"), - ); - response - } + Err(ServerError::Runtime(cellule_runtime::Error::Capacity(_))) => plain( + StatusCode::SERVICE_UNAVAILABLE, + "Repository listing capacity is full; retry the request", + ), Err(error) => { - tracing::error!(error = %error, "repository listing failed"); + tracing::error!(error = ?error, "repository listing failed"); plain(StatusCode::SERVICE_UNAVAILABLE, "Repository listing failed") } } diff --git a/crates/canopy-server/src/schema.sql b/crates/canopy-server/src/schema.sql deleted file mode 100644 index 0948f0ca..00000000 --- a/crates/canopy-server/src/schema.sql +++ /dev/null @@ -1,336 +0,0 @@ -CREATE TABLE object_uploads ( - id BLOB PRIMARY KEY CHECK(length(id) = 16) -) WITHOUT ROWID; - -CREATE TABLE object_chunks ( - upload_id BLOB NOT NULL REFERENCES object_uploads(id), - part INTEGER NOT NULL CHECK(part >= 0), - body BLOB NOT NULL CHECK(length(body) BETWEEN 1 AND 524288), - PRIMARY KEY(upload_id, part) -) WITHOUT ROWID; - -CREATE TABLE objects ( - sequence INTEGER PRIMARY KEY AUTOINCREMENT, - oid BLOB NOT NULL UNIQUE CHECK(length(oid) IN (20, 32)), - kind TEXT NOT NULL CHECK(kind IN ('blob', 'tree', 'commit', 'tag')), - size INTEGER NOT NULL CHECK(size >= 0), - digest BLOB NOT NULL CHECK(length(digest) = 32), - storage TEXT NOT NULL CHECK(storage IN ('inline', 'external', 'chunked', 'packed')), - body BLOB, - external_sha256 BLOB, - chunk_id BLOB UNIQUE REFERENCES object_uploads(id) CHECK(chunk_id IS NULL OR length(chunk_id) = 16), - CHECK( - (storage = 'inline' AND body IS NOT NULL AND external_sha256 IS NULL AND chunk_id IS NULL AND size = length(body)) - OR - (storage = 'external' AND kind = 'blob' AND body IS NULL AND chunk_id IS NULL AND length(external_sha256) = 32) - OR - (storage = 'packed' AND kind = 'blob' AND body IS NULL AND chunk_id IS NULL AND length(external_sha256) = 32 ) - OR - (storage = 'chunked' AND kind != 'blob' AND body IS NULL AND external_sha256 IS NULL AND chunk_id IS NOT NULL AND size > 786432) - ) -); - -CREATE TABLE object_edges ( - parent BLOB NOT NULL REFERENCES objects(oid), - child BLOB NOT NULL REFERENCES objects(oid), - PRIMARY KEY(parent, child) -) WITHOUT ROWID; -CREATE INDEX object_edges_by_child ON object_edges(child, parent); - -CREATE TABLE object_closure ( - oid BLOB PRIMARY KEY REFERENCES objects(oid) CHECK(length(oid) IN (20, 32)) -) WITHOUT ROWID; - -CREATE TABLE refs ( - name TEXT PRIMARY KEY, - oid BLOB CHECK(oid IS NULL OR length(oid) IN (20, 32)), - version INTEGER NOT NULL CHECK(version > 0) -) WITHOUT ROWID; - -CREATE INDEX refs_by_oid ON refs(oid) WHERE oid IS NOT NULL; - -CREATE TABLE ref_generation ( - singleton INTEGER PRIMARY KEY CHECK(singleton = 1), - generation INTEGER NOT NULL CHECK(typeof(generation) = 'integer' AND generation >= 0), - default_branch TEXT NOT NULL, - visibility TEXT NOT NULL DEFAULT 'private' CHECK(visibility IN ('private', 'public')) -) WITHOUT ROWID; -INSERT INTO ref_generation (singleton, generation, default_branch) VALUES (1, 0, 'refs/heads/main'); - -CREATE TABLE lfs_objects ( - sha256 BLOB PRIMARY KEY CHECK(length(sha256) = 32), - size INTEGER NOT NULL CHECK(size >= 0), - digest BLOB NOT NULL CHECK(length(digest) = 32) -) WITHOUT ROWID; - -CREATE TABLE lfs_locks ( - sequence INTEGER PRIMARY KEY AUTOINCREMENT, - id TEXT NOT NULL UNIQUE, - path TEXT NOT NULL UNIQUE, - locked_at TEXT NOT NULL, - owner TEXT NOT NULL -); - -CREATE TABLE repository_identity ( - object_format TEXT NOT NULL CHECK(object_format IN ('sha1', 'sha256')), - singleton INTEGER PRIMARY KEY CHECK(singleton = 1), - repository_id BLOB NOT NULL CHECK(length(repository_id) = 16), - owner TEXT NOT NULL, - push_cert_seed BLOB NOT NULL CHECK(length(push_cert_seed) = 32) -) WITHOUT ROWID; - -CREATE TABLE repository_members ( - account TEXT PRIMARY KEY, - role TEXT NOT NULL CHECK(role IN ('read', 'write')) -) WITHOUT ROWID; - -CREATE TABLE membership_versions ( - account TEXT PRIMARY KEY, - version INTEGER NOT NULL CHECK(typeof(version) = 'integer' AND version > 0) -) WITHOUT ROWID; - -CREATE TABLE pushes ( - id BLOB PRIMARY KEY CHECK(length(id) = 16), - actor TEXT NOT NULL, - request_digest BLOB NOT NULL CHECK(length(request_digest) = 32), - options TEXT NOT NULL DEFAULT '[]' CHECK(length(CAST(options AS BLOB)) <= 32768), - response_id BLOB CHECK(response_id IS NULL OR length(response_id) = 16), - rejected INTEGER CHECK(rejected IN (0, 1)), - rejection_reason TEXT, - CHECK((response_id IS NULL) = (rejected IS NULL)) -) WITHOUT ROWID; - -CREATE TABLE push_certificates ( - digest BLOB PRIMARY KEY CHECK(length(digest) = 32), - push_id BLOB NOT NULL UNIQUE REFERENCES pushes(id), - actor TEXT NOT NULL, - signer TEXT NOT NULL, - key TEXT NOT NULL, - size INTEGER NOT NULL CHECK(size > 0), - recorded_at_ms INTEGER NOT NULL CHECK(recorded_at_ms >= 0) -) WITHOUT ROWID; - -CREATE TABLE push_certificate_chunks ( - push_id BLOB NOT NULL REFERENCES pushes(id), - part INTEGER NOT NULL CHECK(part >= 0), - body BLOB NOT NULL CHECK(length(body) BETWEEN 1 AND 524288), - PRIMARY KEY(push_id, part) -) WITHOUT ROWID; - -CREATE TABLE push_responses ( - id BLOB PRIMARY KEY CHECK(length(id) = 16), - push_id BLOB NOT NULL REFERENCES pushes(id), - status INTEGER NOT NULL CHECK(status BETWEEN 100 AND 599), - headers TEXT NOT NULL, - size INTEGER NOT NULL CHECK(size BETWEEN 0 AND 67108864), - digest BLOB NOT NULL CHECK(length(digest) = 32) -) WITHOUT ROWID; - -CREATE TABLE push_response_chunks ( - response_id BLOB NOT NULL REFERENCES push_responses(id), - part INTEGER NOT NULL CHECK(part >= 0), - body BLOB NOT NULL CHECK(length(body) BETWEEN 1 AND 524288), - PRIMARY KEY(response_id, part) -) WITHOUT ROWID; - -CREATE TABLE push_plan_chunks ( - response_id BLOB NOT NULL REFERENCES push_responses(id), - part INTEGER NOT NULL CHECK(part >= 0), - body BLOB NOT NULL CHECK(length(body) BETWEEN 1 AND 65536), - PRIMARY KEY(response_id, part) -) WITHOUT ROWID; - -CREATE TABLE issues ( - number INTEGER PRIMARY KEY AUTOINCREMENT, - id BLOB NOT NULL UNIQUE CHECK(length(id) = 16), - creation_digest BLOB NOT NULL CHECK(length(creation_digest) = 32), - author TEXT NOT NULL, - title TEXT NOT NULL CHECK(length(CAST(title AS BLOB)) BETWEEN 1 AND 256), - body TEXT NOT NULL CHECK(length(CAST(body AS BLOB)) <= 16384), - state TEXT NOT NULL CHECK(state IN ('open', 'closed')), - version INTEGER NOT NULL CHECK(typeof(version) = 'integer' AND version > 0), - created_ms INTEGER NOT NULL CHECK(created_ms >= 0), - updated_ms INTEGER NOT NULL CHECK(updated_ms >= created_ms) -); -CREATE INDEX issues_by_state ON issues(state, number); - -CREATE TABLE issue_comments ( - number INTEGER PRIMARY KEY AUTOINCREMENT, - issue_number INTEGER NOT NULL REFERENCES issues(number), - id BLOB NOT NULL UNIQUE CHECK(length(id) = 16), - creation_digest BLOB NOT NULL CHECK(length(creation_digest) = 32), - author TEXT NOT NULL, - body TEXT NOT NULL CHECK(length(CAST(body AS BLOB)) BETWEEN 1 AND 16384), - version INTEGER NOT NULL CHECK(typeof(version) = 'integer' AND version > 0), - created_ms INTEGER NOT NULL CHECK(created_ms >= 0), - updated_ms INTEGER NOT NULL CHECK(updated_ms >= created_ms) -); -CREATE INDEX comments_by_issue ON issue_comments(issue_number, number); - -CREATE TABLE check_contexts ( - name TEXT PRIMARY KEY, - reporter TEXT NOT NULL, - enabled INTEGER NOT NULL CHECK(enabled IN (0, 1)), - version INTEGER NOT NULL CHECK(typeof(version) = 'integer' AND version > 0) -) WITHOUT ROWID; - -CREATE INDEX check_contexts_by_enabled ON check_contexts(enabled, name); - -CREATE TABLE check_runs ( - number INTEGER PRIMARY KEY AUTOINCREMENT, - id BLOB NOT NULL UNIQUE CHECK(length(id) = 16), - oid BLOB NOT NULL REFERENCES objects(oid) CHECK(length(oid) IN (20, 32)), - context TEXT NOT NULL REFERENCES check_contexts(name), - context_version INTEGER NOT NULL CHECK(context_version > 0), - reporter TEXT NOT NULL, - state TEXT NOT NULL CHECK(state IN ('queued', 'in_progress', 'success', 'failure', 'cancelled')), - version INTEGER NOT NULL CHECK(typeof(version) = 'integer' AND version > 0), - summary TEXT NOT NULL CHECK(length(CAST(summary AS BLOB)) <= 4096), - created_ms INTEGER NOT NULL CHECK(created_ms >= 0), - updated_ms INTEGER NOT NULL CHECK(updated_ms >= created_ms) -); -CREATE INDEX check_runs_by_commit ON check_runs(oid, context, context_version, number); - -CREATE TABLE commit_parents ( - child BLOB NOT NULL REFERENCES objects(oid) CHECK(length(child) IN (20, 32)), - parent BLOB NOT NULL REFERENCES objects(oid) CHECK(length(parent) IN (20, 32)), - PRIMARY KEY(child, parent) -) WITHOUT ROWID; - -CREATE TABLE commit_ancestry ( - ancestor BLOB NOT NULL REFERENCES objects(oid) CHECK(length(ancestor) IN (20, 32)), - descendant BLOB NOT NULL REFERENCES objects(oid) CHECK(length(descendant) IN (20, 32)), - PRIMARY KEY(ancestor, descendant) -) WITHOUT ROWID; - -CREATE TABLE branch_rules ( - reference TEXT PRIMARY KEY, - version INTEGER NOT NULL CHECK(typeof(version) = 'integer' AND version > 0), - enabled INTEGER NOT NULL CHECK(enabled IN (0, 1)), - deny_deletions INTEGER NOT NULL CHECK(deny_deletions IN (0, 1)), - fast_forward INTEGER NOT NULL CHECK(fast_forward IN (0, 1)), - require_pull_request INTEGER NOT NULL CHECK(require_pull_request IN (0, 1)), - required_approvals INTEGER NOT NULL CHECK(required_approvals BETWEEN 0 AND 16 AND (require_pull_request = 1 OR required_approvals = 0)) -) WITHOUT ROWID; -CREATE TABLE branch_required_checks ( - reference TEXT NOT NULL REFERENCES branch_rules(reference), - context TEXT NOT NULL, - PRIMARY KEY(reference, context) -) WITHOUT ROWID; -CREATE INDEX branch_rules_by_enabled ON branch_rules(enabled, reference); - -CREATE TABLE pull_requests ( - number INTEGER PRIMARY KEY AUTOINCREMENT, - id BLOB NOT NULL UNIQUE CHECK(length(id) = 16), - creation_digest BLOB NOT NULL CHECK(length(creation_digest) = 32), - author TEXT NOT NULL, - title TEXT NOT NULL CHECK(length(CAST(title AS BLOB)) BETWEEN 1 AND 256), - body TEXT NOT NULL CHECK(length(CAST(body AS BLOB)) <= 16384), - state TEXT NOT NULL CHECK(state IN ('open', 'closed', 'merged')), - draft INTEGER NOT NULL CHECK(draft IN (0, 1)), - version INTEGER NOT NULL CHECK(typeof(version) = 'integer' AND version > 0), - source_ref TEXT NOT NULL REFERENCES refs(name), - base_ref TEXT NOT NULL REFERENCES refs(name) CHECK(source_ref != base_ref), - initial_source_oid BLOB NOT NULL REFERENCES objects(oid) CHECK(length(initial_source_oid) IN (20, 32)), - initial_base_oid BLOB NOT NULL REFERENCES objects(oid) CHECK(length(initial_base_oid) IN (20, 32)), - created_ms INTEGER NOT NULL CHECK(created_ms >= 0), - updated_ms INTEGER NOT NULL CHECK(updated_ms >= created_ms) -); -CREATE INDEX pulls_by_state ON pull_requests(state, number); - -CREATE TABLE pull_reviews ( - number INTEGER PRIMARY KEY AUTOINCREMENT, - id BLOB NOT NULL UNIQUE CHECK(length(id) = 16), - creation_digest BLOB NOT NULL CHECK(length(creation_digest) = 32), - pull_number INTEGER NOT NULL REFERENCES pull_requests(number), - reviewer TEXT NOT NULL, - membership_version INTEGER NOT NULL CHECK(membership_version >= 0), - kind TEXT NOT NULL CHECK(kind IN ('comment', 'approve', 'request_changes')), - body TEXT NOT NULL CHECK(length(CAST(body AS BLOB)) <= 16384), - pull_version INTEGER NOT NULL CHECK(pull_version > 0), - source_oid BLOB NOT NULL REFERENCES objects(oid) CHECK(length(source_oid) IN (20, 32)), - source_version INTEGER NOT NULL CHECK(source_version > 0), - base_oid BLOB NOT NULL REFERENCES objects(oid) CHECK(length(base_oid) IN (20, 32)), - base_version INTEGER NOT NULL CHECK(base_version > 0), - created_ms INTEGER NOT NULL CHECK(created_ms >= 0) -); -CREATE INDEX reviews_by_pull ON pull_reviews(pull_number, number); - -CREATE TABLE pull_review_heads ( - pull_number INTEGER NOT NULL REFERENCES pull_requests(number), - reviewer TEXT NOT NULL, - review_number INTEGER NOT NULL REFERENCES pull_reviews(number), - PRIMARY KEY(pull_number, reviewer) -) WITHOUT ROWID; - -CREATE TABLE pull_merges ( - id BLOB PRIMARY KEY CHECK(length(id) = 16), - binding BLOB NOT NULL CHECK(length(binding) = 32), - pull_number INTEGER NOT NULL UNIQUE REFERENCES pull_requests(number), - oid BLOB NOT NULL REFERENCES objects(oid) CHECK(length(oid) IN (20, 32)), - merged_ms INTEGER NOT NULL CHECK(merged_ms >= 0), - pull_version INTEGER NOT NULL CHECK(pull_version > 0), - source_oid BLOB NOT NULL REFERENCES objects(oid) CHECK(length(source_oid) IN (20, 32)), - source_version INTEGER NOT NULL CHECK(source_version > 0), - base_oid BLOB NOT NULL REFERENCES objects(oid) CHECK(length(base_oid) IN (20, 32)), - base_version INTEGER NOT NULL CHECK(base_version > 0) -) WITHOUT ROWID; - -CREATE TABLE merge_candidates ( - id BLOB PRIMARY KEY CHECK(length(id) = 16), - binding BLOB NOT NULL CHECK(length(binding) = 32), - pull_number INTEGER NOT NULL REFERENCES pull_requests(number), - actor TEXT NOT NULL, - request TEXT NOT NULL CHECK(length(CAST(request AS BLOB)) <= 131072), - created_ms INTEGER NOT NULL CHECK(created_ms >= 0), - result TEXT NOT NULL CHECK(length(CAST(result AS BLOB)) <= 262144), - source_oid BLOB NOT NULL REFERENCES objects(oid), - base_oid BLOB NOT NULL REFERENCES objects(oid), - oid BLOB REFERENCES objects(oid) -) WITHOUT ROWID; - -CREATE TABLE pull_threads ( - number INTEGER PRIMARY KEY AUTOINCREMENT, - id BLOB NOT NULL UNIQUE CHECK(length(id) = 16), - creation_digest BLOB NOT NULL CHECK(length(creation_digest) = 32), - pull_number INTEGER NOT NULL REFERENCES pull_requests(number), - author TEXT NOT NULL, - body TEXT NOT NULL CHECK(length(CAST(body AS BLOB)) BETWEEN 1 AND 16384), - resolved INTEGER NOT NULL CHECK(resolved IN (0, 1)), - version INTEGER NOT NULL CHECK(typeof(version) = 'integer' AND version > 0), - pull_version INTEGER NOT NULL CHECK(pull_version > 0), - source_oid BLOB NOT NULL REFERENCES objects(oid), - source_version INTEGER NOT NULL CHECK(source_version > 0), - base_oid BLOB NOT NULL REFERENCES objects(oid), - base_version INTEGER NOT NULL CHECK(base_version > 0), - merge_base BLOB NOT NULL REFERENCES objects(oid), - path BLOB NOT NULL CHECK(length(path) BETWEEN 1 AND 4096), - side TEXT NOT NULL CHECK(side IN ('before', 'after')), - line INTEGER NOT NULL CHECK(line BETWEEN 1 AND 20000), - blob_oid BLOB NOT NULL REFERENCES objects(oid), - created_ms INTEGER NOT NULL CHECK(created_ms >= 0), - updated_ms INTEGER NOT NULL CHECK(updated_ms >= created_ms) -); -CREATE INDEX threads_by_pull ON pull_threads(pull_number, number); - -CREATE TABLE pull_thread_comments ( - number INTEGER PRIMARY KEY AUTOINCREMENT, - id BLOB NOT NULL UNIQUE CHECK(length(id) = 16), - creation_digest BLOB NOT NULL CHECK(length(creation_digest) = 32), - thread_number INTEGER NOT NULL REFERENCES pull_threads(number), - author TEXT NOT NULL, - body TEXT NOT NULL CHECK(length(CAST(body AS BLOB)) BETWEEN 1 AND 16384), - created_ms INTEGER NOT NULL CHECK(created_ms >= 0) -); -CREATE INDEX comments_by_thread ON pull_thread_comments(thread_number, number); - --- Immutable uploaded pack/index pairs. Approval follows canonical verification --- of every native index member, including pre-existing objects and thin bases. -CREATE TABLE git_packs ( - sha256 BLOB PRIMARY KEY CHECK(length(sha256) = 32), - pack_hash BLOB NOT NULL UNIQUE CHECK(length(pack_hash) IN (20, 32)), - pack_oid BLOB NOT NULL, pack_size INTEGER NOT NULL CHECK(pack_size > 0), pack_digest BLOB NOT NULL CHECK(length(pack_digest) = 32), - index_oid BLOB NOT NULL, index_size INTEGER NOT NULL CHECK(index_size > 0), index_digest BLOB NOT NULL CHECK(length(index_digest) = 32), index_sha256 BLOB NOT NULL CHECK(length(index_sha256) = 32), - approved INTEGER NOT NULL DEFAULT 0 CHECK(approved IN (0, 1)), - covered_through INTEGER NOT NULL DEFAULT 0 CHECK(covered_through >= 0) -) WITHOUT ROWID; diff --git a/crates/canopy-server/src/server/catalog_admission/tests.rs b/crates/canopy-server/src/server/catalog_admission/tests.rs index a9c8d33c..ae19daa0 100644 --- a/crates/canopy-server/src/server/catalog_admission/tests.rs +++ b/crates/canopy-server/src/server/catalog_admission/tests.rs @@ -190,26 +190,17 @@ impl Fixture { } #[tokio::test] -async fn supported_current_and_retained_proofs_are_returned_unchanged() -> TestResult { - let retained = Digest::from_bytes( - hex::decode("f7254eda9d5d339566f45457502618ad13cbbf6e5a74595f5b3ce46653ea12f1")? - .try_into() - .map_err(|_| "invalid retained code")?, - ); - for code in [None, Some(retained)] { - let fixture = Fixture::new()?; - fixture.ready().await?; - let proof = fixture - .proof(CatalogRole::Sql, code.unwrap_or(fixture.code()), 1) - .await?; - let entry = proof.entry().clone(); - let revision = proof.revision(); - let release = fixture.releases.load().await?.unwrap().record().clone(); - let admitted = fixture.admit(proof).await?; - assert_eq!(admitted.entry(), &entry); - assert_eq!(admitted.revision(), revision); - assert_eq!(fixture.releases.load().await?.unwrap().record(), &release); - } +async fn current_catalog_proof_is_returned_unchanged() -> TestResult { + let fixture = Fixture::new()?; + fixture.ready().await?; + let proof = fixture.proof(CatalogRole::Sql, fixture.code(), 1).await?; + let entry = proof.entry().clone(); + let revision = proof.revision(); + let release = fixture.releases.load().await?.unwrap().record().clone(); + let admitted = fixture.admit(proof).await?; + assert_eq!(admitted.entry(), &entry); + assert_eq!(admitted.revision(), revision); + assert_eq!(fixture.releases.load().await?.unwrap().record(), &release); Ok(()) } @@ -278,7 +269,7 @@ async fn corrupted_selected_descriptor_closes_existing_admission() -> TestResult #[tokio::test] async fn unsupported_catalog_identity_and_wrong_target_are_not_adopted() -> TestResult { - for case in 0..4 { + for case in 0..5 { let fixture = Fixture::new()?; fixture.ready().await?; let role = if case == 0 { @@ -288,6 +279,12 @@ async fn unsupported_catalog_identity_and_wrong_target_are_not_adopted() -> Test }; let code = if case == 1 { Digest::from_bytes([1; 32]) + } else if case == 4 { + Digest::from_bytes( + hex::decode("f7254eda9d5d339566f45457502618ad13cbbf6e5a74595f5b3ce46653ea12f1")? + .try_into() + .map_err(|_| "old code")?, + ) } else { fixture.code() }; diff --git a/crates/canopy-server/src/server/catalog_initialization/tests.rs b/crates/canopy-server/src/server/catalog_initialization/tests.rs index 519603ce..81027d31 100644 --- a/crates/canopy-server/src/server/catalog_initialization/tests.rs +++ b/crates/canopy-server/src/server/catalog_initialization/tests.rs @@ -68,7 +68,7 @@ impl Fixture { registry .module_code(RepositoryModule::NAME) .ok_or("module")?, - 1, + crate::REPOSITORY_SCHEMA_VERSION, )?) .await?; let incarnation = IncarnationId::from_bytes([93; 16]); diff --git a/crates/canopy-server/src/server/mod.rs b/crates/canopy-server/src/server/mod.rs index 22de0916..30100b37 100644 --- a/crates/canopy-server/src/server/mod.rs +++ b/crates/canopy-server/src/server/mod.rs @@ -854,7 +854,16 @@ async fn acquire_sql_cell( .provision( &catalog, ®istry, - CatalogEntry::new(target, CatalogRole::Sql, code, 1)?, + CatalogEntry::new( + target, + CatalogRole::Sql, + code, + if target.namespace() == crate::REPOSITORIES { + crate::REPOSITORY_SCHEMA_VERSION + } else { + 1 + }, + )?, ) .await? } diff --git a/crates/canopy-server/src/server/residency/recovery.rs b/crates/canopy-server/src/server/residency/recovery.rs index 036e972b..a7728071 100644 --- a/crates/canopy-server/src/server/residency/recovery.rs +++ b/crates/canopy-server/src/server/residency/recovery.rs @@ -49,7 +49,13 @@ impl RecoveryServices { StagingCoordinator::new_resident( client.clone(), target.clone(), - StagingLimits::default(), + StagingLimits { + // Real histories can need more than one minute to replay + // verified metadata. Keep the existing five-minute ceiling; + // custody still renews independently every lease interval. + bound_lifetime_ms: crate::packs::publication::MAX_LEASE_MS, + ..StagingLimits::default() + }, authority.clone(), manager.staging_budget.clone(), coordinator.clone(), diff --git a/crates/canopy-server/src/server/workspace/mod.rs b/crates/canopy-server/src/server/workspace/mod.rs index e7f1ab05..fc3271d8 100644 --- a/crates/canopy-server/src/server/workspace/mod.rs +++ b/crates/canopy-server/src/server/workspace/mod.rs @@ -42,15 +42,17 @@ impl Workspace { fs::create_dir_all(directory)?; let directory = fs::canonicalize(directory)?; let owner = OwnerLock::acquire(&directory.join(".canopy-owner.lock"))?; - match fs::symlink_metadata(directory.join("runtime-v1")) { - Ok(_) => { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "legacy Canopy runtime requires a fresh packed-format data directory", - )); + for previous in ["runtime-v1", "canopy-pack-v1"] { + match fs::symlink_metadata(directory.join(previous)) { + Ok(_) => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "older Canopy state requires a fresh data directory", + )); + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), } - Err(error) if error.kind() == io::ErrorKind::NotFound => {} - Err(error) => return Err(error), } let root = directory.join(crate::deployment::STORAGE_FORMAT); #[cfg(unix)] diff --git a/crates/canopy-server/src/server/workspace/tests.rs b/crates/canopy-server/src/server/workspace/tests.rs index 0235936d..eab91a02 100644 --- a/crates/canopy-server/src/server/workspace/tests.rs +++ b/crates/canopy-server/src/server/workspace/tests.rs @@ -14,7 +14,7 @@ fn legacy_runtime_prevents_cutover_and_preserves_every_file() -> Result { .is_err_and(|error| error.kind() == io::ErrorKind::InvalidData) ); assert_eq!(fs::read(legacy.join("important"))?, b"retain old state"); - assert!(!directory.path().join("canopy-pack-v1").exists()); + assert!(!directory.path().join("canopy-pack-v2").exists()); Ok(()) } @@ -26,7 +26,7 @@ fn packed_workspace_checks_format_before_reclaiming_any_file() -> Result { Some(b"unknown-format"), ] { let directory = tempfile::TempDir::new()?; - let root = directory.path().join("canopy-pack-v1"); + let root = directory.path().join("canopy-pack-v2"); fs::create_dir(&root)?; fs::write(root.join("important"), b"retain unrecognized state")?; if let Some(marker) = marker { diff --git a/crates/canopy-server/tests/directory_cell/compatibility.rs b/crates/canopy-server/tests/directory_cell/compatibility.rs index 41e18165..790fcf2b 100644 --- a/crates/canopy-server/tests/directory_cell/compatibility.rs +++ b/crates/canopy-server/tests/directory_cell/compatibility.rs @@ -4,9 +4,9 @@ use cellule_runtime::Digest; // Captured read-only from the unchanged RustFS deployment. The canonical // descriptor's SHA-256 is 8b85c842995e4a2b0bcc6d1f3d361ccdbd2c33a1c52592241b50ad8cfe9aeb88. // The full historical application is refused by the Repository hard cutover. -// Directory compatibility is checked separately; no old binary restore is implied. +// Both modules reject predecessor code; fresh prefixes are required. #[test] -fn bounded_authentication_retains_the_exact_selected_predecessor_directory() +fn bounded_authentication_rejects_the_selected_predecessor_directory() -> Result<(), Box> { let previous = include_bytes!("fixtures/c51-selected-release.json").trim_ascii_end(); assert_eq!( @@ -24,8 +24,6 @@ fn bounded_authentication_retains_the_exact_selected_predecessor_directory() "rolling release does not retain predecessor module code" )) )); - let scoped = retained_directory::predecessor(®istry, previous)?; - registry.verify_rolling_from(&scoped)?; let predecessor: serde_json::Value = serde_json::from_slice(previous)?; let old_directory = predecessor["modules"] .as_array() @@ -38,7 +36,9 @@ fn bounded_authentication_retains_the_exact_selected_predecessor_directory() .map_err(|_| "invalid predecessor code length")?; let old_code = Digest::from_bytes(bytes); assert_ne!(Some(old_code), registry.module_code("directory")); - assert!(registry.supports_cell(directory::DIRECTORY, CatalogRole::Sql, old_code, 1)); + assert!(!registry.supports_cell(directory::DIRECTORY, CatalogRole::Sql, old_code, 1)); + assert!(DirectoryModule.descriptor().retained_codes.is_empty()); + assert!(RepositoryModule.descriptor().retained_codes.is_empty()); for schema in [0, 2] { assert!(!registry.supports_cell(directory::DIRECTORY, CatalogRole::Sql, old_code, schema)); } @@ -95,7 +95,7 @@ fn packed_repository_rejects_an_unqualified_predecessor_upgrade() )); assert!( registry.verify_rolling_from(previous).is_err(), - "pack storage requires a qualified repository migration before rolling upgrade" + "pack storage requires a fresh deployment prefix" ); Ok(()) } diff --git a/crates/canopy-server/tests/directory_cell/main.rs b/crates/canopy-server/tests/directory_cell/main.rs index 9f029af8..d0a97f84 100644 --- a/crates/canopy-server/tests/directory_cell/main.rs +++ b/crates/canopy-server/tests/directory_cell/main.rs @@ -2,8 +2,6 @@ mod accounts; mod capacity; mod compatibility; mod expiry; -#[path = "../support/retained_directory.rs"] -mod retained_directory; mod ssh_keys; use std::{ @@ -156,7 +154,7 @@ async fn directory_reservations_recover_two_distinct_repository_cells() ®istry, &layout, &first_target, - (RepositoryModule::NAME, include_str!("../../src/schema.sql")), + (RepositoryModule::NAME, canopy_server::REPOSITORY_SCHEMA), first_session, &files.path().join("first-alpha.sqlite"), ) @@ -176,7 +174,7 @@ async fn directory_reservations_recover_two_distinct_repository_cells() ®istry, &layout, &second_target, - (RepositoryModule::NAME, include_str!("../../src/schema.sql")), + (RepositoryModule::NAME, canopy_server::REPOSITORY_SCHEMA), first_session, &files.path().join("first-beta.sqlite"), ) @@ -461,7 +459,16 @@ async fn bootstrap( let (module, schema) = module_schema; let code = registry.module_code(module).ok_or("module missing")?; let proof = CellCatalog::new(layout.clone(), target.tenant()) - .provision(CatalogEntry::new(target, CatalogRole::Sql, code, 1)?) + .provision(CatalogEntry::new( + target, + CatalogRole::Sql, + code, + if target.namespace() == canopy_server::REPOSITORIES { + canopy_server::REPOSITORY_SCHEMA_VERSION + } else { + 1 + }, + )?) .await?; let authority = CellAuthority::new(layout.clone()); let observed = authority @@ -506,7 +513,16 @@ async fn restore( ) -> Result> { let code = registry.module_code(module).ok_or("module missing")?; let proof = CellCatalog::new(layout.clone(), target.tenant()) - .provision(CatalogEntry::new(target, CatalogRole::Sql, code, 1)?) + .provision(CatalogEntry::new( + target, + CatalogRole::Sql, + code, + if target.namespace() == canopy_server::REPOSITORIES { + canopy_server::REPOSITORY_SCHEMA_VERSION + } else { + 1 + }, + )?) .await?; let authority = CellAuthority::new(layout.clone()); let observed = authority diff --git a/crates/canopy-server/tests/multi_server/catalog_contract.rs b/crates/canopy-server/tests/multi_server/catalog_contract.rs new file mode 100644 index 00000000..8030ef24 --- /dev/null +++ b/crates/canopy-server/tests/multi_server/catalog_contract.rs @@ -0,0 +1,71 @@ +//! Unsupported persisted identities fail before Cell acquisition or rewriting. +use super::*; +use cellule_runtime::{control::authority::CellAuthority, ltx::CellStorageLayout}; +use cellule_store::Store; + +type Result = std::result::Result>; + +async fn unsupported_control_stays_unchanged(code: Option, schema: Option) -> Result { + let files = tempfile::TempDir::new()?; + let address = available_address().await?; + let data = files.path().join("server"); + let configuration = config(address, data.clone()); + let target = canopy_server::directory::directory_target( + configuration.tenant, + configuration.application, + )?; + let store: Arc = Arc::new(InMemory::new()); + let layout = CellStorageLayout::new( + Store::new(Arc::clone(&store)), + configuration.store_prefix.clone(), + *configuration.application.as_bytes(), + ); + let server = CanopyServer::start(configuration, Arc::clone(&store)).await?; + server.shutdown().await?; + let authority = CellAuthority::new(layout.clone()); + let observed = authority + .load(target.cell_id()) + .await? + .ok_or("control missing")?; + let mut unsupported = observed.value().clone(); + if let Some(code) = code { + unsupported.code = code; + } + if let Some(schema) = schema { + unsupported.schema = schema; + } + unsupported.revision += 1; + unsupported.progress += 1; + let path = layout.control_path(target.cell_id().as_bytes()); + let (_, token) = layout.store().get_with_etag(&path).await?; + let encoded = unsupported.encode()?; + layout + .store() + .update(&path, encoded.clone().into(), token) + .await?; + assert!( + CanopyServer::start(config(address, data), Arc::clone(&store)) + .await + .is_err() + ); + assert_eq!( + authority + .load(target.cell_id()) + .await? + .ok_or("control disappeared")? + .value() + .encode()?, + encoded + ); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn unknown_control_code_is_rejected_before_cell_acquisition() -> Result { + unsupported_control_stays_unchanged(Some(Digest::from_bytes([1; 32])), None).await +} + +#[tokio::test(flavor = "multi_thread")] +async fn unknown_control_schema_is_rejected_before_cell_acquisition() -> Result { + unsupported_control_stays_unchanged(None, Some(2)).await +} diff --git a/crates/canopy-server/tests/multi_server/lifecycle/lease.rs b/crates/canopy-server/tests/multi_server/lifecycle/lease.rs index a2bf4983..be32af23 100644 --- a/crates/canopy-server/tests/multi_server/lifecycle/lease.rs +++ b/crates/canopy-server/tests/multi_server/lifecycle/lease.rs @@ -249,13 +249,25 @@ async fn expired_successful_renewal_reply_cannot_reopen_ingress() -> Result { .send() .await; assert!(expired_at_reply > expires); + // Cellule retains the original drain failure through its host/facility + // wrappers. Require the actual typed fencing cause, not its outer shape. + let mut cause = shutdown + .as_ref() + .err() + .map(|error| error as &(dyn std::error::Error + 'static)); + let mut fenced = false; + while let Some(error) = cause { + if matches!( + error.downcast_ref::(), + Some(cellule_runtime::Error::Fenced) + ) { + fenced = true; + break; + } + cause = error.source(); + } assert!( - matches!( - shutdown, - Err(canopy_server::server::ServerError::Runtime( - cellule_runtime::Error::Fenced - )) - ), + fenced, "expired successful refresh did not fence: {shutdown:?}" ); assert!( diff --git a/crates/canopy-server/tests/multi_server/lifecycle/mod.rs b/crates/canopy-server/tests/multi_server/lifecycle/mod.rs index 75d9a4f6..a4dcc571 100644 --- a/crates/canopy-server/tests/multi_server/lifecycle/mod.rs +++ b/crates/canopy-server/tests/multi_server/lifecycle/mod.rs @@ -48,6 +48,60 @@ impl PausedStore { } } +#[tokio::test] +async fn delayed_renewal_pause_uses_first_post_arm_progress() -> Result { + let store = Arc::new(PausedStore::default()); + let path = StorePath::from("renewal-probe"); + let body = |progress: u64, expires: i64| { + serde_json::to_vec(&serde_json::json!({ + "lease": {"progress": progress.to_string(), "expires_at_ms": expires.to_string()} + })) + .unwrap() + }; + // Startup may already have renewed several times before a test arms it. + store + .put_opts(&path, body(8, 100).into(), PutOptions::default()) + .await?; + store.delay_renewals.store(true, Ordering::SeqCst); + let first_body = body(9, 200); + let writer = store.clone(); + let key = path.clone(); + let payload = first_body.clone(); + let first = tokio::spawn(async move { + writer + .put_opts(&key, payload.into(), PutOptions::default()) + .await + }); + store.wait().await?; + let captured = *store.renewal_expiry.lock().unwrap(); + let published = store.inner.get(&path).await?.bytes().await?; + let held = !first.is_finished(); + store.proceed.notify_one(); + let result = timeout(Duration::from_secs(1), first).await??; + assert_eq!(captured, Some(200)); + assert_eq!(published.as_ref(), first_body); + assert!(held, "first successful renewal reply was not held"); + result?; + + let writer = store.clone(); + let key = path.clone(); + let later = tokio::spawn(async move { + writer + .put_opts(&key, body(10, 300).into(), PutOptions::default()) + .await + }); + store.wait().await?; + let published = store.inner.get(&path).await?.bytes().await?; + store.proceed.notify_one(); + let result = timeout(Duration::from_secs(1), later).await??; + assert_eq!(published.as_ref(), first_body); + assert!(matches!( + result, + Err(object_store::Error::PermissionDenied { .. }) + )); + Ok(()) +} + impl fmt::Display for PausedStore { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str("paused-lifecycle-store") @@ -78,8 +132,9 @@ impl ObjectStore for PausedStore { *self.initial_advertisement.lock().unwrap() = advertisement.clone(); } let delayed = self.delay_renewals.load(Ordering::SeqCst); - if delayed && progress.is_some_and(|value| value > 2) { - // Keep later renewals unpublished until the confirmed lease expires. + let renewal = progress.is_some_and(|value| value > 1); + if delayed && renewal && self.renewal_expiry.lock().unwrap().is_some() { + // Keep renewals after the first held reply unpublished until expiry. self.entered.notify_one(); self.proceed.notified().await; return Err(object_store::Error::PermissionDenied { @@ -111,7 +166,7 @@ impl ObjectStore for PausedStore { } } let result = self.inner.put_opts(path, payload, options).await; - if delayed && progress == Some(2) && result.is_ok() { + if delayed && renewal && result.is_ok() { let expires = advertisement.as_ref().unwrap()["lease"]["expires_at_ms"] .as_str() .unwrap() @@ -514,7 +569,7 @@ async fn cancelled_maintenance_recovery_retains_enrollment_until_cell_cleanup() .registry() .module_code(RepositoryModule::NAME) .ok_or("missing module")?, - 1, + canopy_server::REPOSITORY_SCHEMA_VERSION, )?) .await?; let operation = RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()); diff --git a/crates/canopy-server/tests/multi_server/main.rs b/crates/canopy-server/tests/multi_server/main.rs index 9e882b53..ae2c6cac 100644 --- a/crates/canopy-server/tests/multi_server/main.rs +++ b/crates/canopy-server/tests/multi_server/main.rs @@ -14,8 +14,6 @@ mod paused_blobs; mod pulls; mod push_options; mod rebase; -#[path = "../support/retained_directory.rs"] -mod retained_directory; mod sha256; mod size; mod ssh; @@ -29,6 +27,7 @@ use object_store::{ObjectStore, memory::InMemory, path::Path as StorePath}; use tokio::{net::TcpListener, process::Command}; mod branch_rules; +mod catalog_contract; mod checks; mod collaborators; mod default_branch; @@ -41,7 +40,6 @@ mod lifecycle; mod listener_handoff; mod peers; mod residency; -mod retained_catalog; mod tokens; mod transfers; mod visibility; diff --git a/crates/canopy-server/tests/multi_server/partial_clone.rs b/crates/canopy-server/tests/multi_server/partial_clone.rs index 04ea2485..2193b566 100644 --- a/crates/canopy-server/tests/multi_server/partial_clone.rs +++ b/crates/canopy-server/tests/multi_server/partial_clone.rs @@ -133,7 +133,10 @@ async fn cold_single_branch_fetch_skips_unrelated_commit_and_tree_history() -> R ], ) .await?; - assert!(stored_objects(&clone).await?.contains(&last_tree)); + let objects = stored_objects(&clone).await?; + assert!(objects.contains(&last_tree)); + assert!(!objects.contains(&other_commit)); + assert!(!objects.contains(&other_tree)); assert!(!cache_contains(&disk, &other_commit)?); assert!(!cache_contains(&disk, &other_tree)?); run_git( @@ -147,8 +150,17 @@ async fn cold_single_branch_fetch_skips_unrelated_commit_and_tree_history() -> R ], ) .await?; - assert!(cache_contains(&disk, &other_commit)?); - assert!(cache_contains(&disk, &other_tree)?); + let objects = stored_objects(&clone).await?; + assert!(objects.contains(&other_commit)); + assert!(objects.contains(&other_tree)); + for (kind, object) in [("commit", &other_commit), ("tree", &other_tree)] { + assert_eq!( + run_git(Some(&clone), &["cat-file", kind, object]).await?, + run_git(Some(&source), &["cat-file", kind, object]).await? + ); + } + assert!(!cache_contains(&disk, &other_commit)?); + assert!(!cache_contains(&disk, &other_tree)?); restored.shutdown().await?; } Ok(()) @@ -313,8 +325,8 @@ async fn filtered_clones_round_trip(store: Arc) -> Result { assert_eq!(tokio::fs::read(clone.join("two")).await?, bodies[1]); run_git(Some(&clone), &["fsck", "--strict"]).await?; } - // Explicitly cache the branch while referenced, then delete it again. Fetch - // no longer loads orphan blobs, but cached orphans must still be denied. + // Cache an explicit blob want while its branch is referenced, then delete + // the branch. Physically cached orphans must fail reachability validation. run_git( Some(&source), &["-c", AUTH, "push", &url, "refs/heads/discarded"], @@ -327,6 +339,8 @@ async fn filtered_clones_round_trip(store: Arc) -> Result { "-c", AUTH, "clone", + "--filter=blob:none", + "--no-checkout", "--single-branch", "--branch", "discarded", @@ -335,6 +349,15 @@ async fn filtered_clones_round_trip(store: Arc) -> Result { ], ) .await?; + assert_eq!( + run_git( + Some(&discarded_clone), + &["-c", AUTH, "cat-file", "blob", &discarded] + ) + .await?, + run_git(Some(&source), &["cat-file", "blob", &discarded]).await? + ); + assert!(cache_contains(&restored_disk, &discarded)?); run_git( Some(&source), &["-c", AUTH, "push", &url, ":refs/heads/discarded"], diff --git a/crates/canopy-server/tests/multi_server/peers/cold_activation.rs b/crates/canopy-server/tests/multi_server/peers/cold_activation.rs index 7350e360..6d55768f 100644 --- a/crates/canopy-server/tests/multi_server/peers/cold_activation.rs +++ b/crates/canopy-server/tests/multi_server/peers/cold_activation.rs @@ -237,7 +237,7 @@ async fn qualify_competing_cold_gateways(delay_claim_reply: bool) -> Result { .await? .ok_or("missing control")?; let local_path = format!( - "canopy-pack-v1/{}/repository.sqlite", + "canopy-pack-v2/{}/repository.sqlite", repository_id.simple() ); let local_owners = ["first", "second"] diff --git a/crates/canopy-server/tests/multi_server/peers/mod.rs b/crates/canopy-server/tests/multi_server/peers/mod.rs index 6cad8fe2..9de76a18 100644 --- a/crates/canopy-server/tests/multi_server/peers/mod.rs +++ b/crates/canopy-server/tests/multi_server/peers/mod.rs @@ -173,7 +173,7 @@ async fn two_live_nodes_route_git_to_distinct_cell_owners_and_recover_the_direct let node = if name == "left" { "first" } else { "second" }; let other = if node == "first" { "second" } else { "first" }; let path = format!( - "canopy-pack-v1/{}/repository.sqlite", + "canopy-pack-v2/{}/repository.sqlite", hex::encode(id.as_bytes()) ); assert!(files.path().join(node).join(&path).exists()); diff --git a/crates/canopy-server/tests/multi_server/residency/faults/mod.rs b/crates/canopy-server/tests/multi_server/residency/faults/mod.rs index 58ae4ba4..95a322bf 100644 --- a/crates/canopy-server/tests/multi_server/residency/faults/mod.rs +++ b/crates/canopy-server/tests/multi_server/residency/faults/mod.rs @@ -210,7 +210,7 @@ impl Fixture { let oid = run_git(Some(&source), &["rev-parse", "HEAD"]).await?; create(&client, address, "second").await?; create(&client, address, "third").await?; - let local = workspace.path().join("server/canopy-pack-v1"); + let local = workspace.path().join("server/canopy-pack-v2"); Ok(Self { workspace, store, diff --git a/crates/canopy-server/tests/multi_server/residency/mod.rs b/crates/canopy-server/tests/multi_server/residency/mod.rs index f08089f9..e4950c3a 100644 --- a/crates/canopy-server/tests/multi_server/residency/mod.rs +++ b/crates/canopy-server/tests/multi_server/residency/mod.rs @@ -30,7 +30,7 @@ async fn repositories_beyond_resident_capacity_restore_git_and_lfs_on_the_same_n ); let authority = CellAuthority::new(layout.clone()); let server = CanopyServer::start(settings, store).await?; - let local_root = workspace.path().join("server/canopy-pack-v1"); + let local_root = workspace.path().join("server/canopy-pack-v2"); let local = workspace.path().join("source"); run_git(None, &["init", "-b", "main", path_str(&local)?]).await?; run_git(Some(&local), &["config", "user.name", "Canopy Test"]).await?; diff --git a/crates/canopy-server/tests/multi_server/retained_catalog.rs b/crates/canopy-server/tests/multi_server/retained_catalog.rs deleted file mode 100644 index c0e554b3..00000000 --- a/crates/canopy-server/tests/multi_server/retained_catalog.rs +++ /dev/null @@ -1,350 +0,0 @@ -//! Real startup seam for an immutable predecessor Directory catalog entry. -//! This owned in-memory fixture is not an upgrade of the live RustFS corpus -//! or proof that an old executable produced the stored bytes. Only the exact -//! historical Directory contract is selected; other modules remain current. - -use super::*; -use bytes::Bytes; -use canopy_server::{ - CanopyApplication, build_descriptor, - directory::{self, DirectoryCell, DirectoryModule, TokenScope}, -}; -use cellule_app::{ApplicationHandle, CellApplication}; -use cellule_ltx::{CellReplica, DiskBudget, Host, Limits}; -use cellule_runtime::{ - CellClient, CellModule, CellRuntime, MutationIdentity, SessionId, - cell::application::{ApplicationIdentity, ApplicationIdentityStore}, - cell::catalog::{CatalogEntry, CatalogRole, CellCatalog}, - cell::worker::SqlWorkerPool, - control::{ControlState, Owner, authority::CellAuthority}, - identity::{IncarnationId, RequestId}, - ltx::CellStorageLayout, - node::NodeDirectory, - recovery::release::{ReleaseState, ReleaseStore}, -}; -use cellule_store::Store; -use sha2::{Digest as _, Sha256}; -use std::time::{SystemTime, UNIX_EPOCH}; - -type Result = std::result::Result>; - -fn now() -> Result { - Ok(i64::try_from( - SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis(), - )?) -} - -struct RetainedFixture { - _files: tempfile::TempDir, - configuration: ServerConfig, - store: Arc, - layout: CellStorageLayout, - catalog: CellCatalog, - authority: CellAuthority, - target: cellule_runtime::CellTarget, - entry: CatalogEntry, -} - -async fn retained_fixture() -> Result { - let files = tempfile::TempDir::new()?; - let address = available_address().await?; - let configuration = config(address, files.path().join("server")); - let store: Arc = Arc::new(InMemory::new()); - let storage = Store::new(Arc::clone(&store)); - let identity = ApplicationIdentity::new(configuration.tenant, configuration.application); - storage - .create_strict( - &configuration - .store_prefix - .clone() - .join("canopy-root-v1.json"), - Bytes::from_static(br#"{"format":"canopy-pack-v1","purpose":{"kind":"service"}}"#), - ) - .await?; - ApplicationIdentityStore::new(storage.clone(), configuration.store_prefix.clone()) - .initialize(identity) - .await?; - let layout = CellStorageLayout::new( - storage, - configuration.store_prefix.clone(), - *configuration.application.as_bytes(), - ); - let application = Arc::new(CanopyApplication::compile(build_descriptor( - include_bytes!("../../../../Cargo.lock"), - env!("CARGO_PKG_VERSION"), - ))?); - let registry = application.registry(); - let previous = - include_bytes!("../directory_cell/fixtures/c51-selected-release.json").trim_ascii_end(); - assert_eq!( - blake3::hash(previous).to_hex().as_str(), - "e31bf1a951e2fa19d91e9f964b2ddeade1a81b05a20ad628362819a1487c16b1" - ); - assert!(matches!( - registry.verify_rolling_from(previous), - Err(cellule_runtime::Error::Registry( - "rolling release does not retain predecessor module code" - )) - )); - let scoped = retained_directory::predecessor(®istry, previous)?; - registry.verify_rolling_from(&scoped)?; - let descriptor: serde_json::Value = serde_json::from_slice(previous)?; - let module = descriptor["modules"] - .as_array() - .ok_or("modules missing")? - .iter() - .find(|module| module["name"] == DirectoryModule::NAME) - .ok_or("Directory predecessor missing")?; - let old_code = Digest::from_bytes( - hex::decode(module["code"].as_str().ok_or("code missing")?)? - .try_into() - .map_err(|_| "invalid predecessor code")?, - ); - let releases = ReleaseStore::new(layout.clone(), identity)?; - let image = format!("sha256:{}", hex::encode(configuration.image.as_bytes())); - let old_operation = RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()); - let prepared = releases - .prepare( - &scoped, - Digest::from_bytes(*blake3::hash(&scoped).as_bytes()), - 0, - &image, - old_operation, - ) - .await?; - // The fixture has no Cells or advertised writers at first activation. - let activating = releases - .start_activation(prepared.revision(), old_operation) - .await?; - let old_ready = releases - .complete_activation(activating.revision(), old_operation) - .await?; - assert_eq!(old_ready.state(), ReleaseState::Ready); - - let target = directory::directory_target(configuration.tenant, configuration.application)?; - let catalog = CellCatalog::new(layout.clone(), configuration.tenant); - let entry = CatalogEntry::new(&target, CatalogRole::Sql, old_code, 1)?; - let proof = catalog.provision(entry.clone()).await?; - let authority = CellAuthority::new(layout.clone()); - let session = SessionId::from_bytes(uuid::Uuid::new_v4().into_bytes()); - let observed = authority - .create_initial( - &proof, - IncarnationId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - Owner { - session, - endpoint: "https://canopy.test".into(), - }, - ) - .await?; - let runtime = CellRuntime::new_with_replica_host( - SqlWorkerPool::new(1, 4)?, - 64 * 1024 * 1024, - session, - Host::default().with_local_disk_budget(DiskBudget::new(1 << 30)), - )?; - let handle = runtime - .bootstrap( - proof, - CellReplica::new( - layout.clone(), - *target.cell_id().as_bytes(), - *observed.value().incarnation.as_bytes(), - Limits::default(), - )?, - authority.clone(), - observed, - files.path().join("predecessor.sqlite"), - |transaction| { - transaction.execute_batch(directory::SCHEMA)?; - Ok(()) - }, - ) - .await?; - let client = ApplicationHandle::::new( - CellClient::local(registry.clone(), handle), - application, - configuration.tenant, - configuration.application, - )?; - let directory = DirectoryCell::new(&client, target.clone())?; - let issued = now()?; - directory - .create_account( - MutationIdentity { - request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - issued_at_ms: issued, - expires_at_ms: issued + 60_000, - }, - "canopy", - Sha256::digest(configuration.token.as_bytes()).into(), - TokenScope::Admin, - ) - .await?; - runtime.shutdown().await?; - let old_control = authority - .load(target.cell_id()) - .await? - .ok_or("control missing")?; - assert_eq!(old_control.value().code, old_code); - assert_eq!(old_control.value().schema, 1); - assert_eq!(old_control.value().state, ControlState::Idle); - assert!(old_control.value().root.is_some()); - assert!(old_control.value().owner.is_none()); - - // Explicit fixture-only admission, before completing the new release. - // The low-level phase CAS does not itself prove compatibility or drain. - let nodes = NodeDirectory::new( - layout.clone(), - configuration.fleet, - configuration.image, - registry.release_digest(), - ); - assert!(nodes.advertised_sessions(now()?, 4096).await?.is_empty()); - let mut cells = 0; - for shard in 0..=u8::MAX { - let mut scan = catalog.scan_shard(shard).await?; - while let Some(page) = scan.next_page().await? { - for proof in page.entries() { - assert_eq!(proof.entry(), &entry); - assert!(registry.supports_cell( - proof.entry().namespace(), - proof.entry().role(), - proof.entry().initial_code(), - proof.entry().initial_schema(), - )); - cells += 1; - } - } - } - assert_eq!(cells, 1); - let operation = RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()); - let prepared = releases - .prepare( - registry.release_bytes(), - registry.release_digest(), - old_ready.revision(), - &image, - operation, - ) - .await?; - let activating = releases - .start_activation(prepared.revision(), operation) - .await?; - releases - .complete_activation(activating.revision(), operation) - .await?; - - Ok(RetainedFixture { - _files: files, - configuration, - store, - layout, - catalog, - authority, - target, - entry, - }) -} - -#[tokio::test(flavor = "multi_thread")] -async fn server_reopens_a_retained_directory_catalog_after_explicit_fixture_activation() -> Result { - let fixture = retained_fixture().await?; - let address = fixture.configuration.listen; - let RetainedFixture { - _files, - configuration, - store, - catalog, - target, - entry, - .. - } = fixture; - - // Exercise real Canopy startup, not a direct low-level client workaround. - let server = CanopyServer::start(configuration, Arc::clone(&store)).await?; - let url = create_repository(address, "after-activation").await?; - let listing: serde_json::Value = reqwest::Client::new() - .get(format!("http://{address}/api/repositories")) - .bearer_auth("local-test-token") - .send() - .await? - .error_for_status()? - .json() - .await?; - assert_eq!( - listing["repositories"] - .as_array() - .ok_or("list missing")? - .len(), - 1 - ); - assert_eq!( - catalog - .lookup(target.cell_id()) - .await? - .ok_or("catalog missing")? - .entry(), - &entry - ); - assert!(url.ends_with("/canopy/after-activation.git")); - server.shutdown().await?; - Ok(()) -} - -async fn unsupported_control_stays_unchanged(code: Option, schema: Option) -> Result { - let fixture = retained_fixture().await?; - let observed = fixture - .authority - .load(fixture.target.cell_id()) - .await? - .ok_or("control missing")?; - let mut unsupported = observed.value().clone(); - if let Some(code) = code { - unsupported.code = code; - } - if let Some(schema) = schema { - unsupported.schema = schema; - } - unsupported.revision += 1; - unsupported.progress += 1; - let path = fixture - .layout - .control_path(fixture.target.cell_id().as_bytes()); - let (_, token) = fixture.layout.store().get_with_etag(&path).await?; - let encoded = unsupported.encode()?; - // Inject unsupported persisted metadata only in this owned in-memory fixture. - // Its catalog remains supported. A failed startup must reject the control - // before acquisition, not claim it and rely on later SQL/client rejection. - fixture - .layout - .store() - .update(&path, Bytes::from(encoded.clone()), token) - .await?; - assert!( - CanopyServer::start(fixture.configuration, Arc::clone(&fixture.store)) - .await - .is_err() - ); - let after = fixture - .authority - .load(fixture.target.cell_id()) - .await? - .ok_or("control disappeared")?; - assert_eq!( - after.value().encode()?, - encoded, - "unsupported persisted control was acquired or rewritten" - ); - Ok(()) -} - -#[tokio::test(flavor = "multi_thread")] -async fn unknown_control_code_is_rejected_before_cell_acquisition() -> Result { - unsupported_control_stays_unchanged(Some(Digest::from_bytes([1; 32])), None).await -} - -#[tokio::test(flavor = "multi_thread")] -async fn unknown_control_schema_is_rejected_before_cell_acquisition() -> Result { - unsupported_control_stays_unchanged(None, Some(2)).await -} diff --git a/crates/canopy-server/tests/multi_server/size.rs b/crates/canopy-server/tests/multi_server/size.rs index 0b01c4bd..657c8c3f 100644 --- a/crates/canopy-server/tests/multi_server/size.rs +++ b/crates/canopy-server/tests/multi_server/size.rs @@ -32,7 +32,7 @@ impl http_body::Body for Repeated { #[tokio::test(flavor = "multi_thread")] #[ignore = "large transfer qualification; run on a dedicated disk with at least 40 GiB free"] -async fn push_and_database_exceed_512_mib_and_lfs_exceeds_5_gib_after_restore() -> Result { +async fn native_pack_exceeds_512_mib_and_git_lfs_exceed_5_gib_after_restore() -> Result { let _ = tracing_subscriber::fmt() .with_max_level(tracing::Level::WARN) .try_init(); @@ -62,8 +62,8 @@ async fn push_and_database_exceed_512_mib_and_lfs_exceeds_5_gib_after_restore() ] { run_git(Some(&source), &["config", name, value]).await?; } - // Each incompressible blob stays in SQLite. The combined pack and database - // both exceed their former caps, so externalization cannot conceal a quota. + // Incompressible bodies make the native pack exceed the old transfer cap. + // SQLite must retain only metadata, independently of the logical Git bytes. for index in 0_u64..900 { let mut bytes = vec![0; 640 * 1024]; blake3::Hasher::new() @@ -81,7 +81,11 @@ async fn push_and_database_exceed_512_mib_and_lfs_exceeds_5_gib_after_restore() .set_len((PART * parts) as u64) .await?; run_git(Some(&source), &["add", "."]).await?; - run_git(Some(&source), &["commit", "-m", "Large SQLite repository"]).await?; + run_git( + Some(&source), + &["commit", "-m", "Large native-pack repository"], + ) + .await?; run_git(Some(&source), &["repack", "-ad"]).await?; let mut packs = tokio::fs::read_dir(source.join(".git/objects/pack")).await?; let mut pack_bytes = 0; @@ -99,7 +103,7 @@ async fn push_and_database_exceed_512_mib_and_lfs_exceeds_5_gib_after_restore() run_git(Some(&source), &["-c", AUTH, "push", &url, "main"]).await?; let expected = run_git(Some(&source), &["rev-parse", "HEAD"]).await?; tokio::fs::remove_dir_all(&source).await?; - let database_root = workspace.path().join("first/canopy-pack-v1"); + let database_root = workspace.path().join("first/canopy-pack-v2"); let database_bytes = tokio::task::spawn_blocking( move || -> std::result::Result> { for entry in std::fs::read_dir(database_root)? { @@ -121,7 +125,10 @@ async fn push_and_database_exceed_512_mib_and_lfs_exceeds_5_gib_after_restore() ) .await? .map_err(|error| error as Box)?; - assert!(database_bytes > 512 * 1024 * 1024); + assert!( + database_bytes < 16 * 1024 * 1024, + "Git bodies must not expand into SQLite" + ); eprintln!("large push published; SQLite database: {database_bytes} bytes"); // Independent SHA-256 reference (Python hashlib): 641 blocks of 8 MiB zeros. diff --git a/crates/canopy-server/tests/multi_server/ssh/fetch.rs b/crates/canopy-server/tests/multi_server/ssh/fetch.rs index f827cc74..8d7a11a3 100644 --- a/crates/canopy-server/tests/multi_server/ssh/fetch.rs +++ b/crates/canopy-server/tests/multi_server/ssh/fetch.rs @@ -138,7 +138,7 @@ async fn cold_ssh_blobless_clones_hydrate_only_explicit_blob_wants() -> Result { } #[tokio::test(flavor = "multi_thread")] -async fn cold_full_fetches_only_hydrate_blobs_reachable_from_requested_tips() -> Result { +async fn cold_full_fetches_reuse_packs_without_expanding_blobs_into_loose_files() -> Result { let workspace = tempfile::TempDir::new()?; let store: Arc = Arc::new(InMemory::new()); let host = ssh_key::PrivateKey::new( @@ -228,7 +228,28 @@ async fn cold_full_fetches_only_hydrate_blobs_reachable_from_requested_tips() -> vec![0x61; 2 * 1024 * 1024] ); git(Some(&clone), &ssh, &["fsck", "--strict", "--full"]).await?; - assert!(cache_contains(&cold, &blobs[0])?); + let objects = String::from_utf8( + git( + Some(&clone), + &ssh, + &[ + "cat-file", + "--batch-all-objects", + "--batch-check=%(objectname)", + ], + ) + .await?, + )?; + assert!(objects.contains(&blobs[0])); + assert!( + !objects.contains(&blobs[1]), + "{name}: unrelated branch sent to client" + ); + assert!( + !objects.contains(&blobs[2]), + "{name}: deleted branch sent to client" + ); + assert!(!cache_contains(&cold, &blobs[0])?); assert!( !cache_contains(&cold, &blobs[1])?, "{name}: unrelated branch blob hydrated" @@ -268,7 +289,7 @@ async fn cold_full_fetches_only_hydrate_blobs_reachable_from_requested_tips() -> git(Some(&clone), &ssh, &["show", "origin/other:asset"]).await?, vec![0x62; 2 * 1024 * 1024] ); - assert!(cache_contains(&cold, &blobs[1])?); + assert!(!cache_contains(&cold, &blobs[1])?); assert!(!cache_contains(&cold, &blobs[2])?); git(Some(&clone), &ssh, &["fsck", "--strict", "--full"]).await?; restored.shutdown().await?; @@ -311,7 +332,7 @@ async fn cold_full_fetches_only_hydrate_blobs_reachable_from_requested_tips() -> git(Some(&existing), &ssh, &["show", "origin/other:asset"]).await?, vec![0x62; 2 * 1024 * 1024] ); - assert!(cache_contains(&cold, &blobs[1])?); + assert!(!cache_contains(&cold, &blobs[1])?); assert!( !cache_contains(&cold, &blobs[0])?, "{name}: negotiation hydrated an unrequested blob" diff --git a/crates/canopy-server/tests/multi_server/workspace.rs b/crates/canopy-server/tests/multi_server/workspace.rs index 49e35db7..47a284e3 100644 --- a/crates/canopy-server/tests/multi_server/workspace.rs +++ b/crates/canopy-server/tests/multi_server/workspace.rs @@ -7,14 +7,14 @@ async fn old_deployment_format_is_rejected_before_workspace_or_identity_writes() br#"{"kind":"service"}"#.as_slice(), br#"{"purpose":{"kind":"service"}}"#, br#"{"format":"future-format","purpose":{"kind":"service"}}"#, - br#"{"format":"canopy-pack-v1","purpose":{"kind":"backup","source":"source","pin":"11111111-1111-4111-8111-111111111111","complete":true}}"#, - br#"{"format":"canopy-pack-v1","purpose":{"kind":"restore","source":"source","pin":"11111111-1111-4111-8111-111111111111","complete":false}}"#, + br#"{"format":"canopy-pack-v2","purpose":{"kind":"backup","source":"source","pin":"11111111-1111-4111-8111-111111111111","complete":true}}"#, + br#"{"format":"canopy-pack-v2","purpose":{"kind":"restore","source":"source","pin":"11111111-1111-4111-8111-111111111111","complete":false}}"#, ] { let files = tempfile::TempDir::new()?; let data = files.path().join("node"); - let packed = data.join("canopy-pack-v1"); + let packed = data.join("canopy-pack-v2"); std::fs::create_dir_all(&packed)?; - std::fs::write(packed.join(".canopy-runtime"), b"canopy-pack-v1")?; + std::fs::write(packed.join(".canopy-runtime"), b"canopy-pack-v2")?; std::fs::write(packed.join("important"), b"retain existing cache")?; let configuration = config(available_address().await?, data.clone()); let store: Arc = Arc::new(InMemory::new()); @@ -74,7 +74,7 @@ async fn active_node_blocks_workspace_reuse_and_shutdown_allows_durable_restore( matches!(occupied, Err(canopy_server::server::ServerError::Io(error)) if error.kind() == std::io::ErrorKind::WouldBlock) ); server.shutdown().await?; - let sentinel = data.join("canopy-pack-v1/abandoned"); + let sentinel = data.join("canopy-pack-v2/abandoned"); std::fs::write(&sentinel, b"reclaim before restoring")?; let address = available_address().await?; let server = CanopyServer::start(config(address, data), store).await?; diff --git a/crates/canopy-server/tests/repository_cell/batches.rs b/crates/canopy-server/tests/repository_cell/batches.rs deleted file mode 100644 index bc7be274..00000000 --- a/crates/canopy-server/tests/repository_cell/batches.rs +++ /dev/null @@ -1,217 +0,0 @@ -use std::time::{SystemTime, UNIX_EPOCH}; - -use canopy_server::{ - INLINE_OBJECT_LIMIT, ObjectBatch, ObjectKind, ObjectStorage, RepositoryCell, RepositoryModule, - StoredObject, object_id, -}; -use cellule_runtime::{ - InvocationError, MutationIdentity, SqlCell, identity::RequestId, primitives::sql::SqlBatch, - primitives::sql::SqlStatement, primitives::sql::SqlValue, -}; - -type Result = std::result::Result>; - -fn identity() -> Result { - let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; - Ok(MutationIdentity { - request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - issued_at_ms: now, - expires_at_ms: now + 60_000, - }) -} - -fn inline(body: Vec) -> StoredObject { - StoredObject { - oid: object_id(canopy_server::ObjectFormat::Sha1, ObjectKind::Blob, &body), - kind: ObjectKind::Blob, - storage: ObjectStorage::Inline(body), - } -} - -fn batch(objects: impl IntoIterator) -> ObjectBatch { - let mut batch = ObjectBatch::default(); - for object in objects { - assert!(batch.try_push(object).is_ok()); - } - batch -} - -pub async fn exercise(repository: &RepositoryCell, sql: &SqlCell) -> Result { - let foreign = b"foreign hash format"; - let foreign_id = object_id( - canopy_server::ObjectFormat::Sha256, - ObjectKind::Blob, - foreign, - ); - assert!(matches!( - repository - .put_objects( - identity()?, - batch([StoredObject { - oid: foreign_id, - kind: ObjectKind::Blob, - storage: ObjectStorage::Inline(foreign.to_vec()), - }]), - ) - .await, - Err(InvocationError::Rejected(_)) - )); - assert!(repository.object(foreign_id, None).await?.output.is_none()); - - // Every inserted object becomes visible at the same publication receipt. - let objects: Vec<_> = (0..128) - .map(|n| inline(format!("batch-{n}").into_bytes())) - .collect(); - let ids: Vec<_> = objects.iter().map(|object| object.oid).collect(); - let request_id = identity()?; - let committed = repository.put_objects(request_id, batch(objects)).await?; - for (n, oid) in ids.iter().enumerate() { - assert_eq!( - repository - .object(*oid, Some(committed.receipt)) - .await? - .output, - Some((ObjectKind::Blob, format!("batch-{n}").into_bytes())) - ); - } - assert_eq!( - repository.existing_objects(&ids).await?.output, - ids.iter().copied().collect() - ); - let replay = repository - .put_objects( - request_id, - batch((0..128).map(|n| inline(format!("batch-{n}").into_bytes()))), - ) - .await?; - assert_eq!(replay.receipt, committed.receipt); - - // Two bodies exactly fill the shared payload budget without exceeding the wire limit. - let first = vec![17; INLINE_OBJECT_LIMIT / 2]; - let second = vec![18; INLINE_OBJECT_LIMIT / 2]; - let first_id = object_id(canopy_server::ObjectFormat::Sha1, ObjectKind::Blob, &first); - let second_id = object_id(canopy_server::ObjectFormat::Sha1, ObjectKind::Blob, &second); - let stored = repository - .put_objects( - identity()?, - batch([inline(first.clone()), inline(second.clone())]), - ) - .await?; - assert_eq!( - repository - .object(first_id, Some(stored.receipt)) - .await? - .output, - Some((ObjectKind::Blob, first)) - ); - assert_eq!( - repository - .object(second_id, Some(stored.receipt)) - .await? - .output, - Some((ObjectKind::Blob, second)) - ); - - // A later invalid identity rolls back earlier valid records in the same command. - let fresh = inline(b"must roll back with bad OID".to_vec()); - let fresh_id = fresh.oid; - let mut wrong = inline(b"wrong object identity".to_vec()); - wrong.oid = canopy_server::ObjectId::Sha1([13; 20]); - assert!(matches!( - repository - .put_objects(identity()?, batch([fresh, wrong])) - .await, - Err(InvocationError::Rejected(_)) - )); - assert!( - repository - .existing_objects(&[fresh_id]) - .await? - .output - .is_empty() - ); - - // Existing corrupt rows cannot be accepted just because their OID is present. - sql.batch( - identity()?, - SqlBatch { - statements: vec![SqlStatement { - sql: "UPDATE objects SET digest = ?1 WHERE oid = ?2".into(), - parameters: vec![ - SqlValue::Blob(vec![91; 32]), - SqlValue::Blob(ids[0].to_vec()), - ], - }], - }, - ) - .await?; - let fresh = inline(b"must roll back with corrupt existing object".to_vec()); - let fresh_id = fresh.oid; - assert!(matches!( - repository - .put_objects(identity()?, batch([fresh, inline(b"batch-0".to_vec())])) - .await, - Err(InvocationError::Rejected(_)) - )); - assert!( - repository - .existing_objects(&[fresh_id]) - .await? - .output - .is_empty() - ); - - // External bytes are uploaded by the caller; their identity record is immutable here. - let external = |size| StoredObject { - oid: canopy_server::ObjectId::Sha1([27; 20]), - kind: ObjectKind::Blob, - storage: ObjectStorage::External { - size, - blake3: [28; 32], - sha256: [29; 32], - }, - }; - repository - .put_objects(identity()?, batch([external(1_000_000)])) - .await?; - let fresh = inline(b"must roll back with external identity conflict".to_vec()); - let fresh_id = fresh.oid; - assert!(matches!( - repository - .put_objects(identity()?, batch([fresh, external(1_000_001)])) - .await, - Err(InvocationError::Rejected(_)) - )); - assert!( - repository - .existing_objects(&[fresh_id]) - .await? - .output - .is_empty() - ); - for object in [ - StoredObject { - oid: canopy_server::ObjectId::Sha1([31; 20]), - kind: ObjectKind::Tree, - storage: ObjectStorage::External { - size: 1, - blake3: [32; 32], - sha256: [33; 32], - }, - }, - external(u64::MAX), - ] { - assert!(matches!( - repository.put_objects(identity()?, batch([object])).await, - Err(InvocationError::Rejected(_)) - )); - } - assert!(repository.existing_objects(&[]).await.is_err()); - assert!( - repository - .existing_objects(&vec![canopy_server::ObjectId::Sha1([0; 20]); 129]) - .await - .is_err() - ); - Ok(()) -} diff --git a/crates/canopy-server/tests/repository_cell/branch_rules.rs b/crates/canopy-server/tests/repository_cell/branch_rules.rs deleted file mode 100644 index f0cdf167..00000000 --- a/crates/canopy-server/tests/repository_cell/branch_rules.rs +++ /dev/null @@ -1,351 +0,0 @@ -use super::*; -use canopy_server::{ - FinalizePush, - branch_rules::BranchRuleEdit, - checks::{CheckContextEdit, CheckEdit, CheckState, NewCheck}, -}; -use cellule_runtime::{ - SqlCell, primitives::sql::SqlBatch, primitives::sql::SqlStatement, primitives::sql::SqlValue, -}; - -type Result = std::result::Result>; -type Oid = canopy_server::ObjectId; -pub(super) fn identity() -> Result { - let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; - Ok(MutationIdentity { - request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - issued_at_ms: now, - expires_at_ms: now + 60_000, - }) -} -fn rule(version: i64, checks: &[&str]) -> BranchRuleEdit { - BranchRuleEdit { - reference: "refs/heads/protected".into(), - expected_version: version, - enabled: true, - deny_deletions: true, - fast_forward_only: true, - require_pull_request: false, - required_approvals: 0, - required_checks: checks.iter().map(|s| (*s).into()).collect(), - } -} -pub(super) async fn plan(repo: &RepositoryCell, name: &str, oid: Option) -> Result { - Ok(PushPlan { - actor: "canopy".into(), - updates: vec![RefUpdate { - name: name.into(), - expected: repo.ref_state(name, None).await?.output, - new_oid: oid, - }], - }) -} -pub(super) async fn commit( - repo: &RepositoryCell, - tree: Oid, - parents: &[Oid], - message: &str, -) -> Result { - let parents: String = parents - .iter() - .map(|oid| format!("parent {}\n", hex::encode(oid))) - .collect(); - let body = format!( - "tree {}\n{parents}author Test 0 +0000\ncommitter Test 0 +0000\n\n{message}\n", - hex::encode(tree) - ); - Ok( - objects::put(repo, identity()?, ObjectKind::Commit, body.as_bytes()) - .await? - .output, - ) -} -async fn certificate( - sql: &SqlCell, - ancestor: Oid, - descendant: Oid, -) -> Result { - let result = sql - .query( - None, - SqlBatch { - statements: vec![SqlStatement { - sql: "SELECT 1 FROM commit_ancestry WHERE ancestor = ?1 AND descendant = ?2" - .into(), - parameters: vec![ - SqlValue::Blob(ancestor.to_vec()), - SqlValue::Blob(descendant.to_vec()), - ], - }], - }, - ) - .await?; - Ok(!result.output[0].rows.is_empty()) -} - -pub async fn verify( - repo: &RepositoryCell, - sql: &SqlCell, - app: &ApplicationHandle, - target: &CellTarget, -) -> Result { - let name = "refs/heads/protected"; - let tree = objects::put(repo, identity()?, ObjectKind::Tree, b"") - .await? - .output; - let root = commit(repo, tree, &[], "Root").await?; - let child = commit(repo, tree, &[root], "Child").await?; - let unrelated = commit(repo, tree, &[], "Unrelated").await?; - let mut tail = child; - for number in 0..130 { - tail = commit(repo, tree, &[tail], &format!("Chain {number}")).await?; - } - let merge = commit(repo, tree, &[unrelated, tail], "Merge").await?; - repo.finalize_push( - identity()?, - plan(repo, "refs/heads/branch-fixture", Some(merge)).await?, - ) - .await?; - repo.finalize_push(identity()?, plan(repo, name, Some(root)).await?) - .await?; - assert!(matches!( - repo.set_branch_rule(identity()?, "outsider", rule(0, &[])) - .await, - Err(InvocationError::Rejected(rejected)) if !rejected.output - )); - let policy_id = identity()?; - let first = repo - .set_branch_rule(policy_id, "canopy", rule(0, &[])) - .await?; - assert_eq!( - repo.set_branch_rule(policy_id, "canopy", rule(0, &[])) - .await? - .receipt, - first.receipt - ); - assert!(matches!( - repo.set_branch_rule(identity()?, "canopy", rule(0, &[])) - .await, - Err(InvocationError::Rejected(rejected)) if !rejected.output - )); - let advance = plan(repo, name, Some(merge)).await?; - let generation = repo.refs_page("", None).await?.output.generation; - // Even valid Git ancestry needs a server-verified certificate at publication. - assert!(matches!( - app.command::(target, identity()?, advance.clone()) - .await, - Err(InvocationError::Rejected(rejected)) if !rejected.output - )); - // A malicious certificate cannot substitute a caller-asserted link, including - // a valid first step followed by a false one in the same transaction. - assert!(matches!( - app.command::( - target, - identity()?, - Proof { - ancestor: root, - steps: vec![(child, root), (unrelated, child)] - } - ) - .await, - Err(InvocationError::Rejected(rejected)) if !rejected.output - )); - assert!(!certificate(sql, root, child).await?); - assert!(!certificate(sql, root, unrelated).await?); - assert_eq!( - repo.refs_page("", None).await?.output.generation, - generation - ); - // More than one proof page, including a merge's alternate ancestry path. - let mutation = identity()?; - let committed = repo.finalize_push(mutation, advance.clone()).await?; - assert!(certificate(sql, root, merge).await?); - let delete = plan(repo, name, None).await?; - let force = plan(repo, name, Some(unrelated)).await?; - for rejected in [delete, force] { - assert!(matches!( - app.command::(target, identity()?, rejected) - .await, - Err(InvocationError::Rejected(rejected)) if !rejected.output - )); - } - repo.set_check_context( - identity()?, - "canopy", - "branch-ci", - CheckContextEdit { - expected_version: 0, - reporter: "canopy", - enabled: true, - }, - ) - .await?; - repo.set_branch_rule(identity()?, "canopy", rule(1, &["branch-ci"])) - .await?; - // Completed receipt replay returns the prior result without applying new refs. - assert_eq!( - app.command::(target, mutation, advance) - .await? - .receipt, - committed.receipt - ); - assert_eq!( - repo.ref_state(name, None).await?.output.unwrap().oid, - Some(merge) - ); - let candidate = commit(repo, tree, &[merge], "Candidate").await?; - repo.finalize_push( - identity()?, - plan(repo, "refs/heads/branch-fixture", Some(candidate)).await?, - ) - .await?; - app.command::( - target, - identity()?, - Proof { - ancestor: merge, - steps: vec![(candidate, merge)], - }, - ) - .await?; - let proposed = plan(repo, name, Some(candidate)).await?; - let old = uuid::Uuid::new_v4().into_bytes(); - repo.start_check( - identity()?, - "canopy", - NewCheck { - id: old, - oid: candidate, - context: "branch-ci", - context_version: 1, - }, - ) - .await?; - repo.update_check( - identity()?, - "canopy", - old, - CheckEdit { - expected_version: 1, - state: CheckState::Success, - summary: "passed", - }, - ) - .await?; - // Captured push intent remains the same while a new queued attempt arrives. - let newer = uuid::Uuid::new_v4().into_bytes(); - repo.start_check( - identity()?, - "canopy", - NewCheck { - id: newer, - oid: candidate, - context: "branch-ci", - context_version: 1, - }, - ) - .await?; - assert!(matches!( - app.command::(target, identity()?, proposed.clone()) - .await, - Err(InvocationError::Rejected(rejected)) if !rejected.output - )); - repo.update_check( - identity()?, - "canopy", - newer, - CheckEdit { - expected_version: 1, - state: CheckState::Success, - summary: "passed again", - }, - ) - .await?; - // Current context enablement is checked in the same transaction as refs. - repo.set_check_context( - identity()?, - "canopy", - "branch-ci", - CheckContextEdit { - expected_version: 1, - reporter: "canopy", - enabled: false, - }, - ) - .await?; - assert!(matches!( - app.command::(target, identity()?, proposed.clone()) - .await, - Err(InvocationError::Rejected(rejected)) if !rejected.output - )); - repo.set_branch_rule(identity()?, "canopy", rule(2, &[])) - .await?; - app.command::(target, identity()?, proposed) - .await?; - assert_eq!( - repo.ref_state(name, None).await?.output.unwrap().oid, - Some(candidate) - ); - // Rule paging uses names; disabled records retain versions and requirements. - for number in 0..33 { - let mut edit = rule(0, &["branch-ci"]); - edit.reference = format!("refs/heads/page-{number:02}"); - edit.enabled = false; - repo.set_branch_rule(identity()?, "canopy", edit).await?; - } - let first = repo.branch_rules("canopy", None).await?.output.unwrap(); - assert_eq!(first.len(), 32); - let after = &first.last().unwrap().reference; - let remaining = repo - .branch_rules("canopy", Some(after)) - .await? - .output - .unwrap(); - assert_eq!(remaining.len(), 2); - assert!(repo.branch_rules("outsider", None).await?.output.is_none()); - Ok(()) -} - -// Invoke the registered command with independently encoded input, as an untrusted -// client can; the local fixture handler must never be selected by the server. -struct Proof { - ancestor: Oid, - steps: Vec<(Oid, Oid)>, -} -impl cellule_runtime::codec::WireValue for Proof { - fn encode( - &self, - out: &mut cellule_runtime::codec::BoundedEncoder, - ) -> std::result::Result<(), cellule_runtime::codec::CodecError> { - out.write_bytes(&self.ancestor)?; - out.write_count(self.steps.len())?; - for (child, parent) in &self.steps { - out.write_bytes(child)?; - out.write_bytes(parent)?; - } - Ok(()) - } - fn decode( - _: &mut cellule_runtime::codec::BoundedDecoder<'_>, - ) -> std::result::Result { - Err(cellule_runtime::codec::CodecError::Invalid( - "encode-only test client", - )) - } -} -struct CertificateCommand; -impl cellule_runtime::Command for CertificateCommand { - const MODULE: &'static str = "repository"; - const ID: u32 = 7; - const CODEC_VERSION: u32 = 2; - type Input = Proof; - type Output = bool; - fn execute( - _: &mut cellule_runtime::registry::CommandContext<'_, '_>, - _: Proof, - ) -> cellule_runtime::Result> { - Err(cellule_runtime::Error::Command( - "client handler must not execute", - )) - } -} diff --git a/crates/canopy-server/tests/repository_cell/bulk_refs.rs b/crates/canopy-server/tests/repository_cell/bulk_refs.rs deleted file mode 100644 index b1d15592..00000000 --- a/crates/canopy-server/tests/repository_cell/bulk_refs.rs +++ /dev/null @@ -1,183 +0,0 @@ -use super::*; -use canopy_server::{ObjectBatch, ObjectStorage, StoredObject}; - -pub async fn verify(repository: &RepositoryCell) -> Result<(), Box> { - let mut updates = Vec::new(); - for start in (0..1001).step_by(128) { - let mut objects = ObjectBatch::default(); - for index in start..(start + 128).min(1001) { - let body = format!("bulk ref object {index}").into_bytes(); - let oid = object_id(canopy_server::ObjectFormat::Sha1, ObjectKind::Blob, &body); - objects - .try_push(StoredObject { - oid, - kind: ObjectKind::Blob, - storage: ObjectStorage::Inline(body), - }) - .map_err(|_| "fixture exceeds object batch")?; - updates.push(RefUpdate { - name: format!("refs/tags/bulk/{index:04}"), - expected: None, - new_oid: Some(oid), - }); - } - repository - .put_objects(super::graph::identity()?, objects) - .await?; - } - let generation = repository.default_branch(None).await?.output.generation; - repository - .finalize_push( - super::graph::identity()?, - PushPlan { - actor: "canopy".into(), - updates: updates.clone(), - }, - ) - .await?; - assert_eq!( - repository.default_branch(None).await?.output.generation, - generation + 1 - ); - - let oid = updates[0].new_oid; - let mut replacement: Vec<_> = updates - .into_iter() - .map(|update| RefUpdate { - name: update.name, - expected: Some(RefExpectation { - oid: update.new_oid, - version: 1, - }), - new_oid: None, - }) - .collect(); - let last = replacement.pop().ok_or("missing final descendant")?; - replacement.push(RefUpdate { - name: "refs/tags/bulk".into(), - expected: None, - new_oid: oid, - }); - // The remaining descendant is beyond several SQL pages. Reject the whole - // replacement without deleting any earlier siblings or advancing generation. - assert!(matches!( - repository - .finalize_push( - super::graph::identity()?, - PushPlan { - actor: "canopy".into(), - updates: replacement.clone() - } - ) - .await, - Err(InvocationError::Rejected(_)) - )); - assert_eq!( - repository - .ref_state("refs/tags/bulk/0000", None) - .await? - .output - .map(|state| state.oid), - Some(oid) - ); - assert_eq!( - repository.default_branch(None).await?.output.generation, - generation + 1 - ); - - replacement.push(last.clone()); - repository - .finalize_push( - super::graph::identity()?, - PushPlan { - actor: "canopy".into(), - updates: replacement, - }, - ) - .await?; - assert_eq!( - repository - .ref_state("refs/tags/bulk", None) - .await? - .output - .map(|state| state.oid), - Some(oid) - ); - assert_eq!( - repository.ref_state(&last.name, None).await?.output, - Some(RefExpectation { - oid: None, - version: 2 - }) - ); - assert_eq!( - repository.default_branch(None).await?.output.generation, - generation + 2 - ); - let long_prefix = format!( - "refs/tags/long-page/{}", - vec!["r".repeat(200); 21].join("/") - ); - let long_names: Vec<_> = (0..270) - .map(|index| format!("{long_prefix}/{index:04}")) - .collect(); - assert!(long_names[0].len() * 256 > 1 << 20); - for chunk in long_names.chunks(16) { - repository - .finalize_push( - super::graph::identity()?, - PushPlan { - actor: "canopy".into(), - updates: chunk - .iter() - .map(|name| RefUpdate { - name: name.clone(), - expected: None, - new_oid: oid, - }) - .collect(), - }, - ) - .await?; - } - assert!(matches!( - repository - .finalize_push( - super::graph::identity()?, - PushPlan { - actor: "canopy".into(), - updates: vec![RefUpdate { - name: "refs/tags/long-page".into(), - expected: None, - new_oid: oid, - }], - }, - ) - .await, - Err(InvocationError::Rejected(_)) - )); - let mut after = String::new(); - let mut page_generation = None; - let mut observed = Vec::new(); - let mut byte_bounded = false; - loop { - let page = repository.refs_page(&after, page_generation).await?.output; - page_generation = Some(page.generation); - byte_bounded |= page.has_more && page.refs.len() < 256; - if let Some((name, _)) = page.refs.last() { - after.clone_from(name); - } - observed.extend( - page.refs - .into_iter() - .filter(|(name, _)| name.starts_with("refs/tags/long-page/")) - .map(|(name, _)| name), - ); - if !page.has_more { - break; - } - } - assert!(byte_bounded); - assert_eq!(observed, long_names); - Ok(()) -} diff --git a/crates/canopy-server/tests/repository_cell/checks.rs b/crates/canopy-server/tests/repository_cell/checks.rs deleted file mode 100644 index f43021a2..00000000 --- a/crates/canopy-server/tests/repository_cell/checks.rs +++ /dev/null @@ -1,192 +0,0 @@ -use super::*; -use canopy_server::checks::{CheckChange, CheckContextEdit, CheckEdit, CheckState, NewCheck}; - -type Result = std::result::Result>; -fn identity() -> Result { - let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; - Ok(MutationIdentity { - request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - issued_at_ms: now, - expires_at_ms: now + 60_000, - }) -} - -pub async fn exercise(repository: &RepositoryCell, oid: canopy_server::ObjectId) -> Result { - let policy = || CheckContextEdit { - expected_version: 0, - reporter: "ci-agent", - enabled: true, - }; - assert_eq!( - repository - .set_check_context(identity()?, "outsider", "unit", policy()) - .await? - .output, - CheckChange::Forbidden - ); - assert_eq!( - repository - .set_check_context(identity()?, "canopy", "unit", policy()) - .await? - .output, - CheckChange::NotFound - ); - repository - .grant_member(identity()?, "canopy", "ci-agent", TokenScope::Read) - .await?; - let config_identity = identity()?; - let configured = repository - .set_check_context(config_identity, "canopy", "unit", policy()) - .await?; - assert_eq!(configured.output, CheckChange::Applied); - assert_eq!( - repository - .set_check_context(config_identity, "canopy", "unit", policy()) - .await? - .receipt, - configured.receipt - ); - let id = uuid::Uuid::new_v4().into_bytes(); - let new_check = || NewCheck { - id, - oid, - context: "unit", - context_version: 1, - }; - assert_eq!( - repository - .start_check(identity()?, "canopy", new_check()) - .await? - .output, - CheckChange::Forbidden - ); - let creation = identity()?; - let created = repository - .start_check(creation, "ci-agent", new_check()) - .await?; - assert_eq!(created.output, CheckChange::Applied); - assert_eq!( - repository - .start_check(creation, "ci-agent", new_check()) - .await? - .receipt, - created.receipt - ); - assert!( - repository - .check_contexts("outsider", None) - .await? - .output - .is_none() - ); - assert!(repository.check_run("outsider", id).await?.output.is_none()); - assert!( - repository - .commit_checks("outsider", oid, None) - .await? - .output - .is_none() - ); - let summary = "\n".repeat(4096); - let edit = || CheckEdit { - expected_version: 1, - state: CheckState::Success, - summary: &summary, - }; - assert_eq!( - repository - .update_check(identity()?, "canopy", id, edit()) - .await? - .output, - CheckChange::Forbidden - ); - repository - .revoke_member(identity()?, "canopy", "ci-agent") - .await?; - assert_eq!( - repository - .start_check(identity()?, "ci-agent", new_check()) - .await? - .output, - CheckChange::NotFound - ); - assert_eq!( - repository - .update_check(identity()?, "ci-agent", id, edit()) - .await? - .output, - CheckChange::NotFound - ); - repository - .grant_member(identity()?, "canopy", "ci-agent", TokenScope::Read) - .await?; - let completion = identity()?; - let completed = repository - .update_check(completion, "ci-agent", id, edit()) - .await?; - assert_eq!(completed.output, CheckChange::Applied); - assert_eq!( - repository - .update_check(completion, "ci-agent", id, edit()) - .await? - .receipt, - completed.receipt - ); - assert_eq!( - repository - .update_check( - identity()?, - "ci-agent", - id, - CheckEdit { - expected_version: 2, - state: CheckState::Failure, - summary: "Must not overwrite" - } - ) - .await? - .output, - CheckChange::Conflict - ); - assert_eq!( - repository - .check_run("canopy", id) - .await? - .output - .ok_or("missing check")? - .summary, - summary - ); - repository - .set_check_context( - identity()?, - "canopy", - "unit", - CheckContextEdit { - expected_version: 1, - reporter: "ci-agent", - enabled: false, - }, - ) - .await?; - assert_eq!( - repository - .commit_checks("canopy", oid, None) - .await? - .output - .ok_or("missing checks")? - .len(), - 0 - ); - assert_eq!( - repository - .start_check(identity()?, "ci-agent", new_check()) - .await? - .output, - CheckChange::Conflict - ); - repository - .revoke_member(identity()?, "canopy", "ci-agent") - .await?; - Ok(()) -} diff --git a/crates/canopy-server/tests/repository_cell/chunks.rs b/crates/canopy-server/tests/repository_cell/chunks.rs deleted file mode 100644 index 18d17b10..00000000 --- a/crates/canopy-server/tests/repository_cell/chunks.rs +++ /dev/null @@ -1,187 +0,0 @@ -use super::*; -use canopy_server::{INLINE_OBJECT_LIMIT, ObjectBatch, ObjectStorage, StoredObject}; -use cellule_runtime::{ - SqlCell, primitives::sql::SqlBatch, primitives::sql::SqlStatement, primitives::sql::SqlValue, -}; - -type Result = std::result::Result>; - -fn identity() -> Result { - let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; - Ok(MutationIdentity { - request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - issued_at_ms: now, - expires_at_ms: now + 60_000, - }) -} - -fn batch(objects: impl IntoIterator) -> ObjectBatch { - let mut batch = ObjectBatch::default(); - for object in objects { - assert!(batch.try_push(object).is_ok()); - } - batch -} - -pub async fn exercise(repository: &RepositoryCell, sql: &SqlCell) -> Result { - let body = vec![b't'; INLINE_OBJECT_LIMIT + 1]; - let staged_id = identity()?; - let object = repository - .stage_object(staged_id, ObjectKind::Tag, &body) - .await?; - let oid = object.oid; - assert!(repository.object(oid, None).await?.output.is_none()); - assert!(repository.existing_objects(&[oid]).await?.output.is_empty()); - let committed = repository.put_objects(identity()?, batch([object])).await?; - assert_eq!( - repository - .object(oid, Some(committed.receipt)) - .await? - .output, - Some((ObjectKind::Tag, body.clone())) - ); - // Stage retries replay their chunk commands; a second upload of identical bytes converges. - for request in [staged_id, identity()?] { - let duplicate = repository - .stage_object(request, ObjectKind::Tag, &body) - .await?; - repository - .put_objects(identity()?, batch([duplicate])) - .await?; - } - let layout = sql - .query( - None, - SqlBatch { - statements: vec![SqlStatement { - sql: "SELECT storage, body, external_sha256, size FROM objects WHERE oid = ?1" - .into(), - parameters: vec![SqlValue::Blob(oid.to_vec())], - }], - }, - ) - .await?; - assert_eq!( - layout.output[0].rows[0], - vec![ - SqlValue::Text("chunked".into()), - SqlValue::Null, - SqlValue::Null, - SqlValue::Integer(body.len() as i64) - ] - ); - - for case in 0..7 { - let bytes = vec![case + 10; INLINE_OBJECT_LIMIT + 32]; - let mut object = repository - .stage_object(identity()?, ObjectKind::Commit, &bytes) - .await?; - let oid = object.oid; - let ObjectStorage::Chunked { - upload, - size, - blake3, - } = &mut object.storage - else { - return Err("chunk reference missing".into()); - }; - let upload = *upload; - let statement = match case { - 0 => Some(SqlStatement { sql: "DELETE FROM object_chunks WHERE upload_id = ?1 AND part = 1".into(), parameters: vec![SqlValue::Blob(upload.to_vec())] }), - 1 => Some(SqlStatement { sql: "UPDATE object_chunks SET body = zeroblob(length(body)) WHERE upload_id = ?1 AND part = 0".into(), parameters: vec![SqlValue::Blob(upload.to_vec())] }), - 2 => Some(SqlStatement { sql: "INSERT INTO object_chunks (upload_id, part, body) VALUES (?1, 2, ?2)".into(), parameters: vec![SqlValue::Blob(upload.to_vec()), SqlValue::Blob(vec![1])] }), - 3 => { blake3[0] ^= 1; None }, - 4 => { object.oid = match object.oid { canopy_server::ObjectId::Sha1(mut bytes) => { bytes[0] ^= 1; canopy_server::ObjectId::Sha1(bytes) }, canopy_server::ObjectId::Sha256(mut bytes) => { bytes[0] ^= 1; canopy_server::ObjectId::Sha256(bytes) } }; None }, - 5 => { object.kind = ObjectKind::Blob; None }, - _ => { *size -= 1; None }, - }; - if let Some(statement) = statement { - sql.batch( - identity()?, - SqlBatch { - statements: vec![statement], - }, - ) - .await?; - } - let first = format!("rolled back with invalid chunks {case}").into_bytes(); - let first_id = object_id(canopy_server::ObjectFormat::Sha1, ObjectKind::Blob, &first); - let first = StoredObject { - oid: first_id, - kind: ObjectKind::Blob, - storage: ObjectStorage::Inline(first), - }; - assert!( - matches!( - repository - .put_objects(identity()?, batch([first, object])) - .await, - Err(InvocationError::Rejected(_)) - ), - "case {case}" - ); - assert!( - repository - .existing_objects(&[first_id, oid]) - .await? - .output - .is_empty(), - "case {case}" - ); - } - // Chunk storage must preserve typed graph validation, including missing edges. - let mut tree = Vec::new(); - for index in 0..24_000 { - tree.extend_from_slice(format!("100644 file-{index:05}\0").as_bytes()); - tree.extend_from_slice(&[91; 20]); - } - let mut commit = format!("tree {}\nauthor Canopy 0 +0000\ncommitter Canopy 0 +0000\n\n", hex::encode([92; 20])).into_bytes(); - commit.extend(vec![b'c'; INLINE_OBJECT_LIMIT]); - let mut tag = format!( - "object {}\ntype commit\ntag missing\n\n", - hex::encode([93; 20]) - ) - .into_bytes(); - tag.extend(vec![b't'; INLINE_OBJECT_LIMIT]); - for (kind, bytes) in [ - (ObjectKind::Tree, tree), - (ObjectKind::Commit, commit), - (ObjectKind::Tag, tag), - ] { - let object = repository.stage_object(identity()?, kind, &bytes).await?; - let name = format!("refs/tags/chunked-{}", kind.git_name()); - let plan = PushPlan { - actor: "canopy".into(), - updates: vec![RefUpdate { - name: name.clone(), - expected: None, - new_oid: Some(object.oid), - }], - }; - repository.put_objects(identity()?, batch([object])).await?; - assert!(matches!( - repository.finalize_push(identity()?, plan).await, - Err(InvocationError::Rejected(_)) - )); - assert!(repository.ref_state(&name, None).await?.output.is_none()); - } - // Corruption after publication must also be detected during recovery reads. - sql.batch(identity()?, SqlBatch { statements: vec![SqlStatement { - sql: "DELETE FROM object_chunks WHERE upload_id = (SELECT chunk_id FROM objects WHERE oid = ?1) AND part = 1".into(), - parameters: vec![SqlValue::Blob(oid.to_vec())], - }] }).await?; - assert!(repository.object(oid, None).await.is_err()); - assert!( - repository - .stage_object(identity()?, ObjectKind::Blob, &body) - .await - .is_err() - ); - assert!( - repository - .stage_object(identity()?, ObjectKind::Tree, b"small") - .await - .is_err() - ); - Ok(()) -} diff --git a/crates/canopy-server/tests/repository_cell/default_branch.rs b/crates/canopy-server/tests/repository_cell/default_branch.rs deleted file mode 100644 index 07555396..00000000 --- a/crates/canopy-server/tests/repository_cell/default_branch.rs +++ /dev/null @@ -1,173 +0,0 @@ -use super::*; -use canopy_server::RefReadError; - -fn identity() -> Result> { - let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; - Ok(MutationIdentity { - request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - issued_at_ms: now, - expires_at_ms: now + 60_000, - }) -} - -pub async fn empty(repository: &RepositoryCell) -> Result<(), Box> { - let initial = repository.default_branch(None).await?.output; - assert_eq!(initial.reference, "refs/heads/main"); - let changed = repository - .set_default_branch( - identity()?, - "canopy", - initial.generation, - "refs/heads/trunk", - ) - .await?; - assert!(changed.output); - let head = repository - .default_branch(Some(changed.receipt)) - .await? - .output; - assert_eq!(head.reference, "refs/heads/trunk"); - assert_eq!(head.generation, initial.generation + 1); - let page = repository.refs_page("", None).await?.output; - assert!(page.refs.is_empty()); - assert_eq!(page.default_branch, head.reference); - assert!(matches!( - repository.refs_page("zzzz", Some(initial.generation)).await, - Err(RefReadError::Changed) - )); - assert!( - repository - .set_default_branch(identity()?, "canopy", head.generation, "refs/heads/main") - .await? - .output - ); - Ok(()) -} - -pub async fn verify(repository: &RepositoryCell) -> Result<(), Box> { - let initial = repository.default_branch(None).await?.output; - for name in [ - "HEAD", - "refs/tags/release", - "refs/heads/", - "refs/heads/a\nref: refs/heads/b", - "refs/heads/a.lock", - "refs/heads/a..b", - ] { - assert!( - repository - .set_default_branch(identity()?, "canopy", initial.generation, name) - .await - .is_err(), - "{name}" - ); - } - for generation in [-1, i64::MAX] { - assert!( - repository - .set_default_branch(identity()?, "canopy", generation, "refs/heads/main") - .await - .is_err() - ); - } - assert!( - !repository - .set_default_branch( - identity()?, - "canopy", - initial.generation, - "refs/heads/absent" - ) - .await? - .output - ); - repository - .grant_member(identity()?, "canopy", "writer", TokenScope::Write) - .await?; - for actor in ["writer", "outsider"] { - assert!( - !repository - .set_default_branch(identity()?, actor, initial.generation, "refs/heads/main") - .await? - .output - ); - } - assert_eq!(repository.default_branch(None).await?.output, initial); - let oid = repository - .ref_state("refs/heads/main", None) - .await? - .output - .ok_or("missing main")? - .oid; - repository - .finalize_push( - identity()?, - PushPlan { - actor: "canopy".into(), - updates: vec![RefUpdate { - name: "refs/heads/trunk".into(), - expected: None, - new_oid: oid, - }], - }, - ) - .await?; - // A push also fences an administrative update derived from an older ref view. - assert!( - !repository - .set_default_branch( - identity()?, - "canopy", - initial.generation, - "refs/heads/trunk" - ) - .await? - .output - ); - let current = repository.default_branch(None).await?.output; - let replay_id = identity()?; - for _ in 0..2 { - assert!( - repository - .set_default_branch(replay_id, "canopy", current.generation, "refs/heads/trunk") - .await? - .output - ); - } - let changed = repository.default_branch(None).await?.output; - assert_eq!(changed.generation, current.generation + 1); - assert_eq!(changed.reference, "refs/heads/trunk"); - assert!(matches!( - repository.refs_page("zzzz", Some(current.generation)).await, - Err(RefReadError::Changed) - )); - assert!( - repository - .set_default_branch(identity()?, "canopy", changed.generation, "refs/heads/main") - .await? - .output - ); - // Returning HEAD to its former name must not make a stale request valid again. - assert!( - !repository - .set_default_branch( - identity()?, - "canopy", - current.generation, - "refs/heads/trunk" - ) - .await? - .output - ); - let generation = repository.default_branch(None).await?.output.generation; - let (left, right) = tokio::join!( - repository.set_default_branch(identity()?, "canopy", generation, "refs/heads/trunk"), - repository.set_default_branch(identity()?, "canopy", generation, "refs/heads/main"), - ); - assert_ne!(left?.output, right?.output); - assert_eq!( - repository.default_branch(None).await?.output.generation, - generation + 1 - ); - Ok(()) -} diff --git a/crates/canopy-server/tests/repository_cell/graph.rs b/crates/canopy-server/tests/repository_cell/graph.rs deleted file mode 100644 index abe58a58..00000000 --- a/crates/canopy-server/tests/repository_cell/graph.rs +++ /dev/null @@ -1,408 +0,0 @@ -use std::time::{SystemTime, UNIX_EPOCH}; - -use canopy_server::{ObjectKind, PushPlan, RefUpdate, RepositoryCell, RepositoryModule, object_id}; -use cellule_runtime::{ - InvocationError, MutationIdentity, SqlCell, identity::RequestId, primitives::sql::SqlBatch, - primitives::sql::SqlStatement, primitives::sql::SqlValue, -}; - -type Result = std::result::Result>; - -pub(super) fn identity() -> Result { - let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; - Ok(MutationIdentity { - request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - issued_at_ms: now, - expires_at_ms: now + 60_000, - }) -} - -fn plan(name: &str, oid: canopy_server::ObjectId) -> PushPlan { - PushPlan { - actor: "canopy".into(), - updates: vec![RefUpdate { - name: name.into(), - expected: None, - new_oid: Some(oid), - }], - } -} - -async fn put( - repository: &RepositoryCell, - kind: ObjectKind, - body: &[u8], -) -> Result { - Ok(super::objects::put(repository, identity()?, kind, body) - .await? - .output) -} - -fn commit(tree: canopy_server::ObjectId, parent: Option) -> Vec { - let parent = parent.map_or_else(String::new, |oid| format!("parent {}\n", hex::encode(oid))); - format!("tree {}\n{parent}author Canopy 0 +0000\ncommitter Canopy 0 +0000\n\nGraph test\n", hex::encode(tree)).into_bytes() -} - -fn entry(mode: &str, name: &[u8], oid: canopy_server::ObjectId) -> Vec { - let mut body = format!("{mode} ").into_bytes(); - body.extend_from_slice(name); - body.push(0); - body.extend_from_slice(&oid); - body -} - -async fn certificates(sql: &SqlCell) -> Result> { - let result = sql - .query( - None, - SqlBatch { - statements: vec![SqlStatement { - sql: "SELECT oid FROM object_closure ORDER BY oid".into(), - parameters: vec![], - }], - }, - ) - .await?; - Ok(result - .output - .into_iter() - .flat_map(|set| set.rows) - .flatten() - .collect()) -} - -pub async fn verify( - repository: &RepositoryCell, - sql: &SqlCell, - application: &cellule_app::ApplicationHandle, - target: &cellule_runtime::CellTarget, -) -> Result<()> { - let blob_body = b"graph closure leaf"; - let blob = object_id( - canopy_server::ObjectFormat::Sha1, - ObjectKind::Blob, - blob_body, - ); - let tree_body = entry("100644", b"leaf", blob); - let tree = object_id( - canopy_server::ObjectFormat::Sha1, - ObjectKind::Tree, - &tree_body, - ); - let root = put(repository, ObjectKind::Commit, &commit(tree, None)).await?; - - let push = plan("refs/heads/graph", root); - assert!(matches!( - repository.finalize_push(identity()?, push.clone()).await, - Err(InvocationError::Rejected(_)) - )); - assert!( - repository - .ref_state("refs/heads/graph", None) - .await? - .output - .is_none() - ); - put(repository, ObjectKind::Tree, &tree_body).await?; - assert!(matches!( - repository.finalize_push(identity()?, push.clone()).await, - Err(InvocationError::Rejected(_)) - )); - assert!( - !certificates(sql) - .await? - .contains(&SqlValue::Blob(root.to_vec())) - ); - put(repository, ObjectKind::Blob, blob_body).await?; - for candidates in [ - vec![root, tree, blob], - vec![blob, root], - vec![blob, tree, canopy_server::ObjectId::Sha1([71; 20])], - ] { - assert!(matches!( - application - .command::(target, identity()?, CertificateInput(candidates)) - .await, - Err(InvocationError::Rejected(_)) - )); - assert!( - certificates(sql) - .await? - .contains(&SqlValue::Blob(blob.to_vec())) - ); - assert!( - !certificates(sql) - .await? - .contains(&SqlValue::Blob(tree.to_vec())) - ); - } - // A rejected batch must roll back edges as well as closure certificates; - // otherwise a staged tree could authorize a later raw-OID fetch. - let edges = sql - .query( - None, - SqlBatch { - statements: vec![SqlStatement { - sql: "SELECT child FROM object_edges WHERE parent = ?1".into(), - parameters: vec![SqlValue::Blob(tree.to_vec())], - }], - }, - ) - .await?; - assert!(edges.output[0].rows.is_empty()); - assert!( - application - .command::(target, identity()?, CertificateInput(vec![blob, tree])) - .await? - .output - ); - // Calling the low-level command cannot bypass certificate preparation. - assert!(matches!( - application - .command::(target, identity()?, push.clone()) - .await, - Err(InvocationError::Rejected(_)) - )); - repository.finalize_push(identity()?, push).await?; - let after = certificates(sql).await?; - for oid in [root, tree, blob] { - assert!(after.contains(&SqlValue::Blob(oid.to_vec()))); - } - repository - .finalize_push(identity()?, plan("refs/heads/graph-again", root)) - .await?; - assert_eq!(certificates(sql).await?, after); - - let wrong_tree = entry("40000", b"directory", blob); - let malformed_tree = b"100644 truncated\0short"; - let wrong_parent = commit(tree, Some(blob)); - let missing_parent = commit(tree, Some(canopy_server::ObjectId::Sha1([71; 20]))); - let wrong_tag = format!( - "object {}\ntype commit\ntag wrong-kind\n\n", - hex::encode(blob) - ); - let missing_tag = format!( - "object {}\ntype tag\ntag missing\n\n", - hex::encode([72; 20]) - ); - for (name, kind, body) in [ - ("wrong-tree", ObjectKind::Tree, wrong_tree.as_slice()), - ("short-tree", ObjectKind::Tree, malformed_tree.as_slice()), - ("wrong-parent", ObjectKind::Commit, wrong_parent.as_slice()), - ( - "missing-parent", - ObjectKind::Commit, - missing_parent.as_slice(), - ), - ("wrong-tag", ObjectKind::Tag, wrong_tag.as_bytes()), - ("missing-tag", ObjectKind::Tag, missing_tag.as_bytes()), - ( - "bad-commit", - ObjectKind::Commit, - b"tree not-an-object-id\n".as_slice(), - ), - ] { - let oid = put(repository, kind, body).await?; - let name = format!("refs/tags/{name}"); - assert!(matches!( - application - .command::(target, identity()?, CertificateInput(vec![oid])) - .await, - Err(InvocationError::Rejected(_)) - )); - assert!( - matches!( - repository - .finalize_push(identity()?, plan(&name, oid)) - .await, - Err(InvocationError::Rejected(_)) - ), - "{name}" - ); - assert!(repository.ref_state(&name, None).await?.output.is_none()); - } - assert!(matches!( - repository - .finalize_push(identity()?, plan("refs/heads/blob", blob)) - .await, - Err(InvocationError::Rejected(_)) - )); - assert_eq!(certificates(sql).await?, after); - - // A missing root must reject every ref even when another root is valid. - // Certificate preparation is independent of ref publication. - let independent = put(repository, ObjectKind::Blob, b"independent graph leaf").await?; - let mut mixed = plan( - "refs/tags/missing-root", - canopy_server::ObjectId::Sha1([73; 20]), - ); - mixed - .updates - .extend(plan("refs/tags/independent", independent).updates); - assert!(matches!( - repository.finalize_push(identity()?, mixed).await, - Err(InvocationError::Rejected(_)) - )); - assert!( - repository - .ref_state("refs/tags/independent", None) - .await? - .output - .is_none() - ); - - // Gitlinks point outside this repository. Symlinks and non-UTF8 names - // still point to local blobs; repeated edges can share a certified leaf. - let mut linked_tree = entry("120000", b"link", blob); - linked_tree.extend(entry( - "160000", - b"submodule", - canopy_server::ObjectId::Sha1([74; 20]), - )); - linked_tree.extend(entry("100755", b"\xff", blob)); - let linked_tree = put(repository, ObjectKind::Tree, &linked_tree).await?; - let linked_commit = put( - repository, - ObjectKind::Commit, - &commit(linked_tree, Some(root)), - ) - .await?; - let tag = format!( - "object {}\ntype commit\ntag v1\n\n", - hex::encode(linked_commit) - ); - let tag = put(repository, ObjectKind::Tag, tag.as_bytes()).await?; - let nested_tag = format!("object {}\ntype tag\ntag nested\n\n", hex::encode(tag)); - let nested_tag = put(repository, ObjectKind::Tag, nested_tag.as_bytes()).await?; - repository - .finalize_push(identity()?, plan("refs/tags/nested", nested_tag)) - .await?; - assert!(matches!( - repository - .finalize_push(identity()?, plan("refs/heads/tag", tag)) - .await, - Err(InvocationError::Rejected(_)) - )); - resumable(repository, sql).await?; - Ok(()) -} - -async fn resumable(repository: &RepositoryCell, sql: &SqlCell) -> Result<()> { - use canopy_server::{ObjectBatch, ObjectStorage, StoredObject}; - let mut leaves: Vec<_> = (0..270) - .map(|index| { - let body = format!("resumable leaf {index}").into_bytes(); - ( - object_id(canopy_server::ObjectFormat::Sha1, ObjectKind::Blob, &body), - body, - ) - }) - .collect(); - leaves.sort_by_key(|(oid, _)| *oid); - let mut tree = Vec::new(); - for (index, (oid, _)) in leaves.iter().enumerate() { - tree.extend(entry("100644", format!("leaf-{index:03}").as_bytes(), *oid)); - } - let (missing, body) = leaves.pop().ok_or("missing fixture leaf")?; - for group in leaves.chunks(128) { - let mut batch = ObjectBatch::default(); - for (oid, body) in group { - assert!( - batch - .try_push(StoredObject { - oid: *oid, - kind: ObjectKind::Blob, - storage: ObjectStorage::Inline(body.clone()) - }) - .is_ok() - ); - } - repository.put_objects(identity()?, batch).await?; - } - let root = put(repository, ObjectKind::Tree, &tree).await?; - let generation = repository.refs_page("", None).await?.output.generation; - let plan = plan("refs/tags/resumable-graph", root); - assert!(matches!( - repository.finalize_push(identity()?, plan.clone()).await, - Err(InvocationError::Rejected(_)) - )); - let persisted = certificates(sql).await?; - assert_eq!( - leaves - .iter() - .filter(|(oid, _)| persisted.contains(&SqlValue::Blob(oid.to_vec()))) - .count(), - leaves.len() - ); - assert!(!persisted.contains(&SqlValue::Blob(root.to_vec()))); - assert_eq!( - repository.refs_page("", None).await?.output.generation, - generation - ); - assert!( - repository - .ref_state("refs/tags/resumable-graph", None) - .await? - .output - .is_none() - ); - assert_eq!(put(repository, ObjectKind::Blob, &body).await?, missing); - repository.finalize_push(identity()?, plan).await?; - assert_eq!( - repository.refs_page("", None).await?.output.generation, - generation + 1 - ); - assert_eq!( - repository - .ref_state("refs/tags/resumable-graph", None) - .await? - .output - .and_then(|state| state.oid), - Some(root) - ); - let persisted = certificates(sql).await?; - assert!(persisted.contains(&SqlValue::Blob(root.to_vec()))); - for (oid, _) in leaves { - assert!(persisted.contains(&SqlValue::Blob(oid.to_vec()))); - } - Ok(()) -} - -// A separately encoded client invokes the registered server command. Its local -// handler cannot run, so these checks exercise the actual wire trust boundary. -struct CertificateInput(Vec); -impl cellule_runtime::codec::WireValue for CertificateInput { - fn encode( - &self, - encoder: &mut cellule_runtime::codec::BoundedEncoder, - ) -> std::result::Result<(), cellule_runtime::codec::CodecError> { - encoder.write_count(self.0.len())?; - for oid in &self.0 { - encoder.write_bytes(oid)?; - } - Ok(()) - } - fn decode( - _: &mut cellule_runtime::codec::BoundedDecoder<'_>, - ) -> std::result::Result { - Err(cellule_runtime::codec::CodecError::Invalid( - "client fixture is encode-only", - )) - } -} -struct CertificateCommand; -impl cellule_runtime::Command for CertificateCommand { - const MODULE: &'static str = "repository"; - const ID: u32 = 6; - const CODEC_VERSION: u32 = 3; - type Input = CertificateInput; - type Output = bool; - fn execute( - _: &mut cellule_runtime::registry::CommandContext<'_, '_>, - _: Self::Input, - ) -> cellule_runtime::Result> { - Err(cellule_runtime::Error::Command( - "client fixture handler must not execute", - )) - } -} diff --git a/crates/canopy-server/tests/repository_cell/issues.rs b/crates/canopy-server/tests/repository_cell/issues.rs deleted file mode 100644 index 4c4803ec..00000000 --- a/crates/canopy-server/tests/repository_cell/issues.rs +++ /dev/null @@ -1,202 +0,0 @@ -use super::*; -use canopy_server::issues::{ - CommentEdit, IssueChange, IssueEdit, IssueState, NewComment, NewIssue, -}; - -type Result = std::result::Result>; - -fn identity() -> Result { - let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; - Ok(MutationIdentity { - request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - issued_at_ms: now, - expires_at_ms: now + 60_000, - }) -} - -pub async fn exercise(repository: &RepositoryCell) -> Result { - let id = uuid::Uuid::new_v4().into_bytes(); - let title = "界".repeat(85); - let body = "\n".repeat(canopy_server::issues::ISSUE_BODY_LIMIT); - let new_issue = || NewIssue { - id, - title: &title, - body: &body, - }; - assert_eq!( - repository - .create_issue(identity()?, "outsider", new_issue()) - .await? - .output, - IssueChange::NotFound - ); - assert!( - repository - .issues("outsider", 0, None) - .await? - .output - .is_none() - ); - repository - .grant_member(identity()?, "canopy", "reporter", TokenScope::Read) - .await?; - let creation = identity()?; - let created = repository - .create_issue(creation, "reporter", new_issue()) - .await?; - assert_eq!(created.output, IssueChange::Applied(1)); - let replayed = repository - .create_issue(creation, "reporter", new_issue()) - .await?; - assert_eq!(created.receipt, replayed.receipt); - assert_eq!( - repository - .issue("reporter", 1) - .await? - .output - .ok_or("missing issue")? - .body, - body - ); - assert!(repository.issue("outsider", 1).await?.output.is_none()); - assert!( - repository - .issue_comments("outsider", 1, 0) - .await? - .output - .is_none() - ); - assert!( - repository - .issue_comments("canopy", 2, 0) - .await? - .output - .is_none() - ); - let new_comment = || NewComment { - id: uuid::Uuid::new_v4().into_bytes(), - body: "comment", - }; - assert_eq!( - repository - .create_issue_comment(identity()?, "reporter", 1, new_comment()) - .await? - .output, - IssueChange::Applied(1) - ); - let edit = || IssueEdit { - expected_version: 1, - title: "Closed", - body: "Finished", - state: IssueState::Closed, - }; - let comment_edit = || CommentEdit { - expected_version: 1, - body: "Edited", - }; - repository - .grant_member(identity()?, "canopy", "discussion-reader", TokenScope::Read) - .await?; - assert_eq!( - repository - .edit_issue(identity()?, "discussion-reader", 1, edit()) - .await? - .output, - IssueChange::Forbidden - ); - assert_eq!( - repository - .edit_issue_comment(identity()?, "discussion-reader", 1, 1, comment_edit()) - .await? - .output, - IssueChange::Forbidden - ); - repository - .grant_member( - identity()?, - "canopy", - "discussion-reader", - TokenScope::Write, - ) - .await?; - let update = identity()?; - let closed = repository - .edit_issue(update, "discussion-reader", 1, edit()) - .await?; - assert_eq!(closed.output, IssueChange::Applied(1)); - assert_eq!( - repository - .edit_issue(update, "discussion-reader", 1, edit()) - .await? - .receipt, - closed.receipt - ); - assert_eq!( - repository - .edit_issue(identity()?, "discussion-reader", 1, edit()) - .await? - .output, - IssueChange::Conflict - ); - assert_eq!( - repository - .edit_issue_comment(identity()?, "discussion-reader", 1, 1, comment_edit()) - .await? - .output, - IssueChange::Applied(1) - ); - repository - .revoke_member(identity()?, "canopy", "reporter") - .await?; - assert_eq!( - repository - .create_issue(identity()?, "reporter", new_issue()) - .await? - .output, - IssueChange::NotFound - ); - assert_eq!( - repository - .create_issue_comment(identity()?, "reporter", 1, new_comment()) - .await? - .output, - IssueChange::NotFound - ); - assert_eq!( - repository - .edit_issue(identity()?, "reporter", 1, edit()) - .await? - .output, - IssueChange::NotFound - ); - assert_eq!( - repository - .edit_issue_comment(identity()?, "reporter", 1, 1, comment_edit()) - .await? - .output, - IssueChange::NotFound - ); - repository - .revoke_member(identity()?, "canopy", "discussion-reader") - .await?; - assert_eq!( - repository - .issues("canopy", 0, Some(IssueState::Closed)) - .await? - .output - .ok_or("missing page")? - .len(), - 1 - ); - assert_eq!( - repository - .issue("canopy", 1) - .await? - .output - .ok_or("missing issue")? - .summary - .version, - 2 - ); - Ok(()) -} diff --git a/crates/canopy-server/tests/repository_cell/merge.rs b/crates/canopy-server/tests/repository_cell/merge.rs deleted file mode 100644 index 183d5c32..00000000 --- a/crates/canopy-server/tests/repository_cell/merge.rs +++ /dev/null @@ -1,190 +0,0 @@ -use super::branch_rules::{commit, identity, plan}; -use super::*; -use canopy_server::{ - FinalizePush, - branch_rules::BranchRuleEdit, - pulls::{ - NewPull, NewReview, PullChange, PullRevision, PullState, ReviewKind, - merge::{MergeOutcome, MergeRequest, MergeStrategy}, - }, -}; -use cellule_runtime::{ - SqlCell, primitives::sql::SqlBatch, primitives::sql::SqlStatement, primitives::sql::SqlValue, -}; -type Result = std::result::Result>; - -pub async fn verify( - repo: &RepositoryCell, - sql: &SqlCell, - app: &ApplicationHandle, - target: &CellTarget, -) -> Result { - let tree = objects::put(repo, identity()?, ObjectKind::Tree, b"") - .await? - .output; - let base = commit(repo, tree, &[], "Merge base").await?; - let source = commit(repo, tree, &[base], "Merge source").await?; - for name in ["refs/canopy", "refs/canopy/merge-candidates/forged"] { - let plan = plan(repo, name, Some(source)).await?; - assert!(matches!( - app.command::(target, identity()?, plan).await, - Err(InvocationError::Rejected(_)) - )); - assert!(repo.ref_state(name, None).await?.output.is_none()); - } - let base_ref = "refs/heads/merge-base"; - let source_ref = "refs/heads/merge-source"; - repo.finalize_push(identity()?, plan(repo, base_ref, Some(base)).await?) - .await?; - repo.finalize_push(identity()?, plan(repo, source_ref, Some(source)).await?) - .await?; - repo.grant_member(identity()?, "canopy", "merge-author", TokenScope::Read) - .await?; - let PullChange::Applied(number) = repo - .create_pull( - identity()?, - "merge-author", - NewPull { - id: uuid::Uuid::new_v4().into_bytes(), - title: "Merge", - body: "", - draft: false, - source_ref, - source_oid: &hex::encode(source), - base_ref, - base_oid: &hex::encode(base), - }, - ) - .await? - .output - else { - return Err("missing pull".into()); - }; - let edit = |version| BranchRuleEdit { - reference: base_ref.into(), - expected_version: version, - enabled: true, - deny_deletions: false, - fast_forward_only: false, - required_checks: vec![], - require_pull_request: true, - required_approvals: 1, - }; - repo.set_branch_rule(identity()?, "canopy", edit(0)).await?; - let revision = PullRevision { - pull_version: 1, - source_oid: hex::encode(source), - source_version: 1, - base_oid: hex::encode(base), - base_version: 1, - }; - let request = MergeRequest { - id: uuid::Uuid::new_v4().to_string(), - revision: revision.clone(), - strategy: MergeStrategy::FastForward, - candidate_id: None, - }; - assert!( - matches!(repo.merge_pull(identity()?,"outsider",number,request.clone()).await,Err(InvocationError::Rejected(rejected)) if rejected.output == MergeOutcome::NotFound) - ); - assert!( - matches!(repo.merge_pull(identity()?,"merge-author",number,request.clone()).await,Err(InvocationError::Rejected(rejected)) if rejected.output == MergeOutcome::Forbidden) - ); - let denied_identity = identity()?; - assert!( - matches!(repo.merge_pull(denied_identity,"canopy",number,request.clone()).await,Err(InvocationError::Rejected(rejected)) if rejected.output == MergeOutcome::ReviewsRequired) - ); - repo.review_pull( - identity()?, - "canopy", - number, - NewReview { - id: uuid::Uuid::new_v4().into_bytes(), - revision: &revision, - kind: ReviewKind::Approve, - body: "Approved", - }, - ) - .await?; - assert!( - repo.pull_review_policy("canopy", number) - .await? - .output - .unwrap() - .reviews_satisfied - ); - // The raw publication command has no merge capability, even for the owner - // and a tip with an eligible approved proposal. - let direct = plan(repo, base_ref, Some(source)).await?; - assert!( - matches!(app.command::(target,identity()?,direct).await,Err(InvocationError::Rejected(rejected)) if !rejected.output) - ); - assert!( - matches!(repo.merge_pull(denied_identity,"canopy",number,request.clone()).await,Err(InvocationError::Rejected(rejected)) if rejected.output == MergeOutcome::ReviewsRequired) - ); - // Force a constraint failure after the ref update inside the merge command. - // The injected duplicate parent record is removed after proving rollback. - let injected = uuid::Uuid::new_v4().into_bytes(); - sql.batch(identity()?,SqlBatch {statements:vec![SqlStatement { - sql:"INSERT INTO pull_merges (id,binding,pull_number,oid,merged_ms,pull_version,source_oid,source_version,base_oid,base_version) VALUES (?1,?2,?3,?4,0,1,?5,1,?4,1)".into(),parameters:vec![SqlValue::Blob(injected.to_vec()),SqlValue::Blob(vec![0;32]),SqlValue::Integer(number),SqlValue::Blob(base.to_vec()),SqlValue::Blob(source.to_vec())], - }]}).await?; - let before = repo.ref_state(base_ref, None).await?.output; - let generation = repo.refs_page("", None).await?.output.generation; - assert!( - repo.merge_pull(identity()?, "canopy", number, request.clone()) - .await - .is_err() - ); - assert_eq!(repo.ref_state(base_ref, None).await?.output, before); - assert_eq!( - repo.refs_page("", None).await?.output.generation, - generation - ); - assert_eq!( - repo.pull("canopy", number) - .await? - .output - .unwrap() - .summary - .state, - PullState::Open - ); - sql.batch( - identity()?, - SqlBatch { - statements: vec![SqlStatement { - sql: "DELETE FROM pull_merges WHERE id = ?1".into(), - parameters: vec![SqlValue::Blob(injected.to_vec())], - }], - }, - ) - .await?; - let command_id = identity()?; - let merged = repo - .merge_pull(command_id, "canopy", number, request.clone()) - .await?; - let MergeOutcome::Applied { merge: record } = &merged.output else { - return Err("merge not applied".into()); - }; - assert_eq!(record.oid, hex::encode(source)); - assert_eq!( - repo.merge_pull(command_id, "canopy", number, request.clone()) - .await? - .receipt, - merged.receipt - ); - let mut stricter = edit(1); - stricter.required_approvals = 2; - repo.set_branch_rule(identity()?, "canopy", stricter) - .await?; - assert_eq!( - repo.merge_pull(identity()?, "canopy", number, request) - .await? - .output, - merged.output - ); - let published = repo.ref_state(base_ref, None).await?.output.unwrap(); - assert_eq!(published.oid, Some(source)); - assert_eq!(published.version, 2); - Ok(()) -} diff --git a/crates/canopy-server/tests/repository_cell/pages.rs b/crates/canopy-server/tests/repository_cell/pages.rs deleted file mode 100644 index 2b38b285..00000000 --- a/crates/canopy-server/tests/repository_cell/pages.rs +++ /dev/null @@ -1,274 +0,0 @@ -use super::*; -use canopy_server::{INLINE_OBJECT_LIMIT, ObjectBatch, ObjectStorage, StoredObject}; -use cellule_runtime::{ - SqlCell, primitives::sql::SqlBatch, primitives::sql::SqlStatement, primitives::sql::SqlValue, -}; -use sha2::{Digest as _, Sha256}; -use std::collections::BTreeMap; - -type Result = std::result::Result>; - -fn identity() -> Result { - let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; - Ok(MutationIdentity { - request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - issued_at_ms: now, - expires_at_ms: now + 60_000, - }) -} - -fn inline(body: Vec) -> StoredObject { - StoredObject { - oid: object_id(canopy_server::ObjectFormat::Sha1, ObjectKind::Blob, &body), - kind: ObjectKind::Blob, - storage: ObjectStorage::Inline(body), - } -} - -async fn put( - repository: &RepositoryCell, - objects: impl IntoIterator, -) -> Result { - let mut batch = ObjectBatch::default(); - for object in objects { - assert!(batch.try_push(object).is_ok()); - } - repository.put_objects(identity()?, batch).await?; - Ok(()) -} - -async fn clear(sql: &SqlCell) -> Result { - // This exercise runs first in a fresh fixture and owns all these records. - sql.batch( - identity()?, - SqlBatch { - statements: [ - "DELETE FROM object_closure", - "DELETE FROM objects", - "DELETE FROM object_chunks", - "DELETE FROM object_uploads", - ] - .into_iter() - .map(|sql| SqlStatement { - sql: sql.into(), - parameters: vec![], - }) - .collect(), - }, - ) - .await?; - Ok(()) -} - -async fn scan( - repository: &RepositoryCell, -) -> Result> { - let mut after = None; - let mut objects = BTreeMap::new(); - for _ in 0..10 { - let page = repository.object_page(after).await?.output; - if page.is_empty() { - return Ok(objects); - } - assert!(page.len() <= 128); - let bytes: usize = page - .iter() - .map(|object| match &object.storage { - ObjectStorage::Inline(body) => body.len(), - _ => 0, - }) - .sum(); - assert!(bytes <= INLINE_OBJECT_LIMIT); - for object in page { - assert!(after.is_none_or(|old| object.oid > old)); - after = Some(object.oid); - assert!(objects.insert(object.oid, object).is_none()); - } - } - Err("object pagination did not finish".into()) -} - -fn before(oid: canopy_server::ObjectId) -> Option { - let mut bytes = match oid { - canopy_server::ObjectId::Sha1(bytes) => bytes.to_vec(), - canopy_server::ObjectId::Sha256(bytes) => bytes.to_vec(), - }; - for byte in bytes.iter_mut().rev() { - if *byte != 0 { - *byte -= 1; - return canopy_server::ObjectId::try_from(bytes).ok(); - } - *byte = u8::MAX; - } - None -} - -pub async fn exercise(repository: &RepositoryCell, sql: &SqlCell) -> Result { - assert!(repository.object_page(None).await?.output.is_empty()); - let mut expected = BTreeMap::new(); - for start in [0, 128] { - let batch = (start..start + 128) - .map(|n| { - let body = format!("object-page-{n}").into_bytes(); - let object = inline(body.clone()); - expected.insert(object.oid, body); - object - }) - .collect::>(); - put(repository, batch).await?; - } - let first = repository.object_page(None).await?.output; - assert_eq!(first.len(), 128); - let second = repository - .object_page(Some(first.last().ok_or("missing first page")?.oid)) - .await? - .output; - assert_eq!(second.len(), 128); - assert!( - repository - .object_page(Some(second.last().ok_or("missing second page")?.oid)) - .await? - .output - .is_empty() - ); - let actual = scan(repository).await?; - assert_eq!( - actual.keys().copied().collect::>(), - expected.keys().copied().collect::>() - ); - for (oid, object) in actual { - let ObjectStorage::Inline(body) = object.storage else { - return Err("inline body missing".into()); - }; - assert_eq!(body, expected[&oid]); - } - clear(sql).await?; - expected.clear(); - - let halves: Vec<_> = [11, 12] - .map(|byte| inline(vec![byte; INLINE_OBJECT_LIMIT / 2])) - .into(); - for object in &halves { - let ObjectStorage::Inline(body) = &object.storage else { - return Err("inline body missing".into()); - }; - expected.insert(object.oid, body.clone()); - } - put(repository, halves).await?; - let full = repository.object_page(None).await?.output; - assert_eq!(full.len(), 2); - assert_eq!( - full.iter() - .map(|object| match &object.storage { - ObjectStorage::Inline(body) => body.len(), - _ => 0, - }) - .sum::(), - INLINE_OBJECT_LIMIT - ); - for body in [vec![13; INLINE_OBJECT_LIMIT], vec![14], Vec::new()] { - let object = inline(body.clone()); - expected.insert(object.oid, body); - put(repository, [object]).await?; - } - let chunked = vec![15; INLINE_OBJECT_LIMIT + 1]; - let staged = repository - .stage_object(identity()?, ObjectKind::Tag, &chunked) - .await?; - let chunked_oid = staged.oid; - let ObjectStorage::Chunked { upload, .. } = staged.storage else { - return Err("missing chunk reference".into()); - }; - put( - repository, - [StoredObject { - oid: chunked_oid, - kind: ObjectKind::Tag, - storage: ObjectStorage::Chunked { - upload, - size: chunked.len() as u64, - blake3: *blake3::hash(&chunked).as_bytes(), - }, - }], - ) - .await?; - let external = vec![16; INLINE_OBJECT_LIMIT + 1]; - let external_oid = object_id( - canopy_server::ObjectFormat::Sha1, - ObjectKind::Blob, - &external, - ); - let sha256: [u8; 32] = Sha256::digest(&external).into(); - put( - repository, - [StoredObject { - oid: external_oid, - kind: ObjectKind::Blob, - storage: ObjectStorage::External { - size: external.len() as u64, - blake3: *blake3::hash(&external).as_bytes(), - sha256, - }, - }], - ) - .await?; - let mut actual = scan(repository).await?; - assert_eq!(actual.len(), expected.len() + 2); - for (oid, expected_body) in &expected { - let object = actual.remove(oid).ok_or("missing inline object")?; - let ObjectStorage::Inline(body) = object.storage else { - return Err("inline body missing".into()); - }; - assert_eq!(&body, expected_body); - } - let descriptor = actual - .remove(&chunked_oid) - .ok_or("missing chunked object")?; - assert!( - matches!(descriptor.storage, ObjectStorage::Chunked { upload: stored, size, blake3: digest } if stored == upload && size == chunked.len() as u64 && digest == *blake3::hash(&chunked).as_bytes()) - ); - let descriptor = actual - .remove(&external_oid) - .ok_or("missing external object")?; - assert!( - matches!(descriptor.storage, ObjectStorage::External { size, blake3: digest, sha256: stored } if size == external.len() as u64 && digest == *blake3::hash(&external).as_bytes() && stored == sha256) - ); - - let damaged = object_id(canopy_server::ObjectFormat::Sha1, ObjectKind::Blob, &[14]); - for (statement, parameters) in [ - ( - "UPDATE objects SET digest = zeroblob(32) WHERE oid = ?1", - vec![SqlValue::Blob(damaged.to_vec())], - ), - ( - "UPDATE objects SET body = ?2, digest = ?3 WHERE oid = ?1", - vec![ - SqlValue::Blob(damaged.to_vec()), - SqlValue::Blob(vec![17]), - SqlValue::Blob(blake3::hash(&[17]).as_bytes().to_vec()), - ], - ), - ( - "UPDATE objects SET body = zeroblob(?2), size = ?2 WHERE oid = ?1", - vec![ - SqlValue::Blob(damaged.to_vec()), - SqlValue::Integer(INLINE_OBJECT_LIMIT as i64 + 1), - ], - ), - ] { - sql.batch( - identity()?, - SqlBatch { - statements: vec![SqlStatement { - sql: statement.into(), - parameters, - }], - }, - ) - .await?; - assert!(repository.object_page(before(damaged)).await.is_err()); - } - clear(sql).await?; - assert!(repository.object_page(None).await?.output.is_empty()); - Ok(()) -} diff --git a/crates/canopy-server/tests/repository_cell/pulls.rs b/crates/canopy-server/tests/repository_cell/pulls.rs deleted file mode 100644 index 68d461c6..00000000 --- a/crates/canopy-server/tests/repository_cell/pulls.rs +++ /dev/null @@ -1,271 +0,0 @@ -use super::*; -use canopy_server::pulls::{ - NewPull, NewReview, PullChange, PullEdit, PullRevision, PullState, ReviewKind, -}; -type Result = std::result::Result>; -fn identity() -> Result { - let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; - Ok(MutationIdentity { - request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - issued_at_ms: now, - expires_at_ms: now + 60_000, - }) -} -async fn number(repo: &RepositoryCell, actor: &str, input: NewPull<'_>) -> Result { - let PullChange::Applied(number) = repo.create_pull(identity()?, actor, input).await?.output - else { - return Err("pull creation failed".into()); - }; - Ok(number) -} -pub async fn verify(repo: &RepositoryCell) -> Result { - let tree = objects::put(repo, identity()?, ObjectKind::Tree, b"") - .await? - .output; - let body = format!( - "tree {}\nauthor Test 0 +0000\ncommitter Test 0 +0000\n\nPull base\n", - hex::encode(tree) - ); - let base = objects::put(repo, identity()?, ObjectKind::Commit, body.as_bytes()) - .await? - .output; - let body = format!( - "tree {}\nparent {}\nauthor Test 0 +0000\ncommitter Test 0 +0000\n\nPull source\n", - hex::encode(tree), - hex::encode(base) - ); - let source = objects::put(repo, identity()?, ObjectKind::Commit, body.as_bytes()) - .await? - .output; - repo.finalize_push( - identity()?, - PushPlan { - actor: "canopy".into(), - updates: vec![ - RefUpdate { - name: "refs/heads/pull-base".into(), - expected: None, - new_oid: Some(base), - }, - RefUpdate { - name: "refs/heads/pull-source".into(), - expected: None, - new_oid: Some(source), - }, - ], - }, - ) - .await?; - repo.grant_member(identity()?, "canopy", "pull-author", TokenScope::Read) - .await?; - repo.grant_member(identity()?, "canopy", "pull-reviewer", TokenScope::Write) - .await?; - let base_oid = hex::encode(base); - let source_oid = hex::encode(source); - let id = uuid::Uuid::new_v4().into_bytes(); - let title = "é".repeat(128); - let body = "x".repeat(16384); - let new = || NewPull { - id, - title: &title, - body: &body, - draft: false, - source_ref: "refs/heads/pull-source", - source_oid: &source_oid, - base_ref: "refs/heads/pull-base", - base_oid: &base_oid, - }; - assert_eq!( - repo.create_pull(identity()?, "outsider", new()) - .await? - .output, - PullChange::NotFound - ); - let create_id = identity()?; - let created = repo.create_pull(create_id, "pull-author", new()).await?; - assert_eq!( - repo.create_pull(create_id, "pull-author", new()) - .await? - .receipt, - created.receipt - ); - let PullChange::Applied(pull) = created.output else { - return Err("missing pull".into()); - }; - let other = number( - repo, - "pull-author", - NewPull { - id: uuid::Uuid::new_v4().into_bytes(), - ..new() - }, - ) - .await?; - assert!(repo.pull("outsider", pull).await?.output.is_none()); - assert!(repo.pulls("outsider", 0, None).await?.output.is_none()); - assert!( - repo.pull_reviews("outsider", pull, 0) - .await? - .output - .is_none() - ); - let revision = PullRevision { - pull_version: 1, - source_oid: source_oid.clone(), - source_version: 1, - base_oid: base_oid.clone(), - base_version: 1, - }; - let review_id = uuid::Uuid::new_v4().into_bytes(); - let review = || NewReview { - id: review_id, - revision: &revision, - kind: ReviewKind::Approve, - body: &body, - }; - assert_eq!( - repo.review_pull(identity()?, "pull-author", pull, review()) - .await? - .output, - PullChange::Forbidden - ); - assert_eq!( - repo.review_pull(identity()?, "outsider", pull, review()) - .await? - .output, - PullChange::NotFound - ); - let review_mutation = identity()?; - let result = repo - .review_pull(review_mutation, "pull-reviewer", pull, review()) - .await?; - assert_eq!( - repo.review_pull(review_mutation, "pull-reviewer", pull, review()) - .await? - .receipt, - result.receipt - ); - assert_eq!( - repo.review_pull(identity()?, "pull-reviewer", other, review()) - .await? - .output, - PullChange::Conflict - ); - assert!(repo.pull_reviews("canopy", pull, 0).await?.output.unwrap()[0].applicable); - let stale = PullRevision { - source_version: 2, - ..revision.clone() - }; - assert_eq!( - repo.review_pull( - identity()?, - "pull-reviewer", - pull, - NewReview { - id: uuid::Uuid::new_v4().into_bytes(), - revision: &stale, - ..review() - } - ) - .await? - .output, - PullChange::Conflict - ); - // Only authorized ACL changes can advance the reviewer's grant version. - assert!( - !repo - .revoke_member(identity()?, "pull-author", "pull-reviewer") - .await? - .output - ); - assert!(repo.pull_reviews("canopy", pull, 0).await?.output.unwrap()[0].applicable); - repo.revoke_member(identity()?, "canopy", "pull-reviewer") - .await?; - assert_eq!( - repo.review_pull(identity()?, "pull-reviewer", pull, review()) - .await? - .output, - PullChange::NotFound - ); - repo.grant_member(identity()?, "canopy", "pull-reviewer", TokenScope::Write) - .await?; - repo.review_pull(identity()?, "pull-reviewer", pull, review()) - .await?; - assert!(!repo.pull_reviews("canopy", pull, 0).await?.output.unwrap()[0].applicable); - let owner_review = NewReview { - id: uuid::Uuid::new_v4().into_bytes(), - ..review() - }; - assert!(matches!( - repo.review_pull(identity()?, "canopy", pull, owner_review) - .await? - .output, - PullChange::Applied(_) - )); - let edit = || PullEdit { - expected_version: 1, - title: "Edited", - body: "Changed", - state: PullState::Closed, - draft: false, - }; - assert_eq!( - repo.edit_pull(identity()?, "outsider", pull, edit()) - .await? - .output, - PullChange::NotFound - ); - assert_eq!( - repo.edit_pull(identity()?, "pull-author", pull, edit()) - .await? - .output, - PullChange::Applied(pull) - ); - assert_eq!( - repo.edit_pull(identity()?, "pull-author", pull, edit()) - .await? - .output, - PullChange::Conflict - ); - repo.create_pull(identity()?, "pull-author", new()).await?; - assert_eq!( - repo.pull("canopy", pull) - .await? - .output - .unwrap() - .summary - .title, - "Edited" - ); - assert!( - repo.pull_reviews("canopy", pull, 0) - .await? - .output - .unwrap() - .iter() - .all(|r| !r.applicable) - ); - repo.revoke_member(identity()?, "canopy", "pull-author") - .await?; - assert_eq!( - repo.create_pull(identity()?, "pull-author", new()) - .await? - .output, - PullChange::NotFound - ); - assert_eq!( - repo.edit_pull( - identity()?, - "pull-author", - pull, - PullEdit { - expected_version: 2, - ..edit() - } - ) - .await? - .output, - PullChange::NotFound - ); - Ok(()) -} diff --git a/crates/canopy-server/tests/repository_cell/rebase.rs b/crates/canopy-server/tests/repository_cell/rebase.rs deleted file mode 100644 index 94a5330f..00000000 --- a/crates/canopy-server/tests/repository_cell/rebase.rs +++ /dev/null @@ -1,123 +0,0 @@ -use super::branch_rules::{commit, identity, plan}; -use super::*; -use canopy_server::pulls::{ - NewPull, PullChange, PullRevision, - candidates::{CandidateRequest, CandidateResult}, - merge::{MergeOutcome, MergeRequest, MergeStrategy}, -}; -use cellule_runtime::{ - SqlCell, primitives::sql::SqlBatch, primitives::sql::SqlStatement, primitives::sql::SqlValue, -}; -type Result = std::result::Result>; - -pub async fn verify(repo: &RepositoryCell, sql: &SqlCell) -> Result { - let tree = objects::put(repo, identity()?, ObjectKind::Tree, b"") - .await? - .output; - let base = commit(repo, tree, &[], "Rebase base").await?; - let first = commit(repo, tree, &[base], "First").await?; - let source = commit(repo, tree, &[first], "Second").await?; - let base_ref = "refs/heads/rebase-base"; - let source_ref = "refs/heads/rebase-source"; - for (reference, commit) in [(base_ref, base), (source_ref, source)] { - repo.finalize_push(identity()?, plan(repo, reference, Some(commit)).await?) - .await?; - } - let PullChange::Applied(number) = repo - .create_pull( - identity()?, - "canopy", - NewPull { - id: uuid::Uuid::new_v4().into_bytes(), - title: "Certified rebase", - body: "", - draft: false, - source_ref, - source_oid: &hex::encode(source), - base_ref, - base_oid: &hex::encode(base), - }, - ) - .await? - .output - else { - return Err("pull not created".into()); - }; - let revision = PullRevision { - pull_version: 1, - source_oid: hex::encode(source), - source_version: 1, - base_oid: hex::encode(base), - base_version: 1, - }; - let rewritten = |parent, author, message| { - format!( - "tree {}\nparent {}\nauthor {author} 0 +0000\ncommitter canopy 1 +0000\n\n{message}\n", - hex::encode(tree), - hex::encode(parent) - ) - }; - let body = rewritten(base, "Test", "First"); - let correct_first = objects::put(repo, identity()?, ObjectKind::Commit, body.as_bytes()) - .await? - .output; - for (name, parent, author, message, accepted) in [ - ("skip", base, "Test", "Second", false), - ("author", correct_first, "Forged", "Second", false), - ("message", correct_first, "Test", "Replacement", false), - ("valid", correct_first, "Test", "Second", true), - ] { - let body = rewritten(parent, author, message); - let tip = objects::put(repo, identity()?, ObjectKind::Commit, body.as_bytes()) - .await? - .output; - // Certify genuine Git objects first. A valid graph alone must never - // certify a chain with a dropped commit or changed author/message. - repo.finalize_push( - identity()?, - plan(repo, &format!("refs/heads/rebase-proof-{name}"), Some(tip)).await?, - ) - .await?; - let id = uuid::Uuid::new_v4(); - let request = CandidateRequest { - id: id.to_string(), - revision: revision.clone(), - strategy: MergeStrategy::Rebase, - message: String::new(), - }; - let result = CandidateResult::Ready { - oid: hex::encode(tip), - tree_oid: hex::encode(tree), - }; - sql.batch(identity()?, SqlBatch { statements: vec![SqlStatement { - sql: "INSERT INTO merge_candidates (id,binding,pull_number,actor,request,created_ms,result,source_oid,base_oid,oid) VALUES (?1,?2,?3,'canopy',?4,1000,?5,?6,?7,?8)".into(), - parameters: vec![SqlValue::Blob(id.as_bytes().to_vec()), SqlValue::Blob(vec![0;32]), SqlValue::Integer(number), SqlValue::Text(serde_json::to_string(&request)?), SqlValue::Text(serde_json::to_string(&result)?), SqlValue::Blob(source.to_vec()), SqlValue::Blob(base.to_vec()), SqlValue::Blob(tip.to_vec())], - }] }).await?; - let publish = MergeRequest { - id: uuid::Uuid::new_v4().to_string(), - revision: revision.clone(), - strategy: MergeStrategy::Rebase, - candidate_id: Some(id.to_string()), - }; - let outcome = repo - .merge_pull(identity()?, "canopy", number, publish) - .await; - if accepted { - assert!(matches!(outcome?.output, MergeOutcome::Applied { .. })); - assert_eq!( - repo.ref_state(base_ref, None).await?.output.unwrap().oid, - Some(tip) - ); - } else { - assert!( - matches!(outcome, Err(InvocationError::Rejected(result)) if result.output == MergeOutcome::Conflict), - "{name}" - ); - assert_eq!( - repo.ref_state(base_ref, None).await?.output.unwrap().oid, - Some(base) - ); - } - } - Ok(()) -} diff --git a/crates/canopy-server/tests/repository_cell/visibility.rs b/crates/canopy-server/tests/repository_cell/visibility.rs deleted file mode 100644 index fdbe50e7..00000000 --- a/crates/canopy-server/tests/repository_cell/visibility.rs +++ /dev/null @@ -1,116 +0,0 @@ -use super::*; -use canopy_server::{ReadIdentity, Visibility}; - -fn identity() -> Result> { - let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; - Ok(MutationIdentity { - request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - issued_at_ms: now, - expires_at_ms: now + 60_000, - }) -} - -pub async fn verify(repository: &RepositoryCell) -> Result<(), Box> { - let initial = repository.visibility().await?.output; - assert_eq!(initial.visibility, Visibility::Private); - assert_eq!( - repository - .access_level(ReadIdentity::Anonymous, None) - .await? - .output, - None - ); - repository - .grant_member(identity()?, "canopy", "writer", TokenScope::Write) - .await?; - for actor in ["writer", "outsider"] { - assert!( - !repository - .set_visibility(identity()?, actor, initial.generation, Visibility::Public) - .await? - .output - ); - } - for generation in [-1, i64::MAX] { - assert!( - repository - .set_visibility(identity()?, "canopy", generation, Visibility::Public) - .await - .is_err() - ); - } - let request = identity()?; - let published = repository - .set_visibility(request, "canopy", initial.generation, Visibility::Public) - .await?; - assert!(published.output); - let replayed = repository - .set_visibility(request, "canopy", initial.generation, Visibility::Public) - .await?; - assert_eq!(replayed.receipt, published.receipt); - assert!(replayed.output); - for (reader, role) in [ - (ReadIdentity::Anonymous, TokenScope::Read), - (ReadIdentity::Account("outsider"), TokenScope::Read), - (ReadIdentity::Account("writer"), TokenScope::Write), - (ReadIdentity::Account("canopy"), TokenScope::Admin), - ] { - assert_eq!( - repository - .access_level(reader, Some(published.receipt)) - .await? - .output, - Some(role) - ); - } - let public = repository.visibility().await?.output; - assert_eq!(public.generation, initial.generation + 1); - let head = repository.ref_state("refs/heads/main", None).await?.output; - assert!(matches!( - repository - .finalize_push( - identity()?, - PushPlan { - actor: "outsider".into(), - updates: vec![RefUpdate { - name: "refs/heads/main".into(), - expected: head, - new_oid: None - }], - } - ) - .await, - Err(InvocationError::Rejected(_)) - )); - assert!( - repository - .set_visibility( - identity()?, - "canopy", - public.generation, - Visibility::Private - ) - .await? - .output - ); - assert_eq!( - repository - .access_level(ReadIdentity::Anonymous, None) - .await? - .output, - None - ); - // Returning to private must not make an old public request valid again. - assert!( - !repository - .set_visibility( - identity()?, - "canopy", - initial.generation, - Visibility::Public - ) - .await? - .output - ); - Ok(()) -} diff --git a/crates/canopy-server/tests/smart_http/cache_admission.rs b/crates/canopy-server/tests/smart_http/cache_admission.rs deleted file mode 100644 index acdb8a80..00000000 --- a/crates/canopy-server/tests/smart_http/cache_admission.rs +++ /dev/null @@ -1,291 +0,0 @@ -use super::*; - -pub async fn verify( - root: &Path, - repository: &RepositoryCell, - budget: &DiskBudget, - client: &reqwest::Client, - url: &str, -) -> Result<(), Box> { - let source = root.join("quota-source"); - run_git( - None, - &["init", "-b", "main", source.to_str().ok_or("invalid path")?], - ) - .await?; - run_git(Some(&source), &["config", "user.name", "Canopy Test"]).await?; - run_git( - Some(&source), - &["config", "user.email", "canopy@example.invalid"], - ) - .await?; - // A small delta pack expands into many loose objects below Git's unpack limit. - // This distinguishes native cache admission from request-spool admission. - let mut common = vec![0; 8192]; - blake3::Hasher::new().finalize_xof().fill(&mut common); - for index in 0..40 { - let mut body = common.clone(); - body.extend_from_slice(format!("file {index}\n").as_bytes()); - tokio::fs::write(source.join(format!("file-{index:02}")), body).await?; - } - run_git(Some(&source), &["add", "."]).await?; - run_git(Some(&source), &["commit", "-m", "Cache admission fixture"]).await?; - let commit = run_git(Some(&source), &["rev-parse", "HEAD"]).await?; - let commit = std::str::from_utf8(&commit)?.trim(); - run_git(Some(&source), &["tag", "-a", "inner", "-m", "Inner tag"]).await?; - run_git( - Some(&source), - &["tag", "-a", "outer", "inner", "-m", "Nested tag"], - ) - .await?; - let mut advertised = vec![("refs/heads/quota", commit.to_owned())]; - for (name, revision) in [ - ("refs/tags/inner", "inner"), - ("refs/tags/outer", "outer"), - ("refs/tags/tree", "HEAD^{tree}"), - ("refs/tags/blob", "HEAD:file-00"), - ] { - let oid = run_git(Some(&source), &["rev-parse", revision]).await?; - advertised.push((name, std::str::from_utf8(&oid)?.trim().to_owned())); - } - let pack = run_git( - Some(&source), - &["pack-objects", "--stdout", "--all", "--window=50"], - ) - .await?; - assert!( - pack.len() < common.len() * 4, - "fixture requires delta compression" - ); - let mut body = Vec::new(); - for (index, (name, oid)) in advertised.iter().enumerate() { - let capabilities = if index == 0 { "\0report-status" } else { "" }; - let command = format!("{} {oid} {name}{capabilities}\n", "0".repeat(40)); - body.extend_from_slice(format!("{:04x}{command}", command.len() + 4).as_bytes()); - } - body.extend_from_slice(b"0000"); - body.extend_from_slice(&pack); - encoded_input::reject_corruption_and_expansion(repository, budget, client, url, &body).await?; - let id = uuid::Uuid::new_v4().to_string(); - let request = |id: &str| { - client - .post(format!("{url}/git-receive-pack")) - .bearer_auth("local-test-token") - .header("Content-Type", "application/x-git-receive-pack-request") - .header("Idempotency-Key", id) - .body(body.clone()) - }; - let before = repository.refs_page("", None).await?.output.generation; - let retained = budget.used(); - let occupied = budget.try_reserve(budget.capacity() - retained - body.len() as u64 - 4096)?; - let response = request(&id).send().await?; - assert_eq!(response.status(), reqwest::StatusCode::OK); - let refused = response.bytes().await?; - let report = String::from_utf8_lossy(&refused); - for (name, _) in &advertised { - assert!( - report.contains(&format!( - "ng {name} Canopy push failed before publication: cache disk budget exhausted" - )), - "{report}" - ); - } - assert!( - repository - .ref_state("refs/heads/quota", None) - .await? - .output - .is_none() - ); - assert!(repository.object_page(None).await?.output.is_empty()); - assert_eq!(budget.used(), occupied.bytes() + retained); - assert_eq!( - repository.refs_page("", None).await?.output.generation, - before - ); - drop(occupied); - - // Resource recovery cannot change the recorded outcome of the same push ID. - let replay = request(&id) - .send() - .await? - .error_for_status()? - .bytes() - .await?; - assert_eq!(replay, refused); - let id = uuid::Uuid::new_v4().to_string(); - let response = request(&id) - .send() - .await? - .error_for_status()? - .bytes() - .await?; - assert!( - response - .windows(b"ok refs/heads/quota".len()) - .any(|part| part == b"ok refs/heads/quota") - ); - let expected: canopy_server::ObjectId = - hex::decode(commit)?.try_into().map_err(|_| "invalid OID")?; - assert_eq!( - repository - .ref_state("refs/heads/quota", None) - .await? - .output - .and_then(|state| state.oid), - Some(expected) - ); - assert_eq!( - repository.refs_page("", None).await?.output.generation, - before + 1 - ); - let replay = request(&id) - .header("Content-Encoding", "identity") - .send() - .await? - .error_for_status()? - .bytes() - .await?; - assert_eq!(replay, response); - assert_eq!( - repository.refs_page("", None).await?.output.generation, - before + 1 - ); - assert_eq!( - request(&id) - .header("Content-Encoding", "gzip") - .send() - .await? - .status(), - reqwest::StatusCode::CONFLICT - ); - - // Failed hydration may retain completed immutable objects and conservative - // write charges. It must not publish a partial ref generation; retry resumes - // from the verified cache after external pressure is removed. - let retained = budget.used(); - let occupied = budget.try_reserve(budget.capacity() - retained - 512)?; - let advertisement = || { - client - .get(format!("{url}/info/refs?service=git-upload-pack")) - .bearer_auth("local-test-token") - }; - // Protocol v2 initially advertises capabilities only. Published history - // exceeds the available cache budget, but this exchange needs no objects. - let mut capabilities = None; - for query in ["service=git-upload-pack", "service=git%2Dupload%2Dpack"] { - let response = client - .get(format!("{url}/info/refs?{query}")) - .bearer_auth("local-test-token") - .header("Git-Protocol", "version=2") - .send() - .await?; - assert_eq!(response.status(), reqwest::StatusCode::OK); - let body = response.bytes().await?; - assert!(body.starts_with(b"000eversion 2\n")); - if let Some(expected) = &capabilities { - assert_eq!(&body, expected); - } - capabilities = Some(body); - assert_eq!(budget.used(), occupied.bytes() + retained); - } - let response = advertisement().send().await?; - assert_eq!(response.status(), reqwest::StatusCode::INSUFFICIENT_STORAGE); - assert_eq!(response.text().await?, "Git cache disk budget exhausted"); - assert!(budget.used() >= occupied.bytes()); - assert!(budget.used() <= budget.capacity()); - drop(occupied); - // Ref discovery fits even when expanding the repository history cannot. - // Nested tags and direct tree/blob refs must preserve native peeling rules. - let retained = budget.used(); - let occupied = budget.try_reserve(budget.capacity() - retained - 64 * 1024)?; - for service in ["git-upload-pack", "git-receive-pack"] { - let bytes = client - .get(format!("{url}/info/refs?service={service}")) - .bearer_auth("local-test-token") - .send() - .await? - .error_for_status()? - .bytes() - .await?; - for (name, oid) in &advertised { - let entry = format!("{oid} {name}"); - assert!( - bytes - .windows(entry.len()) - .any(|part| part == entry.as_bytes()) - ); - } - if service == "git-upload-pack" { - let peeled = format!("{commit} refs/tags/outer^{{}}"); - assert!( - bytes - .windows(peeled.len()) - .any(|part| part == peeled.as_bytes()) - ); - } - assert_eq!(budget.used(), occupied.bytes() + retained); - } - let bytes = client - .post(format!("{url}/git-upload-pack")) - .bearer_auth("local-test-token") - .header("Content-Type", "application/x-git-upload-pack-request") - .header("Git-Protocol", "version=2") - .body("0014command=ls-refs\n00010009peel\n000csymrefs\n0000") - .send() - .await? - .error_for_status()? - .bytes() - .await?; - let outer = &advertised[2].1; - let peeled = format!("{outer} refs/tags/outer peeled:{commit}"); - assert!( - bytes - .windows(peeled.len()) - .any(|part| part == peeled.as_bytes()) - ); - assert_eq!(budget.used(), occupied.bytes() + retained); - let want = format!("want {commit}\n"); - let fetch = || { - client - .post(format!("{url}/git-upload-pack")) - .bearer_auth("local-test-token") - .header("Content-Type", "application/x-git-upload-pack-request") - .body(format!("{:04x}{want}00000009done\n", want.len() + 4)) - }; - assert_eq!( - fetch().send().await?.status(), - reqwest::StatusCode::INSUFFICIENT_STORAGE - ); - drop(occupied); - fetch().send().await?.error_for_status()?.bytes().await?; - assert!(budget.used() > common.len() as u64 * 40); - let retained = budget.used(); - // A current full cache can serve discovery without recopying its targets. - let occupied = budget.try_reserve(budget.capacity() - retained - 512)?; - advertisement() - .send() - .await? - .error_for_status()? - .bytes() - .await?; - assert_eq!(budget.used(), occupied.bytes() + retained); - drop(occupied); - encoded_input::delete_with_admission_retry(repository, budget, client, url, commit).await?; - assert_eq!(budget.used(), retained); - run_git( - Some(&source), - &[ - "-c", - "http.extraHeader=Authorization: Bearer local-test-token", - "push", - url, - ":refs/tags/inner", - ":refs/tags/outer", - ":refs/tags/tree", - ":refs/tags/blob", - ], - ) - .await?; - Ok(()) -} diff --git a/crates/canopy-server/tests/smart_http/cache_reuse.rs b/crates/canopy-server/tests/smart_http/cache_reuse.rs deleted file mode 100644 index ec86baa4..00000000 --- a/crates/canopy-server/tests/smart_http/cache_reuse.rs +++ /dev/null @@ -1,87 +0,0 @@ -use super::*; -use std::{collections::BTreeMap, path::PathBuf, time::SystemTime}; - -fn cached_objects(root: &Path) -> std::io::Result> { - let mut objects = BTreeMap::new(); - for entry in std::fs::read_dir(root)? { - let entry = entry?; - if !entry - .file_name() - .to_string_lossy() - .starts_with("canopy-git-") - { - continue; - } - let git = entry.path().join("repo.git"); - if git.join("objects/info/alternates").exists() { - continue; - } - for shard in std::fs::read_dir(git.join("objects"))? { - let shard = shard?; - let name = shard.file_name(); - if name.len() != 2 || !name.as_encoded_bytes().iter().all(u8::is_ascii_hexdigit) { - continue; - } - for object in std::fs::read_dir(shard.path())? { - let object = object?; - if object.file_name().len() == 38 { - let metadata = object.metadata()?; - objects.insert(object.path(), (metadata.len(), metadata.modified()?)); - } - } - } - } - Ok(objects) -} - -pub async fn verify( - root: &Path, - local: &Path, - url: &str, -) -> Result<(), Box> { - let before = cached_objects(root)?; - assert!(!before.is_empty()); - tokio::fs::write(local.join("cache-reuse.txt"), b"only this blob is new\n").await?; - run_git(Some(local), &["add", "cache-reuse.txt"]).await?; - run_git(Some(local), &["commit", "-m", "Incremental object cache"]).await?; - let commit = run_git(Some(local), &["rev-parse", "HEAD"]).await?; - run_git( - Some(local), - &["-c", AUTH, "push", url, "HEAD:refs/heads/cache-reuse"], - ) - .await?; - run_git(None, &["-c", AUTH, "ls-remote", url]).await?; - assert_eq!(cached_objects(root)?, before); - let clone = root.join("cache-reuse-clone"); - run_git( - None, - &[ - "-c", - AUTH, - "clone", - "--single-branch", - "--branch", - "cache-reuse", - "--no-checkout", - url, - clone.to_str().ok_or("clone path")?, - ], - ) - .await?; - let after = cached_objects(root)?; - // A new blob, tree and commit are hydrated. Existing bodies keep their - // actual files across receive-pack and the next fetch of the published ref generation. - assert_eq!(after.len(), before.len() + 3); - for (path, metadata) in before { - assert_eq!(after.get(&path), Some(&metadata)); - } - assert_eq!(run_git(Some(&clone), &["rev-parse", "HEAD"]).await?, commit); - assert_eq!( - run_git(Some(&clone), &["show", "HEAD:cache-reuse.txt"]).await?, - b"only this blob is new\n" - ); - run_git(Some(&clone), &["fsck", "--strict", "--full"]).await?; - Ok(()) -} - -const AUTH: &str = "http.extraHeader=Authorization: Bearer local-test-token"; diff --git a/crates/canopy-server/tests/smart_http/encoded_input.rs b/crates/canopy-server/tests/smart_http/encoded_input.rs deleted file mode 100644 index 5d486de7..00000000 --- a/crates/canopy-server/tests/smart_http/encoded_input.rs +++ /dev/null @@ -1,165 +0,0 @@ -use super::*; -use std::io::Write; - -fn gzip(bytes: &[u8]) -> Result, Box> { - let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); - encoder.write_all(bytes)?; - Ok(encoder.finish()?) -} - -pub async fn reject_corruption_and_expansion( - repository: &RepositoryCell, - budget: &DiskBudget, - client: &reqwest::Client, - url: &str, - push: &[u8], -) -> Result<(), Box> { - let retained = budget.used(); - let valid = gzip(push)?; - let mut checksum = valid.clone(); - let crc = checksum.len() - 8; - checksum[crc] ^= 1; - let mut trailing = valid.clone(); - trailing.extend(b"invalid trailing bytes"); - for wire in [checksum, valid[..valid.len() - 1].to_vec(), trailing] { - let id = uuid::Uuid::new_v4().to_string(); - for _ in 0..2 { - let response = client - .post(format!("{url}/git-receive-pack")) - .bearer_auth("local-test-token") - .header("Content-Type", "application/x-git-receive-pack-request") - .header("Content-Encoding", "gzip") - .header("Idempotency-Key", &id) - .body(wire.clone()) - .send() - .await?; - assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST); - assert_eq!(response.text().await?, "Invalid Git gzip stream"); - assert_eq!(budget.used(), retained); - } - assert!( - repository - .ref_state("refs/heads/quota", None) - .await? - .output - .is_none() - ); - assert!(repository.object_page(None).await?.output.is_empty()); - } - let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); - for _ in 0..1024 { - encoder.write_all(&[b'x'; 64 * 1024])?; - } - encoder.write_all(b"x")?; - let response = client - .post(format!("{url}/git-upload-pack")) - .bearer_auth("local-test-token") - .header("Content-Type", "application/x-git-upload-pack-request") - .header("Content-Encoding", "gzip") - .body(encoder.finish()?) - .send() - .await?; - assert_eq!(response.status(), reqwest::StatusCode::PAYLOAD_TOO_LARGE); - assert_eq!(budget.used(), retained); - - // A valid decoded request above Git CGI's default 10 MiB buffer remains - // below Canopy's 64 MiB limit. No ref matches these long prefixes. - let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); - encoder.write_all(b"0014command=ls-refs\n0001")?; - let prefix = format!("ref-prefix {}\n", "z".repeat(60_000)); - let packet = format!("{:04x}{prefix}", prefix.len() + 4); - for _ in 0..190 { - encoder.write_all(packet.as_bytes())?; - } - encoder.write_all(b"0000")?; - assert!(packet.len() * 190 > 10 * 1024 * 1024); - let response = client - .post(format!("{url}/git-upload-pack")) - .bearer_auth("local-test-token") - .header("Content-Type", "application/x-git-upload-pack-request") - .header("Content-Encoding", "gzip") - .header("Git-Protocol", "version=2") - .body(encoder.finish()?) - .send() - .await? - .error_for_status()? - .bytes() - .await?; - assert_eq!(response, b"0000".as_slice()); - Ok(()) -} - -pub async fn delete_with_admission_retry( - repository: &RepositoryCell, - budget: &DiskBudget, - client: &reqwest::Client, - url: &str, - commit: &str, -) -> Result<(), Box> { - let command = format!( - "{commit} {} refs/heads/quota\0report-status delete-refs\n", - "0".repeat(40) - ); - let wire = gzip(format!("{:04x}{command}0000", command.len() + 4).as_bytes())?; - let id = uuid::Uuid::new_v4().to_string(); - let request = || { - client - .post(format!("{url}/git-receive-pack")) - .bearer_auth("local-test-token") - .header("Content-Type", "application/x-git-receive-pack-request") - .header("Content-Encoding", "gzip") - .header("Idempotency-Key", &id) - .body(wire.clone()) - }; - let warm = budget.used(); - let generation = repository.refs_page("", None).await?.output.generation; - let occupied = budget.try_reserve(budget.capacity() - warm - wire.len() as u64 - 1)?; - let response = request().send().await?; - assert_eq!(response.status(), reqwest::StatusCode::INSUFFICIENT_STORAGE); - assert_eq!(budget.used(), occupied.bytes() + warm); - assert_eq!( - repository.refs_page("", None).await?.output.generation, - generation - ); - drop(occupied); - let response = request().send().await?.error_for_status()?.bytes().await?; - assert!( - response - .windows(b"ok refs/heads/quota".len()) - .any(|part| part == b"ok refs/heads/quota") - ); - assert!( - repository - .ref_state("refs/heads/quota", None) - .await? - .output - .is_some_and(|state| state.oid.is_none()) - ); - // Admit only the encoded spool: any repeated gzip expansion must fail. - // Completed replay must resolve the original response before decoding. - let occupied = budget.try_reserve(budget.capacity() - warm - wire.len() as u64)?; - let replay = request().send().await?.error_for_status()?.bytes().await?; - assert_eq!(replay, response); - assert_eq!(budget.used(), occupied.bytes() + warm); - drop(occupied); - // Changing only gzip metadata preserves decoded commands, but is a - // different encoded request and cannot reuse the completed operation ID. - let mut changed = wire.clone(); - changed[4..8].copy_from_slice(&1u32.to_le_bytes()); - let conflict = client - .post(format!("{url}/git-receive-pack")) - .bearer_auth("local-test-token") - .header("Content-Type", "application/x-git-receive-pack-request") - .header("Content-Encoding", "gzip") - .header("Idempotency-Key", &id) - .body(changed) - .send() - .await?; - assert_eq!(conflict.status(), reqwest::StatusCode::CONFLICT); - assert_eq!( - repository.refs_page("", None).await?.output.generation, - generation + 1 - ); - assert_eq!(budget.used(), warm); - Ok(()) -} diff --git a/crates/canopy-server/tests/smart_http/native_resources.rs b/crates/canopy-server/tests/smart_http/native_resources.rs deleted file mode 100644 index 88b72dab..00000000 --- a/crates/canopy-server/tests/smart_http/native_resources.rs +++ /dev/null @@ -1,105 +0,0 @@ -use super::*; - -type Result = std::result::Result>; - -pub async fn verify(root: &Path, url: &str) -> Result { - let fixture = tempfile::TempDir::new_in(root)?; - let root = fixture.path(); - let source = root.join("native-source"); - let clone = root.join("native-clone"); - run_git( - None, - &["init", "-b", "pack-policy", source.to_str().ok_or("path")?], - ) - .await?; - run_git(Some(&source), &["config", "user.name", "Canopy Test"]).await?; - run_git( - Some(&source), - &["config", "user.email", "canopy@example.invalid"], - ) - .await?; - let mut small = b"small text line\n".repeat(32768); - let mut large = b"large text line\n".repeat(640000); - let mut small_oids = Vec::new(); - let mut large_oids = Vec::new(); - for generation in 0..2 { - small[0] = b'A' + generation; - large[0] = b'A' + generation; - tokio::fs::write(source.join("small.txt"), &small).await?; - tokio::fs::write(source.join("large.txt"), &large).await?; - run_git(Some(&source), &["add", "."]).await?; - run_git(Some(&source), &["commit", "-m", "Related objects"]).await?; - for (name, ids) in [ - ("HEAD:small.txt", &mut small_oids), - ("HEAD:large.txt", &mut large_oids), - ] { - ids.push( - String::from_utf8(run_git(Some(&source), &["rev-parse", name]).await?)? - .trim() - .to_owned(), - ); - } - } - run_git( - Some(&source), - &[ - "-c", - "http.extraHeader=Authorization: Bearer local-test-token", - "push", - url, - "pack-policy", - ], - ) - .await?; - run_git( - None, - &[ - "-c", - "fetch.unpackLimit=1", - "-c", - "http.extraHeader=Authorization: Bearer local-test-token", - "clone", - "--single-branch", - "--branch", - "pack-policy", - url, - clone.to_str().ok_or("path")?, - ], - ) - .await?; - assert_eq!(tokio::fs::read(clone.join("large.txt")).await?, large); - assert_eq!(tokio::fs::read(clone.join("small.txt")).await?, small); - run_git(Some(&clone), &["fsck", "--strict", "--full"]).await?; - let index = std::fs::read_dir(clone.join(".git/objects/pack"))? - .collect::>>()? - .into_iter() - .map(|entry| entry.path()) - .find(|path| path.extension().is_some_and(|extension| extension == "idx")) - .ok_or("clone pack index")?; - let listing = String::from_utf8( - run_git( - Some(&clone), - &["verify-pack", "-v", index.to_str().ok_or("path")?], - ) - .await?, - )?; - // Inspect the actual served pack: large similar blobs are whole objects, - // while ordinary source-size blobs retain delta compression. - for oid in large_oids { - let fields: Vec<_> = listing - .lines() - .find(|line| line.starts_with(&oid)) - .ok_or("large object in pack")? - .split_whitespace() - .collect(); - assert_eq!(fields.len(), 5, "large object must not be a delta"); - } - assert!( - listing.lines().any(|line| { - let fields: Vec<_> = line.split_whitespace().collect(); - fields.len() == 7 && small_oids.iter().any(|oid| fields[0] == oid) - }), - "ordinary blobs still use deltas" - ); - Ok(()) -} diff --git a/crates/canopy-server/tests/smart_http/publication.rs b/crates/canopy-server/tests/smart_http/publication.rs deleted file mode 100644 index d58bb8ea..00000000 --- a/crates/canopy-server/tests/smart_http/publication.rs +++ /dev/null @@ -1,502 +0,0 @@ -use super::*; -use crate::paused_blobs::PausedBlobs; -use canopy_server::{PushPlan, RefUpdate, branch_rules::BranchRuleEdit}; -use std::{sync::atomic::Ordering, time::Duration}; - -type Result = std::result::Result>; -const AUTH: &str = "http.extraHeader=Authorization: Bearer late-writer-token"; - -fn packet(line: &str) -> Vec { - format!("{:04x}{line}", line.len() + 4).into_bytes() -} - -pub async fn verify( - root: &Path, - repository: &Arc, - store: &Arc, - url: &str, -) -> Result { - let local = root.join("publication-race"); - run_git(None, &["init", "-b", "main", local.to_str().ok_or("path")?]).await?; - let current = repository - .ref_state("refs/heads/main", None) - .await? - .output - .ok_or("main")? - .oid - .ok_or("main oid")?; - let client = reqwest::Client::new(); - repository - .grant_member( - support::identity()?, - "canopy", - "late-writer", - TokenScope::Write, - ) - .await?; - for kind in [ - "refs", - "policy", - "access", - "no-report", - "storage", - "storage-sideband", - "storage-no-report", - "storage-stock", - "storage-race", - ] { - let storage = kind.starts_with("storage"); - let no_report = kind.ends_with("no-report"); - let raced = kind == "storage-race"; - let reason = if storage { - "Canopy object ingestion failed" - } else { - "Canopy publication rejected" - }; - tokio::fs::write( - local.join("large"), - kind.as_bytes() - .iter() - .copied() - .cycle() - .take(2 * 1024 * 1024) - .collect::>(), - ) - .await?; - run_git(Some(&local), &["add", "."]).await?; - run_git( - Some(&local), - &[ - "-c", - "user.name=Test", - "-c", - "user.email=test@example.invalid", - "-c", - "commit.gpgsign=false", - "commit", - "-m", - kind, - ], - ) - .await?; - let oid = String::from_utf8(run_git(Some(&local), &["rev-parse", "HEAD"]).await?)? - .trim() - .to_owned(); - let reference = format!("refs/heads/late-{kind}"); - // Sort the valid sibling first so a premature per-ref write would leak. - let sibling = format!("refs/heads/early-{kind}"); - let id = uuid::Uuid::new_v4().to_string(); - let capabilities = match kind { - "policy" | "storage-sideband" => "report-status-v2 side-band-64k", - "access" | "storage" | "storage-race" => "report-status", - _ => "", - }; - let mut body = packet(&format!( - "{} {oid} {reference}\0{capabilities}\n", - "0".repeat(40) - )); - body.extend(packet(&format!("{} {oid} {sibling}\n", "0".repeat(40)))); - body.extend_from_slice(b"0000"); - body.extend(run_git(Some(&local), &["pack-objects", "--all", "--stdout"]).await?); - let post = |id: &str| { - client - .post(format!("{url}/git-receive-pack")) - .bearer_auth("late-writer-token") - .header("Content-Type", "application/x-git-receive-pack-request") - .header("Idempotency-Key", id) - .body(body.clone()) - }; - store.armed.store(true, Ordering::SeqCst); - let mut child = if kind == "refs" || kind == "storage-stock" { - Some( - Command::new("git") - .current_dir(&local) - .args([ - "-c", - "credential.helper=", - "-c", - AUTH, - "push", - "--atomic", - url, - &format!("HEAD:{reference}"), - &format!("HEAD:{sibling}"), - ]) - .env("GIT_TERMINAL_PROMPT", "0") - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .spawn()?, - ) - } else { - None - }; - let request = if child.is_none() { - Some(tokio::spawn(post(&id).send())) - } else { - None - }; - tokio::time::timeout(Duration::from_secs(15), store.entered.notified()).await?; - if kind == "refs" || kind == "no-report" { - repository - .finalize_push( - support::identity()?, - PushPlan { - actor: "canopy".into(), - updates: vec![RefUpdate { - name: reference.clone(), - expected: None, - new_oid: Some(current), - }], - }, - ) - .await?; - } else if kind == "policy" { - repository - .set_branch_rule( - support::identity()?, - "canopy", - BranchRuleEdit { - reference: reference.clone(), - expected_version: 0, - enabled: true, - deny_deletions: false, - fast_forward_only: false, - require_pull_request: true, - required_approvals: 0, - required_checks: vec![], - }, - ) - .await?; - } else if kind == "access" { - repository - .revoke_member(support::identity()?, "canopy", "late-writer") - .await?; - } - let generation = repository.default_branch(None).await?.output.generation; - store.fail.store(storage, Ordering::SeqCst); - // A different gateway can commit the same ID while this attempt is paused. - // The losing ingestion failure must return that success, never a local ng. - let winner = if raced { - Some(gateway_request(root, repository, store.clone(), &body, &id).await?) - } else { - None - }; - store.proceed.notify_one(); - if let Some(child) = child.take() { - let output = child.wait_with_output().await?; - let stderr = String::from_utf8_lossy(&output.stderr); - assert!(!output.status.success(), "{stderr}"); - assert!( - stderr.contains("[remote rejected]") && stderr.contains(reason), - "{stderr}" - ); - assert!( - !stderr.contains("HTTP 409") && !stderr.contains("HTTP 500"), - "{stderr}" - ); - } - if let Some(request) = request { - let response = request.await??; - assert_eq!( - response.status().as_u16(), - if no_report { 409 } else { 200 } - ); - let report = response.bytes().await?; - let text = String::from_utf8_lossy(&report); - if let Some((status, saved)) = winner { - assert_eq!(status, 200); - assert_eq!(report.as_ref(), saved); - assert!(text.contains(&format!("ok {reference}")), "{text}"); - } else { - assert!(text.contains(reason), "{text}"); - } - if !no_report && !raced { - assert!( - text.contains(&format!("ng {reference} ")) - && text.contains(&format!("ng {sibling} ")), - "{text}" - ); - assert!(!text.contains(&format!("ok {reference}")), "{text}"); - } - repository - .grant_member( - support::identity()?, - "canopy", - "late-writer", - TokenScope::Write, - ) - .await?; - if kind == "policy" { - repository - .set_branch_rule( - support::identity()?, - "canopy", - BranchRuleEdit { - reference: reference.clone(), - expected_version: 1, - enabled: false, - deny_deletions: false, - fast_forward_only: false, - require_pull_request: false, - required_approvals: 0, - required_checks: vec![], - }, - ) - .await?; - } - // Recreate the gateway with no native/object cache; the saved Cell - // decision must survive permission/policy or storage recovery. - let (status, saved) = - gateway_request(root, repository, Arc::new(InMemory::new()), &body, &id).await?; - assert_eq!(status, if no_report { 409 } else { 200 }); - assert_eq!(saved, report); - } - assert_eq!( - repository.default_branch(None).await?.output.generation, - generation + i64::from(raced) - ); - if !raced { - assert!(repository.ref_state(&sibling, None).await?.output.is_none()); - } - if storage && !raced { - let response = post(&uuid::Uuid::new_v4().to_string()).send().await?; - assert_eq!(response.status().as_u16(), 200); - let report = response.bytes().await?; - if !no_report && kind != "storage-stock" { - assert!(String::from_utf8_lossy(&report).contains(&format!("ok {reference}"))); - } - assert_eq!( - repository.default_branch(None).await?.output.generation, - generation + 1 - ); - } - if storage { - for name in [&reference, &sibling] { - assert_eq!( - hex::encode( - repository - .ref_state(name, None) - .await? - .output - .ok_or("missing ref")? - .oid - .ok_or("missing oid")? - ), - oid - ); - } - } - } - verify_preparation(root, repository, store, &local, current).await?; - Ok(()) -} - -async fn verify_preparation( - root: &Path, - repository: &Arc, - store: &Arc, - source: &Path, - current: canopy_server::ObjectId, -) -> Result { - use sha1::{Digest as _, Sha1}; - let mut pack = b"PACK\0\0\0\x02\0\0\0\0".to_vec(); - pack.extend_from_slice(&Sha1::digest(&pack)); - for kind in ["plain", "sideband", "no-report", "stock", "race"] { - let scratch = tempfile::tempdir_in(root)?; - let gateway = Arc::new(GitGateway::new( - repository.clone(), - scratch.path().into(), - store.clone(), - DiskBudget::new(1 << 30), - canopy_server::native_resources::NativeResources::default(), - )); - let listener = TcpListener::bind("127.0.0.1:0").await?; - let address = listener.local_addr()?; - let api = Arc::new(GitHttpApi::new( - gateway, - "canopy".into(), - "preparation", - &format!("http://{address}"), - Arc::new(|| true), - )?); - let (stop, stopped) = oneshot::channel::<()>(); - let serving = tokio::spawn(async move { - axum::serve(listener, support::git_router(api)) - .with_graceful_shutdown(async move { - let _ = stopped.await; - }) - .await - }); - let url = format!("http://{address}/canopy/preparation.git"); - let reference = format!("refs/heads/準備-{kind}"); - let sibling = format!("refs/heads/preparation-sibling-{kind}"); - let capabilities = match kind { - "sideband" => "report-status-v2 side-band-64k", - "no-report" => "", - _ => "report-status", - }; - let mut body = packet(&format!( - "{} {} {reference}\0{capabilities}\n", - "0".repeat(40), - hex::encode(current) - )); - body.extend(packet(&format!( - "{} {} {sibling}\n", - "0".repeat(40), - hex::encode(current) - ))); - body.extend_from_slice(b"0000"); - body.extend_from_slice(&pack); - let id = uuid::Uuid::new_v4().to_string(); - let client = reqwest::Client::new(); - let post = |id: &str| { - client - .post(format!("{url}/git-receive-pack")) - .bearer_auth("late-writer-token") - .header("Content-Type", "application/x-git-receive-pack-request") - .header("Idempotency-Key", id) - .body(body.clone()) - }; - let generation = repository.default_branch(None).await?.output.generation; - store.read_armed.store(true, Ordering::SeqCst); - let child = if kind == "stock" { - Some( - Command::new("git") - .current_dir(source) - .args([ - "-c", - "credential.helper=", - "-c", - AUTH, - "push", - "--atomic", - &url, - &format!("HEAD:{reference}"), - &format!("HEAD:{sibling}"), - ]) - .env("GIT_TERMINAL_PROMPT", "0") - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .kill_on_drop(true) - .spawn()?, - ) - } else { - None - }; - let request = if child.is_none() { - Some(tokio::spawn(post(&id).send())) - } else { - None - }; - tokio::time::timeout(Duration::from_secs(15), store.entered.notified()).await?; - let winner = if kind == "race" { - Some(gateway_request(root, repository, store.clone(), &body, &id).await?) - } else { - None - }; - store.fail.store(true, Ordering::SeqCst); - store.proceed.notify_one(); - if let Some(child) = child { - let output = child.wait_with_output().await?; - let stderr = String::from_utf8_lossy(&output.stderr); - assert!(!output.status.success(), "{stderr}"); - assert!( - stderr.contains("Canopy push failed before publication"), - "{stderr}" - ); - assert!(!stderr.contains("HTTP 500"), "{stderr}"); - } - if let Some(request) = request { - let response = request.await??; - let status = response.status().as_u16(); - assert_eq!(status, if kind == "no-report" { 409 } else { 200 }); - let report = response.bytes().await?; - if let Some((status, saved)) = &winner { - assert_eq!(*status, 200); - assert_eq!(report.as_ref(), saved); - } else { - let text = String::from_utf8_lossy(&report); - assert!( - text.contains("Canopy push failed before publication"), - "{text}" - ); - if kind != "no-report" { - for name in [&reference, &sibling] { - assert!(text.contains(&format!("ng {name} ")), "{text}"); - } - } - } - let replay = gateway_request(root, repository, store.clone(), &body, &id).await?; - assert_eq!(replay, (status, report.to_vec())); - } - if winner.is_none() { - for name in [&reference, &sibling] { - assert!(repository.ref_state(name, None).await?.output.is_none()); - } - assert_eq!( - repository.default_branch(None).await?.output.generation, - generation - ); - post(&uuid::Uuid::new_v4().to_string()) - .send() - .await? - .error_for_status()?; - } - assert_eq!( - repository.default_branch(None).await?.output.generation, - generation + 1 - ); - for name in [&reference, &sibling] { - assert_eq!( - repository - .ref_state(name, None) - .await? - .output - .ok_or("ref")? - .oid, - Some(current) - ); - } - let _ = stop.send(()); - serving.await??; - } - Ok(()) -} - -async fn gateway_request( - root: &Path, - repository: &Arc, - store: Arc, - body: &[u8], - id: &str, -) -> Result<(u16, Vec)> { - let scratch = tempfile::tempdir_in(root)?; - let gateway = GitGateway::new( - Arc::clone(repository), - scratch.path().into(), - store, - DiskBudget::new(1 << 30), - canopy_server::native_resources::NativeResources::default(), - ); - let response = gateway - .handle( - canopy_server::git_http::GitHttpRequest { - method: "POST".into(), - path_info: "/repo.git/git-receive-pack".into(), - query: String::new(), - content_type: Some("application/x-git-receive-pack-request".into()), - gzip: false, - protocol_v2: false, - authenticated: true, - body: Body::from(body.to_vec()), - }, - "late-writer", - Some(uuid::Uuid::parse_str(id)?.into_bytes()), - None, - ) - .await?; - Ok(( - response.status, - axum::body::to_bytes(response.body, 1 << 20).await?.to_vec(), - )) -} diff --git a/crates/canopy-server/tests/smart_http/push_uploads.rs b/crates/canopy-server/tests/smart_http/push_uploads.rs deleted file mode 100644 index a3cc36d8..00000000 --- a/crates/canopy-server/tests/smart_http/push_uploads.rs +++ /dev/null @@ -1,71 +0,0 @@ -use super::*; -use canopy_server::git_http::GitHttpRequest; -use futures_core::Stream; -use std::{ - future::Future, - io, - pin::Pin, - task::{Context, Poll}, - time::Duration, -}; - -struct PausedUpload { - entered: Option>, - release: oneshot::Receiver<()>, -} - -impl Stream for PausedUpload { - type Item = Result; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - if let Some(entered) = self.entered.take() { - let _ = entered.send(()); - } - match Pin::new(&mut self.release).poll(cx) { - Poll::Pending => Poll::Pending, - Poll::Ready(_) => Poll::Ready(None), - } - } -} - -fn push(body: Body) -> GitHttpRequest { - GitHttpRequest { - method: "POST".into(), - path_info: "/repo.git/git-receive-pack".into(), - query: String::new(), - content_type: Some("application/x-git-receive-pack-request".into()), - gzip: false, - protocol_v2: false, - authenticated: true, - body, - } -} - -pub async fn verify(gateway: &Arc) -> Result<(), Box> { - let (entered_tx, entered_rx) = oneshot::channel(); - let (release_tx, release_rx) = oneshot::channel(); - let first_gateway = Arc::clone(gateway); - let stalled = tokio::spawn(async move { - first_gateway - .handle( - push(Body::from_stream(PausedUpload { - entered: Some(entered_tx), - release: release_rx, - })), - "canopy", - None, - None, - ) - .await - }); - tokio::time::timeout(Duration::from_secs(5), entered_rx).await??; - let second = tokio::time::timeout( - Duration::from_secs(5), - gateway.handle(push(Body::from("invalid")), "canopy", None, None), - ) - .await; - let _ = release_tx.send(()); - assert!(stalled.await?.is_err()); - assert!(second?.is_err()); - Ok(()) -} diff --git a/crates/canopy-server/tests/smart_http/ref_snapshots.rs b/crates/canopy-server/tests/smart_http/ref_snapshots.rs deleted file mode 100644 index cfc4f534..00000000 --- a/crates/canopy-server/tests/smart_http/ref_snapshots.rs +++ /dev/null @@ -1,299 +0,0 @@ -use super::*; -use canopy_server::{ - ObjectBatch, ObjectKind, PushPlan, RefExpectation, RefReadError, RefUpdate, StoredObject, - object_id, -}; -use std::collections::BTreeMap; - -const FIRST: &str = "refs/tags/snapshot-000"; -const LAST: &str = "refs/tags/snapshot-299"; - -async fn pair( - repository: &RepositoryCell, - version: i64, - old: Option, - new: Option, -) -> Result<(), Box> { - repository - .finalize_push(support::identity()?, pair_plan(version, old, new)) - .await?; - Ok(()) -} - -fn pair_plan( - version: i64, - old: Option, - new: Option, -) -> PushPlan { - PushPlan { - actor: "canopy".into(), - updates: [FIRST, LAST] - .into_iter() - .map(|name| RefUpdate { - name: name.into(), - expected: Some(RefExpectation { oid: old, version }), - new_oid: new, - }) - .collect(), - } -} - -pub async fn verify( - repository: &RepositoryCell, - client: &reqwest::Client, - url: &str, -) -> Result<(), Box> { - let initial = repository.refs_page("", None).await?.output.generation; - let mut batch = ObjectBatch::default(); - let a = object_id( - canopy_server::ObjectFormat::Sha1, - ObjectKind::Blob, - b"snapshot a", - ); - let b = object_id( - canopy_server::ObjectFormat::Sha1, - ObjectKind::Blob, - b"snapshot b", - ); - for (oid, body) in [(a, b"snapshot a"), (b, b"snapshot b")] { - batch - .try_push(StoredObject { - oid, - kind: ObjectKind::Blob, - storage: ObjectStorage::Inline(body.to_vec()), - }) - .map_err(|_| "fixture object exceeds batch limit")?; - } - repository.put_objects(support::identity()?, batch).await?; - assert_eq!( - repository.refs_page("", None).await?.output.generation, - initial - ); - for start in (0..300).step_by(64) { - repository - .finalize_push( - support::identity()?, - PushPlan { - actor: "canopy".into(), - updates: (start..(start + 64).min(300)) - .map(|index| RefUpdate { - name: format!("refs/tags/snapshot-{index:03}"), - expected: None, - new_oid: Some(a), - }) - .collect(), - }, - ) - .await?; - } - let first = repository.refs_page("", None).await?.output; - assert_eq!(first.generation, initial + 5); - assert_eq!(first.refs.len(), 256); - let cursor = first.refs.last().ok_or("missing first page")?.0.clone(); - assert!(repository.refs_page(&cursor, None).await.is_err()); - pair(repository, 1, Some(a), Some(b)).await?; - for after in [&cursor, "zzzz"] { - assert!(matches!( - repository.refs_page(after, Some(first.generation)).await, - Err(RefReadError::Changed) - )); - } - let current = repository.refs_page("", None).await?.output; - assert_eq!(current.generation, first.generation + 1); - let mut refs: BTreeMap<_, _> = current.refs.into_iter().collect(); - let second = repository - .refs_page(&cursor, Some(current.generation)) - .await? - .output; - assert_eq!(second.generation, current.generation); - refs.extend(second.refs); - assert_eq!(refs[FIRST].oid, Some(b)); - assert_eq!(refs[LAST].oid, Some(b)); - assert_eq!( - refs.keys() - .filter(|name| name.starts_with("refs/tags/snapshot-")) - .count(), - 300 - ); - assert!( - repository - .refs_page("zzzz", Some(current.generation)) - .await? - .output - .refs - .is_empty() - ); - - // Returning to the same OIDs, and deleting/recreating names, must both - // invalidate old pages even when their ref tips appear unchanged. - pair(repository, 2, Some(b), Some(a)).await?; - pair(repository, 3, Some(a), Some(b)).await?; - assert!(matches!( - repository - .refs_page(&cursor, Some(current.generation)) - .await, - Err(RefReadError::Changed) - )); - pair(repository, 4, Some(b), None).await?; - let deleted = repository.refs_page("", None).await?.output; - assert!( - deleted - .refs - .iter() - .any(|(name, state)| name == FIRST && state.oid.is_none()) - ); - pair(repository, 5, None, Some(b)).await?; - assert!(matches!( - repository - .refs_page(&cursor, Some(deleted.generation)) - .await, - Err(RefReadError::Changed) - )); - let generation = repository.refs_page("", None).await?.output.generation; - assert!( - repository - .finalize_push(support::identity()?, pair_plan(5, Some(b), Some(a))) - .await - .is_err() - ); - assert_eq!( - repository.refs_page("", None).await?.output.generation, - generation - ); - let identity = support::identity()?; - let plan = pair_plan(6, Some(b), Some(a)); - repository.finalize_push(identity, plan.clone()).await?; - repository.finalize_push(identity, plan).await?; - assert_eq!( - repository.refs_page("", None).await?.output.generation, - generation + 1 - ); - - // Both ends of the ordered scan move in one transaction while real HTTP - // advertisements repeatedly acquire, hydrate and replace cache generations. - let writer = async { - let mut old = a; - for version in 7..39 { - let new = if old == a { b } else { a }; - pair(repository, version, Some(old), Some(new)).await?; - old = new; - } - Ok::<_, Box>(()) - }; - let reader = async { - let mut successes = 0; - let mut busy = 0; - for _ in 0..32 { - let response = client - .get(format!("{url}/info/refs?service=git-upload-pack")) - .bearer_auth("local-test-token") - .send() - .await?; - if response.status() == reqwest::StatusCode::SERVICE_UNAVAILABLE { - assert_eq!( - response.text().await?, - "Repository refs are changing; retry the request" - ); - busy += 1; - continue; - } - let bytes = response.error_for_status()?.bytes().await?; - let advertised = advertisement_refs(&bytes)?; - assert_eq!(advertised[FIRST], advertised[LAST]); - assert_eq!( - advertised - .keys() - .filter(|name| name.starts_with("refs/tags/snapshot-")) - .count(), - 300 - ); - successes += 1; - } - assert!(successes > 0); - eprintln!("snapshot advertisements: {successes} consistent, {busy} retriable"); - Ok::<_, Box>(()) - }; - tokio::try_join!(writer, reader)?; - let listing = run_git( - None, - &[ - "-c", - "http.extraHeader=Authorization: Bearer local-test-token", - "ls-remote", - url, - "refs/tags/snapshot-*", - ], - ) - .await?; - assert_eq!(std::str::from_utf8(&listing)?.lines().count(), 300); - - // Git compresses sufficiently large fetch requests. The CGI contract must - // preserve their encoding while retaining the encoded bytes in the spool. - use std::io::Write; - let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); - encoder.write_all(b"0014command=ls-refs\n00010009peel\n000csymrefs\n0000")?; - let body = encoder.finish()?; - for encoding in ["gzip", "x-gzip", "GZIP"] { - let response = client - .post(format!("{url}/git-upload-pack")) - .bearer_auth("local-test-token") - .header("Content-Type", "application/x-git-upload-pack-request") - .header("Content-Encoding", encoding) - .header("Git-Protocol", "version=2") - .body(body.clone()) - .send() - .await? - .error_for_status()? - .bytes() - .await?; - assert_eq!( - advertisement_refs(&response)? - .keys() - .filter(|name| name.starts_with("refs/tags/snapshot-")) - .count(), - 300 - ); - } - for encoding in ["br", "gzip, identity"] { - let response = client - .post(format!("{url}/git-upload-pack")) - .bearer_auth("local-test-token") - .header("Content-Type", "application/x-git-upload-pack-request") - .header("Content-Encoding", encoding) - .body(body.clone()) - .send() - .await?; - assert_eq!( - response.status(), - reqwest::StatusCode::UNSUPPORTED_MEDIA_TYPE - ); - } - Ok(()) -} - -fn advertisement_refs( - mut bytes: &[u8], -) -> Result, Box> { - let mut refs = BTreeMap::new(); - while !bytes.is_empty() { - let header = bytes.get(..4).ok_or("short packet header")?; - let length = usize::from_str_radix(std::str::from_utf8(header)?, 16)?; - if length == 0 { - bytes = &bytes[4..]; - continue; - } - let packet = std::str::from_utf8(bytes.get(4..length).ok_or("short packet")?)?; - bytes = &bytes[length..]; - if let Some((oid, name)) = packet.split_once(' ') - && oid.len() == 40 - { - let name = name - .split('\0') - .next() - .ok_or("missing ref name")? - .trim_end(); - refs.insert(name.into(), oid.into()); - } - } - Ok(refs) -} diff --git a/crates/canopy-server/tests/support/mod.rs b/crates/canopy-server/tests/support/mod.rs deleted file mode 100644 index 7f8c170a..00000000 --- a/crates/canopy-server/tests/support/mod.rs +++ /dev/null @@ -1,53 +0,0 @@ -use std::{ - sync::Arc, - time::{SystemTime, UNIX_EPOCH}, -}; - -use axum::{ - Router, - body::Body, - http::{Request, StatusCode, header}, - middleware::{self, Next}, - response::{IntoResponse, Response}, -}; -use canopy_server::{ - directory::{Principal, TokenScope}, - http::{GitHttpApi, Viewer}, -}; -use cellule_runtime::{MutationIdentity, identity::RequestId}; - -pub fn git_router(api: Arc) -> Router { - api.router().layer(middleware::from_fn(test_identity)) -} - -async fn test_identity(mut request: Request, next: Next) -> Response { - let account = request - .headers() - .get(header::AUTHORIZATION) - .and_then(|value| value.to_str().ok()) - .and_then(|value| match value { - "Bearer local-test-token" => Some("canopy"), - "Bearer late-writer-token" => Some("late-writer"), - _ => None, - }); - let Some(account) = account else { - return StatusCode::UNAUTHORIZED.into_response(); - }; - request - .extensions_mut() - .insert(Viewer::Authenticated(Principal { - token_id: [1; 16], - account: account.into(), - scope: TokenScope::Admin, - })); - next.run(request).await -} - -pub fn identity() -> Result> { - let now_ms = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; - Ok(MutationIdentity { - request_id: RequestId::from_bytes(uuid::Uuid::new_v4().into_bytes()), - issued_at_ms: now_ms, - expires_at_ms: now_ms + 60_000, - }) -} diff --git a/crates/canopy-server/tests/support/objects.rs b/crates/canopy-server/tests/support/objects.rs deleted file mode 100644 index aab80c65..00000000 --- a/crates/canopy-server/tests/support/objects.rs +++ /dev/null @@ -1,26 +0,0 @@ -use canopy_server::{ - ObjectBatch, ObjectKind, ObjectStorage, RepositoryCell, StoredObject, object_id, -}; -use cellule_runtime::{Committed, MutationIdentity}; - -pub async fn put( - repository: &RepositoryCell, - identity: MutationIdentity, - kind: ObjectKind, - body: &[u8], -) -> Result, Box> { - let oid = object_id(repository.object_format(), kind, body); - let mut batch = ObjectBatch::default(); - batch - .try_push(StoredObject { - oid, - kind, - storage: ObjectStorage::Inline(body.to_vec()), - }) - .map_err(|_| "fixture object exceeds batch bounds")?; - let result = repository.put_objects(identity, batch).await?; - Ok(Committed { - output: oid, - receipt: result.receipt, - }) -} diff --git a/crates/canopy-server/tests/support/retained_directory.rs b/crates/canopy-server/tests/support/retained_directory.rs deleted file mode 100644 index b83d588a..00000000 --- a/crates/canopy-server/tests/support/retained_directory.rs +++ /dev/null @@ -1,32 +0,0 @@ -//! Owned fixture for a Directory-only predecessor release. Other module -//! identities stay current: the packed Repository module has a hard cutover. - -use cellule_runtime::Registry; - -type Result = std::result::Result>; - -pub fn predecessor(registry: &Registry, historical: &[u8]) -> Result> { - let historical: serde_json::Value = serde_json::from_slice(historical)?; - let old_directory = historical["modules"] - .as_array() - .ok_or("historical modules missing")? - .iter() - .find(|module| module["name"] == "directory") - .ok_or("historical Directory missing")?; - let mut selected: serde_json::Value = serde_json::from_slice(registry.release_bytes())?; - let modules = selected["modules"] - .as_array_mut() - .ok_or("current modules missing")?; - let directory = modules - .iter_mut() - .find(|module| module["name"] == "directory") - .ok_or("current Directory missing")?; - assert_ne!(directory["code"], old_directory["code"]); - *directory = old_directory.clone(); - // Preserve the exact historical Directory contract, including its code, - // migration and operation bounds. No old Repository code is selected. - selected["build"]["source_revision"] = "owned-directory-predecessor-fixture".into(); - let encoded = serde_json::to_vec(&selected)?; - registry.verify_rolling_from(&encoded)?; - Ok(encoded) -} diff --git a/docs/benchmarks/2026-10-13-rustfs-browse.json b/docs/benchmarks/2026-10-13-rustfs-browse.json new file mode 100644 index 00000000..613d6f5e --- /dev/null +++ b/docs/benchmarks/2026-10-13-rustfs-browse.json @@ -0,0 +1,1622 @@ +{ + "date": "2026-10-13", + "timezone": "America/Vancouver", + "environment": { + "cpu": "Apple M2 Max", + "logical_cpus": 12, + "memory_bytes": 34359738368, + "os": "macOS Darwin 25.5.0 arm64", + "dedicated_host": false, + "rustc": "rustc 1.97.0 (2d8144b78 2026-07-07)", + "git": "git version 2.50.1 (Apple Git-155)", + "cellule_version": "0.2.0", + "head": "9c5f1d1bf837fdc2e38ee229aa409712b9579f69", + "dirty_worktree": true, + "build_profile": "release", + "binary_sha256": "7292067d1a34a796066f32f07e2cba84c69ac619cae64d16c8f7e670ac0095c1", + "cargo_lock_sha256": "1e400d63d86f6f8d364de72e669be1d4db7c183a4e079d419a859bb46e385366", + "transport": "HTTP/1.1 loopback, authenticated single account, no reverse proxy/TLS", + "provider": { + "container": "canopy-api-bench-20261013", + "volume": "canopy-api-bench-20261013-data", + "image": "rustfs/rustfs:1.0.0-beta.8-glibc", + "endpoint": "http://127.0.0.1:32793", + "bucket": "canopy-benchmark", + "cpu_limit": 4, + "memory_limit_gib": 8, + "image_id": "sha256:040304b66e029a5cde4bed140b41513e925909839a9b912a40a98340610d1f66" + }, + "admission": { + "node_active_transfers": 8, + "account_active_transfers": 4, + "account_waiters": 4, + "maximum_wait_seconds": 1 + }, + "max_active_repositories": 8 + }, + "corpus": { + "repositories": [ + { + "name": "bat", + "source_url": "https://github.com/sharkdp/bat.git", + "branch": "master", + "provider_pull_refs_excluded": 2577, + "source_stats": "count: 0\nsize: 0\nin-pack: 20952\npacks: 1\nsize-pack: 35959\nprune-packable: 0\ngarbage: 0\nsize-garbage: 0\n", + "commit_count": 4068, + "object_count": 20952, + "commit": "d9559c69f541873f8c47ac8ad81d985f4441edf6", + "pack_bytes": 36234681, + "object_format": "sha1", + "owner": "canopy", + "repository_id": "c26304a7-f3e4-433a-9332-77ffcbb26cbd", + "clone_url": "http://127.0.0.1:60523/canopy/bat.git", + "import": { + "seconds": 28.921057333005592, + "exit_code": 0, + "objects_per_second": 724.454841285797, + "source_pack_mib_per_second": 1.1948416766020888 + } + }, + { + "name": "starship", + "source_url": "https://github.com/starship/starship.git", + "branch": "main", + "provider_pull_refs_excluded": 4513, + "source_stats": "count: 0\nsize: 0\nin-pack: 38333\npacks: 1\nsize-pack: 48427\nprune-packable: 0\ngarbage: 0\nsize-garbage: 0\n", + "commit_count": 4556, + "object_count": 38333, + "commit": "df688000684138943610bbaa2a6c454db6987d48", + "pack_bytes": 48515539, + "object_format": "sha1", + "owner": "canopy", + "repository_id": "f82d4523-84d1-4ab4-8099-743457d303ae", + "clone_url": "http://127.0.0.1:60523/canopy/starship.git", + "import": { + "seconds": 79.99967504100641, + "exit_code": 0, + "objects_per_second": 479.16444635995316, + "source_pack_mib_per_second": 0.5783526428974817 + } + }, + { + "name": "libgit2", + "source_url": "https://github.com/libgit2/libgit2.git", + "branch": "main", + "provider_pull_refs_excluded": 6804, + "source_stats": "count: 0\nsize: 0\nin-pack: 134967\npacks: 1\nsize-pack: 77541\nprune-packable: 0\ngarbage: 0\nsize-garbage: 0\n", + "commit_count": 19565, + "object_count": 134967, + "commit": "4af47f040cb4d80c75f37f626f511843e0dd8d44", + "pack_bytes": 75622259, + "object_format": "sha1", + "owner": "canopy", + "repository_id": "beb16211-3d72-4397-94e6-09639ba05503", + "clone_url": "http://127.0.0.1:60523/canopy/libgit2.git", + "import": { + "seconds": 813.6219757079962, + "exit_code": 0, + "objects_per_second": 165.8841624607726, + "source_pack_mib_per_second": 0.08863945211677081 + } + } + ] + }, + "verification": { + "repositories": [ + { + "repository": "bat", + "refs_verified": 66, + "clone_seconds": 19.773580291017424, + "clone_fsck_passed": true, + "clone_objects_verified": 20952, + "commit_metadata_and_attribution_verified": 128, + "files": [ + { + "path": "tests/snapshots/output/grid.snapshot.txt", + "bytes": 1025, + "oid": "7061949760a95e1bdacff061c222e7600523c038" + }, + { + "path": "tests/examples/longline.json", + "bytes": 16450, + "oid": "0e11bf142d38256d157782dc928f31dbdc2d6c64" + }, + { + "path": "tests/integration_tests.rs", + "bytes": 153394, + "oid": "9463be375f3c0d2f2e9aebbb5adea07b0d350924" + } + ], + "attribution_query_status": 422 + }, + { + "repository": "starship", + "refs_verified": 164, + "clone_seconds": 48.21346141700633, + "clone_fsck_passed": true, + "clone_objects_verified": 38333, + "commit_metadata_and_attribution_verified": 128, + "files": [ + { + "path": "install/macos_packages/pkg_resources/English.lproj/license.html", + "bytes": 1024, + "oid": "f446f65e4956895ede297314924caf390d3f4f17" + }, + { + "path": ".github/workflows/release.yml", + "bytes": 16378, + "oid": "1b39bb0accbd8be9259e952df20dd059c21b2282" + }, + { + "path": "media/icon.png", + "bytes": 129048, + "oid": "aaf79dc967943414d67f9ff00f7b06775a86f320" + } + ], + "attribution_query_status": 422 + }, + { + "repository": "libgit2", + "refs_verified": 271, + "clone_seconds": 214.87667675002012, + "clone_fsck_passed": true, + "clone_objects_verified": 134967, + "commit_metadata_and_attribution_verified": 128, + "files": [ + { + "path": "tests/libgit2/object/raw/chars.c", + "bytes": 1025, + "oid": "24d630d38614cfa2c97937f7bf896aa478925b40" + }, + { + "path": "deps/pcre2/pcre2_compile.h", + "bytes": 16360, + "oid": "331ca3ad3965ea7d1c67222c736eaabe8830ccc0" + }, + { + "path": "tests/resources/gitgit.index", + "bytes": 134799, + "oid": "215da649e1c68079fb03f4f9bc0f196cca9855c8" + } + ], + "attribution_query_status": 422 + } + ], + "attribution_api": "not implemented; browse kind rejected with 422" + }, + "large_file_preview_limits": [ + { + "repository": "bat", + "path": "assets/syntaxes.bin", + "bytes": 1012454, + "content_status": "too_large", + "status": 200 + }, + { + "repository": "starship", + "path": "media/demo.gif", + "bytes": 2469588, + "content_status": "too_large", + "status": 200 + }, + { + "repository": "libgit2", + "path": "benchmarks/cli/_script/flamegraph/example-dtrace-stacks.txt", + "bytes": 1382457, + "content_status": "too_large", + "status": 200 + } + ], + "warm": { + "method": "closed-loop, persistent HTTP/1.1 connections, full response read, no retries", + "mixed_workload": "equal rotation over one query for each repository/API pair; medium sampled blob", + "duration_per_repeat_seconds": 8, + "repeats": 2, + "rows": [ + { + "repository": "all", + "operation": "mixed", + "concurrency": 1, + "seconds": 16.00467266701162, + "requests": 8787, + "outcomes": { + "ok": 8787 + }, + "latency_ms_all_attempts": { + "p50": 1.0325420298613608, + "p95": 4.82308299979195, + "p99": 6.397457967977971, + "min": 0.5297499592415988, + "max": 92.74037496652454, + "mean": 1.8120717856468307 + }, + "latency_ms_successful": { + "p50": 1.0325420298613608, + "p95": 4.82308299979195, + "p99": 6.397457967977971, + "min": 0.5297499592415988, + "max": 92.74037496652454, + "mean": 1.8120717856468307 + }, + "successful_requests_per_second": 549.0271611809667, + "successful_response_mib_per_second": 8.169318987425775, + "error_percent": 0.0 + }, + { + "repository": "all", + "operation": "mixed", + "concurrency": 4, + "seconds": 16.015075459028594, + "requests": 14714, + "outcomes": { + "ok": 14714 + }, + "latency_ms_all_attempts": { + "p50": 2.1343749831430614, + "p95": 11.538084014318883, + "p99": 15.04329196177423, + "min": 0.8282500202767551, + "max": 88.08912499807775, + "mean": 4.336179986815298 + }, + "latency_ms_successful": { + "p50": 2.1343749831430614, + "p95": 11.538084014318883, + "p99": 15.04329196177423, + "min": 0.8282500202767551, + "max": 88.08912499807775, + "mean": 4.336179986815298 + }, + "successful_requests_per_second": 918.7593300851352, + "successful_response_mib_per_second": 13.670334790148779, + "error_percent": 0.0 + }, + { + "repository": "all", + "operation": "mixed", + "concurrency": 16, + "seconds": 16.030485082999803, + "requests": 145990, + "outcomes": { + "http_503_or_invalid_body": 122815, + "ok": 10565, + "BrokenPipeError": 12458, + "ConnectionResetError": 66, + "RemoteDisconnected": 86 + }, + "latency_ms_all_attempts": { + "p50": 0.8549580234102905, + "p95": 8.482416975311935, + "p99": 21.860542008653283, + "min": 0.02929195761680603, + "max": 47.4909589975141, + "mean": 1.7490142179625363 + }, + "latency_ms_successful": { + "p50": 10.354708007071167, + "p95": 27.166916988790035, + "p99": 32.0184999727644, + "min": 2.8825420304201543, + "max": 47.4909589975141, + "mean": 13.002402145956355 + }, + "successful_requests_per_second": 659.0567874458206, + "successful_response_mib_per_second": 9.770577278489611, + "error_percent": 92.76320295910679 + }, + { + "repository": "bat", + "operation": "blob", + "concurrency": 1, + "seconds": 16.00199016695842, + "requests": 13848, + "outcomes": { + "ok": 13848 + }, + "latency_ms_all_attempts": { + "p50": 1.101000001654029, + "p95": 1.4443330001085997, + "p99": 1.646874996367842, + "min": 0.7540420047007501, + "max": 6.462041987106204, + "mean": 1.12150191992839 + }, + "latency_ms_successful": { + "p50": 1.101000001654029, + "p95": 1.4443330001085997, + "p99": 1.646874996367842, + "min": 0.7540420047007501, + "max": 6.462041987106204, + "mean": 1.12150191992839 + }, + "successful_requests_per_second": 865.3923577952154, + "successful_response_mib_per_second": 62.95434571926579, + "error_percent": 0.0 + }, + { + "repository": "bat", + "operation": "blob", + "concurrency": 4, + "seconds": 16.004783916985616, + "requests": 28449, + "outcomes": { + "ok": 28449 + }, + "latency_ms_all_attempts": { + "p50": 2.164374978747219, + "p95": 2.7350829914212227, + "p99": 3.00733296899125, + "min": 1.1347500258125365, + "max": 55.061583989299834, + "mean": 2.2109012953796157 + }, + "latency_ms_successful": { + "p50": 2.164374978747219, + "p95": 2.7350829914212227, + "p99": 3.00733296899125, + "min": 1.1347500258125365, + "max": 55.061583989299834, + "mean": 2.2109012953796157 + }, + "successful_requests_per_second": 1777.531027445334, + "successful_response_mib_per_second": 129.33296811357653, + "error_percent": 0.0 + }, + { + "repository": "bat", + "operation": "blob", + "concurrency": 16, + "seconds": 16.01165054203011, + "requests": 146284, + "outcomes": { + "ok": 19061, + "http_503_or_invalid_body": 87036, + "BrokenPipeError": 39775, + "RemoteDisconnected": 217, + "ConnectionResetError": 195 + }, + "latency_ms_all_attempts": { + "p50": 0.7972500170581043, + "p95": 7.779459003359079, + "p99": 9.129625046625733, + "min": 0.026124995201826096, + "max": 26.476458995603025, + "mean": 1.7295034789342059 + }, + "latency_ms_successful": { + "p50": 7.500582956708968, + "p95": 9.475333034060895, + "p99": 11.623708007391542, + "min": 3.350542043335736, + "max": 26.476458995603025, + "mean": 7.679052330335933 + }, + "successful_requests_per_second": 1190.4456664205502, + "successful_response_mib_per_second": 87.55830221538214, + "error_percent": 86.96986683437696 + }, + { + "repository": "bat", + "operation": "commits", + "concurrency": 1, + "seconds": 16.017477457993664, + "requests": 3501, + "outcomes": { + "ok": 3501 + }, + "latency_ms_all_attempts": { + "p50": 4.089916998054832, + "p95": 5.8344590361230075, + "p99": 9.358957991935313, + "min": 2.862958994228393, + "max": 28.306666994467378, + "mean": 4.558941061042278 + }, + "latency_ms_successful": { + "p50": 4.089916998054832, + "p95": 5.8344590361230075, + "p99": 9.358957991935313, + "min": 2.862958994228393, + "max": 28.306666994467378, + "mean": 4.558941061042278 + }, + "successful_requests_per_second": 218.57374291185874, + "successful_response_mib_per_second": 4.370237237001032, + "error_percent": 0.0 + }, + { + "repository": "bat", + "operation": "commits", + "concurrency": 4, + "seconds": 16.006958667945582, + "requests": 5430, + "outcomes": { + "ok": 5430 + }, + "latency_ms_all_attempts": { + "p50": 10.898125008679926, + "p95": 15.361499972641468, + "p99": 15.846750000491738, + "min": 8.433957991655916, + "max": 21.503625030163676, + "mean": 11.774078276888236 + }, + "latency_ms_successful": { + "p50": 10.898125008679926, + "p95": 15.361499972641468, + "p99": 15.846750000491738, + "min": 8.433957991655916, + "max": 21.503625030163676, + "mean": 11.774078276888236 + }, + "successful_requests_per_second": 339.2274642948719, + "successful_response_mib_per_second": 6.7831833481177455, + "error_percent": 0.0 + }, + { + "repository": "bat", + "operation": "commits", + "concurrency": 16, + "seconds": 16.05331412499072, + "requests": 151688, + "outcomes": { + "ok": 2867, + "http_503_or_invalid_body": 108631, + "BrokenPipeError": 39776, + "ConnectionResetError": 189, + "RemoteDisconnected": 225 + }, + "latency_ms_all_attempts": { + "p50": 0.7049579871818423, + "p95": 2.1658329642377794, + "p99": 42.14174998924136, + "min": 0.022416003048419952, + "max": 128.7855840055272, + "mean": 1.6847534488501597 + }, + "latency_ms_successful": { + "p50": 42.33108297921717, + "p95": 67.43995798751712, + "p99": 99.09195901127532, + "min": 20.263166981749237, + "max": 128.7855840055272, + "mean": 45.66223086149068 + }, + "successful_requests_per_second": 178.59240638273235, + "successful_response_mib_per_second": 3.5661834712808487, + "error_percent": 98.10993618480038 + }, + { + "repository": "bat", + "operation": "refs", + "concurrency": 1, + "seconds": 16.014794999966398, + "requests": 18622, + "outcomes": { + "ok": 18622 + }, + "latency_ms_all_attempts": { + "p50": 0.8425840060226619, + "p95": 0.9940830059349537, + "p99": 1.0678750113584101, + "min": 0.6624590023420751, + "max": 5.0185840227641165, + "mean": 0.8540398355497097 + }, + "latency_ms_successful": { + "p50": 0.8425840060226619, + "p95": 0.9940830059349537, + "p99": 1.0678750113584101, + "min": 0.6624590023420751, + "max": 5.0185840227641165, + "mean": 0.8540398355497097 + }, + "successful_requests_per_second": 1162.7997735868034, + "successful_response_mib_per_second": 7.57622533144478, + "error_percent": 0.0 + }, + { + "repository": "bat", + "operation": "refs", + "concurrency": 4, + "seconds": 16.01462700002594, + "requests": 35073, + "outcomes": { + "ok": 35073 + }, + "latency_ms_all_attempts": { + "p50": 1.7980000120587647, + "p95": 2.1177500020712614, + "p99": 2.2350000217556953, + "min": 1.0270840139128268, + "max": 10.337167012039572, + "mean": 1.8192831381431933 + }, + "latency_ms_successful": { + "p50": 1.7980000120587647, + "p95": 2.1177500020712614, + "p99": 2.2350000217556953, + "min": 1.0270840139128268, + "max": 10.337167012039572, + "mean": 1.8192831381431933 + }, + "successful_requests_per_second": 2190.0603741781306, + "successful_response_mib_per_second": 14.269344784150112, + "error_percent": 0.0 + }, + { + "repository": "bat", + "operation": "refs", + "concurrency": 16, + "seconds": 16.02138833294157, + "requests": 158165, + "outcomes": { + "ok": 20054, + "http_503_or_invalid_body": 124862, + "BrokenPipeError": 13093, + "RemoteDisconnected": 105, + "ConnectionResetError": 51 + }, + "latency_ms_all_attempts": { + "p50": 0.799959001597017, + "p95": 7.2485419805161655, + "p99": 7.970708014909178, + "min": 0.029250048100948334, + "max": 17.770208010915667, + "mean": 1.6139088356820033 + }, + "latency_ms_successful": { + "p50": 7.107457960955799, + "p95": 8.147875021677464, + "p99": 8.82079201983288, + "min": 2.2863749763928354, + "max": 17.770208010915667, + "mean": 7.126413922704734 + }, + "successful_requests_per_second": 1251.7017616236778, + "successful_response_mib_per_second": 8.155466494954076, + "error_percent": 87.32083583599406 + }, + { + "repository": "bat", + "operation": "tree", + "concurrency": 1, + "seconds": 16.015277875005268, + "requests": 18057, + "outcomes": { + "ok": 18057 + }, + "latency_ms_all_attempts": { + "p50": 0.8279170142486691, + "p95": 1.1947079910896719, + "p99": 1.6283750301226974, + "min": 0.5131660145707428, + "max": 34.24637496937066, + "mean": 0.8809236574188942 + }, + "latency_ms_successful": { + "p50": 0.8279170142486691, + "p95": 1.1947079910896719, + "p99": 1.6283750301226974, + "min": 0.5131660145707428, + "max": 34.24637496937066, + "mean": 0.8809236574188942 + }, + "successful_requests_per_second": 1127.4859007086732, + "successful_response_mib_per_second": 6.566964872894387, + "error_percent": 0.0 + }, + { + "repository": "bat", + "operation": "tree", + "concurrency": 4, + "seconds": 16.027722750033718, + "requests": 37382, + "outcomes": { + "ok": 37382 + }, + "latency_ms_all_attempts": { + "p50": 1.640458998735994, + "p95": 2.4475829559378326, + "p99": 3.062208998017013, + "min": 0.7026249659247696, + "max": 19.468375016003847, + "mean": 1.7061047552683481 + }, + "latency_ms_successful": { + "p50": 1.640458998735994, + "p95": 2.4475829559378326, + "p99": 3.062208998017013, + "min": 0.7026249659247696, + "max": 19.468375016003847, + "mean": 1.7061047552683481 + }, + "successful_requests_per_second": 2332.3338307634103, + "successful_response_mib_per_second": 13.585427441303326, + "error_percent": 0.0 + }, + { + "repository": "bat", + "operation": "tree", + "concurrency": 16, + "seconds": 16.00996450096136, + "requests": 150745, + "outcomes": { + "ok": 27129, + "http_503_or_invalid_body": 82495, + "BrokenPipeError": 40716, + "ConnectionResetError": 201, + "RemoteDisconnected": 204 + }, + "latency_ms_all_attempts": { + "p50": 0.8531250059604645, + "p95": 6.033874989952892, + "p99": 7.056166010443121, + "min": 0.028458016458898783, + "max": 12.190042005386204, + "mean": 1.6899882608471022 + }, + "latency_ms_successful": { + "p50": 5.48045797040686, + "p95": 7.101957977283746, + "p99": 7.9649160034023225, + "min": 1.79462501546368, + "max": 12.190042005386204, + "mean": 5.547666112374419 + }, + "successful_requests_per_second": 1694.50719259065, + "successful_response_mib_per_second": 9.90357989569809, + "error_percent": 82.00338319678929 + }, + { + "repository": "libgit2", + "operation": "blob", + "concurrency": 1, + "seconds": 16.00327133195242, + "requests": 15154, + "outcomes": { + "ok": 15154 + }, + "latency_ms_all_attempts": { + "p50": 1.0045829694718122, + "p95": 1.2737499782815576, + "p99": 1.4454579795710742, + "min": 0.7080409559421241, + "max": 4.052291973493993, + "mean": 1.0254417736195982 + }, + "latency_ms_successful": { + "p50": 1.0045829694718122, + "p95": 1.2737499782815576, + "p99": 1.4454579795710742, + "min": 0.7080409559421241, + "max": 4.052291973493993, + "mean": 1.0254417736195982 + }, + "successful_requests_per_second": 946.9313920675238, + "successful_response_mib_per_second": 61.389964203748015, + "error_percent": 0.0 + }, + { + "repository": "libgit2", + "operation": "blob", + "concurrency": 4, + "seconds": 16.005381707975175, + "requests": 27930, + "outcomes": { + "ok": 27930 + }, + "latency_ms_all_attempts": { + "p50": 2.2345830220729113, + "p95": 2.8366249753162265, + "p99": 3.283208003267646, + "min": 1.2243750388734043, + "max": 20.5275000189431, + "mean": 2.257089662635542 + }, + "latency_ms_successful": { + "p50": 2.2345830220729113, + "p95": 2.8366249753162265, + "p99": 3.283208003267646, + "min": 1.2243750388734043, + "max": 20.5275000189431, + "mean": 2.257089662635542 + }, + "successful_requests_per_second": 1745.0380446773734, + "successful_response_mib_per_second": 113.14824424794928, + "error_percent": 0.0 + }, + { + "repository": "libgit2", + "operation": "blob", + "concurrency": 16, + "seconds": 16.012136042059865, + "requests": 152697, + "outcomes": { + "ok": 15933, + "http_503_or_invalid_body": 93457, + "BrokenPipeError": 42830, + "RemoteDisconnected": 253, + "ConnectionResetError": 224 + }, + "latency_ms_all_attempts": { + "p50": 0.7473749574273825, + "p95": 8.710167021490633, + "p99": 10.828375001437962, + "min": 0.026999972760677338, + "max": 41.841500031296164, + "mean": 1.6610423994319796 + }, + "latency_ms_successful": { + "p50": 8.64333298522979, + "p95": 11.448832985479385, + "p99": 13.542667031288147, + "min": 3.043250006157905, + "max": 41.841500031296164, + "mean": 8.961353011451289 + }, + "successful_requests_per_second": 995.0577460838458, + "successful_response_mib_per_second": 64.75569542375841, + "error_percent": 89.56561032633255 + }, + { + "repository": "libgit2", + "operation": "commits", + "concurrency": 1, + "seconds": 16.02726750000147, + "requests": 3940, + "outcomes": { + "ok": 3940 + }, + "latency_ms_all_attempts": { + "p50": 3.8279169821180403, + "p95": 5.187375005334616, + "p99": 6.551624974235892, + "min": 2.923749969340861, + "max": 50.19733402878046, + "mean": 4.05026076800015 + }, + "latency_ms_successful": { + "p50": 3.8279169821180403, + "p95": 5.187375005334616, + "p99": 6.551624974235892, + "min": 2.923749969340861, + "max": 50.19733402878046, + "mean": 4.05026076800015 + }, + "successful_requests_per_second": 245.83105011503915, + "successful_response_mib_per_second": 4.322186698838102, + "error_percent": 0.0 + }, + { + "repository": "libgit2", + "operation": "commits", + "concurrency": 4, + "seconds": 16.184691207949072, + "requests": 4750, + "outcomes": { + "ok": 4750 + }, + "latency_ms_all_attempts": { + "p50": 11.091416992712766, + "p95": 20.288583997171372, + "p99": 55.95716700190678, + "min": 8.686790999490768, + "max": 278.26662501320243, + "mean": 13.5943113943856 + }, + "latency_ms_successful": { + "p50": 11.091416992712766, + "p95": 20.288583997171372, + "p99": 55.95716700190678, + "min": 8.686790999490768, + "max": 278.26662501320243, + "mean": 13.5943113943856 + }, + "successful_requests_per_second": 293.48721819709783, + "successful_response_mib_per_second": 5.160118765610484, + "error_percent": 0.0 + }, + { + "repository": "libgit2", + "operation": "commits", + "concurrency": 16, + "seconds": 16.069224500039127, + "requests": 146005, + "outcomes": { + "ok": 2324, + "http_503_or_invalid_body": 105713, + "BrokenPipeError": 37551, + "RemoteDisconnected": 200, + "ConnectionResetError": 217 + }, + "latency_ms_all_attempts": { + "p50": 0.7667079917155206, + "p95": 2.0916659850627184, + "p99": 43.76049997517839, + "min": 0.026250025257468224, + "max": 373.9662499865517, + "mean": 1.7513654750468928 + }, + "latency_ms_successful": { + "p50": 44.48579205200076, + "p95": 92.43291697930545, + "p99": 145.10820800205693, + "min": 18.669291981495917, + "max": 373.9662499865517, + "mean": 56.13284755673295 + }, + "successful_requests_per_second": 144.62427853903847, + "successful_response_mib_per_second": 2.5428051366668414, + "error_percent": 98.40827368925721 + }, + { + "repository": "libgit2", + "operation": "refs", + "concurrency": 1, + "seconds": 16.01616116700461, + "requests": 10256, + "outcomes": { + "ok": 10256 + }, + "latency_ms_all_attempts": { + "p50": 1.505708962213248, + "p95": 1.7992090433835983, + "p99": 1.9750000210478902, + "min": 1.3214590144343674, + "max": 7.476333004888147, + "mean": 1.5469342338044034 + }, + "latency_ms_successful": { + "p50": 1.505708962213248, + "p95": 1.7992090433835983, + "p99": 1.9750000210478902, + "min": 1.3214590144343674, + "max": 7.476333004888147, + "mean": 1.5469342338044034 + }, + "successful_requests_per_second": 640.353196565523, + "successful_response_mib_per_second": 15.304461917046138, + "error_percent": 0.0 + }, + { + "repository": "libgit2", + "operation": "refs", + "concurrency": 4, + "seconds": 16.005978625034913, + "requests": 18717, + "outcomes": { + "ok": 18717 + }, + "latency_ms_all_attempts": { + "p50": 3.3329169964417815, + "p95": 4.086041997652501, + "p99": 4.763916949741542, + "min": 1.8169160466641188, + "max": 11.757583008147776, + "mean": 3.4061987174350397 + }, + "latency_ms_successful": { + "p50": 3.3329169964417815, + "p95": 4.086041997652501, + "p99": 4.763916949741542, + "min": 1.8169160466641188, + "max": 11.757583008147776, + "mean": 3.4061987174350397 + }, + "successful_requests_per_second": 1169.3755463801997, + "successful_response_mib_per_second": 27.94811302932185, + "error_percent": 0.0 + }, + { + "repository": "libgit2", + "operation": "refs", + "concurrency": 16, + "seconds": 16.015708165999968, + "requests": 155085, + "outcomes": { + "http_503_or_invalid_body": 106425, + "BrokenPipeError": 36914, + "ok": 11365, + "RemoteDisconnected": 201, + "ConnectionResetError": 180 + }, + "latency_ms_all_attempts": { + "p50": 0.741208961699158, + "p95": 11.474333005025983, + "p99": 13.217750005424023, + "min": 0.025707995519042015, + "max": 22.79691700823605, + "mean": 1.6395113912237211 + }, + "latency_ms_successful": { + "p50": 11.879750003572553, + "p95": 14.615916996262968, + "p99": 19.109999993816018, + "min": 3.878708987031132, + "max": 22.79691700823605, + "mean": 12.176306611578461 + }, + "successful_requests_per_second": 709.6158272992861, + "successful_response_mib_per_second": 16.95984101099721, + "error_percent": 92.67176064738692 + }, + { + "repository": "libgit2", + "operation": "tree", + "concurrency": 1, + "seconds": 16.0273771249922, + "requests": 19854, + "outcomes": { + "ok": 19854 + }, + "latency_ms_all_attempts": { + "p50": 0.7934999885037541, + "p95": 0.9943749755620956, + "p99": 1.1671249521896243, + "min": 0.5053749773651361, + "max": 36.6911250166595, + "mean": 0.8013181317481717 + }, + "latency_ms_successful": { + "p50": 0.7934999885037541, + "p95": 0.9943749755620956, + "p99": 1.1671249521896243, + "min": 0.5053749773651361, + "max": 36.6911250166595, + "mean": 0.8013181317481717 + }, + "successful_requests_per_second": 1238.7554024070962, + "successful_response_mib_per_second": 6.556396755168436, + "error_percent": 0.0 + }, + { + "repository": "libgit2", + "operation": "tree", + "concurrency": 4, + "seconds": 16.003774209006224, + "requests": 35592, + "outcomes": { + "ok": 35592 + }, + "latency_ms_all_attempts": { + "p50": 1.7680840101093054, + "p95": 2.4791660252958536, + "p99": 2.884333021938801, + "min": 0.777917040977627, + "max": 46.537958027329296, + "mean": 1.7930966172506462 + }, + "latency_ms_successful": { + "p50": 1.7680840101093054, + "p95": 2.4791660252958536, + "p99": 2.884333021938801, + "min": 0.777917040977627, + "max": 46.537958027329296, + "mean": 1.7930966172506462 + }, + "successful_requests_per_second": 2223.9753907531626, + "successful_response_mib_per_second": 11.77081581752995, + "error_percent": 0.0 + }, + { + "repository": "libgit2", + "operation": "tree", + "concurrency": 16, + "seconds": 16.023573751037475, + "requests": 141247, + "outcomes": { + "ok": 20254, + "http_503_or_invalid_body": 83517, + "RemoteDisconnected": 173, + "BrokenPipeError": 37141, + "ConnectionResetError": 162 + }, + "latency_ms_all_attempts": { + "p50": 0.8657919825054705, + "p95": 7.516624988056719, + "p99": 9.28116700379178, + "min": 0.026124995201826096, + "max": 77.92679202975705, + "mean": 1.806408299506879 + }, + "latency_ms_successful": { + "p50": 7.014915987383574, + "p95": 9.628708998207003, + "p99": 11.940707976464182, + "min": 2.885083027649671, + "max": 77.92679202975705, + "mean": 7.218813678929034 + }, + "successful_requests_per_second": 1264.0126550226423, + "successful_response_mib_per_second": 6.711990129438018, + "error_percent": 85.66058040170765 + }, + { + "repository": "starship", + "operation": "blob", + "concurrency": 1, + "seconds": 16.00264966697432, + "requests": 16234, + "outcomes": { + "ok": 16234 + }, + "latency_ms_all_attempts": { + "p50": 0.9408749756403267, + "p95": 1.1576669639907777, + "p99": 1.299374969676137, + "min": 0.660583027638495, + "max": 3.7677089567296207, + "mean": 0.956252978667922 + }, + "latency_ms_successful": { + "p50": 0.9408749756403267, + "p95": 1.1576669639907777, + "p99": 1.299374969676137, + "min": 0.660583027638495, + "max": 3.7677089567296207, + "mean": 0.956252978667922 + }, + "successful_requests_per_second": 1014.4570016741122, + "successful_response_mib_per_second": 63.30858390772817, + "error_percent": 0.0 + }, + { + "repository": "starship", + "operation": "blob", + "concurrency": 4, + "seconds": 16.019204750016797, + "requests": 26541, + "outcomes": { + "ok": 26541 + }, + "latency_ms_all_attempts": { + "p50": 2.2444999776780605, + "p95": 3.361875016707927, + "p99": 4.2891670018434525, + "min": 1.1972079519182444, + "max": 14.385791961103678, + "mean": 2.3772514884462317 + }, + "latency_ms_successful": { + "p50": 2.2444999776780605, + "p95": 3.361875016707927, + "p99": 4.2891670018434525, + "min": 1.1972079519182444, + "max": 14.385791961103678, + "mean": 2.3772514884462317 + }, + "successful_requests_per_second": 1656.823819545235, + "successful_response_mib_per_second": 103.39808580733056, + "error_percent": 0.0 + }, + { + "repository": "starship", + "operation": "blob", + "concurrency": 16, + "seconds": 16.01056633295957, + "requests": 149181, + "outcomes": { + "ok": 17041, + "http_503_or_invalid_body": 88827, + "BrokenPipeError": 42854, + "RemoteDisconnected": 236, + "ConnectionResetError": 223 + }, + "latency_ms_all_attempts": { + "p50": 0.7764159818179905, + "p95": 8.426666958257556, + "p99": 10.070791991893202, + "min": 0.0272089964710176, + "max": 19.993667025119066, + "mean": 1.6990347348808084 + }, + "latency_ms_successful": { + "p50": 8.269332989584655, + "p95": 10.661875014193356, + "p99": 12.635833001695573, + "min": 3.636666981037706, + "max": 19.993667025119066, + "mean": 8.451818296576976 + }, + "successful_requests_per_second": 1064.3596013789447, + "successful_response_mib_per_second": 66.54636524212845, + "error_percent": 88.57696355434003 + }, + { + "repository": "starship", + "operation": "commits", + "concurrency": 1, + "seconds": 16.004689000023063, + "requests": 3663, + "outcomes": { + "ok": 3663 + }, + "latency_ms_all_attempts": { + "p50": 4.277792002540082, + "p95": 5.101667018607259, + "p99": 6.068875023629516, + "min": 3.3681250060908496, + "max": 11.682791984640062, + "mean": 4.358136191547304 + }, + "latency_ms_successful": { + "p50": 4.277792002540082, + "p95": 5.101667018607259, + "p99": 6.068875023629516, + "min": 3.3681250060908496, + "max": 11.682791984640062, + "mean": 4.358136191547304 + }, + "successful_requests_per_second": 228.87042666025698, + "successful_response_mib_per_second": 4.041330736553808, + "error_percent": 0.0 + }, + { + "repository": "starship", + "operation": "commits", + "concurrency": 4, + "seconds": 16.037280582997482, + "requests": 5826, + "outcomes": { + "ok": 5826 + }, + "latency_ms_all_attempts": { + "p50": 10.686875029932708, + "p95": 12.921207991894335, + "p99": 16.019333037547767, + "min": 8.48954200046137, + "max": 32.082749996334314, + "mean": 10.97801928644032 + }, + "latency_ms_successful": { + "p50": 10.686875029932708, + "p95": 12.921207991894335, + "p99": 16.019333037547767, + "min": 8.48954200046137, + "max": 32.082749996334314, + "mean": 10.97801928644032 + }, + "successful_requests_per_second": 363.27854774684494, + "successful_response_mib_per_second": 6.4148433799877305, + "error_percent": 0.0 + }, + { + "repository": "starship", + "operation": "commits", + "concurrency": 16, + "seconds": 16.044194416026585, + "requests": 155344, + "outcomes": { + "http_503_or_invalid_body": 102128, + "BrokenPipeError": 49763, + "ok": 2908, + "ConnectionResetError": 289, + "RemoteDisconnected": 256 + }, + "latency_ms_all_attempts": { + "p50": 0.6870829965919256, + "p95": 2.0948330056853592, + "p99": 44.14208303205669, + "min": 0.023916014470160007, + "max": 79.94050002889708, + "mean": 1.645229124241228 + }, + "latency_ms_successful": { + "p50": 44.31204200955108, + "p95": 49.60324999410659, + "p99": 68.43279098393396, + "min": 21.835583029314876, + "max": 79.94050002889708, + "mean": 44.99723243675155 + }, + "successful_requests_per_second": 181.24936189349538, + "successful_response_mib_per_second": 3.1978944968471947, + "error_percent": 98.12802554331033 + }, + { + "repository": "starship", + "operation": "refs", + "concurrency": 1, + "seconds": 16.017063624982256, + "requests": 12267, + "outcomes": { + "ok": 12267 + }, + "latency_ms_all_attempts": { + "p50": 1.2107499642297626, + "p95": 1.5799999819137156, + "p99": 3.1397080165334046, + "min": 0.9955419809557498, + "max": 20.694000006187707, + "mean": 1.2947523266364327 + }, + "latency_ms_successful": { + "p50": 1.2107499642297626, + "p95": 1.5799999819137156, + "p99": 3.1397080165334046, + "min": 0.9955419809557498, + "max": 20.694000006187707, + "mean": 1.2947523266364327 + }, + "successful_requests_per_second": 765.8707168314435, + "successful_response_mib_per_second": 11.012839001068597, + "error_percent": 0.0 + }, + { + "repository": "starship", + "operation": "refs", + "concurrency": 4, + "seconds": 16.027744332968723, + "requests": 26701, + "outcomes": { + "ok": 26701 + }, + "latency_ms_all_attempts": { + "p50": 2.362457977142185, + "p95": 2.665541018359363, + "p99": 2.864292007870972, + "min": 1.3547090347856283, + "max": 4.374417010694742, + "mean": 2.388159744118872 + }, + "latency_ms_successful": { + "p50": 2.362457977142185, + "p95": 2.665541018359363, + "p99": 2.864292007870972, + "min": 1.3547090347856283, + "max": 4.374417010694742, + "mean": 2.388159744118872 + }, + "successful_requests_per_second": 1665.9237535425757, + "successful_response_mib_per_second": 23.95515285102363, + "error_percent": 0.0 + }, + { + "repository": "starship", + "operation": "refs", + "concurrency": 16, + "seconds": 16.02671875001397, + "requests": 157651, + "outcomes": { + "ok": 14602, + "http_503_or_invalid_body": 111964, + "BrokenPipeError": 30744, + "RemoteDisconnected": 194, + "ConnectionResetError": 147 + }, + "latency_ms_all_attempts": { + "p50": 0.7643330027349293, + "p95": 9.344458987470716, + "p99": 10.822499985806644, + "min": 0.026958005037158728, + "max": 16.665208037011325, + "mean": 1.613133108635022 + }, + "latency_ms_successful": { + "p50": 9.434541978407651, + "p95": 11.390750005375594, + "p99": 12.63729203492403, + "min": 3.121916961390525, + "max": 16.665208037011325, + "mean": 9.575170548322289 + }, + "successful_requests_per_second": 911.1035282869285, + "successful_response_mib_per_second": 13.101214408407504, + "error_percent": 90.73776886921111 + }, + { + "repository": "starship", + "operation": "tree", + "concurrency": 1, + "seconds": 16.013972209009808, + "requests": 19365, + "outcomes": { + "ok": 19365 + }, + "latency_ms_all_attempts": { + "p50": 0.74954197043553, + "p95": 1.1352500296197832, + "p99": 2.40679201669991, + "min": 0.4905420355498791, + "max": 11.217541992664337, + "mean": 0.8226428006385161 + }, + "latency_ms_successful": { + "p50": 0.74954197043553, + "p95": 1.1352500296197832, + "p99": 2.40679201669991, + "min": 0.4905420355498791, + "max": 11.217541992664337, + "mean": 0.8226428006385161 + }, + "successful_requests_per_second": 1209.256500963879, + "successful_response_mib_per_second": 3.833752690815503, + "error_percent": 0.0 + }, + { + "repository": "starship", + "operation": "tree", + "concurrency": 4, + "seconds": 16.004292124009226, + "requests": 31545, + "outcomes": { + "ok": 31545 + }, + "latency_ms_all_attempts": { + "p50": 1.613333006389439, + "p95": 3.273292037192732, + "p99": 10.248832986690104, + "min": 0.6747079896740615, + "max": 74.0720410249196, + "mean": 2.024663219069402 + }, + "latency_ms_successful": { + "p50": 1.613333006389439, + "p95": 3.273292037192732, + "p99": 10.248832986690104, + "min": 0.6747079896740615, + "max": 74.0720410249196, + "mean": 2.024663219069402 + }, + "successful_requests_per_second": 1971.033754918595, + "successful_response_mib_per_second": 6.2487241553177215, + "error_percent": 0.0 + }, + { + "repository": "starship", + "operation": "tree", + "concurrency": 16, + "seconds": 16.009098416019697, + "requests": 152799, + "outcomes": { + "ok": 24050, + "http_503_or_invalid_body": 85628, + "BrokenPipeError": 42647, + "RemoteDisconnected": 236, + "ConnectionResetError": 238 + }, + "latency_ms_all_attempts": { + "p50": 0.8189580403268337, + "p95": 6.577374995686114, + "p99": 7.767125032842159, + "min": 0.02887495793402195, + "max": 14.533292036503553, + "mean": 1.6718909593842297 + }, + "latency_ms_successful": { + "p50": 6.068458023946732, + "p95": 7.914166955742985, + "p99": 8.801666961517185, + "min": 2.592250006273389, + "max": 14.533292036503553, + "mean": 6.141423780936748 + }, + "successful_requests_per_second": 1502.2707322439894, + "successful_response_mib_per_second": 4.78240448478861, + "error_percent": 84.26036819612693 + } + ], + "overload_outcomes": { + "concurrency": 16, + "attempts": 1962881, + "http_503": 1303498, + "transport_errors": 471230, + "successful": 188153, + "invalid_successful_bodies": 0, + "note": "Single account, closed-loop workload without Retry-After backoff; failed attempts retained separately from successful latency and throughput." + } + }, + "cold": { + "method": "one first authenticated read per repository/API after graceful restart with a new empty Canopy data directory; RustFS and OS caches remain warm", + "reads": [ + { + "repository": "bat", + "operation": "refs", + "startup_seconds": 0.6589105000020936, + "latency_ms": 474.42112502176315, + "status": 200, + "valid": true, + "response_bytes": 6832, + "query": { + "kind": "refs" + } + }, + { + "repository": "bat", + "operation": "tree", + "startup_seconds": 1.0333095410023816, + "latency_ms": 745.7135000149719, + "status": 200, + "valid": true, + "response_bytes": 4541, + "query": { + "kind": "tree", + "commit": "d9559c69f541873f8c47ac8ad81d985f4441edf6", + "path_base64": "" + } + }, + { + "repository": "bat", + "operation": "blob", + "startup_seconds": 1.0326245839823969, + "latency_ms": 381.3612499507144, + "status": 200, + "valid": true, + "response_bytes": 22279, + "query": { + "kind": "file", + "commit": "d9559c69f541873f8c47ac8ad81d985f4441edf6", + "path_base64": "dGVzdHMvZXhhbXBsZXMvbG9uZ2xpbmUuanNvbg" + } + }, + { + "repository": "bat", + "operation": "commits", + "startup_seconds": 1.0474249579710886, + "latency_ms": 236.28650000318885, + "status": 200, + "valid": true, + "response_bytes": 20535, + "query": { + "kind": "history", + "commit": "d9559c69f541873f8c47ac8ad81d985f4441edf6" + } + }, + { + "repository": "libgit2", + "operation": "refs", + "startup_seconds": 0.5767115830094554, + "latency_ms": 975.2154999878258, + "status": 200, + "valid": true, + "response_bytes": 25061, + "query": { + "kind": "refs" + } + }, + { + "repository": "libgit2", + "operation": "tree", + "startup_seconds": 1.0377434589900076, + "latency_ms": 991.5179999661632, + "status": 200, + "valid": true, + "response_bytes": 5398, + "query": { + "kind": "tree", + "commit": "4af47f040cb4d80c75f37f626f511843e0dd8d44", + "path_base64": "" + } + }, + { + "repository": "libgit2", + "operation": "blob", + "startup_seconds": 1.0659633749746718, + "latency_ms": 859.9925839807838, + "status": 200, + "valid": true, + "response_bytes": 22154, + "query": { + "kind": "file", + "commit": "4af47f040cb4d80c75f37f626f511843e0dd8d44", + "path_base64": "ZGVwcy9wY3JlMi9wY3JlMl9jb21waWxlLmg" + } + }, + { + "repository": "libgit2", + "operation": "commits", + "startup_seconds": 0.5351114589720964, + "latency_ms": 1275.7054580142722, + "status": 200, + "valid": true, + "response_bytes": 18498, + "query": { + "kind": "history", + "commit": "4af47f040cb4d80c75f37f626f511843e0dd8d44" + } + }, + { + "repository": "starship", + "operation": "refs", + "startup_seconds": 0.5408446250366978, + "latency_ms": 352.6365830330178, + "status": 200, + "valid": true, + "response_bytes": 15078, + "query": { + "kind": "refs" + } + }, + { + "repository": "starship", + "operation": "tree", + "startup_seconds": 0.581210583972279, + "latency_ms": 409.1240420239046, + "status": 200, + "valid": true, + "response_bytes": 5088, + "query": { + "kind": "tree", + "commit": "df688000684138943610bbaa2a6c454db6987d48", + "path_base64": "" + } + }, + { + "repository": "starship", + "operation": "blob", + "startup_seconds": 0.5271440829965286, + "latency_ms": 396.08150004642084, + "status": 200, + "valid": true, + "response_bytes": 22185, + "query": { + "kind": "file", + "commit": "df688000684138943610bbaa2a6c454db6987d48", + "path_base64": "LmdpdGh1Yi93b3JrZmxvd3MvcmVsZWFzZS55bWw" + } + }, + { + "repository": "starship", + "operation": "commits", + "startup_seconds": 1.040699000004679, + "latency_ms": 499.0982919698581, + "status": 200, + "valid": true, + "response_bytes": 19598, + "query": { + "kind": "history", + "commit": "df688000684138943610bbaa2a6c454db6987d48" + } + } + ] + }, + "resources": { + "server": { + "sampling_interval_seconds": 2, + "max_observed_rss_mib": 40.375, + "max_observed_ps_cpu_percent": 424.5, + "median_observed_ps_cpu_percent": 199.2 + }, + "client": { + "sampling_interval_seconds": 2, + "max_observed_rss_mib": 82.375, + "max_observed_ps_cpu_percent": 201.1, + "median_observed_ps_cpu_percent": 18.4 + }, + "scope": "Two-second ps samples of the Canopy parent and benchmark client only; excludes SQLite/Git children, RustFS, OS page cache and true between-sample peaks." + }, + "storage": { + "repositories": [ + { + "repository_id": "c26304a7f3e4433a933277ffcbb26cbd", + "sqlite_bytes": 190627840, + "objects": 20952, + "refs": 66 + }, + { + "repository_id": "beb162113d72439794e609639ba05503", + "sqlite_bytes": 2093539328, + "objects": 134967, + "refs": 271 + }, + { + "repository_id": "f82d452384d14ab48099743457d303ae", + "sqlite_bytes": 685969408, + "objects": 38333, + "refs": 164 + } + ], + "provider_df": "Filesystem 1B-blocks Used Available Use% Mounted on\n/dev/vdb1 83953606656 76902137856 2739830784 97% /data\n", + "rustfs_data_directory_apparent_bytes": 22525385112, + "provider_df_after_cold_reads": "Filesystem 1B-blocks Used Available Use% Mounted on\n/dev/vdb1 83953606656 76903243776 2738724864 97% /data\n" + }, + "harness_sha256": { + "benchmark_repositories.py": "17515a1a8471fe8a35433b9eeeb8bc809d377dea44252f34c06ccedad4767320", + "benchmark_browse.py": "fb2945e0ca503413d5d5fd81351cb3e7b4ca6ab76c441693459a6417ec24d97a" + }, + "sample_count": 2425079, + "normal_load_requests": 462198, + "normal_load_errors": 0, + "raw_evidence_directory": "~/.local/share/canopy-benchmark-20261013/results" +} diff --git a/docs/benchmarks/2026-10-14-storage-optimization.md b/docs/benchmarks/2026-10-14-storage-optimization.md new file mode 100644 index 00000000..c28cdf5f --- /dev/null +++ b/docs/benchmarks/2026-10-14-storage-optimization.md @@ -0,0 +1,189 @@ +# RustFS storage optimization qualification — 2026-10-14 + +The current integration uses the native catalog/publication implementation from +`main` at `aa18585`, published Cellule 0.2.0, repository schema 4 and storage +format `canopy-pack-v2`. It retains same-release recovery and requires fresh +storage prefixes. The [October 15 qualification](2026-10-15-native-storage.md) +records the current integration and its separate source, binary and measurements. +All measurements below belong to the earlier implementation and its recorded +binary; they remain historical evidence. + +The earlier storage changes keep Git bodies in verified, immutable native pack/index +pairs, batch metadata publication, and expose Cellule's retention collector +through drained deployment maintenance. Repository schema 3 requires a fresh +storage prefix. Schema-2 measurements below remain historical evidence. The current product contains no reader or migration path for +the earlier Git-body representation. + +The full-history corpus is unchanged from the +[original benchmark](2026-10-13-rustfs-browse.json): bat, starship and libgit2, +194,252 reachable objects, 28,189 commits and 501 branch/tag refs. Its source Git +packs total 152.94 MiB. All builds use published Cellule 0.2.0, with exact pins +and the lockfile checked together. The five direct crates remained at 0.2.0 in +the crates.io check on 2026-10-14. + +## Implemented priorities + +| Priority | Implementation | Measured evidence | +| --- | --- | --- | +| Retention-aware GC | Verified live roots and backup pins; grace boundary; bounded deletes; durable claim; drained owners and node advertisements | The completed three-repository native trial deleted 276,723 obsolete Cellule objects and retained 810 reachable objects. | +| Immutable Git packs | Separate self-contained structural/blob pack pairs; native and BLAKE3 verification; conditional publication; metadata only in SQLite | Durable Git pack bodies total 194.87 MiB, versus 3.07 GiB of expanded bodies in the original format. | +| Less directory rewriting | Smaller body-free SQLite state and fewer bounded metadata transactions | Before GC, directory backing files fell from 7.36 GiB / 351,653 keys to 5.72 GiB / 270,152 keys. | +| Batched import metadata | Up to 512 object metadata records per transaction, blob certificates in that batch, and up to 8,192 dependency rows per SQL write | Current SQLite files total 776,110,080 bytes, approximately 740 MiB, versus 2.77 GiB originally. SHA-256 wire-size and 8,193-edge boundary tests passed. | + +Directory rewrite reduction is the combined effect of fewer modified database +pages and fewer metadata publications. The comparison is not an isolated +measurement of either change. The repository namespace's existing `shards: 1` +is unrelated to LTX page-index sharding and was not changed by this work. + +## Completed native storage trial + +The storage trial used binary +`81a8084224da322a73eb1c73e6af90499da9a5cedba56a072e6d58f867b8a07c`. +Imports completed in 48.75, 90.53 and 910.33 seconds respectively. These are +observations on a shared host, rather than an isolated throughput comparison. + +| RustFS apparent backing bytes | Before GC | After GC | +| --- | ---: | ---: | +| All prefix files | 16,313,052,025 | 1,062,134,213 | +| Directory revisions | 6,141,486,337 | 17,132,798 | +| LTX segments | 9,678,730,257 | 810,393,284 | +| Native Git packs | 204,345,936 | 204,345,936 | +| Native Git indexes | 5,448,648 | 5,448,648 | + +The final 0.989 GiB is approximately 95.3% below the original 20.98 GiB. +The comparison counts regular files under each exact RustFS prefix, including +provider framing and metadata; it is not a bucket-wide allocated-disk measure. +The collector ran three bounded passes and did not delete native Git packs. +Receipts and before/after inventories are retained under +`~/.local/share/canopy-storage-optimization/medium-widebatch-20261014/results/`. + +The original schema-1 fixture's collection also completed on its exact archived +release. Its remaining sweep removed 260,957 obsolete objects and retained +2,985 reachable objects. That original representation occupies 2,937,040,978 +apparent bytes (2.735 GiB) after GC. Thus GC accounts for most of the reduction +from 20.98 GiB; the native representation is approximately 64% smaller than the +collected original representation. No old reader was added to the current +product to collect the archived fixture. + +Native packs are larger than the source's 152.94 MiB because Canopy creates +self-contained structural/blob pairs under its bounded native Git memory +policy. They still avoid the original expansion to 3.07 GiB. Cold reads fetch +the complete referenced pair; sparse S3 range reads are not implemented. +Selected serving objects use disposable, disk-accounted loose projections. + +The older process profile's `--structural-objects` flag used 1,100,000-byte +commit/tag messages and a 32,000-entry tree. The historical references below to +65 MiB structural coverage refer to the ordinary large-object integration test, +not that isolated process profile. Current qualification specifies the structural +message size explicitly; retained raw logs and source snapshots are unchanged. + +## Index-warming checkpoint qualification + +The index-warming checkpoint is +`08d4e52c17d89808251d2516d99f797c821f7b48fc56b50034cf353f7e736674`. +Before its first uncertified object, preparation scans both existing graph +indexes using read-only 512-pair pages and indexed tuple seeks. That materializes +cold pages outside the write transaction without producing directory revisions. +Certification retains the 128-object / 64 MiB batching policy and 512-row edge +inserts. The real SQLite regression checks duplicate cursor keys, both index +orders, exact completeness and absence of full scans. +Both complete existing indexes are traversed during graph preparation; page +and statement sizes are bounded, but total warmup read work grows with history. + +Ref-target reachability checks obey Cellule's 128-statement SQL limit. The +regression uses 129 distinct commit tips. The earlier native storage trial +exposed this limit when cloning Starship's 164 refs; its storage result does +not constitute a successful three-repository clone qualification. + +That source passed formatting, all-target Clippy with warnings denied, +the exact dependency policy, Python's 19 tests, doctests, all eight ignored +real-S3 probes and the paired SHA-1/SHA-256 native-pack probe. The library +suite passed serially: 120 passed, one ignored. Two earlier parallel +workspace-lock failures passed unchanged focused rechecks; those failure +logs remain alongside the successful serial run. Integration results are +Main one, Directory 10, Git HTTP one, multi-server 93 with nine ignored, owner restart +one, repository one and smart HTTP one, all passed. + +The complete isolated process profile passed with 80 MiB Git/LFS objects, +65 MiB structural objects, 256 additional objects, 300 refs, SIGKILL and +takeover, two TLS peers, backup with every source object deleted, retained-pin +GC and an exact cold stock-Git clone. Final source and gate evidence are frozen +in `~/.local/share/canopy-storage-optimization/format2-index-warm-final-20261014/`. + +At that checkpoint the full 5 GiB Git/LFS and 512 MiB incompressible-pack +qualification was not run: its required 40 GiB free scratch and provider +capacity was unavailable. +Historical evidence retains each original binary and dependency version. + +An exploratory build, +`e89d1caea09a29a2bce6a09161ec4e4c6355a3aa4425468b230c813c9be02798`, +capped each certificate batch at 1,024 dependencies to avoid the published +runtime's five-second SQL execution deadline on cold indexes. Repeated crash +trials increased publication history until the provider filled. The final +build instead warms existing indexes with read-only seeks and retains the +larger certificate batches. The failed trial was collected under its exact +compiled release; its logs and partial-import evidence remain separate. + +## Limits and retained failure evidence + +The shared Docker VM filled during exploratory imports. The first disposable +512-dependency-row fixture was discarded by its exact prefix, with its logs +retained. The final build's libgit2 import was interrupted after durable object +staging; its recovery and retry are recorded separately. These attempts must +not be reported as successful clean import timings. + +The subsequent `08d4` three-repository run completed imports and collection, +leaving 1,191,805,853 apparent backing bytes (1.110 GiB). Its bat and starship +clone/API checks passed, but libgit2 exposed two serving limits: a cold SQL read +deadline and a fixed 120-second native CGI lifetime. That run is storage +evidence, not a complete final serving qualification. + +The current transport-budget build is +`7b16ef3133c39ad84290459e0c256765bcd0acb0fd3e4ac04eb85e2ebd1a0ec3`. +Reachability queries now use 16 statements per batch within Cellule's +128-statement ceiling, and native Git CGI has a finite 30-minute lifetime. +Storage schema 2 and command codecs 4/3 are unchanged. This build passed the +entire serial ordinary test run, formatting, all-target Clippy, the dependency +policy and Python's 19 tests. The complete isolated size/provider/process qualification passed on this exact +source and frozen release binary in 2,960.21 seconds. It transferred a Git blob +and an LFS object each larger than 5 GiB, plus over 512 MiB of incompressible +pack data. The native Git pack measured 595,260,812 bytes while SQLite measured +675,840 bytes after push. A fresh-server stock clone and strict fsck passed; +restored Git/LFS sizes and hashes matched. All eight real-provider protocol +probes, the paired SHA-1/SHA-256 pack probe and the extended process profile +passed, including retained-pin GC, source-deleted backup/restore and SIGKILL +recovery. Library/provider tests used the debug profile; process qualification +used the frozen release binary. + +The current medium run imported bat in 56.05 seconds and starship in 124.66 +seconds. Libgit2's first attempt failed after 1,492.34 seconds when graph +certification exceeded Cellule's five-second SQL deadline. Exact-release +maintenance recovery and retry completed: retry took 903.49 seconds, and the +failed attempt plus retry totaled 2,395.83 seconds, excluding recovery. This is +a retained failure and recovered import, not successful clean-import throughput. +Collection completed in three bounded passes: 276,750 obsolete objects were +deleted and all 824 marked objects were retained. The passes took 3,373.37 +seconds (56.22 minutes), excluding maintenance setup and restart. The post-GC +prefix occupies 1,259,651,402 apparent bytes (1.173 GiB), 94.4% below the +original measurement. Bat and Starship passed fresh mirror/fsck/object/ref/API comparisons. Libgit2 +failed its fresh full mirror on a five-second Cellule read deadline immediately +after explicit wants were hydrated. This build's medium serving qualification +is incomplete; its storage result remains valid. + +The next source revision adds a covering `(oid, kind)` index and uses bounded +keyset graph pages, limiting edge rows before type lookup and blob filtering. +Dependency certification checks kind and closure through the covering index, +without fetching unused wide metadata rows. It retains the 128-object / 64 MiB +certificate publication batches and the 512-object metadata batch. Repository +schema 3 has one fresh-cell initialization descriptor; certificate command codec +is 4 and object metadata codec remains 4. Qualification requires a fresh prefix; +no old reader or in-place index upgrade is provided. Its results are pending. + +Cellule GC is operator-invoked under maintenance. It does not reclaim orphaned +native Git/LFS bodies or replace a product-level object retention policy. +The separate Cellule collector performance patch is unpublished and is not +consumed by Canopy's registry pins. + +Refs, tree, exact blob bytes and commit history are checked against source Git. +Author/committer metadata is checked separately. A line-attribution query still +returns HTTP 422; there is no measured blame endpoint. diff --git a/docs/benchmarks/2026-10-15-native-storage.json b/docs/benchmarks/2026-10-15-native-storage.json new file mode 100644 index 00000000..a9cfb1ed --- /dev/null +++ b/docs/benchmarks/2026-10-15-native-storage.json @@ -0,0 +1,1980 @@ +{ + "captured_utc": "2026-10-15T19:22:15.998649+00:00", + "source_commit": "12dc9a19881ca4c87353a7643a374d20bc800d58", + "qualification_source_commit": "81925e121c2fad14c25ab993001835b1bc5dc3d3", + "test_stack_source_equivalence": { + "state": "passed", + "source_commit": "930e6c7a4018b02081ac1421d1f1d5ce9e63691e", + "runtime_base_commit": "12dc9a19881ca4c87353a7643a374d20bc800d58", + "files_compared": 727, + "changed_files": [ + "crates/canopy-server/src/packs/publication/tests/native_head.rs", + "crates/canopy-server/src/packs/publication/tests/native_merge/retirement.rs" + ], + "compiled_runtime_change": false, + "cfg_test_boundary": "crates/canopy-server/src/packs/publication/mod.rs:304", + "original_source_archive": "source.tar", + "qualification_source_archive": "source-test-retirement-heaps.tar", + "captured_utc": "2026-10-15T07:46:02.402713+00:00" + }, + "original_stack_reproduction": { + "binary": "canopy_server-4c94d44cbce73e9e", + "stack_bytes": 2097152, + "exit_code": -6, + "seconds": 73.89485187502578 + }, + "fixed_stack_reproduction": { + "state": "passed", + "stack_bytes": 2097152, + "source_commit": "930e6c7a4018b02081ac1421d1f1d5ce9e63691e", + "started_utc": "2026-10-15T07:46:02.507893+00:00", + "exit_code": 0, + "seconds": 196.71240079205018 + }, + "additional_stack_diagnosis": { + "state": "diagnosis-complete", + "source_commit": "7362d076748d851db3d9e82a5e832e7090c34bbe", + "binary_sha256": "40b38d0c10dbfc08798fe80252c8a4d0d4d95ddda3f75a68d2a6307cb5c44ba0", + "stack_bytes": 2097152, + "previously_passed": 432, + "known_failed": [ + "packs::publication::tests::native_head::native_head_reuses_ref_tree_and_preserves_original_receipt_after_typed_retirement" + ], + "runs": [ + { + "exit_code": -6, + "seconds": 15.571115583996288, + "finished": 12, + "stack_overflows": [ + "packs::publication::tests::native_merge::retirement::negative_merge_archive_keeps_its_denial_after_same_uuid_succeeds" + ], + "log": "remaining-unit-stacks-1.log" + }, + { + "exit_code": 0, + "seconds": 803.8268396250205, + "finished": 303, + "stack_overflows": [], + "log": "remaining-unit-stacks-2.log" + } + ], + "started_utc": "2026-10-15T07:31:31.349814+00:00", + "passed": 747, + "failed": [ + "packs::publication::tests::native_head::native_head_reuses_ref_tree_and_preserves_original_receipt_after_typed_retirement", + "packs::publication::tests::native_merge::retirement::negative_merge_archive_keeps_its_denial_after_same_uuid_succeeds" + ], + "remaining": 0, + "finished_utc": "2026-10-15T07:45:11.495453+00:00" + }, + "final_harness_source_equivalence": { + "state": "passed", + "source_commit": "81925e121c2fad14c25ab993001835b1bc5dc3d3", + "rust_qualified_source_commit": "930e6c7a4018b02081ac1421d1f1d5ce9e63691e", + "changed_paths": [ + "scripts/smoke_s3_backup.py", + "scripts/smoke_s3_peers.py", + "scripts/smoke_s3_process.py" + ], + "all_other_724_source_files_identical": true, + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8" + }, + "backup_harness_source_equivalence": { + "state": "passed", + "source_commit": "81925e121c2fad14c25ab993001835b1bc5dc3d3", + "previous_source_commit": "1a0fc93d526d205979e03523435ba0c6a9a027c1", + "changed_paths": [ + "scripts/smoke_s3_backup.py" + ], + "rust_source_unchanged": true, + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "python_tests": 105, + "python_exit_code": 0, + "python_seconds": 45.085, + "python_log": "python-backup-harness.log", + "backup_script_sha256": "ffb5a2ca1a7b5ab20bc735a980628f28c0217ee56aac8ca948fe6adeb783af0a" + }, + "backup_count_diagnosis": { + "state": "diagnosed", + "expected_legacy_count": 3, + "actual_external_manifest_count": 26, + "physical_key_count": 61, + "families": { + "git-catalogs": 9, + "git-inputs": 12, + "git-packs": 3, + "lfs": 2 + }, + "inventory_sha256": "7a1feb76fc5896bd0443c447522a7f19bbc69bccb5123b31de686f19cebbc9df" + }, + "backup_retention_probe": { + "state": "passed", + "started_utc": "2026-10-15T19:12:39.473607+00:00", + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "script_sha256": { + "smoke_s3_backup.py": "ffb5a2ca1a7b5ab20bc735a980628f28c0217ee56aac8ca948fe6adeb783af0a", + "smoke_s3_retention.py": "551afec5c8e4cff728bbb48f0605967e838a28d67b3e45322be58ef7938558b1" + }, + "backup": { + "saved": { + "pin": "4d4de244-3545-4f5e-91ee-ed1126244eb4", + "root": "qualification/backup-native-inventory/process-smoke/d3c8abc2-9d5b-4da6-8ef6-bc504cb508bc/backup-copy", + "cells": 2, + "external_objects": 26 + }, + "restored": { + "pin": "4d4de244-3545-4f5e-91ee-ed1126244eb4", + "root": "qualification/backup-native-inventory/process-smoke/d3c8abc2-9d5b-4da6-8ef6-bc504cb508bc/backup-restored", + "cells": 2, + "external_objects": 26 + }, + "external_manifest_count": 26, + "external_inventory_sha256": "73183146886bfb8c69fdde0b93e6a0abb0c170a402ed1407fe427ddf9ffe06fd", + "source_deleted": true, + "corrupt_copy_rejected": true, + "cold_clone_verified": true, + "issue_verified": true + }, + "retention": { + "deleted_objects": 248, + "last_report": { + "listed_objects": 410, + "immutable_candidates": 299, + "reachable_objects": 52, + "retained_objects": 52, + "grace_objects": 0, + "eligible_objects": 247, + "deleted_objects": 247, + "current_controls": 2, + "retained_pins": 1, + "complete": true + }, + "backup_verified": true, + "cold_clone_verified": true + }, + "seconds": 120.83030666701961, + "finished_utc": "2026-10-15T19:14:40.315911+00:00" + }, + "superseded_extended_process": { + "state": "failed", + "started_utc": "2026-10-15T19:03:33.782137+00:00", + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "peer_script_sha256": "62566d1eb9bb7910053af81040e070b3d583e270869f0e4ed0473f2d26b62b77", + "source_commit": "1a0fc93d526d205979e03523435ba0c6a9a027c1", + "profile": [ + "--large-clone", + "--many-objects", + "256", + "--structural-objects", + "--structural-object-bytes", + "68157440" + ], + "error": "", + "seconds": 416.4211825419916, + "finished_utc": "2026-10-15T19:10:30.257295+00:00" + }, + "peer_harness_source_equivalence": { + "state": "passed", + "source_commit": "1a0fc93d526d205979e03523435ba0c6a9a027c1", + "previous_source_commit": "624bd26d7a3d04ae64aaf0ca1f3e50b6a7c89e05", + "changed_paths": [ + "scripts/smoke_s3_peers.py" + ], + "rust_source_unchanged": true, + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "python_tests": 105, + "python_exit_code": 0, + "python_seconds": 47.655, + "python_log": "python-peer-harness.log", + "peer_script_sha256": "62566d1eb9bb7910053af81040e070b3d583e270869f0e4ed0473f2d26b62b77" + }, + "peer_tls_probe": { + "state": "passed", + "started_utc": "2026-10-15T19:02:05.942183+00:00", + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "peer_script_sha256": "62566d1eb9bb7910053af81040e070b3d583e270869f0e4ed0473f2d26b62b77", + "seconds": 157.06976595905144, + "finished_utc": "2026-10-15T19:04:43.108462+00:00" + }, + "extended_process": { + "state": "passed", + "started_utc": "2026-10-15T19:13:22.285310+00:00", + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "backup_script_sha256": "ffb5a2ca1a7b5ab20bc735a980628f28c0217ee56aac8ca948fe6adeb783af0a", + "source_commit": "81925e121c2fad14c25ab993001835b1bc5dc3d3", + "profile": [ + "--large-clone", + "--many-objects", + "256", + "--structural-objects", + "--structural-object-bytes", + "68157440" + ], + "seconds": 493.9588900420349, + "finished_utc": "2026-10-15T19:21:36.316138+00:00" + }, + "process_harness_python": { + "state": "passed", + "source_commit": "624bd26d7a3d04ae64aaf0ca1f3e50b6a7c89e05", + "started_utc": "2026-10-15T17:44:11.910904+00:00", + "exit_code": 0, + "seconds": 42.9214055420598, + "finished_utc": "2026-10-15T17:44:54.832541+00:00" + }, + "process_harness_activation": { + "state": "passed", + "started_utc": "2026-10-15T17:44:12.823561+00:00", + "source_commit": "624bd26d7a3d04ae64aaf0ca1f3e50b6a7c89e05", + "rust_qualified_source_commit": "930e6c7a4018b02081ac1421d1f1d5ce9e63691e", + "changed_files": [ + "scripts/smoke_s3_process.py" + ], + "all_other_files_unchanged": true, + "binary_sha256": "0be8986fe9d9a9c12780dead22f9c8dc8c63a3adf88cdc7f682778e5bfc9f254", + "files_verified": 727, + "finished_utc": "2026-10-15T18:34:34.242039+00:00" + }, + "process_cache_recovery_probe": { + "state": "passed", + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "harness_sha256": "02a170d0925ae6f05e4c559001b7c4926331dfa83048587c45f061f9664f97ef", + "storage_url": "s3://canopy-benchmark/qualification/cache-hydration/cf6fead6-2877-4c18-8018-02ceaa875581", + "started_utc": "2026-10-15T17:40:35.009927+00:00", + "blob": "12735a460af03df3b27e7a8dd63e6e1d1c2188f4", + "stale_cache_count": 1, + "hydrated_cache_count": 1, + "stale_cache_reclaimed": true, + "corrupt_sqlite_discarded": true, + "acknowledged_git_and_head_restored": true, + "clean_shutdown": true, + "seconds": 35.77593683404848, + "finished_utc": "2026-10-15T17:41:10.786269+00:00" + }, + "binary": { + "source_commit": "12dc9a19881ca4c87353a7643a374d20bc800d58", + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "binary_bytes": 70621392, + "profile": "release", + "cellule": "0.2.0", + "schema": 4, + "storage_format": "canopy-pack-v2", + "qualification_status": "repository verification running; full workspace/provider/process gates pending", + "runtime_equivalent_source_commit": "624bd26d7a3d04ae64aaf0ca1f3e50b6a7c89e05", + "runtime_image_equivalence": "runtime-image-equivalence.json", + "immutable_qualified_artifact": true + }, + "dependency_check": { + "captured_utc": "2026-10-15T05:35:35.029928+00:00", + "source": "Official crates.io API, live requests", + "releases": [ + { + "name": "cellule-app", + "source": "https://crates.io/api/v1/crates/cellule-app", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0", + "published_utc": "2026-10-13T01:16:20.482357Z" + }, + { + "name": "cellule-host", + "source": "https://crates.io/api/v1/crates/cellule-host", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0", + "published_utc": "2026-10-13T01:16:36.982608Z" + }, + { + "name": "cellule-ltx", + "source": "https://crates.io/api/v1/crates/cellule-ltx", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0", + "published_utc": "2026-10-13T01:14:56.127567Z" + }, + { + "name": "cellule-runtime", + "source": "https://crates.io/api/v1/crates/cellule-runtime", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0", + "published_utc": "2026-10-13T01:15:54.928938Z" + }, + { + "name": "cellule-store", + "source": "https://crates.io/api/v1/crates/cellule-store", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0", + "published_utc": "2026-10-13T01:14:45.791071Z" + }, + { + "name": "cellule-types", + "source": "https://crates.io/api/v1/crates/cellule-types", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0", + "published_utc": "2026-10-13T01:12:54.510864Z" + } + ], + "aligned_with_compiled_release": true + }, + "final_dependency_check": { + "checked_utc": "2026-10-15T07:08:45.122451+00:00", + "source": "Official crates.io API", + "crates": [ + { + "crate": "cellule-app", + "url": "https://crates.io/api/v1/crates/cellule-app", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0", + "published_at": "2026-10-13T01:16:20.482357Z" + }, + { + "crate": "cellule-host", + "url": "https://crates.io/api/v1/crates/cellule-host", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0", + "published_at": "2026-10-13T01:16:36.982608Z" + }, + { + "crate": "cellule-ltx", + "url": "https://crates.io/api/v1/crates/cellule-ltx", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0", + "published_at": "2026-10-13T01:14:56.127567Z" + }, + { + "crate": "cellule-runtime", + "url": "https://crates.io/api/v1/crates/cellule-runtime", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0", + "published_at": "2026-10-13T01:15:54.928938Z" + }, + { + "crate": "cellule-store", + "url": "https://crates.io/api/v1/crates/cellule-store", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0", + "published_at": "2026-10-13T01:14:45.791071Z" + }, + { + "crate": "cellule-types", + "url": "https://crates.io/api/v1/crates/cellule-types", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0", + "published_at": "2026-10-13T01:12:54.510864Z" + } + ] + }, + "evening_dependency_check": { + "checked_utc": "2026-10-15T18:41:35.144852+00:00", + "source": "Official crates.io API", + "crates": [ + { + "crate": "cellule-app", + "url": "https://crates.io/api/v1/crates/cellule-app", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0" + }, + { + "crate": "cellule-host", + "url": "https://crates.io/api/v1/crates/cellule-host", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0" + }, + { + "crate": "cellule-ltx", + "url": "https://crates.io/api/v1/crates/cellule-ltx", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0" + }, + { + "crate": "cellule-runtime", + "url": "https://crates.io/api/v1/crates/cellule-runtime", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0" + }, + { + "crate": "cellule-store", + "url": "https://crates.io/api/v1/crates/cellule-store", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0" + }, + { + "crate": "cellule-types", + "url": "https://crates.io/api/v1/crates/cellule-types", + "max_stable_version": "0.2.0", + "newest_version": "0.2.0" + } + ] + }, + "runtime_image_equivalence": { + "state": "passed", + "source_commit": "624bd26d7a3d04ae64aaf0ca1f3e50b6a7c89e05", + "measured_runtime_source_commit": "12dc9a19881ca4c87353a7643a374d20bc800d58", + "measured_binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "current_build_binary_sha256": "cc08a2361f792147047b7c1ff1ca30ef2ca6ab0a4445db93adfda45510634190", + "binary_bytes": 70621392, + "differing_bytes": 47, + "excluded_metadata_ranges": [ + { + "kind": "Mach-O LC_UUID", + "start": 2024, + "end": 2040 + }, + { + "kind": "derived SHA-256 code-sign first-page hash", + "start": 70073936, + "end": 70073968 + } + ], + "all_other_file_bytes_equal": true, + "first_signature_page_hashes_verified": true, + "normalized_file_sha256": "7f3703573f7c4ff20ca1d302011d3462df08c4c7d4e47dd0b48651225b831956", + "method": "Compare every byte; only the explicit linker UUID and its verified derived ad-hoc signature page hash differ. Neither executable is modified.", + "captured_utc": "2026-10-15T18:47:57.518869+00:00" + }, + "corpus": { + "state": "passed", + "captured_utc": "2026-10-15T05:49:21.123694+00:00", + "source_commit": "58b4b52a783a08c7d28f6a320f1e689ad5a87492+working-91f9507aafeec4ba", + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "cellule": "0.2.0", + "repository_schema": 4, + "storage_format": "canopy-pack-v2", + "repositories": [ + { + "repository": "bat", + "objects": 20952, + "refs": 66, + "all_object_bytes_equal": true, + "clone_fsck_passed": true, + "import_seconds": 21.917903000023216, + "clone_seconds": 30.795469499891624, + "api_blob_samples": 3, + "api_commit_records_checked": 128, + "head_reference": "refs/heads/master", + "head_oid": "d9559c69f541873f8c47ac8ad81d985f4441edf6", + "all_object_bytes_sha256": "ef957f77437a906af80f6381aaa0a78998ea8ce7f700876427f0624cb0c87100" + }, + { + "repository": "starship", + "objects": 38333, + "refs": 164, + "all_object_bytes_equal": true, + "clone_fsck_passed": true, + "import_seconds": 38.66624029201921, + "clone_seconds": 47.59193941601552, + "api_blob_samples": 3, + "api_commit_records_checked": 128, + "head_reference": "refs/heads/main", + "head_oid": "df688000684138943610bbaa2a6c454db6987d48", + "all_object_bytes_sha256": "b4a625af0cc416e04af88e614d0dcd2f8a2f4b4d82bb41805b980c90b7304b88" + }, + { + "repository": "libgit2", + "objects": 134967, + "refs": 271, + "all_object_bytes_equal": true, + "clone_fsck_passed": true, + "import_seconds": 363.5411464580102, + "clone_seconds": 240.1334371669218, + "api_blob_samples": 3, + "api_commit_records_checked": 128, + "head_reference": "refs/heads/main", + "head_oid": "4af47f040cb4d80c75f37f626f511843e0dd8d44", + "all_object_bytes_sha256": "9a2d91500b281267cf1e5f92493a8b76a3a5a3c626a7f9b3f6f7e770714a6821" + } + ], + "tree_pagination_paths": [ + { + "repository": "bat", + "path_base64": "", + "pages": 1, + "entries": 24, + "all_equal": true + }, + { + "repository": "bat", + "path_base64": "dGVzdHM", + "pages": 1, + "entries": 16, + "all_equal": true + }, + { + "repository": "bat", + "path_base64": "dGVzdHMvZXhhbXBsZXM", + "pages": 2, + "entries": 39, + "all_equal": true + }, + { + "repository": "bat", + "path_base64": "dGVzdHMvc25hcHNob3RzL291dHB1dA", + "pages": 2, + "entries": 33, + "all_equal": true + }, + { + "repository": "starship", + "path_base64": "", + "pages": 1, + "entries": 26, + "all_equal": true + }, + { + "repository": "starship", + "path_base64": "LmdpdGh1Yi93b3JrZmxvd3M", + "pages": 1, + "entries": 8, + "all_equal": true + }, + { + "repository": "starship", + "path_base64": "aW5zdGFsbC9tYWNvc19wYWNrYWdlcy9wa2dfcmVzb3VyY2VzL0VuZ2xpc2gubHByb2o", + "pages": 1, + "entries": 3, + "all_equal": true + }, + { + "repository": "starship", + "path_base64": "bWVkaWE", + "pages": 1, + "entries": 20, + "all_equal": true + }, + { + "repository": "libgit2", + "path_base64": "", + "pages": 1, + "entries": 29, + "all_equal": true + }, + { + "repository": "libgit2", + "path_base64": "ZGVwcy9wY3JlMg", + "pages": 2, + "entries": 42, + "all_equal": true + }, + { + "repository": "libgit2", + "path_base64": "dGVzdHMvbGliZ2l0Mi9vYmplY3QvcmF3", + "pages": 1, + "entries": 10, + "all_equal": true + }, + { + "repository": "libgit2", + "path_base64": "dGVzdHMvcmVzb3VyY2Vz", + "pages": 3, + "entries": 89, + "all_equal": true + } + ], + "api_scope": "Every ref; complete pages for selected tree paths; three exact blob previews and four history pages (128 commit records) per repository. Every stored Git object body compared separately through stock Git.", + "dedicated_attribution_endpoint": "not implemented (422), outside the four requested API kinds", + "wider_workspace_provider_process_qualification": "tracked separately in build root", + "equivalent_commit": "12dc9a19881ca4c87353a7643a374d20bc800d58", + "source_equivalence": "All 727 tracked files match the committed source; original build snapshot identity is retained.", + "post_gc_verification": { + "state": "passed", + "captured_utc": "2026-10-15T05:59:50.756120+00:00", + "objects": 194252, + "refs": 501, + "all_object_bytes_heads_fsck_and_apis_equal": true, + "full_selected_tree_pages": true, + "fresh_local_workspace": true, + "deleted_cellule_objects": 1664, + "proof_directory": "after-gc-verification" + }, + "fresh_workspace_api_reads": { + "state": "passed", + "verified": 12, + "evidence": "results/cold-reads.json" + } + }, + "before_gc": { + "captured_utc": "2026-10-15T05:49:28.440391+00:00", + "storage_url": "s3://canopy-benchmark/qualification/native-main/88272c1d-7d05-41a4-a287-e31c17cf50f9", + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "physical_method": "Apparent bytes of every regular backing file under the exact RustFS fixture prefix. Longest listed-key prefix maps files to S3 objects; unmatched files are counted separately. Includes provider framing and metadata; excludes filesystem allocation overhead.", + "physical": { + "directory": { + "bytes": 7834274, + "files": 574, + "object_keys": 574, + "artifacts": 574 + }, + "input_body": { + "bytes": 160676473, + "files": 177, + "object_keys": 114, + "artifacts": 48 + }, + "input_root": { + "bytes": 94093, + "files": 189, + "object_keys": 126, + "artifacts": 63 + }, + "native_catalog_node": { + "bytes": 167004, + "files": 171, + "object_keys": 114, + "artifacts": 57 + }, + "native_directory": { + "bytes": 33846366, + "files": 13, + "object_keys": 8, + "artifacts": 3 + }, + "native_index": { + "bytes": 5445596, + "files": 9, + "object_keys": 6, + "artifacts": 3 + }, + "native_metadata": { + "bytes": 230728186, + "files": 107, + "object_keys": 66, + "artifacts": 25 + }, + "native_pack": { + "bytes": 160389768, + "files": 45, + "object_keys": 24, + "artifacts": 3 + }, + "other": { + "bytes": 436977, + "files": 591, + "object_keys": 591, + "artifacts": 591 + }, + "provider_unlisted": { + "bytes": 92898, + "files": 8, + "object_keys": 0, + "artifacts": 0 + }, + "root": { + "bytes": 4233657, + "files": 574, + "object_keys": 574, + "artifacts": 574 + }, + "cellule_pack": { + "bytes": 7908265, + "files": 574, + "object_keys": 574, + "artifacts": 574 + } + }, + "physical_bytes": 611853557, + "logical": { + "directory": { + "bytes": 7540960, + "files": 574, + "object_keys": 574, + "artifacts": 574 + }, + "input_body": { + "bytes": 160614876, + "files": 114, + "object_keys": 114, + "artifacts": 48 + }, + "input_root": { + "bytes": 30859, + "files": 126, + "object_keys": 126, + "artifacts": 63 + }, + "native_catalog_node": { + "bytes": 109777, + "files": 114, + "object_keys": 114, + "artifacts": 57 + }, + "native_directory": { + "bytes": 33841360, + "files": 8, + "object_keys": 8, + "artifacts": 3 + }, + "native_index": { + "bytes": 5442416, + "files": 6, + "object_keys": 6, + "artifacts": 3 + }, + "native_metadata": { + "bytes": 230688432, + "files": 66, + "object_keys": 66, + "artifacts": 25 + }, + "native_pack": { + "bytes": 160373199, + "files": 24, + "object_keys": 24, + "artifacts": 3 + }, + "other": { + "bytes": 136256, + "files": 591, + "object_keys": 591, + "artifacts": 591 + }, + "root": { + "bytes": 3939195, + "files": 574, + "object_keys": 574, + "artifacts": 574 + }, + "cellule_pack": { + "bytes": 7613803, + "files": 574, + "object_keys": 574, + "artifacts": 574 + } + }, + "logical_bytes": 610331133, + "logical_object_keys": 2771, + "unlisted_physical_files": [ + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/root-lineage/v1/e982bc491da741cda687cb9bbd3a2698/21d07a037cad956e727ffefbf523a5f8e7cca33beab11a9f13420533c948a093.bin/xl.meta", + "bytes": 727 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/root-lineage/v1/e982bc491da741cda687cb9bbd3a2698/2e0cfced117cd2f51fecc967dfc7a46dfc5da73b7c8762cded14c2eadfc94ad1.bin/xl.meta", + "bytes": 727 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/079ee476323918658a7c32475cb9226f1cedec4d09c90f29c37f2f01c424b121.pack/xl.meta", + "bytes": 13516 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/2e0cfced117cd2f51fecc967dfc7a46dfc5da73b7c8762cded14c2eadfc94ad1.root/xl.meta", + "bytes": 11838 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/c94376062284b1605fb4152f080c568493654da5479bd63ce48bc8a1df4bdbf1.dir/xl.meta", + "bytes": 18583 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/de780288e09ee843e9119ffcbb06c956a0d56c34dda7f0960d52c0c06aea0b39.pack/xl.meta", + "bytes": 17758 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/21d07a037cad956e727ffefbf523a5f8e7cca33beab11a9f13420533c948a093.root/xl.meta", + "bytes": 11342 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/56b34a68a53d518d4221f7c896b4e6f2f4a3348d77859057d899688fe6cdcab2.dir/xl.meta", + "bytes": 18407 + } + ], + "classification_correction": { + "source": "storage-before-gc.json", + "reason": "Every extension-only captures entry is a cells/v1/.../objects/*.pack Cellule packed data object. Corrected family name only; original reports, exact keys, counts and byte totals preserved.", + "verified_logical_keys": 574 + } + }, + "after_gc": { + "captured_utc": "2026-10-15T05:49:45.322035+00:00", + "storage_url": "s3://canopy-benchmark/qualification/native-main/88272c1d-7d05-41a4-a287-e31c17cf50f9", + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "physical_method": "Apparent bytes of every regular backing file under the exact RustFS fixture prefix. Longest listed-key prefix maps files to S3 objects; unmatched files are counted separately. Includes provider framing and metadata; excludes filesystem allocation overhead.", + "physical": { + "directory": { + "bytes": 64051, + "files": 5, + "object_keys": 5, + "artifacts": 5 + }, + "input_body": { + "bytes": 160676473, + "files": 177, + "object_keys": 114, + "artifacts": 48 + }, + "input_root": { + "bytes": 94093, + "files": 189, + "object_keys": 126, + "artifacts": 63 + }, + "native_catalog_node": { + "bytes": 167004, + "files": 171, + "object_keys": 114, + "artifacts": 57 + }, + "native_directory": { + "bytes": 33846366, + "files": 13, + "object_keys": 8, + "artifacts": 3 + }, + "native_index": { + "bytes": 5445596, + "files": 9, + "object_keys": 6, + "artifacts": 3 + }, + "native_metadata": { + "bytes": 230728186, + "files": 107, + "object_keys": 66, + "artifacts": 25 + }, + "native_pack": { + "bytes": 160389768, + "files": 45, + "object_keys": 24, + "artifacts": 3 + }, + "other": { + "bytes": 452348, + "files": 611, + "object_keys": 611, + "artifacts": 611 + }, + "root": { + "bytes": 42293, + "files": 5, + "object_keys": 5, + "artifacts": 5 + }, + "cellule_pack": { + "bytes": 2118052, + "files": 75, + "object_keys": 75, + "artifacts": 75 + } + }, + "physical_bytes": 594024230, + "logical": { + "directory": { + "bytes": 61496, + "files": 5, + "object_keys": 5, + "artifacts": 5 + }, + "input_body": { + "bytes": 160614876, + "files": 114, + "object_keys": 114, + "artifacts": 48 + }, + "input_root": { + "bytes": 30859, + "files": 126, + "object_keys": 126, + "artifacts": 63 + }, + "native_catalog_node": { + "bytes": 109777, + "files": 114, + "object_keys": 114, + "artifacts": 57 + }, + "native_directory": { + "bytes": 33841360, + "files": 8, + "object_keys": 8, + "artifacts": 3 + }, + "native_index": { + "bytes": 5442416, + "files": 6, + "object_keys": 6, + "artifacts": 3 + }, + "native_metadata": { + "bytes": 230688432, + "files": 66, + "object_keys": 66, + "artifacts": 25 + }, + "native_pack": { + "bytes": 160373199, + "files": 24, + "object_keys": 24, + "artifacts": 3 + }, + "other": { + "bytes": 141649, + "files": 611, + "object_keys": 611, + "artifacts": 611 + }, + "root": { + "bytes": 39728, + "files": 5, + "object_keys": 5, + "artifacts": 5 + }, + "cellule_pack": { + "bytes": 2079577, + "files": 75, + "object_keys": 75, + "artifacts": 75 + } + }, + "logical_bytes": 593423369, + "logical_object_keys": 1154, + "unlisted_physical_files": [], + "classification_correction": { + "source": "storage-after-gc.json", + "reason": "Every extension-only captures entry is a cells/v1/.../objects/*.pack Cellule packed data object. Corrected family name only; original reports, exact keys, counts and byte totals preserved.", + "verified_logical_keys": 75 + } + }, + "post_browse": { + "captured_utc": "2026-10-15T06:23:52.158347+00:00", + "storage_url": "s3://canopy-benchmark/qualification/native-main/88272c1d-7d05-41a4-a287-e31c17cf50f9", + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "physical_method": "Apparent bytes of every regular backing file under the exact RustFS fixture prefix. Longest listed-key prefix maps files to S3 objects; unmatched files are counted separately. Includes provider framing and metadata; excludes filesystem allocation overhead.", + "physical": { + "directory": { + "bytes": 17264316, + "files": 1535, + "object_keys": 1535, + "artifacts": 1535 + }, + "input_body": { + "bytes": 160676473, + "files": 177, + "object_keys": 114, + "artifacts": 48 + }, + "input_root": { + "bytes": 94093, + "files": 189, + "object_keys": 126, + "artifacts": 63 + }, + "native_catalog_node": { + "bytes": 167004, + "files": 171, + "object_keys": 114, + "artifacts": 57 + }, + "native_directory": { + "bytes": 33846366, + "files": 13, + "object_keys": 8, + "artifacts": 3 + }, + "native_index": { + "bytes": 5445596, + "files": 9, + "object_keys": 6, + "artifacts": 3 + }, + "native_metadata": { + "bytes": 230728186, + "files": 107, + "object_keys": 66, + "artifacts": 25 + }, + "native_pack": { + "bytes": 160389768, + "files": 45, + "object_keys": 24, + "artifacts": 3 + }, + "other": { + "bytes": 1036636, + "files": 1365, + "object_keys": 1365, + "artifacts": 1365 + }, + "provider_unlisted": { + "bytes": 247164, + "files": 22, + "object_keys": 0, + "artifacts": 0 + }, + "root": { + "bytes": 6417438, + "files": 690, + "object_keys": 690, + "artifacts": 690 + }, + "cellule_pack": { + "bytes": 15350335, + "files": 770, + "object_keys": 767, + "artifacts": 767 + } + }, + "physical_bytes": 631663375, + "logical": { + "directory": { + "bytes": 16481696, + "files": 1535, + "object_keys": 1535, + "artifacts": 1535 + }, + "input_body": { + "bytes": 160614876, + "files": 114, + "object_keys": 114, + "artifacts": 48 + }, + "input_root": { + "bytes": 30859, + "files": 126, + "object_keys": 126, + "artifacts": 63 + }, + "native_catalog_node": { + "bytes": 109777, + "files": 114, + "object_keys": 114, + "artifacts": 57 + }, + "native_directory": { + "bytes": 33841360, + "files": 8, + "object_keys": 8, + "artifacts": 3 + }, + "native_index": { + "bytes": 5442416, + "files": 6, + "object_keys": 6, + "artifacts": 3 + }, + "native_metadata": { + "bytes": 230688432, + "files": 66, + "object_keys": 66, + "artifacts": 25 + }, + "native_pack": { + "bytes": 160373199, + "files": 24, + "object_keys": 24, + "artifacts": 3 + }, + "other": { + "bytes": 342223, + "files": 1365, + "object_keys": 1365, + "artifacts": 1365 + }, + "root": { + "bytes": 6063468, + "files": 690, + "object_keys": 690, + "artifacts": 690 + }, + "cellule_pack": { + "bytes": 14957033, + "files": 767, + "object_keys": 767, + "artifacts": 767 + } + }, + "logical_bytes": 628945339, + "logical_object_keys": 4815, + "unlisted_physical_files": [ + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/20c93a9d1441f5b0ead0e241db02196726a612072363e50bb9d4916952c55878/root-lineage/v1/256aaa7da0ca448db66d775282cc434a/4833f053d41d17f1c5a14ecffef451dfc8f1e30a1e0ef69dda6ac0960b984171.bin/xl.meta", + "bytes": 727 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/20c93a9d1441f5b0ead0e241db02196726a612072363e50bb9d4916952c55878/root-lineage/v1/256aaa7da0ca448db66d775282cc434a/6aa76b970848254ef30a6b15dcd548c094edc24b38bee64c4b2481b56ba20a4f.bin/xl.meta", + "bytes": 727 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/20c93a9d1441f5b0ead0e241db02196726a612072363e50bb9d4916952c55878/inc/256aaa7da0ca448db66d775282cc434a/objects/fe3caf82c835e7844a053aa2c929f0c203259073c3ccaa5e3da15f87dbfddff0.dir/xl.meta", + "bytes": 8463 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/20c93a9d1441f5b0ead0e241db02196726a612072363e50bb9d4916952c55878/inc/256aaa7da0ca448db66d775282cc434a/objects/7bc461f204d00f376cd0bb8fb08d8931dd0b3cf0d266497ca3dcda11e0282e55.dir/xl.meta", + "bytes": 651 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/20c93a9d1441f5b0ead0e241db02196726a612072363e50bb9d4916952c55878/inc/256aaa7da0ca448db66d775282cc434a/objects/acc7bd0e463400ac8addd34c941f1d8fe1b660349aa9e078e6c3e740bdeea2d3.pack/xl.meta", + "bytes": 16511 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/20c93a9d1441f5b0ead0e241db02196726a612072363e50bb9d4916952c55878/inc/256aaa7da0ca448db66d775282cc434a/objects/6aa76b970848254ef30a6b15dcd548c094edc24b38bee64c4b2481b56ba20a4f.root/xl.meta", + "bytes": 13348 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/20c93a9d1441f5b0ead0e241db02196726a612072363e50bb9d4916952c55878/inc/256aaa7da0ca448db66d775282cc434a/objects/5963374840b39631342788675bd3d59c9792042ef472226a17cf734dff89e9b8.dir/xl.meta", + "bytes": 23071 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/20c93a9d1441f5b0ead0e241db02196726a612072363e50bb9d4916952c55878/inc/256aaa7da0ca448db66d775282cc434a/objects/c16091c1c1cb9807f789dbcc042c91effd9282f8f52b40e7c978b9eecd1e8311.dir/xl.meta", + "bytes": 8551 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/20c93a9d1441f5b0ead0e241db02196726a612072363e50bb9d4916952c55878/inc/256aaa7da0ca448db66d775282cc434a/objects/36992fcc90fe62f7e35fcbdb573c25a019b6bb3c5f8c4e196a06eb4e8713fe6c.dir/xl.meta", + "bytes": 23071 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/20c93a9d1441f5b0ead0e241db02196726a612072363e50bb9d4916952c55878/inc/256aaa7da0ca448db66d775282cc434a/objects/a07dcd20e6e2684282c51e9acfd6ddbb76274ad258f93b10f20a85db484c49e1.dir/xl.meta", + "bytes": 651 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/20c93a9d1441f5b0ead0e241db02196726a612072363e50bb9d4916952c55878/inc/256aaa7da0ca448db66d775282cc434a/objects/b4d48365fc62b95cd04c6d09492b6d548efee80f5b7dde8c8af89292756d5add.pack/xl.meta", + "bytes": 17488 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/20c93a9d1441f5b0ead0e241db02196726a612072363e50bb9d4916952c55878/inc/256aaa7da0ca448db66d775282cc434a/objects/4833f053d41d17f1c5a14ecffef451dfc8f1e30a1e0ef69dda6ac0960b984171.root/xl.meta", + "bytes": 12852 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/68b8a5074e4e282249b967b72692f0505b6d6a7863cbe3e3a346df04ffb16f03.dir/xl.meta", + "bytes": 651 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/2814d8e2a8a860c8fe62e4264ba2c1e9b334280c634eaa6d07eb4b5e85288019.dir/xl.meta", + "bytes": 23071 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/f00ac31b926bf87b201998e0db07233837467434524c94d1cd6dd402045fba01.root/xl.meta", + "bytes": 12356 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/79ad7899319af1b309a8b1faf507bafc6d5141f5baf714a64bc8f4fd8ba100dc.dir/xl.meta", + "bytes": 7847 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/585cca6e199e6c5dfe2f148495afbea8499140303e500a8d43c82abeb7a6b5a2.dir/xl.meta", + "bytes": 651 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/792a52fa8873e3896d21881e983e9720b6ac4c6e270944d2fca4ca4c25c918d7.pack/xl.meta", + "bytes": 16595 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/5273a9d9f28483a3af93ded3b823c1a77267ffe6c08fadafe1350d9d7833c574.dir/xl.meta", + "bytes": 7759 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/03a662847b49221ae89494148eccce6f3b7b109262541a0772739f2feecd59bf.dir/xl.meta", + "bytes": 23071 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/0284cf37e7ea55e6d6ad7c76cf236c33262dd9100bd308b0a402116ce434e556.root/xl.meta", + "bytes": 11860 + }, + { + "path": "cells/v1/apps/f2e9c34612d241ebbce3311c08eb4a62/cells/5671374e58805959a7262e6bb640494779201554fcf3bc70190b83f806b05ec7/inc/e982bc491da741cda687cb9bbd3a2698/objects/d2cc64649b97603cd605d9a836a2a2a3541822aee60b769ce2dc1a3d4660287d.pack/xl.meta", + "bytes": 17192 + } + ], + "classification_correction": { + "source": "storage-post-browse.json", + "reason": "All extension-only captures entries are Cellule objects/*.pack keys. Renamed category without changing totals.", + "verified_logical_keys": 767 + }, + "scope_note": "Online observation after full post-GC verification, 12 cold-workspace reads and 11,474 warm benchmark requests; serving heartbeat/publication may continue during this non-atomic inventory. Includes unlisted backing files. No second GC performed." + }, + "baseline": { + "captured_utc": "2026-10-15T01:55:50.286170+00:00", + "baseline_cellule": "0.2.0", + "reported_rustfs_data_tree_apparent_bytes": 22525385112, + "bucket_regular_file_bytes": 22525370449, + "outside_bucket_difference_bytes": 14663, + "directory_regular_file_bytes": 7899358413, + "directory_immutable_keys": 351653, + "method": "Read-only scan of regular files under the RustFS bucket directory; apparent bytes include provider metadata and data-part framing. Object counts deduplicate backing files by immutable key. No deletion or reachability claim.", + "baseline_json_sha256": "57b401005da4eb5682e2c290d1b044aa41ff798e7d39d561b85267055b39032b", + "bucket_breakdown_sha256": "0eed4d56a8f79d580e3653f40b5e594cab1bea9984aaf4550c4d1780c87dc03f", + "scope_note": "Original report counts the whole RustFS data tree; dedicated corpus bucket inventory differs by 14663 bytes. Fresh native measurements scan the exact deployment prefix and report unlisted provider files separately. Compare apparent regular-file bytes, not allocated VM filesystem usage." + }, + "garbage_collection": { + "operation": "99b49e85-976e-4406-9930-de70098ad3a7", + "passes": [ + { + "seconds": 9.297116999980062, + "report": { + "listed_objects": 2350, + "immutable_candidates": 1751, + "reachable_objects": 87, + "retained_objects": 87, + "grace_objects": 0, + "eligible_objects": 1664, + "deleted_objects": 1664, + "current_controls": 4, + "retained_pins": 0, + "complete": true + } + } + ], + "state": "complete" + }, + "cold_reads": { + "method": "One first full authenticated HTTP read after graceful shutdown and fresh local workspace per repository/API. Excludes healthz startup; RustFS and host filesystem caches may be warm. Not a percentile or fully cold provider benchmark.", + "results": [ + { + "repository": "bat", + "operation": "blob", + "latency_ms": 1717.557291034609, + "response_bytes": 22279, + "startup_seconds": 1.0512839580187574, + "status": 200, + "response_verified": true + }, + { + "repository": "bat", + "operation": "commits", + "latency_ms": 3115.7047499436885, + "response_bytes": 20535, + "startup_seconds": 0.5792585840681568, + "status": 200, + "response_verified": true + }, + { + "repository": "bat", + "operation": "refs", + "latency_ms": 297.9790000244975, + "response_bytes": 6832, + "startup_seconds": 0.5556554580107331, + "status": 200, + "response_verified": true + }, + { + "repository": "bat", + "operation": "tree", + "latency_ms": 2241.450082976371, + "response_bytes": 4541, + "startup_seconds": 1.0334134169388562, + "status": 200, + "response_verified": true + }, + { + "repository": "libgit2", + "operation": "blob", + "latency_ms": 3872.913834056817, + "response_bytes": 22154, + "startup_seconds": 1.0310157909989357, + "status": 200, + "response_verified": true + }, + { + "repository": "libgit2", + "operation": "commits", + "latency_ms": 5007.826166925952, + "response_bytes": 18498, + "startup_seconds": 0.6664082080824301, + "status": 200, + "response_verified": true + }, + { + "repository": "libgit2", + "operation": "refs", + "latency_ms": 208.63866701256484, + "response_bytes": 25061, + "startup_seconds": 0.6468817499699071, + "status": 200, + "response_verified": true + }, + { + "repository": "libgit2", + "operation": "tree", + "latency_ms": 2760.773833026178, + "response_bytes": 5398, + "startup_seconds": 1.0328317920211703, + "status": 200, + "response_verified": true + }, + { + "repository": "starship", + "operation": "blob", + "latency_ms": 1210.0983749842271, + "response_bytes": 22185, + "startup_seconds": 1.049035458941944, + "status": 200, + "response_verified": true + }, + { + "repository": "starship", + "operation": "commits", + "latency_ms": 2843.330917065032, + "response_bytes": 19598, + "startup_seconds": 1.0459265420213342, + "status": 200, + "response_verified": true + }, + { + "repository": "starship", + "operation": "refs", + "latency_ms": 490.4339590575546, + "response_bytes": 15078, + "startup_seconds": 1.0539884169120342, + "status": 200, + "response_verified": true + }, + { + "repository": "starship", + "operation": "tree", + "latency_ms": 1907.0911249145865, + "response_bytes": 5088, + "startup_seconds": 1.0474654169520363, + "status": 200, + "response_verified": true + } + ] + }, + "warm_measurement_method": { + "method": "closed-loop, persistent HTTP/1.1 connections, full response read, no retries", + "mixed_workload": "equal rotation over one query for each repository/API pair; medium sampled blob", + "duration_per_group_seconds": 8.0, + "concurrency": [ + 1, + 4 + ], + "repeats": 2 + }, + "warm_aggregation": "Pool raw latency samples across both trials; nearest-rank percentiles. Successful requests and response bytes divided by the sum of actual trial durations; no summed rates or averaged percentiles.", + "warm_results": [ + { + "repository": "all", + "operation": "mixed", + "concurrency": 1, + "attempts": 32, + "successes": 32, + "errors": 0, + "elapsed_seconds": 19.138353791902773, + "requests_per_second": 1.6720351367701671, + "response_mib_per_second": 0.02444835302691302, + "latency_ms": { + "p50": 150.08925006259233, + "p95": 2850.6643750006333, + "p99": 2937.793624936603 + } + }, + { + "repository": "all", + "operation": "mixed", + "concurrency": 4, + "attempts": 97, + "successes": 97, + "errors": 0, + "elapsed_seconds": 21.218805790995248, + "requests_per_second": 4.571416551687583, + "response_mib_per_second": 0.06963764920448519, + "latency_ms": { + "p50": 208.67841690778732, + "p95": 3701.124874991365, + "p99": 4382.5643750606105 + } + }, + { + "repository": "bat", + "operation": "blob", + "concurrency": 1, + "attempts": 51, + "successes": 51, + "errors": 0, + "elapsed_seconds": 16.540011209086515, + "requests_per_second": 3.0834320095250205, + "response_mib_per_second": 0.21264022614481842, + "latency_ms": { + "p50": 292.68170800060034, + "p95": 647.6248749531806, + "p99": 1145.3940420178697 + } + }, + { + "repository": "bat", + "operation": "blob", + "concurrency": 4, + "attempts": 148, + "successes": 148, + "errors": 0, + "elapsed_seconds": 16.584007749101147, + "requests_per_second": 8.92426018119906, + "response_mib_per_second": 0.6450469724266509, + "latency_ms": { + "p50": 443.28795897308737, + "p95": 834.6678329398856, + "p99": 1487.2602919349447 + } + }, + { + "repository": "bat", + "operation": "commits", + "concurrency": 1, + "attempts": 6, + "successes": 6, + "errors": 0, + "elapsed_seconds": 19.923467208980583, + "requests_per_second": 0.30115240169118135, + "response_mib_per_second": 0.005701520422337849, + "latency_ms": { + "p50": 3142.1082080341876, + "p95": 4125.456375069916, + "p99": 4125.456375069916 + } + }, + { + "repository": "bat", + "operation": "commits", + "concurrency": 4, + "attempts": 28, + "successes": 28, + "errors": 0, + "elapsed_seconds": 18.9588872079039, + "requests_per_second": 1.476879929341364, + "response_mib_per_second": 0.02953123233652151, + "latency_ms": { + "p50": 3110.6809999328107, + "p95": 4257.9138750443235, + "p99": 4293.876167037524 + } + }, + { + "repository": "bat", + "operation": "refs", + "concurrency": 1, + "attempts": 713, + "successes": 713, + "errors": 0, + "elapsed_seconds": 16.042291291989386, + "requests_per_second": 44.445022660574175, + "response_mib_per_second": 0.28958167535499835, + "latency_ms": { + "p50": 24.8757500667125, + "p95": 44.15645799599588, + "p99": 57.99899995326996 + } + }, + { + "repository": "bat", + "operation": "refs", + "concurrency": 4, + "attempts": 2400, + "successes": 2400, + "errors": 0, + "elapsed_seconds": 16.042705750092864, + "requests_per_second": 149.60069936993688, + "response_mib_per_second": 0.9747237950281226, + "latency_ms": { + "p50": 16.39533299021423, + "p95": 61.51379202492535, + "p99": 78.73658300377429 + } + }, + { + "repository": "bat", + "operation": "tree", + "concurrency": 1, + "attempts": 72, + "successes": 72, + "errors": 0, + "elapsed_seconds": 16.54015316697769, + "requests_per_second": 4.353043123188699, + "response_mib_per_second": 0.025355624328285, + "latency_ms": { + "p50": 158.60500000417233, + "p95": 610.6297079240903, + "p99": 784.1266669565812 + } + }, + { + "repository": "bat", + "operation": "tree", + "concurrency": 4, + "attempts": 250, + "successes": 250, + "errors": 0, + "elapsed_seconds": 16.611242791172117, + "requests_per_second": 15.050047918922722, + "response_mib_per_second": 0.08757715109656573, + "latency_ms": { + "p50": 183.34029102697968, + "p95": 649.8822499997914, + "p99": 1036.7236660094932 + } + }, + { + "repository": "libgit2", + "operation": "blob", + "concurrency": 1, + "attempts": 41, + "successes": 41, + "errors": 0, + "elapsed_seconds": 16.558681790949777, + "requests_per_second": 2.4760425085533524, + "response_mib_per_second": 0.152900446179729, + "latency_ms": { + "p50": 359.654291998595, + "p95": 785.2027079788968, + "p99": 1080.859125009738 + } + }, + { + "repository": "libgit2", + "operation": "blob", + "concurrency": 4, + "attempts": 146, + "successes": 146, + "errors": 0, + "elapsed_seconds": 16.568943708087318, + "requests_per_second": 8.811666125025052, + "response_mib_per_second": 0.5636727630207855, + "latency_ms": { + "p50": 504.0497080190107, + "p95": 792.1191659988835, + "p99": 885.5659590335563 + } + }, + { + "repository": "libgit2", + "operation": "commits", + "concurrency": 1, + "attempts": 7, + "successes": 7, + "errors": 0, + "elapsed_seconds": 18.889531416003592, + "requests_per_second": 0.37057562974110936, + "response_mib_per_second": 0.006530836886871976, + "latency_ms": { + "p50": 2596.7865829588845, + "p95": 4647.928624995984, + "p99": 4647.928624995984 + } + }, + { + "repository": "libgit2", + "operation": "commits", + "concurrency": 4, + "attempts": 28, + "successes": 28, + "errors": 0, + "elapsed_seconds": 16.994784374022856, + "requests_per_second": 1.6475642987738646, + "response_mib_per_second": 0.028967376148409815, + "latency_ms": { + "p50": 2054.943374940194, + "p95": 3526.8444169778377, + "p99": 3527.486833045259 + } + }, + { + "repository": "libgit2", + "operation": "refs", + "concurrency": 1, + "attempts": 1039, + "successes": 1039, + "errors": 0, + "elapsed_seconds": 16.035073166014627, + "requests_per_second": 64.79546362171256, + "response_mib_per_second": 1.5486136568295845, + "latency_ms": { + "p50": 10.329957935027778, + "p95": 35.10695800650865, + "p99": 56.80791602935642 + } + }, + { + "repository": "libgit2", + "operation": "refs", + "concurrency": 4, + "attempts": 1904, + "successes": 1904, + "errors": 0, + "elapsed_seconds": 16.02469187497627, + "requests_per_second": 118.81663715314461, + "response_mib_per_second": 2.839721435255963, + "latency_ms": { + "p50": 18.749042064882815, + "p95": 89.0887079294771, + "p99": 129.4307919451967 + } + }, + { + "repository": "libgit2", + "operation": "tree", + "concurrency": 1, + "attempts": 59, + "successes": 59, + "errors": 0, + "elapsed_seconds": 16.37868070800323, + "requests_per_second": 3.602243736955591, + "response_mib_per_second": 0.018992690322077416, + "latency_ms": { + "p50": 203.3157499972731, + "p95": 666.7511670384556, + "p99": 841.6428329655901 + } + }, + { + "repository": "libgit2", + "operation": "tree", + "concurrency": 4, + "attempts": 222, + "successes": 222, + "errors": 0, + "elapsed_seconds": 16.703519250033423, + "requests_per_second": 13.290612395921045, + "response_mib_per_second": 0.0702799524570234, + "latency_ms": { + "p50": 208.73466599732637, + "p95": 688.8832499971613, + "p99": 874.1668340517208 + } + }, + { + "repository": "starship", + "operation": "blob", + "concurrency": 1, + "attempts": 61, + "successes": 61, + "errors": 0, + "elapsed_seconds": 16.687753124977462, + "requests_per_second": 3.6553752649119673, + "response_mib_per_second": 0.22452658926736563, + "latency_ms": { + "p50": 207.47679204214364, + "p95": 564.7345830220729, + "p99": 667.3221660312265 + } + }, + { + "repository": "starship", + "operation": "blob", + "concurrency": 4, + "attempts": 141, + "successes": 141, + "errors": 0, + "elapsed_seconds": 16.785071666003205, + "requests_per_second": 8.400321595622616, + "response_mib_per_second": 0.5134882748119307, + "latency_ms": { + "p50": 434.05975005589426, + "p95": 885.4027499910444, + "p99": 1100.3043330274522 + } + }, + { + "repository": "starship", + "operation": "commits", + "concurrency": 1, + "attempts": 9, + "successes": 9, + "errors": 0, + "elapsed_seconds": 18.718768041115254, + "requests_per_second": 0.4808008721637957, + "response_mib_per_second": 0.008545219324381596, + "latency_ms": { + "p50": 2330.3796669933945, + "p95": 3049.1317079868168, + "p99": 3049.1317079868168 + } + }, + { + "repository": "starship", + "operation": "commits", + "concurrency": 4, + "attempts": 32, + "successes": 32, + "errors": 0, + "elapsed_seconds": 18.660066998912953, + "requests_per_second": 1.7148920205840723, + "response_mib_per_second": 0.030281963971266444, + "latency_ms": { + "p50": 2094.548290944658, + "p95": 3081.987417070195, + "p99": 3101.8335840199143 + } + }, + { + "repository": "starship", + "operation": "refs", + "concurrency": 1, + "attempts": 834, + "successes": 834, + "errors": 0, + "elapsed_seconds": 16.03657645895146, + "requests_per_second": 52.00611253497747, + "response_mib_per_second": 0.747821965029135, + "latency_ms": { + "p50": 11.184707982465625, + "p95": 40.05862504709512, + "p99": 53.62862499896437 + } + }, + { + "repository": "starship", + "operation": "refs", + "concurrency": 4, + "attempts": 2812, + "successes": 2812, + "errors": 0, + "elapsed_seconds": 16.121262750122696, + "requests_per_second": 174.42802363472416, + "response_mib_per_second": 2.508188000072833, + "latency_ms": { + "p50": 12.591624981723726, + "p95": 54.89604198373854, + "p99": 85.55033407174051 + } + }, + { + "repository": "starship", + "operation": "tree", + "concurrency": 1, + "attempts": 63, + "successes": 63, + "errors": 0, + "elapsed_seconds": 16.077648540958762, + "requests_per_second": 3.9184834672498137, + "response_mib_per_second": 0.012581444608665482, + "latency_ms": { + "p50": 231.43070796504617, + "p95": 533.7357500102371, + "p99": 596.9165000133216 + } + }, + { + "repository": "starship", + "operation": "tree", + "concurrency": 4, + "attempts": 279, + "successes": 279, + "errors": 0, + "elapsed_seconds": 16.647743915906176, + "requests_per_second": 16.759027614151847, + "response_mib_per_second": 0.05278188486752141, + "latency_ms": { + "p50": 183.54470806661993, + "p95": 648.0151249561459, + "p99": 757.4171250453219 + } + } + ], + "workspace_gates": { + "state": "passed", + "source_commit": "930e6c7a4018b02081ac1421d1f1d5ce9e63691e", + "cellule": "0.2.0", + "schema": 4, + "storage_format": "canopy-pack-v2", + "started_utc": "2026-10-15T17:28:34.280422+00:00", + "gates": [ + { + "name": "dependency-policy", + "exit_code": 0, + "seconds": 0.0703032499877736, + "log": "dependency-policy.log" + }, + { + "name": "format", + "exit_code": 0, + "seconds": 1.4924878330202773, + "log": "format.log" + }, + { + "name": "all-target-clippy", + "exit_code": 0, + "seconds": 49.77081887505483, + "log": "all-target-clippy.log" + }, + { + "name": "ordinary-tests", + "exit_code": 0, + "seconds": 3332.7650082920445, + "log": "ordinary-tests.log" + }, + { + "name": "python", + "exit_code": 0, + "seconds": 44.6627316250233, + "log": "python.log" + }, + { + "name": "release-build", + "exit_code": 0, + "seconds": 530.0212494999869, + "log": "release-build.log" + } + ], + "stage": "release-build", + "binary_sha256": "0be8986fe9d9a9c12780dead22f9c8dc8c63a3adf88cdc7f682778e5bfc9f254", + "finished_utc": "2026-10-15T18:34:33.859565+00:00" + }, + "final_process_backup": { + "saved": { + "pin": "6005db5e-94ca-476e-a5bd-70cbf8f20e84", + "root": "qualification/extended-process-native-backup-harness/process-smoke/f81dade2-bb2a-4612-ace2-ce3ea5efd04b/backup-copy", + "cells": 2, + "external_objects": 26 + }, + "restored": { + "pin": "6005db5e-94ca-476e-a5bd-70cbf8f20e84", + "root": "qualification/extended-process-native-backup-harness/process-smoke/f81dade2-bb2a-4612-ace2-ce3ea5efd04b/backup-restored", + "cells": 2, + "external_objects": 26 + }, + "external_manifest_count": 26, + "external_inventory_sha256": "93df98819341b2ebec553973f270fe72f84674e270be51d1095edc9f7e1f44ae", + "source_deleted": true, + "corrupt_copy_rejected": true, + "cold_clone_verified": true, + "issue_verified": true + }, + "final_process_retention": { + "deleted_objects": 256, + "last_report": { + "listed_objects": 418, + "immutable_candidates": 305, + "reachable_objects": 50, + "retained_objects": 50, + "grace_objects": 0, + "eligible_objects": 255, + "deleted_objects": 255, + "current_controls": 2, + "retained_pins": 1, + "complete": true + }, + "backup_verified": true, + "cold_clone_verified": true + }, + "prior_provider_process_attempt": { + "state": "failed", + "cellule": "0.2.0", + "schema": 4, + "storage_format": "canopy-pack-v2", + "profile": "release", + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "source_commit": "624bd26d7a3d04ae64aaf0ca1f3e50b6a7c89e05", + "rust_qualified_source_commit": "930e6c7a4018b02081ac1421d1f1d5ce9e63691e", + "alternate_target_compile_check_sha256": "0be8986fe9d9a9c12780dead22f9c8dc8c63a3adf88cdc7f682778e5bfc9f254", + "runtime_image_equivalence_gate": "runtime-image-equivalence.json", + "process_profile": [ + "--large-clone", + "--many-objects", + "256", + "--structural-objects", + "--structural-object-bytes", + "68157440", + "--retain-work-dir" + ], + "started_utc": "2026-10-15T18:47:58.035643+00:00", + "error": "Command '['/opt/homebrew/opt/python@3.14/bin/python3.14', '-B', 'native-medium-cache-final-20261015/source/scripts/smoke_s3_process.py', '--binary', '/private/var/folders/rn/k2b7mvbs2dgb__6ys6_h94pm0000gn/T/canopy-size-95af668fccdf98z1k51f/canopy', '--storage-url', 's3://canopy-size/process-qualification', '--work-parent', 'native-medium-cache-final-20261015/process-work', '--large-clone', '--many-objects', '256', '--structural-objects', '--structural-object-bytes', '68157440', '--retain-work-dir']' died with .", + "exit_code": 1, + "seconds": 1779.200018582982, + "finished_utc": "2026-10-15T19:17:37.221470+00:00" + }, + "isolated_provider_process": { + "state": "passed", + "captured_utc": "2026-10-15T19:21:36.522060+00:00", + "source_commit": "81925e121c2fad14c25ab993001835b1bc5dc3d3", + "provider_source_commit": "624bd26d7a3d04ae64aaf0ca1f3e50b6a7c89e05", + "rust_qualified_source_commit": "930e6c7a4018b02081ac1421d1f1d5ce9e63691e", + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "provider_cases_expected": [ + "size::native_pack_exceeds_512_mib_and_git_lfs_exceed_5_gib_after_restore", + "sha256::sha256_real_provider_round_trip", + "sha256::sha256_real_provider_native_merge_candidates", + "push_options::signed_push_real_provider_round_trip", + "ssh::signed_sha256_ssh_real_provider_round_trip", + "ssh::stock_ssh_real_provider_round_trip", + "bulk_refs::bulk_mirror_real_provider_round_trip", + "partial_clone::filtered_clones_real_provider_round_trip", + "ssh::lfs::stock_lfs_ssh_real_provider_round_trip" + ], + "provider_cases_passed": [ + "size::native_pack_exceeds_512_mib_and_git_lfs_exceed_5_gib_after_restore", + "sha256::sha256_real_provider_round_trip", + "sha256::sha256_real_provider_native_merge_candidates", + "push_options::signed_push_real_provider_round_trip", + "ssh::signed_sha256_ssh_real_provider_round_trip", + "ssh::stock_ssh_real_provider_round_trip", + "bulk_refs::bulk_mirror_real_provider_round_trip", + "partial_clone::filtered_clones_real_provider_round_trip", + "ssh::lfs::stock_lfs_ssh_real_provider_round_trip" + ], + "extended_process": { + "state": "passed", + "started_utc": "2026-10-15T19:13:22.285310+00:00", + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "backup_script_sha256": "ffb5a2ca1a7b5ab20bc735a980628f28c0217ee56aac8ca948fe6adeb783af0a", + "source_commit": "81925e121c2fad14c25ab993001835b1bc5dc3d3", + "profile": [ + "--large-clone", + "--many-objects", + "256", + "--structural-objects", + "--structural-object-bytes", + "68157440" + ], + "seconds": 493.9588900420349, + "finished_utc": "2026-10-15T19:21:36.316138+00:00" + }, + "current_harness_source_equivalence": { + "state": "passed", + "source_commit": "81925e121c2fad14c25ab993001835b1bc5dc3d3", + "previous_source_commit": "1a0fc93d526d205979e03523435ba0c6a9a027c1", + "changed_paths": [ + "scripts/smoke_s3_backup.py" + ], + "rust_source_unchanged": true, + "binary_sha256": "ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8", + "python_tests": 105, + "python_exit_code": 0, + "python_seconds": 45.085, + "python_log": "python-backup-harness.log", + "backup_script_sha256": "ffb5a2ca1a7b5ab20bc735a980628f28c0217ee56aac8ca948fe6adeb783af0a" + }, + "provider_log_sha256": "6101a15acdf53255d008d1950898fcee5a33d980b76b21bf7a9548be2279e175", + "provider_log": "isolated-provider-process-full-final.log", + "scope_note": "The nine Rust provider cases use identical Rust source. The complete extended process profile uses the corrected peer TLS proxy and native backup inventory at the current Python harness commit. Original attempts and their failures are preserved separately." + }, + "state": "passed", + "ordinary_rust_test_summary": { + "unique_passed": 896, + "failed": 0, + "provider_cases_deferred": 9, + "nested_child_result_duplicates_excluded": 2 + }, + "evidence_sha256": { + "medium-metadata-cache-final-20261015/verify_mirrors.py": "b1038426fd92c44b16d052e78b642b618007c6896563438a80d5d7e8648dfe9a", + "medium-metadata-cache-final-20261015/verify_tree_pagination.py": "75e01b0109515c6ff9f54e8ca9045d27eb7a8e9da360c2161fab9e29a0d94b4c", + "medium-metadata-cache-final-20261015/complete-verification-report.json": "745721a6b19594d78e4d09d5cdb6d067659cdc5a6cdf4b7543dccdf545890622", + "medium-metadata-cache-final-20261015/complete-repository-verification/mirror-ref-proof.json": "3c8b4f603399ca33490ba47c25792780accf6f69699ef79b365f2ba0d7769011", + "medium-metadata-cache-final-20261015/after-gc-verification/mirror-ref-proof.json": "a22be4ec4486b541336afba70d92823428459c3ebe8e1ed725996947391e3edd", + "medium-metadata-cache-final-20261015/results/samples.jsonl": "246f723c9afc33818deb179d3dc326848f696e4b456d624056e2ade9af58d90e", + "medium-metadata-cache-final-20261015/results/measurements.json": "f8873ef943819f5fa7a61a54e65151ca2efcaaec05efc7b6fa309214d742f685", + "medium-metadata-cache-final-20261015/results/inventory-before-gc.json": "6c813af07877231e2c6163dd357c7d9431dabbfad48982d136f0927b5ea63610", + "medium-metadata-cache-final-20261015/results/inventory-after-gc.json": "8a76e17cc846dab4c7e6b696aefdd09a78a50f322919abd2584f9f0bd18184a8", + "medium-metadata-cache-final-20261015/results/inventory-post-browse.json": "01d1c46c35c9c892e0d0fb9be780560969423bb57c4f3045f5b2edc93cf7faeb", + "native-medium-cache-final-20261015/verification-results.json": "03c2d081914c4201a10edf0775fe9db943b3c01a9f5f5fc4f716f15c3ed2ae13", + "native-medium-cache-final-20261015/source-manifest-rust-qualified-930.json": "048eca61b95b6a5c75c7d6b4e7cd18155f8414509139cc895f93fd55de5e9402", + "native-medium-cache-final-20261015/source-manifest-native-backup-harness.json": "632c2a1f075b7df7a33fc56d274092228759881d7513ff91ec83a74f449c4121", + "native-medium-cache-final-20261015/final-harness-source-equivalence.json": "315144c9b9feaf479abc9967a90431ba192c0add3d292076e12aa588483b774d", + "native-medium-cache-final-20261015/current-provider-process-results.json": "1da5190587d0db9e07645efae915879cae4feed33dd67d8c9248baa6c43b9ad5", + "native-medium-cache-final-20261015/runtime-image-equivalence.json": "6099d2744c54d7853874548d19df149185cc1e9f0915db4249a0f2b9ad16fedb", + "native-medium-cache-final-20261015/python-backup-harness.log": "ba2da238849eaf5a8f9ece2ef8b91639e89430039f9d4676dc39c87686371ac8", + "native-medium-cache-final-20261015/isolated-provider-process-full-final.log": "6101a15acdf53255d008d1950898fcee5a33d980b76b21bf7a9548be2279e175", + "native-medium-cache-final-20261015/extended-process-native-backup-harness.log": "f2ee4b194f362e7001663827f57928e0197c3dce672191bb9d83187394bd40db" + }, + "superseded_process_interruption": { + "state": "interrupted_superseded_process_fixture", + "signal": "SIGINT", + "pid": 48734, + "provider_cases_passed": [ + "bulk_refs::bulk_mirror_real_provider_round_trip", + "partial_clone::filtered_clones_real_provider_round_trip", + "push_options::signed_push_real_provider_round_trip", + "sha256::sha256_real_provider_native_merge_candidates", + "sha256::sha256_real_provider_round_trip", + "size::native_pack_exceeds_512_mib_and_git_lfs_exceed_5_gib_after_restore", + "ssh::lfs::stock_lfs_ssh_real_provider_round_trip", + "ssh::signed_sha256_ssh_real_provider_round_trip", + "ssh::stock_ssh_real_provider_round_trip" + ], + "replacement_source_commit": "81925e121c2fad14c25ab993001835b1bc5dc3d3", + "replacement_profile": "extended-process-native-backup-harness-20261015/results.json", + "reason": "Nine required Rust provider cases passed. The old POST-only TLS proxy and fixed external-object count are superseded by the independently running complete current harness. Interrupt only the old Python process child; its finally blocks reclaim its servers and isolated RustFS volume.", + "utc": "2026-10-15T19:17:35.274969+00:00" + } +} diff --git a/docs/benchmarks/2026-10-15-native-storage.md b/docs/benchmarks/2026-10-15-native-storage.md new file mode 100644 index 00000000..0c614a68 --- /dev/null +++ b/docs/benchmarks/2026-10-15-native-storage.md @@ -0,0 +1,136 @@ +# Native storage and full repository qualification — 2026-10-15 + +The three-repository integrity checks passed before and after Cellule retention GC on RustFS. This run uses committed source `12dc9a19881ca4c87353a7643a374d20bc800d58`, published Cellule **0.2.0**, Repository schema **4**, Directory schema **1** and **`canopy-pack-v2`**. The release executable SHA-256 is `ffa22ca297d764537df7a72870f267e1f58fbacf26267e6243910fdb882919f8`. + +All 727 tracked source files in the frozen build snapshot match that commit. The earlier snapshot identifier is retained in the machine-readable evidence. All five direct Cellule pins are exact `=0.2.0`; all six lockfile entries use the published registry release. The official crates.io release check was recorded on 2026-10-15. Breaking format changes require a fresh storage prefix. Only the compiled release and its current formats are supported. + +Workspace qualification source `930e6c7a4018b02081ac1421d1f1d5ce9e63691e` differs from the corpus-tested commit only in two `cfg(test)` files. Three large retirement scenarios are constructed on the heap before polling so they fit the default 2 MiB stack. Original cases fail at that limit; all nine HEAD/merge-retirement regressions now pass, covering both Git formats and retaining every receipt, pin, rejection and restore assertion. Production source is unchanged; the corpus and process fixtures execute the selected immutable artifact described below. Interrupted/failed full runs and the first ineffective candidate remain separate evidence. + +The cache process harness is qualified separately at `624bd26d7a3d04ae64aaf0ca1f3e50b6a7c89e05`. Its only change from the Rust-qualified source is `scripts/smoke_s3_process.py`: a real blobless clone/lazy read creates hydration scratch, and the fixture checks the current nested `repo.git` location before owner death. All 105 Python tests and a real RustFS crash/corrupt-SQLite/recovery probe pass with those exact script bytes. + +The peer proxy harness commit is `1a0fc93d526d205979e03523435ba0c6a9a027c1`. Its only change from that cache harness is `scripts/smoke_s3_peers.py`: the fixture uses the existing bounded stream proxy for TLS termination. The old POST-only proxy returned HTTP 501 for cross-owner Git discovery GETs. All 105 Python tests pass, and the complete two-node scenario passes with eight Git/LFS repositories, opposite-owner routing, public/private reads, owner death and interrupted-maintenance recovery. Rust source and the selected executable are unchanged. + +The final Python harness commit is `81925e121c2fad14c25ab993001835b1bc5dc3d3`. Its only additional change is `scripts/smoke_s3_backup.py`: the backup receipt must match the actual external-manifest inventory, require the current native pack/catalog/input families and both LFS objects, and retain that exact inventory on restore. The earlier assertion of three external objects predates native storage; the current fixture produces 26 manifests. All corruption, conditional-copy, source-deletion and exact restored Git/LFS/issue checks remain. All 105 Python tests pass with the final script bytes. + +The selected immutable deployment artifact remains `ffa22ca2…919f8`, so corpus, storage, browse and provider measurements refer to one executable. A current-source rebuild in the original target directory matches every file byte except the 16-byte Mach-O linker UUID and its independently verified 32-byte ad-hoc signature page hash (47 bytes differ). Code, data, loader commands and all remaining bytes match exactly. Neither executable was patched. The rebuild and the alternate-target build, whose generated protobuf panic path differs, are preserved separately; the original raw-hash reproduction checks remain recorded as unsuccessful. + +## Implemented storage changes + +| Priority | Implementation | Evidence | +| --- | --- | --- | +| Cellule retention GC | Verified live roots and backup pins, bounded grace/deletes, confirmed deployment drain and durable worker exclusion. | One complete pass deleted 1,664 eligible Cellule objects; all repository bytes and APIs passed again after restore. | +| Immutable Git packs | Authenticated native pack/index pairs, reusable metadata shards and catalog roots; full fetch reuses verified packs. | Native Git packs occupy 160,389,768 provider bytes for 194,252 objects; no persistent loose-body expansion is required. | +| Less directory rewriting | Native directory runs and catalog subtrees are immutable and reused; Cellule holds bounded publication state. | Cellule directory state is 7,834,274 bytes / 574 keys before GC and 64,051 bytes / 5 keys after GC. Native directory runs are reported separately below. | +| Batched import metadata | Bounded 512-object metadata pages, up to 8,192 objects per shard, and fenced publication of the completed root. | All three full histories imported successfully; 25 metadata artifacts replace per-object authoritative body rows. | + +The qualification also fixed renewal during staged metadata replay and final assembly, separated the aggregate closure-spool limit from per-shard limits, retained physical pack completion across repeated fetches, and raised the metadata cache from 16 to 24 files under the existing shared disk budget. The last change keeps libgit2’s 17 shards plus its directory resident; the deterministic control downloads 36 files with 16 slots versus 18 with the current default. Same-release recovery, authorization, fencing and bounded admission remain enforced. + +## Complete corpus checks + +| Repository | Stored objects compared byte for byte | Refs | Import seconds | Cold mirror seconds | +| --- | ---: | ---: | ---: | ---: | +| bat | 20,952 | 66 | 21.92 | 30.80 | +| starship | 38,333 | 164 | 38.67 | 47.59 | +| libgit2 | 134,967 | 271 | 363.54 | 240.13 | + +The pinned source repositories contain 28,189 commits. Their Git packs total 160,372,479 bytes (152.94 MiB); canonical uncompressed bodies total 3,297,080,585 bytes (3.07 GiB). Source and mirror repositories pass `git fsck --full`. Every stored object header and body is compared through paired stock-Git `cat-file --batch-all-objects --batch --buffer` streams. All 501 ref names/targets, symbolic HEAD names and HEAD object IDs match. This is a full body comparison, in addition to object-ID checks. + +API checks compare every paginated ref, all pages of 12 selected tree paths (17 pages total), nine exact blob previews and 384 commit records against stock Git. They do not exhaustively enumerate every tree path or every blob through the API; all underlying stored object bodies are checked independently. The complete object/ref/HEAD/fsck/API checks pass again after GC and restoration from S3 into a fresh local workspace. Twelve additional first API reads after separate fresh-workspace restarts return verified responses. + +The dedicated attribution endpoint returns 422 and remains unimplemented. It is outside the four API kinds qualified here. + +## Storage accounting + +Counts below are apparent bytes of regular RustFS backing files under the exact fresh deployment prefix, including provider framing and metadata. They exclude filesystem allocation overhead. Object keys, backing files and logical artifacts are distinct counts. The raw inventories and original classification are retained; `.pack` objects under Cellule paths are correctly classified as Cellule packed objects in this report. + +| Category | Before GC bytes | After GC bytes | After GC logical artifacts | +| --- | ---: | ---: | ---: | +| native_pack | 160,389,768 | 160,389,768 | 3 | +| native_index | 5,445,596 | 5,445,596 | 3 | +| native_metadata | 230,728,186 | 230,728,186 | 25 | +| native_directory | 33,846,366 | 33,846,366 | 3 | +| native_catalog_node | 167,004 | 167,004 | 57 | +| input_body | 160,676,473 | 160,676,473 | 48 | +| input_root | 94,093 | 94,093 | 63 | +| cellule_pack | 7,908,265 | 2,118,052 | 75 | +| directory | 7,834,274 | 64,051 | 5 | +| root | 4,233,657 | 42,293 | 5 | +| other | 436,977 | 452,348 | 611 | +| **Total** | **611,853,557** | **594,024,230** | | + +Before GC there are 2,771 listed S3 keys plus 92,898 bytes of unlisted provider files; after GC there are 1,154 keys and no unlisted files. The totals include those files. Logical S3 body bytes after GC total 593,423,369. One 9.30-second collector pass deleted 1,664 eligible objects and retained 87 reachable objects; new control objects explain why the net key-count difference is smaller than the deletion count. The complete maintenance/drain/GC sequence took 14.27 seconds. + +The original corpus bucket used 22,525,370,449 apparent bytes, close to the previously reported whole-RustFS total of 22,525,385,112 bytes (20.98 GiB). The fresh prefix uses 594,024,230 bytes (566.5 MiB), about **97.36% less**. This compares the historical bucket with a fresh current-format deployment and reflects the combined storage changes; it is not an isolated measurement of GC alone. Historical Cellule directory state used 7,899,358,413 bytes / 351,653 keys. Current native directory runs occupy 33,846,366 bytes separately from the 64,051 bytes of Cellule directory state. + +The retained compressed input bodies occupy another 160,676,473 provider bytes and support durable input/replay state. GC intentionally retains product-owned Git packs, catalogs and LFS bodies. Reclaiming unreferenced product artifacts requires its own root/retention policy; this collector does not implement that policy. + +After the complete post-GC verification, 12 fresh-workspace reads and 11,474 warm benchmark requests, an additional online inventory observed **631,663,375 bytes (602.4 MiB)** / 4,815 listed keys. Native packs, indexes, metadata, directory runs, catalogs and retained input families were unchanged; Cellule serving/recovery state accounted for the growth. Cellule directory objects then occupied 17,264,316 bytes / 1,535 keys. This was a non-atomic online observation with the server still running, including 247,164 bytes of unlisted provider files; it is not another post-GC bound. No second GC was performed. + +## Browse throughput and latency + +The client and Canopy run on the shared host over loopback HTTP without TLS; S3 requests reach RustFS in the Docker VM. The warm benchmark uses persistent HTTP/1.1 connections, closed-loop concurrency 1 and 4, two eight-second trials per repository/API group, randomized with seed 42. It reads and validates full response bodies, performs no retries and includes all attempts in latency percentiles. Rates divide successful requests by the sum of actual elapsed trial times; percentiles pool raw samples across both trials using nearest rank. The mixed workload rotates equally across the 12 repository/API pairs. These are short trials on a shared host, not a saturation or capacity guarantee. + +| Repository | API | Concurrency | Requests/s | p50 ms | p95 ms | p99 ms | Errors / attempts | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | +| all | mixed | 1 | 1.67 | 150.1 | 2850.7 | 2937.8 | 0 / 32 | +| all | mixed | 4 | 4.57 | 208.7 | 3701.1 | 4382.6 | 0 / 97 | +| bat | blob | 1 | 3.08 | 292.7 | 647.6 | 1145.4 | 0 / 51 | +| bat | blob | 4 | 8.92 | 443.3 | 834.7 | 1487.3 | 0 / 148 | +| bat | commits | 1 | 0.30 | 3142.1 | 4125.5 | 4125.5 | 0 / 6 | +| bat | commits | 4 | 1.48 | 3110.7 | 4257.9 | 4293.9 | 0 / 28 | +| bat | refs | 1 | 44.45 | 24.9 | 44.2 | 58.0 | 0 / 713 | +| bat | refs | 4 | 149.60 | 16.4 | 61.5 | 78.7 | 0 / 2400 | +| bat | tree | 1 | 4.35 | 158.6 | 610.6 | 784.1 | 0 / 72 | +| bat | tree | 4 | 15.05 | 183.3 | 649.9 | 1036.7 | 0 / 250 | +| libgit2 | blob | 1 | 2.48 | 359.7 | 785.2 | 1080.9 | 0 / 41 | +| libgit2 | blob | 4 | 8.81 | 504.0 | 792.1 | 885.6 | 0 / 146 | +| libgit2 | commits | 1 | 0.37 | 2596.8 | 4647.9 | 4647.9 | 0 / 7 | +| libgit2 | commits | 4 | 1.65 | 2054.9 | 3526.8 | 3527.5 | 0 / 28 | +| libgit2 | refs | 1 | 64.80 | 10.3 | 35.1 | 56.8 | 0 / 1039 | +| libgit2 | refs | 4 | 118.82 | 18.7 | 89.1 | 129.4 | 0 / 1904 | +| libgit2 | tree | 1 | 3.60 | 203.3 | 666.8 | 841.6 | 0 / 59 | +| libgit2 | tree | 4 | 13.29 | 208.7 | 688.9 | 874.2 | 0 / 222 | +| starship | blob | 1 | 3.66 | 207.5 | 564.7 | 667.3 | 0 / 61 | +| starship | blob | 4 | 8.40 | 434.1 | 885.4 | 1100.3 | 0 / 141 | +| starship | commits | 1 | 0.48 | 2330.4 | 3049.1 | 3049.1 | 0 / 9 | +| starship | commits | 4 | 1.71 | 2094.5 | 3082.0 | 3101.8 | 0 / 32 | +| starship | refs | 1 | 52.01 | 11.2 | 40.1 | 53.6 | 0 / 834 | +| starship | refs | 4 | 174.43 | 12.6 | 54.9 | 85.6 | 0 / 2812 | +| starship | tree | 1 | 3.92 | 231.4 | 533.7 | 596.9 | 0 / 63 | +| starship | tree | 4 | 16.76 | 183.5 | 648.0 | 757.4 | 0 / 279 | + +Commit history is the slowest measured API: concurrency-4 p95 is 3.1–4.3 seconds. No service-level latency target or large-team capacity claim is established by this run. + +Each cold result below is one first verified API request after a graceful shutdown and fresh local workspace. Health/startup time is excluded. RustFS and host filesystem caches may remain warm, so these are individual cold-local-workspace observations, not percentile estimates or a fully cold provider benchmark. + +| Repository | Refs ms | Tree ms | Blob ms | Commits ms | +| --- | ---: | ---: | ---: | ---: | +| bat | 298.0 | 2241.5 | 1717.6 | 3115.7 | +| starship | 490.4 | 1907.1 | 1210.1 | 2843.3 | +| libgit2 | 208.6 | 2760.8 | 3872.9 | 5007.8 | + +## Qualification and environment + +The dependency-policy check, workspace formatting, all-target Clippy with warnings denied, ordinary workspace tests and release compilation pass. The ordinary Rust suites contain 896 unique passing tests, zero failures and nine provider-only cases deferred to the isolated runner. All 105 Python harness tests also pass at the current harness commit. + +All nine isolated RustFS provider cases pass: the 595,260,811-byte native pack and 5,377,097,728-byte Git/LFS bodies after restore, SHA-256 round trips and native merge candidates, signed pushes, signed SHA-256 SSH, stock SSH, bulk refs, filtered clones and SSH LFS. The full current extended process profile passes in 493.96 seconds, including large v0/v2 clones, structural objects, repeated restart/takeover, cache corruption recovery, eight repositories through opposite HTTPS peers, interrupted maintenance, backup and retention. + +The final backup copies 26 external manifests, rejects corrupt retained bytes, verifies after deleting the disposable source prefix, restores the same inventory, and returns exact Git/LFS content and issue state. The process retention fixture rejects collection outside maintenance, enforces a one-delete partial pass, then completes collection of 256 obsolete Cellule objects while preserving its backup pin and cold stock-Git clone. An independent focused fixture also passes backup/retention and collects 248 objects; these are separate from the 1,664-object corpus collection above. + +The nine Rust provider cases use source `624bd26`, whose Rust bytes match the final harness source. After those cases passed, its superseded Python process child was deliberately interrupted; that original whole-run receipt remains failed/interrupted. The complete successful process profile uses `81925e1`. All 724 other source files match the Rust-qualified `930e6c7`; only the three Python fixtures described above differ. The combined qualification receipt records both scopes and their source identities. Earlier failures are retained with their original source versions, rather than relabeled as passes. + +The standard workspace gates use: + +```sh +python3 scripts/check_cellule.py +cargo fmt --all -- --check +cargo clippy --workspace --all-targets --locked -- -D warnings +cargo test --workspace --locked -- --test-threads=1 +python3 -B -m unittest discover -s scripts -p 'test_*.py' +``` + +The nine isolated provider cases use `python3 scripts/qualify_size.py --release --docker-volume`. The extended process profile uses `scripts/smoke_s3_process.py` with a frozen release executable, disposable S3 prefix/work directory, `--large-clone --many-objects 256 --structural-objects --structural-object-bytes 68157440 --retain-work-dir`, and the fixture credentials in the environment. The 256-object option creates small files; it does not create 256 large structural objects. + +Hardware: Apple M2 Max, 12 CPU cores, 32 GiB RAM; macOS 26.5.2; Rust/Cargo 1.97; Apple Git 2.50.1; Git LFS 3.7.1; Python 3.14.6. The shared Docker VM has 6 CPUs and about 7.67 GiB RAM. The corpus RustFS `1.0.0-beta.8-glibc` container is limited to 4 CPUs / 6 GiB RAM. The corpus uses an isolated fresh prefix in the dedicated RustFS fixture; measurements are from the same release executable throughout. + +Historical results remain in [the October 14 report](2026-10-14-storage-optimization.md), with their original revisions and format versions. [Machine-readable results](2026-10-15-native-storage.json) retain the source/binary identities, corpus proofs, storage accounting, pooled metrics, qualification receipts and hashes of local raw evidence. The full local evidence lives under `~/.local/share/canopy-storage-optimization/{native-medium-cache-final-20261015,medium-metadata-cache-final-20261015}`. Failed earlier trials remain separate and are not counted as passes. diff --git a/docs/contracts.md b/docs/contracts.md index 73b14c18..162ab6a7 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -1,6 +1,6 @@ # Understand Canopy's persisted contracts -This reference defines the identities, storage boundaries, authorization checks and retry behavior that Canopy must preserve across node loss. It describes the current development format. [Versioned migrations](../ROADMAP.md#r01-safe-upgrades-and-schema-migration) are still required before persistent customer repositories can survive upgrades. +This reference defines the identities, storage boundaries, authorization checks and retry behavior that Canopy must preserve across node loss. It describes the current compiled format. Breaking changes require a fresh storage prefix; older deployments and their readers are not retained. > **Document type:** Reference. **Goal:** preserve durable behavior and make conflicts, retries, and recovery observable. @@ -24,8 +24,8 @@ request ▼ Directory Cell ── repository UUID ──► Repository Cell │ refs, ACL, issues, PRs, checks - │ Git object metadata and small bodies - ├──► immutable large Git/LFS bodies + │ accepted catalog and ref roots + ├──► immutable native Git/LFS artifacts │ in object storage └──► disposable native Git cache for wire protocol work @@ -80,18 +80,19 @@ The Directory Cell binds names and accounts to durable identities; the Repositor ### Git and LFS storage -Small Git objects remain in SQLite. Large Git blobs and Git LFS bodies use immutable object-store bytes with SQLite references and verification metadata. +All Git bodies use verified immutable native pack/index artifacts. Repository +SQLite publishes accepted catalog/ref roots and business metadata; it has no +per-object body, chunk, placement or graph-certificate tables. -| Surface | Current value | Owner | +| Surface | Current representation | Owner | | --- | --- | --- | -| Git object ID | SHA-1 or SHA-256, according to the repository's immutable object format, over canonical Git type, decimal length, NUL and body | `object_id` | -| Small Git objects | SQLite `objects.body`, maximum 768 KiB | Repository Cell | -| Large trees, commits and tags | SQLite chunks of at most 512 KiB; object size above 768 KiB without a fixed byte quota | `object_chunks`, verified before object publication | -| Object publication | at most 128 records, 3 MiB aggregate inline payload; SQLite verification targets 64 MiB, with larger objects verified individually | `PutObjects`, operation 5, codec 3 | -| Large Git blobs | immutable `repos//git-blobs/` body, SQLite digest/size/reference | `LargeBlobStore` | -| LFS objects | immutable `repos//lfs/` body, SQLite digest/size/reference | `LfsService` | -| LFS locks | unique path per repository; UUID identity, owning account and UTC second-precision timestamp in SQLite | `LfsService` / lock API | -| External bodies | No fixed logical byte quota; 8 MiB immutable parts, a 16-byte Git manifest or a part-digest LFS manifest | Each conditional S3 copy handles one bounded part | +| Git identity | SHA-1 or SHA-256 canonical Git type, size and body; repository format is immutable | Native verifier and catalog | +| Git bodies | Immutable native pack/index pairs, scoped to repository and creating operation | `ArtifactStore` | +| Canonical metadata and typed graph | Immutable checked metadata shards and directory runs; pages of up to 512 objects | `packs::metadata`, `packs::directory`, closure verifier | +| Accepted refs and object membership | Joint accepted immutable roots, checked against current authority and policy | Repository Cell | +| LFS objects | Immutable `repos//lfs/` parts; SQLite pins size and manifest digest | `LfsService` | +| LFS locks | Unique repository path, UUID, account owner and timestamp in SQLite | LFS locking API | +| Artifact parts | At most 8 MiB per part, authenticated `CANOPY02` manifest; at most 65,536 parts (512 GiB) per artifact | Conditional part publication and verified readers | ### Admission and publication @@ -102,12 +103,12 @@ These limits bound individual requests and publication work. They do not cap tot | Node transfer admission | eight active Git/LFS requests across repositories; immediate 503 with `Retry-After: 1` when full | repository HTTP router | | LFS request deadline | Batch: 120 seconds; object PUT: 120-second input idle timeout with no whole-transfer deadline; timeout returns 408 | Git HTTP router / LFS service | | Git request admission | No receive-pack byte quota; 64 MiB for other requests; 120-second input idle deadline | anonymous request spool | -| Ref mutation | check actor's write role, compare expected optional OID and monotonic version; retain deletion records and apply all updates in one Cell transaction | `FinalizePush` | +| Ref mutation | Recheck current actor, expected OID/version and policy; atomically select accepted catalog/ref roots and exact outcome | `CompleteRootPush`, operation 36, codec 2 | | Symbolic HEAD | Accepted immutable `RefStateSnapshot.default_branch`, initially `refs/heads/main`; owner-authorized joint-root compare-and-set | `PublishNativeHead`, operation 54, codec 1 | | HTTP push identity | repository-local UUID bound to account and BLAKE3 request digest; different IDs identify independent operations | `pushes` | -| HTTP push outcome | status, headers and BLAKE3-verified body in SQLite chunks; publish response pointer, rejection decision and ordered push notes atomically with accepted refs | `CompletePush`, codec 5 | -| Graph certificates | at most 128 candidates; SQLite verification targets 64 MiB, with larger objects verified individually; typed dependencies must already be certified | `CertifyObjects`, operation 6, codec 1 | -| Git connectivity at ref publication | at most 100,000 certified new tips, with commit-only branch tips; same transaction as ACL, ref CAS and outcome | `object_closure`, shared ref finalization | +| HTTP push outcome | Immutable recorded native outcome; publish its authenticated pointer and decision atomically with accepted roots | Root completion and outcome commands | +| Graph certification | Native bytes, canonical identities and typed dependencies verified before catalog attestation; bounded metadata pages and disk-backed traversal | Canonical/closure verifier and owner-bound attestation | +| Git connectivity at ref publication | Prepared certified catalog/ref transition; commit-only branch tips; final ACL, policy, observation and fencing checks | Root publication | | LFS metadata publication | check actor's write role in the SQLite insert transaction | `record_lfs_object` | ## Git publication and retry @@ -127,31 +128,19 @@ verified object records ──► graph certificates ──► staged ref plan Git report to client ``` -The `objects` table stores one verified kind, size and independent BLAKE3 -digest per Git OID. External objects also store SHA-256. Readers verify the -bytes against the SQLite record and recompute the Git OID or LFS SHA-256. -The `refs` table stores name, optional OID and version. Deletion sets the OID -to NULL and advances the version; recreation advances it again. Never-seen -names and deleted names have different expectations, so stale plans fail even -after a ref returns to the same OID or to a deleted state. Deleted rows remain -in SQLite and are excluded from Git advertisements and namespace conflicts. -They must not be collected without a replacement mechanism for fencing stale -ref plans. `FinalizePush` uses codec version 3 for this expectation shape. -Before reporting accepted refs, a push persists all new objects and publishes -the accepted ref changes and replayable response through `CompletePush`. Both -`CompletePush` and the direct `FinalizePush` command share the same ref checks. -HTTP plans use `push_plan_chunks`, bound to one response UUID, with 128 updates -and at most 64 KiB per chunk. Completion carries the count and BLAKE3 digest, -loads every ordered chunk inside its transaction, verifies actor/count/digest, -and applies the whole plan. Ref generation advances once; the saved response -and deletion of consumed plan chunks commit atomically with refs. Interrupted -staging never publishes refs. Unconsumed chunks from failed attempts remain -until a future retention policy collects them. Direct typed `FinalizePush` -inputs retain their 1 MiB wire bound; HTTP bulk plans are not limited by it. -This changes the development schema and completion codec; use a fresh deployment -prefix, with no compatibility reader for the old inline completion shape. -Rejected or interrupted pushes may leave unreferenced objects; collection is not -implemented yet. +Canonical object metadata and typed closure facts live in authenticated native +catalog artifacts. Accepted immutable ref snapshots retain optional OIDs and +monotonic versions, so deletion/recreation and return-to-the-same-OID cannot +revive stale plans. Physical pack presence is not object membership authority. + +Before acknowledging accepted refs, the owner publishes the certified catalog, +ref roots and exact recorded native outcome through `CompleteRootPush`. +Bounded immutable request/outcome artifacts carry large plans and replies; +the final Cell command checks their authenticated roots, current authority and +preconditions. Interrupted preparation cannot publish a partial ref snapshot. +Unreferenced product artifacts remain until a separate retention policy can +prove them safe to remove. Older Git-body formats and completion APIs are not +exposed through a compatibility reader. ### Ref snapshots and default branch @@ -242,65 +231,26 @@ report unchanged. A rejected atomic push changes no refs. Malformed packs retain Git's unpack failure report and publish no refs. The gateway does not infer transaction success by searching diagnostic text for status fragments. -Both ref commands require certified reachable Git graphs in the ref transaction. -Commit trees and parents, tree entries, and annotated tag targets must exist -with the required object kind. Branch tips must be commits. Tree symlinks and -regular files require blobs; gitlinks refer to another repository and do not -require a local object. The traversal follows Git's -[commit headers](https://github.com/git/git/blob/v2.50.1/commit.c#L435), -[tag targets](https://github.com/git/git/blob/v2.50.1/tag.c#L142), and -[tree connectivity rules](https://github.com/git/git/blob/v2.50.1/fsck.c#L334). -It checks graph structure and content hashes; it is not a replacement for all -of `git fsck`'s metadata, filename and portability checks. - -An async postorder traversal prepares certificates before either ref command. -It reads only uncertified history, deduplicates repeated typed roots/edges and -checks child status in pages of 128. Certified children need no separate Cell -call; uncertified children carry their observed immutable metadata into traversal. -It groups at most 128 candidate object IDs in dependency order. Its cache of objects ready -for certification covers only the pending batch. `CertifyObjects` independently -reads each candidate from SQLite, recomputes hashes and graph edges, and requires -all typed children to be certified. It checks child metadata in groups of at most -128 rows. The same transaction records deduplicated parent/child edges in -`object_edges`, with a reverse child index for fetch authorization. Only -certified typed edges enter this index; gitlinks do not create local edges. -Newly verified inline and chunked bodies target 64 MiB per command; a larger -chunked object is verified individually; references to verified external blob bodies do not consume that budget. -The final ref transaction checks up to 100,000 new tips and their certificates -in groups of 128, together with current ACL, versions and namespace conflicts. -An ordered update map detects duplicates and planned ancestor conflicts. Exact -ancestor lookups and indexed descendant pages check existing namespaces; every -conflict must be removed by a deletion in the same plan, without truncating at a -fixed number of matches. - -The certificate command is the trust boundary: incorrect candidate order, -missing or mismatched children, corrupt data or an invalid verification batch -rejects its whole application savepoint. The client traversal cannot assert a -certificate. Certificates and refs are separate publications. A failed or -interrupted preparation may retain valid certificates from earlier batches; -retries reuse them. Partially certified roots cannot publish refs, and certificate -commits do not advance `ref_generation`. An external blob certificate relies on -the verified immutable upload before its SQLite record; there is no external -network I/O inside certificate or ref transactions. - -`RepositoryCell::finalize_push` and HTTP completion share this preparation path. -The low-level `FinalizePush` command requires certificates already present. -A preparation failure is reported as ref invocation `NotStarted`, retaining its -underlying error, since the final ref command has not been dispatched. Certificate -work may itself have a pending outcome; retry rechecks committed state. Direct -callers must keep their supplied finalization identity valid through preparation; -HTTP completion creates its final command identity after preparation. - -Certificates rely on immutable object records and retained object bytes. There -is currently no object mutation or collector through the product API. Any -future collector or repair that removes or changes objects must invalidate all -affected ancestor certificates before ref publication resumes; clearing the -entire certificate table and its object-edge index is the conservative -implementation. Backup and restore -must preserve the database and its referenced external bodies together. Each -certificate transaction has explicit object/byte limits, but large individual -objects, traversal frontier memory and production-scale latency still need -capacity qualification. +Publication requires a certified native catalog. Commit trees and parents, tree +entries and tag targets must exist with the expected kind; gitlinks can refer +to a different repository. Native verification checks canonical Git identities, +sizes and BLAKE3 digests. Structural parsing streams bounded chunks rather than +reconstructing bodies in a Cell SQL transaction. Metadata shards and disk-backed +closure traversal bound working buffers; complete traversal still grows with +new history and needs capacity qualification. + +Only trusted verification can issue the catalog attestation. The final receiver +checks its exact repository, operation, base, lease/owner and publication facts, +then rechecks current actor access, ref observations and branch/review/check +policy before selecting the new roots and outcome. A client, cached object or +partially prepared artifact cannot assert certified membership. The native +publisher retains exact request identity and same-release recovery registration. + +Git bodies are immutable after publication. Backup/restore must preserve all +artifacts reachable from the accepted roots, retained outcomes and recovery +records. Cellule collection and product-artifact retention are separate: +operator-invoked Cellule GC reclaims superseded Cellule objects after a verified +drain, but does not reclaim abandoned product pack/catalog/LFS artifacts. ### HTTP push identity and exact replay @@ -315,15 +265,15 @@ Authorization secrets and host addresses are not part of the digest. Explicit identity encoding and absent encoding have the same interpretation; changing to gzip under a completed request UUID produces a conflict. -A reservation binds the ID before running Git. Response metadata and 512 KiB -body chunks are durable before finalization. `CompletePush` checks the binding, -chunk completeness, then applies the ref plan under current ACL, policy and CAS -checks. It publishes the response pointer and a rejection decision in one Cell -transaction. `apply_refs` returns false only before any ref writes; SQL errors -roll back completion. A completed ID returns its existing outcome. Only a -published pointer can expose a staged response; replay verifies the stored body -length and BLAKE3 digest before applying the saved decision. The HTTP boundary -checks current token scope and repository access even for completed requests. +The current staging lifecycle binds the original request ID and exact commands +before native execution. Immutable request and result roots retain the replay +material. Root completion authenticates those roots and atomically records the +accepted catalog/ref transition or terminal refusal. Its recovery registration +retains the original command and input facts across owner loss. +Only an accepted published pointer can expose a completed outcome. Replay checks +its bytes and current token/repository access without reapplying an old plan to +newer refs. Preparation, policy and publication failures remain distinguishable +from a completed native result. No-op, rejected and non-200 Git backend responses are also terminal outcomes. Late ACL, policy and ref-CAS refusals are terminal rejected outcomes. Native @@ -646,186 +596,46 @@ collector removes those bodies yet. Account creation is idempotent only for the same account, active token digest and scope. Bootstrap accepts an issued active owner admin token and does not replace or revive credentials. -Git packs and the bare repository cache are transport and acceleration -artifacts. Neither is authoritative. The gateway can reconstruct cache objects -from the Cell and external store. Integration tests prove exact-root restore -after clean node shutdown and local SQLite loss. The process smoke also proves -Git and LFS fetch after an unclean owner exit, lease expiry and a third process -claiming the Cell from an S3-compatible object store. A proxy also drops a -successful push reply; exact replay after takeover returns its recorded report -and preserves a later ref deletion. Crashes during -individual staging/publication boundaries, the full multi-node fault matrix and -backup restore still need proof before service readiness. - -Cache hydration reads objects by an indexed, repository-local insertion sequence. -`objects.sequence` is an `INTEGER PRIMARY KEY AUTOINCREMENT`; `oid` remains unique -and non-null. Only committed insertions matter, duplicates retain their existing -sequence, and gaps are valid. SQLite never reuses a committed automatic sequence, -including after deletion; this is why the extra AUTOINCREMENT bookkeeping is -required. See [SQLite's contract](https://www.sqlite.org/autoinc.html). - -Each refresh captures the maximum stored sequence after reading its ref snapshot. -Header pages query `after < sequence <= maximum` in sequence order, so concurrent -new publications wait for a subsequent refresh. Each page contains at most 128 -records and 768 KiB of aggregate inline bodies. A second query reads the exact -missing IDs, sorted by OID, with the header receipt as its minimum observation. -A byte-limited page advances only through its selected prefix. The gateway advances -its in-memory cursor only after every selected body has verified and completed its -cache write. Failure or cancellation leaves the failed page eligible for retry; -completed files are reused. The cursor belongs to the same cache owner and resets -when that owner is discarded. `RepositoryCell::object_page(after)` retains its -OID-ordered full enumeration and uses the same header decoder and body reader. - -Each gateway retains one disk-accounted cache of verified immutable objects. -Ref snapshots, pushes and native merge candidates get separate bare directories -that borrow those objects through Git's -[alternates contract](https://git-scm.com/docs/gitrepository-layout). -The shared cache receives only verified durable Cell records and external bodies. -Native writes stay in the private generation; failed or unpublished pushes never -populate the shared cache. A successful push's new objects enter it through the -next durable hydration read. Snapshot HEAD/refs remain private and unchanged for -the lifetime of a fetch, even while new snapshots reuse the same object bytes. - -An object is compressed into an accounted temporary file, then installed at its -OID only after verification and compression finish. Incomplete files never count -as cache hits. Existing OIDs skip body queries, external downloads and compression; -refresh seeks past the completed sequence and checks local paths only for new -headers or a retried page. A ref-only refresh reads the high-water mark without -scanning old headers. No cache serves as -durable authority, and authorization still precedes cache access. - -Snapshot owners retain the shared cache through native work and streaming. If a -worker fence prevents generation cleanup, both its disk reservation and its -borrowed-object owner remain retained until process restart. Startup acquires all -generation fences before deleting any cache. This preserves borrowed objects even -when an orphan worker outlives the gateway. Failed/short writes retain conservative -charges until cache destruction; concurrent cancelled writers cannot refund another -writer's reservation through a read-then-resize race. - -Records are immutable through product APIs, and no collector removes them. -This makes the two observations safe; a future collector must fence active -reads before deleting records or bodies. The reader rejects changed/missing IDs, -invalid descriptors, and inline size, Git OID or BLAKE3 mismatches. Inline bodies -move into one bounded blocking verification task without a payload clone. -Chunked and external records carry descriptors; their body readers verify the -actual bytes before the gateway writes them to the disposable cache. Refs become -visible there only after successful hydration for the selected fetch mode. -This bounds each query, -not total repository hydration time or native Git scratch usage. The previous -single-record `next_object` API was removed from this unreleased crate. - -Debug events separate Repository Cell acquisition from successful cache -hydration. Hydration reports object count, raw bytes, admitted cache bytes, total -time, starting/completed insertion sequence, scanned/reused object counts, page-read/verification time, external/chunk body-read time and cache-write -time. Cache writes include worker scheduling, OID verification and compression; for -external Git blobs this interval also includes streamed object-store reads. -These elapsed wall times include waiting; they are not CPU profiles or a claim -that all page time belongs to SQLite execution. Acquisition includes authority -and root validation, sparse activation and publication. Failed phases remain -errors and do not emit a successful-completion event. - -Push object ingestion runs two Git processes regardless of object count: -[`rev-list --objects --no-object-names --stdin`](https://git-scm.com/docs/git-rev-list) -enumerates objects reachable from accepted new ref tips, excluding the previous -live tips; [`cat-file --batch`](https://git-scm.com/docs/git-cat-file) reads only -candidates missing from SQLite. Deletion-only pushes skip both. Enumeration is -consumed incrementally; ref roots travel over stdin rather than an unbounded -argument list. Incoming objects reachable only from rejected ref tips are not -newly persisted. - -Both commands disable replacement refs so stored bytes retain their canonical -identity. The batch reader caps headers at 128 bytes, validates kind and size -before allocation, reads the exact body and delimiter, and recomputes its Git -OID. Missing, truncated, malformed or corrupt output fails the push. Enumeration, -headers, inline/non-blob body reads, delimiters and process completion have -120-second deadlines. External blobs stream in at most 8 MiB chunks, with -120-second input/part idle deadlines and no whole-upload duration ceiling. -OID hashing uses a blocking worker so large bodies do not occupy an async -executor thread. Stderr is drained -concurrently, retaining at most 64 KiB per process. Dropping the reader kills -both direct children and aborts pipe tasks; normal completion requires both -successful exits before publishing refs or recording the successful report. -Previously published graph closure makes exclusions safe; ref publication -still requires every new tip to have a durable certificate. - -External Git blobs use multipart staging for bounded parts under a unique -repository-scoped prefix. Canopy hashes the exact declared body as it streams, -checks the Git OID, conditionally copies the parts, and creates the immutable -SHA-256 manifest. It verifies the destination before returning metadata. Conflicting canonical bytes fail; they -are never overwritten. Staging deletion and incomplete multipart abort are -attempted on ordinary failure. Process loss can leave staging/multipart debris; -operator lifecycle cleanup and qualified collection remain required. - -Hydration and backup verification read one part of at most 8 MiB at a time, -pinning the manifest version/ETag. The final range is withheld until Git SHA-1, -SHA-256 and BLAKE3 all match the SQLite reference. A failed or canceled range -invalidates its reader. Hydration compresses into disk-accounted cache files on -blocking workers; each writer retains the cache reservation until it exits. -The ref generation is not published for Git service until hydration finishes. - -External Git blobs use an immutable `CANOPY01` manifest (eight magic bytes, -then an eight-byte little-endian logical size). LFS bodies use `CANOPY02` with -one BLAKE3 digest per part after the size; SQLite pins the manifest's BLAKE3 -digest. The body bytes live at `.parts/<16-digit hex index>`. Each part is -at most 8 MiB; empty bodies have one empty part. -Parts are conditionally copied before the manifest is created, and the whole -logical body is reverified before SQLite metadata is published. Backup copies -all referenced parts before copying the manifest. There is no whole-object S3 -copy and no fixed logical file-size quota. `CANOPY02` replaces the previous LFS -manifest and digest meaning. This hard cutover requires a fresh preview prefix -or re-uploaded LFS bodies; no old-body decoder or migration is provided. - -Trees, commits and tags have no fixed byte quota, but publication and graph -parsing currently materialize individual bodies. Available worker memory remains -a practical constraint; this is not production capacity qualification. - -Candidate existence queries group up to 128 IDs. Missing records accumulate in -an `ObjectBatch` with at most 128 records and 3 MiB of aggregate inline bodies, -leaving room for metadata beneath the 4 MiB operation input limit. External -blob records count toward the record limit; their bytes are verified and -uploaded before publication. Chunk references count toward the record limit and -a 64 MiB aggregate verification target shared with inline bytes. An oversized -individual object is verified alone. The -decoder enforces these bounds before publication. One typed Cell command publishes the batch and returns one receipt. -It recomputes inline Git OIDs and BLAKE3 digests, then compares every inserted -or existing row with the complete expected record. A mismatch rejects the -command and rolls back all inserts in that batch. Repeating the same mutation -identity and payload returns the recorded result through Cellule deduplication. - -Trees, commits and tags larger than 768 KiB are staged in `object_uploads` and -`object_chunks`; their bodies stay in SQLite. Each part is at most 512 KiB and -there is no fixed number of parts per object. Part command identities derive from -the caller's upload identity and part index, so retrying the same staging -operation replays committed parts. Staging does not create an `objects` row: -queries and ref validation cannot see incomplete uploads. - -`PutObjects` codec 3 accepts a chunk reference containing upload ID, size and -BLAKE3. In the publication transaction it requires the exact chunk count and -part lengths, reconstructs the body, verifies the canonical Git OID and BLAKE3, -and then inserts the immutable object record. A failure rolls back every object -in that batch. Inline and external-blob records use the same publication command. -Command verification checks the total chunk count on the first part only: its -transaction holds a stable snapshot for the remaining indexed part reads. -Graph certification uses the same rule. Separate asynchronous object reads -retain a count check with every part because they do not share that snapshot. -Repeated identical objects converge on the existing record; unused duplicate -uploads remain staged until a collector can prove they are unreferenced. - -Hydration and direct SQLite object reads use the same chunk count, length and -hash checks. Every SQL result contains at most one chunk, fitting Cellule's -1 MiB result limit. Reads reconstruct a bounded whole object in memory. Graph -certification reconstructs chunked objects before applying the same typed edge -checks as inline objects; chunking does not certify missing edges. Part data is -immutable through product APIs, and no collector exists yet. Any future collector -must preserve chunks reachable from objects and invalidate graph certificates -before removing their dependencies. - -Object batches remain separate from ref and push-outcome publication. A later -failure can leave earlier batches unreferenced; a retry finds those records -through the bounded existence query. The final transaction still validates -graph closure before exposing new refs. Retention and collection of abandoned -objects remain open. Per-object buffers, complete cold-cache hydration and -large initial graph traversals still need capacity qualification. +Native Git packs are durable product artifacts only when the accepted catalog +certifies them. Local bare repositories remain disposable serving workspaces. +SQLite stores accepted roots and business state, not expanded Git object bodies. +The old `ObjectBatch`, chunk staging, object-page readers and mutable graph +certificate commands are removed from the library. + +Artifacts bind repository UUID, creating operation, content digest and kind. +Native packs, indexes, metadata shards, directory runs, input bodies and roots +use separate authenticated artifact paths. Reusing identical content after +retirement uses a new operation path, so a delayed delete cannot remove a later +incarnation. Publication copies bounded parts conditionally, verifies collisions +and writes the manifest only after the parts are complete. Ordinary failure +attempts staging cleanup; process loss can leave artifacts requiring a separate +product retention policy. + +Metadata builders collect canonical object/typed-edge records into immutable +files with pages of at most 512 objects. Directory runs target 64 MiB files. +The prepared catalog and joint ref snapshot replace per-object Repository Cell +writes; their final publication carries bounded roots and proofs. Building +metadata still consumes local I/O and native work. This design reduces modified +Cellule pages and publications; its actual storage reduction and throughput must +be measured on the exact executable and corpus. + +A serving session pins a certified catalog and coherent ref snapshot and checks +current actor access. Explicit fetch wants must be reachable in that snapshot; +physical pack presence alone grants no object authority. Native indexes and +selected ranges are authenticated before decoding. Selected objects are verified +against canonical metadata before entering private native workspaces. Snapshot +owners retain disk reservations and worker fences until all work closes. +Corruption or missing bytes fail the request; there is no old-body fallback. + +The current external-part format is `CANOPY02`: size and per-part BLAKE3 hashes +are pinned by the authenticated manifest digest. Readers verify requested parts +and withhold success until their complete identity checks pass. Unhashed +`CANOPY01` manifest readers and writers are removed. The separate `CANOPY01` +operation-ID namespace remains the current command identity format. +LFS uses the same verified part primitive and its own repository-scoped keys. +Backup copies referenced parts before the manifest and verifies the independent +copy. Logical artifact size is bounded by the current manifest format, not by +an S3 single-part copy limit. Git CGI responses use one subprocess/stream implementation. The HTTP gateway streams advertisements and fetch replies through four queued chunks of at most @@ -860,8 +670,8 @@ annotated-tag chains. Native Git checks target existence, peels tags and emits all protocol bytes. Commit ancestors and tree children are not traversed for this exchange. Direct blob/tree refs still require their own verified bodies; a large blob tag can therefore consume substantial discovery cache space. -Selected SQLite object reads reuse the 128-record/768-KiB inline page limit; -chunked and external objects use the same verified hydration path as transfers. +Selected objects come from the certified catalog and authenticated native +artifacts; no per-object Repository SQL body reader is used. Missing or corrupt targets fail preparation instead of disappearing from refs. A current ready cache is reused when its mutex is immediately available. Discovery @@ -1158,7 +968,7 @@ complete OOM/CPU/process/crash fault matrix require separate qualification. `data_dir/.canopy-owner.lock` prevents concurrent nodes from using one local directory. The server and repository manager retain the lock throughout their -lifetime, including detached request work. `data_dir/canopy-pack-v1/` is private to +lifetime, including detached request work. `data_dir/canopy-pack-v2/` is private to the node (created with mode 0700 on Unix) and identified by a version marker. Startup validates that marker and reclaims local SQLite files, Git caches and spools before opening Cells or advertising the node. These are disposable copies; @@ -1222,17 +1032,14 @@ workspace without confirmed drain. OS power loss, unusual filesystems and Window process containment still require qualification. This cleanup does not enforce native peak disk usage or filesystem allocation overhead. -Schema version 1 is still changing in this unreleased repository. The chunk, -HEAD, discovery, token-metadata, issue, check, branch-rule, pull/review, review-head, merge, candidate, object insertion-sequence and membership-version layouts, operations 7–10, -and the operation-5/8/9 codec changes -require a fresh development storage prefix; -there is no upgrade reader for older development databases. The module -descriptor and object paths will become compatibility boundaries at the first -persistent preview. The current source pins Cellule revision -`0dc04a658bd99668936f7ec58032d054f6fbc141`; qualification artifacts -remain bound to their recorded revisions, not silently replaced by this pin. -The earlier entity-partition -cutover also made pre-cutover prefixes incompatible; no migration is available. +The current packed Repository uses schema 4 and storage format `canopy-pack-v2`. +It requires a fresh storage prefix and local data directory. Directory keeps +schema 1 and only its current module code. The compiled module descriptors, +operation codecs, release, identity and format are checked at admission and +recovery. Each module declares one initialization descriptor; neither is an +upgrade chain. Published Cellule `=0.2.0` pins and the lockfile advance together. +Historical qualification artifacts remain bound to their original dependency +versions and binaries. ## Accounts and authentication @@ -1768,20 +1575,12 @@ ref mutation, including deletion and recreation. Otherwise deletion depends on `deny_deletions`; checks and fast-forward policy govern non-deletions. Branch creation needs checks but has no old ancestry to prove. -In the retired SQL graph contract, verified commit objects provided -`commit_parents(child, parent)` when graph closure was certified. Native -preparation reads verified catalog commit headers and uses MAC-bound ancestry -evidence; it does not populate these retired tables. The historical contract -below does not describe native production authority. Only commit-parent edges enter that table. Immutable -`commit_ancestry(ancestor, descendant)` certificates avoid graph traversal in the -ref transaction. Operation 7, codec 1 accepts at most 128 child/parent steps. -Every step must exist in verified parent links and lead either to the claimed -ancestor or to an existing certificate. Any false step rejects and rolls back -the complete command. Preparation reads parents in pages of 128, follows all -merge parents and publishes proof chunks from the known ancestor outward. -Interrupted preparation may retain positive immutable facts without changing -refs. A collector must invalidate graph and ancestry certificates before deleting -objects or their required chunks/edges. No collector is implemented yet. +Ancestry preparation reads verified catalog commit headers and produces +owner-bound native evidence. It follows merge parents through the certified +native graph and checks that evidence at final publication. It neither populates +mutable `commit_parents` / `commit_ancestry` tables nor publishes retired graph +commands. An interrupted preparation can retain immutable artifacts but cannot +move refs without passing the current final authority and policy checks. Native Git receives an executable, disposable `update` hook. Git's documented [update-hook contract](https://git-scm.com/docs/githooks#_update) supplies exact @@ -1826,13 +1625,12 @@ completion. A different push ID cannot reuse the same signed bytes: the final ref transaction records the certificate SHA-256 digest or rejects the update. The certificate body remains in SQLite chunks with its signer, key and Cell recording time. Exact request-ID retries replay the original push response. -Completed notes are stored in the same `CompletePush` transaction as the Git -response and refs. `GET /api/repositories//pushes/` returns the push +Completed notes and the native outcome are bound in the same root-completion +transaction as the accepted refs. `GET /api/repositories//pushes/` returns the push author, notes and optional certificate SHA-256, signer, key and recording time to the current repository owner or the author while they retain read access; other readers receive 404. A missing or unfinished push also returns -404. Notes are audit metadata; they do not run hooks or alter checks. This -unreleased schema and operation 4 codec 6 require a fresh development prefix. +404. Notes are audit metadata; they do not run hooks or alter checks. The current compiled schema requires a fresh storage prefix. Policy reads use batches of at most 128 statements. An `(enabled, reference)` index bounds the presence probe; rule and requirement primary keys and the check-attempt index bound final policy lookups. @@ -2749,9 +2547,9 @@ Stock `git lfs pull` without credentials verifies this behavior. Both Directory and Repository initialization schemas changed in this unreleased build. Use a fresh development prefix; existing data is not migrated or removed. -Source digest validation rejects unsupported module codes. Exact Directory -predecessor retention is a contract check, not a migration. Mixed-build rolling -upgrades are not supported. Production upgrade migration remains a delivery gate. +Source digest validation rejects unsupported module codes. Only the compiled +Directory and Repository contracts are executable. There are no retained +predecessor codes, upgrade migrations or mixed-build operation. ## Deployment release enrollment and maintenance @@ -2839,10 +2637,10 @@ acquiring any Cells. Existing tombstones are retained; competing, unexpired recovery claims remain conflicts rather than being overwritten. A bounded scan of the catalog then skips already settled controls. Each remaining -entry must match a known Canopy namespace and a namespace/role/code/schema contract -explicitly supported by the compiled registry. A supported retained predecessor -contract need not be the current module contract. This restores the existing -catalog identity; it does not upgrade its code or schema. Unknown contracts fail +entry must match a known Canopy namespace and its exact current SQL module code +and schema version. Repository schema 4 and Directory schema 1 each have one +fresh-cell initialization descriptor. This restores the existing same-release +catalog identity; it does not upgrade its code or schema. Older contracts fail before acquisition, and acquisition still validates the persisted Control. Missing controls bootstrap from the published catalog proof. Unpublished owners use `takeover_unpublished`; published owners use `takeover_restored`, which verifies @@ -2866,8 +2664,9 @@ closure, not data integrity: it is not an fsck or backup-verification result, an operators must resolve a reported restoration error before resuming. CLI logs go to stderr so successful administration output remains a single JSON status object. -The [retained-contract recovery checkpoint](performance/2026-10-02-retained-maintenance-recovery.md) -describes the regression fixtures and the separate, incomplete corpus rebuild. +The [historical retained-contract checkpoint](performance/2026-10-02-retained-maintenance-recovery.md) +describes an earlier release's fixtures. It does not describe the current +fresh-prefix contract or qualify the current executable. Crash qualification at every individual publication/claim boundary, long restore lease failures and production-store power loss remain part of the fault matrix. @@ -3058,16 +2857,17 @@ remain required. No new configuration surface was added for diagnostic output. ## Cellule integration Canopy directly uses `cellule-app`, `cellule-host`, `cellule-runtime`, -`cellule-ltx` and `cellule-store`, pinned to Cellule commit -`0dc04a658bd99668936f7ec58032d054f6fbc141`. `cellule-types` is transitive. +`cellule-ltx` and `cellule-store`, each pinned to published Cellule `=0.2.0`. +`cellule-types` is transitive and locked to the same registry release. +`python3 scripts/check_cellule.py` checks all workspace manifests and the lockfile. The lockfile contains no Crab Cell, Crab product/server or Xet packages. Historical qualification runs in the delivery/performance logs retain their original dependency revisions; they are not performance evidence for this build. The [current revision checkpoint](performance/2026-10-01-cellule-main-qualification.md) records release correctness and fresh RustFS compatibility separately from performance. The [owned retained-store qualification](performance/2026-10-01-old-binary-rustfs-upgrade.md) -records actual old-executable restore and fresh-owner recovery; it does not -provide a general upgrade controller or qualify the existing 10K corpus. +records historical old-executable restore and fresh-owner recovery. That +dependency revision and its compatibility support are not part of this build. The [listener handoff checkpoint](performance/2026-10-01-listener-handoff.md) qualifies the optional prebound HTTP listener through the same startup supervisor. Address mismatch is rejected before workspace/storage writes; diff --git a/docs/design/admitted-sqlite-growth.md b/docs/design/admitted-sqlite-growth.md index a91a9ddf..43a21e1f 100644 --- a/docs/design/admitted-sqlite-growth.md +++ b/docs/design/admitted-sqlite-growth.md @@ -32,6 +32,10 @@ Sealing closes SQLite, removes its journal, synchronizes and hashes the file, th At the 256 MiB main-file default, one empty construction spool initially charges 192 KiB instead of 768 MiB. Two such spools initially charge 384 KiB. Growing to the full ceiling can still charge 768 MiB per spool, and old immutable files, cached downloads, native workspaces and edge files retain their independent reservations. Geometric growth can deny a larger quantum before all shared free space is consumed; this is bounded backpressure, not permission to oversubscribe disk. Do not equate the initial charge with peak operation capacity. +Production staged catalog preparation gives the aggregate closure spool a separate 1 GiB main-file ceiling. It initially reserves the same 192 KiB and can grow only under the shared disk budget, reserving up to 3 GiB at the ceiling. Metadata shards retain their 256 MiB ceiling and directory output runs retain their 64 MiB bound. A full-history graph can exceed one shard’s size: the libgit2 verification corpus contains 134,967 objects and 3,158,723 typed dependency edges; a diagnostic reconstruction with the current closure schema measured 377,081,856 bytes after lookup creation. This is scratch sizing evidence, not a successful import or resource qualification. Explicit caller ceilings, denied-growth behavior, cancellation, lease fencing and the finite residence budget remain enforced. + +The default authenticated file cache retains at most 24 files under the unchanged 32 open-file limit and shared disk budget. This leaves eight slots for borrowed or queued work while accommodating a directory and the 17 metadata files produced by the libgit2 corpus. The cache regression cycles 18 authenticated files twice: a 16-file control downloads 36 files, while the default cache downloads each file once. This is a bounded working-set regression; cold-clone and provider resource qualification remain separate evidence. + Ancestry now uses admitted growth too. Native Git descendant RSS, CPU and I/O admission, large-file profiles, full-history throughput, durable authenticated input adoption, producer integration and continuous maintenance remain separate release gates. These construction changes do not select the production hard-cutover registry or prove capacity for 10,000 engineers. ## Evidence diff --git a/docs/design/bound-preparation-lifecycle.md b/docs/design/bound-preparation-lifecycle.md index 667aac42..34478442 100644 --- a/docs/design/bound-preparation-lifecycle.md +++ b/docs/design/bound-preparation-lifecycle.md @@ -14,7 +14,7 @@ A public Bound result remains recoverable after graceful stop and failed fresh c After binding or bound Claim, the same exact slot owns RenewPreparation. Its identity is created once, retained through absent/lost/panicked replies, and resolved before another renewal or checkpoint begins. A known renewal is saved in bound_renewal before fresh CheckPreparation. Renewing preserves the original attempt, namespace and generation floor; a current authorization/expiry/superseding-Claim failure fences the session without erasing the original renewal result. Unknown evidence remains admitted until exact recovery, including after close. -The existing overall staging lifetime still applies. bound_lifetime_ms adds a separate local ceiling from before preparation of Bind, or admission of bound Claim, rather than resetting it after a delayed/replayed reply. Default is 60 seconds; valid profiles use a nonzero ceiling up to MAX_LEASE_MS. The ceiling is fixed in the shared session and all derived bases. A direct fresh lease query or renewal cannot extend local use past it. The supervisor uses the raw fresh SQL deadline to schedule renewal and the residence ceiling to stop work; clipping the renewal schedule to the ceiling would cause a tight renewal loop near the limit. +The existing overall staging lifetime still applies. bound_lifetime_ms adds a separate local ceiling from before preparation of Bind, or admission of bound Claim, rather than resetting it after a delayed/replayed reply. The library default is 60 seconds; production repository residency selects the existing MAX_LEASE_MS ceiling of five minutes to admit metadata replay for real histories. SQL custody leases still renew independently at the normal interval. Valid profiles use a nonzero ceiling up to MAX_LEASE_MS. The ceiling is fixed in the shared session and all derived bases. A direct fresh lease query or renewal cannot extend local use past it. The supervisor uses the raw fresh SQL deadline to schedule renewal and the residence ceiling to stop work; clipping the renewal schedule to the ceiling would cause a tight renewal loop near the limit. The local ceiling does not shorten or remove the independent SQL pin. Already admitted or queued renewals can leave remaining SQL retention beyond local shutdown. Remaining-floor/fact capacity, command queue delay, server clock bounds and lease configuration still need qualification against the [large-team budgets](../large-team-scalability.md). Long physical imports belong in unbound staging; indefinitely renewing a bound floor is not permitted by this lifecycle. An expired local session cannot revive through exact replay or a successful later query. diff --git a/docs/git-compatibility.md b/docs/git-compatibility.md index 3d65676e..31893270 100644 --- a/docs/git-compatibility.md +++ b/docs/git-compatibility.md @@ -135,14 +135,13 @@ dedicated disk but no host-volume sharing, use `--docker-volume`; it creates and removes an isolated named volume while retaining the full size assertions. Use `--release` to run the same gate against optimized server/test code. -The gate pushes an incompressible pack above 512 MiB, checks the SQLite database -exceeds 512 MiB, stores a Git blob and LFS object above 5 GiB, restarts on fresh +The gate pushes an incompressible pack above 512 MiB, checks Repository SQLite +stays below 16 MiB, stores a Git blob and LFS object above 5 GiB, restarts on fresh local storage, then checks clone bytes, strict fsck, and the LFS download hash. The ordinary large-object integration test covers a 65 MiB commit and recovery. These are qualification points, not configured maxima or performance targets. -The pinned runtime retains its five-second SQL command deadline. Very large -non-blob verification still materializes a complete object inside a transaction; +The pinned runtime retains its five-second SQL command deadline. Structural Git verification uses bounded native streams and immutable metadata; progressive verification is needed before claiming arbitrary structural-object capacity. LTX artifact promotion and backup copies still use the configured provider's conditional-copy implementation; very large LTX artifacts need diff --git a/docs/implementation.md b/docs/implementation.md index c84097c2..3bc15eff 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -45,7 +45,7 @@ These are current admission limits, not measured production throughput: | Concurrent Git/LFS transfers | Eight per node; excess requests return `503` and `Retry-After: 1`. | | Fetch request body | Up to 64 MiB; clone/fetch response packs stream with backpressure. | | Push report | Buffered up to 64 MiB; push bodies have no fixed byte quota. | -| Git blob and LFS body | Immutable 8 MiB parts, with no fixed logical file-size quota. | +| Native Git artifacts and LFS bodies | Verified immutable parts up to 8 MiB; current manifest format supports up to 512 GiB per artifact. | | Active repository gateways | `max_active_repositories` per node (1–9,999); stored repositories may exceed this. | | Local admitted disk | Set by `local_disk_limit_bytes`; exhaustion returns `507`. | @@ -53,14 +53,13 @@ The current service supports one repository owner. Incoming Git requests stream to temporary files charged to the same disk budget as the node's SQLite files. Push bodies have no fixed byte quota; fetch requests remain bounded at 64 MiB. Push reports are buffered up to 64 MiB, while clone/fetch -packs stream with backpressure. Git blobs and LFS objects use immutable 8 MiB -parts without a fixed logical file-size quota. Trees, commits and tags above -768 KiB use SQLite chunks without a fixed individual object-size quota. -Git LFS basic downloads resume with a tail `Range` request and a verified -`206` response. The server reads only the requested parts and verifies each -against a manifest digest pinned in SQLite. -Publication and graph parsing still materialize non-blob bodies, so very large -structural objects depend on available worker memory. +packs stream with backpressure. Git bodies remain in verified native pack/index +artifacts, with canonical metadata and typed graph facts in immutable shards and +directory runs. Repository SQLite stores accepted roots and business metadata. +Structural verification streams bounded chunks. LFS downloads can resume with +verified tail ranges. Each artifact uses authenticated parts up to 8 MiB and the +current manifest's 512 GiB format bound. These bounds are not measured production +capacity. Partial uploads stay invisible to Git. Each node admits eight Git/LFS transfers across all repositories. Overload returns 503 with `Retry-After: 1`; retry after capacity is available. Health, @@ -89,7 +88,7 @@ See [native pack policy](contracts.md#native-pack-resource-policy) and ### Local workspace and shutdown -The node locks its `data_dir` and owns `canopy-pack-v1/` beneath it. On Unix, +The node locks its `data_dir` and owns `canopy-pack-v2/` beneath it. On Unix, restart removes abandoned local state before restoring Cells from object storage; live Git descendants prevent cleanup. Unknown runtime markers and cleanup errors stop startup. Keep the lock files in place; files outside the managed runtime @@ -136,10 +135,13 @@ A terminal ownership-release failure leaves that repository unavailable until node restart; confirmed-release cleanup errors are retried on later admission. There is no account deletion API, organization model or production capacity evidence. -`Cargo.toml` pins Cellule's runtime, app, host, LTX and store crates to revision -`a28de7bc09ce36d87e642adc4f4b6be50d6fcb69`. Canopy remains a separate -product crate and builds without a local Cellule checkout. There are no Crab -product/server or Xet dependencies. +The server manifest pins Cellule's runtime, app, host, LTX and store crates to +published `=0.2.0`; `Cargo.lock` binds all six Cellule crates to that registry +release. `scripts/check_cellule.py` checks every workspace member and the lockfile. +Only the compiled module code and current storage formats are supported. Directory +has no retained predecessor allowlist. Repository schema 4 has one fresh-cell +initialization descriptor and uses the native catalog/publication path from the +modular workspace. Storage format `canopy-pack-v2` rejects older deployments. ### Storage compatibility and graph verification @@ -150,40 +152,27 @@ prefixes/backups are unsupported. Startup rejects unmarked application roots, and release admission rejects a different compiled release. See the [runtime integration contract](contracts.md#cellule-integration). -Before ref publication, bounded certificate batches verify the durable Git -graph: commit trees and parents, tree entries and tag targets must exist with -the correct object type. Each batch covers at most 128 objects, targeting 64 MiB of -SQLite object bytes. Ref publication checks certified tips atomically with -permissions and ref versions; branch tips must be commits. Submodule gitlinks may name commits in another repository. -SQLite certificates let later pushes reuse validated history. Push ingestion streams -candidates from accepted ref tips, excludes previously published history, and -reads missing objects through one persistent Git batch process. Object sizes -are checked before allocation and canonical OIDs before storage. SQLite lookups -group up to 128 candidate IDs; object publication groups up to 128 records and -3 MiB of inline bytes in one Cell transaction, targeting 64 MiB of SQLite -object bytes verified per batch. A conflicting record rejects -the whole batch. Recovery tests include annotated tags, submodules and -`git fsck` on the restored clone. +Before publication, trusted native and closure verification checks canonical Git +identities and typed dependencies. Accepted roots bind the exact repository, +operation, source catalog and owner/lease. Final publication rechecks actor +access, branch policy, ref observations and exact retry identity atomically with +the recorded native outcome. There are no Git-body rows, chunk staging or mutable +object-certificate readers in the Repository Cell. ### Cache hydration and partial clone -Cold cache hydration reads insertion-ordered pages of at most 128 records and 768 KiB -of inline bodies. It verifies inline identities on a blocking worker; chunked -and external bodies retain their own verification before cache writes. Pages -reduce SQLite query overhead. Blobless fetch (`--filter=blob:none`) skips ordinary -blob bodies until explicitly requested. Other fetches enumerate missing blobs -reachable from requested tips, applying the native filter before hydration. -Tree and object-type constraints, including those in combined filters, skip -omitted bodies; size filters still need missing blob bodies. Advertised ref/tag targets are always prepared. -Non-blob history is still prepared across the repository. Supported filters and remaining gaps are listed in -[Git compatibility](git-compatibility.md). -While a gateway remains resident, verified object files are shared across private -ref snapshots, pushes and merge candidates. An indexed insertion cursor limits -refresh to newly published object headers and missing bodies. Each refresh captures -a fixed upper bound; each fully verified page advances the cursor. Failed pages -retry without skipping bytes. Native output stays private until it is -published to the Cell and subsequently verified into the reusable cache. See the -[process proof and remaining limits](performance-plan.md#indexed-object-refresh). +Serving snapshots select a certified catalog and coherent refs and recheck live +access. Native workspaces authenticate indexes and required artifact ranges; +verified selected objects enter private disk-accounted caches. A cached OID alone +cannot authorize a fetch want. Blobless and supported native filters follow the +same certified reachability rules. See [Git compatibility](git-compatibility.md) +for supported filters and qualification limits. + +Publication batches canonical metadata into immutable files instead of mutating +a Cell for each object. Metadata pages contain up to 512 objects; directory runs +target 64 MiB. Sparse reads and decoded native working sets still consume I/O, +CPU and disk. Final qualification must measure the complete corpus on one frozen +executable, including cold clones and APIs after retention collection. ## Verify a build @@ -232,7 +221,7 @@ Optional flags expand the qualification: | --- | --- | | `--large-clone` | Push two 80 MiB random blobs, then clone with protocol v0 and v2 after takeover. Each pack exceeds 64 MiB; both file hashes and `git fsck` must match. This checks transfer size, not production capacity. | | `--many-objects 256` | Push 256 files, update one file with an annotated tag, then clone after takeover. Combined with `--large-clone`, exercise resident eviction across four repositories and Git/LFS recovery before restart. | -| `--sqlite-chunks` | Push a 32,000-entry tree and commit/tag messages above 1 MiB; verify raw bytes and OIDs after takeover with strict `git fsck`. | +| `--structural-objects` | Push a 32,000-entry tree and commit/tag messages above 1 MiB; verify exact bytes after takeover and strict fsck. Add `--structural-object-bytes 68157440` to exercise 65 MiB messages. | | `--corpus-repository /path/to/existing/repository` | Read that checkout's HEAD history, bundle it into temporary fixtures under `--work-parent`, then compare every reachable object's type, size and bytes in v0/v2 clones after takeover, with strict `git fsck`. Other source branches and tags are outside this check. | For a local container store, verify that the intended data/log volume is shared @@ -243,18 +232,15 @@ completed on a dedicated Docker volume, ending at 5.4 GiB with 5.1 GiB under its internal temporary directory. Size the test store accordingly; this is an observation from that run, not a production storage bound. -To measure cold recovery, set -`RUST_LOG=warn,canopy_server::git_gateway=debug,canopy_server::server::residency=debug`. -The node reports Cell acquisition time and cache hydration time, with object -count, raw/cache bytes and time spent in page reads, body retrieval and cache -writes. Page time includes inline integrity verification; cache time includes -worker scheduling, OID verification, compression and admitted disk writes. -Use release builds for performance measurements. +Use release builds for performance measurements and retain the exact source, +lockfile, binary identity, configuration and raw timings. Measure complete cold +recovery, imports, clones, APIs and pre/post-GC storage independently. Historical +row/chunk hydration logs do not qualify the current native catalog path. -The chunk, default-branch, repository-discovery, token-metadata, collaboration, -visibility and object insertion-sequence layouts change the unreleased schema; use a fresh development storage prefix when moving -from older builds. No upgrade migration or mixed-build rolling upgrade is supported yet. -Do not reuse an existing development prefix with this changed initialization schema. +The current schema and compiled release require a fresh storage prefix and data +directory. Older-state readers, upgrade chains and mixed-release operation are +not supported. Same-release recovery, backup/restore, retries and fencing remain +required. ## Contributing @@ -269,3 +255,17 @@ Keep behavior, tests, and documentation aligned when you change Canopy: ## License Canopy declares the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) in `Cargo.toml`. + + +The `gc` administration command wraps Cellule's retention-aware collector in a +verified maintenance drain. A persistent operation/worker/workspace claim +serializes collection, recovery and maintenance completion. The sweep keeps +verified live roots and backup pins, enforces its grace and deletion bounds, and +joins work before releasing the claim. Product pack/catalog/LFS retention is a +separate policy. The operational sequence is in [operations](operations.md#cellule-retention-collection). + +The [October 15 qualification](benchmarks/2026-10-15-native-storage.md) records +full object-body/ref/HEAD/fsck/API checks for bat, Starship and libgit2 before and +after Cellule GC and fresh-workspace restoration, plus browse measurements. Full +workspace and isolated provider/process qualification are recorded separately in +that report. Historical evidence retains its original binaries and formats. diff --git a/docs/large-repository-implementation-status.md b/docs/large-repository-implementation-status.md index 8c3adb30..922b2a6b 100644 --- a/docs/large-repository-implementation-status.md +++ b/docs/large-repository-implementation-status.md @@ -1,5 +1,17 @@ # Large-repository implementation status +Current storage integration (2026-10-15): source `12dc9a1` uses published +Cellule 0.2.0, repository schema 4 and `canopy-pack-v2`, with fresh-prefix format +changes. Immutable native packs, bounded metadata publication and Cellule +retention collection are integrated. Full repository verification for bat, +Starship and libgit2 passes before and after GC/fresh-workspace restore; see the +[current qualification](benchmarks/2026-10-15-native-storage.md) for scope, +measurements and gate results. Product Git/LFS artifact collection and the +[large-team capacity amendment](large-team-scalability.md) remain open. + +The following implementation checkpoints are historical, including their +original dependency revisions, incomplete gates and source layouts. + Updated during implementation on 2026-10-05. **The full implementation and capacity goal remains open.** The [large-team amendment](large-team-scalability.md) is mandatory scope alongside the original storage design. Passing primitive tests is not completion of the hard cutover or proof of capacity. Packed cutover [PR #34](https://github.com/crabbuild/canopy/pull/34) was merged into diff --git a/docs/operations.md b/docs/operations.md index 747dac00..d103559c 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -63,8 +63,8 @@ takeover. Cross-node placement races and larger hot sets still need qualificatio Each storage prefix has one durable tenant/application identity and a selected compiled release. First startup initializes an empty deployment; later nodes must match its release and configured image identity. A nonempty catalog without -release metadata is rejected. Existing preview prefixes require explicit future -migration; do not point this build at them as an upgrade. +release metadata is rejected. Breaking schema or format changes require a fresh +storage prefix; this build has no compatibility reader or in-place upgrade path. To close admission and drain the fleet, choose a fresh operation UUID and use the same binary and configuration as the deployment: @@ -113,8 +113,9 @@ failed root verification return an error. Retry the same operation after the reported condition is resolved; recovery never resumes serving automatically. Do not remove authority records or force `drained` to bypass an error. -Upgrade/migration and object collection remain pending. Maintenance and owner -recovery do not provide a separate backup copy. +Cellule object collection uses the [retention maintenance procedure](#cellule-retention-collection). +Product-owned pack/catalog/LFS collection remains a separate policy. Maintenance +and owner recovery do not provide a separate backup copy. ### Backup and restore @@ -157,9 +158,9 @@ failed operation after repairing its reported cause. Replaying a completed resto does not reset subsequently published service state. This is a same-provider copy, not protection from losing the entire bucket or -provider. Cross-provider export, old-release migration, automated retention and -the complete interruption fault matrix remain pending. Do not use older binaries -that lack prefix reservations with these backup/restore destinations. +provider. Cross-provider export, automated retention and the complete interruption +fault matrix remain pending. Restore requires the same compiled release and +current formats; breaking changes use a fresh storage prefix. ### Recover a lost push reply @@ -189,3 +190,30 @@ publication can still return transport errors. Response bodies are limited to staging chunks currently have no expiry or collector and consume repository storage. Native Git requires an explicit native_limits configuration. See the [native resource admission contract](design/native-resource-admission.md) for vector profiles, reserved maintenance shares, HTTP 503 on exhaustion and drain ownership. Choose measured claims within OS limits and server/filesystem headroom; the default production and small evaluation profiles are estimates. The container checker includes these claims in memory and descriptor headroom checks. Missing configuration rejects startup; each foreground/maintenance share must fit a read/pack pipeline. Shutdown permanently closes native admission and waits for both class owners before Cell shutdown, workspace release and lease withdrawal. Poisoned accounting or quarantined native work can retain this drain until process restart; an elapsed shutdown deadline is not a successful drain acknowledgement. + + +## Cellule retention collection + +Use the exact deployed executable, image identity, credentials and configuration. +The current `canopy-pack-v2` release requires a fresh storage prefix; it cannot +collect an earlier release through a compatibility reader. + +1. Run `canopy maintenance config.json begin ` and stop every + serving process gracefully. For a crashed owner, run the same release's + maintenance recovery with its enrolled worker configuration. +2. Confirm `canopy maintenance config.json status` reports `drained: true`. + Expired advertisements alone do not prove closure. +3. Run `canopy gc config.json `. + The deletion bound is at most 100,000 objects per pass. Repeat bounded passes + until the JSON report has `complete: true`. +4. Verify the retained data/backup, then run + `canopy maintenance config.json end ` before restarting. + +The collector uses Cellule's verified roots and backup pins and leaves the +release in maintenance. The persisted worker claim excludes recovery and +maintenance completion during a sweep. If the worker process dies, retry with +its same node/signing identity and canonical workspace after its process has +closed; time passing does not permit a different worker to take over a paused +delete. Policy, release, workspace and drain checks fail before deletion. +Cellule collection does not delete product-owned native packs, catalogs or LFS +bodies; reclaiming unreferenced product artifacts remains a separate policy. diff --git a/docs/repository-cell-primitives.md b/docs/repository-cell-primitives.md index 47170fcf..9f8af9b7 100644 --- a/docs/repository-cell-primitives.md +++ b/docs/repository-cell-primitives.md @@ -49,8 +49,8 @@ recovery lifecycle. Nodes must host thousands of these repository identities with bounded memory, disk, descriptors, workers and queued operations. All existing Git smart-HTTP, LFS, collaboration, ACL and recovery requirements -remain. Ordinary Git objects stay in SQLite; large bodies stay in immutable -object storage. A SQL-only density measurement does not qualify the complete +remain. Git bodies stay in immutable native pack artifacts; accepted roots and business +metadata stay in the Repository Cell. A SQL-only density measurement does not qualify the complete primitive-enabled repository design. ## Current contract gap @@ -63,7 +63,7 @@ in `crates/canopy-server/src/lib.rs`. `RepositoryCell` holds `SqlCell 4096), "commit truncation mismatch") + + +def tree_entries(repo, revision, path=b""): + target = revision + (":" + os.fsdecode(path) if path else "^{tree}") + result = [] + for record in git(repo, "ls-tree", "-z", target).split(b"\0"): + if record: + meta, name = record.split(b"\t", 1) + mode, kind, oid = meta.split() + result.append((name, mode.decode(), kind.decode(), oid.decode())) + return sorted(result) + + +def verify(args, client): + corpus = json.loads(args.corpus.read_text()) + queries, proofs = [], [] + for repo in corpus["repositories"]: + clone = args.output / "clones" / f"{repo['name']}.git" + clone.parent.mkdir(exist_ok=True) + started = time.monotonic() + cloned = subprocess.run(["git", "clone", "--mirror", repo["clone_url"], str(clone)], + env=git_environment(client.token), capture_output=True, timeout=1800) + clone_seconds = time.monotonic() - started + (args.output / f"{repo['name']}-clone.log").write_bytes(cloned.stdout + cloned.stderr) + require(cloned.returncode == 0, "clone-back failed") + checked = subprocess.run(["git", "-C", str(clone), "fsck", "--full"], capture_output=True, timeout=600) + (args.output / f"{repo['name']}-fsck.log").write_bytes(checked.stdout + checked.stderr) + require(checked.returncode == 0, "clone-back fsck failed") + original_objects = set(git(repo, "rev-list", "--objects", "--all", "--no-object-names").splitlines()) + restored_objects = set(subprocess.check_output(["git", "-C", str(clone), "rev-list", "--objects", "--all", "--no-object-names"]).splitlines()) + require(original_objects == restored_objects, "clone-back object graph differs") + expected_refs = dict(line.split() for line in git(repo, "for-each-ref", "--format=%(refname) %(objectname)", + "refs/heads", "refs/tags").decode().splitlines()) + seen, query = {}, {"kind": "refs"} + while True: + view, _ = post(client, repo, query) + page = view["refs"] + seen.update((entry["name"], entry["oid"]) for entry in page["entries"]) + if page["next_after"] is None: + break + query = {"kind": "refs", "after": page["next_after"], "generation": page["generation"]} + require(seen == expected_refs, f"{repo['name']}: imported refs mismatch") + variants = [("refs", {"kind": "refs"})] + files = [] + for record in git(repo, "ls-tree", "-rlz", repo["commit"]).split(b"\0"): + if not record: + continue + meta, path = record.split(b"\t", 1) + mode, kind, oid, size = meta.split() + if kind == b"blob" and mode == b"100644" and 0 < int(size) <= 256 * 1024: + files.append((int(size), path, oid.decode())) + selected = [] + for target in (1024, 16 * 1024, 128 * 1024): + candidate = min(files, key=lambda item: abs(item[0] - target)) + if candidate not in selected: + selected.append(candidate) + directories = {b"", *(path.rpartition(b"/")[0] for _, path, _ in selected)} + for path in sorted(directories): + query = {"kind": "tree", "commit": repo["commit"], "path_base64": encoded(path)} + view, _ = post(client, repo, query) + actual = view["tree"] + check_commit(repo, actual["commit"]) + expected = tree_entries(repo, repo["commit"], path) + require([(decoded(entry["name_base64"]), entry["mode"], entry["oid"]) for entry in actual["entries"]] + == [(name, mode, oid) for name, mode, _, oid in expected[:32]], "tree page mismatch") + require(actual["next_after"] == (encoded(expected[31][0]) if len(expected) > 32 else None), "tree cursor mismatch") + variants.append(("tree", query)) + for size, path, oid in selected: + query = {"kind": "file", "commit": repo["commit"], "path_base64": encoded(path)} + view, _ = post(client, repo, query) + actual = view["file"] + require(actual["content_status"] == "included" and actual["size"] == size and actual["oid"] == oid, + "file metadata mismatch") + require(decoded(actual["content_base64"]) == git(repo, "cat-file", "blob", oid), "blob bytes mismatch") + variants.append(("blob", query)) + history = git(repo, "rev-list", "--first-parent", "-n", "545", repo["commit"]).decode().splitlines() + for offset in (0, 32, 128, 512): + if offset >= len(history): + continue + query = {"kind": "history", "commit": history[offset]} + view, _ = post(client, repo, query) + actual = view["history"] + expected = history[offset:offset + 32] + require([commit["oid"] for commit in actual["commits"]] == expected, "first-parent history mismatch") + for commit in actual["commits"]: + check_commit(repo, commit) + require(actual["next_commit"] == (history[offset + 32] if len(history) > offset + 32 else None), "history cursor mismatch") + variants.append(("commits", query)) + status, _ = client.request(f"/api/repositories/{repo['name']}/browse", + {"repository_id": repo["repository_id"], "query": {"kind": "attribution"}}) + require(status == 422, "unexpected attribution support; update benchmark coverage") + for operation, query in variants: + _, body = post(client, repo, query) + queries.append({"repository": repo["name"], "operation": operation, + "path": f"/api/repositories/{repo['name']}/browse", + "payload": {"repository_id": repo["repository_id"], "query": query}, + "response_sha256": hashlib.sha256(body).hexdigest(), "response_bytes": len(body)}) + proofs.append({"repository": repo["name"], "refs_verified": len(seen), + "clone_seconds": clone_seconds, "clone_fsck_passed": True, + "clone_objects_verified": len(restored_objects), + "commit_metadata_and_attribution_verified": sum(min(32, len(history) - offset) for offset in (0, 32, 128, 512) if offset < len(history)), + "files": [{"path": os.fsdecode(path), "bytes": size, "oid": oid} for size, path, oid in selected], + "attribution_query_status": status}) + print(f"VERIFIED {repo['name']}: {len(seen)} refs, trees, exact blob bytes, history and author/committer metadata", flush=True) + save(args.output / "queries.json", {"queries": queries}) + save(args.output / "verification.json", {"repositories": proofs, "attribution_api": "not implemented; browse kind rejected with 422"}) + + +def distribution(values): + values = sorted(values) + return {**{f"p{p}": values[max(0, math.ceil(len(values) * p / 100) - 1)] for p in (50, 95, 99)}, + "min": values[0], "max": values[-1], "mean": sum(values) / len(values)} if values else None + + +def measure(args, token): + queries = json.loads((args.output / "queries.json").read_text())["queries"] + groups = {} + for query in queries: + groups.setdefault((query["repository"], query["operation"]), []).append(query) + if len(groups) > 1: + groups[("all", "mixed")] = [variants[min(1, len(variants) - 1)] if operation == "blob" else variants[0] + for (_, operation), variants in list(groups.items())] + plan = [(key, concurrency, repeat) for key in groups for concurrency in args.concurrency for repeat in range(args.repeats)] + random.Random(42).shuffle(plan) + report = {"method": "closed-loop, persistent HTTP/1.1 connections, full response read, no retries", + "mixed_workload": "equal rotation over one query for each repository/API pair; medium sampled blob", + "duration_per_group_seconds": args.duration, "concurrency": args.concurrency, + "repeats": args.repeats, "results": []} + samples_path = args.output / "samples.jsonl" + require(not samples_path.exists(), "measurement output already exists") + with samples_path.open("w") as samples: + for (repo, operation), concurrency, repeat in plan: + group_unix_seconds = time.time() + client = Client(args.base_url, token, 130) + variants = groups[(repo, operation)] + # Establish and validate the entire working set before timed requests. + for variant in variants: + status, body = client.request(variant["path"], variant["payload"]) + require(status == 200 and hashlib.sha256(body).hexdigest() == variant["response_sha256"], "warmup validation failed") + gate = threading.Barrier(concurrency + 1) + started = [0.0] + + def worker(index): + observations = [] + gate.wait() + sequence = index + while time.monotonic() < started[0] + args.duration: + variant_index = sequence % len(variants) + variant = variants[variant_index] + before = time.monotonic() + try: + status, body = client.request(variant["path"], variant["payload"]) + elapsed = (time.monotonic() - before) * 1000 + outcome = "ok" if status == 200 and hashlib.sha256(body).hexdigest() == variant["response_sha256"] else f"http_{status}_or_invalid_body" + size = len(body) + except Exception as error: + elapsed = (time.monotonic() - before) * 1000 + status, size, outcome = 0, 0, type(error).__name__ + observations.append({"latency_ms": elapsed, "status": status, "bytes": size, + "outcome": outcome, "variant": variant_index}) + sequence += 1 + return observations + + try: + with ThreadPoolExecutor(max_workers=concurrency) as pool: + tasks = [pool.submit(worker, index) for index in range(concurrency)] + started[0] = time.monotonic() + gate.wait() + observations = [sample for task in tasks for sample in task.result()] + elapsed = time.monotonic() - started[0] + finally: + client.close() + identity = {"repository": repo, "operation": operation, "concurrency": concurrency, "repeat": repeat + 1} + outcomes = Counter(sample["outcome"] for sample in observations) + result = {**identity, "unix_seconds": group_unix_seconds, + "requests": len(observations), "outcomes": dict(outcomes), "elapsed_seconds": elapsed, + "successful_requests_per_second": outcomes["ok"] / elapsed, + "response_mib_per_second": sum(sample["bytes"] for sample in observations if sample["outcome"] == "ok") / 1024**2 / elapsed, + "latency_ms_all_attempts": distribution([sample["latency_ms"] for sample in observations]), + "latency_ms_successful": distribution([sample["latency_ms"] for sample in observations if sample["outcome"] == "ok"])} + for sample in observations: + samples.write(json.dumps({**identity, **sample}) + "\n") + samples.flush() + report["results"].append(result) + save(args.output / "measurements.json", report) + successful = result["latency_ms_successful"] + successful_p95 = f"{successful['p95']:.1f}" if successful else "n/a" + print(f"MEASURED {repo}/{operation} c={concurrency} repeat={repeat + 1}: {result['successful_requests_per_second']:.1f} successful req/s, p95(success)={successful_p95} ms, p95(all)={result['latency_ms_all_attempts']['p95']:.1f} ms, errors={len(observations) - outcomes['ok']}", flush=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("import", "verify", "measure")) + parser.add_argument("--base-url", required=True) + parser.add_argument("--corpus", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--duration", type=float, default=8) + parser.add_argument("--concurrency", type=int, nargs="+", default=[1, 4, 16]) + parser.add_argument("--repeats", type=int, default=2) + args = parser.parse_args() + require(args.duration > 0 and args.repeats > 0 and all(1 <= n <= 64 for n in args.concurrency), "invalid workload") + args.output.mkdir(parents=True, exist_ok=True) + token = os.environ["CANOPY_GIT_TOKEN"] + if args.action == "measure": + measure(args, token) + return + client = Client(args.base_url, token, 130) + try: + if args.action == "import": + import_repositories(args, client, token) + else: + verify(args, client) + finally: + client.close() + + +if __name__ == "__main__": + main() diff --git a/scripts/benchmark_large_repository.py b/scripts/benchmark_large_repository.py index 8ffe8966..28239a08 100644 --- a/scripts/benchmark_large_repository.py +++ b/scripts/benchmark_large_repository.py @@ -114,7 +114,7 @@ def sample(): "process_tree_rss_bytes": rss}) + "\n") # MAX(sequence) uses the integer primary key. Avoid # table scans or long transactions on the live Cell. - for database in (args.state_dir / "node" / "canopy-pack-v1").glob("*/repository.sqlite"): + for database in (args.state_dir / "node" / "canopy-pack-v2").glob("*/repository.sqlite"): try: connection = sqlite3.connect(database.as_uri() + "?mode=ro", uri=True, timeout=0.2) try: @@ -258,7 +258,7 @@ def api(path, payload=None, method=None): git("prepare-pull-client", "clone", "--shared", "--single-branch", "--branch", default_ref.removeprefix("refs/heads/"), str(args.work_dir / "warm-v2.git"), str(pull_work)) git("configure-pull-client", "-C", str(pull_work), "remote", "set-url", "origin", url) - # The binary's startup contract removes canopy-pack-v1 before recovering + # The binary's startup contract removes canopy-pack-v2 before recovering # authoritative Cells from the provider; no manual deletion is needed. commit_env = {**git_env, "GIT_AUTHOR_NAME": "Canopy Evaluation", "GIT_AUTHOR_EMAIL": "evaluation@example.invalid", diff --git a/scripts/check_cellule.py b/scripts/check_cellule.py new file mode 100644 index 00000000..6588b276 --- /dev/null +++ b/scripts/check_cellule.py @@ -0,0 +1,68 @@ +"""Require one exact crates.io Cellule release, including transitive crates.""" + +from pathlib import Path +import re +import tomllib + + +def workspace_manifests(root): + """Read the virtual/root manifest and every explicitly declared member.""" + manifest = tomllib.loads((root / "Cargo.toml").read_text()) + manifests = [manifest] + members = set() + for pattern in manifest.get("workspace", {}).get("members", []): + members.update(path / "Cargo.toml" for path in root.glob(pattern)) + for path in sorted(members): + if path != root / "Cargo.toml": + manifests.append(tomllib.loads(path.read_text())) + return manifests + + +def check(manifest, lock): + pins = {} + + def dependencies(table): + for key, value in table.items(): + if key in ("dependencies", "dev-dependencies", "build-dependencies"): + for alias, dependency in value.items(): + spec = {"version": dependency} if isinstance(dependency, str) else dependency + name = spec.get("package", alias) + if not name.startswith("cellule-"): + continue + version = spec.get("version", "") + if (not re.fullmatch(r"=\d+\.\d+\.\d+", version) + or any(field in spec for field in ("git", "path", "registry", "workspace"))): + raise ValueError(f"{name} must pin an exact stable crates.io release") + pins.setdefault(name, set()).add(version[1:]) + elif isinstance(value, dict): + dependencies(value) + + for item in manifest if isinstance(manifest, list) else [manifest]: + dependencies(item) + versions = {version for values in pins.values() for version in values} + if len(versions) != 1: + raise ValueError("all direct Cellule dependencies must pin the same release") + version = versions.pop() + locked = {} + for package in lock["package"]: + name = package["name"] + if not name.startswith("cellule-"): + continue + if (name in locked or package["version"] != version + or package.get("source") != "registry+https://github.com/rust-lang/crates.io-index" + or not package.get("checksum")): + raise ValueError(f"{name} must resolve once to crates.io {version}") + locked[name] = package + if missing := pins.keys() - locked.keys(): + raise ValueError(f"Cellule dependencies missing from Cargo.lock: {sorted(missing)}") + return version, len(locked) + + +if __name__ == "__main__": + root = Path(__file__).resolve().parents[1] + try: + with (root / "Cargo.lock").open("rb") as lock: + version, count = check(workspace_manifests(root), tomllib.load(lock)) + except ValueError as error: + raise SystemExit(f"Cellule dependency policy: {error}") from error + print(f"Cellule {version}: {count} crates locked to one exact registry release") diff --git a/scripts/qualify_size.py b/scripts/qualify_size.py index 8f5b8113..fa7a900f 100644 --- a/scripts/qualify_size.py +++ b/scripts/qualify_size.py @@ -4,6 +4,7 @@ from pathlib import Path import shutil import subprocess +import sys import tempfile import time import uuid @@ -38,10 +39,19 @@ def main(): parser.add_argument("--provider-only", action="store_true") parser.add_argument("--docker-volume", action="store_true") parser.add_argument("--release", action="store_true") + parser.add_argument("--process-binary", type=Path, + help="Also exercise this Canopy binary through restart, takeover, peers and backup") args = parser.parse_args() + if args.process_binary is not None: + args.process_binary = args.process_binary.resolve(strict=True) name = f"canopy-size-{uuid.uuid4().hex[:12]}" with tempfile.TemporaryDirectory(prefix=name) as temporary: directory = Path(temporary).resolve() + process_binary = directory / "canopy" + if args.process_binary is not None: + # Every restart must execute the same build, even if Cargo replaces + # its output while the fixture is running. + shutil.copy2(args.process_binary, process_binary) image = "rustfs/rustfs:1.0.0-beta.8-glibc" owner = f"{os.getuid()}:{os.getgid()}" volume = None @@ -102,7 +112,7 @@ def main(): env["CANOPY_TEST_S3_ENDPOINT"] = endpoint env["CANOPY_TEST_S3_BUCKET"] = "canopy-size" tests = [] if args.provider_only else [ - "size::push_and_database_exceed_512_mib_and_lfs_exceeds_5_gib_after_restore" + "size::native_pack_exceeds_512_mib_and_git_lfs_exceed_5_gib_after_restore" ] tests.extend([ "sha256::sha256_real_provider_round_trip", @@ -126,6 +136,15 @@ def main(): subprocess.run([*cargo, test, "--", "--exact", "--ignored", "--nocapture"], env=env, check=True, cwd=Path(__file__).resolve().parents[1]) + if args.process_binary is not None: + process_env = {**env, "AWS_ENDPOINT": endpoint, "AWS_ALLOW_HTTP": "true", + "CANOPY_NODE_SIGNING_KEY_HEX": os.urandom(32).hex()} + subprocess.run([ + sys.executable, "-B", str(Path(__file__).with_name("smoke_s3_process.py")), + "--binary", str(process_binary), + "--storage-url", "s3://canopy-size/process-qualification", + "--work-parent", str(directory), + ], env=process_env, check=True) except Exception: if started: subprocess.run(["docker", "logs", "--tail", "60", name]) diff --git a/scripts/smoke_s3_backup.py b/scripts/smoke_s3_backup.py index c79cf9a9..5197244e 100644 --- a/scripts/smoke_s3_backup.py +++ b/scripts/smoke_s3_backup.py @@ -69,12 +69,28 @@ def administer(action, *roots, fails=False): raise RuntimeError(f"backup {action} failed: {error}") return json.loads(result.stdout) - saved = administer("create", backup_prefix) - assert saved["cells"] == 2 and saved["external_objects"] == 3 - assert administer("create", backup_prefix) == saved aws = ["aws"] if os.environ.get("AWS_ENDPOINT"): aws.extend(["--endpoint-url", os.environ["AWS_ENDPOINT"]]) + + def external_roots(prefix): + root = f"{prefix}/repos/{uuid.UUID(repository).hex}/" + listing = subprocess.run([ + *aws, "s3api", "list-objects-v2", "--bucket", + urllib.parse.urlsplit(backup_url).netloc, "--prefix", root, + ], check=True, capture_output=True) + return sorted(item["Key"][len(root):] + for item in json.loads(listing.stdout).get("Contents", []) + if ".parts/" not in item["Key"]) + + saved = administer("create", backup_prefix) + roots = external_roots(backup_prefix) + assert {root.split("/")[0] for root in roots} == { + "git-packs", "git-catalogs", "git-inputs", "lfs"} + assert [root for root in roots if root.startswith("lfs/")] == sorted([ + f"lfs/{empty_oid}", f"lfs/{hashlib.sha256(lfs).hexdigest()}"]) + assert saved["cells"] == 2 and saved["external_objects"] == len(roots) + assert administer("create", backup_prefix) == saved lfs_url = f"{backup_url}/repos/{uuid.UUID(repository).hex}/lfs/{hashlib.sha256(lfs).hexdigest()}.parts/0000000000000000" corrupt = directory / "corrupt-backup-lfs" corrupt.write_bytes(b"corrupt fixture bytes") @@ -96,7 +112,9 @@ def administer(action, *roots, fails=False): assert json.loads(listing.stdout).get("KeyCount", 0) == 0 assert administer("verify", backup_prefix) == saved restored = administer("restore", backup_prefix, restore_prefix) - assert restored["cells"] == 2 and restored["external_objects"] == 3 + assert restored["cells"] == saved["cells"] + assert restored["external_objects"] == saved["external_objects"] + assert external_roots(restore_prefix) == roots node, restored_base = start(binary, directory, {**settings, "storage_url": restored_url, "node_id": str(uuid.uuid4())}, "backup-new-node") processes.append(node) @@ -107,5 +125,12 @@ def administer(action, *roots, fails=False): node.send_signal(signal.SIGTERM) node.wait(timeout=30) assert node.returncode == 0 + (directory / "backup-results.json").write_text(json.dumps({ + "saved": saved, "restored": restored, + "external_manifest_count": len(roots), + "external_inventory_sha256": hashlib.sha256("\n".join(roots).encode()).hexdigest(), + "source_deleted": True, "corrupt_copy_rejected": True, + "cold_clone_verified": True, "issue_verified": True, + }, indent=2)) print(f"LFS qualification: bytes={len(lfs)}, stock push={push_seconds:.2f}s, restored clone+verification={clone_seconds:.2f}s", flush=True) - print("PASS: stock 80 MiB Git blob plus 80 MiB and empty LFS transfers, then real backup CLI copies SQLite and external bodies; deleting every original source object still permits verification, isolated restore, exact stock clone/LFS and issue recovery", flush=True) + print("PASS: stock 80 MiB Git blob plus 80 MiB and empty LFS transfers, then real backup CLI copies SQLite and native artifacts; deleting every original source object still permits verification, isolated restore, exact stock clone/LFS and issue recovery", flush=True) diff --git a/scripts/smoke_s3_cache.py b/scripts/smoke_s3_cache.py index f69bbb17..0f571e5b 100644 --- a/scripts/smoke_s3_cache.py +++ b/scripts/smoke_s3_cache.py @@ -113,7 +113,7 @@ def clone(name, protocol): def cached_objects(data): objects = {} - for cache in (data / "canopy-pack-v1").glob("canopy-git-*/repo.git"): + for cache in (data / "canopy-pack-v2").glob("canopy-git-*/repo.git"): if (cache / "objects/info/alternates").exists(): continue for path in (cache / "objects").glob("*/*"): diff --git a/scripts/smoke_s3_peers.py b/scripts/smoke_s3_peers.py index 351af300..b6d2838b 100644 --- a/scripts/smoke_s3_peers.py +++ b/scripts/smoke_s3_peers.py @@ -1,57 +1,25 @@ """Two live HTTPS peers, remote Git/LFS, and takeover without gateway restart.""" from contextlib import contextmanager -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -import http.client import json import os import signal import ssl import subprocess -import threading import time import uuid from lease_contract import NODE_LEASE_WAIT_SECONDS +from local_tcp_proxy import LocalProxy @contextmanager def proxy(upstream, certificate, key): - class Handler(BaseHTTPRequestHandler): - protocol_version = "HTTP/1.1" - - def log_message(self, *_args): - pass - - def do_POST(self): - body = self.rfile.read(int(self.headers["Content-Length"])) - connection = http.client.HTTPConnection(upstream, timeout=30) - try: - connection.request("POST", self.path, body, - {"Content-Type": "application/octet-stream"}) - response = connection.getresponse() - reply = response.read() - self.send_response(response.status) - self.send_header("Content-Length", str(len(reply))) - self.end_headers() - self.wfile.write(reply) - except (OSError, http.client.HTTPException): - self.send_error(502) - finally: - connection.close() - context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) context.load_cert_chain(certificate, key) - server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) - server.socket = context.wrap_socket(server.socket, server_side=True) - worker = threading.Thread(target=server.serve_forever, daemon=True) - worker.start() - try: - yield f"https://127.0.0.1:{server.server_port}" - finally: - server.shutdown() - server.server_close() - worker.join() + # Preserve Git/LFS methods, authenticated headers, and streaming bodies. + with LocalProxy([upstream], tls_context=context) as relay: + yield relay.url def peer_certificate(directory): diff --git a/scripts/smoke_s3_process.py b/scripts/smoke_s3_process.py index 4eb932a9..2dba2894 100644 --- a/scripts/smoke_s3_process.py +++ b/scripts/smoke_s3_process.py @@ -28,6 +28,7 @@ from lease_contract import NODE_LEASE_WAIT_SECONDS import smoke_s3_account_audit import smoke_s3_backup +import smoke_s3_retention import smoke_s3_branch_rules import smoke_s3_checks import smoke_s3_corpus @@ -335,6 +336,20 @@ def clone_and_verify(url, directory, expected_oid, expected_readme, expected_lfs assert (directory / "asset.lfs").read_bytes() == expected_lfs +def hydrate_fetch_cache(url, directory, expected_oid, expected_readme, branch="main"): + authorization = "http.extraHeader=Authorization: Bearer local-test-token" + git("-c", authorization, "clone", "--filter=blob:none", "--no-checkout", + "--single-branch", "--branch", branch, url, str(directory)) + readme = git("rev-parse", f"{expected_oid.decode()}:README.md", cwd=directory).decode() + actual = subprocess.run( + ["git", "-c", "credential.helper=", "-c", authorization, "cat-file", "blob", readme], + cwd=directory, capture_output=True, check=True, + env={**os.environ, "GIT_TERMINAL_PROMPT": "0"}, + ).stdout + assert actual == expected_readme + return readme + + def push_with_lost_reply(base_url, local): captured = {} push_id = str(uuid.uuid4()) @@ -492,8 +507,8 @@ def verify_many_objects(base_url, directory, count, expected): print(f"PASS: {count}-file clone restored both commits, annotated tag and 300 refs after takeover in {time.monotonic() - started:.2f}s", flush=True) -def seed_sqlite_chunks(base_url, directory): - url, _ = create_repository(base_url, "sqlite-chunks") +def seed_structural_objects(base_url, directory, message_bytes): + url, _ = create_repository(base_url, "structural-objects") local = directory / "chunk-source" git("init", "--bare", str(local)) expected = {} @@ -513,26 +528,26 @@ def write(kind, body): commit = write("commit", ( f"tree {tree}\nauthor Canopy 0 +0000\n" "committer Canopy 0 +0000\n\n" - ).encode() + b"c" * 1_100_000 + b"\n") + ).encode() + b"c" * message_bytes + b"\n") tag = write("tag", ( f"object {commit}\ntype commit\ntag release\n" "tagger Canopy 0 +0000\n\n" - ).encode() + b"t" * 1_100_000 + b"\n") + ).encode() + b"t" * message_bytes + b"\n") git("update-ref", "refs/heads/main", commit, cwd=local) git("symbolic-ref", "HEAD", "refs/heads/main", cwd=local) git("update-ref", "refs/tags/release", tag, cwd=local) started = time.monotonic() git("-c", "http.extraHeader=Authorization: Bearer local-test-token", "push", url, "refs/heads/main", "refs/tags/release", cwd=local) - print(f"PASS: large tree, commit and tag pushed into SQLite chunks in {time.monotonic() - started:.2f}s", flush=True) + print(f"PASS: large tree, commit and tag pushed into native packs in {time.monotonic() - started:.2f}s", flush=True) return expected -def verify_sqlite_chunks(base_url, directory, expected): +def verify_structural_objects(base_url, directory, expected): clone = directory / "chunk-restored" started = time.monotonic() git("-c", "http.extraHeader=Authorization: Bearer local-test-token", - "clone", "--bare", f"{base_url}/canopy/sqlite-chunks.git", str(clone)) + "clone", "--bare", f"{base_url}/canopy/structural-objects.git", str(clone)) for kind, (oid, path) in expected.items(): # run()/git() strip whitespace for command output. Compare raw object bytes here. actual = subprocess.run(["git", "cat-file", kind, oid], cwd=clone, @@ -541,7 +556,7 @@ def verify_sqlite_chunks(base_url, directory, expected): assert git("rev-parse", "refs/heads/main", cwd=clone).decode() == expected["commit"][0] assert git("rev-parse", "refs/tags/release", cwd=clone).decode() == expected["tag"][0] git("fsck", "--strict", "--full", cwd=clone) - print(f"PASS: SQLite chunks restored exact tree/commit/tag bytes and OIDs after takeover in {time.monotonic() - started:.2f}s", flush=True) + print(f"PASS: Native packs restored exact tree/commit/tag bytes and OIDs after takeover in {time.monotonic() - started:.2f}s", flush=True) def main(): @@ -551,11 +566,15 @@ def main(): parser.add_argument("--work-parent", type=Path, required=True) parser.add_argument("--large-clone", action="store_true", help="Qualify a push and v0/v2 clones above 64 MiB, including takeover") parser.add_argument("--many-objects", type=int, default=0, metavar="COUNT", help="Qualify many small objects, an incremental push, and takeover recovery") - parser.add_argument("--sqlite-chunks", action="store_true", help="Qualify large tree, commit and tag objects stored in SQLite and restored after takeover") + parser.add_argument("--structural-objects", action="store_true", help="Qualify large tree, commit and tag objects stored in native packs and restored after takeover") + parser.add_argument("--structural-object-bytes", type=int, default=1_100_000, + help="Commit/tag message bytes for --structural-objects") parser.add_argument("--corpus-repository", type=Path, help="Read HEAD history from an existing repository without changing it; qualify exact object bytes after takeover") parser.add_argument("--peers-only", action="store_true", help="Run the two-node HTTPS Git/LFS and takeover gate without the other process suites") parser.add_argument("--retain-work-dir", action="store_true", help="Retain the temporary workspace and server logs for diagnosis") args = parser.parse_args() + if args.structural_object_bytes < 1: + parser.error("--structural-object-bytes must be positive") if args.many_objects < 0: parser.error("--many-objects must be nonnegative") for name in ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "CANOPY_NODE_SIGNING_KEY_HEX"): @@ -718,7 +737,7 @@ def main(): expiring_token = seed_expiring_token(base_url) disabled_tokens = seed_disabled_account(base_url) audit_state = smoke_s3_account_audit.seed(base_url) - chunks = seed_sqlite_chunks(base_url, directory) if args.sqlite_chunks else None + chunks = seed_structural_objects(base_url, directory, args.structural_object_bytes) if args.structural_objects else None large = seed_large_repository(base_url, directory) if args.large_clone else None many = seed_many_objects(base_url, directory, args.many_objects) if args.many_objects else None corpus = None @@ -753,11 +772,15 @@ def main(): verify_discovery(base_url, reader_token, ["renamed"]) clone_and_verify(url, directory / "clean-clone", oid, b"Canopy process smoke\n", lfs_body, branch="trunk") clone_and_verify(f"{base_url}/canopy/other.git", directory / "clean-other", other_oid, other_readme) + # Materialize real lazy-fetch hydration scratch for owner-death cleanup. + readme = hydrate_fetch_cache(url, directory / "partial-before-owner-death", + oid, b"Canopy process smoke\n", branch="trunk") second.kill() second.wait(timeout=10) - abandoned = directory / "second" / "canopy-pack-v1" - stale_caches = list(abandoned.glob("canopy-git-*")) - assert stale_caches, "expected a retained fetch cache at owner death" + abandoned = directory / "second" / "canopy-pack-v2" + stale_caches = list(abandoned.rglob("canopy-git-*")) + assert any((cache / "repo.git" / "objects" / readme[:2] / readme[2:]).is_file() + for cache in stale_caches), "expected the hydrated blob cache at owner death" sentinel = abandoned / "abandoned-upload" sentinel.write_bytes(b"disposable scratch") # SQLite is a recovered cache: corrupting the killed node's copy must @@ -851,7 +874,7 @@ def main(): if large is not None: verify_large_clone(base_url, directory, large) if chunks is not None: - verify_sqlite_chunks(base_url, directory, chunks) + verify_structural_objects(base_url, directory, chunks) if many is not None: verify_many_objects(base_url, directory, args.many_objects, many) if corpus is not None: @@ -862,6 +885,7 @@ def main(): raise RuntimeError("takeover owner did not shut down cleanly") smoke_s3_peers.qualify(args.binary, directory, settings, processes) smoke_s3_backup.qualify(args.binary, directory, settings, processes) + smoke_s3_retention.qualify(args.binary, directory, settings, processes) assert {path.name for path in (directory / "host-git").iterdir()} == {"config"} print("PASS: native Git ignores host config, object paths, exec path, protocol and trace settings", flush=True) print("PASS: Git/LFS, repository discovery, default branch, ACL, ref outcomes, and a dropped push reply survived restart, disk loss, and lease takeover") diff --git a/scripts/smoke_s3_retention.py b/scripts/smoke_s3_retention.py new file mode 100644 index 00000000..9244bc18 --- /dev/null +++ b/scripts/smoke_s3_retention.py @@ -0,0 +1,92 @@ +"""Qualify fenced Cellule collection with retained backup pins and cold clones.""" + +import json +import signal +import subprocess +import time +import urllib.parse +import uuid + + +def qualify(binary, directory, settings, processes): + from smoke_s3_process import start, create_repository, git + + settings = {**settings, "storage_url": settings["storage_url"] + "/retention"} + server, base = start(binary, directory, settings, "retention-source") + processes.append(server) + url, _ = create_repository(base, "retained") + source = directory / "retention-client" + git("init", "-b", "main", str(source)) + git("config", "user.name", "Retention qualification", cwd=source) + git("config", "user.email", "retention@example.invalid", cwd=source) + config = directory / "retention-source.json" + + def cli(*args, fails=False): + result = subprocess.run([str(binary), *args], capture_output=True, timeout=180) + if fails: + assert result.returncode != 0, "administration accepted an unsafe collection" + return + if result.returncode: + raise RuntimeError(result.stderr.decode(errors="replace")) + return json.loads(result.stdout) + + for revision in range(6): + for index in range(64): + (source / f"file-{index}").write_bytes( + (f"revision={revision}, file={index}\n".encode() * 1024)) + git("add", ".", cwd=source) + git("commit", "-m", f"Revision {revision}", cwd=source) + git("-c", "http.extraHeader=Authorization: Bearer local-test-token", + "push", url, "HEAD:refs/heads/main", cwd=source) + if revision == 0: + # Freeze a published snapshot before backup capture. A serving + # owner's asynchronous checkpoint may legitimately change controls. + server.send_signal(signal.SIGTERM) + server.wait(timeout=90) + assert server.returncode == 0 + pin = str(uuid.uuid4()) + backup_prefix = urllib.parse.urlsplit(settings["storage_url"]).path.lstrip("/") + "-backup" + # An enrolled backup needs a workspace independent of the serving node. + backup_config = directory / "retention-backup.json" + backup = json.loads(config.read_text()) + backup.update(node_id=str(uuid.uuid4()), data_dir=str(directory / "retention-backup-work")) + backup_config.write_text(json.dumps(backup)) + saved = cli("backup", str(backup_config), "create", pin, backup_prefix) + server, base = start(binary, directory, settings, "retention-next", data_instance="retention-source") + processes.append(server) + url = f"{base}/canopy/retained.git" + + operation = str(uuid.uuid4()) + cli("gc", str(config), operation, "1", "128", fails=True) + cli("maintenance", str(config), "begin", operation) + server.send_signal(signal.SIGTERM) + server.wait(timeout=90) + assert server.returncode == 0 + time.sleep(0.02) + partial = cli("gc", str(config), operation, "1", "1") + assert partial["deleted_objects"] == 1 and not partial["complete"] + assert partial["retained_pins"] == 1 + deleted = partial["deleted_objects"] + for _ in range(100): + report = cli("gc", str(config), operation, "1", "100000") + deleted += report["deleted_objects"] + if report["complete"]: + break + else: + raise AssertionError("bounded collection did not converge") + assert deleted > 1 + assert cli("backup", str(backup_config), "verify", pin, backup_prefix) == saved + cli("maintenance", str(config), "end", operation) + recovered, base = start(binary, directory, settings, "retention-cold") + processes.append(recovered) + clone = directory / "retention-clone" + git("-c", "http.extraHeader=Authorization: Bearer local-test-token", "clone", + f"{base}/canopy/retained.git", str(clone)) + git("fsck", "--full", cwd=clone) + assert git("rev-parse", "HEAD", cwd=clone) == git("rev-parse", "HEAD", cwd=source) + assert git("rev-list", "--objects", "--all", cwd=clone).splitlines() == git("rev-list", "--objects", "--all", cwd=source).splitlines() + result = {"deleted_objects": deleted, "last_report": report, + "backup_verified": True, "cold_clone_verified": True} + (directory / "retention-results.json").write_text(json.dumps(result, indent=2)) + print(f"PASS: collected {deleted} obsolete Cellule objects; backup pin and cold stock-Git clone remain exact", flush=True) + return result diff --git a/scripts/test_benchmark_browse.py b/scripts/test_benchmark_browse.py new file mode 100644 index 00000000..57d6f16c --- /dev/null +++ b/scripts/test_benchmark_browse.py @@ -0,0 +1,70 @@ +"""Ensure a real HTTP measurement counts both failures and corrupt responses.""" + +import contextlib +import hashlib +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import io +import json +from pathlib import Path +import socket +import tempfile +import threading +from types import SimpleNamespace +import unittest + +from benchmark_browse import measure + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *_args): + pass + + def do_POST(self): + self.connection.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + self.rfile.read(int(self.headers["Content-Length"])) + with self.server.guard: + self.server.requests += 1 + sequence = self.server.requests + body = b"verified" if sequence % 3 else b"corrupt" + self.send_response(503 if sequence % 5 == 0 else 200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +class BrowseMeasurementTests(unittest.TestCase): + def test_errors_and_invalid_bodies_are_counted_without_retries(self): + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + server.guard, server.requests = threading.Lock(), 0 + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + (output / "queries.json").write_text(json.dumps({"queries": [{ + "repository": "fixture", "operation": "refs", "path": "/browse", + "payload": {}, "response_sha256": hashlib.sha256(b"verified").hexdigest(), + }]})) + with contextlib.redirect_stdout(io.StringIO()): + measure(SimpleNamespace(output=output, base_url=f"http://127.0.0.1:{server.server_port}", + duration=0.25, concurrency=[2], repeats=1), "fixture-token") + report = json.loads((output / "measurements.json").read_text())["results"][0] + samples = [json.loads(line) for line in (output / "samples.jsonl").read_text().splitlines()] + self.assertEqual(len(samples), server.requests - 1) # One untimed warmup. + self.assertEqual(report["requests"], len(samples)) + self.assertEqual(sum(report["outcomes"].values()), len(samples)) + self.assertGreater(report["outcomes"]["http_503_or_invalid_body"], 0) + self.assertGreater(report["outcomes"]["http_200_or_invalid_body"], 0) + self.assertAlmostEqual(report["successful_requests_per_second"], + report["outcomes"]["ok"] / report["elapsed_seconds"]) + self.assertIsNotNone(report["latency_ms_successful"]) + finally: + server.shutdown() + server.server_close() + thread.join() + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_check_cellule.py b/scripts/test_check_cellule.py new file mode 100644 index 00000000..644c13b9 --- /dev/null +++ b/scripts/test_check_cellule.py @@ -0,0 +1,81 @@ +"""Guard against partial Cellule upgrades and alternate dependency sources.""" + +import copy +from pathlib import Path +import tempfile +import unittest + +from check_cellule import check, workspace_manifests + + +class CelluleReleaseTests(unittest.TestCase): + def setUp(self): + self.manifest = {"dependencies": { + "cellule-app": {"version": "=0.2.0"}, + "cellule-runtime": {"version": "=0.2.0"}, + }} + self.lock = {"package": [ + {"name": name, "version": "0.2.0", "checksum": "digest", + "source": "registry+https://github.com/rust-lang/crates.io-index"} + for name in ("cellule-app", "cellule-runtime", "cellule-types") + ]} + + def test_accepts_aligned_direct_and_transitive_crates(self): + self.assertEqual(check(self.manifest, self.lock), ("0.2.0", 3)) + + def test_rejects_ranges_and_alternate_sources(self): + for spec in ({"version": "0.2.0"}, {"version": "^0.2.0"}, + {"version": "=0.2.0", "git": "https://example.com/cellule"}, + {"version": "=0.2.0", "path": "../cellule"}, + {"version": "=0.2.0", "registry": "private"}): + with self.subTest(spec=spec): + self.manifest["dependencies"]["cellule-app"] = spec + with self.assertRaisesRegex(ValueError, "exact stable crates.io"): + check(self.manifest, self.lock) + + def test_rejects_partial_direct_upgrade(self): + self.manifest["dependencies"]["cellule-app"]["version"] = "=0.3.0" + with self.assertRaisesRegex(ValueError, "same release"): + check(self.manifest, self.lock) + + def test_workspace_members_cannot_hide_another_cellule_release(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "Cargo.toml").write_text('[workspace]\nmembers = ["crates/*"]\n') + for name, version in (("server", "0.2.0"), ("storage", "0.1.0")): + member = root / "crates" / name + member.mkdir(parents=True) + (member / "Cargo.toml").write_text( + f'[dependencies]\ncellule-runtime = "={version}"\n') + with self.assertRaisesRegex(ValueError, "same release"): + check(workspace_manifests(root), self.lock) + + def test_rejects_hidden_target_alias_on_another_release(self): + self.manifest["target"] = {"cfg(unix)": {"dependencies": { + "runtime": {"package": "cellule-runtime", "version": "=0.1.0"}, + }}} + with self.assertRaisesRegex(ValueError, "same release"): + check(self.manifest, self.lock) + + def test_rejects_old_or_replaced_transitive_crate(self): + for field, value in (("version", "0.1.0"), ("source", "git+https://example.com/cellule"), + ("source", None), ("checksum", None)): + with self.subTest(field=field, value=value): + lock = copy.deepcopy(self.lock) + lock["package"][-1][field] = value + with self.assertRaisesRegex(ValueError, "cellule-types must resolve once"): + check(self.manifest, lock) + + def test_rejects_duplicate_locked_crate(self): + self.lock["package"].append(self.lock["package"][-1].copy()) + with self.assertRaisesRegex(ValueError, "resolve once"): + check(self.manifest, self.lock) + + def test_rejects_missing_locked_crate(self): + self.lock["package"].pop(0) + with self.assertRaisesRegex(ValueError, "missing from Cargo.lock"): + check(self.manifest, self.lock) + + +if __name__ == "__main__": + unittest.main()