Add retention GC and pin published Cellule for native storage

.github/dependabot.yml

@@ -0,0 +1,13 @@
+version: 2
+updates:
+  - package-ecosystem: cargo
+    directory: /
+    schedule:
+      interval: daily
+    allow:
+      - dependency-name: "cellule-*"
+    groups:
+      cellule:
+        patterns:
+          - "cellule-*"
+    open-pull-requests-limit: 1

.github/workflows/qualify-release.yml

@@ -20,6 +20,8 @@ jobs:
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - name: Check Cellule release alignment
+        run: python3 scripts/check_cellule.py
       - name: Initialize qualification evidence before tool setup
         run: |
           mkdir qualification
@@ -29,7 +31,7 @@ jobs:
         run: |
           rustup component add clippy rustfmt
           sudo apt-get update
-          sudo apt-get install -y git-lfs openssh-client
+          sudo apt-get install -y git-lfs openssh-client openssl
           aws --version
           git lfs install
       - name: Bind exact source and dependency inputs
@@ -45,23 +47,25 @@ jobs:
 
           metadata = json.loads(Path('qualification/metadata.json').read_text())
           manifest = tomllib.loads(Path('crates/canopy-server/Cargo.toml').read_text())
-          revisions = {value['rev'] for value in manifest['dependencies'].values()
-                       if isinstance(value, dict) and value.get('git') == 'https://github.com/crabbuild/cellule.git'}
-          assert len(revisions) == 1
-          revision = revisions.pop()
+          pins = {value['version'] for name, value in manifest['dependencies'].items()
+                  if name.startswith('cellule-')}
+          assert len(pins) == 1 and next(iter(pins)).startswith('=')
+          version = pins.pop()[1:]
           packages = {package['name']: package['source'] for package in metadata['packages']
                       if package['name'].startswith('cellule-')}
           assert set(packages) == {'cellule-app', 'cellule-host', 'cellule-ltx',
                                    'cellule-runtime', 'cellule-store', 'cellule-types'}
-          assert all(source == f'git+https://github.com/crabbuild/cellule.git?rev={revision}#{revision}'
+          assert all(source == 'registry+https://github.com/rust-lang/crates.io-index'
                      for source in packages.values())
+          assert all(package['version'] == version for package in metadata['packages']
+                     if package['name'].startswith('cellule-'))
           files = subprocess.check_output(['git', 'ls-files', 'crates', 'scripts', 'Cargo.toml',
                                             'Cargo.lock', '.github/workflows'], text=True).splitlines()
           image = json.loads(subprocess.check_output(['docker', 'image', 'inspect',
                                                       'rustfs/rustfs:1.0.0-beta.8-glibc'], text=True))[0]
           result = {
               'head': subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip(),
-              'cellule_revision': revision,
+              'cellule_version': version,
               'packages': packages,
               'source_sha256': {name: hashlib.sha256(Path(name).read_bytes()).hexdigest() for name in files},
               'rustc': subprocess.check_output(['rustc', '-Vv'], text=True),
@@ -80,11 +84,9 @@ jobs:
           cargo fmt --all -- --check
           cargo clippy --release --workspace --all-targets --locked -- -D warnings 2>&1 | tee qualification/release-clippy.log
       - name: Test release workspace
-        run: cargo test --release --workspace --locked -- --test-threads=4 2>&1 | tee qualification/release-workspace.log
+        run: cargo test --release --workspace --locked -- --test-threads=1 2>&1 | tee qualification/release-workspace.log
       - name: Check Python qualification harness
         run: python3 -B -m unittest discover -s scripts -p 'test_*.py' 2>&1 | tee qualification/python-harness.log
-      - name: Qualify release Git compatibility against RustFS
-        run: python3 scripts/qualify_size.py --provider-only --release 2>&1 | tee qualification/release-rustfs.log
       - name: Build and retain release executable
         run: |
           cargo build --release --locked --bin canopy 2>&1 | tee qualification/release-build.log
@@ -98,13 +100,15 @@ jobs:
                      for name, digest in inputs['source_sha256'].items())
           binary = Path('target/release/canopy')
           digest = hashlib.sha256(binary.read_bytes()).hexdigest()
-          result = {'head': inputs['head'], 'cellule_revision': inputs['cellule_revision'],
+          result = {'head': inputs['head'], 'cellule_version': inputs['cellule_version'],
                     'binary_sha256': digest, 'binary_size_bytes': binary.stat().st_size,
                     'bound_source_unchanged': True, 'scope': inputs['scope']}
           Path('qualification/release-output.json').write_text(json.dumps(result, indent=2) + '\n')
           PY
           sha256sum target/release/canopy > qualification/canopy.sha256
           tar -czf qualification/canopy-release-linux.tar.gz -C target/release canopy
+      - name: Qualify release Git compatibility and process recovery against RustFS
+        run: python3 scripts/qualify_size.py --provider-only --release --process-binary target/release/canopy 2>&1 | tee qualification/release-rustfs.log
       - name: Retain qualification evidence including failed attempts
         if: always()
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4

.github/workflows/verify.yml

@@ -15,11 +15,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - name: Check Cellule release alignment
+        run: python3 scripts/check_cellule.py
       - name: Install tools
         run: |
           rustup component add clippy rustfmt
           sudo apt-get update
-          sudo apt-get install -y git-lfs openssh-client
+          sudo apt-get install -y git-lfs openssh-client openssl
           git lfs install
       - name: Report tool versions
         run: |
@@ -32,8 +34,8 @@ jobs:
       - name: Check lints
         run: cargo clippy --workspace --all-targets --locked -- -D warnings
       - name: Test
-        run: cargo test --workspace --locked
-      - name: Qualify Git compatibility against RustFS
-        run: python3 scripts/qualify_size.py --provider-only
+        run: cargo test --workspace --locked -- --test-threads=1
       - name: Build server
         run: cargo build --locked --bin canopy
+      - name: Qualify Git compatibility and process recovery against RustFS
+        run: python3 scripts/qualify_size.py --provider-only --process-binary target/debug/canopy

.gitignore

@@ -5,3 +5,5 @@
 
 /deploy/config.json
 /deploy/.proof-*/
+
+__pycache__/

AGENTS.md

@@ -0,0 +1,19 @@
+# Cellule integration
+
+- Keep Canopy on the latest published Cellule release. Update all direct
+  `cellule-*` exact version pins and `Cargo.lock` together, then adapt callers
+  to the current APIs and run the verification gates.
+- Support only the compiled release and its current storage formats. Breaking
+  Cellule or Canopy schema changes use a fresh storage prefix. Do not add old
+  readers, retained module code, migration chains, compatibility adapters, or
+  mixed-version operation to preserve an earlier deployment.
+- Keep release, identity, format, and fencing validation. Rejection of older
+  state is intentional. Same-release crash recovery, retries, backup/restore,
+  and standard Git/Git LFS protocol support remain required.
+- Cellule's single schema-initialization `MigrationDescriptor` is required for
+  fresh Cells; it is not an upgrade path. Keep command codec identities aligned
+  with the current handlers; do not reset version numbers to hide a change.
+- Run `python3 scripts/check_cellule.py`, formatting, Clippy, ordinary tests,
+  and the isolated provider/process qualification for dependency changes. Record
+  any qualification that was not run; retain historical evidence with its
+  original dependency version.

README.md

@@ -36,7 +36,7 @@ This page keeps the project overview, setup path, and first repository workflow.
 | Browser | Embedded repository, issue, pull-request and account views | [Repository browser](docs/user-guide.md#repository-browser) |
 
 These are implementation capabilities, not a production capacity guarantee.
-The [roadmap](ROADMAP.md) lists remaining work such as migrations, full fault
+The [roadmap](ROADMAP.md) lists remaining work such as release validation, full fault
 qualification, collection, provider qualification, observability and a repeatable
 deployment. The [delivery plan](docs/delivery-plan.md) records release gates;
 the [performance plan](docs/performance-plan.md) separates measured results from
@@ -49,8 +49,9 @@ before relying on a feature in a new environment.
 
 The **Directory Cell** maps an owner and repository name to a stable repository
 UUID and owns accounts. Each UUID identifies a **Repository Cell** containing
-authoritative refs, Git metadata, ACLs and collaboration records. Large Git
-blobs and LFS bodies are immutable external objects referenced by that Cell.
+authoritative catalog/ref roots, ACLs and collaboration records. Native Git
+packs, indexes, catalog metadata and LFS bodies are immutable external objects
+referenced by the committed state.
 Native Git handles wire protocols using a **disposable cache**: after local disk
 loss, Canopy restores published Cell state from the object store and rebuilds
 the cache. See the [persisted contracts](docs/contracts.md) for publication and
@@ -92,7 +93,8 @@ sequenceDiagram
 This is a development path for one node with an existing S3-compatible object
 store. The [bounded Linux deployment](deploy/README.md) describes the container
 profile and its resource limits. Use a **new storage prefix** for this build;
-there is no migration from earlier preview layouts yet.
+breaking schema or format changes require a fresh prefix. This build supports
+only its compiled release and current storage formats.
 
 1. Install Rust 1.97 or newer and Git with `http-backend`,
    `merge-tree --write-tree` (`-z --name-only --no-messages`) and `commit-tree`.

ROADMAP.md

@@ -1,6 +1,6 @@
 # Canopy roadmap
 
-Canopy's Git-hosting core works, but the service is not yet ready for an unattended team deployment. This roadmap tracks the remaining work from the current `main` baseline (`ab43e71`, 2026-09-27). It is a planning checklist, not a claim that every capability mentioned below is absent. See [delivery gates](docs/delivery-plan.md) for the detailed acceptance record, [Git compatibility](docs/git-compatibility.md) for protocol coverage, and [performance evidence](docs/performance-plan.md) for measured limits.
+Canopy implements the Git-hosting core, but the current native-storage build still needs end-to-end qualification before an unattended team deployment. This is a planning checklist, not a claim that every capability mentioned below is absent. See the [current storage qualification status](docs/benchmarks/2026-10-14-storage-optimization.md), [delivery gates](docs/delivery-plan.md), [Git compatibility](docs/git-compatibility.md), and [performance evidence](docs/performance-plan.md).
 
 ## How to use this roadmap
 
@@ -13,14 +13,13 @@ Canopy's Git-hosting core works, but the service is not yet ready for an unatten
 
 This is the first intended production-use milestone. The existing bounded Compose profile is a starting point, not a completed install or capacity claim.
 
-### R01. Safe upgrades and schema migration
+### R01. Current-release deployment and format validation
 
-- [ ] Version persistent Canopy and runtime formats; add a fenced, repeatable upgrade command and migration path for a populated deployment.
-- [ ] Require a verified backup and compatible release/configuration before migration; document failed-upgrade recovery and rollback limits.
-- [ ] Upgrade a prior release containing Git objects, LFS, ACLs, issues, checks, and pull requests; interrupt and retry each migration phase, then clone and inspect the restored state.
-- [ ] Define whether mixed-version nodes are supported. Until proven, enforce a fully drained, single-version upgrade.
+- [ ] Qualify release, identity, storage-format, schema and fencing rejection against the exact release candidate.
+- [ ] Qualify fresh initialization and same-release crash recovery, retries, backup and restore with populated Git, LFS and collaboration state.
+- [ ] Document a fully drained deployment using the latest published Cellule release and a fresh storage prefix for breaking Cellule or Canopy format changes.
 
-**Current state:** schema changes still require a fresh storage prefix; maintenance admission and drain exist, but there is no upgrade path.
+**Current state:** only the compiled release and its current formats are supported. Fresh-prefix deployment for breaking changes is intentional. Old readers, retained module code, migration chains, compatibility adapters and mixed-version operation are outside the supported contract. Cellule's single schema-initialization descriptor remains required for fresh Cells.
 
 ### R02. Backup, restore, and owner-recovery fault qualification
 
@@ -37,7 +36,7 @@ This is the first intended production-use milestone. The existing bounded Compos
 - [ ] Add a fenced collector for abandoned uploads, old outcomes/candidates, and unreachable Git/LFS objects, with an inspectable dry run and bounded work.
 - [ ] Prove with concurrent reads, backup, publication, owner loss, and injected collector failures that required bytes are never removed; collect abandoned bytes only after their grace period.
 
-**Current state:** no production collector; abandoned staging and outcomes can accumulate. Collection must remain conservative until its root proof passes.
+**Current state:** `canopy gc` uses Cellule's retention-aware collector under a verified maintenance drain, backup pins, grace/deletion bounds and a durable worker claim. It collects superseded Cellule state. The [October 15 qualification](docs/benchmarks/2026-10-15-native-storage.md) records three full repositories before and after collection and fresh restore. Product-owned pack/catalog/LFS artifacts and abandoned staging still need their own root and retention policy; R03 remains open.
 
 ### R04. Trustworthy CI and release artifacts
 
@@ -46,7 +45,7 @@ This is the first intended production-use milestone. The existing bounded Compos
 - [ ] Add required failure-path and provider gates to branch protection, then publish versioned, digest-identified images and a repeatable release checklist.
 - [ ] Record toolchain, Git, Git LFS, Cellule revision, image digest, provider, and test outcomes for each release candidate.
 
-**Current state:** `.github/workflows/verify.yml` invokes the full size gate on `ubuntu-latest`; a previous hosted run exhausted disk.
+**Current state:** standard CI runs formatting, Clippy, ordinary tests, Python harness checks and provider/process probes. The separate release workflow repeats correctness checks in release mode with provider/process probes. Both use `--provider-only`; the full size gate still needs a runner with sufficient capacity and recorded evidence for the exact release candidate.
 
 ### R05. Production object-store qualification
 
@@ -99,7 +98,7 @@ This is the first intended production-use milestone. The existing bounded Compos
 ### Private team release exit checks
 
 - [ ] R01–R10 acceptance proof is recorded for the exact release candidate and selected object store.
-- [ ] A clean install, upgrade of a populated prior release, backup, source-prefix loss, and restore all pass the operator runbook.
+- [ ] A clean install, current-format rejection checks, same-release backup, source-prefix loss and restore all pass the operator runbook. Breaking formats use a fresh prefix.
 - [ ] Stock Git and Git LFS work after recovery; permissions, issues, reviews, checks, and pull requests retain their state.
 - [ ] The release notes state measured limits for active repositories, concurrent transfers, local disk, memory, restore time, and object-store cost. Unmeasured capacity is not advertised.
 

crates/canopy-object-storage/src/artifact/tests.rs

@@ -11,6 +11,32 @@ fn key(body: &[u8]) -> ArtifactKey {
     }
 }
 
+#[tokio::test]
+async fn predecessor_manifest_is_rejected_even_with_a_matching_descriptor_digest() -> Result {
+    let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
+    let artifacts = ArtifactStore::new(Arc::clone(&store), [1; 16]);
+    let body = b"current immutable artifact";
+    let key = key(body);
+    let mut descriptor = artifacts
+        .put(
+            key,
+            body.len() as u64,
+            key.binding_digest,
+            &mut body.as_slice(),
+        )
+        .await?;
+    let path = artifacts.path(key, descriptor.digest)?;
+    let mut manifest = store.get(&path).await?.bytes().await?.to_vec();
+    manifest[..8].copy_from_slice(b"CANOPY01");
+    descriptor.manifest_digest = *blake3::hash(&manifest).as_bytes();
+    store.put(&path, manifest.into()).await?;
+    assert!(matches!(
+        artifacts.ranges_owned(key, descriptor, Arc::new(())).await,
+        Err(ArtifactError::Corrupt)
+    ));
+    Ok(())
+}
+
 #[tokio::test]
 async fn selected_parts_authenticate_without_fetching_unrequested_bytes() -> Result {
     let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());