From cdef55775d3ae1bee160bb402e8a7712d2bd3964 Mon Sep 17 00:00:00 2001 From: rain Date: Mon, 16 Nov 2026 00:37:54 +0000 Subject: [PATCH] feat(gc): SQLite compaction (#48) --- .../layerfs-control/src/worker_server.rs | 19 +- .../tests/project_delete_worker.rs | 550 ++++++ .../layerfs-store/src/project/delete.rs | 420 ++++- .../crates/layerfs-store/src/project/frame.rs | 112 +- .../layerfs-store/tests/project_delete.rs | 157 +- .../tests/project_delete_frames.rs | 75 +- .../tests/project_delete_store.rs | 238 +++ .../layerfs-history/src/contract/error.rs | 3 + .../issues/project-41/gc-t8-sqlite-rebuild.md | 128 ++ .../queries/history/gc_epoch_read.sql | 1 + .../sqlite/queries/history/gc_epoch_read.sql | 1 + .../backend/postgresql/history/statements.rs | 3 + .../src/backend/sqlite/compact.rs | 1665 +++++++++++++++++ .../src/backend/sqlite/lane/lane.rs | 11 + .../src/backend/sqlite/lifecycle/seal.rs | 127 +- .../src/backend/sqlite/mod.rs | 1 + .../src/backend/sqlite/session/checkpoint.rs | 34 + .../src/backend/sqlite/session/query.rs | 3 + .../layerfs-persistence/src/history/commit.rs | 4 + .../layerfs-persistence/src/history/epoch.rs | 69 + .../src/history/layerstack.rs | 4 + .../layerfs-persistence/src/history/merge.rs | 9 +- .../layerfs-persistence/src/history/mod.rs | 1 + engine/crates/layerfs-persistence/src/lib.rs | 4 + .../layerfs-persistence/tests/epoch_fence.rs | 442 +++++ .../tests/sqlite_compact.rs | 456 +++++ .../src/history/admission.rs | 10 +- .../src/history/deletion.rs | 19 +- .../src/history/lookup.rs | 1 + .../layerfs-store-service/src/write/files.rs | 6 + .../tests/epoch_fence.rs | 77 + 31 files changed, 4539 insertions(+), 111 deletions(-) create mode 100644 control/crates/layerfs-control/tests/project_delete_worker.rs create mode 100644 control/crates/layerfs-store/tests/project_delete_store.rs create mode 100644 core/docs/issues/project-41/gc-t8-sqlite-rebuild.md create mode 100644 engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_epoch_read.sql create mode 100644 engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_epoch_read.sql create mode 100644 engine/crates/layerfs-persistence/src/backend/sqlite/compact.rs create mode 100644 engine/crates/layerfs-persistence/src/history/epoch.rs create mode 100644 engine/crates/layerfs-persistence/tests/epoch_fence.rs create mode 100644 engine/crates/layerfs-persistence/tests/sqlite_compact.rs create mode 100644 engine/crates/layerfs-store-service/tests/epoch_fence.rs diff --git a/control/crates/layerfs-control/src/worker_server.rs b/control/crates/layerfs-control/src/worker_server.rs index 997d24c27e..3e7d2cf8bd 100644 --- a/control/crates/layerfs-control/src/worker_server.rs +++ b/control/crates/layerfs-control/src/worker_server.rs @@ -28,11 +28,12 @@ use layerfs_home::outcome::{refused, Outcome}; use layerfs_machine::worker::{ read_frame, read_frame_within, write_frame, Hello, HELLO_WAIT, MAX_HELLO, }; -use layerfs_store::project::delete::delete_project; +use layerfs_store::project::delete::{admit_project, delete_project, release_project}; use layerfs_store::project::discovery::home_of; use layerfs_store::project::export::in_worker as export_in_worker; use layerfs_store::project::frame::{ - DeleteFrame, ExportFrame, InitFrame, ListingFrame, RemoveFrame, StageFrame, + DeleteAdmitFrame, DeleteFrame, DeleteReleaseFrame, ExportFrame, InitFrame, ListingFrame, + RemoveFrame, StageFrame, }; use layerfs_store::project::listing::list_projects; use layerfs_store::project::{init::in_worker, remove::remove_project, staged}; @@ -280,11 +281,23 @@ fn run( } // retires one project in the Store under this worker root and // removes the worker own entry (2.0, T4.5b): `{ store, stack_id, - // deleted_ms?, project?, store_record? }` + // deleted_ms?, project?, op_id?, store_record? }` "project.delete" => { let root = home_of(Some(root))?; delete_project(&root, &DeleteFrame::decode(&args)?) } + // registers the caller's project-scope admission before its sidecar + // and marker (2.0, T4.6): `{ store, stack_id, op_id, store_record? }` + "project.delete.admit" => { + let root = home_of(Some(root))?; + admit_project(&root, &DeleteAdmitFrame::decode(&args)?) + } + // releases the caller's project-scope admission after the marker + // (2.0, T4.6): `{ store, op_id, store_record? }` + "project.delete.release" => { + let root = home_of(Some(root))?; + release_project(&root, &DeleteReleaseFrame::decode(&args)?) + } // reads the Store's own filtered project listing under this worker // root, the same `layerfs_project::store::projects` a local Store // uses (2.0, T4.5c): `{ store, store_record? }` diff --git a/control/crates/layerfs-control/tests/project_delete_worker.rs b/control/crates/layerfs-control/tests/project_delete_worker.rs new file mode 100644 index 0000000000..c1ab487c20 --- /dev/null +++ b/control/crates/layerfs-control/tests/project_delete_worker.rs @@ -0,0 +1,550 @@ +//! `project.delete` on a worker-backed Store (T4.6): the caller admits the +//! project scope through the worker before it writes its sidecar and sends +//! the marker. A live mount in the worker's Store refuses the call with +//! `MountsActive` (holders reported) and a live operation with `Busy`, with +//! no sidecar, no marker and no leaked admission row; the successful path +//! releases the admission after the marker. +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_bridge::sdk_values::hex; +use layerfs_history::records::{GcIntent, GcIntentKind, GcScopeKind}; +use layerfs_history::{HistoryName, LayerStackId, NamedSnapshot}; +use layerfs_home::events::Events; +use layerfs_home::home::{self, paths}; +use layerfs_home::names::{ProjectRef, StoreName}; +use layerfs_machine::worker::{self, read_frame, read_frame_within, write_frame, Hello, MAX_HELLO}; +use layerfs_store::open_args::OpenArgs; +use layerfs_store::project::add::{add_project, ProjectAdd}; +use layerfs_store::record::StoreRecord; +use layerfs_store::service::with_service; +use layerfs_store::sqlite::create::create; +use layerfs_store_service::history::lookup; +use std::net::TcpListener; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; + +static NEXT: AtomicU64 = AtomicU64::new(0); + +fn tempdir(tag: &str) -> PathBuf { + let directory = std::env::temp_dir().join(format!( + "layerfs-delete-worker-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&directory); + layerfs_home::home::layout::create(&directory).unwrap(); + directory +} + +fn text(value: &str) -> SdkValue { + SdkValue::text(value) +} + +/// A folder to initialize a project from: no home layout inside it. +fn plaindir(tag: &str) -> PathBuf { + let directory = std::env::temp_dir().join(format!( + "layerfs-delete-worker-folder-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&directory); + std::fs::create_dir_all(&directory).unwrap(); + directory +} + +/// Moves Store `store`'s file from `from`'s Store folder to `to`'s. +fn move_store_file(from: &Path, to: &Path, store: &str) { + let source = paths::store_dir(from, store); + let target = paths::store_dir(to, store); + home::private_dir(&target).unwrap(); + for entry in std::fs::read_dir(&source).unwrap() { + let entry = entry.unwrap(); + let name = entry.file_name().to_string_lossy().into_owned(); + if name.starts_with("store.sqlite") { + std::fs::rename(entry.path(), target.join(&name)).unwrap(); + } + } +} + +/// The Store record of `store` places it on machine `machine`. +fn place_store(home: &Path, store: &str, machine: &str) { + let record_path = paths::store_record(home, store); + let record = home::read_record(&record_path).unwrap().unwrap(); + write_record( + &record_path, + &home::with_field(record, "machine", text(machine)), + ); +} + +fn write_record(path: &Path, value: &SdkValue) { + home::write_record(path, value).unwrap(); +} + +/// A worker's `OpenArgs` for Store `shop` under `root`, with the caller's +/// Store record (the same context every `project.delete*` frame carries). +fn worker_open(root: &Path, record: &StoreRecord) -> OpenArgs { + OpenArgs { + home: Some(root.to_path_buf()), + store: Some("shop".to_owned()), + project: Some("app".to_owned()), + store_path: Some(paths::store_file(root, "shop").display().to_string()), + store_record: Some(record.to_record().clone()), + postgresql: None, + } +} + +fn intents(open: &OpenArgs, stack: LayerStackId) -> Vec { + with_service(open, |service| { + Ok(service + .store + .history() + .gc_intents(GcScopeKind::Project, stack.as_slice()) + .unwrap()) + }) + .unwrap() +} + +fn project_names(open: &OpenArgs) -> Vec { + with_service(open, |service| { + let listed = service + .store + .published(layerfs_project::store::projects) + .unwrap(); + Ok(listed + .into_iter() + .map(|stack| stack.name.as_str().to_owned()) + .collect()) + }) + .unwrap() +} + +fn remove(home: &Path) -> Result { + let mut sink = |_| {}; + layerfs_control::project::delete( + home, + &mut Events::new(&mut sink), + &ProjectRef::parse("shop/app").unwrap(), + ) +} + +#[test] +fn the_worker_path_admits_before_the_sidecar_and_releases_after_the_marker() { + // a home with Store `shop` and project `app`, created locally. A + // sandbox filesystem cannot hold a directly opened Store; the fake-home + // cases in the `layerfs-store` tests cover the same home path, and this + // case runs where the Store's filesystem carries SQLite locks. + let home = tempdir("home"); + match create(&home, &StoreName::parse("shop").unwrap()) { + Ok(_) => {} + Err(refusal) + if refusal.code == "Unsupported" + && refusal.message.contains("does not carry SQLite locks") => + { + eprintln!("NOT RUN: this filesystem cannot hold a directly opened Store: {refusal:?}"); + let _ = std::fs::remove_dir_all(&home); + return; + } + Err(refusal) => panic!("store.create: {refusal:?}"), + } + let source = plaindir("folder"); + std::fs::write(source.join("a.txt"), b"a\n").unwrap(); + add_project(&ProjectAdd { + home: home.clone(), + store: StoreName::parse("shop").unwrap(), + name: "app".to_owned(), + source, + timeout_ms: None, + store_record: None, + }) + .unwrap(); + let record = StoreRecord::read(&home, "shop").unwrap().unwrap(); + let caller_open = OpenArgs { + home: Some(home.clone()), + store: Some("shop".to_owned()), + project: Some("app".to_owned()), + store_path: Some(paths::store_file(&home, "shop").display().to_string()), + store_record: Some(record.to_record().clone()), + postgresql: None, + }; + let (stack, branch) = with_service(&caller_open, |service| { + let stack = lookup::project(&service.store, "app").unwrap(); + let project = HistoryName::new("app").unwrap(); + let main = HistoryName::new("main").unwrap(); + let NamedSnapshot::Found(snapshot) = service + .store + .published(|history| history.branch_snapshot_named(&project, &main)) + .unwrap() + else { + panic!("no main Branch"); + }; + Ok((stack.id, snapshot.branch.id)) + }) + .unwrap(); + + // the Store belongs to machine `box`; its file is under the worker's root + let root = tempdir("root"); + move_store_file(&home, &root, "shop"); + home::write_record( + &paths::machine_record(&home, "box"), + &SdkValue::map([("name", text("box")), ("platform", text("lima"))]), + ) + .unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap().to_string(); + home::write_record( + &paths::worker_record(&home, "box"), + &SdkValue::map([("address", text(&address)), ("secret", text("k"))]), + ) + .unwrap(); + place_store(&home, "shop", "box"); + let served = root.clone(); + std::thread::spawn(move || { + layerfs_control::worker_server::serve(listener, "k".to_owned(), served) + }); + + let worker = worker_open(&root, &record); + + // a live mount in the worker's Store: `MountsActive`, no sidecar, no marker + let mount = GcIntent::shared( + [9; 16], + GcIntentKind::Mount, + GcScopeKind::Branch, + branch.as_slice(), + ) + .in_project(stack.as_slice()) + .named(Some("app"), Some("main")) + .placed(Some([9; 32]), Some("sandbox-9"), Some("machine-9")); + with_service(&worker, |service| { + service.store.history().gc_register(&mount).unwrap(); + Ok(()) + }) + .unwrap(); + let refusal = remove(&home).expect_err("a live mount holds the project"); + assert_eq!(refusal.code, "MountsActive", "{refusal:?}"); + assert!( + intents(&worker, stack).is_empty(), + "no project-scope row yet" + ); + let SdkValue::List(holders) = refusal.detail.get("holders").expect("holders") else { + panic!("{refusal:?}"); + }; + assert_eq!(holders[0].get_text("kind"), Some("mount")); + assert_eq!(holders[0].get_text("sandbox"), Some("sandbox-9")); + assert_eq!( + layerfs_store::project::delete::Intent::read(&home, "shop", "app").unwrap(), + None, + "no sidecar was written" + ); + assert!(paths::project_dir(&home, "shop", "app").is_dir()); + assert_eq!(project_names(&worker), vec!["app".to_owned()], "no marker"); + assert!( + intents(&worker, stack).is_empty(), + "the refused admission left no project-scope row" + ); + + // the mount ends; a live removal at project scope is `Busy` + with_service(&worker, |service| { + service.store.history().gc_release(&[9; 16]).unwrap(); + Ok(()) + }) + .unwrap(); + let holder = GcIntent::operation( + [7; 16], + GcIntentKind::Removal, + GcScopeKind::Project, + stack.as_slice(), + ) + .named(Some("app"), None); + with_service(&worker, |service| { + service.store.history().gc_register(&holder).unwrap(); + Ok(()) + }) + .unwrap(); + let refusal = remove(&home).expect_err("another removal holds the project"); + assert_eq!(refusal.code, "Busy", "{refusal:?}"); + assert_eq!( + layerfs_store::project::delete::Intent::read(&home, "shop", "app").unwrap(), + None + ); + assert_eq!( + intents(&worker, stack).len(), + 1, + "only the test's own holder" + ); + assert!(paths::project_dir(&home, "shop", "app").is_dir()); + + // the holder ends: the delete succeeds through the worker and releases + // the admission it took, with the sidecar unlinked + with_service(&worker, |service| { + service.store.history().gc_release(&[7; 16]).unwrap(); + Ok(()) + }) + .unwrap(); + let outcome = remove(&home).unwrap(); + assert_eq!(outcome.get_text("kind"), Some("Removed")); + assert!(outcome.get_bool("home_entry_removed")); + assert!(!paths::project_dir(&home, "shop", "app").exists()); + assert_eq!( + layerfs_store::project::delete::Intent::read(&home, "shop", "app").unwrap(), + None + ); + assert!( + intents(&worker, stack).is_empty(), + "the admission was released" + ); + assert!(project_names(&worker).is_empty(), "the project is retired"); + let _ = std::fs::remove_dir_all(&home); + let _ = std::fs::remove_dir_all(&root); +} + +/// A fake worker on `listener`: answers its secret, answers each request with +/// `reply`, and shares back the requests it read. +fn fake_worker( + listener: TcpListener, + reply: impl Fn(&SdkValue) -> SdkValue + Send + 'static, +) -> Arc>> { + let seen: Arc>> = Arc::default(); + let requests = seen.clone(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { + return; + }; + let Some(_hello) = read_frame_within(&mut stream, MAX_HELLO).ok().flatten() else { + continue; + }; + let ready = Hello { + protocol: Some(worker::PROTOCOL), + bundle: None, + }; + if write_frame(&mut stream, &ready.to_frame()).is_err() { + continue; + } + let Some(request) = read_frame(&mut stream).ok().flatten() else { + continue; + }; + let answer = reply(&request); + requests.lock().unwrap().push(request); + let _ = write_frame(&mut stream, &answer); + } + }); + seen +} + +/// The `op` of one request the fake worker read. +fn op_of(request: &SdkValue) -> &str { + request.get_text("op").unwrap_or("") +} + +/// A frame's `args`. +fn args_of(request: &SdkValue) -> &SdkValue { + request.get("args").expect("args") +} + +/// The `op_id` a request's args carry. +fn sent_op(request: &SdkValue) -> String { + args_of(request) + .get_text("op_id") + .expect("the frame names its admission") + .to_owned() +} + +fn admitted_for(stack: LayerStackId) -> SdkValue { + SdkValue::map([( + "value", + SdkValue::map([ + ("kind", text("Admitted")), + ("stack_id", text(&hex(&stack.to_bytes()))), + ]), + )]) +} + +fn removed_for(stack: LayerStackId, deleted_ms: i64) -> SdkValue { + SdkValue::map([( + "value", + SdkValue::map([ + ("kind", text("Removed")), + ("project", text("app")), + ("stack_id", text(&hex(&stack.to_bytes()))), + ("deleted_ms", SdkValue::Int(deleted_ms)), + ]), + )]) +} + +fn released_for(request: &SdkValue) -> SdkValue { + SdkValue::map([( + "value", + SdkValue::map([ + ("kind", text("Released")), + ("op_id", text(&sent_op(request))), + ]), + )]) +} + +fn refused(code: &str, detail: SdkValue) -> SdkValue { + SdkValue::map([( + "refused", + SdkValue::map([ + ("code", text(code)), + ("message", text("the fake worker refused")), + ("detail", detail), + ]), + )]) +} + +/// A caller's home naming Store `shop` on machine `box`, whose project `app` +/// records `stack`, with the fake worker's address. +fn caller_home(tag: &str, address: &str, stack: LayerStackId) -> PathBuf { + let home = tempdir(tag); + home::write_record( + &paths::store_record(&home, "shop"), + &SdkValue::map([ + ("v", SdkValue::Int(1)), + ("kind", text("sqlite")), + ("name", text("shop")), + ("machine", text("box")), + ]), + ) + .unwrap(); + home::write_record( + &paths::project_record(&home, "shop", "app"), + &SdkValue::map([ + ("v", SdkValue::Int(1)), + ("name", text("app")), + ("stack", text(&hex(&stack.to_bytes()))), + ("created", SdkValue::Int(1)), + ]), + ) + .unwrap(); + home::write_record( + &paths::worker_record(&home, "box"), + &SdkValue::map([("address", text(address)), ("secret", text("k"))]), + ) + .unwrap(); + home +} + +#[test] +fn a_fake_workers_mounts_active_refusal_writes_no_sidecar_and_no_marker() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap().to_string(); + let stack = LayerStackId::from_authority([3; 16]); + let home = caller_home("fake-mount", &address, stack); + let holders = SdkValue::map([( + "holders", + SdkValue::List(vec![SdkValue::map([ + ("kind", text("mount")), + ("entity", text("main")), + ])]), + )]); + let seen = fake_worker(listener, move |request| { + assert_eq!(op_of(request), "project.delete.admit"); + refused("MountsActive", holders.clone()) + }); + + let refusal = remove(&home).expect_err("the fake worker's Store has a live mount"); + assert_eq!(refusal.code, "MountsActive", "{refusal:?}"); + // the caller wrote no sidecar and sent no marker: only the admission frame + assert_eq!( + layerfs_store::project::delete::Intent::read(&home, "shop", "app").unwrap(), + None + ); + assert!(paths::project_dir(&home, "shop", "app").is_dir()); + let requests = seen.lock().unwrap(); + assert_eq!(requests.len(), 1, "{requests:?}"); + assert_eq!(op_of(&requests[0]), "project.delete.admit"); + drop(requests); + let _ = std::fs::remove_dir_all(home); +} + +#[test] +fn a_fake_workers_marker_refusal_releases_the_admission_it_took() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap().to_string(); + let stack = LayerStackId::from_authority([4; 16]); + let home = caller_home("fake-busy", &address, stack); + let seen = fake_worker(listener, move |request| match op_of(request) { + "project.delete.admit" => admitted_for(stack), + "project.delete" => refused("Busy", SdkValue::Null), + "project.delete.release" => released_for(request), + other => panic!("unexpected op {other}"), + }); + + let refusal = remove(&home).expect_err("a live operation holds the project"); + assert_eq!(refusal.code, "Busy", "{refusal:?}"); + // the admission was taken, named in the sidecar, used by the marker and + // released after the marker's refusal + let requests = seen.lock().unwrap(); + let ops: Vec<&str> = requests.iter().map(op_of).collect(); + assert_eq!( + ops, + [ + "project.delete.admit", + "project.delete", + "project.delete.release" + ], + "{requests:?}" + ); + let op = sent_op(&requests[0]); + assert_eq!(sent_op(&requests[1]), op); + assert_eq!(sent_op(&requests[2]), op); + drop(requests); + let body: [u8; 16] = hex_decode(&op).try_into().unwrap(); + assert_eq!( + layerfs_store::project::delete::Intent::read(&home, "shop", "app") + .unwrap() + .expect("the pending sidecar stays for a retry") + .op, + Some(body) + ); + assert!(paths::project_dir(&home, "shop", "app").is_dir()); + let _ = std::fs::remove_dir_all(home); +} + +#[test] +fn a_fake_workers_success_removes_the_entry_and_releases_the_admission() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap().to_string(); + let stack = LayerStackId::from_authority([5; 16]); + let home = caller_home("fake-ok", &address, stack); + let seen = fake_worker(listener, move |request| match op_of(request) { + "project.delete.admit" => admitted_for(stack), + "project.delete" => removed_for(stack, 42), + "project.delete.release" => released_for(request), + other => panic!("unexpected op {other}"), + }); + + let outcome = remove(&home).unwrap(); + assert_eq!(outcome.get_text("kind"), Some("Removed")); + assert_eq!(outcome.get_int("deleted_ms"), Some(42)); + assert!(outcome.get_bool("home_entry_removed")); + assert!(!paths::project_dir(&home, "shop", "app").exists()); + assert_eq!( + layerfs_store::project::delete::Intent::read(&home, "shop", "app").unwrap(), + None + ); + let requests = seen.lock().unwrap(); + let ops: Vec<&str> = requests.iter().map(op_of).collect(); + assert_eq!( + ops, + [ + "project.delete.admit", + "project.delete", + "project.delete.release" + ], + "{requests:?}" + ); + let op = sent_op(&requests[0]); + assert_eq!(sent_op(&requests[1]), op); + assert_eq!(sent_op(&requests[2]), op); + drop(requests); + let _ = std::fs::remove_dir_all(home); +} + +/// The bytes of a lowercase hex string. +fn hex_decode(text: &str) -> Vec { + (0..text.len()) + .step_by(2) + .map(|at| u8::from_str_radix(&text[at..at + 2], 16).unwrap()) + .collect() +} diff --git a/control/crates/layerfs-store/src/project/delete.rs b/control/crates/layerfs-store/src/project/delete.rs index dbf5dc9885..12fb89b68b 100644 --- a/control/crates/layerfs-store/src/project/delete.rs +++ b/control/crates/layerfs-store/src/project/delete.rs @@ -17,28 +17,41 @@ //! that is there now. //! //! The worker op is `project.delete`; its frame is -//! `{store, stack_id, deleted_ms?, project?, store_record?}`. The two optional -//! context fields are the same ones `project.remove` sends: the worker opens -//! the Store under its own root with the caller's `store_record`, and the -//! project name lets its reply and its own home entry name the project. The -//! worker saves the legacy own-project identity, writes the marker, removes -//! its own entry and replies `{kind, project, stack_id, deleted_ms}`; the -//! caller removes its own entry and echoes `home_entry_removed`. +//! `{store, stack_id, deleted_ms?, project?, op_id?, store_record?}`. The two +//! optional context fields are the same ones `project.remove` sends: the +//! worker opens the Store under its own root with the caller's `store_record`, +//! and the project name lets its reply and its own home entry name the +//! project. The worker saves the legacy own-project identity, writes the +//! marker, removes its own entry and replies `{kind, project, stack_id, +//! deleted_ms}`; the caller removes its own entry and echoes +//! `home_entry_removed`. +//! +//! Before any of that, the home path admits the project scope (T4.6, +//! `project.delete.admit` on the worker route): the same `gc_register` the +//! Store-side removals rely on, taken before the identity save, the sidecar +//! and the marker. A live mount refuses `MountsActive` with its holders and a +//! live operation `Busy`, with no sidecar written and no marker. The marker +//! reuses the admission's operation identity, and the caller releases it on +//! every exit path (`project.delete.release`), so a refused or failed call +//! leaves no admission row behind. use crate::open_args::OpenArgs; use crate::place::{self, Executor, Worker}; -use crate::project::frame::{DeleteFrame, DeleteReply}; +use crate::project::frame::{DeleteAdmitFrame, DeleteFrame, DeleteReleaseFrame, DeleteReply}; use crate::project::record::ProjectRecord; use crate::record::{StoreRecord, StoreRef}; use crate::{identity, lease, service}; use layerfs_bridge::control::{SdkRefusal, SdkValue}; use layerfs_bridge::provision::StoreManifest; use layerfs_bridge::sdk_values::{hex, unhex}; +use layerfs_history::HistoryError; use layerfs_history::LayerStackId; use layerfs_home::events::Events; use layerfs_home::home::{now_ms, paths}; use layerfs_home::names::{ProjectRef, StoreName}; use layerfs_home::outcome::{failed, refused, Outcome}; -use layerfs_store_service::history::deletion::{delete_stack, Deletion}; +use layerfs_store_service::history::admission as store_admission; +use layerfs_store_service::history::deletion::{delete_stack_with, Deletion}; +use layerfs_store_service::history::lookup; use std::fs::{self, File, OpenOptions}; use std::io::Write; use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt}; @@ -98,6 +111,10 @@ pub struct Intent { pub attempt_ms: i64, /// Where the attempt got to. pub state: IntentState, + /// The project-scope admission this attempt took, when it took one. A + /// retry re-registers the same identity, so a call whose reply was lost + /// settles its own row instead of meeting it as another operation. + pub op: Option<[u8; 16]>, } static NEXT_TEMPORARY: AtomicU64 = AtomicU64::new(0); @@ -135,6 +152,14 @@ impl Intent { .get_text("state") .and_then(IntentState::parse) .ok_or_else(malformed)?; + let op = match value.get_text("op_id") { + Some(text) => { + let bytes = unhex(text).ok_or_else(malformed)?; + let body: [u8; 16] = bytes.as_slice().try_into().map_err(|_| malformed())?; + Some(body) + } + None => None, + }; Ok(Some(Self { store, project, @@ -142,6 +167,7 @@ impl Intent { deleted_ms, attempt_ms, state, + op, })) } @@ -165,14 +191,21 @@ impl Intent { std::process::id(), unique )); - let value = SdkValue::map([ - ("store", SdkValue::text(&intent.store)), - ("project", SdkValue::text(&intent.project)), - ("stack_id", SdkValue::text(hex(&intent.stack.to_bytes()))), - ("deleted_ms", SdkValue::Int(intent.deleted_ms)), - ("attempt_ms", SdkValue::Int(intent.attempt_ms)), - ("state", SdkValue::text(intent.state.as_str())), - ]); + let mut fields: Vec<(String, SdkValue)> = vec![ + ("store".to_owned(), SdkValue::text(&intent.store)), + ("project".to_owned(), SdkValue::text(&intent.project)), + ( + "stack_id".to_owned(), + SdkValue::text(hex(&intent.stack.to_bytes())), + ), + ("deleted_ms".to_owned(), SdkValue::Int(intent.deleted_ms)), + ("attempt_ms".to_owned(), SdkValue::Int(intent.attempt_ms)), + ("state".to_owned(), SdkValue::text(intent.state.as_str())), + ]; + if let Some(op) = intent.op { + fields.push(("op_id".to_owned(), SdkValue::text(hex(&op)))); + } + let value = SdkValue::Map(fields); let mut text = layerfs_home::json::print(&value); text.push('\n'); let mut file = OpenOptions::new() @@ -218,6 +251,24 @@ impl Intent { } } +/// The project-scope admission `project.delete` takes before its sidecar and +/// marker: the admission row's operation identity, reused by the marker so the +/// one row is excluded from its own scan and released once. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct Admission { + /// The `gc_intent` operation identity. + pub op: [u8; 16], + /// The project scope it holds. + pub stack: LayerStackId, +} + +impl Admission { + /// One admission of `stack` under operation `op`. + pub const fn new(stack: LayerStackId, op: [u8; 16]) -> Self { + Self { op, stack } + } +} + /// The Store side of one `project.delete` attempt and the caller's entry /// removal, as one seam so the home path is testable without a Store. pub trait Effects { @@ -225,9 +276,21 @@ pub trait Effects { /// marker; idempotent on a retry. fn prepare(&mut self) -> Result<()>; - /// Marks (or reclassifies) `stack`. A retry passes the intent's - /// `deleted_ms`. - fn mark(&mut self, stack: LayerStackId, deleted_ms: i64) -> Result; + /// Takes the project-scope admission before the sidecar and the marker: + /// a live mount is `MountsActive` with its holders, a live operation + /// `Busy`, an uncertain owner `Unresolved`. A refusal registers nothing. + /// `reuse` is the identity the sidecar recorded for a retry; the same row + /// is re-registered under it, so a lost reply is settled, never met as a + /// second operation. + fn admit(&mut self, stack: LayerStackId, reuse: Option<[u8; 16]>) -> Result; + + /// Releases `admission` on every exit path: success, refusal and an + /// unknown Store outcome. A provider without admission releases nothing. + fn release(&mut self, admission: Admission) -> Result<()>; + + /// Marks (or reclassifies) `admission`'s Stack, reusing the admission's + /// operation identity. A retry passes the intent's `deleted_ms`. + fn mark(&mut self, admission: Admission, deleted_ms: i64) -> Result; /// Removes the caller's home entry; `Ok(())` when nothing remains. fn remove_entry(&mut self, entry: &Path) -> Result<()>; @@ -267,12 +330,67 @@ fn removed(store: &str, project: &str, deleted_ms: i64, home_entry_removed: bool ]) } +/// The mutating part of `delete_with` after the project scope is admitted: +/// the identity save, the sidecar, the marker and the entry removal (spec §4). +fn mark_and_finish( + home: &Path, + store: &str, + project: &str, + stack: LayerStackId, + intent: Option, + mut deleted_ms: i64, + effects: &mut impl Effects, + admission: Admission, +) -> Outcome { + effects.prepare()?; + let mut current = match intent { + Some(intent) => intent, + None => Intent { + store: store.to_owned(), + project: project.to_owned(), + stack, + deleted_ms, + attempt_ms: now_ms(), + state: IntentState::Pending, + op: None, + }, + }; + // the sidecar names the admission, so a retry reuses it + current.op = Some(admission.op); + if current.state == IntentState::Pending { + current.attempt_ms = now_ms(); + Intent::write(home, ¤t)?; + } + + let deletion = effects.mark(admission, deleted_ms)?; + deleted_ms = deletion.deleted_ms(deleted_ms); + current.deleted_ms = deleted_ms; + current.state = IntentState::Marked; + current.attempt_ms = now_ms(); + Intent::write(home, ¤t)?; + + let home_entry_removed = effects + .remove_entry(&paths::project_dir(home, store, project)) + .is_ok(); + if home_entry_removed { + current.state = IntentState::Done; + Intent::write(home, ¤t)?; + Intent::unlink(home, store, project)?; + } + Ok(removed(store, project, deleted_ms, home_entry_removed)) +} + /// Runs `project.delete` for a home entry whose Store side is `effects`. /// /// `stack` is the Stack id resolved from the record or the Store listing; a /// record without one whose Store does not list it is a stale entry and is /// removed without a marker. The same function serves the local and worker /// routes, and a fake in tests. +/// +/// The project scope is admitted before the identity save, the sidecar and +/// the marker: a live mount is `MountsActive` with its holders and a live +/// operation `Busy`, reported with no sidecar written and no marker. The +/// admission is released on every exit path. pub fn delete_with( home: &Path, store: &str, @@ -294,7 +412,7 @@ pub fn delete_with( } } - let (stack, mut deleted_ms) = match &intent { + let (stack, deleted_ms) = match &intent { Some(intent) => (intent.stack, intent.deleted_ms), None => { let Some(stack) = stack.or(record_stack) else { @@ -319,6 +437,8 @@ pub fn delete_with( if let Some(done) = &intent { if done.state == IntentState::Done { + // The retired-row entry-removal retry needs no admission: the + // marker is already there and only the home entry is removed. effects.prepare()?; let removed_entry = effects .remove_entry(&paths::project_dir(home, store, project)) @@ -331,41 +451,42 @@ pub fn delete_with( } } - // The identity save comes before the intent, and the intent before the - // marker (spec §4). - effects.prepare()?; - let mut current = match intent { - Some(intent) => intent, - None => Intent { - store: store.to_owned(), - project: project.to_owned(), - stack, - deleted_ms, - attempt_ms: now_ms(), - state: IntentState::Pending, - }, + // Admission before the identity save, the sidecar and the marker. A + // refusal returns here, so neither the sidecar nor the marker is written. + // A retry re-registers the identity its sidecar recorded; a refusal then + // releases that identity, never adopting it as a live operation. + let reuse = intent.as_ref().and_then(|intent| intent.op); + let admission = match effects.admit(stack, reuse) { + Ok(admission) => admission, + Err(refusal) => { + if let Some(op) = reuse { + let _ = effects.release(Admission::new(stack, op)); + } + return Err(refusal); + } }; - if current.state == IntentState::Pending { - current.attempt_ms = now_ms(); - Intent::write(home, ¤t)?; - } - - let deletion = effects.mark(stack, deleted_ms)?; - deleted_ms = deletion.deleted_ms(deleted_ms); - current.deleted_ms = deleted_ms; - current.state = IntentState::Marked; - current.attempt_ms = now_ms(); - Intent::write(home, ¤t)?; - - let home_entry_removed = effects - .remove_entry(&paths::project_dir(home, store, project)) - .is_ok(); - if home_entry_removed { - current.state = IntentState::Done; - Intent::write(home, ¤t)?; - Intent::unlink(home, store, project)?; + let attempt = mark_and_finish( + home, store, project, stack, intent, deleted_ms, effects, admission, + ); + let released = effects.release(admission); + match attempt { + Ok(outcome) => { + released?; + Ok(outcome) + } + // The original failure is the reported one; a failed release is named + // so a leaked admission row is not silent. + Err(error) => Err(match released { + Ok(()) => error, + Err(cause) => SdkRefusal { + message: format!( + "{}; releasing the project admission: {}", + error.message, cause.message + ), + ..error + }, + }), } - Ok(removed(store, project, deleted_ms, home_entry_removed)) } /// Removes a directory tree, treating an absent entry as removed. @@ -395,9 +516,36 @@ impl Effects for Local { identity::save_record(&self.home, &self.store, &id) } - fn mark(&mut self, stack: LayerStackId, deleted_ms: i64) -> Result { + fn admit(&mut self, stack: LayerStackId, reuse: Option<[u8; 16]>) -> Result { + let intent = match reuse { + Some(op) => store_admission::project_removal_at(stack, op), + None => store_admission::project_removal_from_id(stack), + }; + let op = intent.op_id; + service::with_service(&self.open, |service| { + match service.store.history().gc_register(&intent) { + Ok(()) => Ok(Admission::new(stack, op)), + // A provider without admission cannot exclude a removal; the + // marker's own admission still runs, as it did before T6. + Err(HistoryError::Unsupported(_)) => Ok(Admission::new(stack, op)), + Err(error) => Err(lookup::history(error)), + } + }) + } + + fn release(&mut self, admission: Admission) -> Result<()> { + service::with_service(&self.open, |service| { + match service.store.history().gc_release(&admission.op) { + Ok(()) | Err(HistoryError::Unsupported(_)) => Ok(()), + Err(error) => Err(lookup::history(error)), + } + }) + } + + fn mark(&mut self, admission: Admission, deleted_ms: i64) -> Result { + let intent = store_admission::project_removal_at(admission.stack, admission.op); service::with_service(&self.open, |service| { - delete_stack(&service.store, stack, deleted_ms) + delete_stack_with(&service.store, admission.stack, deleted_ms, &intent) }) } @@ -420,19 +568,61 @@ impl Effects for Remote { Ok(()) } - fn mark(&mut self, stack: LayerStackId, deleted_ms: i64) -> Result { + fn admit(&mut self, stack: LayerStackId, reuse: Option<[u8; 16]>) -> Result { + let op = reuse.unwrap_or_else(store_admission::op_id); + let frame = DeleteAdmitFrame { + store: StoreName::parse(&self.store)?, + stack, + op, + store_record: Some(self.store_record.clone()), + }; + let answer = match self + .worker + .call("project.delete.admit", frame.encode(), &mut |_| {}) + { + Ok(answer) => answer, + Err(refusal) => { + // The request went out, so the worker may have registered the + // row before its answer was lost: settle the identity before + // reporting the unknown outcome. + if refusal.code == "Unknown" { + let _ = self.release(Admission::new(stack, op)); + } + return Err(refusal); + } + }; + if let Err(refusal) = admit_reply(&answer, stack) { + let _ = self.release(Admission::new(stack, op)); + return Err(refusal); + } + Ok(Admission::new(stack, op)) + } + + fn release(&mut self, admission: Admission) -> Result<()> { + let frame = DeleteReleaseFrame { + store: StoreName::parse(&self.store)?, + op: admission.op, + store_record: Some(self.store_record.clone()), + }; + self.worker + .call("project.delete.release", frame.encode(), &mut |_| {})?; + Ok(()) + } + + fn mark(&mut self, admission: Admission, deleted_ms: i64) -> Result { let frame = DeleteFrame { store: StoreName::parse(&self.store)?, project: Some(self.project.clone()), - stack, + stack: admission.stack, deleted_ms: Some(deleted_ms), + op_id: Some(admission.op), store_record: Some(self.store_record.clone()), }; let answer = self .worker .call("project.delete", frame.encode(), &mut |_| {})?; let reply = DeleteReply::decode(&answer)?; - if reply.stack != stack { + if reply.stack != admission.stack { return Err(refused( "Failed", "the worker answered for another LayerStack", @@ -448,6 +638,24 @@ impl Effects for Remote { } } +/// Checks the worker's `project.delete.admit` answer. +fn admit_reply(answer: &SdkValue, stack: LayerStackId) -> Result<()> { + if answer.get_text("kind") != Some("Admitted") { + return Err(refused( + "Failed", + "the worker project.delete admission is not `Admitted`", + )); + } + let answered = answer + .get_text("stack_id") + .and_then(unhex) + .and_then(|bytes| LayerStackId::from_slice(&bytes).ok()); + if answered != Some(stack) { + return Err(refused("Failed", "the worker admitted another LayerStack")); + } + Ok(()) +} + /// The `OpenArgs` a local Store call opens with. `record` is absent for a /// Store made by its own project Init, whose identity is the project entry. fn open_args( @@ -569,23 +777,95 @@ pub fn delete(home: &Path, events: &mut Events<'_>, project: &ProjectRef) -> Out } } +/// The `OpenArgs` a worker's `project.delete` ops open the Store with: the +/// Store file under the worker's own root and the caller's Store record. +fn worker_open( + root: &Path, + store: &str, + project: Option, + store_record: SdkValue, +) -> OpenArgs { + OpenArgs { + home: Some(root.to_path_buf()), + store: Some(store.to_owned()), + project, + store_path: Some(paths::store_file(root, store).display().to_string()), + store_record: Some(store_record), + postgresql: None, + } +} + +/// The `{kind: "Admitted", stack_id}` answer. +fn admitted(stack: LayerStackId) -> SdkValue { + SdkValue::map([ + ("kind", SdkValue::text("Admitted")), + ("stack_id", SdkValue::text(hex(&stack.to_bytes()))), + ]) +} + +/// The `{kind: "Released", op_id}` answer. +fn released(op: [u8; 16]) -> SdkValue { + SdkValue::map([ + ("kind", SdkValue::text("Released")), + ("op_id", SdkValue::text(hex(&op))), + ]) +} + +/// The worker's `project.delete.admit`: registers the caller's project-scope +/// admission before the caller writes its sidecar and before the marker, so a +/// live mount refuses with `MountsActive` (its holders reported) and a live +/// operation with `Busy`, with no sidecar written. +pub fn admit_project(root: &Path, frame: &DeleteAdmitFrame) -> Outcome { + let store_record = frame.store_record.clone().ok_or_else(|| { + refused( + "InvalidInput", + "project.delete.admit needs the Store's record", + ) + })?; + let open = worker_open(root, frame.store.as_str(), None, store_record); + let intent = store_admission::project_removal_at(frame.stack, frame.op); + service::with_service(&open, |service| { + match service.store.history().gc_register(&intent) { + Ok(()) => Ok(admitted(frame.stack)), + // A provider without admission cannot exclude a removal; the + // marker still admits, as it did before T6. + Err(HistoryError::Unsupported(_)) => Ok(admitted(frame.stack)), + Err(error) => Err(lookup::history(error)), + } + }) +} + +/// The worker's `project.delete.release`: deletes the caller's admission row +/// after the marker, on every exit path. +pub fn release_project(root: &Path, frame: &DeleteReleaseFrame) -> Outcome { + let store_record = frame.store_record.clone().ok_or_else(|| { + refused( + "InvalidInput", + "project.delete.release needs the Store's record", + ) + })?; + let open = worker_open(root, frame.store.as_str(), None, store_record); + service::with_service(&open, |service| { + match service.store.history().gc_release(&frame.op) { + Ok(()) | Err(HistoryError::Unsupported(_)) => Ok(released(frame.op)), + Err(error) => Err(lookup::history(error)), + } + }) +} + /// The worker's `project.delete` on a Store under `root`: save the legacy /// own-project identity, write the marker, remove the worker's own entry and -/// answer `{kind, project, stack_id, deleted_ms}`. +/// answer `{kind, project, stack_id, deleted_ms}`. The marker reuses the +/// caller's admission operation identity when its frame carries one, so the +/// caller's admitted row is excluded from its own scan; without one the +/// marker takes a fresh identity and its own admission, as before. pub fn delete_project(root: &Path, frame: &DeleteFrame) -> Outcome { let store = frame.store.as_str(); let store_record = frame .store_record .clone() .ok_or_else(|| refused("InvalidInput", "project.delete needs the Store's record"))?; - let open = OpenArgs { - home: Some(root.to_path_buf()), - store: Some(store.to_owned()), - project: frame.project.clone(), - store_path: Some(paths::store_file(root, store).display().to_string()), - store_record: Some(store_record), - postgresql: None, - }; + let open = worker_open(root, store, frame.project.clone(), store_record); // The legacy own-project entry carries the Store's identity until the // marker; save it first, idempotently. let own = frame.project.as_deref() == Some(store) && identity::legacy(root, store)?; @@ -594,9 +874,11 @@ pub fn delete_project(root: &Path, frame: &DeleteFrame) -> Outcome { layerfs_home::home::private_dir(&identity::store_dir(root, store)?)?; identity::save_record(root, store, &id)?; } + let op = frame.op_id.unwrap_or_else(store_admission::op_id); + let intent = store_admission::project_removal_at(frame.stack, op); let requested = frame.deleted_ms.unwrap_or_else(now_ms); let deletion = service::with_service(&open, |service| { - delete_stack(&service.store, frame.stack, requested) + delete_stack_with(&service.store, frame.stack, requested, &intent) })?; let deleted_ms = deletion.deleted_ms(requested); let project = remove_worker_entry(root, store, frame.stack, frame.project.as_deref())? diff --git a/control/crates/layerfs-store/src/project/frame.rs b/control/crates/layerfs-store/src/project/frame.rs index 8de409d3a3..808fbd46e4 100644 --- a/control/crates/layerfs-store/src/project/frame.rs +++ b/control/crates/layerfs-store/src/project/frame.rs @@ -1,6 +1,7 @@ //! The `project` worker frames (layerfs-project#109): `project.init`, -//! `project.remove`, `project.list` (T4.5c), `project.stage` and -//! `project.export`, each encoded by the caller and +//! `project.remove`, `project.list` (T4.5c), `project.stage`, +//! `project.export`, `project.delete` (T4.5b) and its admission ops, each +//! encoded by the caller and //! decoded once in the worker, in the order the worker always refused them. //! The worker sets the home to its own root; a frame names none. use layerfs_bridge::control::{SdkRefusal, SdkValue}; @@ -108,6 +109,17 @@ fn stack_id(text: &str) -> Result { }) } +/// The operation identity an `op_id` field holds: its 16 bytes in hex. +fn op_id(text: &str) -> Result<[u8; 16]> { + let bytes = unhex(text).ok_or_else(|| refused("InvalidInput", "`op_id` must be hex digits"))?; + bytes.as_slice().try_into().map_err(|_| { + refused( + "InvalidInput", + "`op_id` must be an operation identity (16 bytes)", + ) + }) +} + impl InitFrame { /// The `project.init` frame: `{ store, name, staged, timeoutMs, /// store_record }`, each sent when set. @@ -246,7 +258,9 @@ impl ExportFrame { /// worker, spec §4). The caller writes `deleted_ms` to its durable intent /// before the call and resends it on a retry; `project` and `store_record` /// are the caller context for the worker own entry and its Store open. -#[derive(Clone, Debug)] +/// `op_id` is the admission the caller already took, so the marker reuses it +/// instead of scanning past a second row. +#[derive(Clone, Debug, Eq, PartialEq)] pub struct DeleteFrame { /// The Store. pub store: StoreName, @@ -257,6 +271,37 @@ pub struct DeleteFrame { /// The `deleted_ms` the marker should hold; `None` lets the worker take /// the current time. pub deleted_ms: Option, + /// The admission operation identity the marker reuses, when the caller + /// took one. + pub op_id: Option<[u8; 16]>, + /// The Store record. + pub store_record: Option, +} + +/// `project.delete.admit`: register the caller's project-scope admission +/// before it writes its sidecar and sends the marker (T4.6). The frame is +/// `{store, stack_id, op_id, store_record?}`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DeleteAdmitFrame { + /// The Store. + pub store: StoreName, + /// The LayerStack the caller recorded. + pub stack: LayerStackId, + /// The admission operation identity. + pub op: [u8; 16], + /// The Store record. + pub store_record: Option, +} + +/// `project.delete.release`: delete the caller's admission row after the +/// marker, on every exit path (T4.6). The frame is +/// `{store, op_id, store_record?}`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DeleteReleaseFrame { + /// The Store. + pub store: StoreName, + /// The admission operation identity. + pub op: [u8; 16], /// The Store record. pub store_record: Option, } @@ -275,7 +320,7 @@ pub struct DeleteReply { impl DeleteFrame { /// The `project.delete` frame: `{ store, stack_id, deleted_ms?, project?, - /// store_record? }`, each optional field sent when set. + /// op_id?, store_record? }`, each optional field sent when set. pub fn encode(&self) -> SdkValue { let mut fields = vec![ ("store", SdkValue::text(self.store.as_str())), @@ -287,12 +332,14 @@ impl DeleteFrame { .as_deref() .map(|p| ("project", SdkValue::text(p))), ); + fields.extend(self.op_id.map(|op| ("op_id", SdkValue::text(hex(&op))))); fields.extend(self.store_record.clone().map(|r| ("store_record", r))); map(fields) } - /// A `project.delete` frame: no Store, a bad Store name, no Stack id, or - /// a Stack id that is not a LayerStack identity are refused. + /// A `project.delete` frame: no Store, a bad Store name, no Stack id, a + /// Stack id that is not a LayerStack identity, or an `op_id` that is not + /// an operation identity are refused. pub fn decode(frame: &SdkValue) -> Result { let store = StoreName::parse(required(frame.get_text("store"), "store")?)?; let stack = stack_id(required(frame.get_text("stack_id"), "stack_id")?)?; @@ -301,6 +348,59 @@ impl DeleteFrame { project: frame.get_text("project").map(str::to_owned), stack, deleted_ms: frame.get_int("deleted_ms"), + op_id: frame.get_text("op_id").map(op_id).transpose()?, + store_record: frame.get("store_record").cloned(), + }) + } +} + +impl DeleteAdmitFrame { + /// The `project.delete.admit` frame: `{store, stack_id, op_id, + /// store_record?}`. + pub fn encode(&self) -> SdkValue { + let mut fields = vec![ + ("store", SdkValue::text(self.store.as_str())), + ("stack_id", SdkValue::text(hex(&self.stack.to_bytes()))), + ("op_id", SdkValue::text(hex(&self.op))), + ]; + fields.extend(self.store_record.clone().map(|r| ("store_record", r))); + map(fields) + } + + /// A `project.delete.admit` frame: its Store, Stack and operation + /// identity are required and checked. + pub fn decode(frame: &SdkValue) -> Result { + let store = StoreName::parse(required(frame.get_text("store"), "store")?)?; + let stack = stack_id(required(frame.get_text("stack_id"), "stack_id")?)?; + let op = op_id(required(frame.get_text("op_id"), "op_id")?)?; + Ok(Self { + store, + stack, + op, + store_record: frame.get("store_record").cloned(), + }) + } +} + +impl DeleteReleaseFrame { + /// The `project.delete.release` frame: `{store, op_id, store_record?}`. + pub fn encode(&self) -> SdkValue { + let mut fields = vec![ + ("store", SdkValue::text(self.store.as_str())), + ("op_id", SdkValue::text(hex(&self.op))), + ]; + fields.extend(self.store_record.clone().map(|r| ("store_record", r))); + map(fields) + } + + /// A `project.delete.release` frame: its Store and operation identity + /// are required and checked. + pub fn decode(frame: &SdkValue) -> Result { + let store = StoreName::parse(required(frame.get_text("store"), "store")?)?; + let op = op_id(required(frame.get_text("op_id"), "op_id")?)?; + Ok(Self { + store, + op, store_record: frame.get("store_record").cloned(), }) } diff --git a/control/crates/layerfs-store/tests/project_delete.rs b/control/crates/layerfs-store/tests/project_delete.rs index 43a30eab48..727bcb981b 100644 --- a/control/crates/layerfs-store/tests/project_delete.rs +++ b/control/crates/layerfs-store/tests/project_delete.rs @@ -9,7 +9,7 @@ use layerfs_history::LayerStackId; use layerfs_home::home::{paths, write_record}; use layerfs_home::outcome::{failed, refused}; use layerfs_store::project::delete::{ - delete_with, remove_worker_entry, Effects, Intent, IntentState, + delete_with, remove_worker_entry, Admission, Effects, Intent, IntentState, }; use layerfs_store_service::history::deletion::Deletion; use std::collections::VecDeque; @@ -54,11 +54,14 @@ fn intent(_home: &Path, store: &str, project: &str, stack: LayerStackId, id: i64 deleted_ms: id, attempt_ms: 1, state: IntentState::Pending, + op: None, } } -/// A fake Store side and entry remover. `mark` reads the intent as the -/// marker would see it, so a test can see the state that preceded it. +/// A fake home/worker: the Store side and entry remover, with the project +/// scope admission a live mount or operation would refuse. `mark` reads the +/// intent as the marker would see it, so a test can see the state that +/// preceded it. struct Fake { home: PathBuf, store: String, @@ -68,6 +71,14 @@ struct Fake { states: Vec, prepared: usize, fail_removal: bool, + /// What `admit` refuses with, before registering anything. + refuse: Option, + /// What `mark` refuses with, before writing anything. + mark_refusal: Option, + /// Admissions taken, in order. + admitted: Vec, + /// Admissions released, in order. + released: Vec, } impl Fake { @@ -81,6 +92,10 @@ impl Fake { states: Vec::new(), prepared: 0, fail_removal: false, + refuse: None, + mark_refusal: None, + admitted: Vec::new(), + released: Vec::new(), } } @@ -88,6 +103,22 @@ impl Fake { self.fail_removal = true; self } + + /// A Store with a live mount (or another live operation) at the project + /// scope: `admit` refuses with the same code the Store's admission does. + fn refusing(mut self, code: &str, detail: SdkValue) -> Self { + self.refuse = Some(SdkRefusal { + code: code.to_owned(), + message: format!("test admission refusal: {code}"), + detail, + }); + self + } + + /// A fake whose admission refuses. + fn refused(&self) -> bool { + self.refuse.is_some() + } } impl Effects for Fake { @@ -96,8 +127,29 @@ impl Effects for Fake { Ok(()) } - fn mark(&mut self, stack: LayerStackId, deleted_ms: i64) -> Result { - self.seen.push((stack, deleted_ms)); + fn admit( + &mut self, + stack: LayerStackId, + reuse: Option<[u8; 16]>, + ) -> Result { + if self.refused() { + return Err(self.refuse.clone().expect("refused above")); + } + let admission = Admission::new(stack, reuse.unwrap_or([0x5a; 16])); + self.admitted.push(admission); + Ok(admission) + } + + fn release(&mut self, admission: Admission) -> Result<(), SdkRefusal> { + self.released.push(admission); + Ok(()) + } + + fn mark(&mut self, admission: Admission, deleted_ms: i64) -> Result { + if let Some(refusal) = &self.mark_refusal { + return Err(refusal.clone()); + } + self.seen.push((admission.stack, deleted_ms)); let state = Intent::read(&self.home, &self.store, &self.project) .unwrap() .expect("the intent is durable before the marker") @@ -142,11 +194,104 @@ fn a_fresh_delete_marks_then_removes_the_entry_and_unlinks_the_intent() { // the identity save precedes the intent, and the intent precedes the mark assert_eq!(fake.prepared, 1); assert_eq!(fake.states, [IntentState::Pending]); + // the project scope was admitted before the marker and released after + assert_eq!(fake.admitted.len(), 1); + assert_eq!(fake.released, fake.admitted); assert!(!paths::project_dir(&home, store, project).exists()); assert_eq!(Intent::read(&home, store, project).unwrap(), None); let _ = std::fs::remove_dir_all(&home); } +#[test] +fn a_refused_admission_writes_no_sidecar_and_leaves_no_intent() { + let refused_home = home("refused"); + let (store, project, stack) = ("shop", "app", stack(20)); + project_record(&refused_home, store, project, stack); + let holders = SdkValue::map([( + "holders", + SdkValue::List(vec![SdkValue::map([ + ("kind", t("mount")), + ("entity", t("main")), + ])]), + )]); + let mut fake = + Fake::new(&refused_home, store, project, Vec::new()).refusing("MountsActive", holders); + let refusal = delete_with(&refused_home, store, project, Some(stack), &mut fake).unwrap_err(); + // the live mount's refusal is the call's, with its holders + assert_eq!(refusal.code, "MountsActive"); + assert!(refusal.detail.get("holders").is_some()); + // no identity save, no sidecar, no marker + assert_eq!(fake.prepared, 0); + assert!(fake.seen.is_empty(), "{:?}", fake.seen); + assert_eq!(Intent::read(&refused_home, store, project).unwrap(), None); + // the admission refused, so there is no admission to release + assert!(fake.admitted.is_empty()); + assert!(fake.released.is_empty()); + // the home entry and the project's record stay + assert!(paths::project_dir(&refused_home, store, project).is_dir()); + let _ = std::fs::remove_dir_all(&refused_home); + + // a live operation at the project scope is `Busy`, with the same rule + let busy_home = home("busy"); + project_record(&busy_home, store, project, stack); + let conflict = SdkValue::map([ + ("kind", t("removal")), + ("entity", t("app")), + ("phase", SdkValue::Null), + ("op", t("11")), + ("uncertain", SdkValue::Bool(false)), + ]); + let mut fake = Fake::new(&busy_home, store, project, Vec::new()).refusing("Busy", conflict); + let refusal = delete_with(&busy_home, store, project, Some(stack), &mut fake).unwrap_err(); + assert_eq!(refusal.code, "Busy"); + assert_eq!(fake.prepared, 0); + assert!(fake.seen.is_empty()); + assert_eq!(Intent::read(&busy_home, store, project).unwrap(), None); + let _ = std::fs::remove_dir_all(&busy_home); +} + +#[test] +fn a_retry_reuses_the_sidecar_admission_identity() { + let home = home("reuse"); + let (store, project, stack) = ("shop", "app", stack(22)); + project_record(&home, store, project, stack); + let mut pending = intent(&home, store, project, stack, 111); + pending.op = Some([3; 16]); + Intent::write(&home, &pending).unwrap(); + let mut fake = Fake::new( + &home, + store, + project, + vec![Deletion::Retired { deleted_ms: 111 }], + ); + let outcome = delete_with(&home, store, project, Some(stack), &mut fake).unwrap(); + // the retry re-registers the recorded identity, then releases it + assert_eq!(fake.admitted, [Admission::new(stack, [3; 16])]); + assert_eq!(fake.released, fake.admitted); + assert_eq!(outcome.get_int("deleted_ms"), Some(111)); + assert_eq!(Intent::read(&home, store, project).unwrap(), None); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn a_failed_marker_releases_the_admission_it_took() { + let home = home("marker-refused"); + let (store, project, stack) = ("shop", "app", stack(21)); + project_record(&home, store, project, stack); + let mut fake = Fake::new(&home, store, project, Vec::new()); + fake.mark_refusal = Some(refused("Busy", "the project is held")); + let refusal = delete_with(&home, store, project, Some(stack), &mut fake).unwrap_err(); + assert_eq!(refusal.code, "Busy"); + // the admission taken before the sidecar is released on the failure + assert_eq!(fake.admitted.len(), 1); + assert_eq!(fake.released, fake.admitted); + // the sidecar stays `pending` for the retry, and the entry stays + let left = Intent::read(&home, store, project).unwrap().unwrap(); + assert_eq!(left.state, IntentState::Pending); + assert!(paths::project_dir(&home, store, project).is_dir()); + let _ = std::fs::remove_dir_all(&home); +} + #[test] fn a_retired_retry_reuses_the_intent_deleted_ms_and_finishes_the_entry() { let home = home("retired"); @@ -350,9 +495,11 @@ fn the_intent_moves_pending_marked_done_and_unlinks() { value.state = IntentState::Marked; value.deleted_ms = 6; + value.op = Some([4; 16]); Intent::write(&home, &value).unwrap(); let read = Intent::read(&home, store, project).unwrap().unwrap(); assert_eq!((read.state, read.deleted_ms), (IntentState::Marked, 6)); + assert_eq!(read.op, Some([4; 16]), "the admission identity persists"); value.state = IntentState::Done; Intent::write(&home, &value).unwrap(); diff --git a/control/crates/layerfs-store/tests/project_delete_frames.rs b/control/crates/layerfs-store/tests/project_delete_frames.rs index 6d5b05e429..87b7178990 100644 --- a/control/crates/layerfs-store/tests/project_delete_frames.rs +++ b/control/crates/layerfs-store/tests/project_delete_frames.rs @@ -4,7 +4,9 @@ use layerfs_bridge::control::SdkValue; use layerfs_bridge::sdk_values::hex; use layerfs_history::LayerStackId; use layerfs_home::names::StoreName; -use layerfs_store::project::frame::{DeleteFrame, DeleteReply}; +use layerfs_store::project::frame::{ + DeleteAdmitFrame, DeleteFrame, DeleteReleaseFrame, DeleteReply, +}; fn t(text: &str) -> SdkValue { SdkValue::text(text) @@ -21,12 +23,14 @@ fn stack(body: u8) -> LayerStackId { #[test] fn the_frame_names_the_store_stack_and_optional_context() { let stack = stack(1); + let op = [7u8; 16]; let record = map(vec![("name", t("shop")), ("kind", t("sqlite"))]); let frame = DeleteFrame { store: StoreName::parse("shop").unwrap(), project: Some("app".to_owned()), stack, deleted_ms: Some(5), + op_id: Some(op), store_record: Some(record.clone()), }; let encoded = frame.encode(); @@ -37,6 +41,7 @@ fn the_frame_names_the_store_stack_and_optional_context() { ("stack_id", t(&hex(&stack.to_bytes()))), ("deleted_ms", SdkValue::Int(5)), ("project", t("app")), + ("op_id", t(&hex(&op))), ("store_record", record), ]) ); @@ -45,6 +50,7 @@ fn the_frame_names_the_store_stack_and_optional_context() { assert_eq!(decoded.project, frame.project); assert_eq!(decoded.stack, stack); assert_eq!(decoded.deleted_ms, Some(5)); + assert_eq!(decoded.op_id, Some(op)); assert_eq!(decoded.store_record, frame.store_record); } @@ -57,10 +63,77 @@ fn a_frame_with_only_the_store_and_stack_decodes() { ]); let decoded = DeleteFrame::decode(&minimal).unwrap(); assert_eq!((decoded.project, decoded.deleted_ms), (None, None)); + assert_eq!(decoded.op_id, None); assert_eq!(decoded.store_record, None); assert_eq!(decoded.stack, stack); } +#[test] +fn the_admission_and_release_frames_round_trip() { + let stack = stack(4); + let op = [9u8; 16]; + let record = map(vec![("name", t("shop")), ("kind", t("sqlite"))]); + let admit = DeleteAdmitFrame { + store: StoreName::parse("shop").unwrap(), + stack, + op, + store_record: Some(record.clone()), + }; + let encoded = admit.encode(); + assert_eq!( + encoded, + map(vec![ + ("store", t("shop")), + ("stack_id", t(&hex(&stack.to_bytes()))), + ("op_id", t(&hex(&op))), + ("store_record", record.clone()), + ]) + ); + assert_eq!(DeleteAdmitFrame::decode(&encoded).unwrap(), admit); + + let release = DeleteReleaseFrame { + store: StoreName::parse("shop").unwrap(), + op, + store_record: None, + }; + let encoded = release.encode(); + assert_eq!( + encoded, + map(vec![("store", t("shop")), ("op_id", t(&hex(&op)))]) + ); + assert_eq!(DeleteReleaseFrame::decode(&encoded).unwrap(), release); +} + +#[test] +fn the_admission_frames_refuse_a_bad_operation_identity() { + let decode = |fields| DeleteAdmitFrame::decode(&map(fields)).unwrap_err().message; + let stack = stack(5); + assert_eq!( + decode(vec![ + ("store", t("shop")), + ("stack_id", t(&hex(&stack.to_bytes()))), + ("op_id", t("zz")), + ]), + "`op_id` must be hex digits" + ); + assert_eq!( + decode(vec![ + ("store", t("shop")), + ("stack_id", t(&hex(&stack.to_bytes()))), + ("op_id", t("ab")), + ]), + "`op_id` must be an operation identity (16 bytes)" + ); + assert_eq!( + delete_release_decode(&map(vec![("store", t("shop"))])), + "missing text argument `op_id`" + ); +} + +fn delete_release_decode(frame: &SdkValue) -> String { + DeleteReleaseFrame::decode(frame).unwrap_err().message +} + #[test] fn the_worker_refuses_a_bad_frame_in_order() { let decode = |fields| DeleteFrame::decode(&map(fields)).unwrap_err().message; diff --git a/control/crates/layerfs-store/tests/project_delete_store.rs b/control/crates/layerfs-store/tests/project_delete_store.rs new file mode 100644 index 0000000000..1d0e945a70 --- /dev/null +++ b/control/crates/layerfs-store/tests/project_delete_store.rs @@ -0,0 +1,238 @@ +//! `project.delete`'s home path against a real local Store (T4.6): the +//! control layer admits the project scope through `gc_register`/`gc_release` +//! before its sidecar and marker, so a live mount refuses with +//! `MountsActive` (its holders), a live operation with `Busy`, and a refusal +//! leaves no sidecar, no marker and no leaked admission intent. +use layerfs_bridge::control::SdkValue; +use layerfs_history::records::{GcIntent, GcIntentKind, GcScopeKind}; +use layerfs_history::{HistoryName, LayerStackId, NamedSnapshot}; +use layerfs_home::events::Events; +use layerfs_home::home::{layout, paths}; +use layerfs_home::names::{ProjectRef, StoreName}; +use layerfs_store::open_args::OpenArgs; +use layerfs_store::project::add::{add_project, ProjectAdd}; +use layerfs_store::project::delete::{delete, Intent}; +use layerfs_store::record::StoreRecord; +use layerfs_store::service::with_service; +use layerfs_store::sqlite::create::create; +use layerfs_store_service::history::lookup; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; + +static NEXT: AtomicU64 = AtomicU64::new(0); + +fn tempdir(tag: &str) -> PathBuf { + let directory = std::env::temp_dir().join(format!( + "layerfs-delete-store-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&directory); + layout::create(&directory).unwrap(); + directory +} + +/// A folder to initialize a project from: no home layout inside it. +fn plaindir(tag: &str) -> PathBuf { + let directory = std::env::temp_dir().join(format!( + "layerfs-delete-store-folder-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&directory); + std::fs::create_dir_all(&directory).unwrap(); + directory +} + +/// True when the sandbox filesystem cannot hold a directly opened Store, so +/// these real-Store cases cannot run here. The fake-home cases in +/// `project_delete` cover the same home path without a Store. +fn no_direct_store(refusal: &layerfs_bridge::control::SdkRefusal) -> bool { + refusal.code == "Unsupported" && refusal.message.contains("does not carry SQLite locks") +} + +/// A home whose Store `shop` holds the live project `app`, and the +/// `OpenArgs` (with the Store record) that reach it. `None` when this +/// filesystem cannot hold a directly opened Store. +fn shop(tag: &str) -> Option<(PathBuf, OpenArgs, LayerStackId)> { + let home = tempdir(tag); + match create(&home, &StoreName::parse("shop").unwrap()) { + Ok(_) => {} + Err(refusal) if no_direct_store(&refusal) => { + eprintln!("NOT RUN: this filesystem cannot hold a directly opened Store: {refusal:?}"); + let _ = std::fs::remove_dir_all(&home); + return None; + } + Err(refusal) => panic!("store.create: {refusal:?}"), + } + let source = plaindir(tag); + std::fs::write(source.join("a.txt"), b"a\n").unwrap(); + add_project(&ProjectAdd { + home: home.clone(), + store: StoreName::parse("shop").unwrap(), + name: "app".to_owned(), + source, + timeout_ms: None, + store_record: None, + }) + .unwrap(); + let record = StoreRecord::read(&home, "shop").unwrap().unwrap(); + let open = OpenArgs { + home: Some(home.clone()), + store: Some("shop".to_owned()), + project: Some("app".to_owned()), + store_path: Some(paths::store_file(&home, "shop").display().to_string()), + store_record: Some(record.to_record().clone()), + postgresql: None, + }; + let stack = with_service(&open, |service| { + Ok(lookup::project(&service.store, "app").unwrap().id) + }) + .unwrap(); + Some((home, open, stack)) +} + +/// `project.delete shop/app` through the control-layer home path. +fn remove(home: &std::path::Path) -> Result { + let mut sink = |_| {}; + delete( + home, + &mut Events::new(&mut sink), + &ProjectRef::parse("shop/app").unwrap(), + ) +} + +/// The live intents at the project scope. +fn intents(open: &OpenArgs, stack: LayerStackId) -> Vec { + with_service(open, |service| { + Ok(service + .store + .history() + .gc_intents(GcScopeKind::Project, stack.as_slice()) + .unwrap()) + }) + .unwrap() +} + +/// Registers a live mount of the project's `main` Branch, as a Workspace +/// bind does before it demands the root. +fn mount_main(open: &OpenArgs, stack: LayerStackId, op: [u8; 16]) { + // `lookup::branch` refuses `main` by design; the snapshot lookup names it + let branch = with_service(open, |service| { + let project = HistoryName::new("app").unwrap(); + let main = HistoryName::new("main").unwrap(); + match service + .store + .published(|history| history.branch_snapshot_named(&project, &main)) + .unwrap() + { + NamedSnapshot::Found(snapshot) => Ok(snapshot.branch.id), + other => panic!("no main Branch: {other:?}"), + } + }) + .unwrap(); + let intent = GcIntent::shared( + op, + GcIntentKind::Mount, + GcScopeKind::Branch, + branch.as_slice(), + ) + .in_project(stack.as_slice()) + .named(Some("app"), Some("main")) + .placed(Some([9; 32]), Some("sandbox-9"), Some("machine-9")); + with_service(open, |service| { + service.store.history().gc_register(&intent).unwrap(); + Ok(()) + }) + .unwrap(); +} + +fn release(open: &OpenArgs, op: [u8; 16]) { + with_service(open, |service| { + service.store.history().gc_release(&op).unwrap(); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn a_live_mount_refuses_the_delete_before_the_sidecar_and_leaves_no_intent() { + let Some((home, open, stack)) = shop("mount") else { + return; + }; + mount_main(&open, stack, [9; 16]); + assert!(intents(&open, stack).is_empty(), "no project-scope row yet"); + + let refusal = remove(&home).expect_err("a live mount holds the project"); + assert_eq!(refusal.code, "MountsActive", "{refusal:?}"); + let SdkValue::List(holders) = refusal.detail.get("holders").expect("holders") else { + panic!("{refusal:?}"); + }; + assert_eq!(holders[0].get_text("kind"), Some("mount")); + assert_eq!(holders[0].get_text("sandbox"), Some("sandbox-9")); + + // no sidecar pending, no marker and no leaked admission intent + assert_eq!(Intent::read(&home, "shop", "app").unwrap(), None); + assert!(paths::project_dir(&home, "shop", "app").is_dir()); + assert!( + intents(&open, stack).is_empty(), + "the refused admission left no project-scope row" + ); + with_service(&open, |service| { + assert_eq!( + lookup::project(&service.store, "app") + .expect("still live") + .id, + stack + ); + Ok(()) + }) + .unwrap(); + + // the mount goes and the same home path succeeds, releasing its admission + release(&open, [9; 16]); + let outcome = remove(&home).unwrap(); + assert_eq!(outcome.get_text("kind"), Some("Removed")); + assert_eq!(outcome.get_text("project"), Some("shop/app")); + assert!(outcome.get_bool("home_entry_removed")); + assert!(!paths::project_dir(&home, "shop", "app").exists()); + assert_eq!(Intent::read(&home, "shop", "app").unwrap(), None); + assert!( + intents(&open, stack).is_empty(), + "the admission was released" + ); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn a_live_operation_at_the_project_scope_is_busy_and_writes_no_sidecar() { + let Some((home, open, stack)) = shop("busy") else { + return; + }; + assert!(intents(&open, stack).is_empty()); + let holder = GcIntent::operation( + [7; 16], + GcIntentKind::Removal, + GcScopeKind::Project, + stack.as_slice(), + ) + .named(Some("app"), None); + with_service(&open, |service| { + service.store.history().gc_register(&holder).unwrap(); + Ok(()) + }) + .unwrap(); + + let refusal = remove(&home).expect_err("another removal holds the project"); + assert_eq!(refusal.code, "Busy", "{refusal:?}"); + assert_eq!(Intent::read(&home, "shop", "app").unwrap(), None); + assert_eq!(intents(&open, stack).len(), 1, "only the test's own holder"); + assert_eq!(intents(&open, stack)[0].op_id, [7; 16]); + + // the operation ends: the delete proceeds and releases its own admission + release(&open, [7; 16]); + assert!(remove(&home).unwrap().get_bool("home_entry_removed")); + assert_eq!(Intent::read(&home, "shop", "app").unwrap(), None); + assert!(intents(&open, stack).is_empty()); + let _ = std::fs::remove_dir_all(&home); +} diff --git a/core/crates/layerfs-history/src/contract/error.rs b/core/crates/layerfs-history/src/contract/error.rs index 940259da6e..32ab59b264 100644 --- a/core/crates/layerfs-history/src/contract/error.rs +++ b/core/crates/layerfs-history/src/contract/error.rs @@ -148,6 +148,8 @@ pub enum HistoryError { NameTaken(Box), /// A path is hidden by `remove_file`. Removed(Box), + /// A staged name or artifact already exists and no record names it. + Exists(&'static str), /// A required marker or fence row is missing or contradicts the request. Conflict(&'static str), /// The Store was replaced or moved under this handle. @@ -217,6 +219,7 @@ impl std::fmt::Display for HistoryError { Self::Unsupported(what) => write!(formatter, "unsupported: {what}"), Self::NameTaken(state) => write!(formatter, "name taken: {:?}", state.holder), Self::Removed(state) => write!(formatter, "removed: {} paths", state.paths.len()), + Self::Exists(what) => write!(formatter, "exists: {what}"), Self::Conflict(what) => write!(formatter, "conflict: {what}"), Self::StaleStore => formatter.write_str("stale Store"), Self::Busy => formatter.write_str("busy"), diff --git a/core/docs/issues/project-41/gc-t8-sqlite-rebuild.md b/core/docs/issues/project-41/gc-t8-sqlite-rebuild.md new file mode 100644 index 0000000000..3ce29deab2 --- /dev/null +++ b/core/docs/issues/project-41/gc-t8-sqlite-rebuild.md @@ -0,0 +1,128 @@ +# T8: SQLite `store.compact` rebuild + +> Implementation note for the frozen [`LayerFS-GC-spec.md`](LayerFS-GC-spec.md) +> §7 (SQLite rebuild), §9 (failure and leftovers) and §11 (report). It adds no +> contract: the spec wins and `§N` cites it. + +## Placement + +`SqliteHandles::compact` is the host-side primitive T10 wraps as +`store.compact`. It consumes the handles like `SqliteHandles::seal` +(`engine/crates/layerfs-persistence/src/backend/sqlite/compact.rs`): a rebuild +replaces the Store file and the caller reopens it afterwards. The public +result is `CompactReceipt`, one value per §11 field, plus the small +`CompactPath`, `CompactKeptWaste`, `CompactResidue` and `CompactZeroReason` +types. The `GcIntent` admission row is registered through the T6 +`HistoryCatalog::gc_register` API; a mark is the T7.2 `HistoryProvider::mark`. + +## Run (SQLite) + +1. **Lane.** `Session::hold_lane()` takes the Store's FIFO turn and the + cross-process `writer_admission` lock, and keeps both. Every later + transaction of this thread is nested work of that lane, so no admitted + writer can interleave between the mark, the seal and the rename. +2. **Checkpoint.** `Session::checkpoint_holding()` runs + `PRAGMA wal_checkpoint(TRUNCATE)` under the held lane; a nonzero `busy` is + `Busy`, never retried. +3. **Admission.** The compacting call writes `.compact-op` (the SQLite + sidecar equivalent of `gc_op`) and registers one Store-scoped + `gc_intent(kind='compaction')` row with the tuple: generation, mark serial, + mark pack id, digest, totals, stamp, phase, owner tokens. +4. **Mark.** `mark()` runs in its write transaction under the same lane. The + §6.1 history closure is written to `kept_node`; the §6.2 closure follows + logical references and delta bases to its fixpoint, so the packs it keeps + are already the refined live set. `deadBytesBound` and + `deadBytesRefined` are therefore equal on both §3 paths. +5. **Path.** `rebuild iff SUM(pack.length) > 0 AND dead * 10 >= SUM(pack.length)`. + `none` clears its intent and op record and returns; it writes no Store + table and creates no artifact, so no scratch exists to delete. +6. **Seal.** `Session::seal_holding(writer)` checkpoints, checked-closes, + checks `-wal`/`-shm`/`-journal`, and captures `(size, mtime, change + counter)` while the lane is still held (§7 step 1). The sealed tuple and + the §6.4 `live_heads_digest` are then re-read through an `immutable=1` + handle and compared with the mark; a difference is `Conflict`. +7. **Build.** `.compact-visited` (`journal_mode=OFF`) holds the visited + object and live-pack sets. `VACUUM INTO .compact` copies the sealed + file compactly; inside one transaction the retained rows are kept and the + complement deleted (dead packs/pack units, unreferenced locations, + signatures and digests, history/lineage/catalog rows outside `kept_node`, + markers of collected rows, merge scratch, `gc_intent`/`gc_dead`/`gc_op`, + acquisition working rows), `binding_hint_block` is cleared, + `catalog_incarnation` becomes `G + 1`, and `store_totals` is recomputed + from the retained rows with its `init_*` columns untouched. `VACUUM`, a + recreated `sqlite_sequence`, and a WAL switch with a zero-`busy` TRUNCATE + checkpoint finish the artifact. Package immutability triggers are dropped + for the delete and recreated from their stored SQL. +8. **Verify.** An `immutable=1` reader runs `schema::check`, + `foreign_key_check`, row counts against the mark (`object_location`, + `pack`, `kept_node`), both live-heads digests, the allocator tuple, and the + recomputed totals against their row sums. +9. **Install.** Sidecars and `-mj*` on both files are rechecked, the sealed + `(size, mtime, change counter)` and the full tuple are re-read last, the + artifact gets `chmod 0600` and `sync_all`, then `rename` and a directory + fsync. A rename/fsync failure is installed-but-uncertain: the op record is + kept at phase `install` and `UnknownOutcome` is returned. +10. **Settle.** The artifact set is removed, then the op record. `residue` + reports anything that could not be removed. + +## Failure and leftovers (§9) + +The next `store.compact` reads `.compact-op` under the lane. Because +the lane was free, the recorded owner is gone: it releases that op's admission +intent (`gc_release`), removes the named artifact set, and removes the op +record, judging by the generation it read (`G` = not installed, `G + 1` or +later = installed/advanced). An artifact set that no op record names is +refused `Exists` and is never deleted. A definite pre-install failure deletes +the artifact set and the op record; the old Store is unchanged apart from the +checkpoint. Failpoints (`AfterSeal`, `AfterBuild`, `AfterWalSwitch`, +`AfterVerify`, `BeforeInstall`, `AfterInstall`) exercise the crash state +without killing the process. + +## Report (§11) + +`CompactReceipt` carries `path`, `generation {before, after}`, +`deadBytesBound`, `deadBytesRefined`, `logicalDiscarded`, +`backendReusable` (free pages × page size), `physicalShrink` (old size − new), +`keptWaste`, `peakTempBytes`, `residue`, `estimate` (`true`: §13) and one +`zeroReason` per zero field from the closed vocabulary. `logicalDiscarded` is +the collected history rows' `logical` bytes; `parallel` note: `estimate` is +always true until #52 measures the arms. + +## Tests + +`engine/crates/layerfs-persistence/tests/sqlite_compact.rs`: + +- empty Store → `none`, `zeroReason: empty-store`, unchanged generation; +- no wholly dead pack → `none`, `deadBytesRefined == deadBytesBound`, no + artifact, unchanged generation; +- a wholly dead pack → `rebuild`, `G + 1`, dead object gone, retained object + readable, no artifact/sidecar, full doctor clean; +- retained Commit/Layer history and its objects survive a rebuild; the full + doctor passes on the installed Store; +- a stray `.compact` with no op record is `Exists` and not deleted; +- `AfterBuild` leaves the staged Store, visited scratch and op record; the + reopened Store is still at `G`, and the next call settles and rebuilds; +- `AfterInstall` leaves the installed Store at `G + 1` and the op record; the + next call judges it and settles; +- generation increases once per installed rebuild. + +Runs: `cargo test -p layerfs-persistence --offline` (all binaries pass), +`cargo check --workspace --offline` passes. `kernel_guard` in +`layerfs-store-service` fails in this container because `/tmp` is tmpfs and +the guard refuses direct Store access; it is unrelated to this change. + +## Gaps (for T10/T12) + +- The `(dev, ino)` `StaleStore` fence (§10) is not implemented in the SQLite + session; install excludes admitted writers by the held lane, not by a + per-transaction inode check. +- `Capacity` mapping is implemented (`SQLITE_FULL`/`TOOBIG`/`NOMEM`, `ENOSPC`, + `EDQUOT`, `EFBIG`) but no capacity/failpoint test asserts it. +- The 9%/10%/30% boundary and the `GroupRows`/`GroupRowsIndexed` and + acquisition layouts are implemented (unit tables conditional, acquisition + rows cleared, `sqlite_sequence` preserved) but exercised only on the + Monolithic layout. The `postgresql` feature is not built here. +- Peak temp is the new file plus the visited scratch; journal/WAL sizes + after the checkpoint are zero and are not added. +- The mark runs before the seal under the same lane (the spec's step 1.5 runs + it after the seal); the sealed tuple re-read makes the window explicit. diff --git a/engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_epoch_read.sql b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_epoch_read.sql new file mode 100644 index 0000000000..67853fc6f2 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_epoch_read.sql @@ -0,0 +1 @@ +SELECT epoch FROM project_epoch WHERE layer_stack_id = $1; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_epoch_read.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_epoch_read.sql new file mode 100644 index 0000000000..67853fc6f2 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_epoch_read.sql @@ -0,0 +1 @@ +SELECT epoch FROM project_epoch WHERE layer_stack_id = $1; diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/history/statements.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/history/statements.rs index bdc010b348..bc66630699 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/history/statements.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/history/statements.rs @@ -12,6 +12,9 @@ pub(crate) fn statement(key: &str) -> Result<&'static str, BackendError> { "statistics_totals" => Ok(include_str!( "../../../../sql/postgresql/queries/history/statistics_totals.sql" )), + "gc_epoch_read" => Ok(include_str!( + "../../../../sql/postgresql/queries/history/gc_epoch_read.sql" + )), "gc_epoch_share" => Ok(include_str!( "../../../../sql/postgresql/queries/history/gc_epoch_share.sql" )), diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/compact.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/compact.rs new file mode 100644 index 0000000000..b14fa8b4c0 --- /dev/null +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/compact.rs @@ -0,0 +1,1665 @@ +//! §7 SQLite Store compaction: mark, rebuild, immutable verification and the +//! atomic install. +//! +//! One call takes the Store's writer lane before it registers its admission +//! intent, runs the T7.2 mark in the same exclusive window, seals the file +//! (checkpoint, checked close, sidecar check and `(size, mtime, change +//! counter)` capture), and then either stops on the §3 `none` path or builds +//! `.compact`, verifies it through an `immutable=1` URI, and renames it +//! into place at `G → G+1`. The run's phase and mark tuple are kept beside the +//! Store in `.compact-op`, the SQLite equivalent of the PostgreSQL +//! `gc_op` row (§11); the next `store.compact` judges a leftover by the Store +//! generation (§9) and removes the artifact set it names. +//! +//! The T7.2 mark already resolves every logical reference and delta base, so +//! the bytes of packs that hold no reached object are the refined dead bytes: +//! §7's step-4 payload sweep cannot discover another live object on SQLite. +//! `deadBytesRefined` therefore equals `deadBytesBound`, as §11 requires for +//! the `none` path. + +use super::lifecycle::seal::SealedFacts; +use super::schema; +use super::session::connection::{Session, SqlWork}; +use super::session::rows; +use crate::backend::records::BackendError; +use crate::{KeptWaste, MarkResult, SqliteHandles}; +use layerfs_content::ObjectId; +use layerfs_history::error::{HistoryError, HistoryResult}; +use layerfs_history::records::{GcIntent, GcIntentKind, GcScopeKind}; +use layerfs_history::HistoryCatalog; +use layerfs_storage::port::PersistenceError; +use rusqlite::{params, Connection, OpenFlags}; +use std::cell::RefCell; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +/// The sidecar record suffix (§11: SQLite keeps its phase outside the sealed +/// database). +const OP_SUFFIX: &str = ".compact-op"; +/// The staged rebuild Store beside the live one. +const COMPACT_SUFFIX: &str = ".compact"; +/// The §6.3 visited scratch database. +const VISITED_SUFFIX: &str = ".compact-visited"; +/// The full SQLite artifact set of §9, without the variable `-mj*` journals. +const ARTIFACT_SUFFIXES: [&str; 7] = [ + COMPACT_SUFFIX, + ".compact-journal", + ".compact-wal", + ".compact-shm", + VISITED_SUFFIX, + ".compact-visited-journal", + ".compact-vacuum", +]; + +/// The mark tuple a compaction records before its first checkpoint (§8 Phase M +/// on PostgreSQL; the same first-statement tuple in the SQLite sidecar here). +#[derive(Clone, Debug, Eq, PartialEq)] +struct MarkTuple { + catalog_id: Vec, + generation: i64, + next_history_serial: i64, + next_pack_id: i64, + next_ordinal: i64, + logical: i64, + canonical: i64, + stored: i64, + metadata: i64, + stamp: i64, + history_rows: i64, +} + +/// The first-statement tuple, in the column order every reader decodes. +const TUPLE_SQL: &str = "SELECT h.catalog_id,h.catalog_incarnation,h.next_history_serial,\ +p.next_pack_id,p.next_ordinal,t.logical,t.canonical,t.stored,t.metadata,\ +(SELECT COALESCE(MAX(stamp),0) FROM content_signature),\ +(SELECT COUNT(*) FROM \"commit\")+(SELECT COUNT(*) FROM layer) \ +FROM history_meta h, store_policy p, store_totals t \ +WHERE h.id=1 AND p.id=1 AND t.id=1"; + +/// The §6.4 live-heads digest input, in identity order. +const LIVE_STACKS: &str = "SELECT s.layer_stack_id,s.head_layer_id FROM layer_stack s \ +WHERE NOT EXISTS (SELECT 1 FROM layer_stack_retired r WHERE r.layer_stack_id=s.layer_stack_id) \ +ORDER BY s.layer_stack_id"; +/// The marker-free Branches, in identity order. +const LIVE_BRANCHES: &str = "SELECT b.branch_id,b.head_commit_id,b.base_layer_id FROM branch b \ +WHERE NOT EXISTS (SELECT 1 FROM branch_retired r WHERE r.branch_id=b.branch_id) \ +AND NOT EXISTS (SELECT 1 FROM layer_stack_retired s WHERE s.layer_stack_id=b.layer_stack_id) \ +ORDER BY b.branch_id"; + +/// Which §3 path a compaction took. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CompactPath { + /// Below the 10% wholly-dead threshold: mark only, nothing written. + None, + /// A rebuild installed at `G → G+1`. + Rebuild, +} + +impl CompactPath { + /// The report's `path` text. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::None => "none", + Self::Rebuild => "rebuild", + } + } +} + +/// A test-only stop point of the run, so a crash between phases can be +/// exercised. The run leaves the state a crash would leave and reports an +/// unknown outcome; the next `store.compact` must recover it (§9). +#[doc(hidden)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CompactFailpoint { + /// After the seal and the sealed-state comparison, before any artifact. + AfterSeal, + /// After the staged `.compact` is built and compacted. + AfterBuild, + /// After the staged file is switched to WAL and checkpointed. + AfterWalSwitch, + /// After the immutable verification passes. + AfterVerify, + /// Immediately before the rename would run. + BeforeInstall, + /// After the rename and directory fsync. + AfterInstall, +} + +impl CompactFailpoint { + /// The phase the sidecar records for this stop point. + const fn phase(self) -> &'static str { + match self { + Self::AfterSeal => "mark", + Self::AfterBuild => "build", + Self::AfterWalSwitch | Self::AfterVerify | Self::BeforeInstall => "verify", + Self::AfterInstall => "install", + } + } +} + +/// Inputs one `store.compact` attempt uses. +#[derive(Clone, Debug, Default)] +pub struct CompactRequest { + /// 16-byte operation identity, written as the admission intent's `op_id`. + pub op_id: [u8; 16], + /// Owning runtime token, for the op and intent records. + pub owner_runtime: Option, + /// Owning boot token, for the op and intent records. + pub owner_boot: Option, + /// A test-only stop point; production callers leave it `None`. + pub failpoint: Option, +} + +/// The `keptWaste` §11 part of a compaction report. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct CompactKeptWaste { + /// Bytes of unreferenced records inside packs the mark retained. + pub dead_in_retained_packs_bytes: u64, + /// Bytes of objects reached only as a delta base. + pub delta_bases_bytes: u64, + /// The objects counted by [`Self::delta_bases_bytes`], in ascending order. + pub delta_base_references: Vec<[u8; 32]>, + /// Bytes of `inode_value_block` rows, which 2.0 keeps in full. + pub inode_blocks_bytes: u64, +} + +impl CompactKeptWaste { + /// One mark's kept-waste accounting. + fn from_mark(waste: &KeptWaste) -> Self { + Self { + dead_in_retained_packs_bytes: waste.dead_in_retained_packs_bytes(), + delta_bases_bytes: waste.delta_bases_bytes(), + delta_base_references: waste + .delta_base_references() + .iter() + .map(|id| *id.as_bytes()) + .collect(), + inode_blocks_bytes: waste.inode_blocks_bytes(), + } + } + + /// Whether every part is zero. + #[must_use] + pub fn is_empty(&self) -> bool { + self.dead_in_retained_packs_bytes == 0 + && self.delta_bases_bytes == 0 + && self.inode_blocks_bytes == 0 + } +} + +/// The `residue` §11 part of a compaction report. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct CompactResidue { + /// Bytes left behind by the artifact set after settle. + pub bytes: u64, + /// The artifact file names that remain. + pub artifacts: Vec, + /// Why anything remains, when it does. + pub reason: Option, +} + +/// One zero report field and its §11 `zeroReason`. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CompactZeroReason { + /// The report field the reason explains. + pub field: &'static str, + /// One word of the §11 closed vocabulary. + pub reason: &'static str, +} + +/// The §11 report `store.compact` returns. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CompactReceipt { + /// `none` or `rebuild`. + pub path: CompactPath, + /// The generation the mark ran at. + pub generation_before: u64, + /// The installed generation: equal on `none`. + pub generation_after: u64, + /// Bytes in wholly dead packs after the metadata walk; the path's input. + pub dead_bytes_bound: u64, + /// The same bytes after the payload sweep. + pub dead_bytes_refined: u64, + /// Collected history rows' logical bytes. + pub logical_discarded: u64, + /// SQLite free pages of the new (rebuild) or live (`none`) Store. + pub backend_reusable: u64, + /// Old file size minus new. + pub physical_shrink: u64, + /// The bytes the mark keeps that no retained root needs. + pub kept_waste: CompactKeptWaste, + /// Peak temporary bytes the run needed. + pub peak_temp_bytes: u64, + /// The artifact set left after settle; empty on success. + pub residue: CompactResidue, + /// §13: the report is an estimate, not a measurement. + pub estimate: bool, + /// One `zeroReason` per zero report field. + pub zero_reasons: Vec, +} + +/// One recorded compaction run beside the Store. +#[derive(Clone, Debug)] +struct RunOp { + op_id: [u8; 16], + generation: i64, + phase: String, + started_ms: i64, + tuple: MarkTuple, + live_heads_digest: [u8; 32], +} + +impl RunOp { + fn encode(&self) -> String { + format!( + "version=1\nop_id={}\ngeneration={}\nphase={}\nstarted_ms={}\n\ +catalog_id={}\nnext_history_serial={}\nnext_pack_id={}\nnext_ordinal={}\n\ +mark_serial={}\nmark_pack_id={}\nmark_digest={}\nmark_totals={},{},{},{}\nmark_stamp={}\nhistory_rows={}\n\ +live_heads_digest={}\n", + hex(&self.op_id), + self.generation, + self.phase, + self.started_ms, + hex(&self.tuple.catalog_id), + self.tuple.next_history_serial, + self.tuple.next_pack_id, + self.tuple.next_ordinal, + self.tuple.next_history_serial.saturating_sub(1), + self.tuple.next_pack_id, + hex(&self.live_heads_digest), + self.tuple.logical, + self.tuple.canonical, + self.tuple.stored, + self.tuple.metadata, + self.tuple.stamp, + self.tuple.history_rows, + hex(&self.live_heads_digest), + ) + } + + fn write(&self, store: &Path) -> HistoryResult<()> { + let path = op_path(store); + let mut temporary = path.as_os_str().to_os_string(); + temporary.push(".tmp"); + let temporary = PathBuf::from(temporary); + std::fs::write(&temporary, self.encode()).map_err(io_error)?; + if let Ok(file) = std::fs::File::open(&temporary) { + let _ = file.sync_all(); + } + std::fs::rename(&temporary, &path).map_err(io_error)?; + if let Some(parent) = path.parent() { + let _ = std::fs::File::open(parent).and_then(|directory| directory.sync_all()); + } + Ok(()) + } + + fn read(store: &Path) -> Option { + let text = std::fs::read_to_string(op_path(store)).ok()?; + let mut values = BTreeMap::new(); + for line in text.lines() { + let (key, value) = line.split_once('=')?; + values.insert(key, value); + } + if values.get("version") != Some(&"1") { + return None; + } + let op_id = unhex::<16>(values.get("op_id")?)?; + let generation: i64 = values.get("generation")?.parse().ok()?; + let phase = (*values.get("phase")?).to_owned(); + let started_ms: i64 = values.get("started_ms")?.parse().ok()?; + let catalog_id = unhex_vec(values.get("catalog_id")?)?; + let tuple = MarkTuple { + catalog_id, + generation, + next_history_serial: values.get("next_history_serial")?.parse().ok()?, + next_pack_id: values.get("next_pack_id")?.parse().ok()?, + next_ordinal: values.get("next_ordinal")?.parse().ok()?, + logical: values.get("mark_totals")?.split(',').next()?.parse().ok()?, + canonical: values.get("mark_totals")?.split(',').nth(1)?.parse().ok()?, + stored: values.get("mark_totals")?.split(',').nth(2)?.parse().ok()?, + metadata: values.get("mark_totals")?.split(',').nth(3)?.parse().ok()?, + stamp: values.get("mark_stamp")?.parse().ok()?, + history_rows: values.get("history_rows")?.parse().ok()?, + }; + let live_heads_digest = unhex::<32>(values.get("live_heads_digest")?)?; + Some(Self { + op_id, + generation, + phase, + started_ms, + tuple, + live_heads_digest, + }) + } +} + +fn now_ms() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| { + i64::try_from(elapsed.as_millis()).unwrap_or(i64::MAX) + }) +} + +fn hex(bytes: &[u8]) -> String { + use std::fmt::Write as _; + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + let _ = write!(out, "{byte:02x}"); + } + out +} + +fn unhex_vec(text: &str) -> Option> { + if text.len() % 2 != 0 { + return None; + } + (0..text.len()) + .step_by(2) + .map(|index| u8::from_str_radix(&text[index..index + 2], 16).ok()) + .collect() +} + +fn unhex(text: &str) -> Option<[u8; N]> { + let bytes = unhex_vec(text)?; + bytes.try_into().ok() +} + +fn with_suffix(path: &Path, suffix: &str) -> PathBuf { + let mut name = path.as_os_str().to_os_string(); + name.push(suffix); + PathBuf::from(name) +} + +fn compact_path(store: &Path) -> PathBuf { + with_suffix(store, COMPACT_SUFFIX) +} + +fn visited_path(store: &Path) -> PathBuf { + with_suffix(store, VISITED_SUFFIX) +} + +fn op_path(store: &Path) -> PathBuf { + with_suffix(store, OP_SUFFIX) +} + +/// The §9 SQLite artifact set: the fixed suffixes plus every multi-database +/// super-journal the staged file may have left (`.compact-mj*`). +fn artifact_paths(store: &Path) -> Vec { + let mut paths: Vec = ARTIFACT_SUFFIXES + .iter() + .map(|suffix| with_suffix(store, suffix)) + .collect(); + let name = store + .file_name() + .map_or_else(|| store.as_os_str().to_os_string(), |name| name.to_os_string()); + let mut prefix = name; + prefix.push(".compact-mj"); + let prefix = prefix.to_string_lossy().into_owned(); + if let Some(parent) = store.parent() { + if let Ok(entries) = std::fs::read_dir(parent) { + for entry in entries.flatten() { + if entry.file_name().to_string_lossy().starts_with(&prefix) { + paths.push(entry.path()); + } + } + } + } + paths +} + +fn io_error(error: std::io::Error) -> HistoryError { + match error.raw_os_error() { + Some(28) | Some(122) | Some(27) | Some(69) => HistoryError::Capacity("disk space"), + _ => HistoryError::Integrity("Store compaction I/O"), + } +} + +fn db_error(error: rusqlite::Error) -> HistoryError { + rows::error(error).into() +} + +fn seal_error(error: PersistenceError) -> HistoryError { + match error { + PersistenceError::Busy => HistoryError::Busy, + PersistenceError::Refused { status } if status == "Capacity" => { + HistoryError::Capacity("Store compaction") + } + PersistenceError::Refused { status } if status == "Store sidecar remains after seal" => { + HistoryError::Conflict("Store sidecar remains after seal") + } + PersistenceError::Refused { .. } | PersistenceError::Malformed => { + HistoryError::Integrity("Store seal") + } + PersistenceError::Uncertain => HistoryError::UnknownOutcome, + PersistenceError::Missing | PersistenceError::BackendUnavailable => { + HistoryError::Unsupported("SQLite Store compaction") + } + } +} + +/// The read-only URI of one database file that must not be changed while it is +/// read. Path bytes are percent-encoded so `?`, `#` and `%` do not misdirect +/// the URI (§7 step 6). +#[cfg(unix)] +fn immutable_uri(path: &Path) -> String { + use std::os::unix::ffi::OsStrExt; + let mut uri = String::from("file:"); + for byte in path.as_os_str().as_bytes() { + match byte { + b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' | b'/' => { + uri.push(char::from(*byte)); + } + _ => { + use std::fmt::Write as _; + let _ = write!(uri, "%{byte:02X}"); + } + } + } + uri.push_str("?immutable=1"); + uri +} + +fn open_immutable(path: &Path) -> HistoryResult { + #[cfg(unix)] + { + let connection = Connection::open_with_flags( + immutable_uri(path), + OpenFlags::SQLITE_OPEN_READ_ONLY + | OpenFlags::SQLITE_OPEN_URI + | OpenFlags::SQLITE_OPEN_NO_MUTEX, + ) + .map_err(db_error)?; + connection + .busy_timeout(std::time::Duration::ZERO) + .map_err(db_error)?; + Ok(connection) + } + #[cfg(not(unix))] + { + let _ = path; + Err(HistoryError::Unsupported("SQLite Store compaction")) + } +} + +fn live_tuple(session: &Arc) -> HistoryResult { + session + .run::<_, BackendError>(false, |tx| { + let rows = tx.query(TUPLE_SQL, vec![])?; + let row = rows.first().ok_or(BackendError::Integrity)?; + Ok(MarkTuple { + catalog_id: row.get(0)?, + generation: row.get(1)?, + next_history_serial: row.get(2)?, + next_pack_id: row.get(3)?, + next_ordinal: row.get(4)?, + logical: row.get(5)?, + canonical: row.get(6)?, + stored: row.get(7)?, + metadata: row.get(8)?, + stamp: row.get(9)?, + history_rows: row.get(10)?, + }) + }) + .map_err(Into::into) +} + +fn file_tuple(connection: &Connection) -> HistoryResult { + connection + .query_row(TUPLE_SQL, params![], |row| { + Ok(MarkTuple { + catalog_id: row.get(0)?, + generation: row.get(1)?, + next_history_serial: row.get(2)?, + next_pack_id: row.get(3)?, + next_ordinal: row.get(4)?, + logical: row.get(5)?, + canonical: row.get(6)?, + stored: row.get(7)?, + metadata: row.get(8)?, + stamp: row.get(9)?, + history_rows: row.get(10)?, + }) + }) + .map_err(db_error) +} + +fn pack_lengths(session: &Arc) -> HistoryResult> { + session + .run::<_, BackendError>(false, |tx| { + let rows = tx.query("SELECT pack_id,length FROM pack", vec![])?; + let mut lengths = BTreeMap::new(); + for row in &rows { + let id: i64 = row.get(0)?; + let length: i64 = row.get(1)?; + lengths.insert( + id, + u64::try_from(length).map_err(|_| BackendError::Integrity)?, + ); + } + Ok(lengths) + }) + .map_err(Into::into) +} + +fn object_packs(session: &Arc) -> HistoryResult> { + session + .run::<_, BackendError>(false, |tx| { + let rows = tx.query("SELECT object_id,pack_id FROM object_location", vec![])?; + let mut packs = BTreeMap::new(); + for row in &rows { + let bytes: Vec = row.get(0)?; + let pack: i64 = row.get(1)?; + let id = ObjectId::from_bytes(&bytes).map_err(|_| BackendError::Integrity)?; + packs.insert(id, pack); + } + Ok(packs) + }) + .map_err(Into::into) +} + +fn live_page_metrics(session: &Arc) -> HistoryResult<(u64, u64)> { + session + .run::<_, BackendError>(false, |tx| { + let free = tx.query("PRAGMA freelist_count", vec![])?; + let page = tx.query("PRAGMA page_size", vec![])?; + let free = free.first().ok_or(BackendError::Integrity)?.get::(0)?; + let page = page.first().ok_or(BackendError::Integrity)?.get::(0)?; + Ok(( + u64::try_from(free).map_err(|_| BackendError::Integrity)?, + u64::try_from(page).map_err(|_| BackendError::Integrity)?, + )) + }) + .map_err(Into::into) +} + +fn file_page_metrics(connection: &Connection) -> HistoryResult<(u64, u64)> { + let free: i64 = connection + .query_row("PRAGMA freelist_count", params![], |row| row.get(0)) + .map_err(db_error)?; + let page: i64 = connection + .query_row("PRAGMA page_size", params![], |row| row.get(0)) + .map_err(db_error)?; + Ok(( + u64::try_from(free).map_err(|_| HistoryError::Integrity("freelist count"))?, + u64::try_from(page).map_err(|_| HistoryError::Integrity("page size"))?, + )) +} + +fn file_heads_digest(connection: &Connection) -> HistoryResult<[u8; 32]> { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"layerfs/live-heads/v1\0"); + let mut stacks = connection.prepare(LIVE_STACKS).map_err(db_error)?; + let mut rows = stacks.query(params![]).map_err(db_error)?; + while let Some(row) = rows.next().map_err(db_error)? { + let id: Vec = row.get(0).map_err(db_error)?; + let head: Vec = row.get(1).map_err(db_error)?; + if id.len() != 17 || head.len() != 33 { + return Err(HistoryError::Integrity("live-heads stack width")); + } + hasher.update(&id); + hasher.update(&head); + } + drop(rows); + drop(stacks); + let mut branches = connection.prepare(LIVE_BRANCHES).map_err(db_error)?; + let mut rows = branches.query(params![]).map_err(db_error)?; + while let Some(row) = rows.next().map_err(db_error)? { + let id: Vec = row.get(0).map_err(db_error)?; + let head: Option> = row.get(1).map_err(db_error)?; + let base: Vec = row.get(2).map_err(db_error)?; + if id.len() != 17 || base.len() != 33 { + return Err(HistoryError::Integrity("live-heads branch width")); + } + hasher.update(&id); + match head { + Some(head) if head.len() == 33 => hasher.update(&head), + Some(_) => return Err(HistoryError::Integrity("live-heads commit width")), + None => hasher.update(&[0u8; 33]), + }; + hasher.update(&base); + } + Ok(*hasher.finalize().as_bytes()) +} + +/// Writes the §6.3 visited set and the retained pack set to the scratch +/// database the §7 copy joins against. The scratch runs `journal_mode=OFF`, so +/// it leaves no journal of its own. +fn write_visited( + scratch: &Path, + mark: &MarkResult, + packs: &BTreeMap, +) -> HistoryResult { + if std::fs::symlink_metadata(scratch).is_ok() { + return Err(HistoryError::Exists("staged compaction artifact")); + } + let connection = Connection::open_with_flags( + scratch, + OpenFlags::SQLITE_OPEN_READ_WRITE + | OpenFlags::SQLITE_OPEN_CREATE + | OpenFlags::SQLITE_OPEN_NO_MUTEX, + ) + .map_err(db_error)?; + connection + .busy_timeout(std::time::Duration::ZERO) + .map_err(db_error)?; + connection + .execute_batch( + "PRAGMA journal_mode=OFF; PRAGMA synchronous=OFF;\ +CREATE TABLE visited (\ + object_id BLOB PRIMARY KEY CHECK(length(object_id)=32),\ + pack_id INTEGER NOT NULL\ +) STRICT, WITHOUT ROWID;\ +CREATE TABLE live_pack (pack_id INTEGER PRIMARY KEY) STRICT, WITHOUT ROWID;", + ) + .map_err(db_error)?; + connection.execute_batch("BEGIN IMMEDIATE").map_err(db_error)?; + { + let mut visited = connection + .prepare("INSERT OR IGNORE INTO visited(object_id,pack_id) VALUES(?1,?2)") + .map_err(db_error)?; + let mut live = connection + .prepare("INSERT OR IGNORE INTO live_pack(pack_id) VALUES(?1)") + .map_err(db_error)?; + for pack in mark.objects().packs() { + live.execute(params![pack]).map_err(db_error)?; + } + for id in mark.objects().objects() { + let pack = packs + .get(id) + .ok_or(HistoryError::Integrity("mark object locator"))?; + visited + .execute(params![id.as_bytes().as_slice(), pack]) + .map_err(db_error)?; + } + } + connection.execute_batch("COMMIT").map_err(db_error)?; + connection.close().map_err(|(_, error)| db_error(error))?; + for suffix in ["-journal", "-wal", "-shm"] { + let mut sidecar = scratch.as_os_str().to_os_string(); + sidecar.push(suffix); + if std::fs::symlink_metadata(sidecar).is_ok() { + return Err(HistoryError::Integrity("visited scratch sidecar")); + } + } + Ok(std::fs::metadata(scratch).map_err(io_error)?.len()) +} + +/// What the staged rebuild produced, for the report and the install fence. +struct Built { + target: PathBuf, + target_bytes: u64, + visited_bytes: u64, + logical: u64, + canonical: u64, + stored: u64, + metadata: u64, + free_bytes: u64, +} + +impl Built { + fn peak_temp_bytes(&self) -> u64 { + self.target_bytes.saturating_add(self.visited_bytes) + } +} + +/// Escaped SQL text literal for an `ATTACH` path (the statement cannot bind). +fn quote(text: &str) -> String { + text.replace('\'', "''") +} + +fn table_present(connection: &Connection, name: &str) -> HistoryResult { + let found: i64 = connection + .query_row( + "SELECT count(*) FROM sqlite_schema WHERE type='table' AND name=?1", + params![name], + |row| row.get(0), + ) + .map_err(db_error)?; + Ok(found == 1) +} + +/// The §7 steps 2–5: build the staged Store, copy only retained rows, compact +/// it, and switch it to WAL. +fn build( + facts: &SealedFacts, + mark: &MarkResult, + tuple: &MarkTuple, + packs: &BTreeMap, + layout: crate::SqlitePackLayout, + synchronous: i64, +) -> HistoryResult { + let store = &facts.path; + let target = compact_path(store); + let visited = visited_path(store); + for staged in [&target, &visited] { + if std::fs::symlink_metadata(staged).is_ok() { + return Err(HistoryError::Exists("staged compaction artifact")); + } + } + let visited_bytes = write_visited(&visited, mark, packs)?; + + let source = open_immutable(store)?; + let target_text = target + .to_str() + .ok_or(HistoryError::InvalidInput("compaction path"))?; + source + .execute("VACUUM INTO ?1", params![target_text]) + .map_err(db_error)?; + source + .close() + .map_err(|(_, error)| db_error(error))?; + + let connection = Connection::open_with_flags( + &target, + OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_NO_MUTEX, + ) + .map_err(db_error)?; + connection + .busy_timeout(std::time::Duration::ZERO) + .map_err(db_error)?; + connection + .execute_batch(&format!( + "PRAGMA journal_mode=DELETE; PRAGMA synchronous={synchronous};" + )) + .map_err(db_error)?; + // The copied published-pack rows may be deleted in the artifact; the live + // immutability triggers are restored before the file is verified. + // `sqlite_sequence` is the acquisition operation counter: the copy keeps + // every row the source had, and the sequence never falls (§4.1, §7). + let sequences: Vec<(String, i64)> = if table_present(&connection, "sqlite_sequence")? { + let mut statement = connection + .prepare("SELECT name,seq FROM sqlite_sequence") + .map_err(db_error)?; + let mut rows = statement.query(params![]).map_err(db_error)?; + let mut found = Vec::new(); + while let Some(row) = rows.next().map_err(db_error)? { + found.push(( + row.get::<_, String>(0).map_err(db_error)?, + row.get::<_, i64>(1).map_err(db_error)?, + )); + } + found + } else { + Vec::new() + }; + let triggers: Vec = { + let mut statement = connection + .prepare( + "SELECT sql FROM sqlite_schema \ +WHERE type='trigger' AND name LIKE '%immutable%' AND sql IS NOT NULL", + ) + .map_err(db_error)?; + let mut rows = statement.query(params![]).map_err(db_error)?; + let mut found = Vec::new(); + while let Some(row) = rows.next().map_err(db_error)? { + found.push(row.get::<_, String>(0).map_err(db_error)?); + } + found + }; + connection + .execute_batch( + "DROP TRIGGER IF EXISTS pack_immutable_update;\ +DROP TRIGGER IF EXISTS pack_immutable_delete;\ +DROP TRIGGER IF EXISTS pack_unit_immutable_update;\ +DROP TRIGGER IF EXISTS pack_unit_immutable_delete;\ +PRAGMA foreign_keys=OFF;\ +PRAGMA defer_foreign_keys=ON;", + ) + .map_err(db_error)?; + connection + .execute_batch(&format!( + "ATTACH DATABASE '{}' AS scratch;", + quote(visited.to_str().ok_or(HistoryError::InvalidInput("compaction path"))?) + )) + .map_err(db_error)?; + + let units = layout.uses_units(); + let acquisition_tables = table_present(&connection, "init_operation")?; + connection + .execute_batch("BEGIN IMMEDIATE") + .map_err(db_error)?; + let staged = (|| -> HistoryResult<()> { + // §7 step 4: the totals trigger invariant holds before the copy. + let before: (i64, i64) = connection + .query_row( + "SELECT (SELECT logical FROM store_totals WHERE id=1),\ +(SELECT COALESCE(SUM(logical),0) FROM \"commit\")+\ +(SELECT COALESCE(SUM(logical),0) FROM layer)", + params![], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .map_err(db_error)?; + if before.0 != before.1 { + return Err(HistoryError::Integrity( + "Store totals disagree with the history rows", + )); + } + let mut deletes = String::from( + "DELETE FROM object_location WHERE object_id NOT IN \ +(SELECT object_id FROM scratch.visited);\ +DELETE FROM content_signature WHERE object_id NOT IN \ +(SELECT object_id FROM scratch.visited);\ +DELETE FROM file_digest WHERE root NOT IN (SELECT object_id FROM scratch.visited);\ +DELETE FROM binding_hint_block;\ +DELETE FROM \"commit\" WHERE commit_id NOT IN \ +(SELECT node_id FROM kept_node WHERE kind='commit');\ +DELETE FROM layer WHERE layer_id NOT IN \ +(SELECT node_id FROM kept_node WHERE kind='layer');\ +DELETE FROM branch WHERE branch_id NOT IN \ +(SELECT node_id FROM kept_node WHERE kind='branch');\ +DELETE FROM layer_stack WHERE layer_stack_id NOT IN \ +(SELECT node_id FROM kept_node WHERE kind='stack');\ +DELETE FROM layer_lineage WHERE layer_id NOT IN \ +(SELECT node_id FROM kept_node WHERE kind='layer');\ +DELETE FROM commit_lineage WHERE commit_id NOT IN \ +(SELECT node_id FROM kept_node WHERE kind='commit');\ +DELETE FROM absorbed WHERE run NOT IN (SELECT run FROM commit_lineage);\ +DELETE FROM reach WHERE key NOT IN \ +(SELECT layer_stack_id FROM layer_stack UNION SELECT run FROM commit_lineage);\ +DELETE FROM branch_retired WHERE branch_id NOT IN \ +(SELECT node_id FROM kept_node WHERE kind='branch');\ +DELETE FROM layer_stack_retired WHERE layer_stack_id NOT IN \ +(SELECT layer_stack_id FROM layer_stack);\ +DELETE FROM path_removed WHERE layer_stack_id NOT IN \ +(SELECT layer_stack_id FROM layer_stack);\ +DELETE FROM project_epoch WHERE layer_stack_id NOT IN \ +(SELECT layer_stack_id FROM layer_stack);\ +DELETE FROM merge_record;\ +DELETE FROM merge_operation;\ +DELETE FROM gc_intent;\ +DELETE FROM gc_dead;\ +DELETE FROM gc_op;", + ); + if units { + deletes.push_str( + "DELETE FROM pack_unit WHERE pack_id NOT IN \ +(SELECT pack_id FROM scratch.live_pack);", + ); + } + deletes.push_str( + "DELETE FROM pack WHERE pack_id NOT IN (SELECT pack_id FROM scratch.live_pack);", + ); + if acquisition_tables { + for table in ["init_operation", "init_entry", "init_host_file"] { + deletes.push_str(&format!("DELETE FROM {table};")); + } + } + for trigger in &triggers { + deletes.push_str(trigger.trim_end_matches(';')); + deletes.push_str(";\n"); + } + deletes.push_str(&format!( + "UPDATE history_meta SET catalog_incarnation={} WHERE id=1;\ +UPDATE store_totals SET \ +logical=(SELECT COALESCE(SUM(logical),0) FROM \"commit\")+\ +(SELECT COALESCE(SUM(logical),0) FROM layer),\ +canonical=(SELECT COALESCE(SUM(canonical_length),0) FROM object_location),\ +stored=(SELECT COALESCE(SUM(length),0) FROM pack),\ +metadata=(SELECT COALESCE(SUM(length),0) FROM pack WHERE domain=0) WHERE id=1;", + tuple.generation + 1 + )); + connection.execute_batch(&deletes).map_err(db_error)?; + Ok(()) + })(); + match staged { + Ok(()) => { + connection.execute_batch("COMMIT").map_err(db_error)?; + } + Err(error) => { + let _ = connection.execute_batch("ROLLBACK"); + let _ = connection.execute_batch("DETACH DATABASE scratch"); + let _ = connection.close(); + return Err(error); + } + } + connection + .execute_batch("DETACH DATABASE scratch") + .map_err(db_error)?; + // Reclaim the deleted pages: a wholly live Store, no freelist. + connection.execute_batch("VACUUM").map_err(db_error)?; + if !sequences.is_empty() && table_present(&connection, "sqlite_sequence")? { + connection + .execute_batch("DELETE FROM sqlite_sequence") + .map_err(db_error)?; + let mut statement = connection + .prepare("INSERT INTO sqlite_sequence(name,seq) VALUES(?1,?2)") + .map_err(db_error)?; + for (name, seq) in &sequences { + statement.execute(params![name, seq]).map_err(db_error)?; + } + } + let totals: (i64, i64, i64, i64) = connection + .query_row( + "SELECT logical,canonical,stored,metadata FROM store_totals WHERE id=1", + params![], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .map_err(db_error)?; + connection + .close() + .map_err(|(_, error)| db_error(error))?; + + // §7 step 5: switch to WAL, checkpoint, and leave no sidecar. + let connection = Connection::open_with_flags( + &target, + OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_NO_MUTEX, + ) + .map_err(db_error)?; + connection + .busy_timeout(std::time::Duration::ZERO) + .map_err(db_error)?; + let mode: String = connection + .query_row("PRAGMA journal_mode=wal", params![], |row| row.get(0)) + .map_err(db_error)?; + if mode != "wal" { + return Err(HistoryError::Integrity("staged Store is not in WAL mode")); + } + connection + .execute_batch(&format!("PRAGMA synchronous={synchronous}")) + .map_err(db_error)?; + let busy: i64 = connection + .query_row("PRAGMA wal_checkpoint(TRUNCATE)", params![], |row| { + row.get(0) + }) + .map_err(db_error)?; + if busy != 0 { + return Err(HistoryError::Busy); + } + connection + .close() + .map_err(|(_, error)| db_error(error))?; + for suffix in ["-wal", "-shm", "-journal"] { + let mut sidecar = target.as_os_str().to_os_string(); + sidecar.push(suffix); + if std::fs::symlink_metadata(sidecar).is_ok() { + return Err(HistoryError::Integrity("staged Store sidecar")); + } + } + let target_bytes = std::fs::metadata(&target).map_err(io_error)?.len(); + let free_bytes = file_free_bytes(&target)?; + Ok(Built { + target, + target_bytes, + visited_bytes, + logical: u64::try_from(totals.0).unwrap_or(0), + canonical: u64::try_from(totals.1).unwrap_or(0), + stored: u64::try_from(totals.2).unwrap_or(0), + metadata: u64::try_from(totals.3).unwrap_or(0), + free_bytes, + }) +} + +/// The §7 step 6 verification through an `immutable=1` reader: schema identity, +/// foreign keys, row counts against the mark, both live-heads digests, the +/// allocator tuple and the recomputed totals. `quick_check` and the full doctor +/// stay separate commands. +fn verify( + built: &Built, + mark: &MarkResult, + tuple: &MarkTuple, + layout: crate::SqlitePackLayout, + acquisition: crate::SqliteAcquisitionSchema, +) -> HistoryResult<()> { + let connection = open_immutable(&built.target)?; + schema::check( + &connection, + &RefCell::new(SqlWork::default()), + (layout, acquisition), + ) + .map_err(HistoryError::from)?; + let count = |sql: &str| -> HistoryResult { + let value: i64 = connection + .query_row(sql, params![], |row| row.get(0)) + .map_err(db_error)?; + u64::try_from(value).map_err(|_| HistoryError::Integrity("verification count")) + }; + if count("SELECT count(*) FROM pragma_foreign_key_check")? != 0 { + return Err(HistoryError::Integrity("staged Store foreign key")); + } + if count("SELECT count(*) FROM object_location")? != mark.objects().len() as u64 { + return Err(HistoryError::Integrity("staged Store object count")); + } + if count("SELECT count(*) FROM pack")? != mark.objects().packs().len() as u64 { + return Err(HistoryError::Integrity("staged Store pack count")); + } + if count("SELECT count(*) FROM kept_node")? != mark.history().len() as u64 { + return Err(HistoryError::Integrity("staged Store history count")); + } + if file_heads_digest(&connection)? != mark.live_heads_digest() { + return Err(HistoryError::Conflict( + "staged Store live heads differ from the mark", + )); + } + let checked = file_tuple(&connection)?; + if checked.catalog_id != tuple.catalog_id + || checked.generation != tuple.generation + 1 + || checked.next_history_serial != tuple.next_history_serial + || checked.next_pack_id != tuple.next_pack_id + || checked.next_ordinal != tuple.next_ordinal + || checked.stamp > tuple.stamp + { + return Err(HistoryError::Integrity("staged Store allocator tuple")); + } + let totals = connection + .query_row( + "SELECT \ +(SELECT logical FROM store_totals WHERE id=1),\ +(SELECT COALESCE(SUM(logical),0) FROM \"commit\")+\ +(SELECT COALESCE(SUM(logical),0) FROM layer),\ +(SELECT canonical FROM store_totals WHERE id=1),\ +(SELECT COALESCE(SUM(canonical_length),0) FROM object_location),\ +(SELECT stored FROM store_totals WHERE id=1),\ +(SELECT COALESCE(SUM(length),0) FROM pack),\ +(SELECT metadata FROM store_totals WHERE id=1),\ +(SELECT COALESCE(SUM(length),0) FROM pack WHERE domain=0)", + params![], + |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, i64>(4)?, + row.get::<_, i64>(5)?, + row.get::<_, i64>(6)?, + row.get::<_, i64>(7)?, + )) + }, + ) + .map_err(db_error)?; + if totals.0 != totals.1 + || totals.2 != totals.3 + || totals.4 != totals.5 + || totals.6 != totals.7 + { + return Err(HistoryError::Integrity("staged Store totals")); + } + let built_logical = u64::try_from(checked.logical).unwrap_or(u64::MAX); + let built_canonical = u64::try_from(checked.canonical).unwrap_or(u64::MAX); + let built_stored = u64::try_from(checked.stored).unwrap_or(u64::MAX); + let built_metadata = u64::try_from(checked.metadata).unwrap_or(u64::MAX); + if built_logical != built.logical + || built_canonical != built.canonical + || built_stored != built.stored + || built_metadata != built.metadata + { + return Err(HistoryError::Integrity("staged Store build totals")); + } + connection + .close() + .map_err(|(_, error)| db_error(error))?; + Ok(()) +} + +fn file_free_bytes(target: &Path) -> HistoryResult { + let measure = open_immutable(target)?; + let (free, page) = file_page_metrics(&measure)?; + measure + .close() + .map_err(|(_, error)| db_error(error))?; + Ok(free.saturating_mul(page)) +} + +fn source_state(store: &Path) -> HistoryResult<(u64, SystemTime, u32)> { + let metadata = std::fs::metadata(store).map_err(io_error)?; + let modified = metadata.modified().map_err(io_error)?; + let mut file = std::fs::File::open(store).map_err(io_error)?; + use std::io::{Read as _, Seek as _, SeekFrom}; + let mut header = [0u8; 4]; + file.seek(SeekFrom::Start(24)).map_err(io_error)?; + file.read_exact(&mut header).map_err(io_error)?; + Ok((metadata.len(), modified, u32::from_be_bytes(header))) +} + +/// True when one live or staged file carries a `-wal`, `-shm`, `-journal` or +/// multi-database super-journal (§7 step 7). +fn sidecar_present(path: &Path) -> Option { + for suffix in ["-wal", "-shm", "-journal"] { + let mut sidecar = path.as_os_str().to_os_string(); + sidecar.push(suffix); + let sidecar = PathBuf::from(sidecar); + if std::fs::symlink_metadata(&sidecar).is_ok() { + return Some(sidecar); + } + } + let name = path + .file_name() + .map_or_else(|| path.as_os_str().to_os_string(), |name| name.to_os_string()); + let mut prefix = name; + prefix.push("-mj"); + let prefix = prefix.to_string_lossy().into_owned(); + if let Some(parent) = path.parent() { + if let Ok(entries) = std::fs::read_dir(parent) { + for entry in entries.flatten() { + if entry.file_name().to_string_lossy().starts_with(&prefix) { + return Some(entry.path()); + } + } + } + } + None +} + +/// §7 step 7's fence before the rename: no sidecar, the same `G`, the same +/// catalog identity and the sealed `(size, mtime, change counter)`. +fn install_fence(facts: &SealedFacts, tuple: &MarkTuple) -> HistoryResult<()> { + for path in [&facts.path, &compact_path(&facts.path)] { + if sidecar_present(path).is_some() { + return Err(HistoryError::Conflict("Store sidecar at install")); + } + } + let connection = open_immutable(&facts.path)?; + let current = file_tuple(&connection)?; + connection + .close() + .map_err(|(_, error)| db_error(error))?; + if ¤t != tuple { + return Err(HistoryError::Conflict("the sealed Store changed")); + } + let (bytes, modified, change_counter) = source_state(&facts.path)?; + if bytes != facts.bytes || modified != facts.modified || change_counter != facts.change_counter + { + return Err(HistoryError::Conflict("the sealed Store changed")); + } + Ok(()) +} + +/// §7 step 7: `chmod 0600` and `sync_all` before the rename. +fn prepare_target(target: &Path) -> HistoryResult<()> { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(target, std::fs::Permissions::from_mode(0o600)).map_err(io_error)?; + } + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(target) + .map_err(io_error)?; + file.sync_all().map_err(io_error)?; + Ok(()) +} + +fn sync_directory(store: &Path) -> HistoryResult<()> { + if let Some(parent) = store.parent() { + let directory = std::fs::File::open(parent).map_err(io_error)?; + directory.sync_all().map_err(io_error)?; + } + Ok(()) +} + +/// Removes the fixed artifact set and reports what remains, measuring the +/// bytes before deletion. +fn clear_artifacts(store: &Path) -> CompactResidue { + let mut bytes = 0u64; + let mut left = Vec::new(); + for path in artifact_paths(store) { + let Ok(metadata) = std::fs::symlink_metadata(&path) else { + continue; + }; + bytes = bytes.saturating_add(metadata.len()); + if std::fs::remove_file(&path).is_err() && std::fs::symlink_metadata(&path).is_ok() { + left.push( + path.file_name() + .map_or_else(|| path.display().to_string(), |name| { + name.to_string_lossy().into_owned() + }), + ); + } + } + CompactResidue { + bytes, + artifacts: left, + reason: None, + } +} + +/// Settles one run: the artifact set is removed first, the op record last +/// (§9: the op is settled only after removal). +fn settle(store: &Path) -> CompactResidue { + let mut residue = clear_artifacts(store); + let op = op_path(store); + if std::fs::symlink_metadata(&op).is_ok() { + let bytes = std::fs::metadata(&op).map_or(0, |metadata| metadata.len()); + if std::fs::remove_file(&op).is_err() && std::fs::symlink_metadata(&op).is_ok() { + residue.bytes = residue.bytes.saturating_add(bytes); + residue.artifacts.push( + op.file_name() + .map_or_else(|| op.display().to_string(), |name| { + name.to_string_lossy().into_owned() + }), + ); + } + } + if !residue.artifacts.is_empty() && residue.reason.is_none() { + residue.reason = Some("the artifact set could not be removed".to_owned()); + } + residue +} + +/// Drops every artifact and the op record after a definite failure before the +/// rename; the live Store is unchanged apart from the seal's checkpoint. +fn abort_before_install(store: &Path) { + let residue = clear_artifacts(store); + if residue.artifacts.is_empty() { + let _ = std::fs::remove_file(op_path(store)); + } +} + +/// Judges and removes a leftover an earlier run recorded (§9): the artifact set +/// it names and the admission intent it took. The caller already holds the +/// writer lane, so a leftover owner is gone. +fn settle_leftover( + history: &crate::HistoryProvider, + store: &Path, + _generation: u64, +) -> HistoryResult<()> { + let Some(op) = RunOp::read(store) else { + if artifact_paths(store) + .iter() + .any(|path| std::fs::symlink_metadata(path).is_ok()) + { + return Err(HistoryError::Exists("unrecorded compaction artifact")); + } + return Ok(()); + }; + // Adopt the recorded owner's admission intent before its artifact is gone. + history.gc_release(&op.op_id)?; + let residue = clear_artifacts(store); + if !residue.artifacts.is_empty() { + return Err(HistoryError::Integrity( + "recorded compaction artifact could not be removed", + )); + } + if std::fs::symlink_metadata(op_path(store)).is_ok() { + std::fs::remove_file(op_path(store)).map_err(io_error)?; + } + Ok(()) +} + +/// A writer held across the seal, install and settle, released on every exit. +struct WriterGuard(Option); + +impl WriterGuard { + fn new(writer: super::lane::writer_admission::Writer) -> Self { + Self(Some(writer)) + } + + fn take(&mut self) -> super::lane::writer_admission::Writer { + self.0.take().expect("a compaction holds its writer") + } + + fn put(&mut self, writer: super::lane::writer_admission::Writer) { + self.0 = Some(writer); + } + + fn release(&mut self) -> HistoryResult<()> { + match self.0.take() { + Some(writer) => writer.release().map_err(|_| HistoryError::UnknownOutcome), + None => Ok(()), + } + } +} + +impl Drop for WriterGuard { + fn drop(&mut self) { + if let Some(writer) = self.0.take() { + let _ = writer.release(); + } + } +} + +/// A test-only crash stop: records the phase and leaves the state a crash +/// would leave. +fn stop_at( + request: &CompactRequest, + point: CompactFailpoint, + op: &RunOp, + store: &Path, +) -> HistoryResult<()> { + if request.failpoint != Some(point) { + return Ok(()); + } + let mut stopped = op.clone(); + stopped.phase = point.phase().to_owned(); + stopped.write(store)?; + Err(HistoryError::UnknownOutcome) +} + +fn immutable_tuple(path: &Path) -> HistoryResult { + let connection = open_immutable(path)?; + let tuple = file_tuple(&connection)?; + connection + .close() + .map_err(|(_, error)| db_error(error))?; + Ok(tuple) +} + +fn immutable_digest(path: &Path) -> HistoryResult<[u8; 32]> { + let connection = open_immutable(path)?; + let digest = file_heads_digest(&connection)?; + connection + .close() + .map_err(|(_, error)| db_error(error))?; + Ok(digest) +} + +/// §11: a zero field always carries one word of the closed vocabulary. +#[allow(clippy::too_many_arguments)] +fn receipt( + path: CompactPath, + generation_before: u64, + generation_after: u64, + dead_bytes: u64, + logical_discarded: u64, + backend_reusable: u64, + physical_shrink: u64, + kept_waste: CompactKeptWaste, + peak_temp_bytes: u64, + residue: CompactResidue, + total_stored: u64, + history_rows: u64, +) -> CompactReceipt { + let mut zero_reasons = Vec::new(); + if dead_bytes == 0 { + let reason = if total_stored == 0 { + "empty-store" + } else { + "no-waste" + }; + zero_reasons.push(CompactZeroReason { + field: "deadBytesBound", + reason, + }); + zero_reasons.push(CompactZeroReason { + field: "deadBytesRefined", + reason, + }); + } + if logical_discarded == 0 { + let reason = if history_rows == 0 { + "no-history" + } else if path == CompactPath::None { + "none-path" + } else { + "no-waste" + }; + zero_reasons.push(CompactZeroReason { + field: "logicalDiscarded", + reason, + }); + } + if physical_shrink == 0 { + zero_reasons.push(CompactZeroReason { + field: "physicalShrink", + reason: if path == CompactPath::None { + "none-path" + } else { + "no-waste" + }, + }); + } + if backend_reusable == 0 { + zero_reasons.push(CompactZeroReason { + field: "backendReusable", + reason: if path == CompactPath::None { + "none-path" + } else { + "no-waste" + }, + }); + } + if kept_waste.is_empty() { + zero_reasons.push(CompactZeroReason { + field: "keptWaste", + reason: "no-waste", + }); + } + if peak_temp_bytes == 0 { + zero_reasons.push(CompactZeroReason { + field: "peakTempBytes", + reason: "none-path", + }); + } + CompactReceipt { + path, + generation_before, + generation_after, + dead_bytes_bound: dead_bytes, + dead_bytes_refined: dead_bytes, + logical_discarded, + backend_reusable, + physical_shrink, + kept_waste, + peak_temp_bytes, + residue, + estimate: true, + zero_reasons, + } +} + +impl SqliteHandles { + /// Runs one §7 SQLite Store compaction on this Store's own machine. + /// + /// The call consumes the handles like [`Self::seal`]: it checkpoints and + /// closes the sole session, and a rebuild installs a new file at `G + 1`. + /// It takes the Store's exclusive writer lane before its admission intent, + /// so no admitted writer can interleave between the mark, the seal and the + /// rename. The caller reopens the Store afterwards. + /// + /// A definite failure before the rename removes the artifact set and leaves + /// the old Store unchanged apart from the seal's checkpoint. A rename or + /// directory-fsync failure is installed-but-uncertain: the new file and the + /// op record are kept for the next `store.compact` to judge (§9). + pub fn compact(self, request: &CompactRequest) -> HistoryResult { + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + let _ = request; + return Err(HistoryError::Unsupported("SQLite Store compaction")); + } + #[cfg(any(target_os = "linux", target_os = "macos"))] + { + let SqliteHandles { + storage, + history, + acquisition, + } = self; + drop(acquisition); + let session = storage.session.clone(); + let store = session.path.clone(); + let profile = session.profile.clone(); + let layout = session.layout; + let acquisition_schema = session.acquisition; + let synchronous = profile.synchronous; + + // §7 step 1: the exclusive lane first, so the generation this run + // records cannot move between the mark, the seal and the rename. + let lane = session.hold_lane().map_err(HistoryError::from)?; + // §7 step 1: the sealed source's WAL is emptied before the closure. + // The lane is already held, so no writer can append meanwhile. + session.checkpoint_holding().map_err(HistoryError::from)?; + let live = live_tuple(&session)?; + let generation = u64::try_from(live.generation) + .map_err(|_| HistoryError::Integrity("Store generation"))?; + settle_leftover(&history, &store, generation)?; + + let op_id = request.op_id; + let mut op = RunOp { + op_id, + generation: live.generation, + phase: "admission".to_owned(), + started_ms: now_ms(), + tuple: live.clone(), + live_heads_digest: [0u8; 32], + }; + op.write(&store)?; + let intent = GcIntent::operation( + op_id, + GcIntentKind::Compaction, + GcScopeKind::Store, + history.catalog_id.as_slice().to_vec(), + ) + .owned( + request.owner_runtime.as_deref(), + request.owner_boot.as_deref(), + ) + .with_phase(Some("mark")); + history.gc_register(&intent)?; + let mark = history.mark()?; + let tuple = live_tuple(&session)?; + op.tuple = tuple.clone(); + op.live_heads_digest = mark.live_heads_digest(); + op.phase = "mark".to_owned(); + op.write(&store)?; + + let lengths = pack_lengths(&session)?; + let total = lengths.values().copied().fold(0u64, u64::saturating_add); + let mut live_stored = 0u64; + for pack in mark.objects().packs() { + let length = lengths + .get(pack) + .ok_or(HistoryError::Integrity("mark pack"))?; + live_stored = live_stored.saturating_add(*length); + } + let dead = total.saturating_sub(live_stored); + let rebuild = total > 0 && dead.saturating_mul(10) >= total; + let kept_waste = CompactKeptWaste::from_mark(mark.objects().waste()); + let history_rows = u64::try_from(tuple.history_rows).unwrap_or(0); + + if !rebuild { + // §3 `none`: mark only. The intent is cleared and the op record + // removed; nothing is rebuilt and the generation is unchanged. + let (free, page) = live_page_metrics(&session)?; + history.gc_release(&op_id)?; + let _ = std::fs::remove_file(op_path(&store)); + drop(lane); + return Ok(receipt( + CompactPath::None, + generation, + generation, + dead, + 0, + free.saturating_mul(page), + 0, + kept_waste, + 0, + CompactResidue::default(), + total, + history_rows, + )); + } + + // §7 step 2 onward: seal and rebuild. Only the session remains. + let object_packs = object_packs(&session)?; + drop(history); + drop(storage); + let mut guard = WriterGuard::new(lane.into_writer()); + let session = Arc::try_unwrap(session).map_err(|_| HistoryError::Busy)?; + let writer = guard.take(); + let (facts, writer) = match session.seal_holding(writer) { + Ok(sealed) => sealed, + Err(error) => return Err(seal_error(error)), + }; + guard.put(writer); + + // §7 step 1.5's fence: the seal must not have moved the generation + // or the heads the mark recorded. + let sealed_tuple = match immutable_tuple(&facts.path) { + Ok(tuple) => tuple, + Err(error) => { + abort_before_install(&store); + return Err(error); + } + }; + if sealed_tuple != tuple { + abort_before_install(&store); + return Err(HistoryError::Conflict("the Store changed after the mark")); + } + match immutable_digest(&facts.path) { + Ok(digest) if digest == mark.live_heads_digest() => {} + Ok(_) => { + abort_before_install(&store); + return Err(HistoryError::Conflict( + "the Store heads changed after the mark", + )); + } + Err(error) => { + abort_before_install(&store); + return Err(error); + } + } + stop_at(request, CompactFailpoint::AfterSeal, &op, &store)?; + + let built = match build( + &facts, + &mark, + &tuple, + &object_packs, + layout, + synchronous, + ) { + Ok(built) => built, + Err(error) => { + abort_before_install(&store); + return Err(error); + } + }; + stop_at(request, CompactFailpoint::AfterBuild, &op, &store)?; + stop_at(request, CompactFailpoint::AfterWalSwitch, &op, &store)?; + + if let Err(error) = verify(&built, &mark, &tuple, layout, acquisition_schema) { + abort_before_install(&store); + return Err(error); + } + stop_at(request, CompactFailpoint::AfterVerify, &op, &store)?; + + if let Err(error) = install_fence(&facts, &tuple) { + abort_before_install(&store); + return Err(error); + } + if let Err(error) = prepare_target(&built.target) { + abort_before_install(&store); + return Err(error); + } + stop_at(request, CompactFailpoint::BeforeInstall, &op, &store)?; + + if std::fs::rename(&built.target, &facts.path).is_err() { + // Installed-but-uncertain: never delete the new Store as if + // nothing happened (§7 step 8, §9). + op.phase = "install".to_owned(); + let _ = op.write(&store); + return Err(HistoryError::UnknownOutcome); + } + if let Err(_error) = sync_directory(&facts.path) { + op.phase = "install".to_owned(); + let _ = op.write(&store); + return Err(HistoryError::UnknownOutcome); + } + stop_at(request, CompactFailpoint::AfterInstall, &op, &store)?; + guard.release()?; + let residue = settle(&store); + let logical_discarded = u64::try_from(tuple.logical) + .unwrap_or(0) + .saturating_sub(built.logical); + let physical_shrink = facts.bytes.saturating_sub(built.target_bytes); + Ok(receipt( + CompactPath::Rebuild, + generation, + generation.saturating_add(1), + dead, + logical_discarded, + built.free_bytes, + physical_shrink, + kept_waste, + built.peak_temp_bytes(), + residue, + total, + history_rows, + )) + } + } +} diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/lane/lane.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/lane/lane.rs index 1c31440b0b..3d58695311 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/lane/lane.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/lane/lane.rs @@ -38,6 +38,17 @@ impl Session { } } +impl LaneHold { + /// Takes the cross-process writer lock out of the hold and releases the + /// connection turn, so a seal can consume the session while the lane is + /// still held (§7 step 1). The caller becomes the lane's owner. + pub(crate) fn into_writer(mut self) -> Writer { + self.writer + .take() + .expect("a held Store lane owns its writer") + } +} + impl Drop for LaneHold { fn drop(&mut self) { if let Some(writer) = self.writer.take() { diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/seal.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/seal.rs index f9a02f37c0..178f404237 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/seal.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/seal.rs @@ -1,12 +1,92 @@ //! Sole-owner host seal; never available through the daemon's Store ports. +use crate::backend::sqlite::lane::writer_admission::Writer; use crate::backend::sqlite::session::connection::Session; use crate::backend::sqlite::session::query; use crate::backend::sqlite::session::rows; use crate::{backend::records::BackendError, SealedStore}; use layerfs_storage::port::PersistenceError; +use std::io::{Read, Seek, SeekFrom}; +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +/// The identity of one closed Store file at the moment its last connection +/// closed, captured under the writer lane (§7 step 1). +#[derive(Clone, Debug)] +pub(crate) struct SealedFacts { + /// The closed database file; no sidecar was present when it was captured. + pub(crate) path: PathBuf, + /// Provisioning profile the file was closed under. + pub(crate) profile: crate::SqlitePersistenceProfile, + /// SQLite version that created/sealed the file. + pub(crate) sqlite_version: String, + /// Main-file bytes after the WAL was checkpointed and the connection closed. + pub(crate) bytes: u64, + /// Main-file modification time at the same moment. + pub(crate) modified: SystemTime, + /// The database header change counter (big-endian bytes 24..28). + pub(crate) change_counter: u32, +} + +impl SealedFacts { + /// The public sealed-Store handoff this identity describes. + pub(crate) fn into_store(self) -> SealedStore { + SealedStore { + path: self.path, + profile: self.profile, + sqlite_version: self.sqlite_version, + bytes: self.bytes, + } + } +} + +fn read_change_counter(path: &Path) -> Result { + let refused = |error: std::io::Error| PersistenceError::Refused { + status: error.to_string(), + }; + let mut file = std::fs::File::open(path).map_err(refused)?; + let mut bytes = [0u8; 4]; + file.seek(SeekFrom::Start(24)).map_err(refused)?; + file.read_exact(&mut bytes).map_err(refused)?; + Ok(u32::from_be_bytes(bytes)) +} impl Session { - pub(crate) fn seal(mut self) -> Result { + /// Consumes the sole session, checkpoints once, closes and verifies one file. + /// macOS then releases unused allocation beyond the original file's logical + /// end, without copying its contents or changing its identity or length. + /// + /// A retained provider/Storage/Reader/Save refuses with `Busy` before any + /// checkpoint. Another process can obstruct checkpoint or keep sidecars; + /// neither case triggers a retry or deletion. Provisioning must prevent + /// new openers throughout sealing and handoff. This is not live Store + /// reclamation. Uncertain transfer/temporary-close outcomes retain the owned + /// `.layerfs-allocation-*` sibling; no automatic retry or deletion follows. + pub(crate) fn seal(self) -> Result { + let writer = { + let mut state = self + .state + .lock() + .map_err(|_| PersistenceError::Uncertain)?; + state.writer_hold.take() + }; + let writer = match writer { + Some(writer) => writer, + None => Writer::acquire(&self.path)?, + }; + let (facts, writer) = self.seal_holding(writer)?; + writer.release().map_err(|_| PersistenceError::Uncertain)?; + Ok(facts.into_store()) + } + + /// [`Self::seal`] under a caller-held writer lane (§7 step 1: the guard is + /// passed into the seal, never acquired a second time). The lane is returned + /// so a compaction can keep it through the rename. The checkpoint, checked + /// close, sidecar check and the sealed `(size, mtime, change counter)` + /// capture all run while the lane is held. + pub(crate) fn seal_holding( + mut self, + writer: Writer, + ) -> Result<(SealedFacts, Writer), PersistenceError> { if !self.writable { return Err(BackendError::ReadOnly.into()); } @@ -19,13 +99,6 @@ impl Session { if state.quarantined || !state.connection.is_autocommit() { return Err(PersistenceError::Uncertain); } - // The same advisory lane as a write transaction: a seal is the sole - // owner's offline handoff, so no other opener may write while the - // checkpoint, close and sidecar removal below run. - if state.writer_hold.is_none() { - state.writer_hold = - Some(crate::backend::sqlite::lane::writer_admission::Writer::acquire(&self.path)?); - } #[cfg(target_os = "macos")] let identity = super::seal_allocation::Identity::capture(&self.path)?; #[cfg(target_os = "macos")] @@ -68,22 +141,34 @@ impl Session { } } } - if let Some(writer) = state.writer_hold.take() { - writer.release().map_err(|_| PersistenceError::Uncertain)?; - } #[cfg(target_os = "macos")] - let bytes = super::seal_allocation::release(&self.path, identity)?; + let released = super::seal_allocation::release(&self.path, identity)?; #[cfg(not(target_os = "macos"))] - let bytes = std::fs::metadata(&self.path) + let released = 0u64; + let metadata = std::fs::metadata(&self.path).map_err(|error| PersistenceError::Refused { + status: error.to_string(), + })?; + let modified = metadata + .modified() .map_err(|error| PersistenceError::Refused { status: error.to_string(), - })? - .len(); - Ok(SealedStore { - path: self.path, - profile: self.profile.persistence, - sqlite_version: self.profile.sqlite_version, - bytes, - }) + })?; + let bytes = if released == 0 { + metadata.len() + } else { + released + }; + let change_counter = read_change_counter(&self.path)?; + Ok(( + SealedFacts { + path: self.path.clone(), + profile: self.profile.persistence, + sqlite_version: self.profile.sqlite_version.clone(), + bytes, + modified, + change_counter, + }, + writer, + )) } } diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/mod.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/mod.rs index 2cb0925bbd..d79e929597 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/mod.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/mod.rs @@ -1,6 +1,7 @@ //! SQLite-only connection, transaction, query, schema and physical mechanics, //! grouped by role (#506 A1). pub(crate) mod acquisition; +pub(crate) mod compact; pub(crate) mod heads; pub(crate) mod lane; pub(crate) mod lifecycle; diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/session/checkpoint.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/session/checkpoint.rs index 151d5e3da3..ae2af52d8f 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/session/checkpoint.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/session/checkpoint.rs @@ -68,6 +68,40 @@ impl Session { } } +impl Session { + /// One `PRAGMA wal_checkpoint(TRUNCATE)` for a caller that already holds the + /// Store's writer lane (§7 step 1). No lane and no turn are taken here; a + /// nonzero `busy` is reported, never retried. + pub(crate) fn checkpoint_holding(&self) -> Result<(), BackendError> { + if !self.writable { + return Err(BackendError::ReadOnly); + } + let _pass = self.background.as_ref().map(|pass| pass.hold()); + let mut state = self.state.lock().map_err(|_| BackendError::Unknown)?; + super::thread_work::attributed(&mut state, |state| { + if state.quarantined || !state.connection.is_autocommit() { + state.quarantined = true; + return Err(BackendError::Unknown); + } + let rows = query::run( + &state.connection, + "PRAGMA wal_checkpoint(TRUNCATE)", + vec![], + &state.work, + )?; + if !state.connection.is_autocommit() { + state.quarantined = true; + return Err(BackendError::Unknown); + } + let busy = rows.first().ok_or(BackendError::Integrity)?.get::(0)?; + if busy != 0 { + return Err(BackendError::Busy); + } + Ok(()) + }) + } +} + /// The length of the Store's `-wal` file; zero when there is none. pub(crate) fn wal_bytes(path: &std::path::Path) -> Result { let mut wal = path.as_os_str().to_os_string(); diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs index c9f6f349f8..640fc7b9d9 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs @@ -249,6 +249,9 @@ pub(crate) fn history(key: &str) -> Result<&'static str, BackendError> { "gc_branch_stack_any" => Ok(include_str!( "../../../../sql/sqlite/queries/history/gc_branch_stack_any.sql" )), + "gc_epoch_read" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/gc_epoch_read.sql" + )), "gc_epoch_share" => Ok(include_str!( "../../../../sql/sqlite/queries/history/gc_epoch_share.sql" )), diff --git a/engine/crates/layerfs-persistence/src/history/commit.rs b/engine/crates/layerfs-persistence/src/history/commit.rs index fc7120f3bc..351c0e533b 100644 --- a/engine/crates/layerfs-persistence/src/history/commit.rs +++ b/engine/crates/layerfs-persistence/src/history/commit.rs @@ -95,6 +95,9 @@ pub(crate) fn commit_branch( if stack.profile != request.profile || stack.scope != request.scope { return Err(HistoryError::Integrity("Commit context")); } + // Spec 5: read the project epoch at the start of this write transaction; + // `confirm` below re-checks it before the transaction commits. + let fence = super::epoch::open(tx, branch.stack)?; let moved = branch.base_layer != request.expected_base || branch.head_commit != request.expected_head; if moved && (!request.force || branch.base_layer != request.expected_base) { @@ -162,6 +165,7 @@ pub(crate) fn commit_branch( if advanced != 1 { return Err(HistoryError::Integrity("Branch advance")); } + fence.confirm(tx)?; let displaced = branch.head_commit.filter(|_| moved); Ok(CommitOutcome::Committed { record: Box::new(record), diff --git a/engine/crates/layerfs-persistence/src/history/epoch.rs b/engine/crates/layerfs-persistence/src/history/epoch.rs new file mode 100644 index 0000000000..1df4cb903d --- /dev/null +++ b/engine/crates/layerfs-persistence/src/history/epoch.rs @@ -0,0 +1,69 @@ +//! The `project_epoch` writer fence (spec §5). +//! +//! Every operation that creates a Commit or Layer reads the project's epoch at +//! the start of the write transaction that will insert it and re-checks that +//! same row before the transaction commits. Each removal marker bumps the epoch +//! in its own marker transaction (§4), so a value that changed between the two +//! reads means a removal raced this write: the writer refuses +//! `Conflict("project epoch")` and the whole write transaction rolls back, so +//! no Commit or Layer is left behind. A missing row (a project deleted and +//! compacted away, §4.1/§6) is the same `Conflict`, never "unchanged". +//! +//! The refusal code is `Conflict`: §10's table names it for the sealed tuple +//! and for a missing `project_epoch` row, and §12's marker-epoch case says a +//! one-shot write racing `branch.remove`/`project.delete` gets `Conflict` and +//! leaves no Commit. It carries no detail (§10; gc-spec-gaps G1 fixes the +//! empty detail for every `Conflict`). +//! +//! On SQLite both reads run inside the one `BEGIN IMMEDIATE` transaction that +//! also writes the Commit/Layer, so a marker can never interleave between them +//! (its `BEGIN IMMEDIATE` is `Busy` until this writer finishes). PostgreSQL +//! makes the re-check the writer's last statement before `COMMIT` with +//! `SELECT ... FOR SHARE` (`gc_epoch_share`), which also blocks the marker's +//! `UPDATE project_epoch SET epoch = epoch + 1` until this transaction resolves: +//! a removal that committed first is seen as a changed value, and one still in +//! flight is serialized after this writer. + +use super::rows::{cell, one}; +use super::transaction::Transaction; +use layerfs_history::error::{HistoryError, HistoryResult}; +use layerfs_history::identity::LayerStackId; + +/// Unlocked read at the start of the write transaction. +const READ: &str = "gc_epoch_read"; +/// Re-read before `COMMIT`; `FOR SHARE` on PostgreSQL, a plain read on SQLite. +const SHARE: &str = "gc_epoch_share"; + +/// The epoch one writer observed when its write transaction opened. +#[derive(Clone, Copy)] +pub(crate) struct Fence { + stack: LayerStackId, + epoch: i64, +} + +/// Reads the project epoch at the start of the write transaction. +/// +/// A missing row is `Conflict`: after `project.delete` and compaction there is +/// no fence left to honour, so the write is refused rather than treated as +/// unchanged. +pub(crate) fn open(tx: &Transaction<'_>, stack: LayerStackId) -> HistoryResult { + let epoch = read(tx, READ, stack)?.ok_or(HistoryError::Conflict("project epoch"))?; + Ok(Fence { stack, epoch }) +} + +impl Fence { + /// Re-checks the epoch as the last statement before `COMMIT`; a changed or + /// removed row is `Conflict` and the caller's transaction writes nothing. + pub(crate) fn confirm(&self, tx: &Transaction<'_>) -> HistoryResult<()> { + let current = + read(tx, SHARE, self.stack)?.ok_or(HistoryError::Conflict("project epoch"))?; + if current != self.epoch { + return Err(HistoryError::Conflict("project epoch")); + } + Ok(()) + } +} + +fn read(tx: &Transaction<'_>, statement: &str, stack: LayerStackId) -> HistoryResult> { + one(tx, statement, [stack.as_slice()], |row| cell::(row, 0)) +} diff --git a/engine/crates/layerfs-persistence/src/history/layerstack.rs b/engine/crates/layerfs-persistence/src/history/layerstack.rs index 17ecc6ddeb..ef5f4e4015 100644 --- a/engine/crates/layerfs-persistence/src/history/layerstack.rs +++ b/engine/crates/layerfs-persistence/src/history/layerstack.rs @@ -352,6 +352,9 @@ pub(crate) fn add_layer( if commit.stack != request.stack { return Err(HistoryError::Integrity("Commit ownership")); } + // Spec 5: read the project epoch at the start of this write transaction; + // `confirm` below re-checks it before the transaction commits. + let fence = super::epoch::open(tx, request.stack)?; if let Some(id) = published_source(tx, request.branch, request.commit)? { let stored = layer(tx, id)?.ok_or(HistoryError::Integrity("Layer provenance"))?; if stored.stack != request.stack @@ -455,6 +458,7 @@ pub(crate) fn add_layer( actual: LayerId::from_slice(&actual)?, }); } + fence.confirm(tx)?; Ok(AddLayerOutcome::Added { layer }) } diff --git a/engine/crates/layerfs-persistence/src/history/merge.rs b/engine/crates/layerfs-persistence/src/history/merge.rs index fc074b4e61..59c9ca97f4 100644 --- a/engine/crates/layerfs-persistence/src/history/merge.rs +++ b/engine/crates/layerfs-persistence/src/history/merge.rs @@ -77,6 +77,9 @@ pub(crate) fn merge_branch( if let Some(state) = super::branch::name_taken(tx, request.stack, &request.name)? { return Err(HistoryError::NameTaken(Box::new(state))); } + // Spec 5: read the project epoch at the start of this write transaction; + // `confirm` below re-checks it before the transaction commits. + let fence = super::epoch::open(tx, request.stack)?; let (base, head) = match &request.head { MergeHead::Node(NodeId::Layer(layer)) => (*layer, None), MergeHead::Node(NodeId::Commit(commit)) => { @@ -132,6 +135,8 @@ pub(crate) fn merge_branch( ], ) .map_err(sql)?; - super::branch::branch_snapshot(tx, request.branch)? - .ok_or(HistoryError::Integrity("Branch insertion")) + let snapshot = super::branch::branch_snapshot(tx, request.branch)? + .ok_or(HistoryError::Integrity("Branch insertion"))?; + fence.confirm(tx)?; + Ok(snapshot) } diff --git a/engine/crates/layerfs-persistence/src/history/mod.rs b/engine/crates/layerfs-persistence/src/history/mod.rs index 7e4c26e712..d90a7291b1 100644 --- a/engine/crates/layerfs-persistence/src/history/mod.rs +++ b/engine/crates/layerfs-persistence/src/history/mod.rs @@ -7,6 +7,7 @@ mod branch; mod catalog; mod commit; mod digests; +mod epoch; mod hints; mod kept; mod layerstack; diff --git a/engine/crates/layerfs-persistence/src/lib.rs b/engine/crates/layerfs-persistence/src/lib.rs index bbf172860e..09d0321814 100644 --- a/engine/crates/layerfs-persistence/src/lib.rs +++ b/engine/crates/layerfs-persistence/src/lib.rs @@ -15,6 +15,10 @@ pub use backend::postgresql::status::{ status as postgresql_status, PostgresStatus, PostgresStatusError, }; pub use backend::sqlite::acquisition::AcquisitionProvider; +pub use backend::sqlite::compact::{ + CompactFailpoint, CompactKeptWaste, CompactPath, CompactReceipt, CompactRequest, + CompactResidue, CompactZeroReason, +}; pub use backend::sqlite::heads::{heads_digest, read_heads}; #[cfg(any(target_os = "macos", target_os = "linux"))] pub use backend::sqlite::lifecycle::headroom::STORE_HEADROOM_BYTES; diff --git a/engine/crates/layerfs-persistence/tests/epoch_fence.rs b/engine/crates/layerfs-persistence/tests/epoch_fence.rs new file mode 100644 index 0000000000..48dabe8a29 --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/epoch_fence.rs @@ -0,0 +1,442 @@ +//! #41 T6.5: the writer-side `project_epoch` fence (spec §5). Every operation +//! that creates a Commit or Layer — `commit_branch`, `add_layer`, +//! `merge_branch`, and the Workspace/file Commit path that calls +//! `commit_branch` — reads the project epoch in its write transaction and +//! re-checks it before that transaction commits. A change means a removal +//! marker (which bumps the epoch in its own transaction) raced the write; a +//! missing row means the project was deleted and compacted away. Both are +//! `Conflict`, and the whole transaction rolls back. +mod support; + +use layerfs_history::error::{HistoryError, Missing}; +use layerfs_history::records::{ + AddLayerOutcome, AddLayerRequest, BranchMergeRequest, CommitMeta, CommitOutcome, CommitRecord, + CommitRequest, CommitVia, ForkRequest, ForkSource, MergeCommit, MergeHead, MergeSide, + StackInitialization, +}; +use layerfs_history::{BranchId, HistoryCatalog, HistoryCatalogConfig, LayerStackId, NodeId}; +use layerfs_persistence::{PersistenceConfig, SqliteHandles, SqlitePersistenceProfile}; +use support::{branch_id, name, root, stack, Temp}; + +const BINDING: &[u8] = b"epoch-fence"; +const CURSOR: [u8; 32] = [53; 32]; + +fn config(path: &std::path::Path) -> PersistenceConfig { + PersistenceConfig::sqlite(path).with_sqlite_profile(SqlitePersistenceProfile::Disposable) +} + +fn catalog(temp: &Temp) -> layerfs_persistence::HistoryProvider { + support::sqlite::create( + &temp.join("store.sqlite"), + &HistoryCatalogConfig { + cursor_key: CURSOR, + binding_key: BINDING.to_vec(), + incarnation: 3, + }, + ) + .expect("catalog") +} + +fn writable(path: &std::path::Path) -> layerfs_persistence::HistoryProvider { + SqliteHandles::open_writable(config(path), BINDING, CURSOR) + .expect("writable catalog") + .history +} + +/// Project `proj` with one Branch `work`, at its genesis Layer. +fn project(history: &dyn HistoryCatalog) -> (LayerStackId, BranchId, layerfs_history::LayerId) { + let record = history + .initialize_layerstack(&StackInitialization { + logical: 1, + stack: stack(1), + name: name("proj"), + scope: root(200), + profile: root(201), + genesis_root: root(1), + }) + .unwrap(); + let branch = branch_id(1); + history + .fork(&ForkRequest { + stack: stack(1), + branch, + name: name("work"), + source: ForkSource::Layer(record.head_layer), + }) + .unwrap(); + (stack(1), branch, record.head_layer) +} + +fn commit_request(history: &dyn HistoryCatalog, branch: BranchId, tag: u8) -> CommitRequest { + let record = history.branch(branch).unwrap().unwrap(); + let expected_root = match record.head_commit { + Some(head) => history.commit(head).unwrap().unwrap().root, + None => history.layer(record.base_layer).unwrap().unwrap().root, + }; + CommitRequest { + branch, + expected_head: record.head_commit, + expected_base: record.base_layer, + expected_root, + candidate_root: root(tag), + profile: root(201), + scope: root(200), + second_parent: None, + meta: CommitMeta::new(CommitVia::Files), + force: false, + } +} + +fn commit(history: &dyn HistoryCatalog, branch: BranchId, tag: u8) -> CommitRecord { + match history + .commit_branch(&commit_request(history, branch, tag)) + .unwrap() + { + CommitOutcome::Committed { record, .. } => *record, + other => panic!("commit: {other:?}"), + } +} + +fn layer_request( + history: &dyn HistoryCatalog, + stack: LayerStackId, + branch: BranchId, +) -> AddLayerRequest { + let record = history.branch(branch).unwrap().unwrap(); + let head = history.layer_stack(stack).unwrap().unwrap(); + AddLayerRequest { + stack, + branch, + commit: record.head_commit.expect("a committed Branch"), + expected_stack_head: head.head_layer, + expected_branch_base: record.base_layer, + } +} + +fn publish( + history: &dyn HistoryCatalog, + stack: LayerStackId, + branch: BranchId, +) -> layerfs_history::LayerRecord { + match history + .add_layer(&layer_request(history, stack, branch)) + .unwrap() + { + AddLayerOutcome::Added { layer } => layer, + other => panic!("publish: {other:?}"), + } +} + +/// A merge of `work`'s Commit with a second Branch still at the genesis Layer: +/// one new Commit, second parent the genesis node, root `tag`. +fn merge_request( + history: &dyn HistoryCatalog, + stack: LayerStackId, + branch: BranchId, + commit: layerfs_history::CommitId, + genesis: layerfs_history::LayerId, + tag: u8, +) -> BranchMergeRequest { + let side = branch_id(2); + history + .fork(&ForkRequest { + stack, + branch: side, + name: name("side"), + source: ForkSource::Layer(genesis), + }) + .unwrap(); + BranchMergeRequest { + stack, + branch: branch_id(9), + name: name("merged"), + sides: [ + (MergeSide::Branch(branch), NodeId::Commit(commit)), + (MergeSide::Branch(side), NodeId::Layer(genesis)), + ], + head: MergeHead::Commit(Box::new(MergeCommit { + base: genesis, + parent: Some(commit), + second_parent: NodeId::Layer(genesis), + root: root(tag), + meta: CommitMeta::new(CommitVia::Merge), + })), + } +} + +fn epoch(path: &std::path::Path, stack: LayerStackId) -> i64 { + let connection = rusqlite::Connection::open(path).unwrap(); + connection + .query_row( + "SELECT epoch FROM project_epoch WHERE layer_stack_id = ?1", + [stack.as_slice()], + |row| row.get(0), + ) + .unwrap() +} + +/// `(commit rows, layer rows)` in the Store file, to prove a refusal wrote +/// nothing. +fn counts(path: &std::path::Path) -> (i64, i64) { + let connection = rusqlite::Connection::open(path).unwrap(); + let count = |table: &str| -> i64 { + connection + .query_row(&format!("SELECT count(*) FROM \"{table}\""), [], |row| { + row.get(0) + }) + .unwrap() + }; + (count("commit"), count("layer")) +} + +fn trigger(path: &std::path::Path, table: &str) { + let connection = rusqlite::Connection::open(path).unwrap(); + connection + .execute_batch(&format!( + "CREATE TRIGGER t65_epoch_race BEFORE INSERT ON \"{table}\" BEGIN \ + UPDATE project_epoch SET epoch = epoch + 1 \ + WHERE layer_stack_id = NEW.layer_stack_id; END;" + )) + .unwrap(); +} + +fn drop_trigger(path: &std::path::Path) { + let connection = rusqlite::Connection::open(path).unwrap(); + connection + .execute_batch("DROP TRIGGER t65_epoch_race") + .unwrap(); +} + +#[test] +fn a_normal_commit_publish_and_merge_leave_the_epoch_alone() { + let temp = Temp::new("epoch-fence-normal"); + let path = temp.join("store.sqlite"); + let history = catalog(&temp); + let (shop, work, genesis) = project(&history); + + let first = commit(&history, work, 5); + assert_eq!(epoch(&path, shop), 0, "a Commit does not bump the epoch"); + + let layer = publish(&history, shop, work); + assert_eq!(layer.parent, Some(genesis)); + assert_eq!(epoch(&path, shop), 0, "a published Layer does not bump it"); + + let merge = history + .merge_branch(&merge_request(&history, shop, work, first.id, genesis, 60)) + .unwrap(); + assert_eq!(merge.branch.name.as_str(), "merged"); + assert_eq!(epoch(&path, shop), 0, "a merge does not bump it"); + assert_eq!((counts(&path).0, counts(&path).1), (2, 2)); +} + +#[test] +fn a_missing_epoch_row_refuses_every_writer_with_conflict() { + let temp = Temp::new("epoch-fence-missing"); + let path = temp.join("store.sqlite"); + let history = catalog(&temp); + let (shop, work, genesis) = project(&history); + let first = commit(&history, work, 5); + let side_request = merge_request(&history, shop, work, first.id, genesis, 61); + let before = counts(&path); + + let connection = rusqlite::Connection::open(&path).unwrap(); + connection + .execute( + "DELETE FROM project_epoch WHERE layer_stack_id = ?1", + [shop.as_slice()], + ) + .unwrap(); + drop(connection); + + // No fence row is left after `project.delete` + compaction: every writer + // refuses `Conflict`, never "unchanged". + let refused = history + .commit_branch(&commit_request(&history, work, 6)) + .unwrap_err(); + assert!( + matches!(refused, HistoryError::Conflict("project epoch")), + "{refused:?}" + ); + let refused = history + .add_layer(&layer_request(&history, shop, work)) + .unwrap_err(); + assert!( + matches!(refused, HistoryError::Conflict("project epoch")), + "{refused:?}" + ); + let refused = history.merge_branch(&side_request).unwrap_err(); + assert!( + matches!(refused, HistoryError::Conflict("project epoch")), + "{refused:?}" + ); + + // No partial write: no Commit, no Layer, and the Branch head is unchanged. + assert_eq!(counts(&path), before); + let snapshot = history.branch_snapshot(work).unwrap().unwrap(); + assert_eq!(snapshot.branch.head_commit, Some(first.id)); + assert_eq!( + history.layer_stack(shop).unwrap().unwrap().head_layer, + genesis + ); + assert!(history.branch(branch_id(9)).unwrap().is_none()); +} + +#[test] +fn a_removal_first_refuses_the_later_commit_and_writes_nothing() { + let temp = Temp::new("epoch-fence-removal-first"); + let path = temp.join("store.sqlite"); + let history = catalog(&temp); + let (shop, work, _genesis) = project(&history); + let _first = commit(&history, work, 5); + let before = counts(&path); + + // Capture the request while the Branch is still visible; the removal then + // wins, and the filtered Branch resolution hides the retired Branch before + // the fence opens, so the write is refused and leaves no Commit behind. + let request = commit_request(&history, work, 6); + history.mark_branch_removed(work, 1_000).unwrap(); + assert_eq!(epoch(&path, shop), 1, "the marker bumped the epoch once"); + + let refused = history.commit_branch(&request).unwrap_err(); + assert!( + matches!( + refused, + HistoryError::Missing(Missing::Branch) | HistoryError::Conflict("project epoch") + ), + "{refused:?}" + ); + assert_eq!(counts(&path), before); +} + +#[test] +fn a_concurrent_commit_and_removal_have_one_winner() { + let temp = Temp::new("epoch-fence-race"); + let path = temp.join("store.sqlite"); + let history = catalog(&temp); + let (shop, work, _genesis) = project(&history); + drop(history); + + let left = writable(&path); + let right = writable(&path); + let request = commit_request(&left, work, 7); + let (committed, removed) = std::thread::scope(|scope| { + let committed = scope.spawn(move || left.commit_branch(&request)); + let removed = scope.spawn(move || right.mark_branch_removed(work, 2_000)); + (committed.join().unwrap(), removed.join().unwrap()) + }); + + // The two writes serialize: a Commit first, then the removal retires the + // Branch with it (or is `Busy` and retries); or the removal first, and the + // Commit is refused, leaving no row behind. + match (&committed, &removed) { + (Ok(_), Ok(_)) => { + assert_eq!(epoch(&path, shop), 1); + assert_eq!(counts(&path).0, 1, "the Commit stays under the removal"); + } + (Ok(_), Err(HistoryError::Busy)) => { + assert_eq!(epoch(&path, shop), 0, "the removal has to retry"); + assert_eq!(counts(&path).0, 1); + } + ( + Err( + HistoryError::Busy + | HistoryError::Missing(Missing::Branch) + | HistoryError::Conflict(_), + ), + Ok(_), + ) => { + assert_eq!(epoch(&path, shop), 1); + assert_eq!(counts(&path).0, 0, "the refused Commit left nothing"); + } + other => panic!("unexpected outcome: {other:?}"), + } +} + +#[test] +fn an_epoch_change_during_a_commit_is_detected_and_rolled_back() { + let temp = Temp::new("epoch-fence-trigger-commit"); + let path = temp.join("store.sqlite"); + let history = catalog(&temp); + let (shop, work, _genesis) = project(&history); + + // A trigger stands in for the removal marker that bumps the epoch after + // the writer's start read: it fires inside the writer's own transaction, + // before its re-check. + trigger(&path, "commit"); + let refused = history + .commit_branch(&commit_request(&history, work, 5)) + .unwrap_err(); + assert!( + matches!(refused, HistoryError::Conflict("project epoch")), + "{refused:?}" + ); + assert_eq!(counts(&path).0, 0, "the Commit rolled back"); + assert_eq!(epoch(&path, shop), 0, "the fake bump rolled back too"); + assert!(history.branch(work).unwrap().unwrap().head_commit.is_none()); + + // Without the interleaved change the same Commit lands. + drop_trigger(&path); + let record = commit(&history, work, 5); + assert_eq!(record.stack, shop); + assert_eq!(epoch(&path, shop), 0); + assert_eq!( + history.branch(work).unwrap().unwrap().head_commit, + Some(record.id) + ); +} + +#[test] +fn an_epoch_change_during_a_publish_is_detected_and_rolled_back() { + let temp = Temp::new("epoch-fence-trigger-layer"); + let path = temp.join("store.sqlite"); + let history = catalog(&temp); + let (shop, work, genesis) = project(&history); + let _first = commit(&history, work, 5); + let before = counts(&path); + + trigger(&path, "layer"); + let refused = history + .add_layer(&layer_request(&history, shop, work)) + .unwrap_err(); + assert!( + matches!(refused, HistoryError::Conflict("project epoch")), + "{refused:?}" + ); + assert_eq!(counts(&path), before, "the Layer rolled back"); + assert_eq!(epoch(&path, shop), 0); + assert_eq!( + history.layer_stack(shop).unwrap().unwrap().head_layer, + genesis + ); + + drop_trigger(&path); + let layer = publish(&history, shop, work); + assert_eq!(layer.parent, Some(genesis)); + assert_eq!(counts(&path).1, before.1 + 1); +} + +#[test] +fn an_epoch_change_during_a_merge_is_detected_and_rolled_back() { + let temp = Temp::new("epoch-fence-trigger-merge"); + let path = temp.join("store.sqlite"); + let history = catalog(&temp); + let (shop, work, genesis) = project(&history); + let first = commit(&history, work, 5); + let request = merge_request(&history, shop, work, first.id, genesis, 62); + let before = counts(&path); + + trigger(&path, "commit"); + let refused = history.merge_branch(&request).unwrap_err(); + assert!( + matches!(refused, HistoryError::Conflict("project epoch")), + "{refused:?}" + ); + assert_eq!(counts(&path), before, "the merge Commit rolled back"); + assert_eq!(epoch(&path, shop), 0); + assert!(history.branch(branch_id(9)).unwrap().is_none()); + + drop_trigger(&path); + let merge = history.merge_branch(&request).unwrap(); + assert_eq!(merge.branch.name.as_str(), "merged"); + assert!(merge.branch.head_commit.is_some()); +} diff --git a/engine/crates/layerfs-persistence/tests/sqlite_compact.rs b/engine/crates/layerfs-persistence/tests/sqlite_compact.rs new file mode 100644 index 0000000000..6b4578cded --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/sqlite_compact.rs @@ -0,0 +1,456 @@ +//! #41 T8: the §7 SQLite rebuild. A compaction takes the writer lane, runs the +//! mark, chooses `none`/`rebuild` from the wholly dead packs, builds and +//! verifies `.compact`, installs it at `G → G+1`, and leaves no +//! artifact; a recorded leftover is judged by the generation and removed by +//! the next `store.compact`. +#![cfg(any(target_os = "macos", target_os = "linux"))] +mod support; + +use layerfs_content::{encode_whole_file_payload, FinalizedObject, ObjectId, ObjectRole}; +use layerfs_history::{ + AddLayerOutcome, AddLayerRequest, BranchId, CommitId, CommitMeta, CommitOutcome, + CommitRequest, CommitVia, ForkRequest, ForkSource, HistoryCatalog, HistoryCatalogConfig, + LayerStackId, LayerStackRecord, StackInitialization, +}; +use layerfs_persistence::{ + CompactFailpoint, CompactPath, CompactRequest, PersistenceConfig, SqliteHandles, + SqlitePersistenceProfile, +}; +use layerfs_storage::{ + port::PackPersistence, Storage, StoragePolicy, +}; +use std::path::{Path, PathBuf}; +use support::Temp; + +const BINDING: &[u8] = b"sqlite-compact-test-binding"; + +fn create_store() -> (Temp, SqliteHandles, PathBuf) { + let temp = Temp::new("sqlite-compact"); + let path = temp.join("store.sqlite"); + let handles = SqliteHandles::create( + PersistenceConfig::sqlite(path.clone()) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable), + StoragePolicy::frozen_default(), + &HistoryCatalogConfig { + binding_key: BINDING.to_vec(), + cursor_key: [71; 32], + incarnation: 7, + }, + ) + .unwrap(); + (temp, handles, path) +} + +fn reopen(path: &Path) -> SqliteHandles { + SqliteHandles::open_writable( + PersistenceConfig::sqlite(path.to_path_buf()) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable), + BINDING, + [71; 32], + ) + .unwrap() +} + +fn request(op: u8) -> CompactRequest { + CompactRequest { + op_id: [op; 16], + ..CompactRequest::default() + } +} + +fn request_at(op: u8, failpoint: CompactFailpoint) -> CompactRequest { + CompactRequest { + op_id: [op; 16], + failpoint: Some(failpoint), + ..CompactRequest::default() + } +} + +fn compact_path(store: &Path) -> PathBuf { + let mut name = store.as_os_str().to_os_string(); + name.push(".compact"); + PathBuf::from(name) +} + +fn op_path(store: &Path) -> PathBuf { + let mut name = store.as_os_str().to_os_string(); + name.push(".compact-op"); + PathBuf::from(name) +} + +fn visited_path(store: &Path) -> PathBuf { + let mut name = store.as_os_str().to_os_string(); + name.push(".compact-visited"); + PathBuf::from(name) +} + + +fn whole_file(handles: &SqliteHandles, bytes: &[u8]) -> ObjectId { + let object = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(bytes).unwrap(), + ) + .unwrap(); + save_objects(handles, &[object]) +} + +fn save_objects(handles: &SqliteHandles, objects: &[FinalizedObject]) -> ObjectId { + let storage = Storage::new(handles.storage.clone()).unwrap(); + let save = storage.begin_save().unwrap(); + let mut last = None; + for object in objects { + last = Some(object.id()); + save.accept(object.clone()).unwrap(); + } + save.finish().unwrap(); + last.expect("at least one object") +} + +fn filesystem_root(handles: &SqliteHandles, seed: u8) -> ObjectId { + use layerfs_content::{ + filesystem::{ + attributes::PortableMetadata, + directory::{encode_directory_page, DirectoryPage}, + inode::{encode_inode_page, InodePage}, + root::profile_id, scope_for_seed, FilesystemRoot, + }, + inode_leaf::{InodeKind, InodeMetadata, InodeValue}, + }; + let directory = FinalizedObject::new( + ObjectRole::DirectoryLeaf, + encode_directory_page(&DirectoryPage::Leaf { entries: Vec::new() }).unwrap(), + ) + .unwrap(); + let directory_id = directory.id(); + let inode = FinalizedObject::new( + ObjectRole::InodeLeaf, + encode_inode_page(&InodePage::Leaf { + entries: vec![( + 1, + InodeValue { + kind: InodeKind::Directory, + namespace_ref_count: 0, + content_root: directory_id, + metadata_root: InodeMetadata::Inline(PortableMetadata { + mode: 0o755, + mtime_seconds: i64::from(seed), + mtime_nanoseconds: 0, + }), + }, + )], + }) + .unwrap(), + ) + .unwrap() + .with_references(vec![directory_id]); + let inode_id = inode.id(); + let root = FilesystemRoot::new( + profile_id(), + scope_for_seed([1; 32]), + 1, + inode_id, + ) + .unwrap(); + let object = FinalizedObject::new(ObjectRole::FilesystemRoot, root.encode().unwrap()) + .unwrap() + .with_references(vec![inode_id]); + save_objects(handles, &[directory, inode, object]) +} + +fn initialize(handles: &SqliteHandles, tag: u8, genesis: ObjectId) -> LayerStackRecord { + handles + .history + .initialize_layerstack(&StackInitialization { + logical: 10, + stack: support::stack(tag), + name: support::name(&format!("project-{tag}")), + scope: layerfs_content::filesystem::scope_for_seed([1; 32]).object(), + profile: layerfs_content::filesystem::root::profile_id(), + genesis_root: genesis, + }) + .unwrap() +} + +fn fork(handles: &SqliteHandles, stack: LayerStackId, branch: BranchId, tag: u8) { + let record = handles.history.layer_stack(stack).unwrap().unwrap(); + handles + .history + .fork(&ForkRequest { + stack, + branch, + name: support::name(&format!("branch-{tag}")), + source: ForkSource::Layer(record.head_layer), + }) + .unwrap(); +} + +fn commit(handles: &SqliteHandles, branch: BranchId, candidate_root: ObjectId) -> CommitId { + let snapshot = handles.history.branch(branch).unwrap().unwrap(); + let expected_root = match snapshot.head_commit { + Some(head) => handles.history.commit(head).unwrap().unwrap().root, + None => handles + .history + .layer(snapshot.base_layer) + .unwrap() + .unwrap() + .root, + }; + match handles + .history + .commit_branch(&CommitRequest { + branch, + expected_head: snapshot.head_commit, + expected_base: snapshot.base_layer, + expected_root, + candidate_root, + profile: layerfs_content::filesystem::root::profile_id(), + scope: layerfs_content::filesystem::scope_for_seed([1; 32]).object(), + second_parent: None, + meta: CommitMeta::new(CommitVia::Files), + force: false, + }) + .unwrap() + { + CommitOutcome::Committed { record, .. } => record.id, + other => panic!("commit: {other:?}"), + } +} + +fn publish(handles: &SqliteHandles, stack: LayerStackId, branch: BranchId) { + let branch_record = handles.history.branch(branch).unwrap().unwrap(); + let stack_record = handles.history.layer_stack(stack).unwrap().unwrap(); + match handles + .history + .add_layer(&AddLayerRequest { + stack, + branch, + commit: branch_record.head_commit.expect("a committed Branch"), + expected_stack_head: stack_record.head_layer, + expected_branch_base: branch_record.base_layer, + }) + .unwrap() + { + AddLayerOutcome::Added { .. } => {} + other => panic!("publish: {other:?}"), + } +} + +fn generation(handles: &SqliteHandles) -> u64 { + handles.history.incarnation() +} + +fn located(handles: &SqliteHandles, id: ObjectId) -> bool { + let mut found = Vec::new(); + handles.storage.locate(&[id], &mut found).unwrap(); + !found.is_empty() +} + +fn clean(handles: &SqliteHandles) { + let mut out = Vec::new(); + handles + .diagnose( + layerfs_persistence::diagnostic::Options::default(), + &mut |observation| out.push(observation), + ) + .unwrap(); + assert!( + !out.iter().any(|observation| matches!( + observation.outcome, + layerfs_persistence::diagnostic::Outcome::Fail + | layerfs_persistence::diagnostic::Outcome::Unknown + )), + "{out:?}" + ); +} + +#[test] +fn an_empty_store_takes_the_none_path() { + let (_temp, handles, path) = create_store(); + let before = generation(&handles); + let receipt = handles.compact(&request(1)).unwrap(); + assert_eq!(receipt.path, CompactPath::None); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before); + assert_eq!(receipt.dead_bytes_bound, 0); + assert!(receipt + .zero_reasons + .iter() + .any(|zero| zero.field == "deadBytesBound" && zero.reason == "empty-store")); + assert!(!compact_path(&path).exists()); + assert!(!visited_path(&path).exists()); + assert!(!op_path(&path).exists()); + let reopened = reopen(&path); + assert_eq!(generation(&reopened), before); + reopened.history.mark().unwrap(); +} + +#[test] +fn a_store_without_wholly_dead_packs_takes_the_none_path() { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let before = generation(&handles); + let receipt = handles.compact(&request(2)).unwrap(); + assert_eq!(receipt.path, CompactPath::None); + assert_eq!(receipt.dead_bytes_bound, 0); + assert_eq!( + receipt.dead_bytes_refined, receipt.dead_bytes_bound, + "the none path does not refine" + ); + assert_eq!(receipt.generation_after, before); + assert_eq!(receipt.physical_shrink, 0); + assert!(!compact_path(&path).exists()); + assert!(!op_path(&path).exists()); +} + +#[test] +fn a_wholly_dead_pack_is_rebuilt_away() { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let dead = whole_file(&handles, b"nobody reaches this object"); + let before = generation(&handles); + let receipt = handles.compact(&request(3)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before + 1); + assert!(!compact_path(&path).exists()); + assert!(!visited_path(&path).exists()); + assert!(!op_path(&path).exists()); + for suffix in ["-wal", "-shm", "-journal"] { + let mut name = path.as_os_str().to_os_string(); + name.push(suffix); + assert!(!PathBuf::from(name).exists(), "sidecar {suffix} remains"); + } + + let reopened = reopen(&path); + assert_eq!(generation(&reopened), before + 1); + assert!(located(&reopened, live), "the retained object survives"); + assert!(!located(&reopened, dead), "the dead object is collected"); + clean(&reopened); +} + +#[test] +fn retained_history_and_its_objects_survive_a_rebuild() { + let (_temp, handles, path) = create_store(); + let genesis = filesystem_root(&handles, 1); + let committed = filesystem_root(&handles, 2); + let stack = initialize(&handles, 1, genesis); + let work = support::branch_id(1); + fork(&handles, stack.id, work, 1); + let commit_id = commit(&handles, work, committed); + publish(&handles, stack.id, work); + let dead = whole_file(&handles, b"a pack no retained root reaches"); + let before = generation(&handles); + let receipt = handles.compact(&request(4)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + let reopened = reopen(&path); + assert_eq!(generation(&reopened), before + 1); + assert!(reopened.history.commit(commit_id).unwrap().is_some()); + assert!(reopened.history.layer_stack(stack.id).unwrap().is_some()); + for id in [genesis, committed] { + assert!(located(&reopened, id), "object {id} survives"); + } + assert!(!located(&reopened, dead)); + clean(&reopened); +} + +#[test] +fn a_stray_staged_name_is_refused_and_not_deleted() { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _ = whole_file(&handles, b"dead"); + let stray = compact_path(&path); + std::fs::write(&stray, b"not a Store").unwrap(); + let error = handles.compact(&request(5)).unwrap_err(); + assert!( + matches!(error, layerfs_history::HistoryError::Exists(_)), + "{error:?}" + ); + assert!(stray.exists(), "a name no record names is not deleted"); +} + +#[test] +fn a_build_failpoint_leaves_a_restartable_artifact() { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let dead = whole_file(&handles, b"dead"); + let before = generation(&handles); + let error = handles + .compact(&request_at(6, CompactFailpoint::AfterBuild)) + .unwrap_err(); + assert!( + matches!(error, layerfs_history::HistoryError::UnknownOutcome), + "{error:?}" + ); + assert!(compact_path(&path).exists(), "the staged Store remains"); + assert!(visited_path(&path).exists(), "the visited scratch remains"); + assert!(op_path(&path).exists(), "the op record remains"); + + let reopened = reopen(&path); + assert_eq!( + generation(&reopened), + before, + "the old Store is still the live one" + ); + let receipt = reopened.compact(&request(7)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_after, before + 1); + assert!(!compact_path(&path).exists()); + assert!(!visited_path(&path).exists()); + assert!(!op_path(&path).exists()); + let final_handles = reopen(&path); + assert!(!located(&final_handles, dead)); + clean(&final_handles); +} + +#[test] +fn an_installed_but_unsettled_run_is_judged_at_g_plus_one() { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _ = whole_file(&handles, b"dead"); + let before = generation(&handles); + let error = handles + .compact(&request_at(8, CompactFailpoint::AfterInstall)) + .unwrap_err(); + assert!( + matches!(error, layerfs_history::HistoryError::UnknownOutcome), + "{error:?}" + ); + assert!(op_path(&path).exists()); + let reopened = reopen(&path); + assert_eq!( + generation(&reopened), + before + 1, + "the new Store is already installed" + ); + let receipt = reopened.compact(&request(9)).unwrap(); + assert_eq!(receipt.generation_before, before + 1); + assert!(!op_path(&path).exists()); + assert!(!compact_path(&path).exists()); +} + +#[test] +fn generation_increases_once_per_installed_rebuild() { + let (_temp, handles, path) = create_store(); + let first = filesystem_root(&handles, 1); + initialize(&handles, 1, first); + let _ = whole_file(&handles, b"first dead root"); + let start = generation(&handles); + assert_eq!( + handles.compact(&request(10)).unwrap().generation_after, + start + 1 + ); + let reopened = reopen(&path); + let _ = whole_file(&reopened, b"second dead root"); + let receipt = reopened.compact(&request(11)).unwrap(); + assert_eq!(receipt.generation_before, start + 1); + assert_eq!(receipt.generation_after, start + 2); + let final_handles = reopen(&path); + assert_eq!(generation(&final_handles), start + 2); + assert!(located(&final_handles, first)); + clean(&final_handles); +} diff --git a/engine/crates/layerfs-store-service/src/history/admission.rs b/engine/crates/layerfs-store-service/src/history/admission.rs index 283ed668a3..84b421dca8 100644 --- a/engine/crates/layerfs-store-service/src/history/admission.rs +++ b/engine/crates/layerfs-store-service/src/history/admission.rs @@ -49,8 +49,16 @@ pub fn project_removal(stack: &LayerStackRecord) -> GcIntent { /// A `project.delete` intent when only the stack id is known. pub fn project_removal_from_id(stack: LayerStackId) -> GcIntent { + project_removal_at(stack, op_id()) +} + +/// A `project.delete` intent under an operation identity the caller already +/// registered. The control layer admits the project scope before its sidecar +/// and marker and reuses the identity there, so one row is taken and released +/// once, never two competing rows. +pub fn project_removal_at(stack: LayerStackId, op: [u8; 16]) -> GcIntent { GcIntent::operation( - op_id(), + op, GcIntentKind::Removal, GcScopeKind::Project, stack.as_slice(), diff --git a/engine/crates/layerfs-store-service/src/history/deletion.rs b/engine/crates/layerfs-store-service/src/history/deletion.rs index c4bd8d6e1a..3fb3d38710 100644 --- a/engine/crates/layerfs-store-service/src/history/deletion.rs +++ b/engine/crates/layerfs-store-service/src/history/deletion.rs @@ -10,7 +10,7 @@ use crate::history::lookup; use crate::store::Store; use layerfs_bridge::control::SdkRefusal; -use layerfs_history::records::GcDeletion; +use layerfs_history::records::{GcDeletion, GcIntent}; use layerfs_history::LayerStackId; use std::sync::Arc; @@ -51,10 +51,23 @@ pub fn delete_stack( stack: LayerStackId, deleted_ms: i64, ) -> Result { - let history = store.history(); let intent = crate::history::admission::project_removal_from_id(stack); + delete_stack_with(store, stack, deleted_ms, &intent) +} + +/// [`delete_stack`] under an admission intent the caller already registered. +/// The marker's own admission scan excludes this operation identity, so the +/// caller's row is reused and released once instead of being seen as another +/// removal. A provider without admission writes the marker as before. +pub fn delete_stack_with( + store: &Arc, + stack: LayerStackId, + deleted_ms: i64, + intent: &GcIntent, +) -> Result { + let history = store.history(); match history - .gc_delete_stack(stack, deleted_ms, &intent) + .gc_delete_stack(stack, deleted_ms, intent) .map_err(lookup::history)? { GcDeletion::Retired { deleted_ms } => Ok(Deletion::Retired { deleted_ms }), diff --git a/engine/crates/layerfs-store-service/src/history/lookup.rs b/engine/crates/layerfs-store-service/src/history/lookup.rs index a37773f2d0..c197adebc4 100644 --- a/engine/crates/layerfs-store-service/src/history/lookup.rs +++ b/engine/crates/layerfs-store-service/src/history/lookup.rs @@ -57,6 +57,7 @@ pub fn history(error: HistoryError) -> SdkRefusal { HistoryError::NameTaken(_) => "NameTaken", HistoryError::Removed(_) => "Removed", HistoryError::Conflict(_) => "Conflict", + HistoryError::Exists(_) => "Exists", HistoryError::StaleStore => "StaleStore", HistoryError::Unsupported(_) => "Unsupported", HistoryError::UnknownOutcome => "Unknown", diff --git a/engine/crates/layerfs-store-service/src/write/files.rs b/engine/crates/layerfs-store-service/src/write/files.rs index 281c3f0de8..f15aec7227 100644 --- a/engine/crates/layerfs-store-service/src/write/files.rs +++ b/engine/crates/layerfs-store-service/src/write/files.rs @@ -123,6 +123,12 @@ pub fn commit( crate::store::CommitError::History(layerfs_history::HistoryError::Busy) => { "Busy" } + // Spec 5: the project epoch moved (a removal raced this + // write) or is gone after compaction; the Commit is + // refused with the spec's `Conflict`, nothing written. + crate::store::CommitError::History( + layerfs_history::HistoryError::Conflict(_), + ) => "Conflict", crate::store::CommitError::History( layerfs_history::HistoryError::Capacity(_), ) => "Capacity", diff --git a/engine/crates/layerfs-store-service/tests/epoch_fence.rs b/engine/crates/layerfs-store-service/tests/epoch_fence.rs new file mode 100644 index 0000000000..41674a4421 --- /dev/null +++ b/engine/crates/layerfs-store-service/tests/epoch_fence.rs @@ -0,0 +1,77 @@ +//! #41 T6.5: the writer-side `project_epoch` fence on the Workspace Commit +//! path. `fs.write` (and the SDK `commit` method) go through +//! `files::commit` -> `BoundWorkspace::commit` -> `HistoryCatalog::commit_branch`, +//! so a fence refusal must arrive as the spec's `Conflict`, not as a generic +//! failure, and an ordinary write must be unaffected. +#[path = "support/installed_store.rs"] +#[allow(dead_code)] +mod installed; +#[path = "support/sdk.rs"] +#[allow(dead_code)] +mod support; + +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_store_service::history::lookup; +use std::fs; +use support::Fixture; + +fn text(value: &str) -> SdkValue { + SdkValue::text(value) +} + +fn write(fixture: &Fixture, content: &str) -> Result { + fixture.try_call( + "fs.write", + vec![ + ("branch", text("work")), + ("path", text("/a.txt")), + ("content", text(content)), + ], + ) +} + +fn commits(fixture: &Fixture) -> usize { + support::items(&fixture.call("branch.list_commits", vec![("branch", text("work"))])).len() +} + +#[test] +fn a_missing_epoch_row_is_conflict_on_the_workspace_commit_path() { + let fixture = Fixture::new("epoch-fence-service", |source| { + fs::write(source.join("a.txt"), b"a\n").unwrap(); + 1 + }); + fixture.fork("work"); + + // The ordinary path still commits and leaves the epoch row alone. + write(&fixture, "b\n").expect("an ordinary write commits"); + let epoch_row = |path: &std::path::Path, stack: &[u8]| -> i64 { + rusqlite::Connection::open(path) + .unwrap() + .query_row( + "SELECT epoch FROM project_epoch WHERE layer_stack_id = ?1", + [stack], + |row| row.get(0), + ) + .unwrap() + }; + let stack = lookup::project(&fixture.service.store, "installed").expect("project"); + assert_eq!(epoch_row(&fixture.host.config.path, stack.id.as_slice()), 0); + + // Remove the fence row behind the service's back: the next Commit's + // start-of-transaction read finds no epoch to honour. + rusqlite::Connection::open(&fixture.host.config.path) + .unwrap() + .execute( + "DELETE FROM project_epoch WHERE layer_stack_id = ?1", + [stack.id.as_slice()], + ) + .unwrap(); + + let before = commits(&fixture); + let refused = write(&fixture, "c\n").expect_err("the fence refuses the commit"); + assert_eq!(refused.code, "Conflict", "{refused:?}"); + assert!(refused.message.contains("project epoch"), "{refused:?}"); + assert_eq!(commits(&fixture), before, "nothing was published"); + + fixture.cleanup(); +}