diff --git a/api/crates/layerfs-api/tests/home_layout_3.rs b/api/crates/layerfs-api/tests/home_layout_3.rs index 63eade8633..b9ff16fe24 100644 --- a/api/crates/layerfs-api/tests/home_layout_3.rs +++ b/api/crates/layerfs-api/tests/home_layout_3.rs @@ -87,11 +87,20 @@ fn home_with(tag: &str, extra: &[(&str, SdkValue)]) -> PathBuf { let home = tempdir(tag); home::open(&home).unwrap(); call(&home, "store.create", &[("name", t("shop"))]).unwrap(); - write_record( - &paths::project_record(&home, "shop", "app"), - &SdkValue::map([("name", t("app")), ("created", SdkValue::Int(1))]), + // a real project, so the §4.2 home↔Store intersection lists and gets it + // across the conversion + let source = tempdir(&format!("{tag}-source")); + std::fs::write(source.join("README.md"), "app\n").unwrap(); + call( + &home, + "project.init", + &[ + ("project", t("shop/app")), + ("folder", t(&source.display().to_string())), + ], ) .unwrap(); + let _ = std::fs::remove_dir_all(source); write_record( &paths::machine_record(&home, "dev"), &SdkValue::map([ diff --git a/api/crates/layerfs-api/tests/project_export.rs b/api/crates/layerfs-api/tests/project_export.rs index 85ef103334..0db2299303 100644 --- a/api/crates/layerfs-api/tests/project_export.rs +++ b/api/crates/layerfs-api/tests/project_export.rs @@ -1600,3 +1600,73 @@ fn export_progress_events_follow_time_not_entry_count() { let _ = std::fs::remove_dir_all(d); } } + +#[test] +fn export_omits_a_path_hidden_by_remove_file() { + let home = private_home("removed-home"); + ok(&home, "store.create", &[("name", t("shop"))]); + let source = tempdir("removed-src"); + std::fs::write(source.join("a.txt"), "a\n").unwrap(); + std::fs::write(source.join("b.txt"), "b\n").unwrap(); + std::fs::create_dir(source.join("dir")).unwrap(); + std::fs::write(source.join("dir/c.txt"), "c\n").unwrap(); + ok( + &home, + "project.init", + &[ + ("project", t("shop/app")), + ("folder", t(&source.display().to_string())), + ], + ); + + // The marker is written through the Store engine, the route the fs calls + // use; `project.remove_file` is not on this crate's call table yet. + layerfs_store::service::with_service( + &layerfs_store::open_args::OpenArgs { + home: Some(home.clone()), + store: Some("shop".to_owned()), + store_path: Some(paths::store_file(&home, "shop").display().to_string()), + ..Default::default() + }, + |service| { + let call = layerfs_bridge::control::SdkCall { + method: "project.remove_file".to_owned(), + args: SdkValue::map([ + ("project", t("app")), + ("path", t("/a.txt")), + ]), + }; + layerfs_store_service::methods::call(service, &call) + }, + ) + .unwrap(); + + let out_parent = tempdir("removed-out"); + let dest = out_parent.join("exported"); + let res = ok( + &home, + "project.export", + &[ + ("project", t("shop/app")), + ("destination", t(&dest.display().to_string())), + ], + ); + + // §4.3: a hidden entry is omitted from the exported folder and counts. + assert!( + !dest.join("a.txt").exists(), + "a hidden path must not be exported" + ); + assert_eq!(std::fs::read_to_string(dest.join("b.txt")).unwrap(), "b\n"); + assert_eq!( + std::fs::read_to_string(dest.join("dir/c.txt")).unwrap(), + "c\n" + ); + assert_eq!(res.get_int("files"), Some(2)); + assert_eq!(res.get_int("directories"), Some(1)); + assert_eq!(res.get_int("symlinks"), Some(0)); + + for d in [home, source, out_parent] { + let _ = std::fs::remove_dir_all(d); + } +} diff --git a/api/crates/layerfs-api/tests/project_list_intersection.rs b/api/crates/layerfs-api/tests/project_list_intersection.rs new file mode 100644 index 0000000000..64532d6cbc --- /dev/null +++ b/api/crates/layerfs-api/tests/project_list_intersection.rs @@ -0,0 +1,141 @@ +//! project.list and project.get intersect the home's project entries with +//! the Store's filtered listing (spec §4.2): a home entry whose Store +//! project is retired (hidden) or absent is not listed, and project.get +//! answers it `NoSuchProject`; a live project still lists and gets. +use layerfs_api::home::{self, paths, write_record}; +use layerfs_bridge::control::SdkValue; +use layerfs_bridge::sdk_values::{hex, unhex}; +use layerfs_history::LayerStackId; +use layerfs_store::open_args::OpenArgs; +use layerfs_store::project::record::ProjectRecord; +use layerfs_store::record::StoreRecord; +use layerfs_store::service::with_service; +use layerfs_store_service::history::deletion::delete_stack; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; + +static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +fn tempdir(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "layerfs-project-intersection-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +fn t(s: &str) -> SdkValue { + SdkValue::text(s) +} + +fn call(home: &Path, name: &str, args: &[(&str, SdkValue)]) -> Result { + let args = SdkValue::Map( + args.iter() + .map(|(n, v)| ((*n).to_owned(), v.clone())) + .collect(), + ); + layerfs_api::call(Some(home), name, args, &mut |_| {}).map_err(|r| (r.code, r.message)) +} + +fn ok(home: &Path, name: &str, args: &[(&str, SdkValue)]) -> SdkValue { + call(home, name, args).unwrap_or_else(|e| panic!("{name}: {e:?}")) +} + +/// Marks Store project `name` hidden, as the project.delete write path will +/// (T4.5b); the home entry stays, exactly the stale case §4.2 hides. +fn retire(home: &Path, store: &str, name: &str, stack: &str) { + let record = StoreRecord::read(home, store).unwrap().unwrap(); + let open = OpenArgs { + home: Some(home.to_path_buf()), + store: Some(store.to_owned()), + project: Some(name.to_owned()), + store_path: Some(paths::store_file(home, store).display().to_string()), + store_record: Some(record.to_record().clone()), + postgresql: None, + }; + let stack = LayerStackId::from_slice(&unhex(stack).unwrap()).unwrap(); + with_service(&open, |service| { + delete_stack(&service.store, stack, 1234).unwrap(); + Ok(SdkValue::Null) + }) + .unwrap(); +} + +#[test] +fn a_retired_or_absent_store_project_is_not_listed_or_got() { + let home = tempdir("home"); + std::fs::set_permissions(&home, std::fs::Permissions::from_mode(0o700)).unwrap(); + home::open(&home).unwrap(); + ok(&home, "store.create", &[("name", t("shop"))]); + let source = tempdir("folder"); + std::fs::write(source.join("README.md"), "live\n").unwrap(); + let init = |name: &str| { + ok( + &home, + "project.init", + &[ + ("project", t(&format!("shop/{name}"))), + ("folder", t(&source.display().to_string())), + ], + ) + }; + init("live"); + init("retired"); + let retired = ProjectRecord::read(&home, "shop", "retired") + .unwrap() + .expect("the retired entry"); + let retired_stack = hex(&retired.stack().expect("the entry's manifest stack")); + retire(&home, "shop", "retired", &retired_stack); + + // A home entry whose Store project is absent: a second home's stale + // record, or a failed home-entry removal. Its stack is not in the Store. + let absent = LayerStackId::from_authority([9; 16]).to_bytes(); + write_record( + &paths::project_record(&home, "shop", "absent"), + &SdkValue::map([ + ("name", t("absent")), + ("stack", t(&hex(&absent))), + ("created", SdkValue::Int(1)), + ]), + ) + .unwrap(); + + // Only the live project lists; the retired and absent entries do not. + let list = ok(&home, "project.list", &[]); + let SdkValue::List(items) = list.get("items").unwrap() else { + panic!("{list:?}") + }; + let names: Vec<&str> = items.iter().filter_map(|i| i.get_text("name")).collect(); + assert_eq!(names, vec!["live"]); + // the named listing intersects the same way + let one = ok(&home, "project.list", &[("store", t("shop"))]); + let SdkValue::List(items) = one.get("items").unwrap() else { + panic!("{one:?}") + }; + let names: Vec<&str> = items.iter().filter_map(|i| i.get_text("name")).collect(); + assert_eq!(names, vec!["live"]); + // both stale home entries are still there; the Store, not the home, hides them + for name in ["retired", "absent"] { + assert!(paths::project_record(&home, "shop", name).exists(), "{name}"); + } + + // project.get refuses a hidden project even with its home entry present, + // and an entry the Store does not hold. + for name in ["retired", "absent"] { + let (code, message) = call( + &home, + "project.get", + &[("project", t(&format!("shop/{name}")))], + ) + .unwrap_err(); + assert_eq!(code, "NoSuchProject", "{name}: {message}"); + } + let live = ok(&home, "project.get", &[("project", t("shop/live"))]); + assert_eq!(live.get_text("project"), Some("shop/live")); + + std::fs::remove_dir_all(home).unwrap(); + let _ = std::fs::remove_dir_all(source); +} diff --git a/api/crates/layerfs-api/tests/project_list_worker.rs b/api/crates/layerfs-api/tests/project_list_worker.rs new file mode 100644 index 0000000000..484c17b261 --- /dev/null +++ b/api/crates/layerfs-api/tests/project_list_worker.rs @@ -0,0 +1,296 @@ +//! project.list and project.get on a worker-backed Store (T4.5c): the +//! listing is the Store's own filtered one, read in the machine's worker +//! (the same `layerfs_project::store::projects` a local Store uses), so a +//! project retired there is not listed and is `NoSuchProject` from +//! project.get; a live project still lists and gets, and a worker's refusal +//! is the call's. +use layerfs_api::home::{self, paths, write_record}; +use layerfs_bridge::control::SdkValue; +use layerfs_bridge::sdk_values::hex; +use layerfs_history::LayerStackId; +use layerfs_machine::worker::{self, read_frame, read_frame_within, write_frame, Hello, MAX_HELLO}; +use layerfs_store::open_args::OpenArgs; +use layerfs_store::project::record::ProjectRecord; +use layerfs_store::record::StoreRecord; +use layerfs_store::service::with_service; +use layerfs_store_service::history::deletion::delete_stack; +use std::net::TcpListener; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; + +static NEXT: AtomicU64 = AtomicU64::new(0); + +fn tempdir(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "lfs-list-worker-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +fn t(s: &str) -> SdkValue { + SdkValue::text(s) +} + +fn call(home: &Path, name: &str, args: &[(&str, SdkValue)]) -> Result { + let args = SdkValue::Map( + args.iter() + .map(|(n, v)| ((*n).to_owned(), v.clone())) + .collect(), + ); + layerfs_api::call(Some(home), name, args, &mut |_| {}).map_err(|r| (r.code, r.message)) +} + +fn ok(home: &Path, name: &str, args: &[(&str, SdkValue)]) -> SdkValue { + call(home, name, args).unwrap_or_else(|e| panic!("{name}: {e:?}")) +} + +fn names(list: &SdkValue) -> Vec { + let SdkValue::List(items) = list.get("items").unwrap() else { + panic!("{list:?}") + }; + items + .iter() + .filter_map(|item| item.get_text("name")) + .map(str::to_owned) + .collect() +} + +/// Marks Store project `stack` retired (the `project.delete` write path, +/// T4.5b): the Store hides it, its home entry stays. +fn retire(home: &Path, store: &str, stack: &[u8]) { + let record = StoreRecord::read(home, store).unwrap().unwrap(); + let open = OpenArgs { + home: Some(home.to_path_buf()), + store: Some(store.to_owned()), + project: None, + store_path: Some(paths::store_file(home, store).display().to_string()), + store_record: Some(record.to_record().clone()), + postgresql: None, + }; + let stack = LayerStackId::from_slice(stack).unwrap(); + with_service(&open, |service| { + delete_stack(&service.store, stack, 1234).unwrap(); + Ok(SdkValue::Null) + }) + .unwrap(); +} + +/// Moves Store `store`'s file from `from`'s Store folder to `to`'s. +fn move_store_file(from: &Path, to: &Path, store: &str) { + let source = paths::store_dir(from, store); + let target = paths::store_dir(to, store); + home::private_dir(&target).unwrap(); + for entry in std::fs::read_dir(&source).unwrap() { + let entry = entry.unwrap(); + let name = entry.file_name().to_string_lossy().into_owned(); + if name.starts_with("store.sqlite") { + std::fs::rename(entry.path(), target.join(&name)).unwrap(); + } + } +} + +/// The Store record of `store` places it on machine `machine`. +fn place_store(home: &Path, store: &str, machine: &str) { + let record_path = paths::store_record(home, store); + let record = home::read_record(&record_path).unwrap().unwrap(); + let record = home::with_field(record, "machine", t(machine)); + write_record(&record_path, &record).unwrap(); +} + +/// A home whose Store `shop` holds `live` and `retired`, the latter retired +/// in the Store, and whose machine `box` record names `listener` as its +/// worker; `shop`'s calls run on `box`. With `root`, the Store's file is +/// moved under the worker's root. +fn shop_on_box(tag: &str, listener: &TcpListener, root: Option<&Path>) -> PathBuf { + let home = tempdir(tag); + std::fs::set_permissions(&home, std::fs::Permissions::from_mode(0o700)).unwrap(); + home::open(&home).unwrap(); + ok(&home, "store.create", &[("name", t("shop"))]); + let source = tempdir(&format!("{tag}-folder")); + std::fs::write(source.join("README.md"), "live\n").unwrap(); + for name in ["live", "retired"] { + ok( + &home, + "project.init", + &[ + ("project", t(&format!("shop/{name}"))), + ("folder", t(&source.display().to_string())), + ], + ); + } + let retired = ProjectRecord::read(&home, "shop", "retired") + .unwrap() + .expect("the retired entry"); + retire( + &home, + "shop", + &retired.stack().expect("the entry's manifest stack"), + ); + write_record( + &paths::machine_record(&home, "box"), + &SdkValue::map([("name", t("box")), ("platform", t("lima"))]), + ) + .unwrap(); + let address = listener.local_addr().unwrap().to_string(); + write_record( + &paths::worker_record(&home, "box"), + &SdkValue::map([("address", t(&address)), ("secret", t("k"))]), + ) + .unwrap(); + if let Some(root) = root { + move_store_file(&home, root, "shop"); + } + place_store(&home, "shop", "box"); + home +} + +type Seen = Arc>>; + +/// A fake worker: answers its secret, reads each request and answers +/// `reply`'s frame; the requests it read are shared back. +fn fake_worker( + listener: TcpListener, + reply: impl Fn(&SdkValue) -> SdkValue + Send + 'static, +) -> Seen { + let seen: Seen = Arc::default(); + let requests = seen.clone(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { + return; + }; + let Some(_hello) = read_frame_within(&mut stream, MAX_HELLO).ok().flatten() else { + continue; + }; + let ready = Hello { + protocol: Some(worker::PROTOCOL), + bundle: None, + }; + if write_frame(&mut stream, &ready.to_frame()).is_err() { + continue; + } + let Some(request) = read_frame(&mut stream).ok().flatten() else { + continue; + }; + let answer = reply(&request); + requests.lock().unwrap().push(request); + let _ = write_frame(&mut stream, &answer); + } + }); + seen +} + +#[test] +fn a_worker_s_filtered_listing_hides_a_retired_project() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let home = shop_on_box("fake-list", &listener, None); + let live = ProjectRecord::read(&home, "shop", "live") + .unwrap() + .expect("the live entry"); + let live_stack = hex(&live.stack().expect("the live entry's stack")); + let seen = fake_worker(listener, move |_| { + // The worker's answer envelope is `{ value }`; its value is the + // `project.list` reply. + SdkValue::map([( + "value", + SdkValue::map([ + ("kind", t("Listed")), + ( + "projects", + SdkValue::List(vec![SdkValue::map([ + ("name", t("live")), + ("stack_id", t(&live_stack)), + ])]), + ), + ]), + )]) + }); + + // The worker's listing names only `live`: the retired entry is hidden. + assert_eq!(names(&ok(&home, "project.list", &[])), vec!["live"]); + assert_eq!( + names(&ok(&home, "project.list", &[("store", t("shop"))])), + vec!["live"] + ); + // project.get refuses the retired entry and gets the live one. + let (code, message) = + call(&home, "project.get", &[("project", t("shop/retired"))]).unwrap_err(); + assert_eq!(code, "NoSuchProject", "{message}"); + let live = ok(&home, "project.get", &[("project", t("shop/live"))]); + assert_eq!(live.get_text("project"), Some("shop/live")); + // Both home entries are still there; the Store, not the home, hides one. + for name in ["live", "retired"] { + assert!( + paths::project_record(&home, "shop", name).exists(), + "{name}" + ); + } + + // Every read went to the worker, as the Store listing's own frame. + let requests = seen.lock().unwrap(); + assert_eq!(requests.len(), 4, "{requests:?}"); + for request in requests.iter() { + assert_eq!(request.get_text("op"), Some("project.list")); + let args = request.get("args").expect("args"); + assert_eq!(args.get_text("store"), Some("shop")); + assert!(args.get("store_record").is_some()); + } + drop(requests); + let _ = std::fs::remove_dir_all(home); +} + +#[test] +fn a_worker_refusal_is_the_calls_refusal() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let home = shop_on_box("fake-refusal", &listener, None); + let _seen = fake_worker(listener, |_| { + SdkValue::map([( + "refused", + SdkValue::map([ + ("code", t("Busy")), + ("message", t("the Store is held")), + ("detail", SdkValue::Null), + ]), + )]) + }); + + // project.get propagates the refusal, never the unfiltered record. + let (code, message) = call(&home, "project.get", &[("project", t("shop/live"))]).unwrap_err(); + assert_eq!(code, "Busy"); + assert_eq!(message, "the Store is held"); + // The named listing reports it; the all-Stores one omits the Store. + let (code, _) = call(&home, "project.list", &[("store", t("shop"))]).unwrap_err(); + assert_eq!(code, "Busy"); + let list = ok(&home, "project.list", &[]); + assert!(names(&list).is_empty(), "{list:?}"); + let _ = std::fs::remove_dir_all(home); +} + +#[test] +fn a_worker_store_lists_only_its_live_projects() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let root = tempdir("worker-root"); + let home = shop_on_box("real-list", &listener, Some(&root)); + let guest = root.clone(); + std::thread::spawn(move || { + layerfs_control::worker_server::serve(listener, "k".to_owned(), guest) + }); + + // The Store under the worker's root hides its retired project; the + // caller's home entry for it stays but is not listed. + assert_eq!(names(&ok(&home, "project.list", &[])), vec!["live"]); + let (code, message) = + call(&home, "project.get", &[("project", t("shop/retired"))]).unwrap_err(); + assert_eq!(code, "NoSuchProject", "{message}"); + let live = ok(&home, "project.get", &[("project", t("shop/live"))]); + assert_eq!(live.get_text("project"), Some("shop/live")); + assert!(paths::project_record(&home, "shop", "retired").exists()); + let _ = std::fs::remove_dir_all(home); + let _ = std::fs::remove_dir_all(root); +} diff --git a/api/crates/layerfs-cli/tests/concurrent_reads.rs b/api/crates/layerfs-cli/tests/concurrent_reads.rs index 9dc2db39d5..605eb7bc3a 100644 --- a/api/crates/layerfs-cli/tests/concurrent_reads.rs +++ b/api/crates/layerfs-cli/tests/concurrent_reads.rs @@ -16,7 +16,11 @@ fn parallel_branch_lists_never_trip_over_a_dropped_lease() { vec!["project", "init", "s/p", source.to_str().unwrap()], ] { let out = run(&mut layerfs(&home, &words)); - assert!(out.status.success(), "layerfs {words:?}: {}", text(&out.stderr)); + assert!( + out.status.success(), + "layerfs {words:?}: {}", + text(&out.stderr) + ); } for round in 0..6 { let children: Vec<_> = (0..8) diff --git a/control/crates/layerfs-control/src/project.rs b/control/crates/layerfs-control/src/project.rs index c3505c5d13..baee69b1eb 100644 --- a/control/crates/layerfs-control/src/project.rs +++ b/control/crates/layerfs-control/src/project.rs @@ -11,3 +11,10 @@ use std::path::Path; pub fn remove(home: &Path, events: &mut Events<'_>, project: &ProjectRef) -> Outcome { layerfs_store::project::family::remove(home, events, project, crate::store::RELATED) } + +/// `project.delete(project)`: retires the project in its Store and removes +/// the caller home entry (spec §4). Add the API and MCP call-table entry in +/// T11; this is the service/control layer behind that dispatch. +pub fn delete(home: &Path, events: &mut Events<'_>, project: &ProjectRef) -> Outcome { + layerfs_store::project::delete::delete(home, events, project) +} diff --git a/control/crates/layerfs-control/src/worker_server.rs b/control/crates/layerfs-control/src/worker_server.rs index aaa0206567..997d24c27e 100644 --- a/control/crates/layerfs-control/src/worker_server.rs +++ b/control/crates/layerfs-control/src/worker_server.rs @@ -10,8 +10,8 @@ //! build sends nothing more and closes. Then a request `{ op, args }`, read with at most //! [`REQUEST_WAIT`] between bytes. `worker.ping` answers who it is; //! `store.call`, `store.file`, `store.reach`, `sandbox.host`, `project.stage`, -//! `project.init`, `project.export` and `project.remove` run one operation on -//! this machine, +//! `project.init`, `project.export`, `project.remove`, `project.delete` and +//! `project.list` run one operation on this machine, //! with `home` set to the worker's own root (the machine's data disk), //! never a path the caller sends. Answers are //! `{ value }` or `{ refused }`, after any `{ event }` frames. @@ -28,9 +28,13 @@ use layerfs_home::outcome::{refused, Outcome}; use layerfs_machine::worker::{ read_frame, read_frame_within, write_frame, Hello, HELLO_WAIT, MAX_HELLO, }; +use layerfs_store::project::delete::delete_project; use layerfs_store::project::discovery::home_of; use layerfs_store::project::export::in_worker as export_in_worker; -use layerfs_store::project::frame::{ExportFrame, InitFrame, RemoveFrame, StageFrame}; +use layerfs_store::project::frame::{ + DeleteFrame, ExportFrame, InitFrame, ListingFrame, RemoveFrame, StageFrame, +}; +use layerfs_store::project::listing::list_projects; use layerfs_store::project::{init::in_worker, remove::remove_project, staged}; use std::net::{TcpListener, TcpStream}; use std::path::{Path, PathBuf}; @@ -274,6 +278,17 @@ fn run( let root = home_of(Some(root))?; remove_project(&root, &RemoveFrame::decode(&args)?) } + // retires one project in the Store under this worker root and + // removes the worker own entry (2.0, T4.5b): `{ store, stack_id, + // deleted_ms?, project?, store_record? }` + "project.delete" => { + let root = home_of(Some(root))?; + delete_project(&root, &DeleteFrame::decode(&args)?) + } + // reads the Store's own filtered project listing under this worker + // root, the same `layerfs_project::store::projects` a local Store + // uses (2.0, T4.5c): `{ store, store_record? }` + "project.list" => list_projects(root, &ListingFrame::decode(&args)?), // writes or discards one part of a caller's folder copy under this // worker's root (#536 H11a) "project.stage" => staged::run(root, &StageFrame::decode(args)?), diff --git a/control/crates/layerfs-control/tests/project_delete_call.rs b/control/crates/layerfs-control/tests/project_delete_call.rs new file mode 100644 index 0000000000..cc88aabe6b --- /dev/null +++ b/control/crates/layerfs-control/tests/project_delete_call.rs @@ -0,0 +1,17 @@ +//! `project.delete` is reachable at the control layer behind the call +//! dispatch T11 will add (the API/MCP call table itself is unchanged here). +use layerfs_home::events::Events; +use layerfs_home::names::ProjectRef; + +#[test] +fn project_delete_is_reachable_from_control_and_refuses_an_unknown_store() { + let home = std::env::temp_dir().join(format!("layerfs-control-delete-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&home); + layerfs_home::home::layout::create(&home).unwrap(); + let project = ProjectRef::parse("shop/app").unwrap(); + let mut sink = |_| {}; + let refused = + layerfs_control::project::delete(&home, &mut Events::new(&mut sink), &project).unwrap_err(); + assert_eq!(refused.code, "NoSuchStore"); + let _ = std::fs::remove_dir_all(&home); +} diff --git a/control/crates/layerfs-store/src/project/delete.rs b/control/crates/layerfs-store/src/project/delete.rs new file mode 100644 index 0000000000..dbf5dc9885 --- /dev/null +++ b/control/crates/layerfs-store/src/project/delete.rs @@ -0,0 +1,639 @@ +//! `project.delete`: a project is retired in its Store and its home entry is +//! removed (spec §4, §4.2, §5.1 and §10). +//! +//! The marker is the commit point. The caller first saves the legacy +//! own-project identity, then writes the durable intent +//! `store//delete/.json` with +//! `{store, project, stack_id, deleted_ms, attempt_ms, state}`, then marks the +//! Stack, then removes only its own home entry and unlinks the intent. A +//! retry reads the intent by the recorded Stack id, reuses its `deleted_ms` +//! and finishes the entry: `Marked` means this attempt marked a live Stack, +//! `Retired` means the marker was already there (including a hidden Store +//! project whose home entry is stale), and `Absent` means no live Stack is +//! there, so the stale entry goes. A lost worker reply is `Unknown`; the +//! intent stays and the retry classifies the Stack again. A home record whose +//! Stack differs from the intent is a project made again under the same name: +//! the stale intent is discarded and this call starts over against the record +//! that is there now. +//! +//! The worker op is `project.delete`; its frame is +//! `{store, stack_id, deleted_ms?, project?, store_record?}`. The two optional +//! context fields are the same ones `project.remove` sends: the worker opens +//! the Store under its own root with the caller's `store_record`, and the +//! project name lets its reply and its own home entry name the project. The +//! worker saves the legacy own-project identity, writes the marker, removes +//! its own entry and replies `{kind, project, stack_id, deleted_ms}`; the +//! caller removes its own entry and echoes `home_entry_removed`. +use crate::open_args::OpenArgs; +use crate::place::{self, Executor, Worker}; +use crate::project::frame::{DeleteFrame, DeleteReply}; +use crate::project::record::ProjectRecord; +use crate::record::{StoreRecord, StoreRef}; +use crate::{identity, lease, service}; +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_bridge::provision::StoreManifest; +use layerfs_bridge::sdk_values::{hex, unhex}; +use layerfs_history::LayerStackId; +use layerfs_home::events::Events; +use layerfs_home::home::{now_ms, paths}; +use layerfs_home::names::{ProjectRef, StoreName}; +use layerfs_home::outcome::{failed, refused, Outcome}; +use layerfs_store_service::history::deletion::{delete_stack, Deletion}; +use std::fs::{self, File, OpenOptions}; +use std::io::Write; +use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +type Result = std::result::Result; + +/// The state the durable intent records (`pending`, `marked`, `done`). +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum IntentState { + /// The identity is saved and the intent is durable; the marker may or may + /// not have been written (a lost reply leaves this state). + Pending, + /// The marker is committed. The home entry may still be there. + Marked, + /// The home entry was removed. The intent is unlinked next; a crash + /// between the two leaves this state for the retry to settle. + Done, +} + +impl IntentState { + /// The state's file and frame text. + pub fn as_str(self) -> &'static str { + match self { + Self::Pending => "pending", + Self::Marked => "marked", + Self::Done => "done", + } + } + + /// The state `text` names, if any. + pub fn parse(text: &str) -> Option { + match text { + "pending" => Some(Self::Pending), + "marked" => Some(Self::Marked), + "done" => Some(Self::Done), + _ => None, + } + } +} + +/// The durable delete intent at `store//delete/.json`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Intent { + /// The Store the project was in, as sent. + pub store: String, + /// The project's name in it, as sent. + pub project: String, + /// The LayerStack the home record held when the intent was written; a + /// retry resolves the Store by this id, never by name. + pub stack: LayerStackId, + /// The `deleted_ms` the marker holds, or is asked to hold. A retry reuses + /// this value; a `Retired` marker may replace it with its stored value. + pub deleted_ms: i64, + /// When the last attempt started, in milliseconds. + pub attempt_ms: i64, + /// Where the attempt got to. + pub state: IntentState, +} + +static NEXT_TEMPORARY: AtomicU64 = AtomicU64::new(0); + +impl Intent { + /// The intent's path in `home`. + pub fn path(home: &Path, store: &str, project: &str) -> PathBuf { + paths::store_dir(home, store) + .join("delete") + .join(format!("{project}.json")) + } + + /// The intent of project `project` of Store `store`, or `None`. + pub fn read(home: &Path, store: &str, project: &str) -> Result> { + let path = Self::path(home, store, project); + let Some(value) = layerfs_home::home::read_record(&path)? else { + return Ok(None); + }; + let malformed = || { + refused( + "InvalidInput", + format!("project.delete intent {} is malformed", path.display()), + ) + }; + let store = value.get_text("store").ok_or_else(malformed)?.to_owned(); + let project = value.get_text("project").ok_or_else(malformed)?.to_owned(); + let stack = value + .get_text("stack_id") + .and_then(unhex) + .and_then(|bytes| LayerStackId::from_slice(&bytes).ok()) + .ok_or_else(malformed)?; + let deleted_ms = value.get_int("deleted_ms").ok_or_else(malformed)?; + let attempt_ms = value.get_int("attempt_ms").ok_or_else(malformed)?; + let state = value + .get_text("state") + .and_then(IntentState::parse) + .ok_or_else(malformed)?; + Ok(Some(Self { + store, + project, + stack, + deleted_ms, + attempt_ms, + state, + })) + } + + /// Writes the intent whole: a private temporary file, fsync, one rename, + /// then the directory's fsync. A reader sees the old intent or the new + /// one, never a part of either. + pub fn write(home: &Path, intent: &Self) -> Result<()> { + let path = Self::path(home, &intent.store, &intent.project); + let parent = path + .parent() + .ok_or_else(|| refused("Failed", "a delete intent needs a folder"))?; + fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(parent) + .map_err(failed)?; + let unique = NEXT_TEMPORARY.fetch_add(1, Ordering::Relaxed); + let temporary = parent.join(format!( + ".{}.{}.{}.tmp", + intent.project, + std::process::id(), + unique + )); + let value = SdkValue::map([ + ("store", SdkValue::text(&intent.store)), + ("project", SdkValue::text(&intent.project)), + ("stack_id", SdkValue::text(hex(&intent.stack.to_bytes()))), + ("deleted_ms", SdkValue::Int(intent.deleted_ms)), + ("attempt_ms", SdkValue::Int(intent.attempt_ms)), + ("state", SdkValue::text(intent.state.as_str())), + ]); + let mut text = layerfs_home::json::print(&value); + text.push('\n'); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&temporary) + .map_err(failed)?; + let written = file + .write_all(text.as_bytes()) + .and_then(|()| file.sync_all()) + .map_err(failed); + drop(file); + let renamed = written.and_then(|()| fs::rename(&temporary, &path).map_err(failed)); + if let Err(refusal) = renamed { + let _ = fs::remove_file(&temporary); + return Err(refusal); + } + // the directory entry is durable too; a filesystem that cannot + // fsync a directory has already made the rename durable + let _ = File::open(parent).and_then(|folder| folder.sync_all()); + Ok(()) + } + + /// Removes the intent, if it is there. + pub fn unlink(home: &Path, store: &str, project: &str) -> Result<()> { + let path = Self::path(home, store, project); + match fs::remove_file(&path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(failed(error)), + } + if let Some(parent) = path.parent() { + let _ = File::open(parent).and_then(|folder| folder.sync_all()); + } + Ok(()) + } + + /// Drops an intent that no longer describes the home record (a project + /// made again under the same name). + pub fn discard(home: &Path, store: &str, project: &str) -> Result<()> { + Self::unlink(home, store, project) + } +} + +/// The Store side of one `project.delete` attempt and the caller's entry +/// removal, as one seam so the home path is testable without a Store. +pub trait Effects { + /// Saves the legacy own-project identity before the intent and the + /// marker; idempotent on a retry. + fn prepare(&mut self) -> Result<()>; + + /// Marks (or reclassifies) `stack`. A retry passes the intent's + /// `deleted_ms`. + fn mark(&mut self, stack: LayerStackId, deleted_ms: i64) -> Result; + + /// Removes the caller's home entry; `Ok(())` when nothing remains. + fn remove_entry(&mut self, entry: &Path) -> Result<()>; +} + +/// The `project` part of a Stack record from a home record: the `manifest`'s +/// Stack for a local sqlite record, the record's `stack` for a PostgreSQL or +/// worker record. +fn stack_of_value(value: &SdkValue) -> Option { + if let Some(SdkValue::Bytes(bytes)) = value.get("manifest") { + if let Ok(manifest) = StoreManifest::decode(bytes) { + if let Ok(stack) = LayerStackId::from_slice(&manifest.stack) { + return Some(stack); + } + } + } + value + .get_text("stack") + .and_then(unhex) + .and_then(|bytes| LayerStackId::from_slice(&bytes).ok()) +} + +/// The Stack a home record names, if it names one. +fn stack_of(record: &ProjectRecord) -> Option { + stack_of_value(record.to_record()) +} + +/// `{kind: "Removed", project, deleted_ms, home_entry_removed}` (spec §10; +/// `home_entry_removed: false` is still success, the marker being the commit +/// point). +fn removed(store: &str, project: &str, deleted_ms: i64, home_entry_removed: bool) -> SdkValue { + SdkValue::map([ + ("kind", SdkValue::text("Removed")), + ("project", SdkValue::text(format!("{store}/{project}"))), + ("deleted_ms", SdkValue::Int(deleted_ms)), + ("home_entry_removed", SdkValue::Bool(home_entry_removed)), + ]) +} + +/// Runs `project.delete` for a home entry whose Store side is `effects`. +/// +/// `stack` is the Stack id resolved from the record or the Store listing; a +/// record without one whose Store does not list it is a stale entry and is +/// removed without a marker. The same function serves the local and worker +/// routes, and a fake in tests. +pub fn delete_with( + home: &Path, + store: &str, + project: &str, + stack: Option, + effects: &mut impl Effects, +) -> Outcome { + let record = ProjectRecord::read(home, store, project)?; + let record_stack = record.as_ref().and_then(stack_of); + let mut intent = Intent::read(home, store, project)?; + + // A different Stack id for the same name is a project made again: the + // stale intent must not touch its entry, so it is discarded and this + // call starts over against the record that is there now. + if let (Some(stale), Some(current)) = (&intent, record_stack) { + if stale.stack != current { + Intent::discard(home, store, project)?; + intent = None; + } + } + + let (stack, mut deleted_ms) = match &intent { + Some(intent) => (intent.stack, intent.deleted_ms), + None => { + let Some(stack) = stack.or(record_stack) else { + if record.is_none() { + return Err(refused( + "NoSuchProject", + format!("no project {store}/{project}"), + )); + } + // An untracked stale entry: save a legacy identity before it + // goes, then remove it. There is no Stack id to mark. + effects.prepare()?; + let deleted_ms = now_ms(); + let removed_entry = effects + .remove_entry(&paths::project_dir(home, store, project)) + .is_ok(); + return Ok(removed(store, project, deleted_ms, removed_entry)); + }; + (stack, now_ms()) + } + }; + + if let Some(done) = &intent { + if done.state == IntentState::Done { + effects.prepare()?; + let removed_entry = effects + .remove_entry(&paths::project_dir(home, store, project)) + .is_ok(); + let mut settled = done.clone(); + settled.attempt_ms = now_ms(); + let _ = Intent::write(home, &settled); + let _ = Intent::unlink(home, store, project); + return Ok(removed(store, project, deleted_ms, removed_entry)); + } + } + + // The identity save comes before the intent, and the intent before the + // marker (spec §4). + effects.prepare()?; + let mut current = match intent { + Some(intent) => intent, + None => Intent { + store: store.to_owned(), + project: project.to_owned(), + stack, + deleted_ms, + attempt_ms: now_ms(), + state: IntentState::Pending, + }, + }; + if current.state == IntentState::Pending { + current.attempt_ms = now_ms(); + Intent::write(home, ¤t)?; + } + + let deletion = effects.mark(stack, deleted_ms)?; + deleted_ms = deletion.deleted_ms(deleted_ms); + current.deleted_ms = deleted_ms; + current.state = IntentState::Marked; + current.attempt_ms = now_ms(); + Intent::write(home, ¤t)?; + + let home_entry_removed = effects + .remove_entry(&paths::project_dir(home, store, project)) + .is_ok(); + if home_entry_removed { + current.state = IntentState::Done; + Intent::write(home, ¤t)?; + Intent::unlink(home, store, project)?; + } + Ok(removed(store, project, deleted_ms, home_entry_removed)) +} + +/// Removes a directory tree, treating an absent entry as removed. +fn remove_tree(entry: &Path) -> Result<()> { + match fs::remove_dir_all(entry) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(failed(format!("removing {}: {error}", entry.display()))), + } +} + +/// The Store-side opens of `project.delete` in this process. +struct Local { + home: PathBuf, + store: String, + project: String, + open: OpenArgs, +} + +impl Effects for Local { + fn prepare(&mut self) -> Result<()> { + if self.project != self.store || !identity::legacy(&self.home, &self.store)? { + return Ok(()); + } + let id = identity::identity(&self.open)?; + layerfs_home::home::private_dir(&identity::store_dir(&self.home, &self.store)?)?; + identity::save_record(&self.home, &self.store, &id) + } + + fn mark(&mut self, stack: LayerStackId, deleted_ms: i64) -> Result { + service::with_service(&self.open, |service| { + delete_stack(&service.store, stack, deleted_ms) + }) + } + + fn remove_entry(&mut self, entry: &Path) -> Result<()> { + remove_tree(entry) + } +} + +/// The Store-side opens of `project.delete` in a sqlite Store's worker. +struct Remote { + worker: Worker, + store: String, + project: String, + store_record: SdkValue, +} + +impl Effects for Remote { + fn prepare(&mut self) -> Result<()> { + // the worker saves its own legacy identity before its marker + Ok(()) + } + + fn mark(&mut self, stack: LayerStackId, deleted_ms: i64) -> Result { + let frame = DeleteFrame { + store: StoreName::parse(&self.store)?, + project: Some(self.project.clone()), + stack, + deleted_ms: Some(deleted_ms), + store_record: Some(self.store_record.clone()), + }; + let answer = self + .worker + .call("project.delete", frame.encode(), &mut |_| {})?; + let reply = DeleteReply::decode(&answer)?; + if reply.stack != stack { + return Err(refused( + "Failed", + "the worker answered for another LayerStack", + )); + } + Ok(Deletion::Marked { + deleted_ms: reply.deleted_ms, + }) + } + + fn remove_entry(&mut self, entry: &Path) -> Result<()> { + remove_tree(entry) + } +} + +/// The `OpenArgs` a local Store call opens with. `record` is absent for a +/// Store made by its own project Init, whose identity is the project entry. +fn open_args( + home: &Path, + store: &str, + project: &str, + record: Option<&StoreRecord>, +) -> Result { + let (store_path, postgresql) = match record { + Some(record) => match record.store_ref()? { + StoreRef::Postgres { .. } => ( + record.postgresql_key(home, store), + Some(crate::postgresql::connect::selection(record)?), + ), + StoreRef::Sqlite { .. } => (paths::store_file(home, store), None), + }, + None => (paths::store_file(home, store), None), + }; + Ok(OpenArgs { + home: Some(home.to_path_buf()), + store: Some(store.to_owned()), + project: Some(project.to_owned()), + store_path: Some(store_path.display().to_string()), + store_record: record.map(|record| record.to_record().clone()), + postgresql, + }) +} + +/// The Stack id a Store lists for project `name`, for a record that does +/// not name one (spec §4.2 fills it from the filtered listing by name). +fn listed_stack(open: &OpenArgs, name: &str) -> Result> { + service::with_service(open, |service| { + let stacks = service + .store + .published(layerfs_project::store::projects) + .map_err(failed)?; + Ok(stacks + .into_iter() + .find(|stack| stack.name.as_str() == name) + .map(|stack| stack.id)) + }) +} + +/// `project.delete(project)`: retires project `project` in its Store and +/// removes the caller's home entry. The marker is the commit point; a home +/// removal failure is success with `home_entry_removed: false`, and the +/// durable intent lets a same-home retry finish it. +pub fn delete(home: &Path, events: &mut Events<'_>, project: &ProjectRef) -> Outcome { + let _ = events; + let home = home.to_path_buf(); + let store = project.store.as_str(); + let name = project.project.as_str(); + let record = StoreRecord::read(&home, store)?; + let project_record = ProjectRecord::read(&home, store, name)?; + let stack = project_record.as_ref().and_then(stack_of); + match record { + Some(record) => { + record.unblocked()?; + let store_ref = record.store_ref()?.clone(); + // the caller holds the Store's lease while it sends the call + // there, as every call that runs in a worker does + let _lease = lease::hold(&home, store)?; + match place::plan(&home, &store_ref)? { + Executor::Local => { + let open = open_args(&home, store, name, Some(&record))?; + let stack = match stack { + Some(stack) => Some(stack), + None if project_record.is_some() => listed_stack(&open, name)?, + None => None, + }; + let mut effects = Local { + home: home.clone(), + store: store.to_owned(), + project: name.to_owned(), + open, + }; + delete_with(&home, store, name, stack, &mut effects) + } + Executor::Worker(worker) => { + if stack.is_none() && project_record.is_some() { + return Err(refused( + "Unsupported", + format!( + "project {store}/{name} has no recorded LayerStack id and its Store is on another machine; \ + list it on that machine first" + ), + )); + } + let mut effects = Remote { + worker, + store: store.to_owned(), + project: name.to_owned(), + store_record: record.to_record().clone(), + }; + delete_with(&home, store, name, stack, &mut effects) + } + } + } + None => { + // A Store made by its own project's Init has no Store record: its + // identity lives in the project entry, and saving it before that + // entry goes is exactly the legacy own-project identity save. + if name != store || project_record.is_none() { + return Err(refused("NoSuchStore", format!("no Store named {store}"))); + } + let open = open_args(&home, store, name, None)?; + let stack = match stack { + Some(stack) => Some(stack), + None => listed_stack(&open, name)?, + }; + let mut effects = Local { + home: home.clone(), + store: store.to_owned(), + project: name.to_owned(), + open, + }; + delete_with(&home, store, name, stack, &mut effects) + } + } +} + +/// The worker's `project.delete` on a Store under `root`: save the legacy +/// own-project identity, write the marker, remove the worker's own entry and +/// answer `{kind, project, stack_id, deleted_ms}`. +pub fn delete_project(root: &Path, frame: &DeleteFrame) -> Outcome { + let store = frame.store.as_str(); + let store_record = frame + .store_record + .clone() + .ok_or_else(|| refused("InvalidInput", "project.delete needs the Store's record"))?; + let open = OpenArgs { + home: Some(root.to_path_buf()), + store: Some(store.to_owned()), + project: frame.project.clone(), + store_path: Some(paths::store_file(root, store).display().to_string()), + store_record: Some(store_record), + postgresql: None, + }; + // The legacy own-project entry carries the Store's identity until the + // marker; save it first, idempotently. + let own = frame.project.as_deref() == Some(store) && identity::legacy(root, store)?; + if own { + let id = identity::identity(&open)?; + layerfs_home::home::private_dir(&identity::store_dir(root, store)?)?; + identity::save_record(root, store, &id)?; + } + let requested = frame.deleted_ms.unwrap_or_else(now_ms); + let deletion = service::with_service(&open, |service| { + delete_stack(&service.store, frame.stack, requested) + })?; + let deleted_ms = deletion.deleted_ms(requested); + let project = remove_worker_entry(root, store, frame.stack, frame.project.as_deref())? + .or_else(|| frame.project.clone()) + .unwrap_or_else(|| store.to_owned()); + Ok(DeleteReply { + project, + stack: frame.stack, + deleted_ms, + } + .encode()) +} + +/// Removes the worker's own home entry for `stack`; returns the project name +/// it removed, if any. +#[doc(hidden)] +pub fn remove_worker_entry( + root: &Path, + store: &str, + stack: LayerStackId, + hint: Option<&str>, +) -> Result> { + let names = match hint { + Some(name) => vec![name.to_owned()], + None => paths::names(&paths::store_dir(root, store).join("project")).map_err(failed)?, + }; + let mut removed = None; + for name in names { + let record = paths::project_record(root, store, &name); + let Some(value) = layerfs_home::home::read_record(&record)? else { + continue; + }; + let matches = stack_of_value(&value) == Some(stack) || name == store; + if matches { + remove_tree(&paths::project_dir(root, store, &name))?; + removed.get_or_insert(name); + } + } + Ok(removed) +} diff --git a/control/crates/layerfs-store/src/project/export/walk.rs b/control/crates/layerfs-store/src/project/export/walk.rs index d97f2d4dee..6cd6ab29eb 100644 --- a/control/crates/layerfs-store/src/project/export/walk.rs +++ b/control/crates/layerfs-store/src/project/export/walk.rs @@ -19,6 +19,7 @@ use layerfs_history::NodeId; use layerfs_home::outcome::refused; use layerfs_store_service::history::{lookup, MAIN}; use layerfs_store_service::read::cached::Cached; +use layerfs_store_service::read::visibility::Visibility; use layerfs_store_service::store::StorePorts; use layerfs_store_service::StoreService; use layerfs_telemetry::timer::Timing; @@ -45,6 +46,9 @@ pub(super) struct PinnedSnapshot { pub(super) node: String, pub(super) root: String, tree: TreeTable, + /// §4.3: the pinned node's marker visibility. Hidden entries are omitted + /// from the exported folder and from its counts. + visibility: Visibility, } /// Resolves `(project, branch)` once into an immutable `(NodeId, ObjectId, @@ -65,12 +69,21 @@ pub(super) fn pin_snapshot( let root = lookup::root(&service.store, node)?; let reader = service.store.ports(stack.scope); let tree = TreeTable::open(reader.as_ref(), FilesystemRootId(root)).map_err(lookup::content)?; + let serial = match service + .store + .published(|history| history.node_history_serial(node)) + { + Ok(Some(serial)) => serial, + _ => 0, + }; + let visibility = Visibility::for_stack(&service.store, stack.id, serial); Ok(PinnedSnapshot { reader, label, node: node_text(node), root: hex(root.as_bytes()), tree, + visibility, }) } @@ -163,6 +176,11 @@ pub(super) fn stream_snapshot( rel.push(b'/'); } rel.extend_from_slice(name.as_bytes()); + // A hidden entry is omitted; a hidden directory's subtree with it, as + // the read surface omits it and the marker hides everything below it. + if pinned.visibility.hidden_relative(&rel).is_some() { + continue; + } match value.kind { InodeKind::Directory => { counts.directories += 1; diff --git a/control/crates/layerfs-store/src/project/family.rs b/control/crates/layerfs-store/src/project/family.rs index 6ed7a2c2b6..af8d81619d 100644 --- a/control/crates/layerfs-store/src/project/family.rs +++ b/control/crates/layerfs-store/src/project/family.rs @@ -9,17 +9,19 @@ //! project's history from its Store. use crate::calls::Page; use crate::lease; -use crate::place::{self, Executor}; +use crate::open_args::OpenArgs; +use crate::place::{self, Executor, Worker}; use crate::project::add::{add_project, ProjectAdd}; use crate::project::discovery::home_of; -use crate::project::frame::{InitFrame, RemoveFrame}; +use crate::project::frame::{InitFrame, ListingFrame, ListingReply, RemoveFrame}; use crate::project::postgres_init::PgProjectAdd; use crate::project::record::ProjectRecord; use crate::record::{StoreRecord, StoreRef}; use crate::resolve; +use crate::service; use crate::sqlite::family::progress; use crate::Related; -use layerfs_bridge::control::SdkValue; +use layerfs_bridge::control::{SdkRefusal, SdkValue}; use layerfs_home::events::Events; use layerfs_home::home::paths; use layerfs_home::names::{ProjectRef, StoreName}; @@ -49,6 +51,90 @@ pub struct ProjectInit { pub timeout: Option, } +/// One project a Store's filtered listing names: its name and LayerStack. +struct LiveProject { + name: String, + stack: Vec, +} + +/// The live projects Store `store` lists, after the retired/hidden ones are +/// filtered out (spec §4.2). Every route reads the listing where the +/// Store's calls run (T4.5c): in this process for a sqlite Store on this +/// machine and for a PostgreSQL Store, in the machine's worker for a sqlite +/// Store on another machine. A Store that cannot be listed is `Err`. +fn live_projects( + home: &Path, + store: &str, + record: &StoreRecord, +) -> std::result::Result, SdkRefusal> { + let store_ref = record.store_ref()?; + record.unblocked()?; + if let Executor::Worker(mut worker) = place::plan(home, store_ref)? { + return live_projects_on_worker(&mut worker, store, record); + } + let (store_path, postgresql) = match store_ref { + StoreRef::Postgres { .. } => ( + record.postgresql_key(home, store), + Some(crate::postgresql::connect::selection(record)?), + ), + StoreRef::Sqlite { .. } => (paths::store_file(home, store), None), + }; + let open = OpenArgs { + home: Some(home.to_path_buf()), + store: Some(store.to_owned()), + store_path: Some(store_path.display().to_string()), + store_record: Some(record.to_record().clone()), + postgresql, + ..OpenArgs::default() + }; + let stacks = service::with_service(&open, |service| { + service + .store + .published(layerfs_project::store::projects) + .map_err(failed) + })?; + Ok(stacks + .into_iter() + .map(|stack| LiveProject { + name: stack.name.as_str().to_owned(), + stack: stack.id.to_bytes().to_vec(), + }) + .collect()) +} + +/// The listing a worker-backed Store sends (T4.5c): the worker runs the +/// same `layerfs_project::store::projects` under its own root, so a project +/// retired in the Store is hidden here as it is for a local one. +fn live_projects_on_worker( + worker: &mut Worker, + store: &str, + record: &StoreRecord, +) -> std::result::Result, SdkRefusal> { + let frame = ListingFrame { + store: StoreName::parse(store)?, + store_record: Some(record.to_record().clone()), + }; + let answer = worker.call("project.list", frame.encode(), &mut |_| {})?; + let reply = ListingReply::decode(&answer)?; + Ok(reply + .projects + .into_iter() + .map(|project| LiveProject { + name: project.name, + stack: project.stack.to_bytes().to_vec(), + }) + .collect()) +} + +/// Whether `project` is in `live`: by the stack id its record holds, or, +/// for a record without one, by its name (spec §4.2). +fn listed(project: &ProjectRecord, name: &str, live: &[LiveProject]) -> bool { + match project.stack() { + Some(stack) => live.iter().any(|item| item.stack == stack), + None => live.iter().any(|item| item.name == name), + } +} + /// One project as callers see it; a record that does not read shows no /// `created`. fn summary(home: &Path, store: &str, name: &str, record: &StoreRecord, view: View) -> SdkValue { @@ -58,9 +144,17 @@ fn summary(home: &Path, store: &str, name: &str, record: &StoreRecord, view: Vie project.to_sdk(view, record) } -/// `project.get(project)`: one project's record. +/// `project.get(project)`: one project's record. A project its Store no +/// longer lists (retired, or absent) is `NoSuchProject` even when its home +/// entry remains (spec §4.2). pub fn get(home: &Path, project: &ProjectRef) -> Outcome { let target = resolve::target(home, &project.to_string())?; + let live = live_projects(home, &target.store, &target.record)?; + let record = ProjectRecord::read(home, &target.store, &target.project)? + .ok_or_else(|| refused("NoSuchProject", format!("no project {project}")))?; + if !listed(&record, &target.project, &live) { + return Err(refused("NoSuchProject", format!("no project {project}"))); + } Ok(summary( home, &target.store, @@ -86,7 +180,26 @@ pub fn list(home: &Path, request: &ProjectList) -> Outcome { let Some(record) = StoreRecord::read(home, &store)? else { continue; }; + // The listing is the Store's own filtered one (spec §4.2), read + // where its calls run (T4.5c): a home entry whose project is hidden + // or absent is not listed. An all-Stores listing omits a Store it + // cannot list; a named one reports the refusal. + let live = match live_projects(home, &store, &record) { + Ok(live) => live, + Err(refusal) => { + if request.store.is_some() { + return Err(refusal); + } + continue; + } + }; for name in paths::names(&paths::store_dir(home, &store).join("project")).map_err(failed)? { + let Some(project) = ProjectRecord::read(home, &store, &name)? else { + continue; + }; + if !listed(&project, &name, &live) { + continue; + } items.push(summary(home, &store, &name, &record, View::List)); } } diff --git a/control/crates/layerfs-store/src/project/frame.rs b/control/crates/layerfs-store/src/project/frame.rs index 15850df28a..8de409d3a3 100644 --- a/control/crates/layerfs-store/src/project/frame.rs +++ b/control/crates/layerfs-store/src/project/frame.rs @@ -1,8 +1,11 @@ //! The `project` worker frames (layerfs-project#109): `project.init`, -//! `project.remove`, `project.stage` and `project.export`, each encoded by the caller and +//! `project.remove`, `project.list` (T4.5c), `project.stage` and +//! `project.export`, each encoded by the caller and //! decoded once in the worker, in the order the worker always refused them. //! The worker sets the home to its own root; a frame names none. use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_bridge::sdk_values::{hex, unhex}; +use layerfs_history::LayerStackId; use layerfs_home::helpers::required; use layerfs_home::names::StoreName; use layerfs_home::outcome::refused; @@ -93,6 +96,18 @@ fn staged_id(frame: &SdkValue) -> Result { Ok(id.to_owned()) } +/// The LayerStack identity a `stack_id` field holds: its 17 bytes in hex. +fn stack_id(text: &str) -> Result { + let bytes = + unhex(text).ok_or_else(|| refused("InvalidInput", "`stack_id` must be hex digits"))?; + LayerStackId::from_slice(&bytes).map_err(|_| { + refused( + "InvalidInput", + "`stack_id` must be a LayerStack identity (17 bytes)", + ) + }) +} + impl InitFrame { /// The `project.init` frame: `{ store, name, staged, timeoutMs, /// store_record }`, each sent when set. @@ -226,3 +241,194 @@ impl ExportFrame { }) } } + +/// `project.delete`: retire Stack `stack` of Store `store` (here, or in its +/// worker, spec §4). The caller writes `deleted_ms` to its durable intent +/// before the call and resends it on a retry; `project` and `store_record` +/// are the caller context for the worker own entry and its Store open. +#[derive(Clone, Debug)] +pub struct DeleteFrame { + /// The Store. + pub store: StoreName, + /// The project, as sent; the worker names its own entry with it. + pub project: Option, + /// The LayerStack id the caller recorded (never a name). + pub stack: LayerStackId, + /// The `deleted_ms` the marker should hold; `None` lets the worker take + /// the current time. + pub deleted_ms: Option, + /// The Store record. + pub store_record: Option, +} + +/// `project.delete` worker reply: `{kind: "Removed", project, stack_id, +/// deleted_ms}`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DeleteReply { + /// The project the worker own entry named (or the Store name). + pub project: String, + /// The LayerStack the marker names. + pub stack: LayerStackId, + /// The stored `deleted_ms` (a retry reuses it). + pub deleted_ms: i64, +} + +impl DeleteFrame { + /// The `project.delete` frame: `{ store, stack_id, deleted_ms?, project?, + /// store_record? }`, each optional field sent when set. + pub fn encode(&self) -> SdkValue { + let mut fields = vec![ + ("store", SdkValue::text(self.store.as_str())), + ("stack_id", SdkValue::text(hex(&self.stack.to_bytes()))), + ]; + fields.extend(self.deleted_ms.map(|ms| ("deleted_ms", SdkValue::Int(ms)))); + fields.extend( + self.project + .as_deref() + .map(|p| ("project", SdkValue::text(p))), + ); + fields.extend(self.store_record.clone().map(|r| ("store_record", r))); + map(fields) + } + + /// A `project.delete` frame: no Store, a bad Store name, no Stack id, or + /// a Stack id that is not a LayerStack identity are refused. + pub fn decode(frame: &SdkValue) -> Result { + let store = StoreName::parse(required(frame.get_text("store"), "store")?)?; + let stack = stack_id(required(frame.get_text("stack_id"), "stack_id")?)?; + Ok(Self { + store, + project: frame.get_text("project").map(str::to_owned), + stack, + deleted_ms: frame.get_int("deleted_ms"), + store_record: frame.get("store_record").cloned(), + }) + } +} + +impl DeleteReply { + /// The reply a worker sends. + pub fn encode(&self) -> SdkValue { + map(vec![ + ("kind", SdkValue::text("Removed")), + ("project", SdkValue::text(&self.project)), + ("stack_id", SdkValue::text(hex(&self.stack.to_bytes()))), + ("deleted_ms", SdkValue::Int(self.deleted_ms)), + ]) + } + + /// The reply fields; `kind` must be `Removed`. + pub fn decode(frame: &SdkValue) -> Result { + if frame.get_text("kind") != Some("Removed") { + return Err(refused( + "Failed", + "the worker project.delete reply is not `Removed`", + )); + } + let project = frame + .get_text("project") + .ok_or_else(|| refused("InvalidInput", "reply has no `project`"))? + .to_owned(); + let stack = stack_id(required(frame.get_text("stack_id"), "stack_id")?)?; + let deleted_ms = frame + .get_int("deleted_ms") + .ok_or_else(|| refused("InvalidInput", "reply has no `deleted_ms`"))?; + Ok(Self { + project, + stack, + deleted_ms, + }) + } +} + +/// `project.list` in a worker (T4.5c): Store `store`'s filtered project +/// listing, the same `layerfs_project::store::projects` a local or +/// PostgreSQL Store is read with, so a project retired in the Store is +/// hidden here too. A worker's root does not hold the caller's home +/// entries; the caller intersects this listing with them (`project.list`, +/// `project.get`). +#[derive(Clone, Debug)] +pub struct ListingFrame { + /// The Store to list. + pub store: StoreName, + /// The Store's record: it opens the Store under the worker's root. + pub store_record: Option, +} + +/// One project a worker's Store listing names. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ListedProject { + /// Its name in the Store. + pub name: String, + /// Its LayerStack. + pub stack: LayerStackId, +} + +/// `project.list` worker reply: `{ kind: "Listed", projects: [{ name, +/// stack_id }] }`, in the Store's name order. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ListingReply { + /// The Store's live projects. + pub projects: Vec, +} + +impl ListingFrame { + /// The `project.list` frame: `{ store, store_record? }`. + pub fn encode(&self) -> SdkValue { + let mut fields = vec![("store", SdkValue::text(self.store.as_str()))]; + fields.extend(self.store_record.clone().map(|r| ("store_record", r))); + map(fields) + } + + /// A `project.list` frame: no Store, then a bad Store name are refused. + pub fn decode(frame: &SdkValue) -> Result { + Ok(Self { + store: StoreName::parse(required(frame.get_text("store"), "store")?)?, + store_record: frame.get("store_record").cloned(), + }) + } +} + +impl ListingReply { + /// The reply a worker sends. + pub fn encode(&self) -> SdkValue { + let projects = self + .projects + .iter() + .map(|project| { + map(vec![ + ("name", SdkValue::text(&project.name)), + ("stack_id", SdkValue::text(hex(&project.stack.to_bytes()))), + ]) + }) + .collect(); + map(vec![ + ("kind", SdkValue::text("Listed")), + ("projects", SdkValue::List(projects)), + ]) + } + + /// The reply fields; `kind` must be `Listed`, and every project names + /// its Store name and its LayerStack identity. + pub fn decode(frame: &SdkValue) -> Result { + if frame.get_text("kind") != Some("Listed") { + return Err(refused( + "Failed", + "the worker project.list reply is not `Listed`", + )); + } + let Some(SdkValue::List(items)) = frame.get("projects") else { + return Err(refused("InvalidInput", "reply has no `projects` list")); + }; + let mut projects = Vec::with_capacity(items.len()); + for item in items { + let name = item + .get_text("name") + .ok_or_else(|| refused("InvalidInput", "reply project has no `name`"))? + .to_owned(); + let stack = stack_id(required(item.get_text("stack_id"), "stack_id")?)?; + projects.push(ListedProject { name, stack }); + } + Ok(Self { projects }) + } +} diff --git a/control/crates/layerfs-store/src/project/listing.rs b/control/crates/layerfs-store/src/project/listing.rs new file mode 100644 index 0000000000..99f3f2ac52 --- /dev/null +++ b/control/crates/layerfs-store/src/project/listing.rs @@ -0,0 +1,45 @@ +//! `project.list` in a worker (T4.5c): the Store's filtered project +//! listing, read under the worker's own root with the same +//! `layerfs_project::store::projects` a local or PostgreSQL Store uses, so +//! a project retired in the Store is hidden on that route too. +use crate::open_args::OpenArgs; +use crate::project::frame::{ListedProject, ListingFrame, ListingReply}; +use crate::service; +use layerfs_home::home::paths; +use layerfs_home::outcome::{failed, refused, Outcome}; +use std::path::Path; + +/// The `project.list` worker op: Store `store`'s live projects, in name +/// order, under `root` (the worker's own root). The caller intersects the +/// answer with its own home entries (`project.list`, `project.get`). +pub fn list_projects(root: &Path, frame: &ListingFrame) -> Outcome { + let store = frame.store.as_str(); + let store_record = frame + .store_record + .clone() + .ok_or_else(|| refused("InvalidInput", "project.list needs the Store's record"))?; + let open = OpenArgs { + home: Some(root.to_path_buf()), + store: Some(store.to_owned()), + store_path: Some(paths::store_file(root, store).display().to_string()), + store_record: Some(store_record), + postgresql: None, + ..OpenArgs::default() + }; + let stacks = service::with_service(&open, |service| { + service + .store + .published(layerfs_project::store::projects) + .map_err(failed) + })?; + Ok(ListingReply { + projects: stacks + .into_iter() + .map(|stack| ListedProject { + name: stack.name.as_str().to_owned(), + stack: stack.id, + }) + .collect(), + } + .encode()) +} diff --git a/control/crates/layerfs-store/src/project/mod.rs b/control/crates/layerfs-store/src/project/mod.rs index 127b636a79..65597449ed 100644 --- a/control/crates/layerfs-store/src/project/mod.rs +++ b/control/crates/layerfs-store/src/project/mod.rs @@ -1,12 +1,14 @@ //! Public Project organization over host Init, and a project's home entry //! and discovery records (`discovery`). pub mod add; +pub mod delete; pub mod discovery; pub mod export; #[doc(hidden)] pub mod family; pub mod frame; pub mod init; +pub mod listing; pub mod parts; pub mod postgres_init; #[doc(hidden)] diff --git a/control/crates/layerfs-store/src/project/record.rs b/control/crates/layerfs-store/src/project/record.rs index 02578ba974..7bddbd071b 100644 --- a/control/crates/layerfs-store/src/project/record.rs +++ b/control/crates/layerfs-store/src/project/record.rs @@ -4,6 +4,8 @@ //! `sandbox/files.rs` reads and writes with the rest kept. use crate::record::StoreRecord; use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_bridge::provision::StoreManifest; +use layerfs_bridge::sdk_values::unhex; use layerfs_home::home::{now_ms, paths, read_record, with_field, write_entity}; use layerfs_home::outcome::View; use std::path::Path; @@ -76,6 +78,19 @@ impl ProjectRecord { self.fields.get("created") } + /// The project's LayerStack identity as its record holds it (S §4.2): + /// the `manifest`'s stack for a local SQLite record, the record's + /// `stack` for a PostgreSQL or worker record. A record with neither has + /// none; its Store's filtered listing is then matched by name. + pub fn stack(&self) -> Option> { + if let Some(SdkValue::Bytes(bytes)) = self.fields.get("manifest") { + if let Ok(manifest) = StoreManifest::decode(bytes) { + return Some(manifest.stack.to_vec()); + } + } + self.fields.get_text("stack").and_then(unhex) + } + /// The record as written. pub fn to_record(&self) -> &SdkValue { &self.fields diff --git a/control/crates/layerfs-store/tests/project_delete.rs b/control/crates/layerfs-store/tests/project_delete.rs new file mode 100644 index 0000000000..43a30eab48 --- /dev/null +++ b/control/crates/layerfs-store/tests/project_delete.rs @@ -0,0 +1,414 @@ +//! `project.delete`'s home path (T4.5b): the durable intent before the marker, +//! the retry classifications (`Marked`, `Retired`, `Absent`, a hidden Store +//! project), the `home_entry_removed` echo, and the worker's own entry. A fake +//! `Effects` stands in for the Store so the home state machine is tested +//! without opening one. +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_bridge::sdk_values::hex; +use layerfs_history::LayerStackId; +use layerfs_home::home::{paths, write_record}; +use layerfs_home::outcome::{failed, refused}; +use layerfs_store::project::delete::{ + delete_with, remove_worker_entry, Effects, Intent, IntentState, +}; +use layerfs_store_service::history::deletion::Deletion; +use std::collections::VecDeque; +use std::path::{Path, PathBuf}; + +fn t(text: &str) -> SdkValue { + SdkValue::text(text) +} + +fn stack(body: u8) -> LayerStackId { + LayerStackId::from_authority([body; 16]) +} + +fn home(tag: &str) -> PathBuf { + let directory = std::env::temp_dir().join(format!( + "layerfs-project-delete-{tag}-{}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&directory); + layerfs_home::home::layout::create(&directory).unwrap(); + directory +} + +fn project_record(home: &Path, store: &str, project: &str, stack: LayerStackId) { + write_record( + &paths::project_record(home, store, project), + &SdkValue::map([ + ("v", SdkValue::Int(1)), + ("name", t(project)), + ("stack", t(&hex(&stack.to_bytes()))), + ("created", SdkValue::Int(1)), + ]), + ) + .unwrap(); +} + +fn intent(_home: &Path, store: &str, project: &str, stack: LayerStackId, id: i64) -> Intent { + Intent { + store: store.to_owned(), + project: project.to_owned(), + stack, + deleted_ms: id, + attempt_ms: 1, + state: IntentState::Pending, + } +} + +/// A fake Store side and entry remover. `mark` reads the intent as the +/// marker would see it, so a test can see the state that preceded it. +struct Fake { + home: PathBuf, + store: String, + project: String, + outcomes: VecDeque, + seen: Vec<(LayerStackId, i64)>, + states: Vec, + prepared: usize, + fail_removal: bool, +} + +impl Fake { + fn new(home: &Path, store: &str, project: &str, outcomes: Vec) -> Self { + Self { + home: home.to_path_buf(), + store: store.to_owned(), + project: project.to_owned(), + outcomes: outcomes.into(), + seen: Vec::new(), + states: Vec::new(), + prepared: 0, + fail_removal: false, + } + } + + fn failing(mut self) -> Self { + self.fail_removal = true; + self + } +} + +impl Effects for Fake { + fn prepare(&mut self) -> Result<(), SdkRefusal> { + self.prepared += 1; + Ok(()) + } + + fn mark(&mut self, stack: LayerStackId, deleted_ms: i64) -> Result { + self.seen.push((stack, deleted_ms)); + let state = Intent::read(&self.home, &self.store, &self.project) + .unwrap() + .expect("the intent is durable before the marker") + .state; + self.states.push(state); + Ok(self + .outcomes + .pop_front() + .expect("one fake outcome per marker attempt")) + } + + fn remove_entry(&mut self, entry: &Path) -> Result<(), SdkRefusal> { + if self.fail_removal { + return Err(refused("Failed", "test removal failure")); + } + match std::fs::remove_dir_all(entry) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(failed(error)), + } + } +} + +#[test] +fn a_fresh_delete_marks_then_removes_the_entry_and_unlinks_the_intent() { + let home = home("fresh"); + let (store, project, stack) = ("shop", "app", stack(1)); + project_record(&home, store, project, stack); + let mut fake = Fake::new( + &home, + store, + project, + vec![Deletion::Marked { deleted_ms: 777 }], + ); + let outcome = delete_with(&home, store, project, Some(stack), &mut fake).unwrap(); + assert_eq!(outcome.get_text("kind"), Some("Removed")); + assert_eq!(outcome.get_text("project"), Some("shop/app")); + assert_eq!(outcome.get_int("deleted_ms"), Some(777)); + assert!(outcome.get_bool("home_entry_removed")); + assert_eq!(fake.seen.len(), 1); + assert_eq!(fake.seen[0].0, stack); + // the identity save precedes the intent, and the intent precedes the mark + assert_eq!(fake.prepared, 1); + assert_eq!(fake.states, [IntentState::Pending]); + assert!(!paths::project_dir(&home, store, project).exists()); + assert_eq!(Intent::read(&home, store, project).unwrap(), None); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn a_retired_retry_reuses_the_intent_deleted_ms_and_finishes_the_entry() { + let home = home("retired"); + let (store, project, stack) = ("shop", "app", stack(2)); + project_record(&home, store, project, stack); + Intent::write(&home, &intent(&home, store, project, stack, 111)).unwrap(); + let mut fake = Fake::new( + &home, + store, + project, + vec![Deletion::Retired { deleted_ms: 222 }], + ); + let outcome = delete_with(&home, store, project, Some(stack), &mut fake).unwrap(); + assert_eq!(fake.seen, [(stack, 111)]); + assert_eq!(outcome.get_int("deleted_ms"), Some(222)); + assert!(outcome.get_bool("home_entry_removed")); + assert!(!paths::project_dir(&home, store, project).exists()); + assert_eq!(Intent::read(&home, store, project).unwrap(), None); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn a_live_row_with_a_live_intent_redoes_the_marker_then_finishes() { + let home = home("marked"); + let (store, project, stack) = ("shop", "app", stack(3)); + project_record(&home, store, project, stack); + Intent::write(&home, &intent(&home, store, project, stack, 111)).unwrap(); + let mut fake = Fake::new( + &home, + store, + project, + vec![Deletion::Marked { deleted_ms: 111 }], + ); + let outcome = delete_with(&home, store, project, Some(stack), &mut fake).unwrap(); + // the live row has no marker, so the marker is redone with the same ms + assert_eq!(fake.seen, [(stack, 111)]); + assert_eq!(fake.states, [IntentState::Pending]); + assert_eq!(outcome.get_int("deleted_ms"), Some(111)); + assert!(outcome.get_bool("home_entry_removed")); + assert!(!paths::project_dir(&home, store, project).exists()); + assert_eq!(Intent::read(&home, store, project).unwrap(), None); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn an_absent_stack_finishes_a_stale_entry() { + let home = home("absent"); + let (store, project, stack) = ("shop", "app", stack(4)); + project_record(&home, store, project, stack); + Intent::write(&home, &intent(&home, store, project, stack, 111)).unwrap(); + let mut fake = Fake::new(&home, store, project, vec![Deletion::Absent]); + let outcome = delete_with(&home, store, project, Some(stack), &mut fake).unwrap(); + assert_eq!(fake.seen, [(stack, 111)]); + assert_eq!(outcome.get_int("deleted_ms"), Some(111)); + assert!(outcome.get_bool("home_entry_removed")); + assert!(!paths::project_dir(&home, store, project).exists()); + assert_eq!(Intent::read(&home, store, project).unwrap(), None); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn a_hidden_store_project_still_has_its_stale_entry_removed() { + let home = home("hidden"); + let (store, project, stack) = ("shop", "app", stack(5)); + project_record(&home, store, project, stack); + // The Store listing filters the retired project out; the Store side + // classifies the retry by the recorded stack id as Retired. + let mut fake = Fake::new( + &home, + store, + project, + vec![Deletion::Retired { deleted_ms: 333 }], + ); + let outcome = delete_with(&home, store, project, Some(stack), &mut fake).unwrap(); + assert_eq!(outcome.get_int("deleted_ms"), Some(333)); + assert!(outcome.get_bool("home_entry_removed")); + assert!(!paths::project_dir(&home, store, project).exists()); + assert_eq!(Intent::read(&home, store, project).unwrap(), None); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn a_home_removal_failure_is_success_false_and_leaves_a_marked_intent() { + let home = home("residue"); + let (store, project, stack) = ("shop", "app", stack(6)); + project_record(&home, store, project, stack); + let mut fake = Fake::new( + &home, + store, + project, + vec![Deletion::Marked { deleted_ms: 44 }], + ) + .failing(); + let outcome = delete_with(&home, store, project, Some(stack), &mut fake).unwrap(); + assert_eq!(outcome.get_text("kind"), Some("Removed")); + assert_eq!(outcome.get_int("deleted_ms"), Some(44)); + assert!(!outcome.get_bool("home_entry_removed")); + // the marker is the commit point; the entry and intent stay for the retry + assert!(paths::project_dir(&home, store, project).is_dir()); + let left = Intent::read(&home, store, project).unwrap().unwrap(); + assert_eq!(left.state, IntentState::Marked); + assert_eq!(left.deleted_ms, 44); + + // a same-home retry reuses the stored ms, finishes the entry and returns + // `home_entry_removed: true` + let mut retry = Fake::new( + &home, + store, + project, + vec![Deletion::Retired { deleted_ms: 44 }], + ); + let outcome = delete_with(&home, store, project, Some(stack), &mut retry).unwrap(); + assert_eq!(retry.seen, [(stack, 44)]); + assert_eq!(outcome.get_int("deleted_ms"), Some(44)); + assert!(outcome.get_bool("home_entry_removed")); + assert!(!paths::project_dir(&home, store, project).exists()); + assert_eq!(Intent::read(&home, store, project).unwrap(), None); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn a_done_intent_settles_the_entry_without_a_second_marker() { + let home = home("done"); + let (store, project, stack) = ("shop", "app", stack(7)); + project_record(&home, store, project, stack); + let mut done = intent(&home, store, project, stack, 44); + done.state = IntentState::Done; + Intent::write(&home, &done).unwrap(); + let mut fake = Fake::new(&home, store, project, Vec::new()); + let outcome = delete_with(&home, store, project, Some(stack), &mut fake).unwrap(); + assert!(fake.seen.is_empty(), "a done intent marks nothing"); + assert_eq!(outcome.get_int("deleted_ms"), Some(44)); + assert!(outcome.get_bool("home_entry_removed")); + assert_eq!(Intent::read(&home, store, project).unwrap(), None); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn a_recreated_project_discards_the_stale_intent_and_starts_over() { + let home = home("recreated"); + let (store, project, old, new) = ("shop", "app", stack(8), stack(9)); + project_record(&home, store, project, new); + Intent::write(&home, &intent(&home, store, project, old, 111)).unwrap(); + let mut fake = Fake::new( + &home, + store, + project, + vec![Deletion::Marked { deleted_ms: 555 }], + ); + let outcome = delete_with(&home, store, project, Some(new), &mut fake).unwrap(); + assert_eq!(fake.seen.len(), 1); + assert_eq!(fake.seen[0].0, new, "the record that is there now wins"); + assert_ne!(fake.seen[0].1, 111, "the stale deleted_ms is not reused"); + assert_eq!(outcome.get_int("deleted_ms"), Some(555)); + assert!(outcome.get_bool("home_entry_removed")); + assert_eq!(Intent::read(&home, store, project).unwrap(), None); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn an_untracked_record_with_no_stack_is_removed_without_a_marker() { + let home = home("untracked"); + let (store, project) = ("shop", "app"); + write_record( + &paths::project_record(&home, store, project), + &SdkValue::map([("v", SdkValue::Int(1)), ("name", t(project))]), + ) + .unwrap(); + let mut fake = Fake::new(&home, store, project, Vec::new()); + let outcome = delete_with(&home, store, project, None, &mut fake).unwrap(); + assert!(fake.seen.is_empty()); + assert_eq!(fake.prepared, 1, "a legacy identity is saved first"); + assert!(outcome.get_bool("home_entry_removed")); + assert!(!paths::project_dir(&home, store, project).exists()); + let _ = std::fs::remove_dir_all(&home); + + // no record and no intent is NoSuchProject + let refusal = delete_with(&home, "shop", "gone", None, &mut fake).unwrap_err(); + assert_eq!(refusal.code, "NoSuchProject"); +} + +#[test] +fn the_intent_moves_pending_marked_done_and_unlinks() { + let home = home("intent"); + let (store, project, stack) = ("shop", "app", stack(10)); + let mut value = intent(&home, store, project, stack, 5); + Intent::write(&home, &value).unwrap(); + let path = Intent::path(&home, store, project); + assert!(path.is_file()); + assert_eq!( + Intent::read(&home, store, project).unwrap().unwrap().state, + IntentState::Pending + ); + // no temporary file is left beside it + let mut names: Vec = std::fs::read_dir(path.parent().unwrap()) + .unwrap() + .map(|entry| entry.unwrap().file_name().into_string().unwrap()) + .collect(); + names.sort(); + assert_eq!(names, [format!("{project}.json")]); + + value.state = IntentState::Marked; + value.deleted_ms = 6; + Intent::write(&home, &value).unwrap(); + let read = Intent::read(&home, store, project).unwrap().unwrap(); + assert_eq!((read.state, read.deleted_ms), (IntentState::Marked, 6)); + + value.state = IntentState::Done; + Intent::write(&home, &value).unwrap(); + assert_eq!( + Intent::read(&home, store, project).unwrap().unwrap().state, + IntentState::Done + ); + Intent::unlink(&home, store, project).unwrap(); + assert_eq!(Intent::read(&home, store, project).unwrap(), None); + // unlinking an absent intent is not an error + Intent::unlink(&home, store, project).unwrap(); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn the_worker_removes_only_its_entry_for_the_recorded_stack() { + let root = home("worker-entry"); + let (store, first) = ("shop", stack(11)); + project_record(&root, store, "app", first); + project_record(&root, store, "other", stack(12)); + // the hint names the caller's project; the other entry stays + let removed = remove_worker_entry(&root, store, first, Some("app")).unwrap(); + assert_eq!(removed.as_deref(), Some("app")); + assert!(!paths::project_dir(&root, store, "app").exists()); + assert!(paths::project_dir(&root, store, "other").is_dir()); + // without a hint the recorded stack id finds the entry + let removed = remove_worker_entry(&root, store, stack(12), None).unwrap(); + assert_eq!(removed.as_deref(), Some("other")); + assert!(!paths::project_dir(&root, store, "other").exists()); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn a_worker_reply_names_the_stack_and_deleted_ms() { + use layerfs_store::project::frame::DeleteReply; + let first = stack(13); + let reply = DeleteReply { + project: "app".to_owned(), + stack: first, + deleted_ms: 42, + }; + let encoded = reply.encode(); + assert_eq!(encoded.get_text("kind"), Some("Removed")); + assert_eq!( + encoded.get_text("stack_id"), + Some(hex(&first.to_bytes()).as_str()) + ); + assert_eq!( + DeleteReply::decode(&encoded).unwrap(), + reply, + "the worker reply round-trips" + ); + // a reply for a different Stack is detectable + let other = DeleteReply { + stack: stack(14), + ..reply.clone() + }; + assert_ne!(DeleteReply::decode(&other.encode()).unwrap().stack, first); +} diff --git a/control/crates/layerfs-store/tests/project_delete_frames.rs b/control/crates/layerfs-store/tests/project_delete_frames.rs new file mode 100644 index 0000000000..6d5b05e429 --- /dev/null +++ b/control/crates/layerfs-store/tests/project_delete_frames.rs @@ -0,0 +1,107 @@ +//! The `project.delete` worker frame and its reply: the map the caller +//! builds, the fields the worker decodes, and the reply it sends back. +use layerfs_bridge::control::SdkValue; +use layerfs_bridge::sdk_values::hex; +use layerfs_history::LayerStackId; +use layerfs_home::names::StoreName; +use layerfs_store::project::frame::{DeleteFrame, DeleteReply}; + +fn t(text: &str) -> SdkValue { + SdkValue::text(text) +} + +fn map(fields: Vec<(&str, SdkValue)>) -> SdkValue { + SdkValue::Map(fields.into_iter().map(|(n, v)| (n.to_owned(), v)).collect()) +} + +fn stack(body: u8) -> LayerStackId { + LayerStackId::from_authority([body; 16]) +} + +#[test] +fn the_frame_names_the_store_stack_and_optional_context() { + let stack = stack(1); + let record = map(vec![("name", t("shop")), ("kind", t("sqlite"))]); + let frame = DeleteFrame { + store: StoreName::parse("shop").unwrap(), + project: Some("app".to_owned()), + stack, + deleted_ms: Some(5), + store_record: Some(record.clone()), + }; + let encoded = frame.encode(); + assert_eq!( + encoded, + map(vec![ + ("store", t("shop")), + ("stack_id", t(&hex(&stack.to_bytes()))), + ("deleted_ms", SdkValue::Int(5)), + ("project", t("app")), + ("store_record", record), + ]) + ); + let decoded = DeleteFrame::decode(&encoded).unwrap(); + assert_eq!(decoded.store, frame.store); + assert_eq!(decoded.project, frame.project); + assert_eq!(decoded.stack, stack); + assert_eq!(decoded.deleted_ms, Some(5)); + assert_eq!(decoded.store_record, frame.store_record); +} + +#[test] +fn a_frame_with_only_the_store_and_stack_decodes() { + let stack = stack(2); + let minimal = map(vec![ + ("store", t("shop")), + ("stack_id", t(&hex(&stack.to_bytes()))), + ]); + let decoded = DeleteFrame::decode(&minimal).unwrap(); + assert_eq!((decoded.project, decoded.deleted_ms), (None, None)); + assert_eq!(decoded.store_record, None); + assert_eq!(decoded.stack, stack); +} + +#[test] +fn the_worker_refuses_a_bad_frame_in_order() { + let decode = |fields| DeleteFrame::decode(&map(fields)).unwrap_err().message; + assert_eq!(decode(vec![]), "missing text argument `store`"); + assert!(decode(vec![("store", t("../x"))]).starts_with("store name")); + assert_eq!( + decode(vec![("store", t("shop"))]), + "missing text argument `stack_id`" + ); + assert_eq!( + decode(vec![("store", t("shop")), ("stack_id", t("zz"))]), + "`stack_id` must be hex digits" + ); + assert_eq!( + decode(vec![("store", t("shop")), ("stack_id", t("ab"))]), + "`stack_id` must be a LayerStack identity (17 bytes)" + ); +} + +#[test] +fn the_reply_round_trips_and_requires_removed() { + let stack = stack(3); + let reply = DeleteReply { + project: "app".to_owned(), + stack, + deleted_ms: 42, + }; + let encoded = reply.encode(); + assert_eq!( + encoded, + map(vec![ + ("kind", t("Removed")), + ("project", t("app")), + ("stack_id", t(&hex(&stack.to_bytes()))), + ("deleted_ms", SdkValue::Int(42)), + ]) + ); + assert_eq!(DeleteReply::decode(&encoded).unwrap(), reply); + let other = map(vec![("kind", t("UpToDate"))]); + assert_eq!( + DeleteReply::decode(&other).unwrap_err().message, + "the worker project.delete reply is not `Removed`" + ); +} diff --git a/control/crates/layerfs-store/tests/project_list_frames.rs b/control/crates/layerfs-store/tests/project_list_frames.rs new file mode 100644 index 0000000000..344f1ee3f8 --- /dev/null +++ b/control/crates/layerfs-store/tests/project_list_frames.rs @@ -0,0 +1,122 @@ +//! The `project.list` worker frame and its reply (T4.5c): the map the +//! caller builds, the fields the worker decodes, and the filtered listing +//! it sends back. +use layerfs_bridge::control::SdkValue; +use layerfs_bridge::sdk_values::hex; +use layerfs_history::LayerStackId; +use layerfs_home::names::StoreName; +use layerfs_store::project::frame::{ListedProject, ListingFrame, ListingReply}; + +fn t(text: &str) -> SdkValue { + SdkValue::text(text) +} + +fn map(fields: Vec<(&str, SdkValue)>) -> SdkValue { + SdkValue::Map(fields.into_iter().map(|(n, v)| (n.to_owned(), v)).collect()) +} + +fn stack(body: u8) -> LayerStackId { + LayerStackId::from_authority([body; 16]) +} + +#[test] +fn the_frame_names_the_store_and_carries_its_record() { + let record = map(vec![("name", t("shop")), ("kind", t("sqlite"))]); + let frame = ListingFrame { + store: StoreName::parse("shop").unwrap(), + store_record: Some(record.clone()), + }; + let encoded = frame.encode(); + assert_eq!( + encoded, + map(vec![("store", t("shop")), ("store_record", record.clone())]) + ); + let decoded = ListingFrame::decode(&encoded).unwrap(); + assert_eq!(decoded.store, frame.store); + assert_eq!(decoded.store_record, Some(record)); +} + +#[test] +fn a_frame_with_only_the_store_decodes() { + let decoded = ListingFrame::decode(&map(vec![("store", t("shop"))])).unwrap(); + assert_eq!(decoded.store.as_str(), "shop"); + assert_eq!(decoded.store_record, None); +} + +#[test] +fn the_worker_refuses_a_bad_frame_in_order() { + let decode = |fields| ListingFrame::decode(&map(fields)).unwrap_err().message; + assert_eq!(decode(vec![]), "missing text argument `store`"); + assert!(decode(vec![("store", t("../x"))]).starts_with("store name")); +} + +#[test] +fn the_reply_round_trips_and_requires_listed() { + let reply = ListingReply { + projects: vec![ + ListedProject { + name: "app".to_owned(), + stack: stack(1), + }, + ListedProject { + name: "web".to_owned(), + stack: stack(2), + }, + ], + }; + let listed = |name: &str, stack: &LayerStackId| { + map(vec![ + ("name", t(name)), + ("stack_id", t(&hex(&stack.to_bytes()))), + ]) + }; + let encoded = reply.encode(); + assert_eq!( + encoded, + map(vec![ + ("kind", t("Listed")), + ( + "projects", + SdkValue::List(vec![listed("app", &stack(1)), listed("web", &stack(2))]), + ), + ]) + ); + assert_eq!(ListingReply::decode(&encoded).unwrap(), reply); + + let wrong_kind = map(vec![("kind", t("Removed"))]); + assert_eq!( + ListingReply::decode(&wrong_kind).unwrap_err().message, + "the worker project.list reply is not `Listed`" + ); + let no_list = map(vec![("kind", t("Listed"))]); + assert_eq!( + ListingReply::decode(&no_list).unwrap_err().message, + "reply has no `projects` list" + ); + let no_name = map(vec![ + ("kind", t("Listed")), + ( + "projects", + SdkValue::List(vec![map(vec![("stack_id", t(&hex(&stack(3).to_bytes())))])]), + ), + ]); + assert_eq!( + ListingReply::decode(&no_name).unwrap_err().message, + "reply project has no `name`" + ); + let bad_stack = map(vec![ + ("kind", t("Listed")), + ( + "projects", + SdkValue::List(vec![map(vec![("name", t("app")), ("stack_id", t("zz"))])]), + ), + ]); + assert_eq!( + ListingReply::decode(&bad_stack).unwrap_err().message, + "`stack_id` must be hex digits" + ); + let empty = ListingReply { + projects: Vec::new(), + }; + assert_eq!(ListingReply::decode(&empty.encode()).unwrap(), empty); +} diff --git a/core/crates/layerfs-history/src/contract/catalog.rs b/core/crates/layerfs-history/src/contract/catalog.rs index 431c5563a7..b543464d0f 100644 --- a/core/crates/layerfs-history/src/contract/catalog.rs +++ b/core/crates/layerfs-history/src/contract/catalog.rs @@ -20,9 +20,10 @@ use crate::error::HistoryResult; use crate::identity::{BranchId, CatalogId, CommitId, HistoryName, LayerId, LayerStackId, NodeId}; use crate::records::{ AddLayerOutcome, AddLayerRequest, BranchMergeRequest, BranchRecord, BranchSnapshot, - CommitHistoryRequest, CommitOutcome, CommitRecord, CommitRequest, ForkRequest, - LayerHistoryRequest, LayerRecord, LayerStackRecord, Page, PageResult, PathRemovalOutcome, - PathRemovalRecord, PathRemovalRequest, Reservation, ReserveRequest, StackInitialization, + CommitHistoryRequest, CommitOutcome, CommitRecord, CommitRequest, ForkRequest, GcDeletion, + GcIntent, GcScopeKind, LayerHistoryRequest, LayerRecord, LayerStackRecord, Page, PageResult, + PathRemovalOutcome, PathRemovalRecord, PathRemovalRequest, Reservation, ReserveRequest, + StackInitialization, }; /// Checked inputs that establish one catalog binding. #[derive(Clone, Eq, PartialEq)] @@ -391,6 +392,80 @@ pub trait HistoryCatalog: Send + Sync { "unreachable history removal", )) } + + /// Registers one admission intent (§5) in its own write transaction. + /// + /// The default refuses: a provider without admission cannot make the + /// `MountsActive`/`Busy`/`Unresolved` promise. + fn gc_register(&self, _intent: &GcIntent) -> HistoryResult<()> { + Err(crate::error::HistoryError::Unsupported("GC admission")) + } + + /// Releases one admission intent, deleting its row. + /// + /// The default refuses; a provider that never registered one is unaffected. + fn gc_release(&self, _op_id: &[u8; 16]) -> HistoryResult<()> { + Err(crate::error::HistoryError::Unsupported("GC admission")) + } + + /// Marks one admission intent uncertain, so a later call reports + /// `Unresolved` until its owner settles it. + fn gc_mark_uncertain(&self, _op_id: &[u8; 16]) -> HistoryResult<()> { + Err(crate::error::HistoryError::Unsupported("GC admission")) + } + + /// `branch.remove`: admits at Branch scope and writes the marker in one + /// write transaction. + fn gc_remove_branch( + &self, + branch: BranchId, + removed_ms: i64, + _intent: &GcIntent, + ) -> HistoryResult<()> { + // A provider without admission still writes the marker; the Store + // service only adds admission where the provider keeps intents. + self.mark_branch_removed(branch, removed_ms) + } + + /// `project.delete`: admits at project scope and writes the marker in one + /// write transaction, preserving the same-home retry classification. + fn gc_delete_stack( + &self, + stack: LayerStackId, + deleted_ms: i64, + _intent: &GcIntent, + ) -> HistoryResult { + if let Some(deleted_ms) = self.stack_deleted_ms(stack)? { + return Ok(GcDeletion::Retired { deleted_ms }); + } + if self.layer_stack(stack)?.is_some() { + self.mark_stack_deleted(stack, deleted_ms)?; + return Ok(GcDeletion::Marked { deleted_ms }); + } + Ok(match self.stack_deleted_ms(stack)? { + Some(deleted_ms) => GcDeletion::Retired { deleted_ms }, + None => GcDeletion::Absent, + }) + } + + /// `project.remove_file`: admits at project scope and advances the marker + /// in one write transaction. + fn gc_remove_path( + &self, + request: &PathRemovalRequest, + _intent: &GcIntent, + ) -> HistoryResult { + self.remove_path(request) + } + + /// The live intents recorded at one scope, for `store.status` and tests. + fn gc_intents( + &self, + _scope_kind: GcScopeKind, + _scope_id: &[u8], + ) -> HistoryResult> { + Err(crate::error::HistoryError::Unsupported("GC admission")) + } } impl std::fmt::Debug for HistoryCatalogConfig { diff --git a/core/crates/layerfs-history/src/contract/error.rs b/core/crates/layerfs-history/src/contract/error.rs index e4b686b3d0..940259da6e 100644 --- a/core/crates/layerfs-history/src/contract/error.rs +++ b/core/crates/layerfs-history/src/contract/error.rs @@ -7,6 +7,7 @@ //! that persistence did not answer and therefore proves nothing about rollback. use crate::identity::{CommitId, LayerId}; +use crate::records::GcIntentKind; /// A related record kind, used for a typed absence. #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -153,6 +154,12 @@ pub enum HistoryError { StaleStore, /// Immediate metadata admission was refused; the caller may retry later. Busy, + /// A mount or Sandbox holds the scope; the holders are the ones observed. + MountsActive(Box), + /// A live operation holds the scope; its entity, phase and op are reported. + BusyOp(Box), + /// A dead or uncertain owner's operation holds the scope. + Unresolved(Box), /// The content authority this catalog is bound to is not available. OwnershipUnavailable, /// A declared bound, count or counter is exhausted. @@ -213,6 +220,25 @@ impl std::fmt::Display for HistoryError { Self::Conflict(what) => write!(formatter, "conflict: {what}"), Self::StaleStore => formatter.write_str("stale Store"), Self::Busy => formatter.write_str("busy"), + Self::MountsActive(state) => { + write!( + formatter, + "mounts active: {} holder(s)", + state.holders.len() + ) + } + Self::BusyOp(state) => write!( + formatter, + "busy: {} at {}", + state.kind.as_str(), + state.entity + ), + Self::Unresolved(state) => write!( + formatter, + "unresolved: {} at {}", + state.kind.as_str(), + state.entity + ), Self::OwnershipUnavailable => formatter.write_str("ownership unavailable"), Self::Capacity(what) => write!(formatter, "capacity: {what}"), Self::Integrity(what) => write!(formatter, "integrity: {what}"), @@ -271,6 +297,89 @@ impl RemovedState { } } +/// One admission holder: a live mount or operation in the scope. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct GcHolder { + /// What kind of holder it is (`mount`, `init`, `merge`, ...). + pub kind: GcIntentKind, + /// The project, Branch or Store the holder is on. + pub entity: String, + /// The holder's Workspace, when it has one. + pub workspace: Option, + /// The holder's Sandbox, when it has one. + pub sandbox: Option, + /// The holder's machine, when it has one. + pub machine: Option, + /// The holder operation identity, hex encoded. + pub op: String, +} + +impl GcHolder { + /// Builds one holder report. + #[must_use] + pub fn new(kind: GcIntentKind, entity: impl Into, op: impl Into) -> Self { + Self { + kind, + entity: entity.into(), + workspace: None, + sandbox: None, + machine: None, + op: op.into(), + } + } +} + +/// The holders that refuse a removal or compaction with `MountsActive`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct MountHolders { + /// Every live or uncertain holder observed in the scope. + pub holders: Vec, +} + +impl MountHolders { + /// Builds the refusal context. + #[must_use] + pub fn new(holders: Vec) -> Self { + Self { holders } + } +} + +/// The live or uncertain operation that refuses a call with `Busy` or +/// `Unresolved`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct OpConflict { + /// The conflicting operation kind. + pub kind: GcIntentKind, + /// The project, Branch or Store it is on. + pub entity: String, + /// Its phase, when recorded. + pub phase: Option, + /// Its operation identity, hex encoded. + pub op: String, + /// Whether its owner is uncertain. + pub uncertain: bool, +} + +impl OpConflict { + /// Builds the refusal context. + #[must_use] + pub fn new( + kind: GcIntentKind, + entity: impl Into, + phase: Option, + op: impl Into, + uncertain: bool, + ) -> Self { + Self { + kind, + entity: entity.into(), + phase, + op: op.into(), + uncertain, + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/core/crates/layerfs-history/src/contract/records.rs b/core/crates/layerfs-history/src/contract/records.rs index 394ba3de9f..e90c475720 100644 --- a/core/crates/layerfs-history/src/contract/records.rs +++ b/core/crates/layerfs-history/src/contract/records.rs @@ -683,3 +683,226 @@ pub struct PathRemovalOutcome { /// The stored `removed_ms`. pub removed_ms: i64, } + +/// One kind of operation admitted through the Store's `gc_intent` table (§5.1). +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum GcIntentKind { + /// A mounted or mounting Workspace. + Mount, + /// A project Init. + Init, + /// A Branch merge. + Merge, + /// A soft removal (`branch.remove`, `project.delete`, `project.remove_file`). + Removal, + /// A Store compaction. + Compaction, +} + +impl GcIntentKind { + /// The `gc_intent.kind` text. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Mount => "mount", + Self::Init => "init", + Self::Merge => "merge", + Self::Removal => "removal", + Self::Compaction => "compaction", + } + } + + /// True when the operation shares its scopes instead of excluding them. + #[must_use] + pub const fn shared(self) -> bool { + matches!(self, Self::Mount | Self::Init | Self::Merge) + } +} + +/// The scope an admission intent is taken at. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum GcScopeKind { + /// The whole Store (compaction). + Store, + /// One project (Init, merge, `project.delete`, `remove_file`). + Project, + /// One Branch (mount, `branch.remove`). + Branch, +} + +impl GcScopeKind { + /// The `gc_intent.scope_kind` text. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Store => "store", + Self::Project => "project", + Self::Branch => "branch", + } + } +} + +/// One durable admission row (§5.1), or the operation row a caller writes +/// before touching a Store. Ids are typed per kind: 32 bytes for the Store +/// catalog, 17 for a project or Branch. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct GcIntent { + /// 16-byte operation identity (`gc_intent.op_id`). + pub op_id: [u8; 16], + /// Operation kind. + pub kind: GcIntentKind, + /// Scope the intent is recorded at. + pub scope_kind: GcScopeKind, + /// Typed scope identity. + pub scope_id: Vec, + /// The owning project identity, when the scope is a Branch. It is not a + /// stored column: it lets the PostgreSQL key derivation name the project + /// scope of a Branch intent. + pub project_id: Option>, + /// Mounting Workspace identity, 32 bytes. + pub workspace: Option<[u8; 32]>, + /// Sandbox name, when the caller knows one. + pub sandbox: Option, + /// Machine name, when the caller knows one. + pub machine: Option, + /// Project name, when the caller knows it. + pub project: Option, + /// Branch name, when the caller knows it. + pub branch: Option, + /// Owning runtime token. + pub owner_runtime: Option, + /// Owning boot token; `UNIQUE(scope_kind, scope_id, owner_boot)`. + pub owner_boot: Option, + /// Operation phase, for a holder report. + pub phase: Option, + /// True when the owner is uncertain and only `Unresolved` may report it. + pub uncertain: bool, +} + +impl GcIntent { + /// A shared (`mount`, `init`, `merge`) intent at one scope. + #[must_use] + pub fn shared( + op_id: [u8; 16], + kind: GcIntentKind, + scope_kind: GcScopeKind, + scope_id: impl Into>, + ) -> Self { + Self::new(op_id, kind, scope_kind, scope_id) + } + + /// An exclusive (`removal`, `compaction`) intent at one scope. + #[must_use] + pub fn operation( + op_id: [u8; 16], + kind: GcIntentKind, + scope_kind: GcScopeKind, + scope_id: impl Into>, + ) -> Self { + Self::new(op_id, kind, scope_kind, scope_id) + } + + fn new( + op_id: [u8; 16], + kind: GcIntentKind, + scope_kind: GcScopeKind, + scope_id: impl Into>, + ) -> Self { + Self { + op_id, + kind, + scope_kind, + scope_id: scope_id.into(), + project_id: None, + workspace: None, + sandbox: None, + machine: None, + project: None, + branch: None, + owner_runtime: None, + owner_boot: None, + phase: None, + uncertain: false, + } + } + + /// Records the owning project identity of a Branch-scoped intent. + #[must_use] + pub fn in_project(mut self, project: impl Into>) -> Self { + self.project_id = Some(project.into()); + self + } + + /// Fills the project and Branch names a holder report shows. + #[must_use] + pub fn named(mut self, project: Option<&str>, branch: Option<&str>) -> Self { + self.project = project.map(str::to_owned); + self.branch = branch.map(str::to_owned); + self + } + + /// Fills the Workspace, Sandbox and machine a holder report shows. + #[must_use] + pub fn placed( + mut self, + workspace: Option<[u8; 32]>, + sandbox: Option<&str>, + machine: Option<&str>, + ) -> Self { + self.workspace = workspace; + self.sandbox = sandbox.map(str::to_owned); + self.machine = machine.map(str::to_owned); + self + } + + /// Fills the owner tokens. + #[must_use] + pub fn owned(mut self, runtime: Option<&str>, boot: Option<&str>) -> Self { + self.owner_runtime = runtime.map(str::to_owned); + self.owner_boot = boot.map(str::to_owned); + self + } + + /// Records the phase a holder report shows. + #[must_use] + pub fn with_phase(mut self, phase: Option<&str>) -> Self { + self.phase = phase.map(str::to_owned); + self + } + + /// Marks the owner uncertain: only `Unresolved` may report it. + #[must_use] + pub fn uncertain(mut self) -> Self { + self.uncertain = true; + self + } +} + +/// The outcome of the Store side of `project.delete`: a marker already there, +/// a marker written now, or neither a live stack nor a marker. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum GcDeletion { + /// The marker was already recorded; its `deleted_ms` is reused. + Retired { + /// The `deleted_ms` the marker holds. + deleted_ms: i64, + }, + /// A live stack was marked with the caller's `deleted_ms`. + Marked { + /// The `deleted_ms` written. + deleted_ms: i64, + }, + /// Neither a live stack nor a marker: the caller removes its stale entry. + Absent, +} + +impl GcDeletion { + /// The `deleted_ms` a `project.delete` result reports. + #[must_use] + pub fn deleted_ms(self, requested: i64) -> i64 { + match self { + Self::Retired { deleted_ms } | Self::Marked { deleted_ms } => deleted_ms, + Self::Absent => requested, + } + } +} diff --git a/core/crates/layerfs-history/src/lib.rs b/core/crates/layerfs-history/src/lib.rs index 6f01165217..1f9100c1bd 100644 --- a/core/crates/layerfs-history/src/lib.rs +++ b/core/crates/layerfs-history/src/lib.rs @@ -26,16 +26,17 @@ pub use contract::query; pub use contract::records; pub use catalog::{HistoryCatalog, HistoryCatalogConfig, NamedSnapshot, PublishLane}; -pub use error::{HistoryError, HistoryResult}; +pub use error::{GcHolder, HistoryError, HistoryResult, MountHolders, OpConflict}; pub use identity::{ BranchId, CatalogId, CommitId, HistoryName, LayerId, LayerStackId, NodeId, WorkspaceId, }; pub use records::{ AddLayerOutcome, AddLayerRequest, BindingHint, BranchMergeRequest, BranchRecord, BranchSnapshot, CommitHistoryRequest, CommitMeta, CommitOutcome, CommitRecord, CommitRequest, - CommitStatistics, CommitVia, FileDigest, ForkRequest, ForkSource, InitStatistics, - LayerHistoryRequest, LayerRecord, LayerStackRecord, MergeCommit, MergeHead, MergeSide, Page, - PageResult, PathRemovalKind, PathRemovalOutcome, PathRemovalRecord, PathRemovalRequest, - RecordKind, Reservation, ReserveRequest, StackInitialization, StoreStatistics, - FILE_DIGEST_WINDOW, MAXIMUM_MESSAGE_BYTES, MAXIMUM_PAGE_RECORDS, + CommitStatistics, CommitVia, FileDigest, ForkRequest, ForkSource, GcDeletion, GcIntent, + GcIntentKind, GcScopeKind, InitStatistics, LayerHistoryRequest, LayerRecord, LayerStackRecord, + MergeCommit, MergeHead, MergeSide, Page, PageResult, PathRemovalKind, PathRemovalOutcome, + PathRemovalRecord, PathRemovalRequest, RecordKind, Reservation, ReserveRequest, + StackInitialization, StoreStatistics, FILE_DIGEST_WINDOW, MAXIMUM_MESSAGE_BYTES, + MAXIMUM_PAGE_RECORDS, }; diff --git a/core/crates/layerfs-history/tests/contract_constructors.rs b/core/crates/layerfs-history/tests/contract_constructors.rs new file mode 100644 index 0000000000..94625590a7 --- /dev/null +++ b/core/crates/layerfs-history/tests/contract_constructors.rs @@ -0,0 +1,70 @@ +//! A provider outside this crate builds each refusal context through the +//! public constructors, and readers still reach the public fields. + +use layerfs_history::error::{ + HolderKind, HolderState, NameHolder, NameSource, NameTakenState, RemovedPath, RemovedState, +}; +use layerfs_history::HistoryError; + +#[test] +fn name_taken_context_is_constructed_and_read() { + let holder = NameHolder::new(HolderKind::Branch, "feature", HolderState::Retired); + assert_eq!(holder.kind, HolderKind::Branch); + assert_eq!(holder.name, "feature"); + assert_eq!(holder.state, HolderState::Retired); + + let state = NameTakenState::new(holder, NameSource::Home); + assert_eq!(state.source, NameSource::Home); + assert_eq!(state.holder.name, "feature"); + + let error = HistoryError::NameTaken(Box::new(state)); + match error.cause() { + HistoryError::NameTaken(state) => { + assert_eq!(state.holder.kind, HolderKind::Branch); + assert_eq!(state.holder.state, HolderState::Retired); + assert_eq!(state.source, NameSource::Home); + } + other => panic!("expected NameTaken, got {other:?}"), + } +} + +#[test] +fn removed_context_is_constructed_and_read() { + let path = RemovedPath::new("/gone.txt", 42); + assert_eq!(path.path, "/gone.txt"); + assert_eq!(path.frontier_serial, 42); + + let state = RemovedState::new(vec![path, RemovedPath::new("/other.txt", 7)]); + assert_eq!(state.paths.len(), 2); + assert_eq!(state.paths[1].path, "/other.txt"); + + let error = HistoryError::Removed(Box::new(state)); + match error.cause() { + HistoryError::Removed(state) => { + assert_eq!(state.paths[0].path, "/gone.txt"); + assert_eq!(state.paths[0].frontier_serial, 42); + assert_eq!(state.paths[1].frontier_serial, 7); + } + other => panic!("expected Removed, got {other:?}"), + } +} + +#[test] +fn every_holder_kind_and_state_builds() { + for kind in [HolderKind::Branch, HolderKind::Project] { + for state in [ + HolderState::Live, + HolderState::Retired, + HolderState::Deleted, + ] { + let holder = NameHolder::new(kind, "name", state); + assert_eq!(holder.kind, kind); + assert_eq!(holder.state, state); + } + } + for source in [NameSource::Store, NameSource::Home] { + let holder = NameHolder::new(HolderKind::Project, "project", HolderState::Live); + let state = NameTakenState::new(holder, source); + assert_eq!(state.source, source); + } +} diff --git a/engine/crates/layerfs-daemon/src/control/failure.rs b/engine/crates/layerfs-daemon/src/control/failure.rs index ba948b0207..01730e05e1 100644 --- a/engine/crates/layerfs-daemon/src/control/failure.rs +++ b/engine/crates/layerfs-daemon/src/control/failure.rs @@ -159,6 +159,8 @@ impl Failure { fn history(error: &HistoryError) -> (ControlCode, Option) { match error.cause() { HistoryError::Busy => (ControlCode::Busy, None), + HistoryError::MountsActive(_) | HistoryError::BusyOp(_) => (ControlCode::Busy, None), + HistoryError::Unresolved(_) => (ControlCode::Unknown, None), HistoryError::HeadMoved(value) => (ControlCode::HeadMoved, Some(**value)), HistoryError::UnknownOutcome => (ControlCode::Unknown, None), HistoryError::Missing(_) => (ControlCode::Missing, None), diff --git a/engine/crates/layerfs-daemon/src/control/unmount.rs b/engine/crates/layerfs-daemon/src/control/unmount.rs index 10b5dd4322..84352a6fdf 100644 --- a/engine/crates/layerfs-daemon/src/control/unmount.rs +++ b/engine/crates/layerfs-daemon/src/control/unmount.rs @@ -150,6 +150,7 @@ impl Service { Err(error) => { let failure = match error { crate::store::CloseError::Owner(cause) => Failure::Owner(cause), + crate::store::CloseError::Intent(cause) => Failure::History(cause), crate::store::CloseError::Reader(cause) => Failure::Workspace( layerfs_workspace::WorkspaceError::Service(Box::new(cause)), ), diff --git a/engine/crates/layerfs-daemon/src/operator/mod.rs b/engine/crates/layerfs-daemon/src/operator/mod.rs index 9aead7c370..7408c42927 100644 --- a/engine/crates/layerfs-daemon/src/operator/mod.rs +++ b/engine/crates/layerfs-daemon/src/operator/mod.rs @@ -5,3 +5,9 @@ pub(crate) use dispatch::{admits, dispatch}; pub(crate) use layerfs_store_service::operator::{snapshot, WorkspaceView}; mod workspace; + +/// The installed Store fixture the operator unit test binds without a kernel. +#[cfg(test)] +#[allow(dead_code)] +#[path = "../../tests/support/installed_store.rs"] +mod installed_store; diff --git a/engine/crates/layerfs-daemon/src/operator/workspace.rs b/engine/crates/layerfs-daemon/src/operator/workspace.rs index 19d5cb8a64..536cb89986 100644 --- a/engine/crates/layerfs-daemon/src/operator/workspace.rs +++ b/engine/crates/layerfs-daemon/src/operator/workspace.rs @@ -5,13 +5,16 @@ use crate::{ }; use layerfs_bridge::control::{SdkRefusal, SdkValue}; use layerfs_content::{ - compare::TreeTable, filesystem::FilesystemRootId, object::inode_leaf::InodeKind, + compare::{EntryFacts, TreeTable}, + filesystem::FilesystemRootId, + object::inode_leaf::InodeKind, }; use layerfs_store_service::{ history::lookup::{self as args, Answer}, operator::args::{paging, text}, read::{ snapshot::{self, Kind, Snap}, + visibility::Visibility, window::Walker, }, }; @@ -80,6 +83,46 @@ impl<'a> Live<'a> { } Ok(None) } + + /// The next live entry that is visible at the base node. A path the markers + /// hide is absent unless the overlay holds a local inode there: a + /// re-creation after the bound node is an ordinary addition, as a Commit's + /// is, while an inherited entry of a mount older than the marker is hidden. + fn next_visible(&mut self, seen: &Visibility) -> Result> { + loop { + let Some((path, stat)) = self.next()? else { + return Ok(None); + }; + if seen.hidden(&path).is_some() + && self + .op + .overlay() + .inode(self.view.source(), stat.serial) + .map_err(args::failed)? + .is_none() + { + continue; + } + return Ok(Some((path, stat))); + } + } +} + +/// The base walk, less every path the markers hide at the base node: a removed +/// path is absent from `changes`, like a read of it, and so is its subtree. +struct VisibleBase<'a> { + walker: Walker<'a>, + seen: &'a Visibility, +} +impl VisibleBase<'_> { + fn next(&mut self) -> Result> { + loop { + match self.walker.next()? { + Some((path, _)) if self.seen.hidden(&path).is_some() => continue, + other => return Ok(other), + } + } + } } fn selected(service: &Service, args: &SdkValue) -> Result> { @@ -181,10 +224,17 @@ fn scan( .pending_conflicts() .map_err(args::failed)? .unwrap_or_default(); - let mut base = Walker::new(op.ports(), &snap, "/")?; + // §4.3: the base walk is path-aware, so a path the markers hide at the + // base node is absent from `changes` instead of a `Removed` row against + // the mount's whiteout. The live side keeps a local re-creation. + let seen = Visibility::for_snap(&service.store, &snap); + let mut base = VisibleBase { + walker: Walker::new(op.ports(), &snap, "/")?, + seen: &seen, + }; let mut live = Live::new(view, op); let mut work = layerfs_content::filesystem::attributes::AttributeReadWork::default(); - let (mut a, mut b) = (base.next()?, live.next()?); + let (mut a, mut b) = (base.next()?, live.next_visible(&seen)?); while a.is_some() || b.is_some() { let order = match (&a, &b) { (Some((a, _)), Some((b, _))) => a.split('/').cmp(b.split('/')), @@ -199,14 +249,14 @@ fn scan( } std::cmp::Ordering::Greater => { let (p, e) = b.take().expect("live"); - b = live.next()?; + b = live.next_visible(&seen)?; (p, None, Some(e)) } std::cmp::Ordering::Equal => { let (p, x) = a.take().expect("base"); let (_, y) = b.take().expect("live"); a = base.next()?; - b = live.next()?; + b = live.next_visible(&seen)?; (p, Some(x), Some(y)) } }; @@ -259,16 +309,20 @@ fn scan( break; } } - let _ = service; Ok(()) } pub(crate) fn changes(service: &Service, args: &SdkValue) -> Answer { let workspace = selected(service, args)?; + changes_of(service, &workspace, args) +} + +/// `changes` over an already selected Workspace; a test binds one directly. +fn changes_of(service: &Service, workspace: &BoundWorkspace, args: &SdkValue) -> Answer { let (limit, pos) = paging(args)?; - with_source(&workspace, |op, view| { + with_source(workspace, |op, view| { let (mut seen, mut items, mut spent, mut more) = (0u64, Vec::new(), 0, false); - scan(service, &workspace, op, view, |path, kind, markers| { + scan(service, workspace, op, view, |path, kind, markers| { if seen < pos { seen += 1; return Ok(true); @@ -317,3 +371,125 @@ pub(crate) fn overlay(service: &Service, args: &SdkValue) -> Answer { ("changes", SdkValue::Null), ])) } + +#[cfg(test)] +mod tests { + //! `workspace.changes` over a bound Workspace, without a kernel mount: the + //! path-awareness of its base walk. + use super::*; + use crate::control::Service; + use crate::{Owner, OwnerConfig}; + use layerfs_bridge::control::SdkValue; + use layerfs_history::{PathRemovalKind, PathRemovalRequest, WorkspaceId}; + use layerfs_storage::ReservationBlocks; + use layerfs_store_service::{ + bootstrap::open_store_observed, store::BindRequest, write::mutation::Mutation, + }; + use std::fs; + + use crate::operator::installed_store as installed; + + fn listed(answer: &SdkValue) -> Vec<(String, String)> { + let Some(SdkValue::List(items)) = answer.get("items") else { + panic!("no items: {answer:?}"); + }; + items + .iter() + .map(|item| { + ( + item.get_text("path").expect("path").to_owned(), + item.get_text("kind").expect("kind").to_owned(), + ) + }) + .collect() + } + + /// A file and a directory removed by `project.remove_file` markers at the + /// bound node are absent from `changes`, subtree and all; a visible + /// change is still listed, and a post-bound re-creation is an addition. + #[test] + fn a_removed_file_and_a_hidden_directory_are_absent_from_changes() { + let fixture = installed::Fixture::built("changes-visibility", |source| { + fs::write(source.join("visible.txt"), b"visible\n").unwrap(); + fs::write(source.join("hidden.txt"), b"hidden\n").unwrap(); + fs::create_dir(source.join("hidden-dir")).unwrap(); + fs::write(source.join("hidden-dir/inner.txt"), b"inner\n").unwrap(); + 4 + }); + let opened = open_store_observed( + fixture.config.clone(), + installed::BINDING, + installed::CURSOR, + 64, + 8 * 1024 * 1024, + ReservationBlocks::default(), + ) + .unwrap(); + let stack = opened + .store + .history() + .branch_snapshot(fixture.branch) + .unwrap() + .unwrap() + .branch + .stack; + for path in ["hidden.txt", "hidden-dir"] { + let outcome = opened + .store + .history() + .remove_path(&PathRemovalRequest { + stack, + path: path.as_bytes().to_vec(), + removed_ms: 1, + }) + .unwrap(); + assert_eq!(outcome.kind, PathRemovalKind::Removed, "{path}"); + } + let owner = Owner::start( + &fixture.directory.join("overlay.sqlite"), + layerfs_overlay::ProfileConfig::default(), + OwnerConfig::default(), + ) + .unwrap(); + let bound = opened + .store + .bind( + owner.client(), + BindRequest { + branch: fixture.branch, + workspace: WorkspaceId::from_authority([57; 32]).unwrap(), + }, + ) + .unwrap() + .workspace; + // Bind wrote the whiteouts; a visible change and one re-creation follow. + let mut mutation = Mutation::new(&bound).unwrap(); + mutation.write("/visible.txt", b"after\n", None).unwrap(); + mutation.finish().unwrap(); + let mut mutation = Mutation::new(&bound).unwrap(); + mutation.write("/hidden.txt", b"again\n", None).unwrap(); + mutation.finish().unwrap(); + + let service = Service::new(opened.store.clone(), &owner); + let answer = changes_of( + &service, + &bound, + &SdkValue::map([("pageSize", SdkValue::Int(50))]), + ) + .unwrap(); + assert_eq!( + listed(&answer), + [ + ("/hidden.txt".to_owned(), "Added".to_owned()), + ("/visible.txt".to_owned(), "Modified".to_owned()), + ] + ); + assert_eq!(answer.get("hasMore"), Some(&SdkValue::Bool(false))); + + drop(service); + bound.close().unwrap(); + owner.stop().unwrap(); + drop(opened); + fixture.cleanup(); + } +} diff --git a/engine/crates/layerfs-persistence/Cargo.toml b/engine/crates/layerfs-persistence/Cargo.toml index 090f56a2be..5502b2e1d8 100644 --- a/engine/crates/layerfs-persistence/Cargo.toml +++ b/engine/crates/layerfs-persistence/Cargo.toml @@ -7,6 +7,8 @@ rust-version.workspace = true license.workspace = true [dependencies] +# S8 advisory keys: blake3("layerfs/advisory/v1\0" || catalog_id || kind || id). +blake3 = { version = "=1.8.5", default-features = false } sha2 = "=0.11.0" layerfs-storage = { path = "../../../core/crates/layerfs-storage" } layerfs-history = { path = "../../../core/crates/layerfs-history", default-features = false } diff --git a/engine/crates/layerfs-persistence/sql/postgresql/gc.sql b/engine/crates/layerfs-persistence/sql/postgresql/gc.sql index 713bb0c64e..b80cbe1420 100644 --- a/engine/crates/layerfs-persistence/sql/postgresql/gc.sql +++ b/engine/crates/layerfs-persistence/sql/postgresql/gc.sql @@ -71,3 +71,14 @@ CREATE TABLE gc_dead ( id BYTEA NOT NULL, PRIMARY KEY (kind, id) ); +CREATE TABLE kept_node ( + layer_stack_id BYTEA NOT NULL CHECK (length(layer_stack_id) = 17), + kind TEXT NOT NULL CHECK (kind IN ('commit','layer','branch','stack')), + node_id BYTEA NOT NULL, + serial BIGINT NOT NULL DEFAULT 0 CHECK (serial >= 0), + PRIMARY KEY (layer_stack_id, kind, node_id), + CHECK ( + (kind IN ('commit','layer') AND length(node_id) = 33) + OR (kind IN ('branch','stack') AND length(node_id) = 17) + ) +); diff --git a/engine/crates/layerfs-persistence/sql/postgresql/queries/history/branch_snapshot.sql b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/branch_snapshot.sql index bb079ef25c..8a36f19368 100644 --- a/engine/crates/layerfs-persistence/sql/postgresql/queries/history/branch_snapshot.sql +++ b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/branch_snapshot.sql @@ -1 +1 @@ -SELECT b.branch_id, b.layer_stack_id, b.name, b.base_layer_id, b.head_commit_id, b.forked_from_id, s.layer_stack_id, s.name, s.scope_id, s.profile_id, s.head_layer_id, l.root_id, c.commit_id, c.layer_stack_id, c.root_id, c.parent_commit_id, c.base_layer_id, c.second_parent_id, c.branch_id, c.via, c.message, c.merged_from_id, c.created_ms, c.logical, c.new_canonical, c.new_stored, c.new_metadata, c.lww, c.displaced_id FROM {s}.branch b LEFT JOIN {s}.layer_stack s ON s.layer_stack_id = b.layer_stack_id LEFT JOIN {s}.layer l ON l.layer_id = b.base_layer_id LEFT JOIN {s}."commit" c ON c.commit_id = b.head_commit_id WHERE b.branch_id = $1; +SELECT b.branch_id, b.layer_stack_id, b.name, b.base_layer_id, b.head_commit_id, b.forked_from_id, s.layer_stack_id, s.name, s.scope_id, s.profile_id, s.head_layer_id, l.root_id, c.commit_id, c.layer_stack_id, c.root_id, c.parent_commit_id, c.base_layer_id, c.second_parent_id, c.branch_id, c.via, c.message, c.merged_from_id, c.created_ms, c.logical, c.new_canonical, c.new_stored, c.new_metadata, c.lww, c.displaced_id FROM {s}.branch b LEFT JOIN {s}.layer_stack s ON s.layer_stack_id = b.layer_stack_id LEFT JOIN {s}.layer l ON l.layer_id = b.base_layer_id LEFT JOIN {s}."commit" c ON c.commit_id = b.head_commit_id WHERE b.branch_id = $1 AND NOT EXISTS (SELECT 1 FROM {s}.branch_retired r WHERE r.branch_id = b.branch_id) AND NOT EXISTS (SELECT 1 FROM {s}.layer_stack_retired d WHERE d.layer_stack_id = b.layer_stack_id); diff --git a/engine/crates/layerfs-persistence/sql/postgresql/queries/history/branch_snapshot_named.sql b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/branch_snapshot_named.sql index 8ff1a3808b..715134482e 100644 --- a/engine/crates/layerfs-persistence/sql/postgresql/queries/history/branch_snapshot_named.sql +++ b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/branch_snapshot_named.sql @@ -1 +1 @@ -SELECT b.branch_id, b.layer_stack_id, b.name, b.base_layer_id, b.head_commit_id, b.forked_from_id, s.layer_stack_id, s.name, s.scope_id, s.profile_id, s.head_layer_id, l.root_id, c.commit_id, c.layer_stack_id, c.root_id, c.parent_commit_id, c.base_layer_id, c.second_parent_id, c.branch_id, c.via, c.message, c.merged_from_id, c.created_ms, c.logical, c.new_canonical, c.new_stored, c.new_metadata, c.lww, c.displaced_id FROM {s}.layer_stack p LEFT JOIN {s}.branch b ON p.name = $1 AND b.layer_stack_id = p.layer_stack_id AND b.name = $2 LEFT JOIN {s}.layer_stack s ON s.layer_stack_id = b.layer_stack_id LEFT JOIN {s}.layer l ON l.layer_id = b.base_layer_id LEFT JOIN {s}."commit" c ON c.commit_id = b.head_commit_id WHERE p.name = $1; +SELECT b.branch_id, b.layer_stack_id, b.name, b.base_layer_id, b.head_commit_id, b.forked_from_id, s.layer_stack_id, s.name, s.scope_id, s.profile_id, s.head_layer_id, l.root_id, c.commit_id, c.layer_stack_id, c.root_id, c.parent_commit_id, c.base_layer_id, c.second_parent_id, c.branch_id, c.via, c.message, c.merged_from_id, c.created_ms, c.logical, c.new_canonical, c.new_stored, c.new_metadata, c.lww, c.displaced_id FROM {s}.layer_stack p LEFT JOIN {s}.branch b ON p.name = $1 AND b.layer_stack_id = p.layer_stack_id AND b.name = $2 AND NOT EXISTS (SELECT 1 FROM {s}.branch_retired r WHERE r.branch_id = b.branch_id) LEFT JOIN {s}.layer_stack s ON s.layer_stack_id = b.layer_stack_id LEFT JOIN {s}.layer l ON l.layer_id = b.base_layer_id LEFT JOIN {s}."commit" c ON c.commit_id = b.head_commit_id WHERE p.name = $1 AND NOT EXISTS (SELECT 1 FROM {s}.layer_stack_retired d WHERE d.layer_stack_id = p.layer_stack_id); diff --git a/engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_epoch_share.sql b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_epoch_share.sql new file mode 100644 index 0000000000..e42a6762c2 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_epoch_share.sql @@ -0,0 +1 @@ +SELECT epoch FROM project_epoch WHERE layer_stack_id = $1 FOR SHARE; diff --git a/engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_lock_session_exclusive.sql b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_lock_session_exclusive.sql new file mode 100644 index 0000000000..255c909a83 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_lock_session_exclusive.sql @@ -0,0 +1 @@ +SELECT 1 AS ok WHERE pg_try_advisory_lock($1::bigint); diff --git a/engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_lock_session_shared.sql b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_lock_session_shared.sql new file mode 100644 index 0000000000..12a63f9809 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_lock_session_shared.sql @@ -0,0 +1 @@ +SELECT 1 AS ok WHERE pg_try_advisory_lock_shared($1::bigint); diff --git a/engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_lock_xact_exclusive.sql b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_lock_xact_exclusive.sql new file mode 100644 index 0000000000..b70b5de73b --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_lock_xact_exclusive.sql @@ -0,0 +1 @@ +SELECT 1 AS ok WHERE pg_try_advisory_xact_lock($1::bigint); diff --git a/engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_lock_xact_shared.sql b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_lock_xact_shared.sql new file mode 100644 index 0000000000..5844d19a06 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_lock_xact_shared.sql @@ -0,0 +1 @@ +SELECT 1 AS ok WHERE pg_try_advisory_xact_lock_shared($1::bigint); diff --git a/engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_unlock_session_exclusive.sql b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_unlock_session_exclusive.sql new file mode 100644 index 0000000000..36e9fa061a --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_unlock_session_exclusive.sql @@ -0,0 +1 @@ +SELECT pg_advisory_unlock($1::bigint) AS ok; diff --git a/engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_unlock_session_shared.sql b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_unlock_session_shared.sql new file mode 100644 index 0000000000..a3fdbed9f2 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_unlock_session_shared.sql @@ -0,0 +1 @@ +SELECT pg_advisory_unlock_shared($1::bigint) AS ok; diff --git a/engine/crates/layerfs-persistence/sql/postgresql/schema.sql b/engine/crates/layerfs-persistence/sql/postgresql/schema.sql index 8687c0371c..d007e98f17 100644 --- a/engine/crates/layerfs-persistence/sql/postgresql/schema.sql +++ b/engine/crates/layerfs-persistence/sql/postgresql/schema.sql @@ -11,7 +11,8 @@ CREATE TABLE store_policy ( chunk_delta_max_depth bigint NOT NULL CHECK (chunk_delta_max_depth BETWEEN 0 AND 50), metadata_delta_max_depth bigint NOT NULL CHECK (metadata_delta_max_depth BETWEEN 0 AND 50), next_pack_id bigint NOT NULL DEFAULT 1 CHECK (next_pack_id > 0), - next_ordinal bigint NOT NULL DEFAULT 1 CHECK (next_ordinal BETWEEN 1 AND 4294967296) + next_ordinal bigint NOT NULL DEFAULT 1 CHECK (next_ordinal BETWEEN 1 AND 4294967296), + trace_salt bytea NOT NULL DEFAULT decode(repeat('00',32),'hex') CHECK (length(trace_salt)=32) ); CREATE TABLE pack ( pack_id bigint PRIMARY KEY CHECK (pack_id > 0), @@ -97,6 +98,7 @@ CREATE TABLE layer ( overwrote bytea NOT NULL, source_commit_id bytea CHECK (source_commit_id IS NULL OR (octet_length(source_commit_id) = 33 AND get_byte(source_commit_id, 0) = 18)), + history_serial bigint NOT NULL DEFAULT 0, CHECK ( (parent_layer_id IS NULL AND source_branch_id IS NULL AND source_commit_id IS NULL) OR (parent_layer_id IS NOT NULL AND source_branch_id IS NOT NULL AND source_commit_id IS NOT NULL) @@ -127,6 +129,7 @@ CREATE TABLE "commit" ( new_metadata bigint NOT NULL DEFAULT 0 CHECK (new_metadata >= 0), lww bytea NOT NULL, displaced_id bytea CHECK (displaced_id IS NULL OR octet_length(displaced_id) = 33), + history_serial bigint NOT NULL DEFAULT 0, UNIQUE (layer_stack_id, commit_id) ); CREATE TABLE branch ( diff --git a/engine/crates/layerfs-persistence/sql/sqlite/gc.sql b/engine/crates/layerfs-persistence/sql/sqlite/gc.sql index 10c62a803c..ee565544b9 100644 --- a/engine/crates/layerfs-persistence/sql/sqlite/gc.sql +++ b/engine/crates/layerfs-persistence/sql/sqlite/gc.sql @@ -80,3 +80,17 @@ CREATE TABLE gc_dead ( id BLOB NOT NULL, PRIMARY KEY (kind, id) ) STRICT, WITHOUT ROWID; +CREATE TABLE kept_node ( + layer_stack_id BLOB NOT NULL + CHECK (length(layer_stack_id) = 17 AND substr(layer_stack_id, 1, 1) = X'31'), + kind TEXT NOT NULL CHECK (kind IN ('commit','layer','branch','stack')), + node_id BLOB NOT NULL, + serial INTEGER NOT NULL DEFAULT 0 CHECK (serial >= 0), + PRIMARY KEY (layer_stack_id, kind, node_id), + CHECK ( + (kind = 'commit' AND length(node_id) = 33 AND substr(node_id, 1, 1) = X'12') + OR (kind = 'layer' AND length(node_id) = 33 AND substr(node_id, 1, 1) = X'32') + OR (kind = 'branch' AND length(node_id) = 17 AND substr(node_id, 1, 1) = X'11') + OR (kind = 'stack' AND length(node_id) = 17 AND substr(node_id, 1, 1) = X'31') + ) +) STRICT, WITHOUT ROWID; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_branch_by_id.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_branch_by_id.sql index 8bb8c308fd..1f9cc22f93 100644 --- a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_branch_by_id.sql +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_branch_by_id.sql @@ -1 +1 @@ -SELECT branch_id, layer_stack_id, name, base_layer_id, head_commit_id, forked_from_id FROM branch WHERE branch_id = $1; +SELECT branch_id, layer_stack_id, name, base_layer_id, head_commit_id, forked_from_id FROM branch WHERE branch_id = $1 AND NOT EXISTS (SELECT 1 FROM branch_retired r WHERE r.branch_id = branch.branch_id) AND NOT EXISTS (SELECT 1 FROM layer_stack_retired s WHERE s.layer_stack_id = branch.layer_stack_id); diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_branch_by_id_any.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_branch_by_id_any.sql new file mode 100644 index 0000000000..8bb8c308fd --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_branch_by_id_any.sql @@ -0,0 +1 @@ +SELECT branch_id, layer_stack_id, name, base_layer_id, head_commit_id, forked_from_id FROM branch WHERE branch_id = $1; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_branch_by_name.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_branch_by_name.sql index f32a794a16..0eadc60e31 100644 --- a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_branch_by_name.sql +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_branch_by_name.sql @@ -1 +1 @@ -SELECT branch_id, layer_stack_id, name, base_layer_id, head_commit_id, forked_from_id FROM branch WHERE layer_stack_id = $1 AND name = $2; +SELECT branch_id, layer_stack_id, name, base_layer_id, head_commit_id, forked_from_id FROM branch WHERE layer_stack_id = $1 AND name = $2 AND NOT EXISTS (SELECT 1 FROM branch_retired r WHERE r.branch_id = branch.branch_id) AND NOT EXISTS (SELECT 1 FROM layer_stack_retired s WHERE s.layer_stack_id = branch.layer_stack_id); diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_branch_first_page.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_branch_first_page.sql index b31a7ac3cb..948b07f14a 100644 --- a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_branch_first_page.sql +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_branch_first_page.sql @@ -1 +1 @@ -SELECT branch_id, layer_stack_id, name, base_layer_id, head_commit_id, forked_from_id FROM branch WHERE layer_stack_id = $1 ORDER BY name LIMIT $2; +SELECT branch_id, layer_stack_id, name, base_layer_id, head_commit_id, forked_from_id FROM branch WHERE layer_stack_id = $1 AND NOT EXISTS (SELECT 1 FROM branch_retired r WHERE r.branch_id = branch.branch_id) AND NOT EXISTS (SELECT 1 FROM layer_stack_retired s WHERE s.layer_stack_id = branch.layer_stack_id) ORDER BY name LIMIT $2; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_branch_name_taken.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_branch_name_taken.sql index 66493a918d..e0f54d1707 100644 --- a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_branch_name_taken.sql +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_branch_name_taken.sql @@ -1 +1 @@ -SELECT 1 FROM branch WHERE layer_stack_id = $1 AND name = $2; +SELECT b.name, CASE WHEN r.branch_id IS NULL THEN 0 ELSE 1 END FROM branch b LEFT JOIN branch_retired r ON r.branch_id = b.branch_id WHERE b.layer_stack_id = $1 AND b.name = $2; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_branch_next_page.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_branch_next_page.sql index 9530b4f910..c37d2b136e 100644 --- a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_branch_next_page.sql +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_branch_next_page.sql @@ -1 +1 @@ -SELECT branch_id, layer_stack_id, name, base_layer_id, head_commit_id, forked_from_id FROM branch WHERE layer_stack_id = $1 AND name > $2 ORDER BY name LIMIT $3; +SELECT branch_id, layer_stack_id, name, base_layer_id, head_commit_id, forked_from_id FROM branch WHERE layer_stack_id = $1 AND name > $2 AND NOT EXISTS (SELECT 1 FROM branch_retired r WHERE r.branch_id = branch.branch_id) AND NOT EXISTS (SELECT 1 FROM layer_stack_retired s WHERE s.layer_stack_id = branch.layer_stack_id) ORDER BY name LIMIT $3; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_retire_insert.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_retire_insert.sql new file mode 100644 index 0000000000..700f5dd810 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_retire_insert.sql @@ -0,0 +1 @@ +INSERT INTO branch_retired (branch_id, removed_ms) VALUES ($1, $2) ON CONFLICT(branch_id) DO NOTHING RETURNING branch_id; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_snapshot.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_snapshot.sql index 146b8b8c6a..08db9eb02a 100644 --- a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_snapshot.sql +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_snapshot.sql @@ -1 +1 @@ -SELECT b.branch_id, b.layer_stack_id, b.name, b.base_layer_id, b.head_commit_id, b.forked_from_id, s.layer_stack_id, s.name, s.scope_id, s.profile_id, s.head_layer_id, l.root_id, c.commit_id, c.layer_stack_id, c.root_id, c.parent_commit_id, c.base_layer_id, c.second_parent_id, c.branch_id, c.via, c.message, c.merged_from_id, c.created_ms, c.logical, c.new_canonical, c.new_stored, c.new_metadata, c.lww, c.displaced_id FROM branch b LEFT JOIN layer_stack s ON s.layer_stack_id = b.layer_stack_id LEFT JOIN layer l ON l.layer_id = b.base_layer_id LEFT JOIN "commit" c ON c.commit_id = b.head_commit_id WHERE b.branch_id = $1; +SELECT b.branch_id, b.layer_stack_id, b.name, b.base_layer_id, b.head_commit_id, b.forked_from_id, s.layer_stack_id, s.name, s.scope_id, s.profile_id, s.head_layer_id, l.root_id, c.commit_id, c.layer_stack_id, c.root_id, c.parent_commit_id, c.base_layer_id, c.second_parent_id, c.branch_id, c.via, c.message, c.merged_from_id, c.created_ms, c.logical, c.new_canonical, c.new_stored, c.new_metadata, c.lww, c.displaced_id FROM branch b LEFT JOIN layer_stack s ON s.layer_stack_id = b.layer_stack_id LEFT JOIN layer l ON l.layer_id = b.base_layer_id LEFT JOIN "commit" c ON c.commit_id = b.head_commit_id WHERE b.branch_id = $1 AND NOT EXISTS (SELECT 1 FROM branch_retired r WHERE r.branch_id = b.branch_id) AND NOT EXISTS (SELECT 1 FROM layer_stack_retired d WHERE d.layer_stack_id = b.layer_stack_id); diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_snapshot_named.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_snapshot_named.sql index d44de96eba..beb7aa52ee 100644 --- a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_snapshot_named.sql +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/branch_snapshot_named.sql @@ -1 +1 @@ -SELECT b.branch_id, b.layer_stack_id, b.name, b.base_layer_id, b.head_commit_id, b.forked_from_id, s.layer_stack_id, s.name, s.scope_id, s.profile_id, s.head_layer_id, l.root_id, c.commit_id, c.layer_stack_id, c.root_id, c.parent_commit_id, c.base_layer_id, c.second_parent_id, c.branch_id, c.via, c.message, c.merged_from_id, c.created_ms, c.logical, c.new_canonical, c.new_stored, c.new_metadata, c.lww, c.displaced_id FROM layer_stack p LEFT JOIN branch b ON p.name = $1 AND b.layer_stack_id = p.layer_stack_id AND b.name = $2 LEFT JOIN layer_stack s ON s.layer_stack_id = b.layer_stack_id LEFT JOIN layer l ON l.layer_id = b.base_layer_id LEFT JOIN "commit" c ON c.commit_id = b.head_commit_id WHERE p.name = $1; +SELECT b.branch_id, b.layer_stack_id, b.name, b.base_layer_id, b.head_commit_id, b.forked_from_id, s.layer_stack_id, s.name, s.scope_id, s.profile_id, s.head_layer_id, l.root_id, c.commit_id, c.layer_stack_id, c.root_id, c.parent_commit_id, c.base_layer_id, c.second_parent_id, c.branch_id, c.via, c.message, c.merged_from_id, c.created_ms, c.logical, c.new_canonical, c.new_stored, c.new_metadata, c.lww, c.displaced_id FROM layer_stack p LEFT JOIN branch b ON p.name = $1 AND b.layer_stack_id = p.layer_stack_id AND b.name = $2 AND NOT EXISTS (SELECT 1 FROM branch_retired r WHERE r.branch_id = b.branch_id) LEFT JOIN layer_stack s ON s.layer_stack_id = b.layer_stack_id LEFT JOIN layer l ON l.layer_id = b.base_layer_id LEFT JOIN "commit" c ON c.commit_id = b.head_commit_id WHERE p.name = $1 AND NOT EXISTS (SELECT 1 FROM layer_stack_retired d WHERE d.layer_stack_id = p.layer_stack_id); diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_branch_stack_any.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_branch_stack_any.sql new file mode 100644 index 0000000000..2d1e79787e --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_branch_stack_any.sql @@ -0,0 +1 @@ +SELECT layer_stack_id FROM branch WHERE branch_id = $1; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_epoch_share.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_epoch_share.sql new file mode 100644 index 0000000000..67853fc6f2 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_epoch_share.sql @@ -0,0 +1 @@ +SELECT epoch FROM project_epoch WHERE layer_stack_id = $1; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_inode_blocks.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_inode_blocks.sql new file mode 100644 index 0000000000..433ca5fb2c --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_inode_blocks.sql @@ -0,0 +1 @@ +SELECT length(body) FROM inode_value_block ORDER BY first_ordinal diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_all_branch.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_all_branch.sql new file mode 100644 index 0000000000..42682b5053 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_all_branch.sql @@ -0,0 +1,10 @@ +SELECT op_id, kind, scope_kind, scope_id, workspace, sandbox, machine, project, branch, + owner_runtime, owner_boot, state, uncertain, phase +FROM gc_intent +WHERE state <> 'done' + AND ( + scope_kind = 'store' + OR ($1 IS NOT NULL AND scope_kind = 'project' AND scope_id = $1) + OR (scope_kind = 'branch' AND scope_id = $2) + ) + AND op_id <> $3; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_all_project.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_all_project.sql new file mode 100644 index 0000000000..e9b5bdeac6 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_all_project.sql @@ -0,0 +1,13 @@ +SELECT op_id, kind, scope_kind, scope_id, workspace, sandbox, machine, project, branch, + owner_runtime, owner_boot, state, uncertain, phase +FROM gc_intent +WHERE state <> 'done' + AND ( + scope_kind = 'store' + OR (scope_kind = 'project' AND scope_id = $1) + OR (scope_kind = 'branch' AND ( + ($2 IS NOT NULL AND project = $2) + OR scope_id IN (SELECT branch_id FROM branch WHERE layer_stack_id = $1) + )) + ) + AND op_id <> $3; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_all_store.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_all_store.sql new file mode 100644 index 0000000000..994f81301b --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_all_store.sql @@ -0,0 +1,5 @@ +SELECT op_id, kind, scope_kind, scope_id, workspace, sandbox, machine, project, branch, + owner_runtime, owner_boot, state, uncertain, phase +FROM gc_intent +WHERE state <> 'done' + AND op_id <> $1; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_get.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_get.sql new file mode 100644 index 0000000000..b4a0f1d551 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_get.sql @@ -0,0 +1,5 @@ +SELECT op_id, kind, scope_kind, scope_id, workspace, sandbox, machine, project, branch, + owner_runtime, owner_boot, state, uncertain, phase +FROM gc_intent +WHERE state <> 'done' + AND op_id = $1; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_insert.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_insert.sql new file mode 100644 index 0000000000..adc87b0b7d --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_insert.sql @@ -0,0 +1,8 @@ +INSERT INTO gc_intent (op_id, kind, scope_kind, scope_id, workspace, sandbox, machine, project, branch, owner_runtime, owner_boot, state, uncertain, phase, generation, artifact, high_water, evidence, updated_ms) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, ($13 <> 0), $14, NULL, NULL, NULL, NULL, $15) +ON CONFLICT(op_id) DO UPDATE SET + state = excluded.state, + uncertain = excluded.uncertain, + phase = excluded.phase, + updated_ms = excluded.updated_ms +RETURNING op_id; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_list.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_list.sql new file mode 100644 index 0000000000..56091ddb25 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_list.sql @@ -0,0 +1,6 @@ +SELECT op_id, kind, scope_kind, scope_id, workspace, sandbox, machine, project, branch, + owner_runtime, owner_boot, state, uncertain, phase +FROM gc_intent +WHERE state <> 'done' + AND scope_kind = $1 AND scope_id = $2 +ORDER BY updated_ms, op_id; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_ops_branch.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_ops_branch.sql new file mode 100644 index 0000000000..124e372af9 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_ops_branch.sql @@ -0,0 +1,11 @@ +SELECT op_id, kind, scope_kind, scope_id, workspace, sandbox, machine, project, branch, + owner_runtime, owner_boot, state, uncertain, phase +FROM gc_intent +WHERE state <> 'done' + AND kind IN ('removal', 'compaction') + AND ( + scope_kind = 'store' + OR ($1 IS NOT NULL AND scope_kind = 'project' AND scope_id = $1) + OR (scope_kind = 'branch' AND scope_id = $2) + ) + AND op_id <> $3; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_ops_project.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_ops_project.sql new file mode 100644 index 0000000000..c427223f56 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_ops_project.sql @@ -0,0 +1,14 @@ +SELECT op_id, kind, scope_kind, scope_id, workspace, sandbox, machine, project, branch, + owner_runtime, owner_boot, state, uncertain, phase +FROM gc_intent +WHERE state <> 'done' + AND kind IN ('removal', 'compaction') + AND ( + scope_kind = 'store' + OR (scope_kind = 'project' AND scope_id = $1) + OR (scope_kind = 'branch' AND ( + ($2 IS NOT NULL AND project = $2) + OR scope_id IN (SELECT branch_id FROM branch WHERE layer_stack_id = $1) + )) + ) + AND op_id <> $3; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_ops_store.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_ops_store.sql new file mode 100644 index 0000000000..655f4bf031 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_ops_store.sql @@ -0,0 +1,6 @@ +SELECT op_id, kind, scope_kind, scope_id, workspace, sandbox, machine, project, branch, + owner_runtime, owner_boot, state, uncertain, phase +FROM gc_intent +WHERE state <> 'done' + AND op_id <> $1 + AND kind IN ('removal', 'compaction'); diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_release.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_release.sql new file mode 100644 index 0000000000..4874b5ef1d --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_release.sql @@ -0,0 +1 @@ +DELETE FROM gc_intent WHERE op_id = $1 RETURNING op_id; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_uncertain.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_uncertain.sql new file mode 100644 index 0000000000..b47218d8b2 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_uncertain.sql @@ -0,0 +1 @@ +UPDATE gc_intent SET state = 'uncertain', uncertain = TRUE, updated_ms = $1 WHERE op_id = $2 RETURNING op_id; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_kept_roots.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_kept_roots.sql new file mode 100644 index 0000000000..180f300089 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_kept_roots.sql @@ -0,0 +1,7 @@ +SELECT c.root_id FROM kept_node k + JOIN "commit" c ON c.commit_id=k.node_id + WHERE k.kind='commit' +UNION +SELECT l.root_id FROM kept_node k + JOIN layer l ON l.layer_id=k.node_id + WHERE k.kind='layer' diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_live_branches.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_live_branches.sql new file mode 100644 index 0000000000..a20e525e28 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_live_branches.sql @@ -0,0 +1,4 @@ +SELECT b.branch_id,b.head_commit_id,b.base_layer_id FROM branch b + WHERE NOT EXISTS (SELECT 1 FROM branch_retired r WHERE r.branch_id=b.branch_id) + AND NOT EXISTS (SELECT 1 FROM layer_stack_retired s WHERE s.layer_stack_id=b.layer_stack_id) + ORDER BY b.branch_id diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_live_stacks.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_live_stacks.sql new file mode 100644 index 0000000000..0a20c54b38 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_live_stacks.sql @@ -0,0 +1,3 @@ +SELECT s.layer_stack_id,s.head_layer_id FROM layer_stack s + WHERE NOT EXISTS (SELECT 1 FROM layer_stack_retired r WHERE r.layer_stack_id=s.layer_stack_id) + ORDER BY s.layer_stack_id diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_object_locations.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_object_locations.sql new file mode 100644 index 0000000000..a96c9ecc9d --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_object_locations.sql @@ -0,0 +1,2 @@ +SELECT object_id,pack_id,group_number,record_number FROM object_location + ORDER BY pack_id,group_number,record_number diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/kept_node_clear.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/kept_node_clear.sql new file mode 100644 index 0000000000..2829711e8c --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/kept_node_clear.sql @@ -0,0 +1 @@ +DELETE FROM kept_node diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/kept_node_mark.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/kept_node_mark.sql new file mode 100644 index 0000000000..7dacb749e4 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/kept_node_mark.sql @@ -0,0 +1,81 @@ +WITH RECURSIVE +roots(kind,node_id) AS ( + SELECT 'stack',s.layer_stack_id FROM layer_stack s + WHERE NOT EXISTS (SELECT 1 FROM layer_stack_retired r WHERE r.layer_stack_id=s.layer_stack_id) + UNION + SELECT 'layer',l.layer_id FROM layer l + WHERE NOT EXISTS (SELECT 1 FROM layer_stack_retired r WHERE r.layer_stack_id=l.layer_stack_id) + UNION + SELECT 'branch',b.branch_id FROM branch b + WHERE NOT EXISTS (SELECT 1 FROM branch_retired r WHERE r.branch_id=b.branch_id) + AND NOT EXISTS (SELECT 1 FROM layer_stack_retired r WHERE r.layer_stack_id=b.layer_stack_id) +), +edges(from_kind,from_id,to_kind,to_id) AS ( + SELECT 'branch',b.branch_id,'commit',b.head_commit_id FROM branch b + JOIN "commit" c ON c.commit_id=b.head_commit_id + UNION ALL + SELECT 'branch',b.branch_id,'layer',b.base_layer_id FROM branch b + UNION ALL + SELECT 'branch',b.branch_id,'commit',b.forked_from_id FROM branch b + JOIN "commit" c ON c.commit_id=b.forked_from_id + UNION ALL + SELECT 'branch',b.branch_id,'layer',b.forked_from_id FROM branch b + JOIN layer l ON l.layer_id=b.forked_from_id + UNION ALL + SELECT 'layer',l.layer_id,'layer',l.parent_layer_id FROM layer l + WHERE l.parent_layer_id IS NOT NULL + UNION ALL + SELECT 'layer',l.layer_id,'commit',l.source_commit_id FROM layer l + WHERE l.source_commit_id IS NOT NULL + UNION ALL + SELECT 'layer',l.layer_id,'branch',l.source_branch_id FROM layer l + WHERE l.source_branch_id IS NOT NULL + UNION ALL + SELECT 'commit',c.commit_id,'commit',c.parent_commit_id FROM "commit" c + WHERE c.parent_commit_id IS NOT NULL + UNION ALL + SELECT 'commit',c.commit_id,'commit',c.second_parent_id FROM "commit" c + JOIN "commit" s ON s.commit_id=c.second_parent_id + UNION ALL + SELECT 'commit',c.commit_id,'layer',c.second_parent_id FROM "commit" c + JOIN layer l ON l.layer_id=c.second_parent_id + UNION ALL + SELECT 'commit',c.commit_id,'layer',c.base_layer_id FROM "commit" c + UNION ALL + SELECT 'commit',c.commit_id,'branch',c.branch_id FROM "commit" c + WHERE c.branch_id IS NOT NULL + UNION ALL + SELECT 'commit',c.commit_id,'commit',c.merged_from_id FROM "commit" c + JOIN "commit" s ON s.commit_id=c.merged_from_id + UNION ALL + SELECT 'commit',c.commit_id,'layer',c.merged_from_id FROM "commit" c + JOIN layer l ON l.layer_id=c.merged_from_id + UNION ALL + SELECT 'commit',c.commit_id,'commit',c.displaced_id FROM "commit" c + JOIN "commit" s ON s.commit_id=c.displaced_id + UNION ALL + SELECT 'commit',c.commit_id,'layer',c.displaced_id FROM "commit" c + JOIN layer l ON l.layer_id=c.displaced_id +), +closure(kind,node_id) AS ( + SELECT kind,node_id FROM roots + UNION + SELECT e.to_kind,e.to_id + FROM closure k JOIN edges e ON e.from_kind=k.kind AND e.from_id=k.node_id +) +INSERT INTO kept_node(layer_stack_id,kind,node_id,serial) +SELECT s.layer_stack_id,'stack',s.layer_stack_id,0 + FROM closure k JOIN layer_stack s ON s.layer_stack_id=k.node_id + WHERE k.kind='stack' +UNION ALL +SELECT b.layer_stack_id,'branch',b.branch_id,0 + FROM closure k JOIN branch b ON b.branch_id=k.node_id + WHERE k.kind='branch' +UNION ALL +SELECT l.layer_stack_id,'layer',l.layer_id,l.history_serial + FROM closure k JOIN layer l ON l.layer_id=k.node_id + WHERE k.kind='layer' +UNION ALL +SELECT c.layer_stack_id,'commit',c.commit_id,c.history_serial + FROM closure k JOIN "commit" c ON c.commit_id=k.node_id + WHERE k.kind='commit' diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/kept_node_read.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/kept_node_read.sql new file mode 100644 index 0000000000..3673d561ce --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/kept_node_read.sql @@ -0,0 +1 @@ +SELECT layer_stack_id,kind,node_id,serial FROM kept_node ORDER BY layer_stack_id,kind,node_id diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/layerstack_bump_project_epoch.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/layerstack_bump_project_epoch.sql index a975e5236a..0ef48580ea 100644 --- a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/layerstack_bump_project_epoch.sql +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/layerstack_bump_project_epoch.sql @@ -1 +1 @@ -INSERT INTO project_epoch (layer_stack_id, epoch) VALUES ($1, 1) ON CONFLICT (layer_stack_id) DO UPDATE SET epoch = epoch + 1 RETURNING 1; +UPDATE project_epoch SET epoch = epoch + 1 WHERE layer_stack_id = $1 RETURNING 1; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/layerstack_stack_name_taken.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/layerstack_stack_name_taken.sql index 875ddc4086..b1e577c213 100644 --- a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/layerstack_stack_name_taken.sql +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/layerstack_stack_name_taken.sql @@ -1 +1 @@ -SELECT 1 FROM layer_stack WHERE name = $1; +SELECT s.name, CASE WHEN r.layer_stack_id IS NULL THEN 0 ELSE 1 END FROM layer_stack s LEFT JOIN layer_stack_retired r ON r.layer_stack_id = s.layer_stack_id WHERE s.name = $1; diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/history/statements.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/history/statements.rs index 0063bf9f73..bdc010b348 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/history/statements.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/history/statements.rs @@ -12,6 +12,27 @@ pub(crate) fn statement(key: &str) -> Result<&'static str, BackendError> { "statistics_totals" => Ok(include_str!( "../../../../sql/postgresql/queries/history/statistics_totals.sql" )), + "gc_epoch_share" => Ok(include_str!( + "../../../../sql/postgresql/queries/history/gc_epoch_share.sql" + )), + "gc_lock_xact_shared" => Ok(include_str!( + "../../../../sql/postgresql/queries/history/gc_lock_xact_shared.sql" + )), + "gc_lock_xact_exclusive" => Ok(include_str!( + "../../../../sql/postgresql/queries/history/gc_lock_xact_exclusive.sql" + )), + "gc_lock_session_shared" => Ok(include_str!( + "../../../../sql/postgresql/queries/history/gc_lock_session_shared.sql" + )), + "gc_lock_session_exclusive" => Ok(include_str!( + "../../../../sql/postgresql/queries/history/gc_lock_session_exclusive.sql" + )), + "gc_unlock_session_shared" => Ok(include_str!( + "../../../../sql/postgresql/queries/history/gc_unlock_session_shared.sql" + )), + "gc_unlock_session_exclusive" => Ok(include_str!( + "../../../../sql/postgresql/queries/history/gc_unlock_session_exclusive.sql" + )), other => crate::backend::sqlite::session::query::history(other), } } diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/mechanics.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/mechanics.rs index 7189700f7e..3bca07b93e 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/mechanics.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/mechanics.rs @@ -16,6 +16,9 @@ use layerfs_storage::{ use postgres::types::ToSql; impl HistoryStatements for Transaction<'_> { + fn mechanics(&self) -> &dyn Mechanics { + self + } fn history(&self, key: &str, params: Vec) -> Result, BackendError> { match history::statements::qualified(key) { Some(sql) => self.query(&sql.replace("{s}", self.schema), params), diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs index fa3115e6ba..6bc014d0e9 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs @@ -11,8 +11,10 @@ //! 10b, `acquisition.sql`), recorded in its schema source; one an earlier //! release imported from SQLite has none and refuses acquisition. A Store //! written before the GC tables gains them at its first writable open -//! ([`add_gc`]); any other schema mismatch is still refused. A merge's -//! records live in the writer connection's temporary tables +//! ([`add_gc`]), and one written before the position columns gains them, +//! with the T1.4 serial backfill, at that same open +//! ([`add_position_columns`]); any other schema mismatch is still refused. +//! A merge's records live in the writer connection's temporary tables //! (`merge_records.rs`), not here. use super::session::{ connection::Session, @@ -36,6 +38,12 @@ const PRE_GC_SCHEMA: &str = include_str!("../../../sql/postgresql/schema.sql"); /// The `history_meta` position column as `schema.sql` writes it; it sits on /// its own line, so removing the line leaves the text an older Store recorded. const NEXT_SERIAL: &str = " next_history_serial bigint NOT NULL DEFAULT 1,\n"; +/// The `commit` and `layer` position column as `schema.sql` writes it; it +/// sits on its own line in each of the two tables. +const HISTORY_SERIAL: &str = " history_serial bigint NOT NULL DEFAULT 0,\n"; +/// The `store_policy` position salt column, with the comma the column before +/// it gains, as `schema.sql` writes them; it ends that table's column list. +const TRACE_SALT: &str = ",\n trace_salt bytea NOT NULL DEFAULT decode(repeat('00',32),'hex') CHECK (length(trace_salt)=32)"; /// The GC marker, position and admission tables; a Store written before them /// gains them at open ([`add_gc`]). const GC: &str = include_str!("../../../sql/postgresql/gc.sql"); @@ -49,7 +57,7 @@ const ACQUISITION_TABLES: [&str; 3] = ["init_entry", "init_host_file", "init_ope /// (#90); a version 1 schema is refused at open, never converted. pub(crate) const SCHEMA_VERSION: i64 = 2; /// Every table of one Store schema, in name order. -pub(crate) const TABLES: [&str; 26] = [ +pub(crate) const TABLES: [&str; 27] = [ "absorbed", "binding_hint_block", "branch", @@ -63,6 +71,7 @@ pub(crate) const TABLES: [&str; 26] = [ "gc_op", "history_meta", "inode_value_block", + "kept_node", "layer", "layer_lineage", "layer_stack", @@ -130,20 +139,33 @@ fn pre_gc_source(acquisition: bool) -> String { } } -/// The current source before [`NEXT_SERIAL`] was written. -fn source_without_next_serial(acquisition: bool) -> String { - source(acquisition).replacen(NEXT_SERIAL, "", 1) +/// `text` as a Store wrote it before the position columns ([`HISTORY_SERIAL`] +/// in `commit` and `layer`, [`TRACE_SALT`] in `store_policy`) existed. +fn without_position_columns(text: String) -> String { + text.replacen(HISTORY_SERIAL, "", 2) + .replacen(TRACE_SALT, "", 1) } -/// The pre-GC source before [`NEXT_SERIAL`] was written. -fn pre_gc_source_without_next_serial(acquisition: bool) -> String { - pre_gc_source(acquisition).replacen(NEXT_SERIAL, "", 1) +/// Whether `recorded` is `current` or an earlier revision of it a Store may +/// have written: before [`NEXT_SERIAL`], before the position columns, or +/// before both. An additive migration matches one of those and refuses any +/// text no release wrote. +fn is_earlier_source(recorded: &str, current: String) -> bool { + let without_columns = without_position_columns(current.clone()); + [ + current.clone(), + current.replacen(NEXT_SERIAL, "", 1), + without_columns.clone(), + without_columns.replacen(NEXT_SERIAL, "", 1), + ] + .iter() + .any(|revision| revision == recorded) } /// Creates the new schema `tx.schema` with the Store and acquisition tables, -/// its policy row and its catalog row, inside the caller's transaction. A -/// schema of that name that already exists is refused before anything is -/// written; nothing is merged into it. +/// its policy row with one random trace salt and its catalog row, inside the +/// caller's transaction. A schema of that name that already exists is +/// refused before anything is written; nothing is merged into it. pub(crate) fn create( tx: &Transaction<'_>, policy: StoragePolicy, @@ -164,7 +186,7 @@ pub(crate) fn create( "CREATE SCHEMA {name}; SET LOCAL search_path TO {name}; {SCHEMA}{ACQUISITION}" ))?; tx.query( - "INSERT INTO store_policy(id,schema_version,format_profile,small_file_threshold_bytes,whole_file_delta_max_depth,chunk_delta_max_depth,metadata_delta_max_depth) VALUES(1,$1,$2,$3,$4,$5,$6)", + "INSERT INTO store_policy(id,schema_version,format_profile,small_file_threshold_bytes,whole_file_delta_max_depth,chunk_delta_max_depth,metadata_delta_max_depth,trace_salt) VALUES(1,$1,$2,$3,$4,$5,$6,decode(md5(random()::text||clock_timestamp()::text)||md5(clock_timestamp()::text||random()::text),'hex'))", vec![ Param::I64(SCHEMA_VERSION), Param::I64(i64::from(policy.format_profile())), @@ -189,6 +211,79 @@ pub(crate) fn create( Ok(()) } +/// The `kept_node` table as `gc.sql` writes it: one statement appended to the +/// GC script, so removing exactly this text leaves the source a Store written +/// before T7.1 recorded. +fn kept_node_table() -> &'static str { + let start = GC + .find("CREATE TABLE kept_node (") + .expect("the GC schema holds kept_node"); + let end = start + GC[start..].find(";\n").expect("kept_node ends") + 2; + &GC[start..end] +} + +/// The recorded source of a Store written before [`kept_node_table`]: the +/// current source with that one statement removed. +fn pre_kept_node_source(acquisition: bool) -> String { + source(acquisition).replacen(kept_node_table(), "", 1) +} + +/// Adds the `kept_node` history-closure table (T7.1) to a Store written +/// before it, then records the current schema: an additive migration under +/// the same conditions as [`add_gc`]. A schema that already has the table is +/// left alone; a recorded source or definition that is not an earlier +/// revision of the current one is refused, never migrated. The table is empty +/// until a mark writes it. +pub(crate) fn add_kept_node(session: &Session) -> Result<(), PersistenceError> { + // A Store that already has the table is left alone: one statement, no + // transaction and no lane wait. + let present = session.run_as::<_, BackendError>(Mode::Statement, |tx| { + let sql = format!( + "SELECT 1 FROM information_schema.tables WHERE table_schema='{}' AND table_name='kept_node' AND table_type='BASE TABLE'", + tx.schema + ); + Ok(!tx.query(&sql, vec![])?.is_empty()) + })?; + if present { + return Ok(()); + } + session.run_as(Mode::Write, |tx| -> Result<(), PersistenceError> { + // Re-check under the lane: another process may have migrated first. + let present = tx.query( + "SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='kept_node' AND table_type='BASE TABLE'", + vec![], + )?; + if !present.is_empty() { + return Ok(()); + } + let rows = tx.query( + "SELECT schema_source,schema_definition FROM history_meta WHERE id=1", + vec![], + )?; + let [row] = rows.as_slice() else { + return Err(BackendError::Integrity.into()); + }; + let recorded_source: String = row.get(0)?; + let acquisition = if is_earlier_source(&recorded_source, pre_kept_node_source(false)) { + false + } else if is_earlier_source(&recorded_source, pre_kept_node_source(true)) { + true + } else { + return Err(BackendError::Integrity.into()); + }; + if row.get::(1)? != definition(tx)? { + return Err(BackendError::Integrity.into()); + } + tx.batch(kept_node_table())?; + let definition = definition(tx)?; + tx.query( + "UPDATE history_meta SET schema_source=$1,schema_definition=$2 WHERE id=1", + vec![Param::Text(source(acquisition)), Param::Text(definition)], + )?; + Ok(()) + }) +} + /// Adds the GC marker, position and admission tables to a Store written /// before them, then records the current schema: an additive migration under /// the same conditions as the SQLite Store's `add_gc`. A schema that already @@ -224,13 +319,9 @@ pub(crate) fn add_gc(session: &Session) -> Result<(), PersistenceError> { return Err(BackendError::Integrity.into()); }; let recorded_source: String = row.get(0)?; - let acquisition = if recorded_source == pre_gc_source(false) - || recorded_source == pre_gc_source_without_next_serial(false) - { + let acquisition = if is_earlier_source(&recorded_source, pre_gc_source(false)) { false - } else if recorded_source == pre_gc_source(true) - || recorded_source == pre_gc_source_without_next_serial(true) - { + } else if is_earlier_source(&recorded_source, pre_gc_source(true)) { true } else { return Err(BackendError::Integrity.into()); @@ -262,10 +353,10 @@ pub(crate) fn add_gc(session: &Session) -> Result<(), PersistenceError> { /// Store written before it, then records the current schema: an additive /// migration under the same conditions as [`add_gc`]. A schema that already /// has the column is left alone; a recorded source or definition that is not -/// one of the earlier schemas is refused, never migrated. PostgreSQL has no -/// `commit`/`layer` serial columns yet, so only the column and one -/// `project_epoch` row per project are recorded here; the serial backfill is -/// SQLite's. +/// an earlier revision of the current one is refused, never migrated. The +/// `commit`/`layer` serial columns and their backfill follow in +/// [`add_position_columns`], the next step of the same open; only the column +/// and one `project_epoch` row per project are recorded here. pub(crate) fn add_position_state(session: &Session) -> Result<(), PersistenceError> { // A Store that already has the column is left alone: one statement, no // transaction and no lane wait. @@ -296,13 +387,9 @@ pub(crate) fn add_position_state(session: &Session) -> Result<(), PersistenceErr return Err(BackendError::Integrity.into()); }; let recorded_source: String = row.get(0)?; - let acquisition = if recorded_source == source(false) - || recorded_source == source_without_next_serial(false) - { + let acquisition = if is_earlier_source(&recorded_source, source(false)) { false - } else if recorded_source == source(true) - || recorded_source == source_without_next_serial(true) - { + } else if is_earlier_source(&recorded_source, source(true)) { true } else { return Err(BackendError::Integrity.into()); @@ -323,6 +410,139 @@ pub(crate) fn add_position_state(session: &Session) -> Result<(), PersistenceErr }) } +/// The ALTERs an old Store needs for the position columns, the one random +/// trace salt, and the T1.4 serial backfill. `gen_random_bytes` is used when +/// the server can reach it; without pgcrypto two md5 digests of the clock and +/// `random()` stand in for the same 32 bytes. +const POSITION_COLUMNS: &str = concat!( + "ALTER TABLE \"commit\" ADD COLUMN history_serial bigint NOT NULL DEFAULT 0;", + "ALTER TABLE layer ADD COLUMN history_serial bigint NOT NULL DEFAULT 0;", + "ALTER TABLE store_policy ADD COLUMN trace_salt bytea NOT NULL DEFAULT decode(repeat('00',32),'hex') CHECK (length(trace_salt)=32);", + "DO $$ BEGIN ", + "UPDATE store_policy SET trace_salt=gen_random_bytes(32) WHERE id=1; ", + "EXCEPTION WHEN undefined_function THEN ", + "UPDATE store_policy SET trace_salt=decode(md5(random()::text||clock_timestamp()::text)||md5(clock_timestamp()::text||random()::text),'hex') WHERE id=1; ", + "END $$;" +); + +/// Adds the position columns T1.2c introduced to a Store written before them: +/// `commit.history_serial` and `layer.history_serial` (default 0) and +/// `store_policy.trace_salt` (32 random bytes), then backfills the serials +/// with T1.4's semantics in the same transaction. An additive migration under +/// the same conditions as [`add_gc`]: a schema that already has +/// `commit.history_serial` is left alone, and a recorded source or definition +/// that is not an earlier revision of the current one is refused, never +/// migrated. +pub(crate) fn add_position_columns(session: &Session) -> Result<(), PersistenceError> { + // A Store that already has the columns is left alone: one statement, no + // transaction and no lane wait. + let present = session.run_as::<_, BackendError>(Mode::Statement, |tx| { + let sql = format!( + "SELECT 1 FROM information_schema.columns WHERE table_schema='{}' AND table_name='commit' AND column_name='history_serial'", + tx.schema + ); + Ok(!tx.query(&sql, vec![])?.is_empty()) + })?; + if present { + return Ok(()); + } + session.run_as(Mode::Write, |tx| -> Result<(), PersistenceError> { + // Re-check under the lane: another process may have migrated first. + let present = tx.query( + "SELECT 1 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='commit' AND column_name='history_serial'", + vec![], + )?; + if !present.is_empty() { + return Ok(()); + } + let rows = tx.query( + "SELECT schema_source,schema_definition FROM history_meta WHERE id=1", + vec![], + )?; + let [row] = rows.as_slice() else { + return Err(BackendError::Integrity.into()); + }; + let recorded_source: String = row.get(0)?; + let acquisition = if is_earlier_source(&recorded_source, source(false)) { + false + } else if is_earlier_source(&recorded_source, source(true)) { + true + } else { + return Err(BackendError::Integrity.into()); + }; + if row.get::(1)? != definition(tx)? { + return Err(BackendError::Integrity.into()); + } + tx.batch(POSITION_COLUMNS)?; + backfill_position_serials(tx)?; + let definition = definition(tx)?; + tx.query( + "UPDATE history_meta SET schema_source=$1,schema_definition=$2 WHERE id=1", + vec![Param::Text(source(acquisition)), Param::Text(definition)], + )?; + Ok(()) + }) +} + +/// The T1.4 serial backfill: Commits take the next Store serials in +/// `(created_ms, commit_id)` order, then each project's genesis Layer takes +/// the next one in stack-id order, every other Layer republishes its source +/// Commit's serial, existing serials are kept and `next_history_serial` never +/// falls below `MAX(serial) + 1`. +fn backfill_position_serials(tx: &Transaction<'_>) -> Result<(), BackendError> { + let next: i64 = tx + .query( + "SELECT next_history_serial FROM history_meta WHERE id=1", + vec![], + )? + .first() + .ok_or(BackendError::Integrity)? + .get(0)?; + let commit_max: i64 = tx + .query( + "SELECT COALESCE(MAX(history_serial),0) FROM \"commit\"", + vec![], + )? + .first() + .ok_or(BackendError::Integrity)? + .get(0)?; + let layer_max: i64 = tx + .query("SELECT COALESCE(MAX(history_serial),0) FROM layer", vec![])? + .first() + .ok_or(BackendError::Integrity)? + .get(0)?; + let base = next.max(commit_max.max(layer_max) + 1); + tx.query( + &format!( + "WITH ordered AS (SELECT commit_id,{base}+ROW_NUMBER() OVER (ORDER BY created_ms,commit_id)-1 AS serial FROM \"commit\" WHERE history_serial=0) UPDATE \"commit\" AS c SET history_serial=ordered.serial FROM ordered WHERE ordered.commit_id=c.commit_id AND c.history_serial=0" + ), + vec![], + )?; + let after_commits: i64 = tx + .query( + "SELECT COALESCE(MAX(history_serial),0)+1 FROM \"commit\"", + vec![], + )? + .first() + .ok_or(BackendError::Integrity)? + .get(0)?; + tx.query( + &format!( + "WITH ordered AS (SELECT layer_id,{after_commits}+ROW_NUMBER() OVER (ORDER BY layer_stack_id,layer_id)-1 AS serial FROM layer WHERE history_serial=0 AND parent_layer_id IS NULL) UPDATE layer AS l SET history_serial=ordered.serial FROM ordered WHERE ordered.layer_id=l.layer_id AND l.history_serial=0 AND l.parent_layer_id IS NULL" + ), + vec![], + )?; + tx.query( + "UPDATE layer AS l SET history_serial=(SELECT c.history_serial FROM \"commit\" AS c WHERE c.layer_stack_id=l.layer_stack_id AND c.commit_id=l.source_commit_id) WHERE l.history_serial=0 AND l.source_commit_id IS NOT NULL", + vec![], + )?; + tx.query( + "UPDATE history_meta SET next_history_serial=GREATEST(next_history_serial,(SELECT COALESCE(MAX(history_serial),0)+1 FROM (SELECT history_serial FROM \"commit\" UNION ALL SELECT history_serial FROM layer) AS assigned)) WHERE id=1", + vec![], + )?; + Ok(()) +} + /// What an open's check read: the catalog's incarnation, whether the schema /// has the acquisition tables, the recorded binding, the frozen policy and /// the server's version. diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/schema_diagnostic.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/schema_diagnostic.rs index dc4eb5ea1f..32e5f2cdee 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/schema_diagnostic.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/schema_diagnostic.rs @@ -31,6 +31,7 @@ commit new_stored bigint NO commit new_metadata bigint NO commit lww bytea NO commit displaced_id bytea YES +commit history_serial bigint NO commit_lineage commit_id bytea NO commit_lineage run bytea NO commit_lineage depth bigint NO @@ -126,6 +127,10 @@ init_operation removed_bytes bigint NO inode_value_block first_ordinal bigint NO inode_value_block last_ordinal bigint NO inode_value_block body bytea NO +kept_node layer_stack_id bytea NO +kept_node kind text NO +kept_node node_id bytea NO +kept_node serial bigint NO layer layer_id bytea NO layer layer_stack_id bytea NO layer parent_layer_id bytea YES @@ -134,6 +139,7 @@ layer source_branch_id bytea YES layer logical bigint NO layer overwrote bytea NO layer source_commit_id bytea YES +layer history_serial bigint NO layer_lineage layer_id bytea NO layer_lineage layer_stack_id bytea NO layer_lineage height bigint NO @@ -179,6 +185,7 @@ store_policy chunk_delta_max_depth bigint NO store_policy metadata_delta_max_depth bigint NO store_policy next_pack_id bigint NO store_policy next_ordinal bigint NO +store_policy trace_salt bytea NO store_totals id bigint NO store_totals logical bigint NO store_totals canonical bigint NO diff --git a/engine/crates/layerfs-persistence/src/backend/seam.rs b/engine/crates/layerfs-persistence/src/backend/seam.rs index 5108cbed52..e654cf8877 100644 --- a/engine/crates/layerfs-persistence/src/backend/seam.rs +++ b/engine/crates/layerfs-persistence/src/backend/seam.rs @@ -23,6 +23,8 @@ pub(crate) enum Engine { /// Named history statements in one open engine transaction. pub(crate) trait HistoryStatements { fn history(&self, key: &str, params: Vec) -> Result, BackendError>; + /// The same open transaction's physical object and metadata access. + fn mechanics(&self) -> &dyn Mechanics; /// One statement over parallel column arrays, where the engine binds /// arrays (PostgreSQL: one round trip for many rows). `None`: the /// engine has no such statement and the caller runs its per-row one. diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/mechanics.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/mechanics.rs index ef0b9c1464..1407870041 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/mechanics.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/mechanics.rs @@ -15,6 +15,9 @@ impl HistoryStatements for Transaction<'_> { fn history(&self, key: &str, params: Vec) -> Result, BackendError> { Transaction::history(self, key, params) } + fn mechanics(&self) -> &dyn Mechanics { + self + } } impl Mechanics for Transaction<'_> { diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/schema.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/schema.rs index 188dcb1a9b..d5685f4960 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/schema.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/schema.rs @@ -18,6 +18,30 @@ pub(crate) const DIGESTS: &str = include_str!("../../../sql/sqlite/file_digest.s /// GC markers, positions and admission records; a Store written before them /// gains them at open. pub(crate) const GC: &str = include_str!("../../../sql/sqlite/gc.sql"); +/// The position columns T1.2a/T1.4 append, as `("table", "column", ALTER)`; +/// a Store written before them gains them at open. +pub(crate) const POSITION_COLUMNS: [(&str, &str, &str); 4] = [ + ( + "\"commit\"", + "history_serial", + "ALTER TABLE \"commit\" ADD COLUMN history_serial INTEGER NOT NULL DEFAULT 0", + ), + ( + "layer", + "history_serial", + "ALTER TABLE layer ADD COLUMN history_serial INTEGER NOT NULL DEFAULT 0", + ), + ( + "store_policy", + "trace_salt", + "ALTER TABLE store_policy ADD COLUMN trace_salt BLOB NOT NULL DEFAULT (X'0000000000000000000000000000000000000000000000000000000000000000') CHECK(length(trace_salt)=32)", + ), + ( + "history_meta", + "next_history_serial", + "ALTER TABLE history_meta ADD COLUMN next_history_serial INTEGER NOT NULL DEFAULT 1", + ), +]; const UNIT_INDEX: &str = include_str!("../../../sql/sqlite/objects_units_index.sql"); const UNITS: &str = include_str!("../../../sql/sqlite/objects_units.sql"); const ACQUISITION: &str = include_str!("../../../sql/sqlite/acquisition/schema.sql"); @@ -75,6 +99,17 @@ fn next_history_serial_column() -> &'static str { &HISTORY[start..end] } +/// The `kept_node` table as `gc.sql` writes it: one appended statement at the +/// end of the GC script, so removing exactly this text leaves the source a +/// Store written before T7.1 recorded. +fn kept_node_table() -> &'static str { + let start = GC + .find("CREATE TABLE kept_node (") + .expect("the GC schema holds kept_node"); + let end = start + GC[start..].find(";\n").expect("kept_node ends") + 2; + &GC[start..end] +} + /// The `commit` and `layer` position column as `history.sql` writes it; it /// appears once in each of the two tables. fn history_serial_column() -> &'static str { @@ -85,12 +120,19 @@ fn history_serial_column() -> &'static str { &HISTORY[start..end] } -/// The `store_policy` position salt column as `metadata.sql` writes it. +/// The `store_policy` position salt column as `metadata.sql` writes it, +/// together with the comma that ends `next_ordinal`: it was appended as the +/// table's last column, so a Store written before it recorded neither the +/// line nor the comma. fn trace_salt_column() -> &'static str { let start = METADATA - .find(" trace_salt BLOB NOT NULL DEFAULT") + .find(",\n trace_salt BLOB NOT NULL DEFAULT") .expect("metadata schema holds the salt column"); - let end = start + METADATA[start..].find('\n').expect("column line ends") + 1; + // The search starts past the comma and newline; the newline that ends the + // column is kept, since it is also the one before the table's closing + // `) STRICT;` that the pre-T1.2a text held. + let line = start + 2; + let end = line + METADATA[line..].find('\n').expect("column line ends"); &METADATA[start..end] } @@ -109,18 +151,9 @@ fn without_position_columns(source: String) -> String { /// The ALTERs and salt backfill an old Store needs; existing rows keep serial /// 0 and the Store keeps one random salt. fn add_position_columns(tx: &Transaction<'_>) -> Result<(), BackendError> { - tx.query( - "ALTER TABLE \"commit\" ADD COLUMN history_serial INTEGER NOT NULL DEFAULT 0", - vec![], - )?; - tx.query( - "ALTER TABLE layer ADD COLUMN history_serial INTEGER NOT NULL DEFAULT 0", - vec![], - )?; - tx.query( - "ALTER TABLE store_policy ADD COLUMN trace_salt BLOB NOT NULL DEFAULT (X'0000000000000000000000000000000000000000000000000000000000000000') CHECK(length(trace_salt)=32)", - vec![], - )?; + for (_, _, alter) in POSITION_COLUMNS.iter().take(3) { + tx.query(alter, vec![])?; + } tx.query( "UPDATE store_policy SET trace_salt=randomblob(32) WHERE id=1", vec![], @@ -254,6 +287,7 @@ pub(crate) fn check( "path_removed", "history_meta", "inode_value_block", + "kept_node", "layer", "layer_lineage", "layer_stack", @@ -544,6 +578,44 @@ impl super::session::connection::Session { Ok(()) }) } + /// Adds the `kept_node` history-closure table (T7.1) to a Store written + /// before it, then records the current schema: an additive migration + /// (REBUILD.md kind 2) under the same conditions as + /// [`Self::add_binding_hints`]. The table is empty until a mark writes it. + pub(crate) fn add_kept_node(&self) -> Result<(), BackendError> { + self.run::<_, BackendError>(true, |tx| { + let present = tx.query( + "SELECT 1 FROM sqlite_schema WHERE type='table' AND name='kept_node'", + vec![], + )?; + if !present.is_empty() { + return Ok(()); + } + let rows = tx.query( + "SELECT schema_source,schema_definition FROM history_meta WHERE id=1", + vec![], + )?; + let Some(row) = rows.first() else { + return Ok(()); + }; + let current = source((self.layout, self.acquisition)); + let recorded = row.get::(0)?; + let without = current.replacen(kept_node_table(), "", 1); + let earlier = without.replacen(next_history_serial_column(), "", 1); + if (recorded != earlier && recorded != without) + || row.get::(1)? != definition(tx.connection, tx.work)? + { + return Ok(()); + } + tx.bootstrap(kept_node_table())?; + let definition = definition(tx.connection, tx.work)?; + tx.query( + "UPDATE history_meta SET schema_source=?1,schema_definition=?2 WHERE id=1", + vec![Param::Text(current), Param::Text(definition)], + )?; + Ok(()) + }) + } /// Adds the position state T1.4 introduced (`history_meta.next_history_serial`, /// appended NOT NULL DEFAULT 1) and backfills the `commit`/`layer` serials /// and `project_epoch` rows in the same write transaction: an additive @@ -575,10 +647,7 @@ impl super::session::connection::Session { { return Ok(()); } - tx.query( - "ALTER TABLE history_meta ADD COLUMN next_history_serial INTEGER NOT NULL DEFAULT 1", - vec![], - )?; + tx.query(POSITION_COLUMNS[3].2, vec![])?; backfill_position_state(tx)?; let definition = definition(tx.connection, tx.work)?; tx.query( diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/schema_diagnostic.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/schema_diagnostic.rs index 50d2864f56..7a3ad4d0fc 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/schema_diagnostic.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/schema_diagnostic.rs @@ -3,7 +3,7 @@ use super::{ schema, session::{connection::SqlWork, query, rows, transaction::Transaction}, }; -use crate::backend::records::{BackendError, Param}; +use crate::backend::records::{BackendError, Param, Record}; use std::cell::RefCell; /// The scratch database contains only the finite shipped schema, never Store @@ -19,19 +19,26 @@ pub(crate) fn check( trusted.execute_batch(&script).map_err(rows::error)?; } let work = RefCell::new(SqlWork::default()); - let expected = query::run(&trusted, - "SELECT type,name,sql FROM sqlite_schema WHERE substr(name,1,7)!='sqlite_' ORDER BY type,name", - vec![], &work)?; - let mut definition = String::new(); - for row in &expected { - for index in 0..3 { - definition.push_str(&row.get::(index)?); - definition.push('\n'); - } + let shipped = schema_rows(&trusted, &work)?; + let shipped_definition = definition(&shipped)?; + // A Store the additive position-column migrations brought forward records + // the definition SQLite rewrote: the appended columns end the column list + // instead of sitting where the shipped schema declares them. The doctor + // admits that rewritten schema as well as the shipped one (REBUILD.md kind + // 2), and rejects every other difference. + for (table, column, _) in schema::POSITION_COLUMNS { + trusted + .execute_batch(&format!("ALTER TABLE {table} DROP COLUMN {column}")) + .map_err(rows::error)?; } + for (_, _, alter) in schema::POSITION_COLUMNS { + trusted.execute_batch(alter).map_err(rows::error)?; + } + let migrated = schema_rows(&trusted, &work)?; + let migrated_definition = definition(&migrated)?; let claims = tx.query( - "SELECT count(*) FROM history_meta WHERE id=1 AND typeof(catalog_id)='blob' AND length(catalog_id)=32 AND typeof(catalog_incarnation)='integer' AND typeof(identity_format)='integer' AND typeof(schema_version)='integer' AND typeof(binding_key)='blob' AND length(binding_key) BETWEEN 1 AND 128 AND typeof(schema_source)='text' AND schema_source=?1 AND typeof(schema_definition)='text' AND schema_definition=?2", - vec![Param::Text(schema::source(selection)), Param::Text(definition)], + "SELECT count(*) FROM history_meta WHERE id=1 AND typeof(catalog_id)='blob' AND length(catalog_id)=32 AND typeof(catalog_incarnation)='integer' AND typeof(identity_format)='integer' AND typeof(schema_version)='integer' AND typeof(binding_key)='blob' AND length(binding_key) BETWEEN 1 AND 128 AND typeof(schema_source)='text' AND schema_source=?1 AND typeof(schema_definition)='text'", + vec![Param::Text(schema::source(selection))], )?; if claims .first() @@ -41,6 +48,20 @@ pub(crate) fn check( { return Err(BackendError::Integrity); } + let recorded = tx.query( + "SELECT schema_source,schema_definition FROM history_meta WHERE id=1", + vec![], + )?; + let recorded = recorded + .first() + .filter(|_| recorded.len() == 1) + .ok_or(BackendError::Integrity)?; + let recorded_definition: String = recorded.get(1)?; + if !same_schema(&recorded_definition, &shipped_definition) + && !same_schema(&recorded_definition, &migrated_definition) + { + return Err(BackendError::Integrity); + } let policy = tx.query("SELECT count(*) FROM store_policy WHERE id=1 AND typeof(small_file_threshold_bytes)='integer'", vec![])?; if policy .first() @@ -58,29 +79,111 @@ pub(crate) fn check( .first() .ok_or(BackendError::Integrity)? .get::(0)? - != expected.len() as i64 + != shipped.len() as i64 { return Err(BackendError::Integrity); } - for row in expected { - let matched = tx.query( - "SELECT count(*) FROM sqlite_schema WHERE type=?1 AND name=?2 AND sql=?3", - (0..3) - .map(|index| row.get::(index).map(Param::Text)) - .collect::>()?, + for (index, row) in shipped.iter().enumerate() { + let kind: String = row.get(0)?; + let name: String = row.get(1)?; + let live = tx.query( + "SELECT sql FROM sqlite_schema WHERE type=?1 AND name=?2", + vec![Param::Text(kind), Param::Text(name)], )?; - if matched - .first() - .ok_or(BackendError::Integrity)? - .get::(0)? - != 1 - { + let live: String = live.first().ok_or(BackendError::Integrity)?.get(0)?; + let shipped_sql: String = row.get(2)?; + let migrated_sql: String = migrated[index].get(2)?; + if !same_schema(&live, &shipped_sql) && !same_schema(&live, &migrated_sql) { return Err(BackendError::Integrity); } } Ok(()) } +/// The schema rows the shipped scripts create, ordered as recorded. +fn schema_rows( + connection: &rusqlite::Connection, + work: &RefCell, +) -> Result, BackendError> { + query::run( + connection, + "SELECT type,name,sql FROM sqlite_schema WHERE substr(name,1,7)!='sqlite_' ORDER BY type,name", + vec![], + work, + ) +} + +/// The recorded definition text for one row set. +fn definition(rows: &[Record]) -> Result { + let mut definition = String::new(); + for row in rows { + for index in 0..3 { + definition.push_str(&row.get::(index)?); + definition.push('\n'); + } + } + Ok(definition) +} + +/// Whether two schema texts carry the same SQL. SQLite rewrites a table's +/// stored definition when a column is appended, moving only whitespace and a +/// separator; every token must still match, so no other schema is admitted. +fn same_schema(left: &str, right: &str) -> bool { + tokens(left) == tokens(right) +} + +/// The SQL tokens of a schema text, with whitespace excluded. A quoted +/// identifier, string or blob stays one token, as does a run of identifier +/// characters; every other byte is one token. +fn tokens(sql: &str) -> Vec<&str> { + let bytes = sql.as_bytes(); + let mut tokens = Vec::new(); + let mut index = 0; + while index < bytes.len() { + let start = index; + match bytes[index] { + byte if byte.is_ascii_whitespace() => { + index += 1; + continue; + } + quote @ (b'\'' | b'"' | b'`') => { + index += 1; + while index < bytes.len() { + if bytes[index] == quote { + index += 1; + if index < bytes.len() && bytes[index] == quote { + index += 1; + continue; + } + break; + } + index += 1; + } + } + b'[' => { + index += 1; + while index < bytes.len() && bytes[index] != b']' { + index += 1; + } + index = (index + 1).min(bytes.len()); + } + byte if byte.is_ascii_alphanumeric() || byte == b'_' || byte == b'$' => { + index += 1; + while index < bytes.len() + && (bytes[index].is_ascii_alphanumeric() + || bytes[index] == b'_' + || bytes[index] == b'$') + { + index += 1; + } + } + _ => index += 1, + } + tokens.push(&sql[start..index]); + } + tokens +} + /// Derive storage-class checks from trusted declarations. SQLite affinity does /// not forbid a corrupt integer cell from holding arbitrarily large text. /// Scalar predicates reject that state before generic row conversion copies it. diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs index 345873410b..c9f6f349f8 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs @@ -45,6 +45,12 @@ pub(crate) fn history(key: &str) -> Result<&'static str, BackendError> { "branch_branch_by_id" => Ok(include_str!( "../../../../sql/sqlite/queries/history/branch_branch_by_id.sql" )), + "branch_branch_by_id_any" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/branch_branch_by_id_any.sql" + )), + "branch_retire_insert" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/branch_retire_insert.sql" + )), "branch_branch_by_name" => Ok(include_str!( "../../../../sql/sqlite/queries/history/branch_branch_by_name.sql" )), @@ -207,6 +213,69 @@ pub(crate) fn history(key: &str) -> Result<&'static str, BackendError> { "paths_advance_epoch" => Ok(include_str!( "../../../../sql/sqlite/queries/history/paths_advance_epoch.sql" )), + "gc_intent_insert" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/gc_intent_insert.sql" + )), + "gc_intent_get" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/gc_intent_get.sql" + )), + "gc_intent_release" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/gc_intent_release.sql" + )), + "gc_intent_uncertain" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/gc_intent_uncertain.sql" + )), + "gc_intent_list" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/gc_intent_list.sql" + )), + "gc_intent_ops_branch" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/gc_intent_ops_branch.sql" + )), + "gc_intent_ops_project" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/gc_intent_ops_project.sql" + )), + "gc_intent_ops_store" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/gc_intent_ops_store.sql" + )), + "gc_intent_all_branch" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/gc_intent_all_branch.sql" + )), + "gc_intent_all_project" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/gc_intent_all_project.sql" + )), + "gc_intent_all_store" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/gc_intent_all_store.sql" + )), + "gc_branch_stack_any" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/gc_branch_stack_any.sql" + )), + "gc_epoch_share" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/gc_epoch_share.sql" + )), + "gc_kept_roots" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/gc_kept_roots.sql" + )), + "gc_object_locations" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/gc_object_locations.sql" + )), + "gc_live_stacks" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/gc_live_stacks.sql" + )), + "gc_live_branches" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/gc_live_branches.sql" + )), + "gc_inode_blocks" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/gc_inode_blocks.sql" + )), + "kept_node_clear" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/kept_node_clear.sql" + )), + "kept_node_mark" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/kept_node_mark.sql" + )), + "kept_node_read" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/kept_node_read.sql" + )), _ => Err(BackendError::Integrity), } } diff --git a/engine/crates/layerfs-persistence/src/diagnostic/history.rs b/engine/crates/layerfs-persistence/src/diagnostic/history.rs index f15d464906..efa0a74541 100644 --- a/engine/crates/layerfs-persistence/src/diagnostic/history.rs +++ b/engine/crates/layerfs-persistence/src/diagnostic/history.rs @@ -32,7 +32,7 @@ fn inspect( options: Options, observer: &mut dyn FnMut(Observation), ) -> Result<(), BackendError> { - scratch.0.execute_batch("CREATE TABLE nodes(id BLOB PRIMARY KEY,kind INTEGER NOT NULL,stack BLOB NOT NULL,root BLOB NOT NULL,parent BLOB,base BLOB,second BLOB,source BLOB,source_branch BLOB,done INTEGER NOT NULL DEFAULT 0,height INTEGER,depth INTEGER,newest INTEGER,run BLOB,overwrote INTEGER NOT NULL DEFAULT 0) WITHOUT ROWID; CREATE TABLE edges(child BLOB NOT NULL,parent BLOB NOT NULL,PRIMARY KEY(child,parent)) WITHOUT ROWID; CREATE INDEX parent_edges ON edges(parent,child); CREATE TABLE stacks(id BLOB PRIMARY KEY,scope BLOB NOT NULL,profile BLOB NOT NULL,head BLOB NOT NULL) WITHOUT ROWID; CREATE TABLE branches(id BLOB PRIMARY KEY,stack BLOB NOT NULL,base BLOB NOT NULL,head BLOB,fork BLOB) WITHOUT ROWID; CREATE TABLE contexts(root BLOB,scope BLOB,profile BLOB,PRIMARY KEY(root,scope,profile)) WITHOUT ROWID;").map_err(sql)?; + scratch.0.execute_batch("CREATE TABLE nodes(id BLOB PRIMARY KEY,kind INTEGER NOT NULL,stack BLOB NOT NULL,root BLOB NOT NULL,parent BLOB,base BLOB,second BLOB,source BLOB,source_branch BLOB,done INTEGER NOT NULL DEFAULT 0,height INTEGER,depth INTEGER,newest INTEGER,run BLOB,overwrote INTEGER NOT NULL DEFAULT 0,branch BLOB) WITHOUT ROWID; CREATE TABLE edges(child BLOB NOT NULL,parent BLOB NOT NULL,PRIMARY KEY(child,parent)) WITHOUT ROWID; CREATE INDEX parent_edges ON edges(parent,child); CREATE TABLE stacks(id BLOB PRIMARY KEY,scope BLOB NOT NULL,profile BLOB NOT NULL,head BLOB NOT NULL) WITHOUT ROWID; CREATE TABLE branches(id BLOB PRIMARY KEY,stack BLOB NOT NULL,base BLOB NOT NULL,head BLOB,fork BLOB) WITHOUT ROWID; CREATE TABLE contexts(root BLOB,scope BLOB,profile BLOB,PRIMARY KEY(root,scope,profile)) WITHOUT ROWID;").map_err(sql)?; super::facts::run(view, options, observer)?; let mut failures = 0u64; inventory(view, scratch, options, observer, &mut failures)?; @@ -43,6 +43,7 @@ fn inspect( ("store.history.branches","SELECT b.id,b.base FROM branches b LEFT JOIN nodes n ON n.id=b.base LEFT JOIN stacks s ON s.id=b.stack WHERE n.id IS NULL OR n.kind!=2 OR n.stack!=b.stack OR s.id IS NULL","branch base/stack invalid"), ("store.history.branches","SELECT b.id,b.head FROM branches b LEFT JOIN nodes n ON n.id=b.head WHERE b.head IS NOT NULL AND (n.id IS NULL OR n.kind!=1 OR n.stack!=b.stack)","branch head invalid"), ("store.history.branches","SELECT b.id,b.fork FROM branches b LEFT JOIN nodes n ON n.id=b.fork WHERE b.fork IS NOT NULL AND (n.id IS NULL OR n.stack!=b.stack)","branch fork provenance invalid"), + ("store.history.branches","SELECT n.id,n.branch FROM nodes n LEFT JOIN branches b ON b.id=n.branch WHERE n.kind=1 AND n.branch IS NOT NULL AND (b.id IS NULL OR b.stack!=n.stack)","Commit names absent or cross-stack Branch"), ("store.history.provenance","SELECT n.id,n.source FROM nodes n LEFT JOIN nodes c ON c.id=n.source LEFT JOIN branches b ON b.id=n.source_branch WHERE n.kind=2 AND n.parent IS NOT NULL AND (c.id IS NULL OR c.kind!=1 OR c.stack!=n.stack OR c.root!=n.root OR b.id IS NULL OR b.stack!=n.stack)","Layer source Commit/Branch/root mismatch"), ]{ let mut stmt=scratch.0.prepare(query).map_err(sql)?; @@ -142,7 +143,7 @@ fn inventory( 0=>{let s=crate::history::diagnostic_stack(&row).map_err(|e|e.to_string())?;scratch.0.execute("INSERT INTO stacks(id,scope,profile,head) VALUES(?1,?2,?3,?4)",params![s.id.as_slice(),s.scope.as_bytes().as_slice(),s.profile.as_bytes().as_slice(),s.head_layer.as_slice()]).map_err(|_|"diagnostic scratch".to_owned())?;}, 3=>{let b=crate::history::diagnostic_branch(&row).map_err(|e|e.to_string())?;scratch.0.execute("INSERT INTO branches(id,stack,base,head,fork) VALUES(?1,?2,?3,?4,?5)",params![b.id.as_slice(),b.stack.as_slice(),b.base_layer.as_slice(),b.head_commit.map(|v|v.as_slice().to_vec()),b.forked_from.map(|v|v.as_slice().to_vec())]).map_err(|_|"diagnostic scratch".to_owned())?;}, 2=>{let l=crate::history::diagnostic_layer(&row).map_err(|e|e.to_string())?;scratch.0.execute("INSERT INTO nodes(id,kind,stack,root,parent,source,source_branch,overwrote) VALUES(?1,2,?2,?3,?4,?5,?6,?7)",params![l.id.as_slice(),l.stack.as_slice(),l.root.as_bytes().as_slice(),l.parent.map(|v|v.as_slice().to_vec()),l.source_commit.map(|v|v.as_slice().to_vec()),l.source_branch.map(|v|v.as_slice().to_vec()),i64::try_from(l.overwrote_count).map_err(|_|"Layer overwrite count".to_owned())?]).map_err(|_|"diagnostic scratch".to_owned())?;if LayerId::derive(l.stack,l.parent,l.root)!=l.id{return Err("Layer identity mismatch".to_owned());}}, - _=>{let c=crate::history::diagnostic_commit_graph(&row).map_err(|e|e.to_string())?;scratch.0.execute("INSERT INTO nodes(id,kind,stack,root,parent,base,second) VALUES(?1,1,?2,?3,?4,?5,?6)",params![c.id.as_slice(),c.stack.as_slice(),c.root.as_bytes().as_slice(),c.parent.map(|v|v.as_slice().to_vec()),c.base_layer.as_slice(),c.second_parent.map(|v|v.as_slice().to_vec())]).map_err(|_|"diagnostic scratch".to_owned())?; + _=>{let c=crate::history::diagnostic_commit_graph(&row).map_err(|e|e.to_string())?;scratch.0.execute("INSERT INTO nodes(id,kind,stack,root,parent,base,second,branch) VALUES(?1,1,?2,?3,?4,?5,?6,?7)",params![c.id.as_slice(),c.stack.as_slice(),c.root.as_bytes().as_slice(),c.parent.map(|v|v.as_slice().to_vec()),c.base_layer.as_slice(),c.second_parent.map(|v|v.as_slice().to_vec()),c.meta.branch.map(|v|v.as_slice().to_vec())]).map_err(|_|"diagnostic scratch".to_owned())?; for edge in [c.meta.merged_from,c.meta.displaced.map(layerfs_history::NodeId::Commit)].into_iter().flatten(){scratch.0.execute("INSERT OR IGNORE INTO edges VALUES(?1,?2)",params![c.id.as_slice(),edge.as_slice()]).map_err(|_|"diagnostic scratch".to_owned())?;} if CommitId::derive(c.root,c.parent,c.base_layer,c.second_parent,&c.meta)!=c.id{return Err("Commit v3 identity mismatch".to_owned());} } diff --git a/engine/crates/layerfs-persistence/src/history/admission.rs b/engine/crates/layerfs-persistence/src/history/admission.rs new file mode 100644 index 0000000000..f45a48eef0 --- /dev/null +++ b/engine/crates/layerfs-persistence/src/history/admission.rs @@ -0,0 +1,735 @@ +//! Store-side admission for the four GC calls (spec §5). +//! +//! Admission is one Store-DB table and, on SQLite, two short +//! `BEGIN IMMEDIATE` transactions: a mount/Init/merge registers a shared +//! intent, a removal scans the intents in its scope and then writes its +//! marker in the same write transaction, and a compaction registers an +//! exclusive Store intent. A live mount refuses a removal with +//! `MountsActive`; a live operation refuses a mount or another removal with +//! `Busy`; a dead or uncertain owner is `Unresolved`. +//! +//! PostgreSQL adds transaction-scoped advisory keys to a removal's marker +//! transaction and session-scoped shared keys to a mount, so a removal on one +//! machine sees a mount on another. Its `gc_intent` rows carry the same +//! ownership evidence for `MountsActive`/`Busy`/`Unresolved` reports. + +use super::rows::{cell, one, unsigned}; +use super::transaction::Transaction; +use super::{branch, layerstack, params, paths, position}; +use crate::backend::Engine; +use crate::history::provider::HistoryProvider; +use layerfs_history::error::{ + GcHolder, HistoryError, HistoryResult, Missing, MountHolders, OpConflict, +}; +#[cfg(feature = "postgresql")] +use layerfs_history::identity::CatalogId; +use layerfs_history::identity::{BranchId, LayerStackId}; +use layerfs_history::path::path_key; +use layerfs_history::records::{ + GcDeletion, GcIntent, GcIntentKind, GcScopeKind, PathRemovalKind, PathRemovalOutcome, + PathRemovalRequest, +}; +use std::time::{SystemTime, UNIX_EPOCH}; + +const INSERT: &str = "gc_intent_insert"; +#[cfg(feature = "postgresql")] +const GET: &str = "gc_intent_get"; +const RELEASE: &str = "gc_intent_release"; +const UNCERTAIN: &str = "gc_intent_uncertain"; +const LIST: &str = "gc_intent_list"; +const OPS_BRANCH: &str = "gc_intent_ops_branch"; +const OPS_PROJECT: &str = "gc_intent_ops_project"; +const OPS_STORE: &str = "gc_intent_ops_store"; +const ALL_BRANCH: &str = "gc_intent_all_branch"; +const ALL_PROJECT: &str = "gc_intent_all_project"; +const ALL_STORE: &str = "gc_intent_all_store"; +#[cfg(feature = "postgresql")] +const BRANCH_STACK: &str = "gc_branch_stack_any"; +const EPOCH_SHARE: &str = "gc_epoch_share"; + +#[cfg(feature = "postgresql")] +const LOCK_XACT_SHARED: &str = "gc_lock_xact_shared"; +#[cfg(feature = "postgresql")] +const LOCK_XACT_EXCLUSIVE: &str = "gc_lock_xact_exclusive"; +#[cfg(feature = "postgresql")] +const LOCK_SESSION_SHARED: &str = "gc_lock_session_shared"; +#[cfg(feature = "postgresql")] +const LOCK_SESSION_EXCLUSIVE: &str = "gc_lock_session_exclusive"; +#[cfg(feature = "postgresql")] +const UNLOCK_SESSION_SHARED: &str = "gc_unlock_session_shared"; +#[cfg(feature = "postgresql")] +const UNLOCK_SESSION_EXCLUSIVE: &str = "gc_unlock_session_exclusive"; + +/// True when this provider is backed by PostgreSQL. +fn postgres(provider: &HistoryProvider) -> bool { + #[cfg(feature = "postgresql")] + { + matches!(provider.engine, Engine::Postgres(_)) + } + #[cfg(not(feature = "postgresql"))] + { + let _ = provider; + false + } +} + +fn now_ms() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| { + i64::try_from(elapsed.as_millis()).unwrap_or(i64::MAX) + }) +} + +fn hex(bytes: &[u8]) -> String { + use std::fmt::Write as _; + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + let _ = write!(out, "{byte:02x}"); + } + out +} + +fn kind_of(text: &str) -> HistoryResult { + Ok(match text { + "mount" => GcIntentKind::Mount, + "init" => GcIntentKind::Init, + "merge" => GcIntentKind::Merge, + "removal" => GcIntentKind::Removal, + "compaction" => GcIntentKind::Compaction, + _ => return Err(HistoryError::Integrity("gc intent kind")), + }) +} + +fn scope_of(text: &str) -> HistoryResult { + Ok(match text { + "store" => GcScopeKind::Store, + "project" => GcScopeKind::Project, + "branch" => GcScopeKind::Branch, + _ => return Err(HistoryError::Integrity("gc intent scope")), + }) +} + +fn workspace_of(text: Option) -> HistoryResult> { + let Some(text) = text else { + return Ok(None); + }; + if text.len() != 64 { + return Err(HistoryError::Integrity("gc intent workspace")); + } + let mut body = [0u8; 32]; + for (index, pair) in text.as_bytes().chunks_exact(2).enumerate() { + let digit = |byte: u8| -> HistoryResult { + match byte { + b'0'..=b'9' => Ok(byte - b'0'), + b'a'..=b'f' => Ok(byte - b'a' + 10), + b'A'..=b'F' => Ok(byte - b'A' + 10), + _ => Err(HistoryError::Integrity("gc intent workspace")), + } + }; + body[index] = (digit(pair[0])? << 4) | digit(pair[1])?; + } + Ok(Some(body)) +} + +/// Decodes the fourteen columns every intent query returns, in order: +/// op_id, kind, scope_kind, scope_id, workspace, sandbox, machine, project, +/// branch, owner_runtime, owner_boot, state, uncertain, phase. +fn intent_row(row: &crate::backend::records::Record) -> HistoryResult { + let op_id: Vec = cell(row, 0)?; + let op_id: [u8; 16] = op_id + .as_slice() + .try_into() + .map_err(|_| HistoryError::Integrity("gc intent op_id"))?; + let state: String = cell(row, 11)?; + let uncertain_flag: i64 = cell(row, 12)?; + Ok(GcIntent { + op_id, + kind: kind_of(&cell::(row, 1)?)?, + scope_kind: scope_of(&cell::(row, 2)?)?, + scope_id: cell(row, 3)?, + workspace: workspace_of(cell::>(row, 4)?)?, + sandbox: cell(row, 5)?, + machine: cell(row, 6)?, + project: cell(row, 7)?, + branch: cell(row, 8)?, + owner_runtime: cell(row, 9)?, + owner_boot: cell(row, 10)?, + project_id: None, + phase: cell(row, 13)?, + uncertain: state == "uncertain" || uncertain_flag != 0, + }) +} + +fn entity(intent: &GcIntent) -> String { + match (&intent.project, &intent.branch, intent.scope_kind) { + (_, Some(branch), _) => branch.clone(), + (Some(project), None, _) => project.clone(), + (None, None, GcScopeKind::Store) => "store".to_owned(), + (None, None, _) => hex(&intent.scope_id), + } +} + +fn holder(intent: &GcIntent) -> GcHolder { + GcHolder { + kind: intent.kind, + entity: entity(intent), + workspace: intent.workspace.as_ref().map(|bytes| hex(bytes)), + sandbox: intent.sandbox.clone(), + machine: intent.machine.clone(), + op: hex(&intent.op_id), + } +} + +fn conflict(intent: &GcIntent) -> OpConflict { + OpConflict::new( + intent.kind, + entity(intent), + intent.phase.clone(), + hex(&intent.op_id), + intent.uncertain, + ) +} + +/// The refusal a non-empty conflict set reports: an uncertain owner is +/// `Unresolved`, a live mount is `MountsActive`, any other live operation is +/// `Busy`. +fn classify(found: &[GcIntent]) -> HistoryError { + if let Some(dead) = found.iter().find(|intent| intent.uncertain) { + return HistoryError::Unresolved(Box::new(conflict(dead))); + } + let holders: Vec = found + .iter() + .filter(|intent| intent.kind.shared()) + .map(holder) + .collect(); + if !holders.is_empty() { + return HistoryError::MountsActive(Box::new(MountHolders::new(holders))); + } + match found.first() { + Some(operation) => HistoryError::BusyOp(Box::new(conflict(operation))), + None => HistoryError::Busy, + } +} + +fn query( + tx: &Transaction<'_>, + key: &str, + params: Vec, +) -> HistoryResult> { + tx.query(key, params)?.iter().map(intent_row).collect() +} + +fn conflicts_for(tx: &Transaction<'_>, intent: &GcIntent) -> HistoryResult> { + let shared = intent.kind.shared(); + match intent.scope_kind { + GcScopeKind::Store => query( + tx, + if shared { OPS_STORE } else { ALL_STORE }, + params![intent.op_id.to_vec()], + ), + GcScopeKind::Project => query( + tx, + if shared { OPS_PROJECT } else { ALL_PROJECT }, + params![ + intent.scope_id.clone(), + intent.project.clone(), + intent.op_id.to_vec() + ], + ), + GcScopeKind::Branch => query( + tx, + if shared { OPS_BRANCH } else { ALL_BRANCH }, + params![ + intent.project_id.clone(), + intent.scope_id.clone(), + intent.op_id.to_vec() + ], + ), + } +} + +fn validate_mount(tx: &Transaction<'_>, intent: &GcIntent) -> HistoryResult<()> { + if intent.kind != GcIntentKind::Mount { + return Ok(()); + } + match intent.scope_kind { + GcScopeKind::Store => Ok(()), + GcScopeKind::Project => { + let stack = LayerStackId::from_slice(&intent.scope_id) + .map_err(|_| HistoryError::InvalidInput("GC project scope"))?; + if layerstack::layer_stack(tx, stack)?.is_none() { + return Err(HistoryError::Missing(Missing::LayerStack)); + } + Ok(()) + } + GcScopeKind::Branch => { + let branch = BranchId::from_slice(&intent.scope_id) + .map_err(|_| HistoryError::InvalidInput("GC Branch scope"))?; + if branch::branch(tx, branch)?.is_none() { + return Err(HistoryError::Missing(Missing::Branch)); + } + Ok(()) + } + } +} + +/// One registration in its own write transaction. +fn register_in(tx: &Transaction<'_>, intent: &GcIntent, _postgres: bool) -> HistoryResult<()> { + validate_mount(tx, intent)?; + let found = conflicts_for(tx, intent)?; + if !found.is_empty() { + return Err(classify(&found)); + } + let state = if intent.uncertain { + "uncertain" + } else { + "live" + }; + let uncertain = i64::from(intent.uncertain); + tx.execute( + INSERT, + params![ + intent.op_id.to_vec(), + intent.kind.as_str(), + intent.scope_kind.as_str(), + intent.scope_id.clone(), + intent.workspace.map(|bytes| hex(&bytes)), + intent.sandbox.clone(), + intent.machine.clone(), + intent.project.clone(), + intent.branch.clone(), + intent.owner_runtime.clone(), + intent.owner_boot.clone(), + state, + uncertain, + intent.phase.clone(), + now_ms(), + ], + )?; + Ok(()) +} + +/// The advisory keys a PostgreSQL intent names, Store key first. +#[cfg(feature = "postgresql")] +fn advisory_keys(catalog: CatalogId, intent: &GcIntent) -> Vec<(u8, Vec)> { + let mut keys = vec![(1u8, catalog.as_slice().to_vec())]; + match intent.scope_kind { + GcScopeKind::Store => {} + GcScopeKind::Project => keys.push((2, intent.scope_id.clone())), + GcScopeKind::Branch => { + if let Some(project) = &intent.project_id { + keys.push((2, project.clone())); + } + keys.push((3, intent.scope_id.clone())); + } + } + keys +} + +#[cfg(feature = "postgresql")] +fn advisory_key(catalog: CatalogId, kind: u8, id: &[u8]) -> i64 { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"layerfs/advisory/v1\0"); + hasher.update(catalog.as_slice()); + hasher.update(&[kind]); + hasher.update(id); + let digest = hasher.finalize(); + let mut head = [0u8; 8]; + head.copy_from_slice(&digest.as_bytes()[..8]); + i64::from_be_bytes(head) +} + +#[cfg(feature = "postgresql")] +fn xact_lock(tx: &Transaction<'_>, key: i64, shared: bool) -> HistoryResult<()> { + let found = tx.query( + if shared { + LOCK_XACT_SHARED + } else { + LOCK_XACT_EXCLUSIVE + }, + params![key], + )?; + if found.is_empty() { + return Err(if shared { + HistoryError::MountsActive(Box::new(MountHolders::new(Vec::new()))) + } else { + HistoryError::Busy + }); + } + Ok(()) +} + +/// PostgreSQL: the Store key a removal shares with every mount, taken first. +#[cfg(feature = "postgresql")] +fn lock_store_shared(tx: &Transaction<'_>, catalog: CatalogId) -> HistoryResult<()> { + xact_lock(tx, advisory_key(catalog, 1, catalog.as_slice()), true) +} + +/// PostgreSQL: one project or Branch key. +#[cfg(feature = "postgresql")] +fn lock_scope( + tx: &Transaction<'_>, + catalog: CatalogId, + kind: u8, + id: &[u8], + shared: bool, +) -> HistoryResult<()> { + xact_lock(tx, advisory_key(catalog, kind, id), shared) +} + +fn epoch_share(tx: &Transaction<'_>, postgres: bool, stack: LayerStackId) -> HistoryResult<()> { + if !postgres { + return Ok(()); + } + let found = tx.query(EPOCH_SHARE, [stack.as_slice()])?; + if found.len() != 1 { + return Err(HistoryError::Conflict("project epoch")); + } + Ok(()) +} + +fn release_in(tx: &Transaction<'_>, op_id: &[u8; 16]) -> HistoryResult<()> { + tx.execute(RELEASE, [op_id.as_slice()])?; + Ok(()) +} + +fn uncertain_in(tx: &Transaction<'_>, op_id: &[u8; 16]) -> HistoryResult<()> { + tx.execute(UNCERTAIN, params![now_ms(), op_id.as_slice()])?; + Ok(()) +} + +#[cfg(feature = "postgresql")] +fn get_in(tx: &Transaction<'_>, op_id: &[u8; 16]) -> HistoryResult> { + one(tx, GET, [op_id.as_slice()], intent_row) +} + +#[cfg(feature = "postgresql")] +fn keys_for_release( + tx: &Transaction<'_>, + catalog: CatalogId, + intent: &GcIntent, +) -> HistoryResult<(Vec<(u8, Vec)>, bool)> { + let mut intent = intent.clone(); + if intent.scope_kind == GcScopeKind::Branch && intent.project_id.is_none() { + intent.project_id = one(tx, BRANCH_STACK, [intent.scope_id.as_slice()], |row| { + cell::>(row, 0) + })?; + } + Ok((advisory_keys(catalog, &intent), intent.kind.shared())) +} + +/// Registers one shared (`mount`, `init`, `merge`) or exclusive +/// (`compaction`) intent. +pub(crate) fn register(provider: &HistoryProvider, intent: &GcIntent) -> HistoryResult<()> { + match &provider.engine { + Engine::Sqlite(_) => provider.write(|tx| register_in(tx, intent, false)), + #[cfg(feature = "postgresql")] + Engine::Postgres(session) => { + let keys = advisory_keys(provider.catalog_id, intent); + let shared = intent.kind.shared(); + session_lock(session, provider.catalog_id, &keys, shared)?; + let result = session.run(true, |statements| { + register_in(&Transaction::new(statements), intent, true) + }); + if result.is_err() { + let _ = session_unlock(session, provider.catalog_id, &keys, shared); + } + result + } + } +} + +/// Deletes one intent, and on PostgreSQL releases the session keys it held. +pub(crate) fn release(provider: &HistoryProvider, op_id: &[u8; 16]) -> HistoryResult<()> { + match &provider.engine { + Engine::Sqlite(_) => provider.write(|tx| release_in(tx, op_id)), + #[cfg(feature = "postgresql")] + Engine::Postgres(session) => { + let found = provider.write(|tx| { + let Some(intent) = get_in(tx, op_id)? else { + return Ok(None); + }; + release_in(tx, op_id)?; + Ok(Some(keys_for_release(tx, provider.catalog_id, &intent)?)) + })?; + if let Some((keys, shared)) = found { + let _ = session_unlock(session, provider.catalog_id, &keys, shared); + } + Ok(()) + } + } +} + +/// Marks one intent uncertain; its owner should settle it later. +pub(crate) fn mark_uncertain(provider: &HistoryProvider, op_id: &[u8; 16]) -> HistoryResult<()> { + provider.write(|tx| uncertain_in(tx, op_id)) +} + +/// `branch.remove`: scan the Branch scope and write the marker in one write +/// transaction. +pub(crate) fn remove_branch( + provider: &HistoryProvider, + branch: BranchId, + removed_ms: i64, + intent: &GcIntent, +) -> HistoryResult<()> { + let postgres = postgres(provider); + let catalog = provider.catalog_id; + let _ = catalog; + provider.write(|tx| { + let record = branch::branch(tx, branch)?.ok_or(HistoryError::Missing(Missing::Branch))?; + #[cfg(feature = "postgresql")] + if postgres { + lock_store_shared(tx, catalog)?; + lock_scope(tx, catalog, 2, record.stack.as_slice(), true)?; + let found = query( + tx, + ALL_BRANCH, + params![ + Some(record.stack.as_slice().to_vec()), + branch.as_slice().to_vec(), + intent.op_id.to_vec() + ], + )?; + if !found.is_empty() { + return Err(classify(&found)); + } + lock_scope(tx, catalog, 3, branch.as_slice(), false)?; + } + epoch_share(tx, postgres, record.stack)?; + let found = query( + tx, + ALL_BRANCH, + params![ + Some(record.stack.as_slice().to_vec()), + branch.as_slice().to_vec(), + intent.op_id.to_vec() + ], + )?; + if !found.is_empty() { + return Err(classify(&found)); + } + super::removal::mark_branch_removed(tx, branch, removed_ms) + }) +} + +/// `project.delete`: the same-home retry classification, then admission and +/// the marker in one write transaction. +pub(crate) fn delete_stack( + provider: &HistoryProvider, + stack: LayerStackId, + deleted_ms: i64, + intent: &GcIntent, +) -> HistoryResult { + let postgres = postgres(provider); + let catalog = provider.catalog_id; + let _ = catalog; + provider.write(|tx| { + if let Some(recorded) = layerstack::stack_deleted_ms(tx, stack)? { + return Ok(GcDeletion::Retired { + deleted_ms: recorded, + }); + } + let live = layerstack::layer_stack(tx, stack)?; + let Some(record) = live else { + return Ok(match layerstack::stack_deleted_ms(tx, stack)? { + Some(recorded) => GcDeletion::Retired { + deleted_ms: recorded, + }, + None => GcDeletion::Absent, + }); + }; + #[cfg(feature = "postgresql")] + if postgres { + lock_store_shared(tx, catalog)?; + let found = query( + tx, + ALL_PROJECT, + params![ + stack.as_slice().to_vec(), + Some(record.name.as_str().to_owned()), + intent.op_id.to_vec() + ], + )?; + if !found.is_empty() { + return Err(classify(&found)); + } + lock_scope(tx, catalog, 2, stack.as_slice(), false)?; + } + epoch_share(tx, postgres, stack)?; + let found = query( + tx, + ALL_PROJECT, + params![ + stack.as_slice().to_vec(), + Some(record.name.as_str().to_owned()), + intent.op_id.to_vec() + ], + )?; + if !found.is_empty() { + return Err(classify(&found)); + } + layerstack::mark_stack_deleted(tx, stack, deleted_ms)?; + Ok(GcDeletion::Marked { deleted_ms }) + }) +} + +/// `project.remove_file`: admission first, then the marker and epoch bump in +/// the same write transaction. +pub(crate) fn remove_path( + provider: &HistoryProvider, + request: &PathRemovalRequest, + intent: &GcIntent, +) -> HistoryResult { + let postgres = postgres(provider); + let catalog = provider.catalog_id; + let _ = catalog; + provider.write(|tx| { + if layerstack::layer_stack(tx, request.stack)?.is_none() { + return Err(HistoryError::Missing(Missing::LayerStack)); + } + #[cfg(feature = "postgresql")] + if postgres { + lock_store_shared(tx, catalog)?; + let found = query( + tx, + ALL_PROJECT, + params![ + request.stack.as_slice().to_vec(), + intent.project.clone(), + intent.op_id.to_vec() + ], + )?; + if !found.is_empty() { + return Err(classify(&found)); + } + lock_scope(tx, catalog, 2, request.stack.as_slice(), false)?; + } + epoch_share(tx, postgres, request.stack)?; + let found = query( + tx, + ALL_PROJECT, + params![ + request.stack.as_slice().to_vec(), + intent.project.clone(), + intent.op_id.to_vec() + ], + )?; + if !found.is_empty() { + return Err(classify(&found)); + } + let key = path_key(&request.path); + let stored = one( + tx, + "paths_marker", + [request.stack.as_slice(), key.as_slice()], + |row| { + Ok(( + cell::>(row, 0)?, + unsigned(row, 1)?, + cell::(row, 2)?, + )) + }, + )?; + let stack_high = position::stack_high_water(tx, request.stack)?; + if let Some((stored_path, frontier, removed)) = &stored { + if stored_path != &request.path { + return Err(HistoryError::Integrity("path marker key")); + } + if stack_high <= *frontier { + return Ok(PathRemovalOutcome { + kind: PathRemovalKind::UpToDate, + stack: request.stack, + path: stored_path.clone(), + frontier_serial: *frontier, + removed_ms: *removed, + }); + } + } + let frontier = position::high_water(tx)?; + paths::mark( + tx, + request.stack, + &request.path, + frontier, + request.removed_ms, + )?; + let advanced = tx.execute("paths_advance_epoch", [request.stack.as_slice()])?; + if advanced != 1 { + return Err(HistoryError::Conflict("project epoch")); + } + Ok(PathRemovalOutcome { + kind: PathRemovalKind::Removed, + stack: request.stack, + path: request.path.clone(), + frontier_serial: frontier, + removed_ms: request.removed_ms, + }) + }) +} + +/// Every live intent recorded at one scope, for `store.status` and tests. +pub(crate) fn intents( + provider: &HistoryProvider, + scope_kind: GcScopeKind, + scope_id: &[u8], +) -> HistoryResult> { + provider.read(|tx| query(tx, LIST, params![scope_kind.as_str(), scope_id.to_vec()])) +} + +#[cfg(feature = "postgresql")] +fn session_lock( + session: &std::sync::Arc, + catalog: CatalogId, + keys: &[(u8, Vec)], + shared: bool, +) -> HistoryResult<()> { + let mut taken: Vec<(u8, Vec)> = Vec::new(); + for (kind, id) in keys { + let key = advisory_key(catalog, *kind, id); + let found = session.read_statement(|statements| { + Transaction::new(statements).query( + if shared { + LOCK_SESSION_SHARED + } else { + LOCK_SESSION_EXCLUSIVE + }, + params![key], + ) + })?; + if found.is_empty() { + let _ = session_unlock(session, catalog, &taken, shared); + return Err(if shared { + HistoryError::MountsActive(Box::new(MountHolders::new(Vec::new()))) + } else { + HistoryError::Busy + }); + } + taken.push((*kind, id.clone())); + } + Ok(()) +} + +#[cfg(feature = "postgresql")] +fn session_unlock( + session: &std::sync::Arc, + catalog: CatalogId, + keys: &[(u8, Vec)], + shared: bool, +) -> HistoryResult<()> { + for (kind, id) in keys { + let key = advisory_key(catalog, *kind, id); + let _ = session.read_statement(|statements| { + Transaction::new(statements).query( + if shared { + UNLOCK_SESSION_SHARED + } else { + UNLOCK_SESSION_EXCLUSIVE + }, + params![key], + ) + })?; + } + Ok(()) +} diff --git a/engine/crates/layerfs-persistence/src/history/branch.rs b/engine/crates/layerfs-persistence/src/history/branch.rs index e174b8cdf2..2307c13661 100644 --- a/engine/crates/layerfs-persistence/src/history/branch.rs +++ b/engine/crates/layerfs-persistence/src/history/branch.rs @@ -7,7 +7,10 @@ use super::params; use super::transaction::Transaction; -use layerfs_history::error::{HistoryError, HistoryResult, Missing}; +use layerfs_history::error::{ + HistoryError, HistoryResult, HolderKind, HolderState, Missing, NameHolder, NameSource, + NameTakenState, +}; use layerfs_history::identity::{BranchId, CatalogId, HistoryName, LayerStackId}; use layerfs_history::records::{ BranchRecord, BranchSnapshot, ForkRequest, ForkSource, Page, PageResult, @@ -15,13 +18,17 @@ use layerfs_history::records::{ use layerfs_history::NamedSnapshot; use super::query::{self, Context, Cursor, Range}; -use super::rows::{branch_row, commit_row, many, one, sql, stack_row}; +use super::rows::{branch_row, cell, commit_row, many, one, sql, stack_row}; use crate::backend::records::{Record as Row, Value}; use layerfs_content::ObjectId; use layerfs_history::records::{CommitRecord, LayerStackRecord}; const BRANCH_BY_ID: &str = "branch_branch_by_id"; +/// The same row without the visibility filter: fork provenance and rendering +/// keep a retired Branch's name and stack without resolving it for a caller. +const BRANCH_BY_ID_ANY: &str = "branch_branch_by_id_any"; + const BRANCH_BY_NAME: &str = "branch_branch_by_name"; const BRANCH_FIRST_PAGE: &str = "branch_branch_first_page"; @@ -37,6 +44,55 @@ pub(crate) fn branch(tx: &Transaction<'_>, id: BranchId) -> HistoryResult