From 0574b29a5d61bbf6d13bbe0ad9c57f6cd18a1315 Mon Sep 17 00:00:00 2001 From: rain Date: Mon, 16 Nov 2026 00:37:54 +0000 Subject: [PATCH] feat(gc): mark, closure and kept objects (#47) --- .../layerfs-api/tests/project_export.rs | 70 ++ .../layerfs-api/tests/project_list_worker.rs | 296 +++++++++ .../layerfs-control/src/worker_server.rs | 13 +- .../layerfs-store/src/project/export/walk.rs | 18 + .../layerfs-store/src/project/family.rs | 90 ++- .../crates/layerfs-store/src/project/frame.rs | 127 +++- .../layerfs-store/src/project/listing.rs | 45 ++ .../crates/layerfs-store/src/project/mod.rs | 1 + .../tests/project_list_frames.rs | 122 ++++ .../crates/layerfs-daemon/src/operator/mod.rs | 6 + .../layerfs-daemon/src/operator/workspace.rs | 192 +++++- .../layerfs-persistence/sql/postgresql/gc.sql | 11 + .../layerfs-persistence/sql/sqlite/gc.sql | 14 + .../queries/history/gc_inode_blocks.sql | 1 + .../sqlite/queries/history/gc_kept_roots.sql | 7 + .../queries/history/gc_live_branches.sql | 4 + .../sqlite/queries/history/gc_live_stacks.sql | 3 + .../queries/history/gc_object_locations.sql | 2 + .../queries/history/kept_node_clear.sql | 1 + .../sqlite/queries/history/kept_node_mark.sql | 81 +++ .../sqlite/queries/history/kept_node_read.sql | 1 + .../src/backend/postgresql/mechanics.rs | 3 + .../src/backend/postgresql/schema.rs | 76 ++- .../backend/postgresql/schema_diagnostic.rs | 4 + .../layerfs-persistence/src/backend/seam.rs | 2 + .../src/backend/sqlite/mechanics.rs | 3 + .../src/backend/sqlite/schema.rs | 50 ++ .../src/backend/sqlite/session/query.rs | 24 + .../layerfs-persistence/src/history/kept.rs | 196 ++++++ .../layerfs-persistence/src/history/mod.rs | 4 + .../src/history/objects.rs | 502 ++++++++++++++ .../src/history/transaction.rs | 6 +- engine/crates/layerfs-persistence/src/lib.rs | 2 +- .../layerfs-persistence/src/store/open.rs | 1 + .../src/store/postgresql.rs | 1 + .../layerfs-persistence/tests/kept_node.rs | 390 +++++++++++ .../layerfs-persistence/tests/kept_objects.rs | 466 +++++++++++++ .../tests/lineage_backfill.rs | 4 +- .../tests/old_format_columns.rs | 2 +- .../tests/old_format_open.rs | 2 +- .../tests/sqlite_publication.rs | 5 +- .../layerfs-store-service/src/methods/args.rs | 9 - .../src/methods/file/blame.rs | 11 +- .../src/methods/file/glob.rs | 8 +- .../src/methods/file/grep.rs | 15 +- .../src/methods/file/list.rs | 6 +- .../src/methods/file/revision.rs | 1 + .../src/read/attribution.rs | 19 +- .../layerfs-store-service/src/read/blame.rs | 31 +- .../src/read/branch_compare.rs | 42 +- .../layerfs-store-service/src/read/compare.rs | 24 +- .../src/read/file_compare.rs | 68 +- .../src/read/paged_reads.rs | 16 + .../layerfs-store-service/src/read/reads.rs | 22 +- .../src/read/revision.rs | 41 +- .../src/read/visibility.rs | 222 ++++++- .../layerfs-store-service/src/write/files.rs | 43 +- .../tests/sdk_path_reads.rs | 620 ++++++++++++++++++ .../tests/sdk_path_whiteouts.rs | 7 +- 59 files changed, 3880 insertions(+), 173 deletions(-) create mode 100644 api/crates/layerfs-api/tests/project_list_worker.rs create mode 100644 control/crates/layerfs-store/src/project/listing.rs create mode 100644 control/crates/layerfs-store/tests/project_list_frames.rs create mode 100644 engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_inode_blocks.sql create mode 100644 engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_kept_roots.sql create mode 100644 engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_live_branches.sql create mode 100644 engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_live_stacks.sql create mode 100644 engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_object_locations.sql create mode 100644 engine/crates/layerfs-persistence/sql/sqlite/queries/history/kept_node_clear.sql create mode 100644 engine/crates/layerfs-persistence/sql/sqlite/queries/history/kept_node_mark.sql create mode 100644 engine/crates/layerfs-persistence/sql/sqlite/queries/history/kept_node_read.sql create mode 100644 engine/crates/layerfs-persistence/src/history/kept.rs create mode 100644 engine/crates/layerfs-persistence/src/history/objects.rs create mode 100644 engine/crates/layerfs-persistence/tests/kept_node.rs create mode 100644 engine/crates/layerfs-persistence/tests/kept_objects.rs create mode 100644 engine/crates/layerfs-store-service/tests/sdk_path_reads.rs diff --git a/api/crates/layerfs-api/tests/project_export.rs b/api/crates/layerfs-api/tests/project_export.rs index 85ef103334..0db2299303 100644 --- a/api/crates/layerfs-api/tests/project_export.rs +++ b/api/crates/layerfs-api/tests/project_export.rs @@ -1600,3 +1600,73 @@ fn export_progress_events_follow_time_not_entry_count() { let _ = std::fs::remove_dir_all(d); } } + +#[test] +fn export_omits_a_path_hidden_by_remove_file() { + let home = private_home("removed-home"); + ok(&home, "store.create", &[("name", t("shop"))]); + let source = tempdir("removed-src"); + std::fs::write(source.join("a.txt"), "a\n").unwrap(); + std::fs::write(source.join("b.txt"), "b\n").unwrap(); + std::fs::create_dir(source.join("dir")).unwrap(); + std::fs::write(source.join("dir/c.txt"), "c\n").unwrap(); + ok( + &home, + "project.init", + &[ + ("project", t("shop/app")), + ("folder", t(&source.display().to_string())), + ], + ); + + // The marker is written through the Store engine, the route the fs calls + // use; `project.remove_file` is not on this crate's call table yet. + layerfs_store::service::with_service( + &layerfs_store::open_args::OpenArgs { + home: Some(home.clone()), + store: Some("shop".to_owned()), + store_path: Some(paths::store_file(&home, "shop").display().to_string()), + ..Default::default() + }, + |service| { + let call = layerfs_bridge::control::SdkCall { + method: "project.remove_file".to_owned(), + args: SdkValue::map([ + ("project", t("app")), + ("path", t("/a.txt")), + ]), + }; + layerfs_store_service::methods::call(service, &call) + }, + ) + .unwrap(); + + let out_parent = tempdir("removed-out"); + let dest = out_parent.join("exported"); + let res = ok( + &home, + "project.export", + &[ + ("project", t("shop/app")), + ("destination", t(&dest.display().to_string())), + ], + ); + + // §4.3: a hidden entry is omitted from the exported folder and counts. + assert!( + !dest.join("a.txt").exists(), + "a hidden path must not be exported" + ); + assert_eq!(std::fs::read_to_string(dest.join("b.txt")).unwrap(), "b\n"); + assert_eq!( + std::fs::read_to_string(dest.join("dir/c.txt")).unwrap(), + "c\n" + ); + assert_eq!(res.get_int("files"), Some(2)); + assert_eq!(res.get_int("directories"), Some(1)); + assert_eq!(res.get_int("symlinks"), Some(0)); + + for d in [home, source, out_parent] { + let _ = std::fs::remove_dir_all(d); + } +} diff --git a/api/crates/layerfs-api/tests/project_list_worker.rs b/api/crates/layerfs-api/tests/project_list_worker.rs new file mode 100644 index 0000000000..484c17b261 --- /dev/null +++ b/api/crates/layerfs-api/tests/project_list_worker.rs @@ -0,0 +1,296 @@ +//! project.list and project.get on a worker-backed Store (T4.5c): the +//! listing is the Store's own filtered one, read in the machine's worker +//! (the same `layerfs_project::store::projects` a local Store uses), so a +//! project retired there is not listed and is `NoSuchProject` from +//! project.get; a live project still lists and gets, and a worker's refusal +//! is the call's. +use layerfs_api::home::{self, paths, write_record}; +use layerfs_bridge::control::SdkValue; +use layerfs_bridge::sdk_values::hex; +use layerfs_history::LayerStackId; +use layerfs_machine::worker::{self, read_frame, read_frame_within, write_frame, Hello, MAX_HELLO}; +use layerfs_store::open_args::OpenArgs; +use layerfs_store::project::record::ProjectRecord; +use layerfs_store::record::StoreRecord; +use layerfs_store::service::with_service; +use layerfs_store_service::history::deletion::delete_stack; +use std::net::TcpListener; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; + +static NEXT: AtomicU64 = AtomicU64::new(0); + +fn tempdir(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "lfs-list-worker-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +fn t(s: &str) -> SdkValue { + SdkValue::text(s) +} + +fn call(home: &Path, name: &str, args: &[(&str, SdkValue)]) -> Result { + let args = SdkValue::Map( + args.iter() + .map(|(n, v)| ((*n).to_owned(), v.clone())) + .collect(), + ); + layerfs_api::call(Some(home), name, args, &mut |_| {}).map_err(|r| (r.code, r.message)) +} + +fn ok(home: &Path, name: &str, args: &[(&str, SdkValue)]) -> SdkValue { + call(home, name, args).unwrap_or_else(|e| panic!("{name}: {e:?}")) +} + +fn names(list: &SdkValue) -> Vec { + let SdkValue::List(items) = list.get("items").unwrap() else { + panic!("{list:?}") + }; + items + .iter() + .filter_map(|item| item.get_text("name")) + .map(str::to_owned) + .collect() +} + +/// Marks Store project `stack` retired (the `project.delete` write path, +/// T4.5b): the Store hides it, its home entry stays. +fn retire(home: &Path, store: &str, stack: &[u8]) { + let record = StoreRecord::read(home, store).unwrap().unwrap(); + let open = OpenArgs { + home: Some(home.to_path_buf()), + store: Some(store.to_owned()), + project: None, + store_path: Some(paths::store_file(home, store).display().to_string()), + store_record: Some(record.to_record().clone()), + postgresql: None, + }; + let stack = LayerStackId::from_slice(stack).unwrap(); + with_service(&open, |service| { + delete_stack(&service.store, stack, 1234).unwrap(); + Ok(SdkValue::Null) + }) + .unwrap(); +} + +/// Moves Store `store`'s file from `from`'s Store folder to `to`'s. +fn move_store_file(from: &Path, to: &Path, store: &str) { + let source = paths::store_dir(from, store); + let target = paths::store_dir(to, store); + home::private_dir(&target).unwrap(); + for entry in std::fs::read_dir(&source).unwrap() { + let entry = entry.unwrap(); + let name = entry.file_name().to_string_lossy().into_owned(); + if name.starts_with("store.sqlite") { + std::fs::rename(entry.path(), target.join(&name)).unwrap(); + } + } +} + +/// The Store record of `store` places it on machine `machine`. +fn place_store(home: &Path, store: &str, machine: &str) { + let record_path = paths::store_record(home, store); + let record = home::read_record(&record_path).unwrap().unwrap(); + let record = home::with_field(record, "machine", t(machine)); + write_record(&record_path, &record).unwrap(); +} + +/// A home whose Store `shop` holds `live` and `retired`, the latter retired +/// in the Store, and whose machine `box` record names `listener` as its +/// worker; `shop`'s calls run on `box`. With `root`, the Store's file is +/// moved under the worker's root. +fn shop_on_box(tag: &str, listener: &TcpListener, root: Option<&Path>) -> PathBuf { + let home = tempdir(tag); + std::fs::set_permissions(&home, std::fs::Permissions::from_mode(0o700)).unwrap(); + home::open(&home).unwrap(); + ok(&home, "store.create", &[("name", t("shop"))]); + let source = tempdir(&format!("{tag}-folder")); + std::fs::write(source.join("README.md"), "live\n").unwrap(); + for name in ["live", "retired"] { + ok( + &home, + "project.init", + &[ + ("project", t(&format!("shop/{name}"))), + ("folder", t(&source.display().to_string())), + ], + ); + } + let retired = ProjectRecord::read(&home, "shop", "retired") + .unwrap() + .expect("the retired entry"); + retire( + &home, + "shop", + &retired.stack().expect("the entry's manifest stack"), + ); + write_record( + &paths::machine_record(&home, "box"), + &SdkValue::map([("name", t("box")), ("platform", t("lima"))]), + ) + .unwrap(); + let address = listener.local_addr().unwrap().to_string(); + write_record( + &paths::worker_record(&home, "box"), + &SdkValue::map([("address", t(&address)), ("secret", t("k"))]), + ) + .unwrap(); + if let Some(root) = root { + move_store_file(&home, root, "shop"); + } + place_store(&home, "shop", "box"); + home +} + +type Seen = Arc>>; + +/// A fake worker: answers its secret, reads each request and answers +/// `reply`'s frame; the requests it read are shared back. +fn fake_worker( + listener: TcpListener, + reply: impl Fn(&SdkValue) -> SdkValue + Send + 'static, +) -> Seen { + let seen: Seen = Arc::default(); + let requests = seen.clone(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { + return; + }; + let Some(_hello) = read_frame_within(&mut stream, MAX_HELLO).ok().flatten() else { + continue; + }; + let ready = Hello { + protocol: Some(worker::PROTOCOL), + bundle: None, + }; + if write_frame(&mut stream, &ready.to_frame()).is_err() { + continue; + } + let Some(request) = read_frame(&mut stream).ok().flatten() else { + continue; + }; + let answer = reply(&request); + requests.lock().unwrap().push(request); + let _ = write_frame(&mut stream, &answer); + } + }); + seen +} + +#[test] +fn a_worker_s_filtered_listing_hides_a_retired_project() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let home = shop_on_box("fake-list", &listener, None); + let live = ProjectRecord::read(&home, "shop", "live") + .unwrap() + .expect("the live entry"); + let live_stack = hex(&live.stack().expect("the live entry's stack")); + let seen = fake_worker(listener, move |_| { + // The worker's answer envelope is `{ value }`; its value is the + // `project.list` reply. + SdkValue::map([( + "value", + SdkValue::map([ + ("kind", t("Listed")), + ( + "projects", + SdkValue::List(vec![SdkValue::map([ + ("name", t("live")), + ("stack_id", t(&live_stack)), + ])]), + ), + ]), + )]) + }); + + // The worker's listing names only `live`: the retired entry is hidden. + assert_eq!(names(&ok(&home, "project.list", &[])), vec!["live"]); + assert_eq!( + names(&ok(&home, "project.list", &[("store", t("shop"))])), + vec!["live"] + ); + // project.get refuses the retired entry and gets the live one. + let (code, message) = + call(&home, "project.get", &[("project", t("shop/retired"))]).unwrap_err(); + assert_eq!(code, "NoSuchProject", "{message}"); + let live = ok(&home, "project.get", &[("project", t("shop/live"))]); + assert_eq!(live.get_text("project"), Some("shop/live")); + // Both home entries are still there; the Store, not the home, hides one. + for name in ["live", "retired"] { + assert!( + paths::project_record(&home, "shop", name).exists(), + "{name}" + ); + } + + // Every read went to the worker, as the Store listing's own frame. + let requests = seen.lock().unwrap(); + assert_eq!(requests.len(), 4, "{requests:?}"); + for request in requests.iter() { + assert_eq!(request.get_text("op"), Some("project.list")); + let args = request.get("args").expect("args"); + assert_eq!(args.get_text("store"), Some("shop")); + assert!(args.get("store_record").is_some()); + } + drop(requests); + let _ = std::fs::remove_dir_all(home); +} + +#[test] +fn a_worker_refusal_is_the_calls_refusal() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let home = shop_on_box("fake-refusal", &listener, None); + let _seen = fake_worker(listener, |_| { + SdkValue::map([( + "refused", + SdkValue::map([ + ("code", t("Busy")), + ("message", t("the Store is held")), + ("detail", SdkValue::Null), + ]), + )]) + }); + + // project.get propagates the refusal, never the unfiltered record. + let (code, message) = call(&home, "project.get", &[("project", t("shop/live"))]).unwrap_err(); + assert_eq!(code, "Busy"); + assert_eq!(message, "the Store is held"); + // The named listing reports it; the all-Stores one omits the Store. + let (code, _) = call(&home, "project.list", &[("store", t("shop"))]).unwrap_err(); + assert_eq!(code, "Busy"); + let list = ok(&home, "project.list", &[]); + assert!(names(&list).is_empty(), "{list:?}"); + let _ = std::fs::remove_dir_all(home); +} + +#[test] +fn a_worker_store_lists_only_its_live_projects() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let root = tempdir("worker-root"); + let home = shop_on_box("real-list", &listener, Some(&root)); + let guest = root.clone(); + std::thread::spawn(move || { + layerfs_control::worker_server::serve(listener, "k".to_owned(), guest) + }); + + // The Store under the worker's root hides its retired project; the + // caller's home entry for it stays but is not listed. + assert_eq!(names(&ok(&home, "project.list", &[])), vec!["live"]); + let (code, message) = + call(&home, "project.get", &[("project", t("shop/retired"))]).unwrap_err(); + assert_eq!(code, "NoSuchProject", "{message}"); + let live = ok(&home, "project.get", &[("project", t("shop/live"))]); + assert_eq!(live.get_text("project"), Some("shop/live")); + assert!(paths::project_record(&home, "shop", "retired").exists()); + let _ = std::fs::remove_dir_all(home); + let _ = std::fs::remove_dir_all(root); +} diff --git a/control/crates/layerfs-control/src/worker_server.rs b/control/crates/layerfs-control/src/worker_server.rs index 049b97b9e8..997d24c27e 100644 --- a/control/crates/layerfs-control/src/worker_server.rs +++ b/control/crates/layerfs-control/src/worker_server.rs @@ -10,8 +10,8 @@ //! build sends nothing more and closes. Then a request `{ op, args }`, read with at most //! [`REQUEST_WAIT`] between bytes. `worker.ping` answers who it is; //! `store.call`, `store.file`, `store.reach`, `sandbox.host`, `project.stage`, -//! `project.init`, `project.export`, `project.remove` and `project.delete` run -//! one operation on this machine, +//! `project.init`, `project.export`, `project.remove`, `project.delete` and +//! `project.list` run one operation on this machine, //! with `home` set to the worker's own root (the machine's data disk), //! never a path the caller sends. Answers are //! `{ value }` or `{ refused }`, after any `{ event }` frames. @@ -31,7 +31,10 @@ use layerfs_machine::worker::{ use layerfs_store::project::delete::delete_project; use layerfs_store::project::discovery::home_of; use layerfs_store::project::export::in_worker as export_in_worker; -use layerfs_store::project::frame::{DeleteFrame, ExportFrame, InitFrame, RemoveFrame, StageFrame}; +use layerfs_store::project::frame::{ + DeleteFrame, ExportFrame, InitFrame, ListingFrame, RemoveFrame, StageFrame, +}; +use layerfs_store::project::listing::list_projects; use layerfs_store::project::{init::in_worker, remove::remove_project, staged}; use std::net::{TcpListener, TcpStream}; use std::path::{Path, PathBuf}; @@ -282,6 +285,10 @@ fn run( let root = home_of(Some(root))?; delete_project(&root, &DeleteFrame::decode(&args)?) } + // reads the Store's own filtered project listing under this worker + // root, the same `layerfs_project::store::projects` a local Store + // uses (2.0, T4.5c): `{ store, store_record? }` + "project.list" => list_projects(root, &ListingFrame::decode(&args)?), // writes or discards one part of a caller's folder copy under this // worker's root (#536 H11a) "project.stage" => staged::run(root, &StageFrame::decode(args)?), diff --git a/control/crates/layerfs-store/src/project/export/walk.rs b/control/crates/layerfs-store/src/project/export/walk.rs index d97f2d4dee..6cd6ab29eb 100644 --- a/control/crates/layerfs-store/src/project/export/walk.rs +++ b/control/crates/layerfs-store/src/project/export/walk.rs @@ -19,6 +19,7 @@ use layerfs_history::NodeId; use layerfs_home::outcome::refused; use layerfs_store_service::history::{lookup, MAIN}; use layerfs_store_service::read::cached::Cached; +use layerfs_store_service::read::visibility::Visibility; use layerfs_store_service::store::StorePorts; use layerfs_store_service::StoreService; use layerfs_telemetry::timer::Timing; @@ -45,6 +46,9 @@ pub(super) struct PinnedSnapshot { pub(super) node: String, pub(super) root: String, tree: TreeTable, + /// §4.3: the pinned node's marker visibility. Hidden entries are omitted + /// from the exported folder and from its counts. + visibility: Visibility, } /// Resolves `(project, branch)` once into an immutable `(NodeId, ObjectId, @@ -65,12 +69,21 @@ pub(super) fn pin_snapshot( let root = lookup::root(&service.store, node)?; let reader = service.store.ports(stack.scope); let tree = TreeTable::open(reader.as_ref(), FilesystemRootId(root)).map_err(lookup::content)?; + let serial = match service + .store + .published(|history| history.node_history_serial(node)) + { + Ok(Some(serial)) => serial, + _ => 0, + }; + let visibility = Visibility::for_stack(&service.store, stack.id, serial); Ok(PinnedSnapshot { reader, label, node: node_text(node), root: hex(root.as_bytes()), tree, + visibility, }) } @@ -163,6 +176,11 @@ pub(super) fn stream_snapshot( rel.push(b'/'); } rel.extend_from_slice(name.as_bytes()); + // A hidden entry is omitted; a hidden directory's subtree with it, as + // the read surface omits it and the marker hides everything below it. + if pinned.visibility.hidden_relative(&rel).is_some() { + continue; + } match value.kind { InodeKind::Directory => { counts.directories += 1; diff --git a/control/crates/layerfs-store/src/project/family.rs b/control/crates/layerfs-store/src/project/family.rs index 452b82d960..af8d81619d 100644 --- a/control/crates/layerfs-store/src/project/family.rs +++ b/control/crates/layerfs-store/src/project/family.rs @@ -10,10 +10,10 @@ use crate::calls::Page; use crate::lease; use crate::open_args::OpenArgs; -use crate::place::{self, Executor}; +use crate::place::{self, Executor, Worker}; use crate::project::add::{add_project, ProjectAdd}; use crate::project::discovery::home_of; -use crate::project::frame::{InitFrame, RemoveFrame}; +use crate::project::frame::{InitFrame, ListingFrame, ListingReply, RemoveFrame}; use crate::project::postgres_init::PgProjectAdd; use crate::project::record::ProjectRecord; use crate::record::{StoreRecord, StoreRef}; @@ -58,19 +58,20 @@ struct LiveProject { } /// The live projects Store `store` lists, after the retired/hidden ones are -/// filtered out (spec §4.2). `None` means this route cannot list the Store -/// here (a sqlite Store on another machine, T4.5b); the home entries are -/// then left as the records say. A Store that cannot be listed is `Err`. +/// filtered out (spec §4.2). Every route reads the listing where the +/// Store's calls run (T4.5c): in this process for a sqlite Store on this +/// machine and for a PostgreSQL Store, in the machine's worker for a sqlite +/// Store on another machine. A Store that cannot be listed is `Err`. fn live_projects( home: &Path, store: &str, record: &StoreRecord, -) -> std::result::Result>, SdkRefusal> { +) -> std::result::Result, SdkRefusal> { let store_ref = record.store_ref()?; - if let Executor::Worker(_) = place::plan(home, store_ref)? { - return Ok(None); - } record.unblocked()?; + if let Executor::Worker(mut worker) = place::plan(home, store_ref)? { + return live_projects_on_worker(&mut worker, store, record); + } let (store_path, postgresql) = match store_ref { StoreRef::Postgres { .. } => ( record.postgresql_key(home, store), @@ -92,15 +93,37 @@ fn live_projects( .published(layerfs_project::store::projects) .map_err(failed) })?; - Ok(Some( - stacks - .into_iter() - .map(|stack| LiveProject { - name: stack.name.as_str().to_owned(), - stack: stack.id.to_bytes().to_vec(), - }) - .collect(), - )) + Ok(stacks + .into_iter() + .map(|stack| LiveProject { + name: stack.name.as_str().to_owned(), + stack: stack.id.to_bytes().to_vec(), + }) + .collect()) +} + +/// The listing a worker-backed Store sends (T4.5c): the worker runs the +/// same `layerfs_project::store::projects` under its own root, so a project +/// retired in the Store is hidden here as it is for a local one. +fn live_projects_on_worker( + worker: &mut Worker, + store: &str, + record: &StoreRecord, +) -> std::result::Result, SdkRefusal> { + let frame = ListingFrame { + store: StoreName::parse(store)?, + store_record: Some(record.to_record().clone()), + }; + let answer = worker.call("project.list", frame.encode(), &mut |_| {})?; + let reply = ListingReply::decode(&answer)?; + Ok(reply + .projects + .into_iter() + .map(|project| LiveProject { + name: project.name, + stack: project.stack.to_bytes().to_vec(), + }) + .collect()) } /// Whether `project` is in `live`: by the stack id its record holds, or, @@ -126,12 +149,11 @@ fn summary(home: &Path, store: &str, name: &str, record: &StoreRecord, view: Vie /// entry remains (spec §4.2). pub fn get(home: &Path, project: &ProjectRef) -> Outcome { let target = resolve::target(home, &project.to_string())?; - if let Some(live) = live_projects(home, &target.store, &target.record)? { - let record = ProjectRecord::read(home, &target.store, &target.project)? - .ok_or_else(|| refused("NoSuchProject", format!("no project {project}")))?; - if !listed(&record, &target.project, &live) { - return Err(refused("NoSuchProject", format!("no project {project}"))); - } + let live = live_projects(home, &target.store, &target.record)?; + let record = ProjectRecord::read(home, &target.store, &target.project)? + .ok_or_else(|| refused("NoSuchProject", format!("no project {project}")))?; + if !listed(&record, &target.project, &live) { + return Err(refused("NoSuchProject", format!("no project {project}"))); } Ok(summary( home, @@ -158,10 +180,10 @@ pub fn list(home: &Path, request: &ProjectList) -> Outcome { let Some(record) = StoreRecord::read(home, &store)? else { continue; }; - // The listing is the Store's own filtered one (spec §4.2): a home - // entry whose project is hidden or absent is not listed. An - // all-Stores listing omits a Store it cannot list; a named one - // reports the refusal. + // The listing is the Store's own filtered one (spec §4.2), read + // where its calls run (T4.5c): a home entry whose project is hidden + // or absent is not listed. An all-Stores listing omits a Store it + // cannot list; a named one reports the refusal. let live = match live_projects(home, &store, &record) { Ok(live) => live, Err(refusal) => { @@ -172,13 +194,11 @@ pub fn list(home: &Path, request: &ProjectList) -> Outcome { } }; for name in paths::names(&paths::store_dir(home, &store).join("project")).map_err(failed)? { - if let Some(live) = &live { - let Some(project) = ProjectRecord::read(home, &store, &name)? else { - continue; - }; - if !listed(&project, &name, live) { - continue; - } + let Some(project) = ProjectRecord::read(home, &store, &name)? else { + continue; + }; + if !listed(&project, &name, &live) { + continue; } items.push(summary(home, &store, &name, &record, View::List)); } diff --git a/control/crates/layerfs-store/src/project/frame.rs b/control/crates/layerfs-store/src/project/frame.rs index 6ca0d0b304..8de409d3a3 100644 --- a/control/crates/layerfs-store/src/project/frame.rs +++ b/control/crates/layerfs-store/src/project/frame.rs @@ -1,5 +1,6 @@ //! The `project` worker frames (layerfs-project#109): `project.init`, -//! `project.remove`, `project.stage` and `project.export`, each encoded by the caller and +//! `project.remove`, `project.list` (T4.5c), `project.stage` and +//! `project.export`, each encoded by the caller and //! decoded once in the worker, in the order the worker always refused them. //! The worker sets the home to its own root; a frame names none. use layerfs_bridge::control::{SdkRefusal, SdkValue}; @@ -95,6 +96,18 @@ fn staged_id(frame: &SdkValue) -> Result { Ok(id.to_owned()) } +/// The LayerStack identity a `stack_id` field holds: its 17 bytes in hex. +fn stack_id(text: &str) -> Result { + let bytes = + unhex(text).ok_or_else(|| refused("InvalidInput", "`stack_id` must be hex digits"))?; + LayerStackId::from_slice(&bytes).map_err(|_| { + refused( + "InvalidInput", + "`stack_id` must be a LayerStack identity (17 bytes)", + ) + }) +} + impl InitFrame { /// The `project.init` frame: `{ store, name, staged, timeoutMs, /// store_record }`, each sent when set. @@ -282,15 +295,7 @@ impl DeleteFrame { /// a Stack id that is not a LayerStack identity are refused. pub fn decode(frame: &SdkValue) -> Result { let store = StoreName::parse(required(frame.get_text("store"), "store")?)?; - let text = required(frame.get_text("stack_id"), "stack_id")?; - let bytes = - unhex(text).ok_or_else(|| refused("InvalidInput", "`stack_id` must be hex digits"))?; - let stack = LayerStackId::from_slice(&bytes).map_err(|_| { - refused( - "InvalidInput", - "`stack_id` must be a LayerStack identity (17 bytes)", - ) - })?; + let stack = stack_id(required(frame.get_text("stack_id"), "stack_id")?)?; Ok(Self { store, project: frame.get_text("project").map(str::to_owned), @@ -324,15 +329,7 @@ impl DeleteReply { .get_text("project") .ok_or_else(|| refused("InvalidInput", "reply has no `project`"))? .to_owned(); - let text = required(frame.get_text("stack_id"), "stack_id")?; - let bytes = - unhex(text).ok_or_else(|| refused("InvalidInput", "`stack_id` must be hex digits"))?; - let stack = LayerStackId::from_slice(&bytes).map_err(|_| { - refused( - "InvalidInput", - "`stack_id` must be a LayerStack identity (17 bytes)", - ) - })?; + let stack = stack_id(required(frame.get_text("stack_id"), "stack_id")?)?; let deleted_ms = frame .get_int("deleted_ms") .ok_or_else(|| refused("InvalidInput", "reply has no `deleted_ms`"))?; @@ -343,3 +340,95 @@ impl DeleteReply { }) } } + +/// `project.list` in a worker (T4.5c): Store `store`'s filtered project +/// listing, the same `layerfs_project::store::projects` a local or +/// PostgreSQL Store is read with, so a project retired in the Store is +/// hidden here too. A worker's root does not hold the caller's home +/// entries; the caller intersects this listing with them (`project.list`, +/// `project.get`). +#[derive(Clone, Debug)] +pub struct ListingFrame { + /// The Store to list. + pub store: StoreName, + /// The Store's record: it opens the Store under the worker's root. + pub store_record: Option, +} + +/// One project a worker's Store listing names. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ListedProject { + /// Its name in the Store. + pub name: String, + /// Its LayerStack. + pub stack: LayerStackId, +} + +/// `project.list` worker reply: `{ kind: "Listed", projects: [{ name, +/// stack_id }] }`, in the Store's name order. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ListingReply { + /// The Store's live projects. + pub projects: Vec, +} + +impl ListingFrame { + /// The `project.list` frame: `{ store, store_record? }`. + pub fn encode(&self) -> SdkValue { + let mut fields = vec![("store", SdkValue::text(self.store.as_str()))]; + fields.extend(self.store_record.clone().map(|r| ("store_record", r))); + map(fields) + } + + /// A `project.list` frame: no Store, then a bad Store name are refused. + pub fn decode(frame: &SdkValue) -> Result { + Ok(Self { + store: StoreName::parse(required(frame.get_text("store"), "store")?)?, + store_record: frame.get("store_record").cloned(), + }) + } +} + +impl ListingReply { + /// The reply a worker sends. + pub fn encode(&self) -> SdkValue { + let projects = self + .projects + .iter() + .map(|project| { + map(vec![ + ("name", SdkValue::text(&project.name)), + ("stack_id", SdkValue::text(hex(&project.stack.to_bytes()))), + ]) + }) + .collect(); + map(vec![ + ("kind", SdkValue::text("Listed")), + ("projects", SdkValue::List(projects)), + ]) + } + + /// The reply fields; `kind` must be `Listed`, and every project names + /// its Store name and its LayerStack identity. + pub fn decode(frame: &SdkValue) -> Result { + if frame.get_text("kind") != Some("Listed") { + return Err(refused( + "Failed", + "the worker project.list reply is not `Listed`", + )); + } + let Some(SdkValue::List(items)) = frame.get("projects") else { + return Err(refused("InvalidInput", "reply has no `projects` list")); + }; + let mut projects = Vec::with_capacity(items.len()); + for item in items { + let name = item + .get_text("name") + .ok_or_else(|| refused("InvalidInput", "reply project has no `name`"))? + .to_owned(); + let stack = stack_id(required(item.get_text("stack_id"), "stack_id")?)?; + projects.push(ListedProject { name, stack }); + } + Ok(Self { projects }) + } +} diff --git a/control/crates/layerfs-store/src/project/listing.rs b/control/crates/layerfs-store/src/project/listing.rs new file mode 100644 index 0000000000..99f3f2ac52 --- /dev/null +++ b/control/crates/layerfs-store/src/project/listing.rs @@ -0,0 +1,45 @@ +//! `project.list` in a worker (T4.5c): the Store's filtered project +//! listing, read under the worker's own root with the same +//! `layerfs_project::store::projects` a local or PostgreSQL Store uses, so +//! a project retired in the Store is hidden on that route too. +use crate::open_args::OpenArgs; +use crate::project::frame::{ListedProject, ListingFrame, ListingReply}; +use crate::service; +use layerfs_home::home::paths; +use layerfs_home::outcome::{failed, refused, Outcome}; +use std::path::Path; + +/// The `project.list` worker op: Store `store`'s live projects, in name +/// order, under `root` (the worker's own root). The caller intersects the +/// answer with its own home entries (`project.list`, `project.get`). +pub fn list_projects(root: &Path, frame: &ListingFrame) -> Outcome { + let store = frame.store.as_str(); + let store_record = frame + .store_record + .clone() + .ok_or_else(|| refused("InvalidInput", "project.list needs the Store's record"))?; + let open = OpenArgs { + home: Some(root.to_path_buf()), + store: Some(store.to_owned()), + store_path: Some(paths::store_file(root, store).display().to_string()), + store_record: Some(store_record), + postgresql: None, + ..OpenArgs::default() + }; + let stacks = service::with_service(&open, |service| { + service + .store + .published(layerfs_project::store::projects) + .map_err(failed) + })?; + Ok(ListingReply { + projects: stacks + .into_iter() + .map(|stack| ListedProject { + name: stack.name.as_str().to_owned(), + stack: stack.id, + }) + .collect(), + } + .encode()) +} diff --git a/control/crates/layerfs-store/src/project/mod.rs b/control/crates/layerfs-store/src/project/mod.rs index 8aadb31747..65597449ed 100644 --- a/control/crates/layerfs-store/src/project/mod.rs +++ b/control/crates/layerfs-store/src/project/mod.rs @@ -8,6 +8,7 @@ pub mod export; pub mod family; pub mod frame; pub mod init; +pub mod listing; pub mod parts; pub mod postgres_init; #[doc(hidden)] diff --git a/control/crates/layerfs-store/tests/project_list_frames.rs b/control/crates/layerfs-store/tests/project_list_frames.rs new file mode 100644 index 0000000000..344f1ee3f8 --- /dev/null +++ b/control/crates/layerfs-store/tests/project_list_frames.rs @@ -0,0 +1,122 @@ +//! The `project.list` worker frame and its reply (T4.5c): the map the +//! caller builds, the fields the worker decodes, and the filtered listing +//! it sends back. +use layerfs_bridge::control::SdkValue; +use layerfs_bridge::sdk_values::hex; +use layerfs_history::LayerStackId; +use layerfs_home::names::StoreName; +use layerfs_store::project::frame::{ListedProject, ListingFrame, ListingReply}; + +fn t(text: &str) -> SdkValue { + SdkValue::text(text) +} + +fn map(fields: Vec<(&str, SdkValue)>) -> SdkValue { + SdkValue::Map(fields.into_iter().map(|(n, v)| (n.to_owned(), v)).collect()) +} + +fn stack(body: u8) -> LayerStackId { + LayerStackId::from_authority([body; 16]) +} + +#[test] +fn the_frame_names_the_store_and_carries_its_record() { + let record = map(vec![("name", t("shop")), ("kind", t("sqlite"))]); + let frame = ListingFrame { + store: StoreName::parse("shop").unwrap(), + store_record: Some(record.clone()), + }; + let encoded = frame.encode(); + assert_eq!( + encoded, + map(vec![("store", t("shop")), ("store_record", record.clone())]) + ); + let decoded = ListingFrame::decode(&encoded).unwrap(); + assert_eq!(decoded.store, frame.store); + assert_eq!(decoded.store_record, Some(record)); +} + +#[test] +fn a_frame_with_only_the_store_decodes() { + let decoded = ListingFrame::decode(&map(vec![("store", t("shop"))])).unwrap(); + assert_eq!(decoded.store.as_str(), "shop"); + assert_eq!(decoded.store_record, None); +} + +#[test] +fn the_worker_refuses_a_bad_frame_in_order() { + let decode = |fields| ListingFrame::decode(&map(fields)).unwrap_err().message; + assert_eq!(decode(vec![]), "missing text argument `store`"); + assert!(decode(vec![("store", t("../x"))]).starts_with("store name")); +} + +#[test] +fn the_reply_round_trips_and_requires_listed() { + let reply = ListingReply { + projects: vec![ + ListedProject { + name: "app".to_owned(), + stack: stack(1), + }, + ListedProject { + name: "web".to_owned(), + stack: stack(2), + }, + ], + }; + let listed = |name: &str, stack: &LayerStackId| { + map(vec![ + ("name", t(name)), + ("stack_id", t(&hex(&stack.to_bytes()))), + ]) + }; + let encoded = reply.encode(); + assert_eq!( + encoded, + map(vec![ + ("kind", t("Listed")), + ( + "projects", + SdkValue::List(vec![listed("app", &stack(1)), listed("web", &stack(2))]), + ), + ]) + ); + assert_eq!(ListingReply::decode(&encoded).unwrap(), reply); + + let wrong_kind = map(vec![("kind", t("Removed"))]); + assert_eq!( + ListingReply::decode(&wrong_kind).unwrap_err().message, + "the worker project.list reply is not `Listed`" + ); + let no_list = map(vec![("kind", t("Listed"))]); + assert_eq!( + ListingReply::decode(&no_list).unwrap_err().message, + "reply has no `projects` list" + ); + let no_name = map(vec![ + ("kind", t("Listed")), + ( + "projects", + SdkValue::List(vec![map(vec![("stack_id", t(&hex(&stack(3).to_bytes())))])]), + ), + ]); + assert_eq!( + ListingReply::decode(&no_name).unwrap_err().message, + "reply project has no `name`" + ); + let bad_stack = map(vec![ + ("kind", t("Listed")), + ( + "projects", + SdkValue::List(vec![map(vec![("name", t("app")), ("stack_id", t("zz"))])]), + ), + ]); + assert_eq!( + ListingReply::decode(&bad_stack).unwrap_err().message, + "`stack_id` must be hex digits" + ); + let empty = ListingReply { + projects: Vec::new(), + }; + assert_eq!(ListingReply::decode(&empty.encode()).unwrap(), empty); +} diff --git a/engine/crates/layerfs-daemon/src/operator/mod.rs b/engine/crates/layerfs-daemon/src/operator/mod.rs index 9aead7c370..7408c42927 100644 --- a/engine/crates/layerfs-daemon/src/operator/mod.rs +++ b/engine/crates/layerfs-daemon/src/operator/mod.rs @@ -5,3 +5,9 @@ pub(crate) use dispatch::{admits, dispatch}; pub(crate) use layerfs_store_service::operator::{snapshot, WorkspaceView}; mod workspace; + +/// The installed Store fixture the operator unit test binds without a kernel. +#[cfg(test)] +#[allow(dead_code)] +#[path = "../../tests/support/installed_store.rs"] +mod installed_store; diff --git a/engine/crates/layerfs-daemon/src/operator/workspace.rs b/engine/crates/layerfs-daemon/src/operator/workspace.rs index 19d5cb8a64..536cb89986 100644 --- a/engine/crates/layerfs-daemon/src/operator/workspace.rs +++ b/engine/crates/layerfs-daemon/src/operator/workspace.rs @@ -5,13 +5,16 @@ use crate::{ }; use layerfs_bridge::control::{SdkRefusal, SdkValue}; use layerfs_content::{ - compare::TreeTable, filesystem::FilesystemRootId, object::inode_leaf::InodeKind, + compare::{EntryFacts, TreeTable}, + filesystem::FilesystemRootId, + object::inode_leaf::InodeKind, }; use layerfs_store_service::{ history::lookup::{self as args, Answer}, operator::args::{paging, text}, read::{ snapshot::{self, Kind, Snap}, + visibility::Visibility, window::Walker, }, }; @@ -80,6 +83,46 @@ impl<'a> Live<'a> { } Ok(None) } + + /// The next live entry that is visible at the base node. A path the markers + /// hide is absent unless the overlay holds a local inode there: a + /// re-creation after the bound node is an ordinary addition, as a Commit's + /// is, while an inherited entry of a mount older than the marker is hidden. + fn next_visible(&mut self, seen: &Visibility) -> Result> { + loop { + let Some((path, stat)) = self.next()? else { + return Ok(None); + }; + if seen.hidden(&path).is_some() + && self + .op + .overlay() + .inode(self.view.source(), stat.serial) + .map_err(args::failed)? + .is_none() + { + continue; + } + return Ok(Some((path, stat))); + } + } +} + +/// The base walk, less every path the markers hide at the base node: a removed +/// path is absent from `changes`, like a read of it, and so is its subtree. +struct VisibleBase<'a> { + walker: Walker<'a>, + seen: &'a Visibility, +} +impl VisibleBase<'_> { + fn next(&mut self) -> Result> { + loop { + match self.walker.next()? { + Some((path, _)) if self.seen.hidden(&path).is_some() => continue, + other => return Ok(other), + } + } + } } fn selected(service: &Service, args: &SdkValue) -> Result> { @@ -181,10 +224,17 @@ fn scan( .pending_conflicts() .map_err(args::failed)? .unwrap_or_default(); - let mut base = Walker::new(op.ports(), &snap, "/")?; + // §4.3: the base walk is path-aware, so a path the markers hide at the + // base node is absent from `changes` instead of a `Removed` row against + // the mount's whiteout. The live side keeps a local re-creation. + let seen = Visibility::for_snap(&service.store, &snap); + let mut base = VisibleBase { + walker: Walker::new(op.ports(), &snap, "/")?, + seen: &seen, + }; let mut live = Live::new(view, op); let mut work = layerfs_content::filesystem::attributes::AttributeReadWork::default(); - let (mut a, mut b) = (base.next()?, live.next()?); + let (mut a, mut b) = (base.next()?, live.next_visible(&seen)?); while a.is_some() || b.is_some() { let order = match (&a, &b) { (Some((a, _)), Some((b, _))) => a.split('/').cmp(b.split('/')), @@ -199,14 +249,14 @@ fn scan( } std::cmp::Ordering::Greater => { let (p, e) = b.take().expect("live"); - b = live.next()?; + b = live.next_visible(&seen)?; (p, None, Some(e)) } std::cmp::Ordering::Equal => { let (p, x) = a.take().expect("base"); let (_, y) = b.take().expect("live"); a = base.next()?; - b = live.next()?; + b = live.next_visible(&seen)?; (p, Some(x), Some(y)) } }; @@ -259,16 +309,20 @@ fn scan( break; } } - let _ = service; Ok(()) } pub(crate) fn changes(service: &Service, args: &SdkValue) -> Answer { let workspace = selected(service, args)?; + changes_of(service, &workspace, args) +} + +/// `changes` over an already selected Workspace; a test binds one directly. +fn changes_of(service: &Service, workspace: &BoundWorkspace, args: &SdkValue) -> Answer { let (limit, pos) = paging(args)?; - with_source(&workspace, |op, view| { + with_source(workspace, |op, view| { let (mut seen, mut items, mut spent, mut more) = (0u64, Vec::new(), 0, false); - scan(service, &workspace, op, view, |path, kind, markers| { + scan(service, workspace, op, view, |path, kind, markers| { if seen < pos { seen += 1; return Ok(true); @@ -317,3 +371,125 @@ pub(crate) fn overlay(service: &Service, args: &SdkValue) -> Answer { ("changes", SdkValue::Null), ])) } + +#[cfg(test)] +mod tests { + //! `workspace.changes` over a bound Workspace, without a kernel mount: the + //! path-awareness of its base walk. + use super::*; + use crate::control::Service; + use crate::{Owner, OwnerConfig}; + use layerfs_bridge::control::SdkValue; + use layerfs_history::{PathRemovalKind, PathRemovalRequest, WorkspaceId}; + use layerfs_storage::ReservationBlocks; + use layerfs_store_service::{ + bootstrap::open_store_observed, store::BindRequest, write::mutation::Mutation, + }; + use std::fs; + + use crate::operator::installed_store as installed; + + fn listed(answer: &SdkValue) -> Vec<(String, String)> { + let Some(SdkValue::List(items)) = answer.get("items") else { + panic!("no items: {answer:?}"); + }; + items + .iter() + .map(|item| { + ( + item.get_text("path").expect("path").to_owned(), + item.get_text("kind").expect("kind").to_owned(), + ) + }) + .collect() + } + + /// A file and a directory removed by `project.remove_file` markers at the + /// bound node are absent from `changes`, subtree and all; a visible + /// change is still listed, and a post-bound re-creation is an addition. + #[test] + fn a_removed_file_and_a_hidden_directory_are_absent_from_changes() { + let fixture = installed::Fixture::built("changes-visibility", |source| { + fs::write(source.join("visible.txt"), b"visible\n").unwrap(); + fs::write(source.join("hidden.txt"), b"hidden\n").unwrap(); + fs::create_dir(source.join("hidden-dir")).unwrap(); + fs::write(source.join("hidden-dir/inner.txt"), b"inner\n").unwrap(); + 4 + }); + let opened = open_store_observed( + fixture.config.clone(), + installed::BINDING, + installed::CURSOR, + 64, + 8 * 1024 * 1024, + ReservationBlocks::default(), + ) + .unwrap(); + let stack = opened + .store + .history() + .branch_snapshot(fixture.branch) + .unwrap() + .unwrap() + .branch + .stack; + for path in ["hidden.txt", "hidden-dir"] { + let outcome = opened + .store + .history() + .remove_path(&PathRemovalRequest { + stack, + path: path.as_bytes().to_vec(), + removed_ms: 1, + }) + .unwrap(); + assert_eq!(outcome.kind, PathRemovalKind::Removed, "{path}"); + } + let owner = Owner::start( + &fixture.directory.join("overlay.sqlite"), + layerfs_overlay::ProfileConfig::default(), + OwnerConfig::default(), + ) + .unwrap(); + let bound = opened + .store + .bind( + owner.client(), + BindRequest { + branch: fixture.branch, + workspace: WorkspaceId::from_authority([57; 32]).unwrap(), + }, + ) + .unwrap() + .workspace; + // Bind wrote the whiteouts; a visible change and one re-creation follow. + let mut mutation = Mutation::new(&bound).unwrap(); + mutation.write("/visible.txt", b"after\n", None).unwrap(); + mutation.finish().unwrap(); + let mut mutation = Mutation::new(&bound).unwrap(); + mutation.write("/hidden.txt", b"again\n", None).unwrap(); + mutation.finish().unwrap(); + + let service = Service::new(opened.store.clone(), &owner); + let answer = changes_of( + &service, + &bound, + &SdkValue::map([("pageSize", SdkValue::Int(50))]), + ) + .unwrap(); + assert_eq!( + listed(&answer), + [ + ("/hidden.txt".to_owned(), "Added".to_owned()), + ("/visible.txt".to_owned(), "Modified".to_owned()), + ] + ); + assert_eq!(answer.get("hasMore"), Some(&SdkValue::Bool(false))); + + drop(service); + bound.close().unwrap(); + owner.stop().unwrap(); + drop(opened); + fixture.cleanup(); + } +} diff --git a/engine/crates/layerfs-persistence/sql/postgresql/gc.sql b/engine/crates/layerfs-persistence/sql/postgresql/gc.sql index 713bb0c64e..b80cbe1420 100644 --- a/engine/crates/layerfs-persistence/sql/postgresql/gc.sql +++ b/engine/crates/layerfs-persistence/sql/postgresql/gc.sql @@ -71,3 +71,14 @@ CREATE TABLE gc_dead ( id BYTEA NOT NULL, PRIMARY KEY (kind, id) ); +CREATE TABLE kept_node ( + layer_stack_id BYTEA NOT NULL CHECK (length(layer_stack_id) = 17), + kind TEXT NOT NULL CHECK (kind IN ('commit','layer','branch','stack')), + node_id BYTEA NOT NULL, + serial BIGINT NOT NULL DEFAULT 0 CHECK (serial >= 0), + PRIMARY KEY (layer_stack_id, kind, node_id), + CHECK ( + (kind IN ('commit','layer') AND length(node_id) = 33) + OR (kind IN ('branch','stack') AND length(node_id) = 17) + ) +); diff --git a/engine/crates/layerfs-persistence/sql/sqlite/gc.sql b/engine/crates/layerfs-persistence/sql/sqlite/gc.sql index 10c62a803c..ee565544b9 100644 --- a/engine/crates/layerfs-persistence/sql/sqlite/gc.sql +++ b/engine/crates/layerfs-persistence/sql/sqlite/gc.sql @@ -80,3 +80,17 @@ CREATE TABLE gc_dead ( id BLOB NOT NULL, PRIMARY KEY (kind, id) ) STRICT, WITHOUT ROWID; +CREATE TABLE kept_node ( + layer_stack_id BLOB NOT NULL + CHECK (length(layer_stack_id) = 17 AND substr(layer_stack_id, 1, 1) = X'31'), + kind TEXT NOT NULL CHECK (kind IN ('commit','layer','branch','stack')), + node_id BLOB NOT NULL, + serial INTEGER NOT NULL DEFAULT 0 CHECK (serial >= 0), + PRIMARY KEY (layer_stack_id, kind, node_id), + CHECK ( + (kind = 'commit' AND length(node_id) = 33 AND substr(node_id, 1, 1) = X'12') + OR (kind = 'layer' AND length(node_id) = 33 AND substr(node_id, 1, 1) = X'32') + OR (kind = 'branch' AND length(node_id) = 17 AND substr(node_id, 1, 1) = X'11') + OR (kind = 'stack' AND length(node_id) = 17 AND substr(node_id, 1, 1) = X'31') + ) +) STRICT, WITHOUT ROWID; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_inode_blocks.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_inode_blocks.sql new file mode 100644 index 0000000000..433ca5fb2c --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_inode_blocks.sql @@ -0,0 +1 @@ +SELECT length(body) FROM inode_value_block ORDER BY first_ordinal diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_kept_roots.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_kept_roots.sql new file mode 100644 index 0000000000..180f300089 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_kept_roots.sql @@ -0,0 +1,7 @@ +SELECT c.root_id FROM kept_node k + JOIN "commit" c ON c.commit_id=k.node_id + WHERE k.kind='commit' +UNION +SELECT l.root_id FROM kept_node k + JOIN layer l ON l.layer_id=k.node_id + WHERE k.kind='layer' diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_live_branches.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_live_branches.sql new file mode 100644 index 0000000000..a20e525e28 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_live_branches.sql @@ -0,0 +1,4 @@ +SELECT b.branch_id,b.head_commit_id,b.base_layer_id FROM branch b + WHERE NOT EXISTS (SELECT 1 FROM branch_retired r WHERE r.branch_id=b.branch_id) + AND NOT EXISTS (SELECT 1 FROM layer_stack_retired s WHERE s.layer_stack_id=b.layer_stack_id) + ORDER BY b.branch_id diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_live_stacks.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_live_stacks.sql new file mode 100644 index 0000000000..0a20c54b38 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_live_stacks.sql @@ -0,0 +1,3 @@ +SELECT s.layer_stack_id,s.head_layer_id FROM layer_stack s + WHERE NOT EXISTS (SELECT 1 FROM layer_stack_retired r WHERE r.layer_stack_id=s.layer_stack_id) + ORDER BY s.layer_stack_id diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_object_locations.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_object_locations.sql new file mode 100644 index 0000000000..a96c9ecc9d --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_object_locations.sql @@ -0,0 +1,2 @@ +SELECT object_id,pack_id,group_number,record_number FROM object_location + ORDER BY pack_id,group_number,record_number diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/kept_node_clear.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/kept_node_clear.sql new file mode 100644 index 0000000000..2829711e8c --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/kept_node_clear.sql @@ -0,0 +1 @@ +DELETE FROM kept_node diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/kept_node_mark.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/kept_node_mark.sql new file mode 100644 index 0000000000..7dacb749e4 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/kept_node_mark.sql @@ -0,0 +1,81 @@ +WITH RECURSIVE +roots(kind,node_id) AS ( + SELECT 'stack',s.layer_stack_id FROM layer_stack s + WHERE NOT EXISTS (SELECT 1 FROM layer_stack_retired r WHERE r.layer_stack_id=s.layer_stack_id) + UNION + SELECT 'layer',l.layer_id FROM layer l + WHERE NOT EXISTS (SELECT 1 FROM layer_stack_retired r WHERE r.layer_stack_id=l.layer_stack_id) + UNION + SELECT 'branch',b.branch_id FROM branch b + WHERE NOT EXISTS (SELECT 1 FROM branch_retired r WHERE r.branch_id=b.branch_id) + AND NOT EXISTS (SELECT 1 FROM layer_stack_retired r WHERE r.layer_stack_id=b.layer_stack_id) +), +edges(from_kind,from_id,to_kind,to_id) AS ( + SELECT 'branch',b.branch_id,'commit',b.head_commit_id FROM branch b + JOIN "commit" c ON c.commit_id=b.head_commit_id + UNION ALL + SELECT 'branch',b.branch_id,'layer',b.base_layer_id FROM branch b + UNION ALL + SELECT 'branch',b.branch_id,'commit',b.forked_from_id FROM branch b + JOIN "commit" c ON c.commit_id=b.forked_from_id + UNION ALL + SELECT 'branch',b.branch_id,'layer',b.forked_from_id FROM branch b + JOIN layer l ON l.layer_id=b.forked_from_id + UNION ALL + SELECT 'layer',l.layer_id,'layer',l.parent_layer_id FROM layer l + WHERE l.parent_layer_id IS NOT NULL + UNION ALL + SELECT 'layer',l.layer_id,'commit',l.source_commit_id FROM layer l + WHERE l.source_commit_id IS NOT NULL + UNION ALL + SELECT 'layer',l.layer_id,'branch',l.source_branch_id FROM layer l + WHERE l.source_branch_id IS NOT NULL + UNION ALL + SELECT 'commit',c.commit_id,'commit',c.parent_commit_id FROM "commit" c + WHERE c.parent_commit_id IS NOT NULL + UNION ALL + SELECT 'commit',c.commit_id,'commit',c.second_parent_id FROM "commit" c + JOIN "commit" s ON s.commit_id=c.second_parent_id + UNION ALL + SELECT 'commit',c.commit_id,'layer',c.second_parent_id FROM "commit" c + JOIN layer l ON l.layer_id=c.second_parent_id + UNION ALL + SELECT 'commit',c.commit_id,'layer',c.base_layer_id FROM "commit" c + UNION ALL + SELECT 'commit',c.commit_id,'branch',c.branch_id FROM "commit" c + WHERE c.branch_id IS NOT NULL + UNION ALL + SELECT 'commit',c.commit_id,'commit',c.merged_from_id FROM "commit" c + JOIN "commit" s ON s.commit_id=c.merged_from_id + UNION ALL + SELECT 'commit',c.commit_id,'layer',c.merged_from_id FROM "commit" c + JOIN layer l ON l.layer_id=c.merged_from_id + UNION ALL + SELECT 'commit',c.commit_id,'commit',c.displaced_id FROM "commit" c + JOIN "commit" s ON s.commit_id=c.displaced_id + UNION ALL + SELECT 'commit',c.commit_id,'layer',c.displaced_id FROM "commit" c + JOIN layer l ON l.layer_id=c.displaced_id +), +closure(kind,node_id) AS ( + SELECT kind,node_id FROM roots + UNION + SELECT e.to_kind,e.to_id + FROM closure k JOIN edges e ON e.from_kind=k.kind AND e.from_id=k.node_id +) +INSERT INTO kept_node(layer_stack_id,kind,node_id,serial) +SELECT s.layer_stack_id,'stack',s.layer_stack_id,0 + FROM closure k JOIN layer_stack s ON s.layer_stack_id=k.node_id + WHERE k.kind='stack' +UNION ALL +SELECT b.layer_stack_id,'branch',b.branch_id,0 + FROM closure k JOIN branch b ON b.branch_id=k.node_id + WHERE k.kind='branch' +UNION ALL +SELECT l.layer_stack_id,'layer',l.layer_id,l.history_serial + FROM closure k JOIN layer l ON l.layer_id=k.node_id + WHERE k.kind='layer' +UNION ALL +SELECT c.layer_stack_id,'commit',c.commit_id,c.history_serial + FROM closure k JOIN "commit" c ON c.commit_id=k.node_id + WHERE k.kind='commit' diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/kept_node_read.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/kept_node_read.sql new file mode 100644 index 0000000000..3673d561ce --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/kept_node_read.sql @@ -0,0 +1 @@ +SELECT layer_stack_id,kind,node_id,serial FROM kept_node ORDER BY layer_stack_id,kind,node_id diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/mechanics.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/mechanics.rs index 7189700f7e..3bca07b93e 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/mechanics.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/mechanics.rs @@ -16,6 +16,9 @@ use layerfs_storage::{ use postgres::types::ToSql; impl HistoryStatements for Transaction<'_> { + fn mechanics(&self) -> &dyn Mechanics { + self + } fn history(&self, key: &str, params: Vec) -> Result, BackendError> { match history::statements::qualified(key) { Some(sql) => self.query(&sql.replace("{s}", self.schema), params), diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs index be40e0f957..6bc014d0e9 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs @@ -57,7 +57,7 @@ const ACQUISITION_TABLES: [&str; 3] = ["init_entry", "init_host_file", "init_ope /// (#90); a version 1 schema is refused at open, never converted. pub(crate) const SCHEMA_VERSION: i64 = 2; /// Every table of one Store schema, in name order. -pub(crate) const TABLES: [&str; 26] = [ +pub(crate) const TABLES: [&str; 27] = [ "absorbed", "binding_hint_block", "branch", @@ -71,6 +71,7 @@ pub(crate) const TABLES: [&str; 26] = [ "gc_op", "history_meta", "inode_value_block", + "kept_node", "layer", "layer_lineage", "layer_stack", @@ -210,6 +211,79 @@ pub(crate) fn create( Ok(()) } +/// The `kept_node` table as `gc.sql` writes it: one statement appended to the +/// GC script, so removing exactly this text leaves the source a Store written +/// before T7.1 recorded. +fn kept_node_table() -> &'static str { + let start = GC + .find("CREATE TABLE kept_node (") + .expect("the GC schema holds kept_node"); + let end = start + GC[start..].find(";\n").expect("kept_node ends") + 2; + &GC[start..end] +} + +/// The recorded source of a Store written before [`kept_node_table`]: the +/// current source with that one statement removed. +fn pre_kept_node_source(acquisition: bool) -> String { + source(acquisition).replacen(kept_node_table(), "", 1) +} + +/// Adds the `kept_node` history-closure table (T7.1) to a Store written +/// before it, then records the current schema: an additive migration under +/// the same conditions as [`add_gc`]. A schema that already has the table is +/// left alone; a recorded source or definition that is not an earlier +/// revision of the current one is refused, never migrated. The table is empty +/// until a mark writes it. +pub(crate) fn add_kept_node(session: &Session) -> Result<(), PersistenceError> { + // A Store that already has the table is left alone: one statement, no + // transaction and no lane wait. + let present = session.run_as::<_, BackendError>(Mode::Statement, |tx| { + let sql = format!( + "SELECT 1 FROM information_schema.tables WHERE table_schema='{}' AND table_name='kept_node' AND table_type='BASE TABLE'", + tx.schema + ); + Ok(!tx.query(&sql, vec![])?.is_empty()) + })?; + if present { + return Ok(()); + } + session.run_as(Mode::Write, |tx| -> Result<(), PersistenceError> { + // Re-check under the lane: another process may have migrated first. + let present = tx.query( + "SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='kept_node' AND table_type='BASE TABLE'", + vec![], + )?; + if !present.is_empty() { + return Ok(()); + } + let rows = tx.query( + "SELECT schema_source,schema_definition FROM history_meta WHERE id=1", + vec![], + )?; + let [row] = rows.as_slice() else { + return Err(BackendError::Integrity.into()); + }; + let recorded_source: String = row.get(0)?; + let acquisition = if is_earlier_source(&recorded_source, pre_kept_node_source(false)) { + false + } else if is_earlier_source(&recorded_source, pre_kept_node_source(true)) { + true + } else { + return Err(BackendError::Integrity.into()); + }; + if row.get::(1)? != definition(tx)? { + return Err(BackendError::Integrity.into()); + } + tx.batch(kept_node_table())?; + let definition = definition(tx)?; + tx.query( + "UPDATE history_meta SET schema_source=$1,schema_definition=$2 WHERE id=1", + vec![Param::Text(source(acquisition)), Param::Text(definition)], + )?; + Ok(()) + }) +} + /// Adds the GC marker, position and admission tables to a Store written /// before them, then records the current schema: an additive migration under /// the same conditions as the SQLite Store's `add_gc`. A schema that already diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/schema_diagnostic.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/schema_diagnostic.rs index a8c792c290..32e5f2cdee 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/schema_diagnostic.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/schema_diagnostic.rs @@ -127,6 +127,10 @@ init_operation removed_bytes bigint NO inode_value_block first_ordinal bigint NO inode_value_block last_ordinal bigint NO inode_value_block body bytea NO +kept_node layer_stack_id bytea NO +kept_node kind text NO +kept_node node_id bytea NO +kept_node serial bigint NO layer layer_id bytea NO layer layer_stack_id bytea NO layer parent_layer_id bytea YES diff --git a/engine/crates/layerfs-persistence/src/backend/seam.rs b/engine/crates/layerfs-persistence/src/backend/seam.rs index 5108cbed52..e654cf8877 100644 --- a/engine/crates/layerfs-persistence/src/backend/seam.rs +++ b/engine/crates/layerfs-persistence/src/backend/seam.rs @@ -23,6 +23,8 @@ pub(crate) enum Engine { /// Named history statements in one open engine transaction. pub(crate) trait HistoryStatements { fn history(&self, key: &str, params: Vec) -> Result, BackendError>; + /// The same open transaction's physical object and metadata access. + fn mechanics(&self) -> &dyn Mechanics; /// One statement over parallel column arrays, where the engine binds /// arrays (PostgreSQL: one round trip for many rows). `None`: the /// engine has no such statement and the caller runs its per-row one. diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/mechanics.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/mechanics.rs index ef0b9c1464..1407870041 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/mechanics.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/mechanics.rs @@ -15,6 +15,9 @@ impl HistoryStatements for Transaction<'_> { fn history(&self, key: &str, params: Vec) -> Result, BackendError> { Transaction::history(self, key, params) } + fn mechanics(&self) -> &dyn Mechanics { + self + } } impl Mechanics for Transaction<'_> { diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/schema.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/schema.rs index 353eec0f9a..d5685f4960 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/schema.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/schema.rs @@ -99,6 +99,17 @@ fn next_history_serial_column() -> &'static str { &HISTORY[start..end] } +/// The `kept_node` table as `gc.sql` writes it: one appended statement at the +/// end of the GC script, so removing exactly this text leaves the source a +/// Store written before T7.1 recorded. +fn kept_node_table() -> &'static str { + let start = GC + .find("CREATE TABLE kept_node (") + .expect("the GC schema holds kept_node"); + let end = start + GC[start..].find(";\n").expect("kept_node ends") + 2; + &GC[start..end] +} + /// The `commit` and `layer` position column as `history.sql` writes it; it /// appears once in each of the two tables. fn history_serial_column() -> &'static str { @@ -276,6 +287,7 @@ pub(crate) fn check( "path_removed", "history_meta", "inode_value_block", + "kept_node", "layer", "layer_lineage", "layer_stack", @@ -566,6 +578,44 @@ impl super::session::connection::Session { Ok(()) }) } + /// Adds the `kept_node` history-closure table (T7.1) to a Store written + /// before it, then records the current schema: an additive migration + /// (REBUILD.md kind 2) under the same conditions as + /// [`Self::add_binding_hints`]. The table is empty until a mark writes it. + pub(crate) fn add_kept_node(&self) -> Result<(), BackendError> { + self.run::<_, BackendError>(true, |tx| { + let present = tx.query( + "SELECT 1 FROM sqlite_schema WHERE type='table' AND name='kept_node'", + vec![], + )?; + if !present.is_empty() { + return Ok(()); + } + let rows = tx.query( + "SELECT schema_source,schema_definition FROM history_meta WHERE id=1", + vec![], + )?; + let Some(row) = rows.first() else { + return Ok(()); + }; + let current = source((self.layout, self.acquisition)); + let recorded = row.get::(0)?; + let without = current.replacen(kept_node_table(), "", 1); + let earlier = without.replacen(next_history_serial_column(), "", 1); + if (recorded != earlier && recorded != without) + || row.get::(1)? != definition(tx.connection, tx.work)? + { + return Ok(()); + } + tx.bootstrap(kept_node_table())?; + let definition = definition(tx.connection, tx.work)?; + tx.query( + "UPDATE history_meta SET schema_source=?1,schema_definition=?2 WHERE id=1", + vec![Param::Text(current), Param::Text(definition)], + )?; + Ok(()) + }) + } /// Adds the position state T1.4 introduced (`history_meta.next_history_serial`, /// appended NOT NULL DEFAULT 1) and backfills the `commit`/`layer` serials /// and `project_epoch` rows in the same write transaction: an additive diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs index e9d99d1bc2..c9f6f349f8 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs @@ -252,6 +252,30 @@ pub(crate) fn history(key: &str) -> Result<&'static str, BackendError> { "gc_epoch_share" => Ok(include_str!( "../../../../sql/sqlite/queries/history/gc_epoch_share.sql" )), + "gc_kept_roots" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/gc_kept_roots.sql" + )), + "gc_object_locations" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/gc_object_locations.sql" + )), + "gc_live_stacks" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/gc_live_stacks.sql" + )), + "gc_live_branches" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/gc_live_branches.sql" + )), + "gc_inode_blocks" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/gc_inode_blocks.sql" + )), + "kept_node_clear" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/kept_node_clear.sql" + )), + "kept_node_mark" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/kept_node_mark.sql" + )), + "kept_node_read" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/kept_node_read.sql" + )), _ => Err(BackendError::Integrity), } } diff --git a/engine/crates/layerfs-persistence/src/history/kept.rs b/engine/crates/layerfs-persistence/src/history/kept.rs new file mode 100644 index 0000000000..1e45197d47 --- /dev/null +++ b/engine/crates/layerfs-persistence/src/history/kept.rs @@ -0,0 +1,196 @@ +//! The §6.1 history closure: the retained roots and every history node they +//! reach, written to `kept_node` by a mark. +//! +//! The roots are every LayerStack with no `layer_stack_retired` row and every +//! Branch with no own or stack marker. A live stack keeps all its Layers; from +//! there the walk follows every history edge (Branch head/base/forked-from, +//! Layer parent/source Commit/source Branch, Commit parents/base Branch/merge +//! origin/displaced node) to its fixpoint. A retired Branch a retained node +//! reaches is kept through that node with its own edges, and a deleted project +//! is excluded because every history edge is stack-local. +use super::provider::HistoryProvider; +use super::rows::cell; +use super::transaction::Transaction; +use crate::backend::records::{Param, Record}; +use layerfs_history::error::{HistoryError, HistoryResult}; +use layerfs_history::identity::{BranchId, CommitId, LayerId, LayerStackId}; + +const CLEAR: &str = "kept_node_clear"; +const MARK: &str = "kept_node_mark"; +const READ: &str = "kept_node_read"; + +/// The identity of one node in a Store's retained history. +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub enum KeptNodeId { + /// A LayerStack (a project) no `layer_stack_retired` marker hides. + Stack(LayerStackId), + /// A Branch: a live one, or a retired one a retained node still reaches. + Branch(BranchId), + /// One immutable Commit. + Commit(CommitId), + /// One published Layer. + Layer(LayerId), +} + +impl KeptNodeId { + /// The `kept_node.kind` text of this identity. + pub fn kind(&self) -> &'static str { + match self { + Self::Stack(_) => "stack", + Self::Branch(_) => "branch", + Self::Commit(_) => "commit", + Self::Layer(_) => "layer", + } + } + + /// The tagged identity bytes. + pub fn as_slice(&self) -> &[u8] { + match self { + Self::Stack(id) => id.as_slice(), + Self::Branch(id) => id.as_slice(), + Self::Commit(id) => id.as_slice(), + Self::Layer(id) => id.as_slice(), + } + } + + /// Checks `kind` and its tagged identity bytes. + pub fn from_slice(kind: &str, bytes: &[u8]) -> HistoryResult { + match kind { + "stack" => Ok(Self::Stack(LayerStackId::from_slice(bytes)?)), + "branch" => Ok(Self::Branch(BranchId::from_slice(bytes)?)), + "commit" => Ok(Self::Commit(CommitId::from_slice(bytes)?)), + "layer" => Ok(Self::Layer(LayerId::from_slice(bytes)?)), + _ => Err(HistoryError::Integrity("kept node kind")), + } + } +} + +/// One node a mark retains, with the history serial the mark saw for it. +/// +/// `serial` is the node's own `history_serial`. A Branch and a LayerStack have +/// no serial column, so their nodes carry 0. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct KeptNode { + stack: LayerStackId, + id: KeptNodeId, + serial: i64, +} + +impl KeptNode { + /// The LayerStack the node belongs to. + pub fn stack(&self) -> LayerStackId { + self.stack + } + + /// The node identity. + pub fn id(&self) -> KeptNodeId { + self.id + } + + /// The node's history serial (0 for a Branch or a LayerStack). + pub fn serial(&self) -> i64 { + self.serial + } +} + +/// The retained history closure a mark wrote to `kept_node` (§6.1), in +/// `(layer_stack_id, kind, node_id)` order. +/// +/// The object closure (§6.2) starts from [`Self::commits`] and [`Self::layers`]. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct KeptHistory { + nodes: Vec, +} + +impl KeptHistory { + /// Every retained node, in `kept_node` order. + pub fn nodes(&self) -> &[KeptNode] { + &self.nodes + } + + /// How many nodes the closure holds. + pub fn len(&self) -> usize { + self.nodes.len() + } + + /// Whether the closure holds no node. + pub fn is_empty(&self) -> bool { + self.nodes.is_empty() + } + + /// Whether `id` is retained. + pub fn contains(&self, id: KeptNodeId) -> bool { + self.nodes.iter().any(|node| node.id == id) + } + + /// The node for `id`, when retained. + pub fn node(&self, id: KeptNodeId) -> Option { + self.nodes.iter().copied().find(|node| node.id == id) + } + + /// The retained Commits, in order. + pub fn commits(&self) -> impl Iterator + '_ { + self.nodes.iter().filter_map(|node| match node.id { + KeptNodeId::Commit(id) => Some(id), + _ => None, + }) + } + + /// The retained Layers, in order. + pub fn layers(&self) -> impl Iterator + '_ { + self.nodes.iter().filter_map(|node| match node.id { + KeptNodeId::Layer(id) => Some(id), + _ => None, + }) + } + + /// The retained Branches, in order. + pub fn branches(&self) -> impl Iterator + '_ { + self.nodes.iter().filter_map(|node| match node.id { + KeptNodeId::Branch(id) => Some(id), + _ => None, + }) + } + + /// The retained LayerStacks, in order. + pub fn stacks(&self) -> impl Iterator + '_ { + self.nodes.iter().filter_map(|node| match node.id { + KeptNodeId::Stack(id) => Some(id), + _ => None, + }) + } +} + +impl HistoryProvider { + /// Recomputes the §6.1 history closure into `kept_node` and returns it. + /// + /// It runs in one write transaction: `kept_node` is cleared and refilled, + /// so a second mark over unchanged history returns the same set and writes + /// the same rows. Nothing else is written; the object closure (§6.2) starts + /// from the returned [`KeptHistory::commits`] and [`KeptHistory::layers`]. + pub fn mark_history_closure(&self) -> HistoryResult { + self.write(mark) + } +} + +pub(super) fn mark(tx: &Transaction<'_>) -> HistoryResult { + tx.execute(CLEAR, Vec::::new())?; + tx.execute(MARK, Vec::::new())?; + let rows = tx.query(READ, Vec::::new())?; + let mut nodes = Vec::with_capacity(rows.len()); + for row in &rows { + nodes.push(kept_node(row)?); + } + Ok(KeptHistory { nodes }) +} + +fn kept_node(row: &Record) -> HistoryResult { + let stack: Vec = cell(row, 0)?; + let kind: String = cell(row, 1)?; + let id: Vec = cell(row, 2)?; + Ok(KeptNode { + stack: LayerStackId::from_slice(&stack)?, + id: KeptNodeId::from_slice(&kind, &id)?, + serial: cell(row, 3)?, + }) +} diff --git a/engine/crates/layerfs-persistence/src/history/mod.rs b/engine/crates/layerfs-persistence/src/history/mod.rs index e4a81e2b25..7e4c26e712 100644 --- a/engine/crates/layerfs-persistence/src/history/mod.rs +++ b/engine/crates/layerfs-persistence/src/history/mod.rs @@ -8,16 +8,20 @@ mod catalog; mod commit; mod digests; mod hints; +mod kept; mod layerstack; mod lineage; mod lineage_backfill; mod merge; +mod objects; mod paths; mod position; mod provider; mod removal; mod rows; mod transaction; +pub use kept::{KeptHistory, KeptNode, KeptNodeId}; +pub use objects::{KeptObjects, KeptWaste, MarkResult}; pub(crate) use lineage_backfill::backfill_lineage; pub use provider::HistoryProvider; pub(crate) use rows::{ diff --git a/engine/crates/layerfs-persistence/src/history/objects.rs b/engine/crates/layerfs-persistence/src/history/objects.rs new file mode 100644 index 0000000000..5a2af90032 --- /dev/null +++ b/engine/crates/layerfs-persistence/src/history/objects.rs @@ -0,0 +1,502 @@ +//! The §6.2 object closure of a mark, its `keptWaste` and the §6.4 live-heads +//! digest. +//! +//! A mark starts at the roots of every Commit and Layer the §6.1 history +//! closure retained. Each object is admitted once through `FinalizedObject`, so +//! its logical references (`references()`) are followed, and each locator's +//! direct delta base (`ChainBases::edge`) is followed, to the fixpoint. The +//! retained pack of every reached object is recorded, and the bytes of records +//! in those packs that no reached object owns are reported as `keptWaste`. +//! Nothing is deleted, rewritten or compacted here; the closure is kept in +//! memory for the rebuild path to use. +use super::kept::{self, KeptHistory}; +use super::provider::HistoryProvider; +use super::rows::{cell, object}; +use super::transaction::Transaction; +use crate::backend::records::Param; +use layerfs_content::filesystem::{FilesystemRoot, InodeScope}; +use layerfs_content::inode_leaf::INODE_VALUE_BYTES; +use layerfs_content::{ConstructionPolicy, FinalizedObject, ObjectId, ObjectRole}; +use layerfs_history::error::{HistoryError, HistoryResult}; +use layerfs_storage::{ + encoding::pool::PoolReader, + encoding::{ + delta::read::{BodyCaches, ChainBases, ChainCounters, Resolver}, + framed_record, DecompressionWorkspace, GroupCache, PackCache, + }, + location::{ObjectLocation, SignatureRow}, + pack::layout::{self, GroupCodec}, + source::Source, + StorageCapacities, StorageError, +}; +use layerfs_telemetry::timer::Timing; +use std::collections::{BTreeMap, BTreeSet, VecDeque}; + +const ROOTS: &str = "gc_kept_roots"; +const LOCATIONS: &str = "gc_object_locations"; +const LIVE_STACKS: &str = "gc_live_stacks"; +const LIVE_BRANCHES: &str = "gc_live_branches"; +const INODE_BLOCKS: &str = "gc_inode_blocks"; + +/// The bytes a mark keeps that no retained root actually needs (§6.2, §11). +/// +/// The report names three parts: bytes of unreferenced records inside a pack +/// the mark retains, bytes of objects reached only as a delta base, and the +/// inode-value block bytes 2.0 keeps for every live Store. A wholly dead pack is +/// not part of this value: it is not retained, so the rebuild path decides on it +/// from the refined live set. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct KeptWaste { + dead_in_retained_packs_bytes: u64, + delta_bases_bytes: u64, + delta_base_references: Vec, + inode_blocks_bytes: u64, +} + +impl KeptWaste { + /// Stored bytes of records in retained packs that no reached object owns. + pub fn dead_in_retained_packs_bytes(&self) -> u64 { + self.dead_in_retained_packs_bytes + } + + /// Stored bytes of objects reached only as a delta base. + pub fn delta_bases_bytes(&self) -> u64 { + self.delta_bases_bytes + } + + /// The objects counted by [`Self::delta_bases_bytes`], in ascending order. + pub fn delta_base_references(&self) -> &[ObjectId] { + &self.delta_base_references + } + + /// Bytes of `inode_value_block` rows, which 2.0 keeps in full (§6.4). + pub fn inode_blocks_bytes(&self) -> u64 { + self.inode_blocks_bytes + } + + /// Whether every part is zero. + pub fn is_empty(&self) -> bool { + self.dead_in_retained_packs_bytes == 0 + && self.delta_bases_bytes == 0 + && self.inode_blocks_bytes == 0 + } +} + +/// The object half of a mark: every object and pack the retained history +/// reaches (§6.2), in ascending order. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct KeptObjects { + objects: Vec, + packs: Vec, + waste: KeptWaste, +} + +impl KeptObjects { + /// Every reached canonical object, in ascending identity order. + pub fn objects(&self) -> &[ObjectId] { + &self.objects + } + + /// Every pack holding a reached object, in ascending pack order. + pub fn packs(&self) -> &[i64] { + &self.packs + } + + /// The bytes the mark keeps without a retained root needing them. + pub fn waste(&self) -> &KeptWaste { + &self.waste + } + + /// Whether `id` is reached. + pub fn contains(&self, id: ObjectId) -> bool { + self.objects.binary_search(&id).is_ok() + } + + /// How many objects the closure holds. + pub fn len(&self) -> usize { + self.objects.len() + } + + /// Whether the closure holds no object. + pub fn is_empty(&self) -> bool { + self.objects.is_empty() + } +} + +/// One complete mark: the §6.1 history closure, the §6.2 object closure and the +/// §6.4 digest of the live heads. +/// +/// A rebuild path chooses its plan from [`Self::objects`] and reports +/// [`Self::live_heads_digest`] to prove the generation it installed; nothing in +/// this value has been compacted or deleted. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct MarkResult { + history: KeptHistory, + objects: KeptObjects, + live_heads_digest: [u8; 32], +} + +impl MarkResult { + /// The retained history closure, exactly as written to `kept_node`. + pub fn history(&self) -> &KeptHistory { + &self.history + } + + /// The retained object and pack closure. + pub fn objects(&self) -> &KeptObjects { + &self.objects + } + + /// BLAKE3 over the §6.4 live-heads vector at the mark's fixed generation. + pub fn live_heads_digest(&self) -> [u8; 32] { + self.live_heads_digest + } +} + +impl HistoryProvider { + /// Recomputes the §6.1 history closure, then the §6.2 object closure over + /// its retained Commit and Layer roots, and the §6.4 live-heads digest. + /// + /// It runs in the same write transaction as [`Self::mark_history_closure`], + /// so a missing locator, a corrupt dependency or a failed admission aborts + /// the mark and leaves no `kept_node` row behind. Nothing is compacted; the + /// returned [`MarkResult`] is the rebuild path's input. + pub fn mark(&self) -> HistoryResult { + self.write(mark) + } +} + +fn mark(tx: &Transaction<'_>) -> HistoryResult { + let history = kept::mark(tx)?; + let policy = tx.mechanics().policy().map_err(storage)?; + let capacities = StorageCapacities::from_policy(policy).map_err(storage)?; + let source = ClosureSource { + mechanics: tx.mechanics(), + }; + let roots = roots(tx)?; + let closure = walk(&source, &capacities, policy.construction(), roots)?; + let waste = waste(tx, &source, &closure)?; + let digest = live_heads_digest(tx)?; + let objects = KeptObjects { + objects: closure.visited.keys().copied().collect(), + packs: closure.packs(), + waste, + }; + Ok(MarkResult { + history, + objects, + live_heads_digest: digest, + }) +} + +/// The roots of every retained Commit and Layer, deduplicated and ordered. +fn roots(tx: &Transaction<'_>) -> HistoryResult> { + let rows = tx.query(ROOTS, Vec::::new())?; + let mut roots = BTreeSet::new(); + for row in &rows { + roots.insert(object(row, 0)?); + } + Ok(roots.into_iter().collect()) +} + +/// The in-memory closure of one mark, before `keptWaste` is attributed. +struct Closure { + /// Every reached object and its winning locator. + visited: BTreeMap, + /// The stored width a read charges each reached object's record. + widths: BTreeMap, + /// Objects reached as a direct delta base. + bases: BTreeSet, + /// Objects reached as a root or through a logical reference. + logical: BTreeSet, +} + +impl Closure { + /// The retained packs: those holding at least one reached object. + fn packs(&self) -> Vec { + self.visited + .values() + .map(|location| location.pack_id) + .collect::>() + .into_iter() + .collect() + } +} + +/// Physical input for the shared encoding readers over one mark transaction. +struct ClosureSource<'a> { + mechanics: &'a dyn crate::backend::Mechanics, +} + +impl Source for ClosureSource<'_> { + fn ordered_dependencies(&self) -> bool { + false + } + + fn location(&self, id: ObjectId, ceiling: i64) -> Result, StorageError> { + let mut found = Vec::new(); + self.mechanics + .locate(&[id], &mut found) + .map_err(StorageError::from)?; + Ok(found + .into_iter() + .find(|row| row.location.object_id == id && row.location.pack_id <= ceiling) + .map(|row| row.location)) + } + + fn pack_bytes(&self, pack_id: i64) -> Result, StorageError> { + let mut found = Vec::new(); + self.mechanics + .read_packs(&[pack_id], &mut found) + .map_err(StorageError::from)?; + let pack = found.pop().ok_or(StorageError::Integrity("mark pack"))?; + if !found.is_empty() || pack.info().pack_id != pack_id { + return Err(StorageError::Integrity("mark pack binding")); + } + Ok(pack.into_parts().1) + } + + fn inode_values(&self, ordinals: &[u32]) -> Result, StorageError> { + let mut rows = Vec::new(); + self.mechanics + .inode_values(ordinals, &mut rows) + .map_err(StorageError::from)?; + ordinals + .iter() + .map(|ordinal| { + rows.iter() + .find(|row| row.ordinal == *ordinal) + .map(|row| row.value) + .ok_or(StorageError::Integrity("mark inode value")) + }) + .collect() + } + + fn signatures(&self) -> Result, StorageError> { + let mut rows = Vec::new(); + self.mechanics + .signatures(&mut rows) + .map_err(StorageError::from)?; + Ok(rows) + } + + fn write_signatures(&self, _rows: &[SignatureRow]) -> Result { + Err(StorageError::Integrity("mark is read-only")) + } +} + +/// Admits every object reachable from `roots` through logical references and +/// delta bases, exactly once, in a queue that runs to empty. +fn walk( + source: &ClosureSource<'_>, + capacities: &StorageCapacities, + construction: ConstructionPolicy, + roots: Vec, +) -> HistoryResult { + let mut packs = PackCache::new(); + let mut pool = PoolReader::new(); + let mut groups = GroupCache::new(); + let mut workspace = DecompressionWorkspace::new().map_err(storage)?; + let mut counters = ChainCounters::default(); + let mut closure = Closure { + visited: BTreeMap::new(), + widths: BTreeMap::new(), + bases: BTreeSet::new(), + logical: roots.iter().copied().collect(), + }; + let mut queued: BTreeSet = roots.iter().copied().collect(); + let mut queue: VecDeque = roots.into_iter().collect(); + while let Some(id) = queue.pop_front() { + let location = source + .location(id, i64::MAX) + .map_err(storage)? + .ok_or(HistoryError::Integrity("mark object locator"))?; + let canonical = { + let mut resolver = Resolver::new( + source, + i64::MAX, + capacities, + BodyCaches { + packs: &mut packs, + pool: &mut pool, + }, + &mut groups, + &mut workspace, + &mut counters, + ); + resolver.resolve(id).map_err(storage)?.0 + }; + let role = location.role; + let scope = if role == ObjectRole::FilesystemRoot { + FilesystemRoot::decode(&canonical) + .map_err(|_| HistoryError::Integrity("mark root scope"))? + .scope() + } else { + // No other admitted role reads the scope; the leaf encoders that + // derive references ignore it. + InodeScope::from_object(ObjectId::for_bytes(b"gc.mark/object-closure")) + }; + let admitted = Timing::disabled("gc.mark.admit", |timing| { + FinalizedObject::admit( + id, + role, + canonical, + construction, + scope, + timing.child("admit"), + ) + }) + .0 + .map_err(|_| HistoryError::Integrity("mark object admission"))?; + for reference in admitted.references() { + closure.logical.insert(*reference); + if queued.insert(*reference) { + queue.push_back(*reference); + } + } + let edge = { + let mut chain = ChainBases::new(&mut packs); + chain + .edge(source, &mut workspace, &location) + .map_err(storage)? + }; + closure.widths.insert(id, edge.width); + if let Some(base) = edge.base { + closure.bases.insert(base); + if queued.insert(base) { + queue.push_back(base); + } + } + closure.visited.insert(id, location); + } + Ok(closure) +} + +/// Attributes the bytes a retained pack keeps for records no reached object +/// owns, the delta-base-only bytes and the inode-value block bytes. +fn waste( + tx: &Transaction<'_>, + source: &ClosureSource<'_>, + closure: &Closure, +) -> HistoryResult { + let rows = tx.query(LOCATIONS, Vec::::new())?; + // pack -> group -> (record, object): the pack/object inventory. + let mut inventory: BTreeMap>> = BTreeMap::new(); + for row in &rows { + let id = object(row, 0)?; + let pack: i64 = cell(row, 1)?; + let group = usize::try_from(cell::(row, 2)?) + .map_err(|_| HistoryError::Integrity("mark pack inventory"))?; + let record = usize::try_from(cell::(row, 3)?) + .map_err(|_| HistoryError::Integrity("mark pack inventory"))?; + inventory + .entry(pack) + .or_default() + .entry(group) + .or_default() + .insert(record, id); + } + let mut dead_in_retained_packs_bytes = 0u64; + for pack_id in closure.packs() { + let body = source.pack_bytes(pack_id).map_err(storage)?; + let header = layout::parse_header(&body).map_err(storage)?; + let groups = inventory.get(&pack_id); + for group_number in 0..header.group_count { + let view = layout::group_view(&body, header, group_number).map_err(storage)?; + let records = groups.and_then(|groups| groups.get(&group_number)); + let live = records + .is_some_and(|records| records.values().any(|id| closure.visited.contains_key(id))); + if !live { + // No reached object owns any record of this group, so its + // stored extent stays only because the pack is retained. + dead_in_retained_packs_bytes = dead_in_retained_packs_bytes + .saturating_add(u64::try_from(view.end - view.start).unwrap_or(u64::MAX)); + } else if view.codec == GroupCodec::Raw { + // An exact per-record split is possible only when the body is + // stored uncompressed. A compressed group's bytes cannot be + // attributed to one record, so a retained group contributes + // nothing and the error stays conservative. + if let Some(records) = records { + for (record, id) in records { + if closure.visited.contains_key(id) { + continue; + } + let framed = + framed_record(&body[view.start..view.end], *record).map_err(storage)?; + dead_in_retained_packs_bytes = dead_in_retained_packs_bytes + .saturating_add(u64::try_from(framed.len()).unwrap_or(u64::MAX)); + } + } + } + } + } + let deltas: Vec = closure + .bases + .difference(&closure.logical) + .copied() + .collect(); + let delta_bases_bytes = deltas.iter().fold(0u64, |total, id| { + total.saturating_add(closure.widths.get(id).copied().unwrap_or(0)) + }); + let inode = tx.query(INODE_BLOCKS, Vec::::new())?; + let mut inode_blocks_bytes = 0u64; + for row in &inode { + let length: i64 = cell(row, 0)?; + inode_blocks_bytes = + inode_blocks_bytes.saturating_add(u64::try_from(length).unwrap_or(u64::MAX)); + } + Ok(KeptWaste { + dead_in_retained_packs_bytes, + delta_bases_bytes, + delta_base_references: deltas, + inode_blocks_bytes, + }) +} + +/// The §6.4 live-heads digest: BLAKE3 over `layerfs/live-heads/v1\0`, then the +/// non-retired stacks in identity order as `(stack_id, head_layer_id)`, then the +/// marker-free Branches in identity order as `(branch_id, head_commit_id or 33 +/// zero bytes, base_layer_id)`. Every field is fixed width, so a retired row is +/// absent rather than hashed as zeros and timestamps never enter the vector. +fn live_heads_digest(tx: &Transaction<'_>) -> HistoryResult<[u8; 32]> { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"layerfs/live-heads/v1\0"); + let stacks = tx.query(LIVE_STACKS, Vec::::new())?; + for row in &stacks { + let id: Vec = cell(row, 0)?; + let head: Vec = cell(row, 1)?; + if id.len() != 17 || head.len() != 33 { + return Err(HistoryError::Integrity("live-heads stack width")); + } + hasher.update(&id); + hasher.update(&head); + } + let branches = tx.query(LIVE_BRANCHES, Vec::::new())?; + for row in &branches { + let id: Vec = cell(row, 0)?; + let head: Option> = cell(row, 1)?; + let base: Vec = cell(row, 2)?; + if id.len() != 17 || base.len() != 33 { + return Err(HistoryError::Integrity("live-heads branch width")); + } + hasher.update(&id); + match head { + Some(head) if head.len() == 33 => hasher.update(&head), + Some(_) => return Err(HistoryError::Integrity("live-heads commit width")), + None => hasher.update(&[0u8; 33]), + }; + hasher.update(&base); + } + Ok(*hasher.finalize().as_bytes()) +} + +/// One physical read failure as a mark failure; an unproven outcome is never +/// folded into an integrity refusal. +fn storage(error: impl Into) -> HistoryError { + let error = error.into(); + if error.is_unknown_outcome() { + HistoryError::UnknownOutcome + } else { + HistoryError::Integrity("mark object closure") + } +} diff --git a/engine/crates/layerfs-persistence/src/history/transaction.rs b/engine/crates/layerfs-persistence/src/history/transaction.rs index 189bb915fd..7a5a1bbdfc 100644 --- a/engine/crates/layerfs-persistence/src/history/transaction.rs +++ b/engine/crates/layerfs-persistence/src/history/transaction.rs @@ -1,6 +1,6 @@ //! Shared history transaction facade; backend owns acknowledgement and errors. use super::{bindings::Parameters, provider::HistoryProvider}; -use crate::backend::{records::Record, Column, Engine, HistoryStatements}; +use crate::backend::{records::Record, Column, Engine, HistoryStatements, Mechanics}; use layerfs_history::HistoryResult; pub(crate) struct Transaction<'a> { backend: &'a dyn HistoryStatements, @@ -17,6 +17,10 @@ impl<'a> Transaction<'a> { pub(crate) fn execute(&self, key: &str, params: impl Parameters) -> HistoryResult { self.query(key, params).map(|r| r.len()) } + /// The same transaction's physical object and metadata access. + pub(crate) fn mechanics(&self) -> &dyn Mechanics { + self.backend.mechanics() + } /// The engine's array form of a statement, if it has one. pub(crate) fn arrays( &self, diff --git a/engine/crates/layerfs-persistence/src/lib.rs b/engine/crates/layerfs-persistence/src/lib.rs index 6845998af3..bbf172860e 100644 --- a/engine/crates/layerfs-persistence/src/lib.rs +++ b/engine/crates/layerfs-persistence/src/lib.rs @@ -29,7 +29,7 @@ pub use config::{ PersistenceConfig, PostgresCheck, PostgresSelection, SqliteAcquisitionSchema, SqlitePackLayout, SqlitePersistenceProfile, SynchronousCommit, }; -pub use history::HistoryProvider; +pub use history::{HistoryProvider, KeptHistory, KeptNode, KeptNodeId, KeptObjects, KeptWaste, MarkResult}; pub(crate) use storage::provider as storage_provider; pub(crate) use storage::publication; pub use storage_provider::StorageProvider; diff --git a/engine/crates/layerfs-persistence/src/store/open.rs b/engine/crates/layerfs-persistence/src/store/open.rs index 213707bcc7..f5073ebf29 100644 --- a/engine/crates/layerfs-persistence/src/store/open.rs +++ b/engine/crates/layerfs-persistence/src/store/open.rs @@ -108,6 +108,7 @@ impl SqliteHandles { // before `add_gc` records the full schema. session.add_gc_columns()?; session.add_gc()?; + session.add_kept_node()?; session.add_position_state()?; } let handles = Self::validated(session, directory, binding, cursor_key)?; diff --git a/engine/crates/layerfs-persistence/src/store/postgresql.rs b/engine/crates/layerfs-persistence/src/store/postgresql.rs index 5e11202868..078a767dc7 100644 --- a/engine/crates/layerfs-persistence/src/store/postgresql.rs +++ b/engine/crates/layerfs-persistence/src/store/postgresql.rs @@ -182,6 +182,7 @@ impl PostgresHandles { let session = Arc::new(Session::connect(selection, writable)?); if writable { schema::add_gc(&session)?; + schema::add_kept_node(&session)?; schema::add_position_state(&session)?; schema::add_position_columns(&session)?; } diff --git a/engine/crates/layerfs-persistence/tests/kept_node.rs b/engine/crates/layerfs-persistence/tests/kept_node.rs new file mode 100644 index 0000000000..310b003fb5 --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/kept_node.rs @@ -0,0 +1,390 @@ +//! #41 T7.1: the §6.1 history closure. A mark keeps every non-retired +//! LayerStack and every Branch with no own or stack marker, then follows every +//! history edge to its fixpoint and records the result in `kept_node`. A +//! retired Branch a retained node reaches stays; a deleted project is +//! excluded; a Store written before the table gains it at its first writable +//! open. +#![cfg(any(target_os = "macos", target_os = "linux"))] +mod support; + +use layerfs_history::{ + AddLayerOutcome, AddLayerRequest, BranchId, CommitMeta, CommitOutcome, CommitRecord, + CommitRequest, CommitVia, ForkRequest, ForkSource, HistoryCatalog, HistoryCatalogConfig, + LayerId, LayerRecord, LayerStackId, NodeId, StackInitialization, +}; +use layerfs_persistence::{KeptNodeId, PersistenceConfig, SqliteHandles, SqlitePersistenceProfile}; +use layerfs_storage::StoragePolicy; +use support::{branch_id, name, root, stack, Temp, BINDING}; + +fn config(path: &std::path::Path) -> PersistenceConfig { + PersistenceConfig::sqlite(path).with_sqlite_profile(SqlitePersistenceProfile::Disposable) +} + +fn create(path: &std::path::Path) -> SqliteHandles { + SqliteHandles::create( + config(path), + StoragePolicy::frozen_default(), + &HistoryCatalogConfig { + binding_key: BINDING.to_vec(), + cursor_key: [71; 32], + incarnation: 7, + }, + ) + .expect("Store creation") +} + +fn writable( + path: &std::path::Path, +) -> Result { + SqliteHandles::open_writable(config(path), BINDING, [71; 32]) +} + +fn definition(sql: &rusqlite::Connection) -> String { + let mut statement = sql + .prepare("SELECT type,name,sql FROM sqlite_schema WHERE substr(name,1,7)!='sqlite_' ORDER BY type,name") + .unwrap(); + let rows = statement + .query_map([], |r| Ok([r.get::<_, String>(0)?, r.get(1)?, r.get(2)?])) + .unwrap(); + rows.map(|r| r.unwrap().map(|v| v + "\n").concat()) + .collect() +} + +/// Rewrites a fresh Store into the format written before T7.1: no +/// `kept_node` table, recorded as that revision wrote it. Returns the fresh +/// source string it replaced. +fn strip_kept_node(path: &std::path::Path) -> String { + let sql = rusqlite::Connection::open(path).unwrap(); + let fresh: String = sql + .query_row("SELECT schema_source FROM history_meta", [], |r| r.get(0)) + .unwrap(); + let start = fresh + .find("CREATE TABLE kept_node (") + .expect("a fresh Store holds the table"); + let end = start + fresh[start..].find(";\n").expect("the table ends") + 2; + let old = format!("{}{}", &fresh[..start], &fresh[end..]); + assert_ne!(old, fresh); + sql.execute_batch("DROP TABLE kept_node;").unwrap(); + sql.execute( + "UPDATE history_meta SET schema_source=?1,schema_definition=?2", + rusqlite::params![old, definition(&sql)], + ) + .unwrap(); + fresh +} + +fn initialize(handles: &SqliteHandles, tag: u8) -> layerfs_history::LayerStackRecord { + handles + .history + .initialize_layerstack(&StackInitialization { + logical: 10, + stack: stack(tag), + name: name(&format!("project-{tag}")), + scope: root(200), + profile: root(201), + genesis_root: root(1), + }) + .unwrap() +} + +fn fork(handles: &SqliteHandles, stack: LayerStackId, branch: BranchId, tag: u8) { + let record = handles.history.layer_stack(stack).unwrap().unwrap(); + handles + .history + .fork(&ForkRequest { + stack, + branch, + name: name(&format!("branch-{tag}")), + source: ForkSource::Layer(record.head_layer), + }) + .unwrap(); +} + +fn commit(handles: &SqliteHandles, branch: BranchId, tag: u8) -> CommitRecord { + let snapshot = handles.history.branch(branch).unwrap().unwrap(); + let expected_root = match snapshot.head_commit { + Some(head) => handles.history.commit(head).unwrap().unwrap().root, + None => { + handles + .history + .layer(snapshot.base_layer) + .unwrap() + .unwrap() + .root + } + }; + match handles + .history + .commit_branch(&CommitRequest { + branch, + expected_head: snapshot.head_commit, + expected_base: snapshot.base_layer, + expected_root, + candidate_root: root(tag), + profile: root(201), + scope: root(200), + second_parent: None, + meta: CommitMeta::new(CommitVia::Files), + force: false, + }) + .unwrap() + { + CommitOutcome::Committed { record, .. } => *record, + other => panic!("commit: {other:?}"), + } +} + +fn publish(handles: &SqliteHandles, stack: LayerStackId, branch: BranchId) -> LayerRecord { + let branch_record = handles.history.branch(branch).unwrap().unwrap(); + let stack_record = handles.history.layer_stack(stack).unwrap().unwrap(); + match handles + .history + .add_layer(&AddLayerRequest { + stack, + branch, + commit: branch_record.head_commit.expect("a committed Branch"), + expected_stack_head: stack_record.head_layer, + expected_branch_base: branch_record.base_layer, + }) + .unwrap() + { + AddLayerOutcome::Added { layer } => layer, + other => panic!("publish: {other:?}"), + } +} + +fn genesis(handles: &SqliteHandles, stack: LayerStackId) -> LayerId { + let mut layer = handles + .history + .layer_stack(stack) + .unwrap() + .unwrap() + .head_layer; + loop { + let record = handles.history.layer(layer).unwrap().unwrap(); + match record.parent { + Some(parent) => layer = parent, + None => return layer, + } + } +} + +fn kept_rows(path: &std::path::Path) -> Vec<(Vec, String, Vec, i64)> { + let sql = rusqlite::Connection::open(path).unwrap(); + let mut statement = sql + .prepare("SELECT layer_stack_id,kind,node_id,serial FROM kept_node ORDER BY layer_stack_id,kind,node_id") + .unwrap(); + statement + .query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?))) + .unwrap() + .map(|r| r.unwrap()) + .collect() +} + +#[test] +fn a_marker_free_branch_is_a_root() { + let temp = Temp::new("kept-root"); + let path = temp.join("store.sqlite"); + let handles = create(&path); + let one = initialize(&handles, 1); + let work = branch_id(1); + fork(&handles, one.id, work, 1); + + let kept = handles.history.mark_history_closure().unwrap(); + assert!(kept.contains(KeptNodeId::Stack(one.id))); + assert!(kept.contains(KeptNodeId::Branch(work))); + assert!(kept.contains(KeptNodeId::Layer(genesis(&handles, one.id)))); + assert_eq!(kept.len(), 3, "a stack, its genesis Layer and one Branch"); + assert!(kept.commits().next().is_none()); + + // The returned set is exactly what the table holds, serials included. + let rows = kept_rows(&path); + assert_eq!(rows.len(), kept.len()); + for node in kept.nodes() { + assert_eq!(node.stack(), one.id); + let found = rows + .iter() + .find(|row| row.1 == node.id().kind() && row.2 == node.id().as_slice()) + .unwrap_or_else(|| panic!("{:?} not in kept_node", node.id())); + assert_eq!(found.3, node.serial()); + } + let genesis_node = kept + .node(KeptNodeId::Layer(genesis(&handles, one.id))) + .unwrap(); + let serial = handles + .history + .node_history_serial(NodeId::Layer(genesis(&handles, one.id))) + .unwrap() + .unwrap(); + assert_eq!(genesis_node.serial(), serial as i64); + assert_eq!(kept.node(KeptNodeId::Branch(work)).unwrap().serial(), 0); +} + +#[test] +fn a_retired_branch_a_retained_commit_reaches_is_kept() { + let temp = Temp::new("kept-retired"); + let path = temp.join("store.sqlite"); + let handles = create(&path); + let one = initialize(&handles, 1); + let work = branch_id(1); + fork(&handles, one.id, work, 1); + let record = commit(&handles, work, 9); + let layer = publish(&handles, one.id, work); + + handles.history.mark_branch_removed(work, 1_000).unwrap(); + assert!( + handles.history.branch(work).unwrap().is_none(), + "the marker hides the Branch from callers" + ); + + let kept = handles.history.mark_history_closure().unwrap(); + // A retained Commit names its Branch, so the retired row and its edges stay. + assert!(kept.contains(KeptNodeId::Commit(record.id))); + assert!(kept.contains(KeptNodeId::Branch(work))); + assert!(kept.contains(KeptNodeId::Layer(record.base_layer))); + assert!(kept.contains(KeptNodeId::Layer(layer.id))); + assert!(kept.contains(KeptNodeId::Stack(one.id))); + assert_eq!( + kept.node(KeptNodeId::Commit(record.id)).unwrap().serial(), + handles + .history + .node_history_serial(NodeId::Commit(record.id)) + .unwrap() + .unwrap() as i64 + ); +} + +#[test] +fn a_retired_branch_nothing_reaches_is_not_kept() { + let temp = Temp::new("kept-retired-unreferenced"); + let path = temp.join("store.sqlite"); + let handles = create(&path); + let one = initialize(&handles, 1); + let work = branch_id(1); + fork(&handles, one.id, work, 1); + let kept = handles.history.mark_history_closure().unwrap(); + assert!(kept.contains(KeptNodeId::Branch(work))); + + handles.history.mark_branch_removed(work, 1_000).unwrap(); + let kept = handles.history.mark_history_closure().unwrap(); + assert!( + !kept.contains(KeptNodeId::Branch(work)), + "a retired Branch no retained node reaches is collected" + ); + assert!(kept.contains(KeptNodeId::Stack(one.id))); +} + +#[test] +fn a_deleted_project_is_excluded() { + let temp = Temp::new("kept-deleted"); + let path = temp.join("store.sqlite"); + let handles = create(&path); + let one = initialize(&handles, 1); + let two = initialize(&handles, 2); + let first = branch_id(1); + let second = branch_id(2); + fork(&handles, one.id, first, 1); + let first_commit = commit(&handles, first, 9); + let first_layer = publish(&handles, one.id, first); + fork(&handles, two.id, second, 2); + let second_commit = commit(&handles, second, 10); + let second_layer = publish(&handles, two.id, second); + + handles.history.mark_stack_deleted(one.id, 2_000).unwrap(); + let kept = handles.history.mark_history_closure().unwrap(); + + assert!(!kept.contains(KeptNodeId::Stack(one.id))); + assert!(!kept.contains(KeptNodeId::Commit(first_commit.id))); + assert!(!kept.contains(KeptNodeId::Layer(first_layer.id))); + assert!(!kept.contains(KeptNodeId::Branch(first))); + assert!( + kept.stacks().all(|stack| stack == two.id), + "no node of the deleted project is reached" + ); + assert!(kept.contains(KeptNodeId::Stack(two.id))); + assert!(kept.contains(KeptNodeId::Commit(second_commit.id))); + assert!(kept.contains(KeptNodeId::Layer(second_layer.id))); + assert!(kept.contains(KeptNodeId::Branch(second))); + assert!(kept_rows(&path) + .iter() + .all(|row| row.0 == two.id.as_slice())); +} + +#[test] +fn a_mark_is_idempotent() { + let temp = Temp::new("kept-idempotent"); + let path = temp.join("store.sqlite"); + let handles = create(&path); + let one = initialize(&handles, 1); + let work = branch_id(1); + fork(&handles, one.id, work, 1); + commit(&handles, work, 9); + publish(&handles, one.id, work); + + let first = handles.history.mark_history_closure().unwrap(); + let rows = kept_rows(&path); + // A stale row from an interrupted earlier mark is cleared, never kept. + { + let sql = rusqlite::Connection::open(&path).unwrap(); + sql.execute( + "INSERT INTO kept_node(layer_stack_id,kind,node_id,serial) VALUES(?1,'branch',?2,0)", + rusqlite::params![one.id.as_slice(), branch_id(9).as_slice()], + ) + .unwrap(); + } + let second = handles.history.mark_history_closure().unwrap(); + assert_eq!(first, second, "the same history yields the same set"); + assert_eq!(rows, kept_rows(&path), "the same rows are written"); + assert!(!second.contains(KeptNodeId::Branch(branch_id(9)))); +} + +#[test] +fn a_store_before_kept_node_gains_the_table() { + let temp = Temp::new("kept-migration"); + let path = temp.join("store.sqlite"); + drop(create(&path)); + let fresh = strip_kept_node(&path); + + // The marker and its edges survive the migration; the table starts empty. + let handles = writable(&path).expect("a pre-T7.1 Store opens writable"); + let one = initialize(&handles, 1); + let work = branch_id(1); + fork(&handles, one.id, work, 1); + drop(handles); + drop(writable(&path).expect("an upgraded Store reopens")); + + let sql = rusqlite::Connection::open(&path).unwrap(); + let kept: i64 = sql + .query_row("SELECT count(*) FROM kept_node", [], |r| r.get(0)) + .unwrap(); + assert_eq!(kept, 0, "the migration writes no mark"); + let (source, recorded): (String, String) = sql + .query_row( + "SELECT schema_source,schema_definition FROM history_meta", + [], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .unwrap(); + assert_eq!(source, fresh, "the Store records the current schema"); + assert_eq!( + recorded, + definition(&sql), + "the recorded definition is the live one" + ); + drop(sql); + + let handles = writable(&path).expect("a migrated Store reopens"); + let kept = handles.history.mark_history_closure().unwrap(); + assert!(kept.contains(KeptNodeId::Branch(work))); + assert!(kept.contains(KeptNodeId::Stack(one.id))); +} + +#[test] +fn an_empty_store_marks_nothing() { + let temp = Temp::new("kept-empty"); + let path = temp.join("store.sqlite"); + let handles = create(&path); + let kept = handles.history.mark_history_closure().unwrap(); + assert!(kept.is_empty()); + assert!(kept_rows(&path).is_empty()); +} diff --git a/engine/crates/layerfs-persistence/tests/kept_objects.rs b/engine/crates/layerfs-persistence/tests/kept_objects.rs new file mode 100644 index 0000000000..8845281877 --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/kept_objects.rs @@ -0,0 +1,466 @@ +//! #41 T7.2: the §6.2 object closure, `keptWaste` and the §6.4 live-heads +//! digest. A mark keeps every object reachable from a retained Commit's or +//! Layer's root through logical references and delta bases, records the packs +//! those objects live in, reports the bytes of records in retained packs that +//! nothing reaches, and hashes the live heads. An unreferenced record, a record +//! only kept as a delta base, a deleted project and a moved head each move the +//! answer; nothing is compacted. +#![cfg(any(target_os = "macos", target_os = "linux"))] +mod support; + +use layerfs_content::{ + encode_whole_file_payload, + file::mapping::{self, ExtentNode, ExtentSlice, FileState}, + filesystem::attributes::PortableMetadata, + inode_leaf::{ + encode_inode_value, InodeKind, InodeLeaf, InodeLeafRow, InodeMetadata, InodeValue, + LEAF_ROW_BYTES, + }, + FinalizedObject, ObjectId, ObjectRole, +}; +use layerfs_history::{ + AddLayerOutcome, AddLayerRequest, BranchId, CommitMeta, CommitOutcome, CommitRecord, + CommitRequest, CommitVia, ForkRequest, ForkSource, HistoryCatalog, LayerRecord, LayerStackId, + LayerStackRecord, StackInitialization, +}; +use layerfs_persistence::SqliteHandles; +use layerfs_storage::{ + encoding::{encode_full, encode_prefix, raw_payload, CompressionWorkspace}, + location::{ObjectLocation, PackDomain}, + pack::{assemble, build_group, layout, PackLane}, + port::{PackPersistence, Publication, PublishedPack, Reserve}, + save::SaveProfile, + StorageCapacities, +}; +use std::sync::Arc; +use support::doctor::Fixture; +use support::{branch_id, name, root}; + +fn initialize(handles: &SqliteHandles, tag: u8, genesis: ObjectId) -> LayerStackRecord { + handles + .history + .initialize_layerstack(&StackInitialization { + logical: 10, + stack: support::stack(tag), + name: name(&format!("project-{tag}")), + scope: root(200), + profile: root(201), + genesis_root: genesis, + }) + .unwrap() +} + +fn fork(handles: &SqliteHandles, stack: LayerStackId, branch: BranchId, tag: u8) { + let record = handles.history.layer_stack(stack).unwrap().unwrap(); + handles + .history + .fork(&ForkRequest { + stack, + branch, + name: name(&format!("branch-{tag}")), + source: ForkSource::Layer(record.head_layer), + }) + .unwrap(); +} + +fn commit(handles: &SqliteHandles, branch: BranchId, candidate_root: ObjectId) -> CommitRecord { + let snapshot = handles.history.branch(branch).unwrap().unwrap(); + let expected_root = match snapshot.head_commit { + Some(head) => handles.history.commit(head).unwrap().unwrap().root, + None => { + handles + .history + .layer(snapshot.base_layer) + .unwrap() + .unwrap() + .root + } + }; + match handles + .history + .commit_branch(&CommitRequest { + branch, + expected_head: snapshot.head_commit, + expected_base: snapshot.base_layer, + expected_root, + candidate_root, + profile: root(201), + scope: root(200), + second_parent: None, + meta: CommitMeta::new(CommitVia::Files), + force: false, + }) + .unwrap() + { + CommitOutcome::Committed { record, .. } => *record, + other => panic!("commit: {other:?}"), + } +} + +fn publish(handles: &SqliteHandles, stack: LayerStackId, branch: BranchId) -> LayerRecord { + let branch_record = handles.history.branch(branch).unwrap().unwrap(); + let stack_record = handles.history.layer_stack(stack).unwrap().unwrap(); + match handles + .history + .add_layer(&AddLayerRequest { + stack, + branch, + commit: branch_record.head_commit.expect("a committed Branch"), + expected_stack_head: stack_record.head_layer, + expected_branch_base: branch_record.base_layer, + }) + .unwrap() + { + AddLayerOutcome::Added { layer } => layer, + other => panic!("publish: {other:?}"), + } +} + +fn whole_file(f: &Fixture, bytes: &[u8]) -> FinalizedObject { + f.put( + ObjectRole::WholeFile, + encode_whole_file_payload(bytes).unwrap(), + ) +} + +fn pack_of(f: &Fixture, id: ObjectId) -> i64 { + f.raw() + .query_row( + "SELECT pack_id FROM object_location WHERE object_id=?1", + rusqlite::params![id.as_bytes().as_slice()], + |row| row.get(0), + ) + .unwrap() +} + +/// Publishes one pack holding each entry as its own group, so a retained pack +/// can hold both a reached and an unreached record. +fn publish_groups(f: &Fixture, objects: &[FinalizedObject]) -> (i64, Vec) { + let capacities = StorageCapacities::from_policy(f.policy).unwrap(); + let mut workspace = CompressionWorkspace::new().unwrap(); + let mut profile = SaveProfile::default(); + let groups: Vec<_> = objects + .iter() + .map(|object| { + let record = encode_full( + object.canonical(), + object.role(), + &capacities, + &mut workspace, + &mut profile, + ) + .unwrap(); + build_group(PackLane::WholeFile, &[record.record], None).unwrap() + }) + .collect(); + let body = assemble(PackLane::WholeFile, &groups).unwrap(); + let reserved = f + .handles + .storage + .reserve(Reserve { + packs: 1, + ordinals: 0, + }) + .unwrap(); + f.handles + .storage + .publish(&Publication { + packs: vec![PublishedPack::seal( + reserved.first_pack_id, + PackDomain::Payload, + Arc::new(body.clone()), + )], + objects: objects + .iter() + .enumerate() + .map(|(group, object)| ObjectLocation { + object_id: object.id(), + role: object.role(), + canonical_length: object.canonical_len(), + pack_id: reserved.first_pack_id, + group_number: group, + record_number: 0, + }) + .collect(), + ..Publication::default() + }) + .unwrap(); + (reserved.first_pack_id, body) +} + +#[test] +fn a_retained_tree_and_inode_are_kept_with_their_packs() { + let f = Fixture::new(); + let chunk = f + .put( + ObjectRole::Chunk, + mapping::encode_chunk_object(b"tree payload").unwrap(), + ) + .id(); + let leaf = f + .put( + ObjectRole::ExtentLeaf, + mapping::encode_node( + &ExtentNode::Leaf { + subtree_logical_bytes: 12, + extents: vec![ExtentSlice::new(chunk, 0, 12).unwrap()], + }, + true, + ) + .unwrap(), + ) + .id(); + let tree = f + .put( + ObjectRole::FileState, + mapping::encode_file_state(FileState { + logical_len: 12, + extent_count: 1, + tree_level: 0, + profile_id: mapping::profile_id(), + mapping_root: leaf, + }) + .unwrap(), + ) + .id(); + let value = encode_inode_value(InodeValue { + kind: InodeKind::RegularFile, + namespace_ref_count: 1, + content_root: tree, + metadata_root: InodeMetadata::Inline(PortableMetadata { + mode: 0o644, + mtime_seconds: 0, + mtime_nanoseconds: 0, + }), + }); + let inode = f.inode( + InodeLeaf { + subtree_bytes: LEAF_ROW_BYTES as u64, + rows: vec![InodeLeafRow { serial: 7, value }], + } + .encode() + .unwrap(), + vec![value], + ); + + let stack = initialize(&f.handles, 1, tree); + let work = branch_id(1); + fork(&f.handles, stack.id, work, 1); + commit(&f.handles, work, inode); + publish(&f.handles, stack.id, work); + + let mark = f.handles.history.mark().unwrap(); + let objects = mark.objects(); + for id in [chunk, leaf, tree, inode] { + assert!(objects.contains(id), "object {id} is retained"); + assert!( + objects.packs().contains(&pack_of(&f, id)), + "the pack of {id} is retained" + ); + } + let blocks: i64 = f + .raw() + .query_row( + "SELECT COALESCE(SUM(length(body)),0) FROM inode_value_block", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(objects.waste().inode_blocks_bytes(), blocks as u64); + assert_eq!(objects.waste().dead_in_retained_packs_bytes(), 0); +} + +#[test] +fn an_unreferenced_record_in_a_retained_pack_is_kept_waste() { + let f = Fixture::new(); + let live = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(b"the live record").unwrap(), + ) + .unwrap(); + let dead = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(b"nobody reaches this record").unwrap(), + ) + .unwrap(); + let (pack, body) = publish_groups(&f, &[live.clone(), dead.clone()]); + let header = layout::parse_header(&body).unwrap(); + let dead_view = layout::group_view(&body, header, 1).unwrap(); + let expected = u64::try_from(dead_view.end - dead_view.start).unwrap(); + + let stack = initialize(&f.handles, 1, live.id()); + fork(&f.handles, stack.id, branch_id(1), 1); + + let mark = f.handles.history.mark().unwrap(); + let objects = mark.objects(); + assert!(objects.contains(live.id())); + assert!(!objects.contains(dead.id())); + assert_eq!(objects.packs(), &[pack]); + assert_eq!( + objects.waste().dead_in_retained_packs_bytes(), + expected, + "the unreferenced group's stored extent is the kept waste" + ); + assert!(objects.waste().delta_base_references().is_empty()); +} + +#[test] +fn the_digest_is_stable_and_moves_with_a_head() { + let f = Fixture::new(); + let start = whole_file(&f, b"first root").id(); + let stack = initialize(&f.handles, 1, start); + let work = branch_id(1); + fork(&f.handles, stack.id, work, 1); + + let first = f.handles.history.mark().unwrap(); + let second = f.handles.history.mark().unwrap(); + assert_eq!(first.live_heads_digest(), second.live_heads_digest()); + assert_eq!(first.objects(), second.objects()); + + let next = whole_file(&f, b"second root").id(); + commit(&f.handles, work, next); + let committed = f.handles.history.mark().unwrap(); + assert_ne!( + committed.live_heads_digest(), + first.live_heads_digest(), + "a moved Branch head changes the digest" + ); + + publish(&f.handles, stack.id, work); + let published = f.handles.history.mark().unwrap(); + assert_ne!( + published.live_heads_digest(), + committed.live_heads_digest(), + "a moved Stack head changes the digest" + ); +} + +#[test] +fn a_deleted_project_is_not_kept_unless_a_retained_root_reaches_it() { + let f = Fixture::new(); + // Stack one's genesis root is a chunk stack two's tree still references. + let shared = f + .put( + ObjectRole::Chunk, + mapping::encode_chunk_object(b"shared payload").unwrap(), + ) + .id(); + let only_one = whole_file(&f, b"only the deleted project").id(); + let leaf = f + .put( + ObjectRole::ExtentLeaf, + mapping::encode_node( + &ExtentNode::Leaf { + subtree_logical_bytes: 14, + extents: vec![ExtentSlice::new(shared, 0, 14).unwrap()], + }, + true, + ) + .unwrap(), + ) + .id(); + let other = whole_file(&f, b"the live project").id(); + + let one = initialize(&f.handles, 1, shared); + let first = branch_id(1); + fork(&f.handles, one.id, first, 1); + commit(&f.handles, first, only_one); + publish(&f.handles, one.id, first); + + let two = initialize(&f.handles, 2, other); + let second = branch_id(2); + fork(&f.handles, two.id, second, 2); + commit(&f.handles, second, leaf); + publish(&f.handles, two.id, second); + + f.handles.history.mark_stack_deleted(one.id, 2_000).unwrap(); + let mark = f.handles.history.mark().unwrap(); + let objects = mark.objects(); + assert!( + objects.contains(shared), + "the deleted project's object a retained root still references stays" + ); + assert!(objects.contains(leaf)); + assert!(objects.contains(other)); + assert!( + !objects.contains(only_one), + "an object only the deleted project reached is collected" + ); + assert!(!objects.packs().contains(&pack_of(&f, only_one))); +} + +#[test] +fn a_delta_base_is_enqueued_and_reported() { + let f = Fixture::new(); + let base = whole_file(&f, b"0123456789abcdef base payload"); + let target = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(b"0123456789abcdef base payload, extended").unwrap(), + ) + .unwrap(); + let capacities = StorageCapacities::from_policy(f.policy).unwrap(); + let record = encode_prefix( + target.canonical(), + target.role(), + base.id(), + raw_payload(base.canonical(), base.role()).unwrap(), + &capacities, + &mut CompressionWorkspace::new().unwrap(), + &mut SaveProfile::default(), + ) + .unwrap(); + let body = assemble( + PackLane::WholeFile, + &[build_group(PackLane::WholeFile, &[record.record], None).unwrap()], + ) + .unwrap(); + let reserved = f + .handles + .storage + .reserve(Reserve { + packs: 1, + ordinals: 0, + }) + .unwrap(); + f.handles + .storage + .publish(&Publication { + packs: vec![PublishedPack::seal( + reserved.first_pack_id, + PackDomain::Payload, + Arc::new(body), + )], + objects: vec![ObjectLocation { + object_id: target.id(), + role: target.role(), + canonical_length: target.canonical_len(), + pack_id: reserved.first_pack_id, + group_number: 0, + record_number: 0, + }], + ..Publication::default() + }) + .unwrap(); + + let stack = initialize(&f.handles, 1, target.id()); + fork(&f.handles, stack.id, branch_id(1), 1); + + let mark = f.handles.history.mark().unwrap(); + let objects = mark.objects(); + assert!(objects.contains(target.id())); + assert!( + objects.contains(base.id()), + "the delta base is reached through the stored record" + ); + assert!(objects.packs().contains(&pack_of(&f, target.id()))); + assert!( + objects.packs().contains(&pack_of(&f, base.id())), + "the base's pack is enqueued too" + ); + assert_eq!( + objects.waste().delta_base_references(), + &[base.id()], + "the base is kept only because the delta names it" + ); + assert!(objects.waste().delta_bases_bytes() > 0); +} diff --git a/engine/crates/layerfs-persistence/tests/lineage_backfill.rs b/engine/crates/layerfs-persistence/tests/lineage_backfill.rs index 6f535c0f99..d3f30df93d 100644 --- a/engine/crates/layerfs-persistence/tests/lineage_backfill.rs +++ b/engine/crates/layerfs-persistence/tests/lineage_backfill.rs @@ -19,7 +19,7 @@ const GC: &str = include_str!("../sql/sqlite/gc.sql"); /// appears once in each of the two tables (T1.2a). const HISTORY_SERIAL: &str = " history_serial INTEGER NOT NULL DEFAULT 0,\n"; /// The `store_policy` salt column as `metadata.sql` writes it (T1.2a). -const TRACE_SALT: &str = " trace_salt BLOB NOT NULL DEFAULT (X'0000000000000000000000000000000000000000000000000000000000000000') CHECK(length(trace_salt)=32)\n"; +const TRACE_SALT: &str = ",\n trace_salt BLOB NOT NULL DEFAULT (X'0000000000000000000000000000000000000000000000000000000000000000') CHECK(length(trace_salt)=32)"; /// The `history_meta` next serial column as `history.sql` writes it (T1.4). const NEXT_SERIAL: &str = " next_history_serial INTEGER NOT NULL DEFAULT 1,\n"; @@ -74,7 +74,7 @@ fn make_old(path: &std::path::Path) { DROP TABLE layer_lineage; DROP TABLE commit_lineage; DROP TABLE absorbed; DROP TABLE reach; \ DROP TABLE merge_operation; DROP TABLE merge_record; DROP TABLE file_digest; \ DROP TABLE branch_retired; DROP TABLE layer_stack_retired; DROP TABLE path_removed; \ - DROP TABLE project_epoch; DROP TABLE gc_intent; DROP TABLE gc_op; DROP TABLE gc_dead;", + DROP TABLE project_epoch; DROP TABLE gc_intent; DROP TABLE gc_op; DROP TABLE gc_dead; DROP TABLE kept_node;", ) .unwrap(); let source: String = sql diff --git a/engine/crates/layerfs-persistence/tests/old_format_columns.rs b/engine/crates/layerfs-persistence/tests/old_format_columns.rs index 24199e9840..25de101ee1 100644 --- a/engine/crates/layerfs-persistence/tests/old_format_columns.rs +++ b/engine/crates/layerfs-persistence/tests/old_format_columns.rs @@ -40,7 +40,7 @@ fn make_old(path: &std::path::Path) { ALTER TABLE store_policy DROP COLUMN trace_salt; \ ALTER TABLE history_meta DROP COLUMN next_history_serial; \ DROP TABLE branch_retired; DROP TABLE layer_stack_retired; DROP TABLE path_removed; \ - DROP TABLE project_epoch; DROP TABLE gc_intent; DROP TABLE gc_op; DROP TABLE gc_dead;", + DROP TABLE project_epoch; DROP TABLE gc_intent; DROP TABLE gc_op; DROP TABLE gc_dead; DROP TABLE kept_node;", ) .unwrap(); let source: String = sql diff --git a/engine/crates/layerfs-persistence/tests/old_format_open.rs b/engine/crates/layerfs-persistence/tests/old_format_open.rs index 55c99ab8c2..1af9b679d7 100644 --- a/engine/crates/layerfs-persistence/tests/old_format_open.rs +++ b/engine/crates/layerfs-persistence/tests/old_format_open.rs @@ -55,7 +55,7 @@ fn make_old(path: &std::path::Path) { DROP TABLE absorbed; DROP TABLE reach; DROP TABLE merge_operation; \ DROP TABLE merge_record; DROP TABLE file_digest; \ DROP TABLE branch_retired; DROP TABLE layer_stack_retired; DROP TABLE path_removed; \ - DROP TABLE project_epoch; DROP TABLE gc_intent; DROP TABLE gc_op; DROP TABLE gc_dead;", + DROP TABLE project_epoch; DROP TABLE gc_intent; DROP TABLE gc_op; DROP TABLE gc_dead; DROP TABLE kept_node;", ) .unwrap(); let source: String = sql diff --git a/engine/crates/layerfs-persistence/tests/sqlite_publication.rs b/engine/crates/layerfs-persistence/tests/sqlite_publication.rs index 3700b91c85..db33639b18 100644 --- a/engine/crates/layerfs-persistence/tests/sqlite_publication.rs +++ b/engine/crates/layerfs-persistence/tests/sqlite_publication.rs @@ -112,8 +112,9 @@ fn wal_full_settings_and_unavailable_engine_are_explicit() { // #327 dropped workspace_stage and added store_totals; #352 added binding_hint; // #353 added layer_lineage, commit_lineage, absorbed and reach; #355 // added merge_operation and merge_record; #364 added file_digest; - // the GC work added the markers, project_epoch, gc_intent, gc_op and gc_dead. - assert_eq!(names, 27); + // the GC work added the markers, project_epoch, gc_intent, gc_op, gc_dead + // and kept_node. + assert_eq!(names, 28); } #[test] fn final_body_multirow_locators_and_first_wins_have_one_atomic_ack() { diff --git a/engine/crates/layerfs-store-service/src/methods/args.rs b/engine/crates/layerfs-store-service/src/methods/args.rs index ce5e687878..f8bc3424cd 100644 --- a/engine/crates/layerfs-store-service/src/methods/args.rs +++ b/engine/crates/layerfs-store-service/src/methods/args.rs @@ -21,15 +21,6 @@ pub fn project(service: &StoreService, args: &SdkValue) -> Result Result<(Arc, Snap), SdkRefusal> { - let (_, reader, snap) = crate::operator::args::snapshot(&service.store, args)?; - Ok((reader, snap)) -} - /// The project, its reader and the snapshot a read names, in that order. pub fn snapshot_full( service: &StoreService, diff --git a/engine/crates/layerfs-store-service/src/methods/file/blame.rs b/engine/crates/layerfs-store-service/src/methods/file/blame.rs index 77704d466e..1c9a8dcaf4 100644 --- a/engine/crates/layerfs-store-service/src/methods/file/blame.rs +++ b/engine/crates/layerfs-store-service/src/methods/file/blame.rs @@ -1,17 +1,22 @@ //! `fs.blame`. -use crate::methods::args::{line, path, snapshot, Answer}; -use crate::{read::blame, StoreService}; +use crate::methods::args::{line, path, snapshot_full, Answer}; +use crate::{ + read::{blame, visibility}, + StoreService, +}; use layerfs_bridge::control::SdkValue; /// `blame(path, { branch, commit?, fromLine? })`: each line with the Commit that last changed it. pub fn blame(service: &StoreService, args: &SdkValue) -> Answer { - let (reader, snap) = snapshot(service, args)?; + let (stack, reader, snap) = snapshot_full(service, args)?; let path = path(args, "path")?; + let seen = visibility::Visibility::load(&service.store, &stack, &snap); blame::blame( &service.store, &reader, &snap, &path, + &seen, line(args, "fromLine", 1), line(args, "lineCount", 2000), &mut blame::Limits { diff --git a/engine/crates/layerfs-store-service/src/methods/file/glob.rs b/engine/crates/layerfs-store-service/src/methods/file/glob.rs index c7b21f7548..bde2596c77 100644 --- a/engine/crates/layerfs-store-service/src/methods/file/glob.rs +++ b/engine/crates/layerfs-store-service/src/methods/file/glob.rs @@ -17,6 +17,7 @@ pub fn glob(service: &StoreService, args: &SdkValue) -> Answer { return Err(refusal); } } + let removed = seen.hidden_matches(&reader, &snap, &path, pattern)?; if args.get("page").is_some() { return paged_reads::pin( &service.store, @@ -27,16 +28,17 @@ pub fn glob(service: &StoreService, args: &SdkValue) -> Answer { &snap, pattern, &path, + &seen, query::result_page(args)?, ) - .map(|value| visibility::filter(value, &seen)), + .map(|value| visibility::settle(value, &seen, removed)), ); } let (limit, pos) = paging(args)?; paged_reads::pin( &service.store, &snap, - reads::glob(&reader, &snap, pattern, &path, limit, pos) - .map(|value| visibility::filter(value, &seen)), + reads::glob(&reader, &snap, pattern, &path, &seen, limit, pos) + .map(|value| visibility::settle(value, &seen, removed)), ) } diff --git a/engine/crates/layerfs-store-service/src/methods/file/grep.rs b/engine/crates/layerfs-store-service/src/methods/file/grep.rs index c3737b7ebd..e9ca4d9ea4 100644 --- a/engine/crates/layerfs-store-service/src/methods/file/grep.rs +++ b/engine/crates/layerfs-store-service/src/methods/file/grep.rs @@ -1,19 +1,25 @@ //! `fs.grep`. -use crate::methods::args::{invalid, optional_path, paging, snapshot, text, Answer}; +use crate::methods::args::{invalid, optional_path, paging, snapshot_full, text, Answer}; use crate::{ - read::{paged_reads, query, reads}, + read::{paged_reads, query, reads, snapshot, visibility}, StoreService, }; use layerfs_bridge::control::SdkValue; /// `grep(pattern, { branch, commit?, path?, pageSize?, pagePos? })`: literal matches per line in text files. pub fn grep(service: &StoreService, args: &SdkValue) -> Answer { - let (reader, snap) = snapshot(service, args)?; + let (stack, reader, snap) = snapshot_full(service, args)?; let pattern = text(args, "pattern")?; if pattern.is_empty() { return Err(invalid("pattern is empty")); } let path = optional_path(args)?; + let seen = visibility::Visibility::load(&service.store, &stack, &snap); + if snapshot::entry(&reader, &snap.tree, &path)?.is_some() { + if let Some(refusal) = visibility::refuse(&path, &seen) { + return Err(refusal); + } + } if args.get("page").is_some() { return paged_reads::pin( &service.store, @@ -24,6 +30,7 @@ pub fn grep(service: &StoreService, args: &SdkValue) -> Answer { &snap, pattern, &path, + &seen, query::result_page(args)?, ), ); @@ -32,6 +39,6 @@ pub fn grep(service: &StoreService, args: &SdkValue) -> Answer { paged_reads::pin( &service.store, &snap, - reads::grep(&reader, &snap, pattern, &path, limit, pos), + reads::grep(&reader, &snap, pattern, &path, &seen, limit, pos), ) } diff --git a/engine/crates/layerfs-store-service/src/methods/file/list.rs b/engine/crates/layerfs-store-service/src/methods/file/list.rs index e983ab9080..4a5f57c52a 100644 --- a/engine/crates/layerfs-store-service/src/methods/file/list.rs +++ b/engine/crates/layerfs-store-service/src/methods/file/list.rs @@ -16,17 +16,19 @@ pub fn list(service: &StoreService, args: &SdkValue) -> Answer { return Err(refusal); } } + let removed = seen.hidden_children(&reader, &snap, &path)?; let answer = if args.get("page").is_some() { paged_reads::list( &service.store, &reader, &snap, &path, + &seen, query::result_page(args)?, ) } else { - reads::list(&reader, &snap, &path, paging(args)) + reads::list(&reader, &snap, &path, &seen, paging(args)) } - .map(|value| visibility::filter(value, &seen)); + .map(|value| visibility::settle(value, &seen, removed)); paged_reads::pin(&service.store, &snap, answer) } diff --git a/engine/crates/layerfs-store-service/src/methods/file/revision.rs b/engine/crates/layerfs-store-service/src/methods/file/revision.rs index 2081b2df18..7cadd3a46e 100644 --- a/engine/crates/layerfs-store-service/src/methods/file/revision.rs +++ b/engine/crates/layerfs-store-service/src/methods/file/revision.rs @@ -20,6 +20,7 @@ pub fn revision(service: &StoreService, args: &SdkValue) -> Answer { page: query::result_page(args)?, branch, continuation: args.get_text("continuation"), + stack: stack.id, }, ) } diff --git a/engine/crates/layerfs-store-service/src/read/attribution.rs b/engine/crates/layerfs-store-service/src/read/attribution.rs index 45dd860a82..e08f7df31a 100644 --- a/engine/crates/layerfs-store-service/src/read/attribution.rs +++ b/engine/crates/layerfs-store-service/src/read/attribution.rs @@ -2,6 +2,7 @@ use super::{ blame, file_facts, query, snapshot::{self, Snap}, + visibility::Visibility, }; use crate::history::lookup as args; use crate::store::{Store, StorePorts}; @@ -47,6 +48,7 @@ fn owners( reader: &StorePorts, snap: &Snap, path: &str, + visibility: &Visibility, budget: &mut AttributionBudget, ) -> Result { let (mut lines, mut from, mut exact) = (BTreeMap::new(), 1u64, true); @@ -59,7 +61,8 @@ fn owners( versions: budget.work, bytes: budget.bytes, }; - let page = blame::blame(store, reader, snap, path, Ok(from), Ok(2000), &mut limits)?; + let page = + blame::blame(store, reader, snap, path, visibility, Ok(from), Ok(2000), &mut limits)?; budget.work = limits.versions; budget.bytes = limits.bytes; exact &= !page.get_bool("comparisonCoarse") @@ -102,7 +105,7 @@ fn identity(owner: &SdkValue) -> Option<(&str, &str)> { pub fn align( store: &Arc, reader: &StorePorts, - sides: ((&Snap, &str), (&Snap, &str)), + sides: ((&Snap, &str, &Visibility), (&Snap, &str, &Visibility)), texts: (&str, &str), include: bool, ) -> Result { @@ -119,12 +122,12 @@ pub fn align( fn align_bounded( store: &Arc, reader: &StorePorts, - sides: ((&Snap, &str), (&Snap, &str)), + sides: ((&Snap, &str, &Visibility), (&Snap, &str, &Visibility)), texts: (&str, &str), include: bool, budget: &mut AttributionBudget, ) -> Result { - let ((left, left_path), (right, right_path)) = sides; + let ((left, left_path, left_seen), (right, right_path, right_seen)) = sides; let diff = lines(texts.0, texts.1); if diff.len() as u64 > query::MAXIMUM_QUERY_WORK { return Err(query::capacity("diff row budget reached")); @@ -135,12 +138,12 @@ fn align_bounded( exact: true, }; let before = if include && !texts.0.is_empty() { - owners(store, reader, left, left_path, budget)? + owners(store, reader, left, left_path, left_seen, budget)? } else { empty() }; let after = if include && !texts.1.is_empty() { - owners(store, reader, right, right_path, budget)? + owners(store, reader, right, right_path, right_seen, budget)? } else { empty() }; @@ -314,10 +317,12 @@ pub fn compare_bounded( let (Some(a), Some(b)) = (snapshot::text(reader, a)?, snapshot::text(reader, b)?) else { return Ok(unsupported("binary")); }; + let left_seen = Visibility::for_snap(store, left); + let right_seen = Visibility::for_snap(store, right); Ok(align_bounded( store, reader, - ((left, path), (right, path)), + ((left, path, &left_seen), (right, path, &right_seen)), (&a, &b), true, budget, diff --git a/engine/crates/layerfs-store-service/src/read/blame.rs b/engine/crates/layerfs-store-service/src/read/blame.rs index a5fabab769..a0c1ba9bd1 100644 --- a/engine/crates/layerfs-store-service/src/read/blame.rs +++ b/engine/crates/layerfs-store-service/src/read/blame.rs @@ -5,6 +5,7 @@ //! and from the common prefix only where both texts end it. use super::{ snapshot::{self, Snap}, + visibility::Visibility, window, }; use crate::{ @@ -135,11 +136,15 @@ pub fn blame( reader: &StorePorts, snap: &Snap, path: &str, + visibility: &Visibility, from: Arg, count: Arg, limits: &mut Limits, ) -> Answer { let tip = snapshot::existing(reader, snap, path)?; + if let Some(frontier) = visibility.hidden(path) { + return Err(Visibility::refusal(path, frontier)); + } if tip.value.kind != InodeKind::RegularFile { return Err(args::refuse( "NotFile", @@ -207,13 +212,25 @@ pub fn blame( limits.versions -= 1; let (_, parent, value) = revision(store, node)?; versions += 1; - let older = parent - .map(|p| revision(store, p)) - .transpose()? - .map(|r| entry(reader, r.0, path)) - .transpose()? - .flatten() - .filter(|e| e.value.kind == InodeKind::RegularFile); + // §4.3: a hidden older version is absent; blame stops there and + // attributes the remaining lines to the newest visible version. + let older = match parent { + Some(p) => { + let found = entry(reader, revision(store, p)?.0, path)? + .filter(|e| e.value.kind == InodeKind::RegularFile); + match found { + Some(_) + if visibility + .hidden_at(path, Visibility::node_serial(store, p)) + .is_some() => + { + None + } + other => other, + } + } + None => None, + }; if tracks.iter().all(|t| t.owner.is_some()) { break; } diff --git a/engine/crates/layerfs-store-service/src/read/branch_compare.rs b/engine/crates/layerfs-store-service/src/read/branch_compare.rs index c993ff1363..82492419d7 100644 --- a/engine/crates/layerfs-store-service/src/read/branch_compare.rs +++ b/engine/crates/layerfs-store-service/src/read/branch_compare.rs @@ -3,6 +3,7 @@ use super::{ attribution, compare, query::{result_page, MAXIMUM_QUERY_WORK}, snapshot::{self, Snap}, + visibility::Visibility, walk::Walker, }; use crate::{ @@ -22,17 +23,29 @@ fn walker<'a>( snap: &'a Snap, path: &str, after: Option<&str>, + visibility: &Visibility, ) -> Result>> { - if snapshot::entry(reader, &snap.tree, path)?.is_some() { + if snapshot::entry(reader, &snap.tree, path)?.is_some() + && visibility.hidden(path).is_none() + { Walker::after(reader, snap, path, after).map(Some) } else { Ok(None) } } -fn next(walk: &mut Option>) -> Result> { - match walk { - Some(walk) => walk.next(), - None => Ok(None), +/// The next entry that is visible at its snapshot; a hidden entry is absent. +fn next( + walk: &mut Option>, + visibility: &Visibility, +) -> Result> { + loop { + let Some(walker) = walk else { + return Ok(None); + }; + match walker.next()? { + Some((path, _)) if visibility.hidden(&path).is_some() => continue, + other => return Ok(other), + } } } fn set(item: &mut SdkValue, name: &str, value: SdkValue) { @@ -96,18 +109,23 @@ pub fn diff( let exhausted = left.node == right.node || matches!(cached, Some(None)); let after = cached.as_ref().and_then(|v| v.as_deref()); let skip = if cached.is_some() { 0 } else { page.skip }; + let left_seen = Visibility::for_snap(store, left); + let right_seen = Visibility::for_snap(store, right); let mut lw = if exhausted { None } else { - walker(reader, left, &prefix, after)? + walker(reader, left, &prefix, after, &left_seen)? }; let mut rw = if exhausted { None } else { - walker(reader, right, &prefix, after)? + walker(reader, right, &prefix, after, &right_seen)? }; let mut last = None; - let (mut l, mut r) = (next(&mut lw)?, next(&mut rw)?); + let (mut l, mut r) = ( + next(&mut lw, &left_seen)?, + next(&mut rw, &right_seen)?, + ); let (mut seen, mut examined, mut bytes, mut more) = (0u64, 0u64, 0usize, false); let mut items = Vec::new(); let mut incomplete = false; @@ -130,19 +148,19 @@ pub fn diff( let (path, before, after) = match ordering { std::cmp::Ordering::Less => { let (p, e) = l.take().unwrap(); - l = next(&mut lw)?; + l = next(&mut lw, &left_seen)?; (p, Some(e), None) } std::cmp::Ordering::Greater => { let (p, e) = r.take().unwrap(); - r = next(&mut rw)?; + r = next(&mut rw, &right_seen)?; (p, None, Some(e)) } std::cmp::Ordering::Equal => { let (p, a) = l.take().unwrap(); let (_, b) = r.take().unwrap(); - l = next(&mut lw)?; - r = next(&mut rw)?; + l = next(&mut lw, &left_seen)?; + r = next(&mut rw, &right_seen)?; (p, Some(a), Some(b)) } }; diff --git a/engine/crates/layerfs-store-service/src/read/compare.rs b/engine/crates/layerfs-store-service/src/read/compare.rs index ce21aa0ea7..5f2a0c4304 100644 --- a/engine/crates/layerfs-store-service/src/read/compare.rs +++ b/engine/crates/layerfs-store-service/src/read/compare.rs @@ -8,6 +8,7 @@ //! for a two-file difference to show. use super::reads::REPLY_BUDGET; use super::snapshot::{self, answer, existing, kind_name, Kind, Snap}; +use super::visibility::Visibility; use crate::history::lookup::{self as args, invalid, refuse, Answer, Arg}; use crate::store::{CatalogHints, Store, StorePorts}; use layerfs_bridge::control::{SdkRefusal, SdkValue}; @@ -281,7 +282,11 @@ fn changes( None => ChangeWalk::full(&memo, &left.tree, &right.tree, first.as_ref()), } .map_err(args::content)?; - emit(reader, &mut walk, &request) + // §4.3: a hidden entry is absent on its own side, so a removed path is an + // \`Added\`/\`Removed\` change, never a spurious \`Modified\`. + let left_seen = Visibility::for_snap(store, left); + let right_seen = Visibility::for_snap(store, right); + emit(reader, &mut walk, &request, &left_seen, &right_seen) } /// The component of `cursor` right below directory `path`, when `cursor` @@ -310,6 +315,8 @@ fn emit( reader: &StorePorts, walk: &mut ChangeWalk<'_>, request: &Request, + left_seen: &Visibility, + right_seen: &Visibility, ) -> Result<(Vec, Next), SdkRefusal> { let (limit, pos) = (request.limit, request.pos); let skip = if request.cursor.is_some() { 0 } else { pos }; @@ -317,7 +324,14 @@ fn emit( let mut work = layerfs_content::filesystem::attributes::AttributeReadWork::default(); let mut last = None; while let Some(pair) = walk.next_pair().map_err(args::content)? { - let (path, before, after) = &pair; + let (path, raw_before, raw_after) = &pair; + let before = (*raw_before).filter(|_| left_seen.hidden(path).is_none()); + let after = (*raw_after).filter(|_| right_seen.hidden(path).is_none()); + // Both hidden: the path is absent on both sides, not a change. + if before.is_none() && after.is_none() { + walk.enter(&pair).map_err(args::content)?; + continue; + } if let Some(cursor) = &request.cursor { if order(path, cursor) != std::cmp::Ordering::Greater { let on_path = cursor == path @@ -362,7 +376,7 @@ fn emit( &mut work, ) .map_err(args::content)? - && same_content(reader, x, y)?); + && same_content(reader, &x, &y)?); !same } _ => before != after, @@ -385,8 +399,8 @@ fn emit( let listed = PathChange { path: path.clone(), kind, - left: *before, - right: *after, + left: before, + right: after, }; if let Some(item) = change(reader, &listed, &mut budget, request.hunks)? { items.push(item); diff --git a/engine/crates/layerfs-store-service/src/read/file_compare.rs b/engine/crates/layerfs-store-service/src/read/file_compare.rs index 5356a200cf..f6b055764e 100644 --- a/engine/crates/layerfs-store-service/src/read/file_compare.rs +++ b/engine/crates/layerfs-store-service/src/read/file_compare.rs @@ -4,6 +4,7 @@ use super::{ compare::{self, Selector}, file_facts, query, snapshot::{self, Snap}, + visibility::Visibility, }; use crate::{ history::{lookup as args, saved}, @@ -28,6 +29,8 @@ struct Side { path: String, snap: Snap, entry: Option, + /// The governing marker frontier when this side is hidden at its node. + hidden: Option, text: Option, } @@ -50,28 +53,51 @@ fn side( "saved file selector belongs to another project", )); } - let entry = if allow_missing { - snapshot::entry(reader, &snap.tree, &selector.path)? + // §4.3: a side is visible, hidden or absent; a hidden side is treated as + // absent for the comparison but keeps its own `Removed` side object and + // never refuses, even when `allowMissing` is false. + let raw = snapshot::entry(reader, &snap.tree, &selector.path)?; + let hidden = if raw.is_some() { + Visibility::for_snap(store, &snap).hidden(&selector.path) + } else { + None + }; + let entry = if hidden.is_some() { + None + } else if allow_missing { + raw } else { Some(snapshot::existing(reader, &snap, &selector.path)?) }; - if entry.is_some_and(|e| e.value.kind == InodeKind::Directory) { + if hidden.is_none() && entry.is_some_and(|e| e.value.kind == InodeKind::Directory) { return Err(args::refuse( "IsDirectory", format!("{} is a directory", selector.path), SdkValue::map([("path", SdkValue::text(&selector.path))]), )); } - let text = snapshot::text(reader, entry)?; + let text = if hidden.is_some() { + None + } else { + snapshot::text(reader, entry)? + }; Ok(Side { path: selector.path, snap, entry, + hidden, text, }) } fn label(side: &Side) -> SdkValue { + if let Some(frontier) = side.hidden { + return SdkValue::map([ + ("removed", SdkValue::Bool(true)), + ("path", SdkValue::text(&side.path)), + ("frontier_serial", SdkValue::Int(frontier as i64)), + ]); + } if side.entry.is_none() { return SdkValue::Null; } @@ -83,6 +109,9 @@ fn label(side: &Side) -> SdkValue { } fn omitted(reader: &StorePorts, side: &Side) -> Result, SdkRefusal> { + if side.hidden.is_some() { + return Ok(None); + } Ok(match side.entry { None => None, Some(e) if e.value.kind != InodeKind::RegularFile => Some("unsupported_type"), @@ -103,7 +132,13 @@ pub fn diff( let reader = snapshot::reader(store, stack); let left = side(store, stack, &reader, left, options.allow_missing)?; let right = side(store, stack, &reader, right, options.allow_missing)?; - if left.entry.is_none() && right.entry.is_none() { + // Only a genuinely absent side is `NoSuchPath`; a hidden side is the + // `Removed` state, and two hidden sides are a removal with both objects. + if left.entry.is_none() + && right.entry.is_none() + && left.hidden.is_none() + && right.hidden.is_none() + { return Err(args::refuse( "NoSuchPath", "both file comparison paths are absent", @@ -134,10 +169,15 @@ pub fn diff( let left_text = left.text.as_deref().unwrap_or(""); let right_text = right.text.as_deref().unwrap_or(""); let mut aligned = if reason.is_none() && options.extended { + let left_seen = Visibility::for_snap(store, &left.snap); + let right_seen = Visibility::for_snap(store, &right.snap); attribution::align( store, &reader, - ((&left.snap, &left.path), (&right.snap, &right.path)), + ( + (&left.snap, &left.path, &left_seen), + (&right.snap, &right.path, &right_seen), + ), (left_text, right_text), options.include_attribution, )? @@ -159,7 +199,12 @@ pub fn diff( }, } }; - if options.include_attribution && left.snap.node == right.snap.node && left.path == right.path { + if options.include_attribution + && left.hidden.is_none() + && right.hidden.is_none() + && left.snap.node == right.snap.node + && left.path == right.path + { aligned.summary = attribution::AttributionSummary { changed: Some(false), count: Some(0), @@ -189,10 +234,13 @@ pub fn diff( &left_text.split('\n').collect::>(), &right_text.split('\n').collect::>(), ); + // The visible/hidden/absent table: a hidden side is absent for the + // change kind, and two hidden sides are a removal, never a refusal. let change_kind = match (left.entry, right.entry) { - (None, _) => "Added", - (_, None) => "Removed", - _ => { + (None, Some(_)) => "Added", + (Some(_), None) => "Removed", + (None, None) => "Removed", + (Some(_), Some(_)) => { if content || metadata || identity || aligned.summary.changed == Some(true) { "Modified" } else { diff --git a/engine/crates/layerfs-store-service/src/read/paged_reads.rs b/engine/crates/layerfs-store-service/src/read/paged_reads.rs index f3caabb2a4..f7352f459d 100644 --- a/engine/crates/layerfs-store-service/src/read/paged_reads.rs +++ b/engine/crates/layerfs-store-service/src/read/paged_reads.rs @@ -5,6 +5,7 @@ use super::{ query::{self, ResultPage, Work}, search, snapshot::{self, Snap}, + visibility::Visibility, walk, }; use crate::{ @@ -129,6 +130,7 @@ pub fn list( reader: &StorePorts, snap: &Snap, path: &str, + visibility: &Visibility, page: ResultPage, ) -> args::Answer { let key = key(snap, "list", path, "", page); @@ -165,6 +167,12 @@ pub fn list( .map_err(args::content)?; let mut selected = Vec::new(); for (name, serial) in &batch.entries { + if visibility + .hidden(&join(path, name)) + .is_some() + { + continue; + } work.step()?; if seen < skip { seen += 1; @@ -225,6 +233,7 @@ pub fn glob( snap: &Snap, pattern: &str, path: &str, + visibility: &Visibility, page: ResultPage, ) -> args::Answer { let key = key(snap, "glob", path, pattern, page); @@ -249,6 +258,9 @@ pub fn glob( let (mut work, mut seen, mut items, mut more, mut last) = (Work::default(), 0u64, Vec::new(), false, None); while let Some((file, _)) = walker.next()? { + if visibility.hidden(&file).is_some() { + continue; + } work.step()?; if !rule.matches_path(&file) { continue; @@ -290,6 +302,7 @@ pub fn grep( snap: &Snap, pattern: &str, path: &str, + visibility: &Visibility, page: ResultPage, ) -> args::Answer { let key = key(snap, "grep", path, pattern, page); @@ -325,6 +338,9 @@ pub fn grep( None => walker.next()?, }; let Some((file, entry)) = next else { break }; + if visibility.hidden(&file).is_some() { + continue; + } work.step()?; if entry.value.kind != InodeKind::RegularFile { continue; diff --git a/engine/crates/layerfs-store-service/src/read/reads.rs b/engine/crates/layerfs-store-service/src/read/reads.rs index 26ca567023..cb58357382 100644 --- a/engine/crates/layerfs-store-service/src/read/reads.rs +++ b/engine/crates/layerfs-store-service/src/read/reads.rs @@ -4,6 +4,7 @@ use super::{ cached::{self, Cached}, search, snapshot::{answer, existing, header, kind_name, size, Snap}, + visibility::Visibility, walk, window, }; use crate::history::lookup::{self as args, invalid, Answer, Arg}; @@ -100,7 +101,13 @@ pub fn stat(reader: &StorePorts, snap: &Snap, path: &str) -> Answer { /// One page of the directory at `path`, `paging` (limit, offset) from the /// caller. -pub fn list(reader: &StorePorts, snap: &Snap, path: &str, paging: Arg<(usize, u64)>) -> Answer { +pub fn list( + reader: &StorePorts, + snap: &Snap, + path: &str, + seen: &Visibility, + paging: Arg<(usize, u64)>, +) -> Answer { let cached = Cached::new(reader)?; // One memo for the call (#352 011.a1): the entries' inode lookups share // the inode table's upper pages, read once, so an entry costs its leaf. @@ -128,6 +135,12 @@ pub fn list(reader: &StorePorts, snap: &Snap, path: &str, paging: Arg<(usize, u6 .map_err(args::content)?; let mut taken = Vec::new(); for (name, serial) in &page.entries { + if seen + .hidden(&layerfs_content::compare::tree::join(path, name)) + .is_some() + { + continue; + } if skipped < pos { skipped += 1; continue; @@ -181,6 +194,7 @@ pub fn grep( snap: &Snap, pattern: &str, path: &str, + seen: &Visibility, limit: usize, pos: u64, ) -> Answer { @@ -200,7 +214,7 @@ pub fn grep( let cached = Cached::new(reader)?; let mut window = Vec::new(); walk::walk_within(&cached, snap, path, None, |file, entry| { - if entry.value.kind == InodeKind::RegularFile { + if entry.value.kind == InodeKind::RegularFile && seen.hidden(file).is_none() { window.push((file.to_owned(), entry)); } if window.len() == cached::ROOTS { @@ -302,6 +316,7 @@ pub fn glob( snap: &Snap, pattern: &str, path: &str, + seen: &Visibility, limit: usize, pos: u64, ) -> Answer { @@ -311,6 +326,9 @@ pub fn glob( let cached = Cached::new(reader)?; let within = walk::leading(pattern); walk::walk_within(&cached, snap, path, within, |file, _| { + if seen.hidden(file).is_some() { + return Ok(true); + } if !rule.matches_path(file) { return Ok(true); } diff --git a/engine/crates/layerfs-store-service/src/read/revision.rs b/engine/crates/layerfs-store-service/src/read/revision.rs index 9f87dbf75e..06e12dd06c 100644 --- a/engine/crates/layerfs-store-service/src/read/revision.rs +++ b/engine/crates/layerfs-store-service/src/read/revision.rs @@ -3,12 +3,13 @@ use super::{ file_facts, query, revision_pages::{self, Boundary, Key}, snapshot, + visibility::Visibility, }; use crate::history::{lookup as args, saved}; use crate::operator::files::{entry, revision}; use crate::store::{Store, StorePorts}; use layerfs_bridge::control::{SdkRefusal, SdkValue}; -use layerfs_history::NodeId; +use layerfs_history::{LayerStackId, NodeId}; use std::sync::Arc; fn parent( @@ -35,6 +36,7 @@ pub struct Request<'a> { pub page: query::ResultPage, pub branch: &'a str, pub continuation: Option<&'a str>, + pub stack: LayerStackId, } /// A complete page or Capacity: never advance past unreturned matches. @@ -50,12 +52,21 @@ pub fn list( mut page, branch, continuation, + stack, } = request; let provenance = match traversal { "provenance" => true, "branch" => false, _ => return Err(args::invalid("traversal must be provenance or branch")), }; + // §4.3: a hidden anchor refuses; a hidden older version ends the walk + // instead of being skipped over. + let visibility = Visibility::for_stack(store, stack, Visibility::node_serial(store, start)); + if entry(reader, args::root(store, start)?, path)?.is_some() { + if let Some(frontier) = visibility.hidden_at(path, visibility.serial()) { + return Err(Visibility::refusal(path, frontier)); + } + } let key = Key { start, path, @@ -120,11 +131,35 @@ pub fn list( examined += 1; let previous = parent(store, node, provenance)?; let now = entry(reader, args::root(store, node)?, path)?; + // A node the marker hides is never a returned version: the walk stops + // at the first one rather than listing it or walking past it. + if now.is_some() + && visibility + .hidden_at(path, Visibility::node_serial(store, node)) + .is_some() + { + break; + } + let mut hidden_before = false; let before = previous - .map(|p| entry(reader, args::root(store, p)?, path)) + .map(|p| { + let found = entry(reader, args::root(store, p)?, path)?; + if found.is_some() + && visibility + .hidden_at(path, Visibility::node_serial(store, p)) + .is_some() + { + hidden_before = true; + Ok(None) + } else { + Ok(found) + } + }) .transpose()? .flatten(); - next = if !provenance && branch != "main" && matches!(previous, Some(NodeId::Layer(_))) { + next = if hidden_before + || (!provenance && branch != "main" && matches!(previous, Some(NodeId::Layer(_)))) + { None } else { previous diff --git a/engine/crates/layerfs-store-service/src/read/visibility.rs b/engine/crates/layerfs-store-service/src/read/visibility.rs index 756d417b2f..ef60b2cf4c 100644 --- a/engine/crates/layerfs-store-service/src/read/visibility.rs +++ b/engine/crates/layerfs-store-service/src/read/visibility.rs @@ -5,13 +5,18 @@ //! least `N`'s `history_serial` (`§4.3`). A marked-but-absent path hides //! nothing, so callers check existence first. Reads and stats refuse with one //! `Removed` detail; listings and globs omit hidden entries and report one -//! top-level `removed_count`. +//! top-level `removed_count` over the queried subtree, filtered before paging. -use crate::read::snapshot::Snap; -use crate::store::Store; +use super::{snapshot, walk}; +use crate::store::{Store, StorePorts}; use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_content::merge::Rule; +use layerfs_content::object::inode_leaf::InodeKind; use layerfs_history::error::{RemovedPath, RemovedState}; -use layerfs_history::{HistoryError, LayerStackRecord, PathRemovalRecord}; +use layerfs_history::{ + HistoryError, LayerStackId, LayerStackRecord, NodeId, PathRemovalRecord, +}; +use std::collections::BTreeSet; use std::sync::Arc; /// The markers and node position one snapshot's reads filter against. @@ -23,33 +28,99 @@ pub struct Visibility { impl Visibility { /// Loads the snapshot's serial and its project's markers. A provider that /// does not implement the marker reads (a test double) sees none. - pub fn load(store: &Arc, stack: &LayerStackRecord, snap: &Snap) -> Self { - let serial = snap.serial; - let markers = match store.published(|history| history.path_removals(stack.id)) { + pub fn load(store: &Arc, stack: &LayerStackRecord, snap: &snapshot::Snap) -> Self { + Self::for_stack(store, stack.id, snap.serial) + } + + /// `load` when only the resolved snapshot is at hand. + pub fn for_snap(store: &Arc, snap: &snapshot::Snap) -> Self { + match Self::stack_of(store, snap.node) { + Some(stack) => Self::for_stack(store, stack, snap.serial), + None => Self { + serial: snap.serial, + markers: Vec::new(), + }, + } + } + + /// `for_snap` for a node that has not been resolved into a snapshot. + pub fn for_node(store: &Arc, node: NodeId) -> Self { + match Self::stack_of(store, node) { + Some(stack) => Self::for_stack(store, stack, Self::node_serial(store, node)), + None => Self { + serial: 0, + markers: Vec::new(), + }, + } + } + + /// The stack a saved node belongs to. + fn stack_of(store: &Arc, node: NodeId) -> Option { + use crate::history::lookup; + match node { + NodeId::Commit(id) => lookup::commit(store, id).ok().map(|record| record.stack), + NodeId::Layer(id) => lookup::layer(store, id).ok().map(|record| record.stack), + } + } + + /// `load` for a stack id and an explicit `history_serial`, for reads that + /// walk several nodes and therefore ask `hidden_at` per node. + pub fn for_stack(store: &Arc, stack: LayerStackId, serial: u64) -> Self { + let markers = match store.published(|history| history.path_removals(stack)) { Ok(markers) => markers, Err(_) => Vec::new(), }; Self { serial, markers } } - /// The governing frontier for a path: the maximum marker frontier over the - /// path and its `/`-boundary ancestors, if any marker applies. - fn frontier(&self, path: &str) -> Option { - let relative = path.strip_prefix('/').unwrap_or(path); + /// The node's `history_serial`, or 0 for a provider that keeps none. + pub fn node_serial(store: &Arc, node: NodeId) -> u64 { + match store.published(|history| history.node_history_serial(node)) { + Ok(Some(serial)) => serial, + _ => 0, + } + } + + /// The snapshot position the marker frontier is compared against. + pub fn serial(&self) -> u64 { + self.serial + } + + /// The governing frontier for a relative path: the maximum marker frontier + /// over the path and its `/`-boundary ancestors, if any marker applies. + fn frontier(&self, relative: &[u8]) -> Option { if relative.is_empty() { return None; } self.markers .iter() - .filter(|marker| layerfs_history::path::governs(&marker.path, relative.as_bytes())) + .filter(|marker| layerfs_history::path::governs(&marker.path, relative)) .map(|marker| marker.frontier_serial) .max() } + /// The governing frontier when the entry at `relative` (without the leading + /// `/`) is hidden at node `serial`. + pub fn hidden_relative_at(&self, relative: &[u8], serial: u64) -> Option { + let frontier = self.frontier(relative)?; + (frontier >= serial).then_some(frontier) + } + + /// The governing frontier when the entry at `relative` is hidden at this + /// node. Callers that already hold the path bytes (the export walk) avoid a + /// UTF-8 round trip. + pub fn hidden_relative(&self, relative: &[u8]) -> Option { + self.hidden_relative_at(relative, self.serial) + } + + /// The governing frontier when the entry at `path` is hidden at `serial`. + pub fn hidden_at(&self, path: &str, serial: u64) -> Option { + self.hidden_relative_at(path.strip_prefix('/').unwrap_or(path).as_bytes(), serial) + } + /// The governing frontier when the entry at `path` is hidden at this node. pub fn hidden(&self, path: &str) -> Option { - let frontier = self.frontier(path)?; - (frontier >= self.serial).then_some(frontier) + self.hidden_at(path, self.serial) } /// The refusal for one hidden read or stat, `{paths: [{path, frontier_serial}]}`. @@ -58,6 +129,112 @@ impl Visibility { RemovedPath::new(path, frontier_serial), ])))) } + + /// The number of hidden entries directly inside `dir` whose entries exist + /// at this snapshot: the `removed_count` of a listing, independent of the + /// page. A marker on a path below a direct child is not counted here. + pub fn hidden_children( + &self, + reader: &StorePorts, + snap: &snapshot::Snap, + dir: &str, + ) -> Result { + if self.markers.is_empty() { + return Ok(0); + } + let mut count = 0; + for marker in &self.markers { + let Ok(relative) = std::str::from_utf8(&marker.path) else { + continue; + }; + if relative.is_empty() { + continue; + } + let parent = match relative.rsplit_once('/') { + Some((parent, _)) => format!("/{parent}"), + None => "/".to_owned(), + }; + if parent != dir { + continue; + } + let full = format!("/{relative}"); + if snapshot::entry(reader, &snap.tree, &full)?.is_some() + && self.hidden(&full).is_some() + { + count += 1; + } + } + Ok(count) + } + + /// The number of hidden entries at or below `path` that the glob `pattern` + /// matches: the `removed_count` of a glob, independent of the page. Only + /// hidden subtrees are walked, so a snapshot with no marker counts nothing. + pub fn hidden_matches( + &self, + reader: &StorePorts, + snap: &snapshot::Snap, + path: &str, + pattern: &str, + ) -> Result { + if self.markers.is_empty() { + return Ok(0); + } + let rule = Rule::glob(pattern); + let root = path.trim_start_matches('/'); + let mut found: BTreeSet = BTreeSet::new(); + let mut walked: Vec = Vec::new(); + for marker in &self.markers { + let Ok(relative) = std::str::from_utf8(&marker.path) else { + continue; + }; + if relative.is_empty() { + continue; + } + if Self::under(root, relative).is_none() { + continue; + } + if walked + .iter() + .any(|hidden| layerfs_history::path::governs(hidden.as_bytes(), relative.as_bytes())) + { + // Already counted when the governing hidden directory was walked. + continue; + } + let full = format!("/{relative}"); + let Some(entry) = snapshot::entry(reader, &snap.tree, &full)? else { + continue; + }; + if self.hidden(&full).is_none() { + continue; + } + if rule.matches_path(&full) { + found.insert(full.clone()); + } + if entry.value.kind == InodeKind::Directory { + walked.push(relative.to_owned()); + walk::walk_within(reader, snap, &full, None, |child, _| { + if rule.matches_path(child) { + found.insert(child.to_owned()); + } + Ok(true) + })?; + } + } + Ok(found.len() as i64) + } + + /// The path `relative` within the query root `root` (both without a + /// leading `/`; an empty root is the whole tree), or `None` outside it. + fn under<'p>(root: &str, relative: &'p str) -> Option<&'p str> { + if root.is_empty() || relative == root { + Some(relative) + } else { + relative + .strip_prefix(root) + .and_then(|rest| rest.strip_prefix('/')) + } + } } /// The `Removed` refusal a hidden read or stat reports. @@ -67,10 +244,9 @@ pub fn refuse(path: &str, visibility: &Visibility) -> Option { .map(|frontier| Visibility::refusal(path, frontier)) } -/// Removes hidden entries from a list or glob answer and adds `removed_count`. -/// Entries themselves exist, so only marked-and-present paths are counted. -pub fn filter(mut value: SdkValue, visibility: &Visibility) -> SdkValue { - let mut removed = 0i64; +/// Replaces a list or glob answer: hidden entries are dropped and the one +/// top-level `removed_count` is the caller's page-independent total. +pub fn settle(mut value: SdkValue, visibility: &Visibility, removed: i64) -> SdkValue { if let SdkValue::Map(fields) = &mut value { if let Some((_, SdkValue::List(items))) = fields.iter_mut().find(|(k, _)| k == "items") { items.retain(|item| { @@ -81,15 +257,7 @@ pub fn filter(mut value: SdkValue, visibility: &Visibility) -> SdkValue { .map(str::to_owned) .unwrap_or_default(), }; - if path.is_empty() { - return true; - } - if visibility.hidden(&path).is_some() { - removed += 1; - false - } else { - true - } + path.is_empty() || visibility.hidden(&path).is_none() }); } fields.push(("removed_count".into(), SdkValue::Int(removed))); diff --git a/engine/crates/layerfs-store-service/src/write/files.rs b/engine/crates/layerfs-store-service/src/write/files.rs index 195efecfc8..281c3f0de8 100644 --- a/engine/crates/layerfs-store-service/src/write/files.rs +++ b/engine/crates/layerfs-store-service/src/write/files.rs @@ -4,9 +4,13 @@ //! the StoreService with its failure. use super::mutation::Mutation; use crate::history::lookup::{self as args, Answer, Arg}; +use crate::read::{snapshot, visibility::Visibility}; use crate::StoreService; use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_content::compare::TreeTable; +use layerfs_content::filesystem::FilesystemRootId; use layerfs_history::{CommitOutcome, CommitVia, WorkspaceId}; +use std::sync::Arc; pub fn path(value: &str) -> Result { if value.len() > 4096 @@ -193,16 +197,51 @@ fn change( workspace: &crate::store::BoundWorkspace, changes: &[Arg>], ) -> Result, SdkRefusal> { + // The bound node's raw tree and markers: the private Workspace's overlay + // already whiteouts a hidden path, so its absence there is ambiguous. + // The base view distinguishes a hidden path (a `Removed` refusal for + // `edit` and a cp source) from a genuinely absent one (`NoSuchPath`). + let base = workspace + .snapshot() + .map_err(crate::write::mutation::failure)?; + let node = crate::removal::node_of(&base); + let visibility = Visibility::for_stack( + &workspace.store, + base.branch.stack, + Visibility::node_serial(&workspace.store, node), + ); + let reader: Arc = workspace.store.ports(base.scope); + let tree = TreeTable::open(&*reader, FilesystemRootId(base.effective_root)) + .map_err(args::content)?; + let removed = |path: &str| -> Result, SdkRefusal> { + let exists = snapshot::entry(&*reader, &tree, path)?.is_some(); + Ok(exists + .then(|| visibility.hidden(path)) + .flatten() + .map(|frontier| Visibility::refusal(path, frontier))) + }; let mut mutation = Mutation::new(workspace)?; let mut result = Ok(()); for (index, change) in changes.iter().enumerate() { result = (|| match change.as_ref().map_err(Clone::clone)? { Change::Write { path, bytes, mode } => mutation.write(path, bytes, *mode), - Change::Edit { path, old, new } => mutation.edit(path, old, new), + Change::Edit { path, old, new } => match mutation.edit(path, old, new) { + Err(refusal) if refusal.code == "NoSuchPath" => match removed(path)? { + Some(hidden) => Err(hidden), + None => Err(refusal), + }, + other => other, + }, Change::Mkdir { path, mode } => mutation.mkdir(path, *mode), Change::Chmod { path, mode } => mutation.chmod(path, *mode), Change::Rm { path } => mutation.remove(path), - Change::Cp { from, to } => mutation.copy(from, to, index as u64), + Change::Cp { from, to } => match mutation.copy(from, to, index as u64) { + Err(refusal) if refusal.code == "NoSuchPath" => match removed(from)? { + Some(hidden) => Err(hidden), + None => Err(refusal), + }, + other => other, + }, Change::Mv { from, to } => mutation.rename(from, to), })() .map_err(|mut e| { diff --git a/engine/crates/layerfs-store-service/tests/sdk_path_reads.rs b/engine/crates/layerfs-store-service/tests/sdk_path_reads.rs new file mode 100644 index 0000000000..4bf1c159ed --- /dev/null +++ b/engine/crates/layerfs-store-service/tests/sdk_path_reads.rs @@ -0,0 +1,620 @@ +//! T5.6: path-aware reads (revision, blame, grep, diff, one-shot edit/cp, +//! rm/mv) and paged list/glob. A hidden path is `Removed` for a read or a +//! cp-from/edit, absent for rm/mv-from, and a visible re-creation for a write, +//! mkdir, cp-to or mv-to; a listing or glob omits it before paging and counts +//! it in one page-independent `removed_count`. +#[path = "support/installed_store.rs"] +#[allow(dead_code)] +mod installed; +#[path = "support/sdk.rs"] +#[allow(dead_code)] +mod support; + +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use std::fs; +use support::{items, paths, write, Fixture}; + +fn text(value: &str) -> SdkValue { + SdkValue::text(value) +} + +fn remove(fixture: &Fixture, path: &str) -> SdkValue { + fixture.call("project.remove_file", vec![("path", text(path))]) +} + +fn read(fixture: &Fixture, branch: &str, path: &str) -> Result { + fixture.try_call( + "fs.read", + vec![("branch", text(branch)), ("path", text(path))], + ) +} + +fn visible(fixture: &Fixture, branch: &str, path: &str) -> String { + let value = read(fixture, branch, path) + .unwrap_or_else(|refusal| panic!("{branch}{path}: {refusal:?}")); + value.get_text("text").unwrap_or_default().to_owned() +} + +fn selector(path: &str) -> SdkValue { + SdkValue::map([("path", text(path))]) +} + +fn head(fixture: &Fixture, method: &str, fields: Vec<(&str, SdkValue)>) -> String { + fixture + .call(method, fields) + .get_text("head") + .unwrap() + .to_owned() +} + +fn hidden_frontier(refusal: &SdkRefusal) -> i64 { + assert_eq!(refusal.code, "Removed", "{refusal:?}"); + let Some(SdkValue::List(hidden)) = refusal.detail.get("paths") else { + panic!("no paths detail: {:?}", refusal.detail); + }; + assert_eq!(hidden.len(), 1, "{refusal:?}"); + hidden[0].get_int("frontier_serial").unwrap() +} + +#[test] +fn revision_stops_at_a_hidden_version_and_refuses_a_hidden_anchor() { + let fixture = Fixture::new("t56-revision", |source| { + fs::write(source.join("a"), "one\ntwo\n").unwrap(); + fs::write(source.join("b"), "b\n").unwrap(); + 2 + }); + fixture.fork("work"); + fixture.apply("work", vec![write("/a", "one\nTWO\n")]); + let second = head( + &fixture, + "fs.write", + vec![ + ("branch", text("work")), + ("path", text("/a")), + ("content", text("one\nTHREE\n")), + ], + ); + remove(&fixture, "/a"); + + // The anchor itself is hidden. + let refusal = fixture + .try_call( + "fs.revision", + vec![ + ("branch", text("work")), + ("commit", text(&second)), + ("path", text("/a")), + ], + ) + .expect_err("hidden revision anchor"); + hidden_frontier(&refusal); + + // A visible re-creation above the frontier: the walk stops at the first + // hidden version instead of skipping past it. + let third = head( + &fixture, + "fs.write", + vec![ + ("branch", text("work")), + ("path", text("/a")), + ("content", text("four\n")), + ], + ); + let revision = fixture.call( + "fs.revision", + vec![("branch", text("work")), ("path", text("/a"))], + ); + let found = items(&revision); + assert_eq!(found.len(), 1, "only the visible re-creation: {revision:?}"); + assert_eq!(found[0].get_text("kind"), Some("Added")); + assert_eq!(found[0].get("before"), Some(&SdkValue::Null)); + assert_eq!( + found[0].get("ref").and_then(|r| r.get_text("id")), + Some(third.as_str()) + ); + fixture.cleanup(); +} + +#[test] +fn blame_stops_at_a_hidden_version_and_refuses_a_hidden_anchor() { + let fixture = Fixture::new("t56-blame", |source| { + fs::write(source.join("a"), "one\ntwo\n").unwrap(); + fs::write(source.join("b"), "b\n").unwrap(); + 2 + }); + fixture.fork("work"); + fixture.apply("work", vec![write("/a", "one\nTWO\n")]); + let second = head( + &fixture, + "fs.write", + vec![ + ("branch", text("work")), + ("path", text("/a")), + ("content", text("one\nTHREE\n")), + ], + ); + remove(&fixture, "/a"); + + let refusal = fixture + .try_call( + "fs.blame", + vec![ + ("branch", text("work")), + ("commit", text(&second)), + ("path", text("/a")), + ], + ) + .expect_err("hidden blame anchor"); + hidden_frontier(&refusal); + + let third = head( + &fixture, + "fs.write", + vec![ + ("branch", text("work")), + ("path", text("/a")), + ("content", text("one\nfour\n")), + ], + ); + let blame = fixture.call( + "fs.blame", + vec![("branch", text("work")), ("path", text("/a"))], + ); + let Some(SdkValue::List(lines)) = blame.get("lines") else { + panic!("no blame lines: {blame:?}"); + }; + assert!(!lines.is_empty(), "{blame:?}"); + for line in lines { + assert_eq!(line.get_text("commit"), Some(third.as_str()), "{blame:?}"); + } + fixture.cleanup(); +} + +#[test] +fn grep_omits_hidden_hits_and_refuses_a_hidden_target() { + let fixture = Fixture::new("t56-grep", |source| { + fs::write(source.join("a"), "needle one\n").unwrap(); + fs::write(source.join("b"), "needle two\n").unwrap(); + 2 + }); + fixture.fork("work"); + remove(&fixture, "/a"); + + let refusal = fixture + .try_call( + "fs.grep", + vec![ + ("branch", text("work")), + ("pattern", text("needle")), + ("path", text("/a")), + ], + ) + .expect_err("hidden grep target"); + hidden_frontier(&refusal); + + // A hidden file's hits are omitted, and a paged call does not let them + // consume the page. + let page = fixture.call( + "fs.grep", + vec![ + ("branch", text("work")), + ("pattern", text("needle")), + ("path", text("/")), + ], + ); + let found = items(&page); + assert_eq!(found.len(), 1, "{page:?}"); + assert_eq!(found[0].get_text("path"), Some("/b")); + let paged = fixture.call( + "fs.grep", + vec![ + ("branch", text("work")), + ("pattern", text("needle")), + ("path", text("/")), + ("page", SdkValue::Int(1)), + ("pageSize", SdkValue::Int(1)), + ], + ); + assert_eq!(items(&paged).len(), 1, "{paged:?}"); + assert_eq!(items(&paged)[0].get_text("path"), Some("/b")); + + // A post-frontier re-creation is searched again. + fixture.call( + "fs.write", + vec![ + ("branch", text("work")), + ("path", text("/a")), + ("content", text("needle again\n")), + ], + ); + let again = fixture.call( + "fs.grep", + vec![ + ("branch", text("work")), + ("pattern", text("needle")), + ("path", text("/")), + ], + ); + assert_eq!(items(&again).len(), 2, "{again:?}"); + fixture.cleanup(); +} + +#[test] +fn file_diff_reports_hidden_sides_as_removed_added_and_both() { + let fixture = Fixture::new("t56-diff", |source| { + fs::write(source.join("a"), "a\n").unwrap(); + fs::write(source.join("b"), "b\n").unwrap(); + fs::write(source.join("c"), "c\n").unwrap(); + 3 + }); + fixture.fork("work"); + let frontier_a = remove(&fixture, "/a") + .get_int("frontier_serial") + .unwrap(); + let frontier_c = remove(&fixture, "/c") + .get_int("frontier_serial") + .unwrap(); + + // Left visible, right hidden: Removed with a hidden right side object. + let removed = fixture.call( + "fs.diff_files", + vec![ + ("branch", text("work")), + ("left", selector("/b")), + ("right", selector("/a")), + ], + ); + assert_eq!( + removed.get_text("changeKind"), + Some("Removed"), + "{removed:?}" + ); + assert_eq!(removed.get("left").unwrap().get_text("path"), Some("/b")); + let right = removed.get("right").unwrap(); + assert!(right.get_bool("removed")); + assert_eq!(right.get_text("path"), Some("/a")); + assert_eq!(right.get_int("frontier_serial"), Some(frontier_a)); + + // Left hidden, right visible: Added. + let added = fixture.call( + "fs.diff_files", + vec![ + ("branch", text("work")), + ("left", selector("/a")), + ("right", selector("/b")), + ], + ); + assert_eq!(added.get_text("changeKind"), Some("Added"), "{added:?}"); + assert!(added.get("left").unwrap().get_bool("removed")); + + // Both hidden: Removed with both side objects. + let both = fixture.call( + "fs.diff_files", + vec![ + ("branch", text("work")), + ("left", selector("/a")), + ("right", selector("/c")), + ], + ); + assert_eq!( + both.get_text("changeKind"), + Some("Removed"), + "{both:?}" + ); + assert_eq!( + both.get("left").unwrap().get_int("frontier_serial"), + Some(frontier_a) + ); + assert_eq!( + both.get("right").unwrap().get_int("frontier_serial"), + Some(frontier_c) + ); + + // A genuinely absent side is still NoSuchPath when allowMissing is false. + let refusal = fixture + .try_call( + "fs.diff_files", + vec![ + ("branch", text("work")), + ("left", selector("/b")), + ("right", selector("/absent")), + ], + ) + .expect_err("absent diff side"); + assert_eq!(refusal.code, "NoSuchPath", "{refusal:?}"); + + // A hidden side beats allowMissing. + let hidden_missing = fixture.call( + "fs.diff_files", + vec![ + ("branch", text("work")), + ("left", selector("/b")), + ("right", selector("/a")), + ("allowMissing", SdkValue::Bool(false)), + ], + ); + assert_eq!(hidden_missing.get_text("changeKind"), Some("Removed")); + fixture.cleanup(); +} + +#[test] +fn one_shot_edit_and_cp_from_a_hidden_path_refuse_removed() { + let fixture = Fixture::new("t56-one-shot", |source| { + fs::write(source.join("a"), "a\n").unwrap(); + fs::write(source.join("b"), "b\n").unwrap(); + fs::create_dir(source.join("d")).unwrap(); + fs::write(source.join("d/f"), "f\n").unwrap(); + 4 + }); + for branch in ["rm", "mvfrom", "cpfrom", "edit", "cpto"] { + fixture.fork(branch); + } + remove(&fixture, "/a"); + + // edit and cp-from are `Removed`, not `NoSuchPath`. + let refusal = fixture + .try_call( + "fs.edit", + vec![ + ("branch", text("edit")), + ("path", text("/a")), + ("oldText", text("a")), + ("newText", text("z")), + ], + ) + .expect_err("hidden edit"); + hidden_frontier(&refusal); + let refusal = fixture + .try_call( + "fs.cp", + vec![ + ("branch", text("cpfrom")), + ("from", text("/a")), + ("to", text("/copy")), + ], + ) + .expect_err("hidden cp source"); + hidden_frontier(&refusal); + + // rm and mv-from treat it as absent (the tree has no entry there). + let refusal = fixture + .try_call( + "fs.rm", + vec![("branch", text("rm")), ("path", text("/a"))], + ) + .expect_err("hidden rm"); + assert_eq!(refusal.code, "NoSuchPath", "{refusal:?}"); + let refusal = fixture + .try_call( + "fs.mv", + vec![ + ("branch", text("mvfrom")), + ("from", text("/a")), + ("to", text("/moved")), + ], + ) + .expect_err("hidden mv source"); + assert_eq!(refusal.code, "NoSuchPath", "{refusal:?}"); + + // A never-existing path stays NoSuchPath for edit and rm. + let refusal = fixture + .try_call( + "fs.edit", + vec![ + ("branch", text("edit")), + ("path", text("/missing")), + ("oldText", text("a")), + ("newText", text("z")), + ], + ) + .expect_err("absent edit"); + assert_eq!(refusal.code, "NoSuchPath", "edit: {refusal:?}"); + let refusal = fixture + .try_call( + "fs.rm", + vec![("branch", text("rm")), ("path", text("/missing"))], + ) + .expect_err("absent rm"); + assert_eq!(refusal.code, "NoSuchPath", "rm: {refusal:?}"); + + // cp-to a hidden path is a visible re-creation. + fixture.call( + "fs.cp", + vec![ + ("branch", text("cpto")), + ("from", text("/b")), + ("to", text("/a")), + ], + ); + assert_eq!(visible(&fixture, "cpto", "/a"), "b\n"); + assert_eq!(visible(&fixture, "cpto", "/b"), "b\n"); + fixture.cleanup(); +} + +#[test] +fn paged_list_and_glob_skip_hidden_entries_before_paging() { + let fixture = Fixture::new("t56-paged", |source| { + fs::write(source.join("a"), "a\n").unwrap(); + fs::write(source.join("b"), "b\n").unwrap(); + fs::write(source.join("c"), "c\n").unwrap(); + fs::create_dir(source.join("d")).unwrap(); + fs::write(source.join("d/f"), "f\n").unwrap(); + 5 + }); + fixture.fork("work"); + remove(&fixture, "/a"); + remove(&fixture, "/d"); + + // A non-paged listing omits the hidden entries and counts both. + let listed = fixture.call( + "fs.list", + vec![("branch", text("work")), ("path", text("/"))], + ); + assert_eq!(paths(&listed), vec!["/b", "/c"], "{listed:?}"); + assert_eq!(listed.get_int("removed_count"), Some(2)); + + // Each page skips hidden entries before its page slot and reports the + // same page-independent removed_count. + let first = fixture.call( + "fs.list", + vec![ + ("branch", text("work")), + ("path", text("/")), + ("page", SdkValue::Int(1)), + ("pageSize", SdkValue::Int(1)), + ], + ); + assert_eq!(paths(&first), vec!["/b"], "{first:?}"); + assert_eq!(first.get_int("removed_count"), Some(2)); + let second = fixture.call( + "fs.list", + vec![ + ("branch", text("work")), + ("path", text("/")), + ("page", SdkValue::Int(2)), + ("pageSize", SdkValue::Int(1)), + ], + ); + assert_eq!(paths(&second), vec!["/c"], "{second:?}"); + assert_eq!(second.get_int("removed_count"), Some(2)); + + // A legacy numeric page also counts only visible matches. + let legacy = fixture.call( + "fs.list", + vec![ + ("branch", text("work")), + ("path", text("/")), + ("pageSize", SdkValue::Int(1)), + ("pagePos", SdkValue::Int(1)), + ], + ); + assert_eq!(paths(&legacy), vec!["/c"], "{legacy:?}"); + + // A glob omits hidden matches and counts them on every page. + let globbed = fixture.call( + "fs.glob", + vec![ + ("branch", text("work")), + ("pattern", text("/*")), + ("path", text("/")), + ], + ); + let matched: Vec<&str> = match globbed.get("items") { + Some(SdkValue::List(items)) => items + .iter() + .filter_map(|item| match item { + SdkValue::Text(name) => Some(name.as_str()), + _ => None, + }) + .collect(), + other => panic!("no glob items: {other:?}"), + }; + assert_eq!(matched, vec!["/b", "/c"], "{globbed:?}"); + assert_eq!(globbed.get_int("removed_count"), Some(2)); + + let glob_first = fixture.call( + "fs.glob", + vec![ + ("branch", text("work")), + ("pattern", text("/*")), + ("path", text("/")), + ("page", SdkValue::Int(1)), + ("pageSize", SdkValue::Int(1)), + ], + ); + assert_eq!(items(&glob_first).len(), 1, "{glob_first:?}"); + assert_eq!(glob_first.get_int("removed_count"), Some(2)); + let glob_second = fixture.call( + "fs.glob", + vec![ + ("branch", text("work")), + ("pattern", text("/*")), + ("path", text("/")), + ("page", SdkValue::Int(2)), + ("pageSize", SdkValue::Int(1)), + ], + ); + assert_eq!(items(&glob_second).len(), 1, "{glob_second:?}"); + assert_eq!(glob_second.get_int("removed_count"), Some(2)); + fixture.cleanup(); +} + +#[test] +fn branch_diff_treats_a_hidden_side_as_absent_on_both_routes() { + let fixture = Fixture::new("t56-branch-diff", |source| { + fs::write(source.join("a"), "a\n").unwrap(); + fs::write(source.join("b"), "b\n").unwrap(); + 2 + }); + fixture.fork("left"); + fixture.fork("right"); + remove(&fixture, "/a"); + // `right` re-creates /a above the frontier; `left` stays at the hidden base. + fixture.call( + "fs.write", + vec![ + ("branch", text("right")), + ("path", text("/a")), + ("content", text("new\n")), + ], + ); + + let kind_of = |value: &SdkValue| -> Option { + let Some(SdkValue::List(items)) = value.get("items") else { + return None; + }; + items + .iter() + .find(|item| item.get_text("path") == Some("/a")) + .and_then(|item| item.get_text("kind")) + .map(str::to_owned) + }; + + // The legacy offset route: a hidden left side is an addition, not a spurious + // modification or removal. + let legacy = fixture.call( + "branch.diff_branches", + vec![("from", text("left")), ("to", text("right"))], + ); + assert_eq!(kind_of(&legacy).as_deref(), Some("Added"), "{legacy:?}"); + + // The extended (page/continuation) route applies the same per-side rule. + let extended = fixture.call( + "branch.diff_branches", + vec![ + ("from", text("left")), + ("to", text("right")), + ("page", SdkValue::Int(1)), + ], + ); + assert_eq!( + kind_of(&extended).as_deref(), + Some("Added"), + "{extended:?}" + ); + + // The reverse direction lists the same path as Removed on both routes. + let legacy = fixture.call( + "branch.diff_branches", + vec![("from", text("right")), ("to", text("left"))], + ); + assert_eq!( + kind_of(&legacy).as_deref(), + Some("Removed"), + "{legacy:?}" + ); + let extended = fixture.call( + "branch.diff_branches", + vec![ + ("from", text("right")), + ("to", text("left")), + ("page", SdkValue::Int(1)), + ], + ); + assert_eq!( + kind_of(&extended).as_deref(), + Some("Removed"), + "{extended:?}" + ); + fixture.cleanup(); +} diff --git a/engine/crates/layerfs-store-service/tests/sdk_path_whiteouts.rs b/engine/crates/layerfs-store-service/tests/sdk_path_whiteouts.rs index 806b1b37ff..6fff9e6b2c 100644 --- a/engine/crates/layerfs-store-service/tests/sdk_path_whiteouts.rs +++ b/engine/crates/layerfs-store-service/tests/sdk_path_whiteouts.rs @@ -126,7 +126,8 @@ fn one_shot_creators_see_a_removed_path_as_absent_and_recreate_it_visibly() { } remove(&fixture, "/a"); - // Removing or moving from a hidden path is the absence the tree has. + // Removing or moving from a hidden path is the absence the tree has; + // edit and a cp source are `Removed` (T5.6, §4.3). let refusal = fixture .try_call("fs.rm", vec![("branch", text("rm")), ("path", text("/a"))]) .expect_err("hidden rm"); @@ -152,7 +153,7 @@ fn one_shot_creators_see_a_removed_path_as_absent_and_recreate_it_visibly() { ], ) .expect_err("hidden cp source"); - assert_eq!(refusal.code, "NoSuchPath", "{refusal:?}"); + assert_eq!(refusal.code, "Removed", "{refusal:?}"); let refusal = fixture .try_call( "fs.edit", @@ -164,7 +165,7 @@ fn one_shot_creators_see_a_removed_path_as_absent_and_recreate_it_visibly() { ], ) .expect_err("hidden edit"); - assert_eq!(refusal.code, "NoSuchPath", "{refusal:?}"); + assert_eq!(refusal.code, "Removed", "{refusal:?}"); // Writing, moving or copying to the path, or writing it, is a visible // re-creation.