diff --git a/crates/tracedecay-cli/src/commands/stale_store_reset.rs b/crates/tracedecay-cli/src/commands/stale_store_reset.rs index 2d9287d1c31..2b45f55d5ae 100644 --- a/crates/tracedecay-cli/src/commands/stale_store_reset.rs +++ b/crates/tracedecay-cli/src/commands/stale_store_reset.rs @@ -12,7 +12,7 @@ use tracedecay_runtime_core::storage::{ use super::storage::{ ProfileOfflineAuthority, RestoreWording, join_outcome_and_restore, remove_fixed_profile_path, - take_profile_offline, + remove_store_family, take_profile_offline, }; const OPERATION: &str = "wipe --stale"; @@ -189,7 +189,7 @@ fn print_project_registrations(profile_root: &Path) -> Result<()> { /// What a reset deletes inside a store's directory. enum StoreMembers { - /// A database and every entry named after it. + /// The SQLite family and its registered store artifacts. DatabaseFamily(&'static str), /// These exact directories. Directories(&'static [&'static str]), @@ -229,34 +229,6 @@ fn store_location( }) } -/// Removes `database` and every entry named after it in `directory`: its -/// SQLite sidecars and spools (`.db…`), its host-admission directory -/// (`..db…`), and its session relation graph (`.grafeo…`). -fn remove_store_family(directory: &Path, database: &str) -> Result { - let stem = database.strip_suffix(".db").unwrap_or(database); - let owned_prefix = format!("{stem}."); - let hidden_prefix = format!(".{database}."); - let entries = std::fs::read_dir(directory).map_err(|error| TraceDecayError::Config { - message: format!("failed to list '{}': {error}", directory.display()), - })?; - let mut members = Vec::new(); - for entry in entries { - let entry = entry.map_err(|error| TraceDecayError::Config { - message: format!("failed to list '{}': {error}", directory.display()), - })?; - let name = entry.file_name().to_string_lossy().into_owned(); - if name.starts_with(&owned_prefix) || name.starts_with(&hidden_prefix) { - members.push(name); - } - } - let mut removed = 0; - for name in &members { - removed += usize::from(remove_fixed_profile_path(directory, name)?); - verify_store_path_absent(&directory.join(name))?; - } - Ok(removed) -} - #[cfg(test)] mod tests { use super::*; @@ -270,6 +242,10 @@ mod tests { "sessions.db-wal", "sessions.db-shm", "sessions.grafeo", + "sessions.verified", + "user-sessions.verified", + "tracedecay.verified", + "sessions.grafeo.corruption-lock", "tracedecay.db", "tracedecay.db-wal", "tracedecay.grafeo", @@ -279,19 +255,25 @@ mod tests { "global.db-wal", "global.db-shm", "profile-identity.json", + "sessions.unrelated", + ".sessions.db.notes", ] { std::fs::write(root.join(file), file).expect("write store file"); } for dir in [ ".sessions.db.host-admission", "sessions.db.delivery-settlement-spool-v1", + "sessions.grafeo.wal", + "sessions.grafeo.spill", + "sessions.spill", + "sessions.sealed", "code-index-v1", ] { std::fs::create_dir(root.join(dir)).expect("create store directory"); std::fs::write(root.join(dir).join("entry"), dir).expect("write store entry"); } - assert_eq!(remove_store_family(root, "sessions.db").unwrap(), 6); + assert_eq!(remove_store_family(root, "sessions.db").unwrap(), 10); assert_eq!(remove_store_family(root, GLOBAL_DB_FILENAME).unwrap(), 3); let mut kept: Vec = std::fs::read_dir(root) @@ -302,17 +284,79 @@ mod tests { assert_eq!( kept, [ + ".sessions.db.notes", "code-index-v1", "profile-identity.json", + "sessions.grafeo.corruption-lock", + "sessions.spill", + "sessions.unrelated", "store_manifest.json", "tracedecay.db", "tracedecay.db-wal", "tracedecay.grafeo", + "tracedecay.verified", "user-sessions.db", + "user-sessions.verified", ] ); } + #[test] + fn profile_sessions_reset_removes_only_its_graph_verification_marker() { + let directory = tempfile::TempDir::new().unwrap(); + let root = directory.path(); + for name in [ + "sessions.db", + "sessions.verified", + "user-sessions.db", + "user-sessions.verified", + ] { + std::fs::write(root.join(name), name).unwrap(); + } + + assert_eq!( + remove_store_family(root, USER_SESSIONS_DB_FILENAME).unwrap(), + 2 + ); + assert!(!root.join("user-sessions.db").exists()); + assert!(!root.join("user-sessions.verified").exists()); + assert_eq!( + std::fs::read(root.join("sessions.db")).unwrap(), + b"sessions.db" + ); + assert_eq!( + std::fs::read(root.join("sessions.verified")).unwrap(), + b"sessions.verified" + ); + } + + #[test] + fn unexpected_graph_verification_marker_kind_preserves_the_store() { + let directory = tempfile::TempDir::new().unwrap(); + let root = directory.path(); + std::fs::write(root.join("sessions.db"), b"store").unwrap(); + let marker = root.join("sessions.verified"); + std::fs::create_dir(&marker).unwrap(); + std::fs::write(marker.join("foreign"), b"preserve").unwrap(); + + assert!(remove_store_family(root, SESSIONS_DB_FILENAME).is_err()); + assert_eq!(std::fs::read(root.join("sessions.db")).unwrap(), b"store"); + assert_eq!(std::fs::read(marker.join("foreign")).unwrap(), b"preserve"); + } + + #[test] + fn unexpected_sqlite_member_kind_preserves_the_whole_family() { + let directory = tempfile::TempDir::new().unwrap(); + let root = directory.path(); + std::fs::write(root.join("sessions.db"), b"main").unwrap(); + let foreign = root.join("sessions.db-wal"); + std::fs::create_dir(&foreign).unwrap(); + std::fs::write(foreign.join("entry"), b"keep").unwrap(); + assert!(remove_store_family(root, "sessions.db").is_err()); + assert_eq!(std::fs::read(root.join("sessions.db")).unwrap(), b"main"); + assert_eq!(std::fs::read(foreign.join("entry")).unwrap(), b"keep"); + } + #[test] fn a_store_that_is_not_resettable_on_its_own_refuses_the_whole_reset() { let store = |label: &str, remedy: &str| StoreResetRequiredV1 { diff --git a/crates/tracedecay-cli/src/commands/storage.rs b/crates/tracedecay-cli/src/commands/storage.rs index 42262122b4c..30729f891fc 100644 --- a/crates/tracedecay-cli/src/commands/storage.rs +++ b/crates/tracedecay-cli/src/commands/storage.rs @@ -7,7 +7,11 @@ use tracedecay_maintenance::retention::footprint::{ MAX_FOOTPRINT_ENTRIES, WalkTerminationV1, WalkedEntryV1, walk_regular_files_with_budget, }; use tracedecay_runtime_core::config::{GLOBAL_DB_FILENAME, ProfileRoot}; -use tracedecay_runtime_core::storage::{USER_MEMORY_DB_FILENAME, USER_SESSIONS_DB_FILENAME}; +use tracedecay_runtime_core::shard_runtime::registry::lock_offline_sqlite_family_for_reset; +use tracedecay_runtime_core::storage::{ + SESSIONS_DB_FILENAME, USER_MEMORY_DB_FILENAME, USER_SESSIONS_DB_FILENAME, + graph_store_data_paths, +}; use tracedecay_global_db::profile_registry_maintenance::{ ProfileRegistryMaintenanceRuntime, remove_store_directory, verify_store_path_absent, @@ -33,27 +37,109 @@ const PROFILE_SQLITE_DATABASES: [&str; 3] = [ USER_SESSIONS_DB_FILENAME, USER_MEMORY_DB_FILENAME, ]; -const PROFILE_DATABASE_PATHS: [&str; 14] = [ - "projects", - "remote", - "agent_managed", - "user-sessions.grafeo", - "user-sessions.grafeo.wal", - "user-memory.grafeo", - "user-memory.grafeo.wal", - ".user-sessions.db.host-admission", - tracedecay_hooks::PROFILE_HOOK_ADMISSIONS_DIR, - tracedecay_runtime_core::storage::RESPONSE_HANDLES_DIRECTORY, - "lcm-payloads", - "hook_analytics.jsonl", - "hook_analytics.jsonl.lock", - "maintenance", +#[derive(Clone, Copy)] +enum StoreMemberKind { + File, + Directory, +} + +use StoreMemberKind::{Directory, File}; + +const PROFILE_DATABASE_PATHS: [(&str, StoreMemberKind); 9] = [ + ("projects", Directory), + ("remote", Directory), + ("agent_managed", Directory), + (tracedecay_hooks::PROFILE_HOOK_ADMISSIONS_DIR, Directory), + ( + tracedecay_runtime_core::storage::RESPONSE_HANDLES_DIRECTORY, + Directory, + ), + ("lcm-payloads", Directory), + ("hook_analytics.jsonl", File), + ("hook_analytics.jsonl.lock", File), + ("maintenance", Directory), ]; -fn sqlite_family_member(database: &Path, suffix: &str) -> PathBuf { - let mut member = database.as_os_str().to_os_string(); - member.push(suffix); - member.into() +fn lock_offline_store_family( + directory: &Path, + database: &str, +) -> tracedecay_domain::errors::Result> { + let path = directory.join(database); + lock_offline_sqlite_family_for_reset(&path) + .map_err(|error| wipe_io("lock offline SQLite family", &path, &error)) +} + +/// Exact data files and directories belonging to the relational shard. +/// Sidecars and projections go before the SQLite main so an interrupted +/// reset can still rediscover its stale authority. +fn sqlite_store_data_members(database: &str) -> Vec<(PathBuf, StoreMemberKind)> { + let sessions = database == SESSIONS_DB_FILENAME || database == USER_SESSIONS_DB_FILENAME; + let mut members = Vec::new(); + if sessions || database == USER_MEMORY_DB_FILENAME { + let (files, directories) = + graph_store_data_paths(&Path::new(database).with_extension("grafeo")); + members.extend(directories.into_iter().map(|path| (path, Directory))); + members.extend(files.into_iter().map(|path| (path, File))); + } + if sessions { + members.push((format!(".{database}.host-admission").into(), Directory)); + members.push(( + format!("{database}.delivery-settlement-spool-v1").into(), + Directory, + )); + } + members.extend( + ["-wal", "-shm", "-journal", ""] + .map(|suffix| (PathBuf::from(format!("{database}{suffix}")), File)), + ); + members +} + +/// Under the exclusive offline lease, refuse every ambiguous member before +/// the first removal, including a directory in the place of a regular file. +fn remove_checked_profile_members( + directory: &Path, + members: &[(PathBuf, StoreMemberKind)], +) -> tracedecay_domain::errors::Result { + for (name, kind) in members { + let path = directory.join(name); + tracedecay_runtime_core::storage::reject_symlink_components(&path, "offline store data") + .map_err(|error| wipe_io("validate store data", &path, &error))?; + match std::fs::symlink_metadata(&path) { + Ok(meta) + if match kind { + File => meta.is_file(), + Directory => meta.is_dir(), + } => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Ok(_) => { + return Err(tracedecay_domain::errors::TraceDecayError::Config { + message: format!( + "unexpected file type for offline store data '{}'", + path.display() + ), + }); + } + Err(error) => return Err(wipe_io("inspect offline store data", &path, &error)), + } + } + let mut removed = 0; + for (name, _) in members { + removed += usize::from(remove_fixed_profile_path(directory, name)?); + verify_store_path_absent(&directory.join(name))?; + } + Ok(removed) +} + +/// The offline lifecycle authority admits deletion of the stale store +/// itself. Similar names, private initialization data and stable coordination +/// locks are not members of its data family. +pub(super) fn remove_store_family( + directory: &Path, + database: &str, +) -> tracedecay_domain::errors::Result { + let _initialization = lock_offline_store_family(directory, database)?; + remove_checked_profile_members(directory, &sqlite_store_data_members(database)) } fn wipe_io( @@ -121,7 +207,7 @@ fn validate_complete_wipe_profile_root( pub(super) fn remove_fixed_profile_path( profile_root: &Path, - name: &str, + name: impl AsRef, ) -> tracedecay_domain::errors::Result { use tracedecay_private_fs::framed_log::{DirectorySyncPolicy, sync_directory}; @@ -318,32 +404,15 @@ pub(crate) fn take_profile_offline( fn wipe_complete_profile_database_state( profile_root: &Path, ) -> tracedecay_domain::errors::Result { - let mut removed = 0usize; - for name in PROFILE_DATABASE_PATHS { - removed += usize::from(remove_fixed_profile_path(profile_root, name)?); - } + let mut leases = Vec::new(); + let mut members = PROFILE_DATABASE_PATHS + .map(|(name, is_directory)| (PathBuf::from(name), is_directory)) + .to_vec(); for database in PROFILE_SQLITE_DATABASES { - let database = profile_root.join(database); - for suffix in ["-wal", "-shm", "-journal", ""] { - let member = sqlite_family_member(&database, suffix); - removed += usize::from( - tracedecay_runtime_core::storage::PrivateStoreIo::remove_file_durable(&member) - .map_err(|error| { - wipe_io("remove profile SQLite family member", &member, &error) - })?, - ); - } + leases.push(lock_offline_store_family(profile_root, database)?); + members.extend(sqlite_store_data_members(database)); } - for name in PROFILE_DATABASE_PATHS { - verify_store_path_absent(&profile_root.join(name))?; - } - for database in PROFILE_SQLITE_DATABASES { - let database = profile_root.join(database); - for suffix in ["-wal", "-shm", "-journal", ""] { - verify_store_path_absent(&sqlite_family_member(&database, suffix))?; - } - } - Ok(removed) + remove_checked_profile_members(profile_root, &members) } /// Handles the `wipe` and `wipe --all` commands. diff --git a/crates/tracedecay-cli/tests/core_cli_suite/cli_non_interactive_test.rs b/crates/tracedecay-cli/tests/core_cli_suite/cli_non_interactive_test.rs index 3391fe3c5d6..4e8b9247306 100644 --- a/crates/tracedecay-cli/tests/core_cli_suite/cli_non_interactive_test.rs +++ b/crates/tracedecay-cli/tests/core_cli_suite/cli_non_interactive_test.rs @@ -1052,7 +1052,10 @@ fn write_empty_sqlite_fixture(path: &Path) { if let Some(parent) = path.parent() { std::fs::create_dir_all(parent).unwrap(); } - drop(rusqlite::Connection::open(path).expect("empty SQLite fixture")); + rusqlite::Connection::open(path) + .expect("empty SQLite fixture") + .execute_batch("PRAGMA user_version = 0") + .expect("materialize an initialized empty-schema SQLite fixture"); } async fn register_profile_sharded_store( @@ -2644,7 +2647,19 @@ fn wipe_all_is_schema_independent_and_removes_every_profile_database_root() { let wal = PathBuf::from(wal); std::fs::create_dir(&wal).unwrap(); std::fs::write(wal.join("segment"), b"graph wal").unwrap(); + std::fs::write(path.with_extension("verified"), b"marker").unwrap(); + for directory in [ + path.with_extension("sealed"), + path.with_added_extension("spill"), + ] { + std::fs::create_dir(&directory).unwrap(); + std::fs::write(directory.join("member"), b"owned projection data").unwrap(); + } + std::fs::write(path.with_added_extension("corruption-lock"), b"keep lock").unwrap(); } + let unrelated = profile.join("user-sessions.spill"); + std::fs::create_dir(&unrelated).unwrap(); + std::fs::write(unrelated.join("member"), b"foreign directory").unwrap(); let host_admission = profile.join(".user-sessions.db.host-admission"); std::fs::create_dir(&host_admission).unwrap(); std::fs::write(host_admission.join("pending"), b"admission spool").unwrap(); @@ -2667,6 +2682,46 @@ fn wipe_all_is_schema_independent_and_removes_every_profile_database_root() { std::fs::write(&path, b"profile state").unwrap(); } + let lease_path = profile.join(".user-memory.db.tracedecay-init.durable-directory.lock"); + let initializer = tracedecay_private_fs::create_private_file(&lease_path).unwrap(); + initializer.try_lock().unwrap(); + let mut busy = tracedecay_command_without_daemon(home.path(), project.path()); + busy.args(["wipe", "--all", "--yes"]); + let refused = run_with_timeout(busy, cli_timeout()); + assert!(!refused.status.success(), "{refused:?}"); + assert!( + String::from_utf8_lossy(&refused.stderr).contains("lock offline SQLite family"), + "{refused:?}" + ); + for database in &database_paths { + assert!(database.is_file()); + } + drop(initializer); + + // A wrong-kind marker in the last top-level store must be discovered + // before even the earlier profile, remote or project data is touched. + let ambiguous = profile.join("user-memory.verified"); + std::fs::remove_file(&ambiguous).unwrap(); + std::fs::create_dir(&ambiguous).unwrap(); + let mut refusal = tracedecay_command_without_daemon(home.path(), project.path()); + refusal.args(["wipe", "--all", "--yes"]); + let refused = run_with_timeout(refusal, cli_timeout()); + assert!(!refused.status.success(), "{refused:?}"); + let stderr = String::from_utf8_lossy(&refused.stderr); + assert!( + stderr.contains("unexpected file type") && stderr.contains("user-memory.verified"), + "{stderr}" + ); + for database in &database_paths { + assert_eq!( + std::fs::read(database).unwrap(), + b"not a compatible SQLite schema" + ); + } + assert!(ambiguous.is_dir()); + std::fs::remove_dir(&ambiguous).unwrap(); + std::fs::write(ambiguous, b"marker").unwrap(); + let mut command = tracedecay_command_without_daemon(home.path(), project.path()); command.args(["wipe", "--all", "--yes"]); let output = run_with_timeout(command, cli_timeout()); @@ -2709,7 +2764,23 @@ fn wipe_all_is_schema_independent_and_removes_every_profile_database_root() { wal.push(".wal"); assert_namespace_absent(&path, "wipe --all left a Grafeo store"); assert_namespace_absent(&PathBuf::from(wal), "wipe --all left a Grafeo WAL"); + for member in [ + path.with_extension("verified"), + path.with_extension("sealed"), + path.with_added_extension("spill"), + ] { + assert_namespace_absent(&member, "wipe --all left graph data"); + } + assert_eq!( + std::fs::read(path.with_added_extension("corruption-lock")).unwrap(), + b"keep lock" + ); } + assert!(lease_path.is_file()); + assert_eq!( + std::fs::read(unrelated.join("member")).unwrap(), + b"foreign directory" + ); assert_namespace_absent( &host_admission, "wipe --all left the profile host-admission database companion", diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs index af0c127c645..41756174d9d 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs @@ -103,15 +103,31 @@ async fn temporal_schema_accepts_only_fresh_or_exact_final_stores() { drop(conn); drop(raw_db); - let db = open_global_db(&db_path) + // Removing an application table does not make a previously opened SQLite + // namespace freshly owned. Existing sidecars remain unclaimed. + let error = match open_global_db(&db_path).await { + Ok(_) => panic!("unpublished existing SQLite must not be reinitialized"), + Err(error) => error, + }; + assert!( + error.to_string().contains("sidecar ownership is ambiguous"), + "unpublished SQLite sidecars must remain unclaimed: {error}" + ); + assert!( + !table_exists(&db_path, "tracedecay_schema_version").await, + "refusal must not publish a schema marker into an unowned database" + ); + + let fresh_path = tmp.path().join(".tracedecay").join("fresh.db"); + let db = open_global_db(&fresh_path) .await - .expect("a fresh store should receive the final temporal schema"); + .expect("a new prospective store should receive the final temporal schema"); drop(db); - let initial_catalog = temporal_schema_object_catalog(&db_path).await; - let initial_version = temporal_schema_version(&db_path).await; + let initial_catalog = temporal_schema_object_catalog(&fresh_path).await; + let initial_version = temporal_schema_version(&fresh_path).await; let restart_path = tmp.path().join(".tracedecay").join("restart.db"); - copy_database_for_temporal_restart(&db_path, &restart_path).await; + copy_database_for_temporal_restart(&fresh_path, &restart_path).await; let reopened = open_global_db(&restart_path) .await .expect("idempotent temporal reopen should succeed"); @@ -460,7 +476,7 @@ async fn temporal_schema_is_not_installed_into_a_nonempty_store() { } #[asupersync::test] -async fn writer_ledger_alone_does_not_refuse_a_fresh_store() { +async fn unpublished_writer_ledger_is_preserved_and_fresh_publication_still_works() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); seed_writer_ledger(&db_path).await; @@ -469,13 +485,32 @@ async fn writer_ledger_alone_does_not_refuse_a_fresh_store() { "the seeded store must hold writer ledger tables" ); - let db = open_global_db(&db_path) + let error = match open_global_db(&db_path).await { + Ok(_) => panic!("an unpublished writer-ledger database is not a fresh store"), + Err(error) => error, + }; + assert!( + error.to_string().contains("sidecar ownership is ambiguous"), + "unpublished writer-ledger sidecars must not be claimed: {error}" + ); + assert!( + table_exists(&db_path, "td_runtime_writer_idempotency_v2").await, + "refusal must preserve the existing writer ledger" + ); + assert!( + !table_exists(&db_path, "tracedecay_schema_version").await, + "refusal must not install a schema into an unowned store" + ); + + let fresh_path = tmp.path().join("fresh").join("sessions.db"); + let db = open_global_db(&fresh_path) .await - .expect("a store that holds only the writer ledger must receive the final temporal schema"); + .expect("new prospective publication must install the final schema and writer ledger"); drop(db); assert!( - table_exists(&db_path, "tracedecay_schema_version").await, - "writer ledger metadata must not block the final schema marker" + table_exists(&fresh_path, "tracedecay_schema_version").await + && table_exists(&fresh_path, "td_runtime_writer_idempotency_v2").await, + "a freshly owned family must publish the full schema and writer ledger" ); } diff --git a/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs b/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs index 92cf28bd455..5711f53f64e 100644 --- a/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs +++ b/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs @@ -1,4 +1,4 @@ -//! Daemon shutdown cancelling a registered store attach mid schema install. +//! Cancellation of registered daemon admission after prospective schema publication. use std::future::Future; use std::path::PathBuf; @@ -16,19 +16,16 @@ use tracedecay_runtime_core::db::{ use super::harness::TEST_RUNTIME_NONCE; use crate::RegisteredGlobalDbOwnerV1; -/// The operation a cancel reports once it rolled back an open schema -/// transaction, rather than stopping before the transaction began. -const INSTALL_ROLLBACK: &str = "roll back attached registered global schema"; - -/// An existing profile-sessions store file with no schema objects, the shape -/// daemon attach installs the full registered schema into. -struct EmptyStore { +/// A newly owned profile-sessions store. Prospective publication installs +/// the schema before daemon admission; an unmarked existing SQLite file +/// cannot be used as a freshly owned initialization fixture. +struct RegisteredStore { path: PathBuf, _scope: DaemonDatabaseScope, _directory: TempDir, } -impl EmptyStore { +impl RegisteredStore { fn create() -> Self { crate::register_registered_schema_installer(); let directory = tempfile::tempdir().unwrap(); @@ -42,7 +39,6 @@ impl EmptyStore { .unwrap(); let path = tracedecay_sessions::runtime::user_sessions_db_path(&profile_root); std::fs::create_dir_all(path.parent().unwrap()).unwrap(); - std::fs::File::create(&path).unwrap(); Self { path, _scope: scope, @@ -54,10 +50,15 @@ impl EmptyStore { let authority = DatabaseAuthority::for_owned_runtime(&self.path, "schema install cancellation test") .unwrap(); + let mode = if self.path.try_exists().unwrap() { + TestDatabaseRuntimeMode::Existing + } else { + TestDatabaseRuntimeMode::Initialize + }; Database::publish_registered_daemon_test_runtime_with_retirement_control( &self.path, &authority, - TestDatabaseRuntimeMode::Existing, + mode, TestDatabaseRuntimeScope::ProfileSessions, ) .await @@ -110,9 +111,45 @@ impl EmptyStore { } } +#[asupersync::test] +async fn unpublished_existing_wal_is_refused_without_deleting_foreign_bytes() { + let store = RegisteredStore::create(); + let raw = rusqlite::Connection::open(&store.path).unwrap(); + raw.execute_batch("CREATE TABLE td_runtime_unpublished_probe(value INTEGER);") + .unwrap(); + drop(raw); + let original_main = std::fs::read(&store.path).unwrap(); + let wal = store.path.with_file_name(format!( + "{}-wal", + store.path.file_name().unwrap().to_string_lossy() + )); + let foreign_wal = b"unclaimed foreign WAL bytes"; + std::fs::write(&wal, foreign_wal).unwrap(); + + let authority = + DatabaseAuthority::for_owned_runtime(&store.path, "unpublished sidecar admission").unwrap(); + let attempted = Database::publish_registered_daemon_test_runtime_with_retirement_control( + &store.path, + &authority, + TestDatabaseRuntimeMode::Existing, + TestDatabaseRuntimeScope::ProfileSessions, + ) + .await; + let error = match attempted { + Ok(_) => panic!("an unmarked SQLite namespace must not claim a foreign WAL"), + Err(error) => error, + }; + assert!( + error.to_string().contains("sidecar ownership is ambiguous"), + "unexpected refusal: {error}" + ); + assert_eq!(std::fs::read(&store.path).unwrap(), original_main); + assert_eq!(std::fs::read(&wal).unwrap(), foreign_wal); +} + #[asupersync::test(flavor = "multi_thread")] -async fn cancelled_daemon_attach_rolls_back_schema_and_reopens() { - let reference = EmptyStore::create(); +async fn cancelled_daemon_attach_preserves_published_schema_and_reopens() { + let reference = RegisteredStore::create(); let (outcome, total_polls) = reference.attach_cancelled_at(usize::MAX).await; outcome.unwrap(); let installed = reference.schema_objects().await; @@ -123,13 +160,16 @@ async fn cancelled_daemon_attach_rolls_back_schema_and_reopens() { let stride = (total_polls / 20).max(1); let mut cancelled = Vec::new(); - let mut rolled_back_installs = Vec::new(); let mut completed = Vec::new(); - for cancel_at in (1..total_polls).step_by(stride).chain([total_polls + 10]) { - let store = EmptyStore::create(); - // Publishing the runtime installs its writer ledger before admission. - // Cancellation must preserve exactly that pre-admission schema. + for cancel_at in std::iter::once(1) + .chain((2..total_polls).step_by(stride)) + .chain([total_polls + 10]) + { + let store = RegisteredStore::create(); + // Cancellation cannot modify the already-published schema or + // prevent the next admission from reopening the same family. let before_admission = store.schema_objects().await; + assert_eq!(before_admission, installed); let (outcome, _) = store.attach_cancelled_at(cancel_at).await; let objects = store.schema_objects().await; match outcome { @@ -140,20 +180,14 @@ async fn cancelled_daemon_attach_rolls_back_schema_and_reopens() { ); assert_eq!( objects, before_admission, - "cancel at poll {cancel_at}/{total_polls} must roll the install back" + "cancel at poll {cancel_at}/{total_polls} must preserve the published schema" ); - if error - .project_route_context() - .is_some_and(|(_, _, detail)| detail.starts_with(INSTALL_ROLLBACK)) - { - rolled_back_installs.push(cancel_at); - } cancelled.push(cancel_at); } Ok(()) => { assert_eq!( objects, installed, - "an attach that commits before observing cancel at poll {cancel_at}/{total_polls} installs everything" + "an admission completed before cancel at poll {cancel_at}/{total_polls} preserves the published schema" ); completed.push(cancel_at); } @@ -168,10 +202,5 @@ async fn cancelled_daemon_attach_rolls_back_schema_and_reopens() { cancelled.contains(&1), "a cancel before admission starts must stop it; cancelled at {cancelled:?}" ); - assert!( - !rolled_back_installs.is_empty(), - "a cancel inside the schema transaction must roll the install back; \ - cancelled at {cancelled:?}, completed at {completed:?} of {total_polls} polls" - ); assert!(completed.contains(&(total_polls + 10))); } diff --git a/crates/tracedecay-graph-db/src/corrupt_store.rs b/crates/tracedecay-graph-db/src/corrupt_store.rs index 594535180fd..b7f6ecd513e 100644 --- a/crates/tracedecay-graph-db/src/corrupt_store.rs +++ b/crates/tracedecay-graph-db/src/corrupt_store.rs @@ -220,8 +220,8 @@ fn delete_container_family(container: &Path) -> Result<(), GraphDbError> { for sidecar in [ wal_sidecar_path(container), - container.with_extension("verified"), - container.with_extension("spill"), + crate::verified_marker::marker_path(container), + crate::location::spill_directory(container), ] { remove_family_member(&sidecar)?; } @@ -257,10 +257,8 @@ fn remove_family_member(member: &Path) -> Result<(), GraphDbError> { } /// `graph.grafeo` -> `graph.grafeo.wal`, matching Grafeo's sidecar layout. -fn wal_sidecar_path(container: &Path) -> PathBuf { - let mut sidecar = container.as_os_str().to_owned(); - sidecar.push(".wal"); - PathBuf::from(sidecar) +pub(crate) fn wal_sidecar_path(container: &Path) -> PathBuf { + container.with_added_extension("wal") } fn container_file_name(container: &Path) -> Result<&str, GraphDbError> { @@ -297,6 +295,7 @@ mod tests { ) .unwrap(); std::fs::write(container.with_extension("verified"), b"marker").unwrap(); + std::fs::create_dir(root.join("graph.grafeo.spill")).unwrap(); container } diff --git a/crates/tracedecay-graph-db/src/lib.rs b/crates/tracedecay-graph-db/src/lib.rs index c7aab2ccf1f..04e3efcec3c 100644 --- a/crates/tracedecay-graph-db/src/lib.rs +++ b/crates/tracedecay-graph-db/src/lib.rs @@ -51,6 +51,7 @@ pub use limits::{ MAX_VERIFIED_GENERATION_BATCH_LIVE_BYTES, MAX_VERIFIED_GENERATION_BATCH_MUTATIONS, MAX_VERIFIED_GENERATION_ENTITIES, MAX_VERIFIED_GENERATION_RELATIONS, }; +pub use location::graph_store_data_paths; #[cfg(any(feature = "test-helpers", feature = "eval-helpers"))] pub use location::{GraphDbLocation, GraphDbOpenOptions, GraphDurability, GraphFormatVersion}; #[cfg(not(any(feature = "test-helpers", feature = "eval-helpers")))] diff --git a/crates/tracedecay-graph-db/src/location.rs b/crates/tracedecay-graph-db/src/location.rs index 50e76a7ffe3..bbf8b6eb30e 100644 --- a/crates/tracedecay-graph-db/src/location.rs +++ b/crates/tracedecay-graph-db/src/location.rs @@ -7,6 +7,28 @@ use grafeo_engine::config::{DurabilityMode, StorageFormat}; use crate::{GraphCancellation, GraphDbError}; +/// Regular files and directories owned by one graph container. Offline store +/// reset discards its sealed projections along with the relational authority. +/// Persistent coordination locks are outside the data family. +pub fn graph_store_data_paths(container: &Path) -> ([PathBuf; 2], [PathBuf; 3]) { + ( + [ + crate::verified_marker::marker_path(container), + container.to_owned(), + ], + [ + crate::corrupt_store::wal_sidecar_path(container), + spill_directory(container), + crate::sealed_store::sealed_store_root(container), + ], + ) +} + +pub(crate) fn spill_directory(container: &Path) -> PathBuf { + // Grafeo derives the default spill directory from the complete filename. + container.with_added_extension("spill") +} + #[derive(Clone, Debug, PartialEq, Eq)] pub enum GraphDbLocation { #[cfg(any(test, feature = "test-helpers", feature = "eval-helpers"))] diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index e6441489824..d1d397a8507 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -496,7 +496,7 @@ fn sealed_store_disabled() -> bool { } /// `/var/db/graph.grafeo` -> `/var/db/graph.sealed`. -fn sealed_store_root(database_path: &Path) -> PathBuf { +pub(crate) fn sealed_store_root(database_path: &Path) -> PathBuf { database_path.with_extension("sealed") } diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index c0c846fb8f4..f1da81003f5 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -12,7 +12,89 @@ use std::io; use cap_fs_ext::DirExt; #[cfg(not(windows))] use cap_fs_ext::OpenOptionsMaybeDirExt; -use cap_std::fs::{Dir, File, OpenOptions}; +use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt}; +pub use cap_std::fs::Dir; +use cap_std::fs::{File, OpenOptions}; + +/// Pins the parent of a cross-directory publication. On Windows the no-delete +/// sharing mode also keeps its resolved Win32 pathname usable for rename. +pub fn pin_parent(path: &std::path::Path) -> io::Result { + #[cfg(windows)] + { + use std::os::windows::fs::OpenOptionsExt; + use windows_sys::Win32::Storage::FileSystem::{ + FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, FILE_SHARE_READ, + FILE_SHARE_WRITE, + }; + let file = std::fs::OpenOptions::new() + .read(true) + .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE) + .custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT) + .open(path)?; + if !file.metadata()?.is_dir() { + return Err(io::Error::new( + io::ErrorKind::NotADirectory, + "publication parent is not a directory", + )); + } + Ok(Dir::from_std_file(file)) + } + #[cfg(not(windows))] + { + Dir::open_ambient_dir(path, cap_std::ambient_authority()) + } +} + +pub fn open_regular_nofollow(directory: &Dir, name: &OsStr) -> io::Result { + open_regular(directory, name, false) +} + +pub fn open_writable_regular_nofollow(directory: &Dir, name: &OsStr) -> io::Result { + open_regular(directory, name, true) +} + +fn open_regular(directory: &Dir, name: &OsStr, writable: bool) -> io::Result { + let mut options = OpenOptions::new(); + options + .read(true) + .write(writable) + .follow(FollowSymlinks::No); + #[cfg(windows)] + { + use cap_std::fs::OpenOptionsExt; + use windows_sys::Win32::Storage::FileSystem::{ + FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, + }; + options.share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE); + } + #[cfg(unix)] + { + use cap_std::fs::OpenOptionsExt; + options.custom_flags(libc::O_NONBLOCK); + } + let file = directory.open_with(name, &options)?.into_std(); + if !file.metadata()?.is_file() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "expected a regular file", + )); + } + Ok(file) +} + +pub fn create_private_file(directory: &Dir, name: &OsStr) -> io::Result { + #[cfg(windows)] + { + crate::create_private_file(&dir_entry_path(directory, name)?) + } + #[cfg(not(windows))] + { + use cap_std::fs::OpenOptionsExt; + let mut options = OpenOptions::new(); + options.read(true).write(true).create_new(true).mode(0o600); + Ok(directory.open_with(name, &options)?.into_std()) + } +} /// How many times a create-or-open that reports the entry missing is asked /// again before the answer is believed. xnu bounds its own retry of the same @@ -62,6 +144,7 @@ fn look_again_on_not_found(mut attempt: impl FnMut() -> io::Result) -> io: /// Atomically renames one directory entry between already-open parent /// capabilities without allowing an occupied destination to be replaced. +/// Both names must be single, unnormalized directory entries, not paths. /// Platforms without a suitable primitive fail closed: retaining bytes is /// always preferable to risking a replacement. pub fn rename_noreplace( @@ -70,6 +153,8 @@ pub fn rename_noreplace( to_parent: &Dir, to: &OsStr, ) -> io::Result<()> { + require_entry_name(from)?; + require_entry_name(to)?; #[cfg(all(target_os = "linux", target_env = "gnu"))] { use std::os::fd::AsRawFd; @@ -121,17 +206,8 @@ fn dir_entry_path(parent: &Dir, name: &OsStr) -> io::Result /// Windows uses the shared namespace-durable rename authority, including its /// NTFS fallback for replacement while delete-sharing readers remain open. pub fn replace_file(parent: &Dir, source: &OsStr, destination: &OsStr) -> io::Result<()> { - for name in [source, destination] { - let mut components = std::path::Path::new(name).components(); - if !matches!(components.next(), Some(std::path::Component::Normal(_))) - || components.next().is_some() - { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "file replacement requires sibling entry names", - )); - } - } + require_entry_name(source)?; + require_entry_name(destination)?; #[cfg(windows)] { let source = dir_entry_path(parent, source)?; @@ -146,6 +222,25 @@ pub fn replace_file(parent: &Dir, source: &OsStr, destination: &OsStr) -> io::Re parent.rename(source, parent, destination) } +/// Native *at renames still resolve absolute paths, parent traversal and +/// intermediate symlinks. A held parent only confines a single entry name. +/// Compare the original spelling as well: Path components normalize trailing +/// separators and interior dots, but the native syscall receives those bytes. +fn require_entry_name(name: &OsStr) -> io::Result<()> { + let mut components = std::path::Path::new(name).components(); + if matches!(components.next(), Some(std::path::Component::Normal(entry)) if entry == name) + && components.next().is_none() + && !name.as_encoded_bytes().contains(&0) + { + Ok(()) + } else { + Err(io::Error::new( + io::ErrorKind::InvalidInput, + "publication requires single directory entry names", + )) + } +} + /// Resolves an open directory capability back to its live filesystem path so /// path-based Win32 primitives can address entries below it. #[cfg(windows)] @@ -453,3 +548,6 @@ mod tests { } } } + +#[cfg(test)] +mod namespace_tests; diff --git a/crates/tracedecay-private-fs/src/capability_dir/namespace_tests.rs b/crates/tracedecay-private-fs/src/capability_dir/namespace_tests.rs new file mode 100644 index 00000000000..c4b7c0c604c --- /dev/null +++ b/crates/tracedecay-private-fs/src/capability_dir/namespace_tests.rs @@ -0,0 +1,163 @@ +use std::{ffi::OsStr, fs, io, path::Path}; + +use super::{Dir, rename_noreplace, replace_file}; + +const FAMILY: [&str; 4] = [ + "sessions.db", + "sessions.db-wal", + "sessions.db-shm", + "sessions.db-journal", +]; + +fn open(path: &Path) -> Dir { + Dir::open_ambient_dir(path, cap_std::ambient_authority()).unwrap() +} + +#[test] +fn publication_refuses_escaped_sources_and_destinations_without_moving_store_files() { + let root = tempfile::tempdir().unwrap(); + let stage_path = root.path().join("stage"); + let target_path = root.path().join("target"); + let foreign_path = root.path().join("foreign"); + for path in [&stage_path, &target_path, &foreign_path] { + fs::create_dir(path).unwrap(); + } + let stage = open(&stage_path); + let target = open(&target_path); + + for name in FAMILY { + let owned = format!("owned {name}"); + let foreign = format!("foreign {name}"); + fs::write(stage_path.join(name), owned.as_bytes()).unwrap(); + fs::write(foreign_path.join(name), foreign.as_bytes()).unwrap(); + for source in [Path::new("../foreign").join(name), foreign_path.join(name)] { + let error = rename_noreplace(&stage, source.as_os_str(), &target, OsStr::new(name)) + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert_eq!( + fs::read(foreign_path.join(name)).unwrap(), + foreign.as_bytes() + ); + assert!(!target_path.join(name).exists()); + } + let destination_name = format!("published-{name}"); + for destination in [ + Path::new("../foreign").join(&destination_name), + foreign_path.join(&destination_name), + ] { + let error = + rename_noreplace(&stage, OsStr::new(name), &target, destination.as_os_str()) + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert_eq!(fs::read(stage_path.join(name)).unwrap(), owned.as_bytes()); + assert_eq!( + fs::read(foreign_path.join(name)).unwrap(), + foreign.as_bytes() + ); + assert!(!foreign_path.join(&destination_name).exists()); + } + } +} + +#[test] +fn publication_rejects_normalized_aliases_and_nul_before_either_rename() { + let root = tempfile::tempdir().unwrap(); + let parent = open(root.path()); + fs::write(root.path().join("payload"), b"owned").unwrap(); + fs::write(root.path().join("published"), b"foreign").unwrap(); + + for name in [ + "", + ".", + "..", + "./payload", + "payload/", + "payload/.", + "child/payload", + "payload\0ignored", + ] { + let invalid = OsStr::new(name); + for result in [ + rename_noreplace(&parent, invalid, &parent, OsStr::new("absent")), + rename_noreplace(&parent, OsStr::new("payload"), &parent, invalid), + replace_file(&parent, invalid, OsStr::new("published")), + replace_file(&parent, OsStr::new("payload"), invalid), + ] { + assert_eq!(result.unwrap_err().kind(), io::ErrorKind::InvalidInput); + } + assert_eq!(fs::read(root.path().join("payload")).unwrap(), b"owned"); + assert_eq!(fs::read(root.path().join("published")).unwrap(), b"foreign"); + assert!(!root.path().join("absent").exists()); + } +} + +#[cfg(unix)] +mod process_replacement { + use std::{env, os::unix::fs::symlink, process::Command}; + + use super::{FAMILY, OsStr, Path, fs, io, open, rename_noreplace}; + + const ROOT: &str = "TRACEDECAY_CAPABILITY_RENAME_TEST_ROOT"; + const COMPLETE: &str = "capability-rename-intermediate-replaced"; + + #[test] + fn replace_intermediate_directory_in_child() { + let Some(root) = env::var_os(ROOT) else { + return; + }; + let root = Path::new(&root); + fs::remove_dir(root.join("stage/bridge")).unwrap(); + symlink(root.join("foreign"), root.join("stage/bridge")).unwrap(); + println!("{COMPLETE}"); + } + + #[test] + fn child_replacing_an_intermediate_directory_cannot_redirect_publication() { + let root = tempfile::tempdir().unwrap(); + let stage_path = root.path().join("stage"); + let foreign_path = root.path().join("foreign"); + fs::create_dir_all(stage_path.join("bridge")).unwrap(); + fs::create_dir(&foreign_path).unwrap(); + for name in FAMILY { + fs::write(foreign_path.join(name), name.as_bytes()).unwrap(); + } + fs::write(stage_path.join("owned.sqlite"), b"owned").unwrap(); + let stage = open(&stage_path); + + // The child changes the namespace after the parent capability is held. + let (_, module) = module_path!().split_once("::").unwrap(); + let output = Command::new(env::current_exe().unwrap()) + .args([ + "--exact", + &format!("{module}::replace_intermediate_directory_in_child"), + "--nocapture", + ]) + .env(ROOT, root.path()) + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success() && stdout.contains("1 passed") && stdout.contains(COMPLETE), + "namespace replacement child failed: {stdout} {}", + String::from_utf8_lossy(&output.stderr) + ); + let rebound = fs::symlink_metadata(stage_path.join("bridge")).unwrap(); + assert!(rebound.is_symlink()); + + for (source, destination) in [ + ("bridge/sessions.db", "published.sqlite"), + ("owned.sqlite", "bridge/published.sqlite"), + ] { + let error = + rename_noreplace(&stage, OsStr::new(source), &stage, OsStr::new(destination)) + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + } + for name in FAMILY { + assert_eq!(fs::read(foreign_path.join(name)).unwrap(), name.as_bytes()); + } + assert_eq!(fs::read(stage_path.join("owned.sqlite")).unwrap(), b"owned"); + assert!(!stage_path.join("published.sqlite").exists()); + assert!(!foreign_path.join("published.sqlite").exists()); + } +} diff --git a/crates/tracedecay-private-fs/src/inode_generation.rs b/crates/tracedecay-private-fs/src/inode_generation.rs index b9896492923..93eac0177ba 100644 --- a/crates/tracedecay-private-fs/src/inode_generation.rs +++ b/crates/tracedecay-private-fs/src/inode_generation.rs @@ -61,44 +61,45 @@ fn generation_bits(generation: libc::c_long) -> u64 { #[cfg(test)] mod tests { + #[cfg(target_os = "linux")] + use std::fs::File; #[cfg(target_os = "linux")] use std::os::unix::ffi::OsStrExt; #[cfg(target_os = "linux")] use std::os::unix::fs::MetadataExt; - #[cfg(target_os = "linux")] - use std::path::Path; #[cfg(target_os = "linux")] use super::inode_generation; - /// Writes `path` and reads back the allocation identity the kernel keeps - /// unique: the inode number with the generation, `None` where the - /// filesystem answers `FS_IOC_GETVERSION` with no generation at all. #[cfg(target_os = "linux")] - fn write_and_identify(path: &Path) -> (u64, Option) { - std::fs::write(path, b"{}\n").unwrap(); - let file = std::fs::File::open(path).unwrap(); + fn allocation_identity(file: &File) -> (u64, Option) { ( file.metadata().unwrap().ino(), - inode_generation(&file).unwrap(), + inode_generation(file).unwrap(), ) } - /// Ext4 recycles the inode number and advances the generation; ZFS keeps - /// the generation and allocates a new inode number; tmpfs reports no - /// generation and allocates a new inode number. The pair differs on all - /// of them, and neither field is required to on its own. #[cfg(target_os = "linux")] #[test] fn replacing_a_file_changes_its_inode_allocation_identity() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("rollout.jsonl"); - let first = write_and_identify(&path); + let original = File::create(&path).unwrap(); + let first = allocation_identity(&original); + + // A reported generation survives close. Without it, filesystems such + // as overlayfs can immediately reuse a closed file's inode number. + let pin = first.1.is_none().then_some(original); std::fs::remove_file(&path).unwrap(); - let second = write_and_identify(&path); + let replacement = File::create(&path).unwrap(); + let second = allocation_identity(&replacement); + if let Some(unlinked) = pin { + assert_eq!(unlinked.metadata().unwrap().nlink(), 0); + assert_eq!(allocation_identity(&unlinked), first); + } assert_ne!( first, second, - "a replacement must not share the predecessor's identity" + "a replacement must differ from a pinned or generation-backed predecessor" ); if is_ext4(temp.path()) { assert!( diff --git a/crates/tracedecay-private-fs/src/windows_file.rs b/crates/tracedecay-private-fs/src/windows_file.rs index 7de7ee9abc6..b88c838e88d 100644 --- a/crates/tracedecay-private-fs/src/windows_file.rs +++ b/crates/tracedecay-private-fs/src/windows_file.rs @@ -5,7 +5,8 @@ use std::mem::MaybeUninit; use std::os::windows::io::AsRawHandle; use windows_sys::Win32::Storage::FileSystem::{ - BY_HANDLE_FILE_INFORMATION, GetFileInformationByHandle, + BY_HANDLE_FILE_INFORMATION, FILE_STANDARD_INFO, FileStandardInfo, GetFileInformationByHandle, + GetFileInformationByHandleEx, }; #[derive(Clone, Copy)] @@ -22,6 +23,27 @@ pub fn information(file: &H) -> io::Result { identity_information(raw_information(file)?) } +/// A name that SQLite has marked for deletion cannot be reopened on Windows +/// until every remaining handle, including our creation reservation, closes. +pub fn deletion_pending(file: &impl AsRawHandle) -> io::Result { + let mut info = MaybeUninit::::uninit(); + // SAFETY: the live handle is borrowed, and FileStandardInfo writes exactly + // one FILE_STANDARD_INFO into the correctly sized aligned destination. + let ok = unsafe { + GetFileInformationByHandleEx( + file.as_raw_handle().cast(), + FileStandardInfo, + info.as_mut_ptr().cast(), + std::mem::size_of::() as u32, + ) + }; + if ok == 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: the successful call initialized the requested structure. + Ok(unsafe { info.assume_init() }.DeletePending) +} + /// Reads only the hard-link count from an open Windows file handle. /// /// Unlike [`information`], this does not require the volume to expose a diff --git a/crates/tracedecay-runtime-core/src/db/engine/connection.rs b/crates/tracedecay-runtime-core/src/db/engine/connection.rs index 3bd7c6f4eb5..62fec9cd718 100644 --- a/crates/tracedecay-runtime-core/src/db/engine/connection.rs +++ b/crates/tracedecay-runtime-core/src/db/engine/connection.rs @@ -186,6 +186,13 @@ impl Connection { Ok(Rows::from_exact(rows)) } + pub(crate) async fn finish_initial_schema(&self) -> Result<()> { + self.runtime + .finish_initial_schema() + .await + .map_err(Into::into) + } + pub async fn checkpoint_wal_truncate(&self) -> Result { let runtime = Arc::clone(&self.runtime); let rows = runtime.checkpoint_wal_truncate_async().await?; diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry.rs index da1e1615004..76015075313 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry.rs @@ -14,6 +14,8 @@ mod capacity; mod close; mod destructive; mod graph; +mod initialization; +pub use initialization::lock_offline_sqlite_family_for_reset; mod leases; mod open; mod ports; diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/close.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/close.rs index d9a8501499a..0576c8ebe6d 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/close.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/close.rs @@ -704,6 +704,15 @@ mod tests { } } + /// An existing attachment needs an initialized SQLite header even when + /// the fixture intentionally has no tables. + fn seed_empty_sqlite(path: &Path) { + rusqlite::Connection::open(path) + .unwrap() + .execute_batch("PRAGMA user_version = 0") + .unwrap(); + } + async fn seed_final_graph_db(path: PathBuf) -> PathBuf { let connection = crate::db::engine::TestConnection::open(&path); crate::db::migrations::create_schema_connection(&connection) @@ -725,7 +734,7 @@ mod tests { let authority = DatabaseAuthority::for_runtime(&path, "mount exact-close code runtime").unwrap(); let profile_path = path.with_file_name("profile.db"); - rusqlite::Connection::open(&profile_path).unwrap(); + seed_empty_sqlite(&profile_path); let profile_path = profile_path.canonicalize().unwrap(); let registry = StoreRuntimeRegistry::new( asupersync::runtime::Runtime::current_handle().unwrap(), @@ -1052,7 +1061,7 @@ mod tests { let opened_identity = code.opened_file_identity().unwrap(); let detached = path.with_extension("detached.db"); let replacement = path.with_extension("replacement.db"); - rusqlite::Connection::open(&replacement).unwrap(); + seed_empty_sqlite(&replacement); std::fs::rename(&path, &detached).unwrap(); std::fs::rename(&replacement, &path).unwrap(); drop(code); @@ -1304,11 +1313,11 @@ mod tests { async fn failed_exact_close_retains_a_faulted_evicting_runtime() { let temporary = tempfile::tempdir().unwrap(); let path = temporary.path().join("runtime.db"); - rusqlite::Connection::open(&path).unwrap(); + seed_empty_sqlite(&path); let path = path.canonicalize().unwrap(); let authority = DatabaseAuthority::for_runtime(&path, "mount failing exact-close").unwrap(); let profile_path = path.with_file_name("profile.db"); - rusqlite::Connection::open(&profile_path).unwrap(); + seed_empty_sqlite(&profile_path); let profile_path = profile_path.canonicalize().unwrap(); let registry = StoreRuntimeRegistry::with_config( asupersync::runtime::Runtime::current_handle().unwrap(), diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs new file mode 100644 index 00000000000..30d25426f56 --- /dev/null +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs @@ -0,0 +1,1195 @@ +//! A prospective SQLite store publishes its completed main last. It moves staged +//! files by no-replace rename and retires only empty directories. +//! A failed schema install therefore cannot delete a file at a reused basename. +use std::ffi::OsStr; +use std::fs::{self, File}; +use std::io::{self, Read, Seek, Write}; +#[cfg(unix)] +use std::os::unix::fs::MetadataExt; +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +mod interrupted; + +use interrupted::InterruptedInstall; +use tracedecay_private_fs::capability_dir::{self, Dir}; +use tracedecay_private_fs::framed_log::{DirectorySyncPolicy, sync_parent_directory}; +use tracedecay_private_fs::{FileLease, create_private_directory, open_private_directory}; + +use crate::sqlite_read_snapshot::with_suffix; + +const SIDECARS: [&str; 3] = ["-wal", "-shm", "-journal"]; +const STAGED_MEMBERS: [&str; 4] = [ + "schema.sqlite", + "schema.sqlite-wal", + "schema.sqlite-shm", + "schema.sqlite-journal", +]; +use tracedecay_rusqlite_runtime::repository::InitializationFiles; + +const OWNER: &str = "schema-installation.owner"; +const SYNC: DirectorySyncPolicy = DirectorySyncPolicy::TolerateUnsupported; + +/// Coordinates an authorized reset of an offline store with the existing +/// initializer. A staged or ambiguous family must finish reconciliation +/// before reset can discard any canonical members. +pub fn lock_offline_sqlite_family_for_reset(destination: &Path) -> io::Result> { + let staging = InitializationDirectory::path(destination); + let lease = crate::storage::try_lock_existing_durable_directory(&staging)?; + if present(&staging)? + || lease + .as_ref() + .map(|file| file.metadata()) + .transpose()? + .is_some_and(|meta| meta.len() != 0) + { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "SQLite initialization requires reconciliation before reset", + )); + } + Ok(lease) +} + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct PreparedFamily { + #[serde(default, skip_serializing_if = "Option::is_none")] + retry: Option, + runtime_identity: u64, + main: LeafIdentity, + // Allocation identity alone cannot detect in-place content replacement. + main_sha256: [u8; 32], + sidecars: [LeafIdentity; 3], +} + +fn surviving_sidecars(sidecars: [File; 3]) -> io::Result<[Option; 3]> { + let retained = sidecars.map(Some); + #[cfg(windows)] + let mut retained = retained; + #[cfg(windows)] + for slot in &mut retained { + if let Some(file) = slot.as_ref() + && tracedecay_private_fs::windows_file::deletion_pending(file)? + { + drop(slot.take()); + } + } + Ok(retained) +} + +impl PreparedFamily { + fn has_durable_generations(&self) -> bool { + self.main.allocation.is_some() + && self + .sidecars + .iter() + .all(|sidecar| sidecar.allocation.is_some()) + } + + fn pin_main(&self, directory: &Dir, name: &OsStr) -> io::Result { + let file = self.main.pin_at(directory, name, false)?; + if sha256_file(&file)? != self.main_sha256 { + return Err(ambiguous("prepared SQLite main file content changed")); + } + Ok(file) + } + + fn load(file: &mut File) -> io::Result> { + let length = file.metadata()?.len(); + Ok(if length == 0 { + None + } else { + if length > 2048 { + return Err(ambiguous("invalid SQLite preparation receipt")); + } + let mut bytes = Vec::new(); + file.take(length).read_to_end(&mut bytes)?; + Some( + serde_json::from_slice(&bytes) + .map_err(|_| ambiguous("invalid SQLite preparation receipt"))?, + ) + }) + } +} + +#[derive(Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +struct LeafIdentity { + device: u64, + file_id: u64, + created: Option, + allocation: Option, +} + +impl LeafIdentity { + fn from_file(file: &File) -> io::Result { + let metadata = file.metadata()?; + #[cfg(unix)] + let (device, file_id) = (metadata.dev(), metadata.ino()); + #[cfg(windows)] + let (device, file_id) = { + let info = tracedecay_private_fs::windows_file::information(file)?; + (u64::from(info.volume_serial_number), info.file_index) + }; + #[cfg(target_os = "macos")] + let allocation = { + use std::os::macos::fs::MetadataExt; + let value = metadata.st_gen(); + (value != 0).then_some(u64::from(value)) + }; + #[cfg(not(target_os = "macos"))] + let allocation = tracedecay_private_fs::inode_generation(file)?; + #[cfg(any(unix, windows))] + { + Ok(Self { + device, + file_id, + created: metadata.created().ok(), + allocation, + }) + } + #[cfg(not(any(unix, windows)))] + { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "SQLite generation identity is unavailable", + )) + } + } + + fn pin_at(&self, directory: &Dir, name: &OsStr, empty: bool) -> io::Result { + let file = capability_dir::open_regular_nofollow(directory, name)?; + if *self != Self::from_file(&file)? || (empty && file.metadata()?.len() != 0) { + return Err(ambiguous( + "SQLite initialization member no longer matches its prepared generation", + )); + } + Ok(file) + } +} + +fn sha256_file(mut file: &File) -> io::Result<[u8; 32]> { + file.rewind()?; + let length = file.metadata()?.len(); + let mut input = file.take(length); + let mut digest = Sha256::new(); + let mut buffer = [0u8; 8192]; + loop { + let n = input.read(&mut buffer)?; + if n == 0 { + break; + } + digest.update(&buffer[..n]); + } + if input.limit() != 0 || file.metadata()?.len() != length { + return Err(ambiguous( + "SQLite recovery member changed length during verification", + )); + } + Ok(digest.finalize().into()) +} + +fn ambiguous(message: &'static str) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, message) +} + +pub(super) struct InitializationDirectory { + destination: PathBuf, + directory: PathBuf, + directory_pin: Option, + parent: Dir, + lease: FileLease, + prepared: Option, +} + +impl InitializationDirectory { + fn path(destination: &Path) -> PathBuf { + with_suffix(destination, ".tracedecay-init") + } + + fn acquire(destination: PathBuf, create: bool) -> io::Result { + let directory = Self::path(&destination); + let parent = capability_dir::pin_parent( + destination + .parent() + .ok_or_else(|| ambiguous("SQLite destination has no parent"))?, + )?; + let mut lease = crate::storage::try_lock_durable_directory(&directory)?; + let prepared = PreparedFamily::load(&mut lease)?; + if create { + if prepared.is_some() && !present(&directory)? { + return Err(ambiguous( + "prepared SQLite initialization directory is missing", + )); + } + match create_private_directory(&directory) { + Ok(()) => sync_parent_directory(&directory, SYNC)?, + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error), + } + } + let directory_pin = open_private_directory(&directory)?; + let directory = directory.canonicalize()?; + Ok(Self { + destination, + directory, + directory_pin: Some(directory_pin), + parent, + lease, + prepared, + }) + } + + pub(super) fn begin(destination: PathBuf) -> io::Result { + let stage = Self::acquire(destination, true)?; + if present(&stage.destination)? { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "SQLite destination exists", + )); + } + if stage.prepared.is_none() { + stage.require_empty()?; + for suffix in SIDECARS { + if present(&with_suffix(&stage.destination, suffix))? { + return Err(ambiguous( + "occupied SQLite sidecar has no prepared generation", + )); + } + } + } + Ok(stage) + } + + pub(super) fn database(&self) -> PathBuf { + self.directory.join("schema.sqlite") + } + + #[cfg(test)] + fn reservation(&self, suffix: &str) -> PathBuf { + self.directory.join(format!("reservation{suffix}")) + } + + fn stage_cap(&self) -> io::Result { + let pin = self + .directory_pin + .as_ref() + .ok_or_else(|| ambiguous("SQLite staging directory is no longer pinned"))?; + Ok(Dir::from_std_file(pin.try_clone()?)) + } + + fn require_empty(&self) -> io::Result<()> { + self.verify_directory_name()?; + if self.stage_cap()?.entries()?.next().transpose()?.is_some() { + return Err(ambiguous( + "unproven SQLite staging contents require offline reconciliation", + )); + } + Ok(()) + } + + fn prepare( + &mut self, + expected: u64, + (main_file, sidecars): (File, [File; 3]), + ) -> io::Result<[File; 4]> { + self.verify_directory_name()?; + let stage = self.stage_cap()?; + let sidecars = surviving_sidecars(sidecars)?; + let main = LeafIdentity::from_file(&main_file)?; + let main_pin = main.pin_at(&stage, OsStr::new("schema.sqlite"), false)?; + require_schema_members(&stage)?; + main_file.sync_all()?; + let main_sha256 = sha256_file(&main_pin)?; + // WAL and rollback-journal bytes may be durable data. The original + // writer has checkpointed and joined; refuse any remaining frames. + // Only the original, now-unused SHM index may be cleared by descriptor. + for (index, suffix) in SIDECARS.iter().enumerate() { + let source = format!("schema.sqlite{suffix}"); + if present_at(&stage, source.as_ref())? { + let file = sidecars[index].as_ref().ok_or_else(|| { + ambiguous("SQLite sidecar has no surviving original reservation") + })?; + LeafIdentity::from_file(file)?.pin_at( + &stage, + source.as_ref(), + *suffix != "-shm", + )?; + } + } + let reserve = |index: usize, suffix: &str| { + let source = format!("schema.sqlite{suffix}"); + let target = format!("reservation{suffix}"); + if present_at(&stage, source.as_ref())? { + let file = sidecars[index].as_ref().ok_or_else(|| { + ambiguous("SQLite sidecar has no surviving original reservation") + })?; + let identity = LeafIdentity::from_file(file)?; + identity.pin_at(&stage, source.as_ref(), suffix != "-shm")?; + if suffix == "-shm" { + file.set_len(0)?; + } + file.sync_all()?; + capability_dir::rename_noreplace(&stage, source.as_ref(), &stage, target.as_ref())?; + let pin = identity.pin_at(&stage, target.as_ref(), true)?; + Ok::<_, io::Error>((identity, pin)) + } else { + let file = capability_dir::create_private_file(&stage, target.as_ref())?; + file.sync_all()?; + let identity = LeafIdentity::from_file(&file)?; + let pin = identity.pin_at(&stage, target.as_ref(), true)?; + Ok((identity, pin)) + } + }; + let (wal, wal_pin) = reserve(0, "-wal")?; + let (shm, shm_pin) = reserve(1, "-shm")?; + let (journal, journal_pin) = reserve(2, "-journal")?; + let prepared = PreparedFamily { + retry: None, + runtime_identity: expected, + main, + main_sha256, + sidecars: [wal, shm, journal], + }; + capability_dir::sync_directory(&stage)?; + self.record(prepared)?; + Ok([main_pin, wal_pin, shm_pin, journal_pin]) + } + + fn record(&mut self, prepared: PreparedFamily) -> io::Result<()> { + let bytes = serde_json::to_vec(&prepared).map_err(io::Error::other)?; + self.lease.rewind()?; + self.lease.write_all(&bytes)?; + self.lease.set_len(bytes.len() as u64)?; + self.lease.sync_all()?; + self.prepared = Some(prepared); + Ok(()) + } + + fn pin_members(&self, stage: &Dir, prepared: &PreparedFamily) -> io::Result> { + self.verify_directory_name()?; + for entry in stage.entries()? { + let name = entry?.file_name(); + if name != "schema.sqlite" + && !SIDECARS + .iter() + .any(|suffix| name == OsStr::new(&format!("reservation{suffix}"))) + { + return Err(ambiguous( + "unexpected SQLite staging member requires offline reconciliation", + )); + } + } + let mut pins = vec![prepared.pin_main(stage, OsStr::new("schema.sqlite"))?]; + for (suffix, identity) in SIDECARS.into_iter().zip(&prepared.sidecars) { + let from = format!("reservation{suffix}"); + let to = with_suffix(&self.destination, suffix); + let to = leaf(&to)?; + pins.push( + match ( + present_at(stage, from.as_ref())?, + present_at(&self.parent, to)?, + ) { + (true, false) => identity.pin_at(stage, from.as_ref(), true)?, + (false, true) => identity.pin_at(&self.parent, to, true)?, + _ => return Err(ambiguous("SQLite reservation is missing or ambiguous")), + }, + ); + } + Ok(pins) + } + + pub(super) fn resume_prepared(&mut self) -> io::Result> { + let Some(prepared) = self.prepared.as_ref() else { + return Ok(None); + }; + if prepared.retry.is_some() { + return Ok(None); + } + // A persistent inode number alone cannot distinguish inode reuse after + // process death. A filesystem without allocation generations can publish + // in the original process, but must preserve interrupted preparation. + if !prepared.has_durable_generations() { + return Err(ambiguous( + "SQLite preparation has no durable file generation for recovery", + )); + } + self.publish_prepared(Vec::new()).map(Some) + } + + fn publish_prepared(&mut self, mut pins: Vec) -> io::Result { + let prepared = self + .prepared + .as_ref() + .ok_or_else(|| ambiguous("SQLite preparation receipt is missing"))?; + let stage = self.stage_cap()?; + pins.extend(self.pin_members(&stage, prepared)?); + for (suffix, identity) in SIDECARS.into_iter().zip(&prepared.sidecars) { + let from = format!("reservation{suffix}"); + let to = with_suffix(&self.destination, suffix); + let to = leaf(&to)?; + if present_at(&stage, from.as_ref())? { + capability_dir::rename_noreplace(&stage, from.as_ref(), &self.parent, to)?; + } + identity.pin_at(&self.parent, to, true)?; + } + capability_dir::sync_directory(&stage)?; + capability_dir::sync_directory(&self.parent)?; + self.pin_members(&stage, prepared)?; + let expected = prepared.runtime_identity; + capability_dir::rename_noreplace( + &stage, + OsStr::new("schema.sqlite"), + &self.parent, + leaf(&self.destination)?, + )?; + capability_dir::sync_directory(&self.parent)?; + prepared.pin_main(&self.parent, leaf(&self.destination)?)?; + drop(stage); + // Read-only admission observes the receipt without taking the writer's + // lease. Close every raw main/SHM pin before clearing that receipt: + // a later close could release the new reader's process-wide POSIX locks. + drop(pins); + self.retire()?; + Ok(expected) + } + + pub(super) fn publish(mut self, expected: u64, files: (File, [File; 3])) -> io::Result<()> { + let pins = self.prepare(expected, files)?; + self.publish_prepared(Vec::from(pins)).map(|_| ()) + } + + fn verify_directory_name(&self) -> io::Result<()> { + let pinned = self + .directory_pin + .as_ref() + .ok_or_else(|| ambiguous("SQLite staging directory is no longer pinned"))?; + verify_named_directory(pinned, &self.directory) + } + + fn retire(&mut self) -> io::Result<()> { + self.require_empty()?; + if self.prepared.is_some() { + // If we crash after clearing the receipt, an empty directory is all + // that remains of the already-published preparation. + self.lease.set_len(0)?; + self.lease.sync_all()?; + } + drop(self.directory_pin.take()); + // The kernel refuses a file or a nonempty directory even if a new + // entry appears after the read. No regular staging leaf is unlinked. + self.parent.remove_dir(leaf(&self.directory)?)?; + capability_dir::sync_directory(&self.parent)?; + if self.prepared.is_some() { + tracing::debug!( + name: "sqlite.initialization.handoff", + path = %self.destination.display(), + "completed SQLite initialization handed off" + ); + } + Ok(()) + } + + fn require_released_receipt(directory: &Path) -> io::Result<()> { + if crate::storage::durable_directory_lock_has_data(directory)? { + return Err(ambiguous( + "SQLite preparation receipt requires writable reconciliation", + )); + } + Ok(()) + } + + pub(super) fn verify_read_only_handoff(destination: &Path) -> io::Result<()> { + let directory = Self::path(destination); + Self::require_released_receipt(&directory)?; + let pin = match open_private_directory(&directory) { + Ok(pin) => pin, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error), + }; + for entry in Dir::from_std_file(pin).entries()? { + // An empty owner-only marker from the previous layout is harmless + // for read-only admission. Never retire it by mutable pathname. + let entry = entry?; + if entry.file_name() != OWNER + || !entry.file_type()?.is_file() + || entry.metadata()?.len() != 0 + { + return Err(ambiguous( + "SQLite staging requires writable reconciliation before read-only attachment", + )); + } + } + Ok(()) + } + + pub(super) fn recover_published(destination: &Path) -> io::Result<()> { + let directory = Self::path(destination); + if !present(&directory)? { + return Self::require_released_receipt(&directory); + } + let mut stage = match Self::acquire(destination.to_owned(), false) { + Ok(stage) => stage, + Err(error) if error.kind() == io::ErrorKind::NotFound && !present(&directory)? => { + return Self::require_released_receipt(&directory); + } + Err(error) => return Err(error), + }; + let mut pins = Vec::new(); + if let Some(prepared) = &stage.prepared { + if prepared.retry.is_some() { + return Err(ambiguous( + "canonical SQLite database appeared during an interrupted schema installation", + )); + } + if !prepared.has_durable_generations() { + return Err(ambiguous( + "SQLite preparation has no durable file generation for recovery", + )); + } + pins.push(prepared.pin_main(&stage.parent, leaf(destination)?)?); + for (suffix, identity) in SIDECARS.into_iter().zip(&prepared.sidecars) { + pins.push(identity.pin_at( + &stage.parent, + leaf(&with_suffix(destination, suffix))?, + true, + )?); + } + } + // Recovery also reopens raw evidence descriptors. They must not outlive + // the receipt that excludes read-only attachments from the family. + drop(pins); + stage.retire() + } +} + +impl Drop for InitializationDirectory { + fn drop(&mut self) { + if self.directory_pin.is_some() + && self.prepared.is_none() + && self.require_empty().is_ok() + && let Err(error) = self.retire() + { + tracing::warn!(%error, "could not retire empty SQLite initialization directory"); + } + } +} + +fn require_schema_members(directory: &Dir) -> io::Result<()> { + for entry in directory.entries()? { + let name = entry?.file_name(); + if !STAGED_MEMBERS + .iter() + .any(|member| name == OsStr::new(member)) + { + return Err(ambiguous("unproven member in SQLite schema directory")); + } + } + Ok(()) +} + +fn verify_named_directory(pinned: &File, path: &Path) -> io::Result<()> { + let named = open_private_directory(path)?; + #[cfg(unix)] + let matches = { + let left = pinned.metadata()?; + let right = named.metadata()?; + left.dev() == right.dev() && left.ino() == right.ino() + }; + #[cfg(windows)] + let matches = { + let left = tracedecay_private_fs::windows_file::information(pinned)?; + let right = tracedecay_private_fs::windows_file::information(&named)?; + left.volume_serial_number == right.volume_serial_number + && left.file_index == right.file_index + }; + #[cfg(not(any(unix, windows)))] + let matches = false; + if !matches { + return Err(ambiguous( + "SQLite staging directory no longer names the pinned directory", + )); + } + Ok(()) +} + +fn leaf(path: &Path) -> io::Result<&OsStr> { + path.file_name() + .ok_or_else(|| ambiguous("SQLite member has no basename")) +} + +fn present_at(directory: &Dir, name: &OsStr) -> io::Result { + match directory.symlink_metadata(name) { + Ok(_) => Ok(true), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } +} + +fn present(path: &Path) -> io::Result { + match fs::symlink_metadata(path) { + Ok(_) => Ok(true), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rusqlite::Connection; + #[cfg(unix)] + use std::os::unix::fs::symlink; + use std::process::Command; + use tempfile::TempDir; + use tracedecay_private_fs::create_private_file; + use tracedecay_private_fs::framed_log::rename_noreplace; + + #[cfg(all(unix, feature = "test-helpers"))] + mod handoff; + + fn seed(path: &Path, value: i64) { + let db = Connection::open(path).unwrap(); + db.execute_batch("CREATE TABLE data(value INTEGER)") + .unwrap(); + db.execute("INSERT INTO data VALUES (?)", [value]).unwrap(); + } + + fn owned_files(stage: &InitializationDirectory) -> (File, [File; 3]) { + let path = stage.database(); + let main = File::options().read(true).write(true).open(&path).unwrap(); + ( + main, + SIDECARS.map(|suffix| create_private_file(&with_suffix(&path, suffix)).unwrap()), + ) + } + + #[test] + fn changed_prepared_sqlite_bytes_are_preserved_and_never_published() { + const CHILD: &str = "TRACEDECAY_SQLITE_PREPARED_SWAP"; + const FOREIGN: &str = "TRACEDECAY_SQLITE_PREPARED_FOREIGN"; + if let Some(target) = std::env::var_os(CHILD) { + let bytes = fs::read(std::env::var_os(FOREIGN).unwrap()).unwrap(); + let mut file = fs::OpenOptions::new().write(true).open(target).unwrap(); + assert_eq!(file.metadata().unwrap().len(), bytes.len() as u64); + file.rewind().unwrap(); + file.write_all(&bytes).unwrap(); + file.sync_all().unwrap(); + return; + } + + let root = TempDir::new().unwrap(); + let destination = root.path().join("sessions.db"); + let mut stage = InitializationDirectory::begin(destination.clone()).unwrap(); + seed(&stage.database(), 7); + let foreign = root.path().join("foreign.sqlite"); + seed(&foreign, 99); + let before = fs::read(stage.database()).unwrap(); + assert_eq!(before.len(), fs::metadata(&foreign).unwrap().len() as usize); + let identity = crate::db::sqlite_generation_identity(&stage.database()).unwrap(); + let files = owned_files(&stage); + stage.prepare(identity, files).unwrap(); + + let (_, module) = module_path!().split_once("::").unwrap(); + let test = + format!("{module}::changed_prepared_sqlite_bytes_are_preserved_and_never_published"); + let child = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", &test]) + .env(CHILD, stage.database()) + .env(FOREIGN, &foreign) + .output() + .unwrap(); + assert!( + child.status.success(), + "cross-process mutation failed: {child:?}" + ); + assert!( + LeafIdentity::from_file(&File::open(stage.database()).unwrap()).unwrap() + == stage.prepared.as_ref().unwrap().main + ); + assert_ne!(fs::read(stage.database()).unwrap(), before); + assert_eq!(read_value(&stage.database()), 99); + assert_eq!( + stage.publish_prepared(Vec::new()).unwrap_err().kind(), + io::ErrorKind::InvalidData + ); + assert!(!destination.exists()); + for suffix in SIDECARS { + assert!(!with_suffix(&destination, suffix).exists()); + assert!(stage.reservation(suffix).exists()); + } + assert_eq!(read_value(&stage.database()), 99); + assert_eq!(read_value(&foreign), 99); + } + + #[test] + fn process_exit_during_initialization_recovers_without_publishing_partial_schema() { + const CHILD: &str = "TRACEDECAY_TEST_EXIT_DURING_INITIALIZATION"; + if let Some(destination) = std::env::var_os(CHILD) { + let mut stage = InitializationDirectory::begin(destination.into()).unwrap(); + seed(&stage.database(), 1); + let state = std::env::var("TRACEDECAY_TEST_INITIALIZATION_STATE").unwrap(); + if state != "installing" { + let identity = crate::db::sqlite_generation_identity(&stage.database()).unwrap(); + let files = owned_files(&stage); + stage.prepare(identity, files).unwrap(); + if state != "prepared" { + let count = if state == "reserved" || state == "unproven" { + 1 + } else { + 3 + }; + for suffix in SIDECARS.into_iter().take(count) { + rename_noreplace( + &stage.reservation(suffix), + &with_suffix(&stage.destination, suffix), + ) + .unwrap(); + } + if state == "published" { + rename_noreplace(&stage.database(), &stage.destination).unwrap(); + } + } + } + std::process::exit(73); + } + let name = format!( + "{}::process_exit_during_initialization_recovers_without_publishing_partial_schema", + module_path!().split_once("::").unwrap().1 + ); + for state in [ + "installing", + "prepared", + "reserved", + "published", + "unproven", + ] { + let root = TempDir::new().unwrap(); + let destination = root.path().join("sessions.db"); + let output = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", &name]) + .env(CHILD, &destination) + .env("TRACEDECAY_TEST_INITIALIZATION_STATE", state) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(73), "{output:?}"); + let directory = InitializationDirectory::path(&destination); + if state == "installing" { + assert!(InitializationDirectory::begin(destination.clone()).is_err()); + assert_eq!(read_value(&directory.join("schema.sqlite")), 1); + assert!(!destination.exists()); + continue; + } + if state == "unproven" { + let wal = with_suffix(&destination, "-wal"); + fs::write(&wal, b"unproven WAL").unwrap(); + let mut retry = InitializationDirectory::begin(destination.clone()).unwrap(); + assert!(retry.resume_prepared().is_err()); + assert_eq!(fs::read(wal).unwrap(), b"unproven WAL"); + assert!(!destination.exists()); + continue; + } + let durable = { + let stage = InitializationDirectory::acquire(destination.clone(), false).unwrap(); + stage.prepared.as_ref().unwrap().has_durable_generations() + }; + if !durable { + let published = state == "published"; + if published { + assert!(InitializationDirectory::recover_published(&destination).is_err()); + } else { + let mut retry = InitializationDirectory::begin(destination.clone()).unwrap(); + assert!(retry.resume_prepared().is_err()); + } + assert_eq!(destination.exists(), published); + assert_eq!( + read_value( + if published { + destination.clone() + } else { + directory.join("schema.sqlite") + } + .as_path() + ), + 1 + ); + assert!(directory.is_dir()); + continue; + } + if state == "published" { + InitializationDirectory::recover_published(&destination).unwrap(); + } else { + let mut retry = InitializationDirectory::begin(destination.clone()).unwrap(); + assert!(retry.resume_prepared().unwrap().is_some()); + } + assert_eq!(read_value(&destination), 1); + let db = Connection::open(&destination).unwrap(); + assert_eq!( + db.query_row("PRAGMA integrity_check", [], |row| row.get::<_, String>(0)) + .unwrap(), + "ok" + ); + assert!(!directory.exists()); + } + } + + fn read_value(path: &Path) -> i64 { + Connection::open(path) + .unwrap() + .query_row("SELECT value FROM data", [], |row| row.get(0)) + .unwrap() + } + + #[test] + fn unproven_private_members_survive_drop_and_retry() { + let root = TempDir::new().unwrap(); + for member in [ + "schema.sqlite", + "schema.sqlite-wal", + "schema.sqlite-shm", + "schema.sqlite-journal", + "prepared.sqlite", + ] { + let destination = root.path().join(member); + let stage = InitializationDirectory::begin(destination.clone()).unwrap(); + let path = stage.directory.join(member); + fs::write(&path, b"unclaimed data").unwrap(); + drop(stage); + assert!(InitializationDirectory::begin(destination.clone()).is_err()); + assert!(!destination.exists()); + assert_eq!(fs::read(path).unwrap(), b"unclaimed data"); + } + } + + #[test] + fn publication_requires_its_original_directory_and_schema() { + let root = TempDir::new().unwrap(); + let destination = root.path().join("sessions.db"); + let mut stage = InitializationDirectory::begin(destination.clone()).unwrap(); + let schema = stage.database(); + seed(&schema, 7); + let identity = crate::db::sqlite_generation_identity(&schema).unwrap(); + let files = owned_files(&stage); + let other = root.path().join("independent-directory"); + create_private_directory(&other).unwrap(); + let independent = other.join("schema.sqlite"); + seed(&independent, 99); + let bytes = fs::read(&independent).unwrap(); + stage.directory = other; + assert_eq!( + stage.publish(identity, files).unwrap_err().kind(), + io::ErrorKind::InvalidData + ); + assert!(!destination.exists()); + for suffix in SIDECARS { + assert!(!with_suffix(&destination, suffix).exists()); + } + assert_eq!(fs::read(&independent).unwrap(), bytes); + assert_eq!(read_value(&schema), 7); + } + + #[test] + fn missing_stage_with_an_outstanding_receipt_denies_handoff_and_reset() { + let root = TempDir::new().unwrap(); + let destination = root.path().join("sessions.db"); + seed(&destination, 7); + let directory = InitializationDirectory::path(&destination); + let mut lease = crate::storage::try_lock_durable_directory(&directory).unwrap(); + lease.write_all(b"unresolved preparation").unwrap(); + drop(lease); + assert!(InitializationDirectory::verify_read_only_handoff(&destination).is_err()); + assert!(InitializationDirectory::recover_published(&destination).is_err()); + assert!(lock_offline_sqlite_family_for_reset(&destination).is_err()); + assert!(!directory.exists()); + let mut lease = crate::storage::try_lock_existing_durable_directory(&directory) + .unwrap() + .unwrap(); + let mut receipt = String::new(); + lease.read_to_string(&mut receipt).unwrap(); + assert_eq!(receipt, "unresolved preparation"); + assert_eq!(read_value(&destination), 7); + } + + #[test] + fn process_exit_during_interrupted_schema_transfer_preserves_or_restores_every_member() { + const DATABASE: &str = "TRACEDECAY_TEST_EXIT_DURING_RETRY_TRANSFER"; + const MOVES: &str = "TRACEDECAY_TEST_RETRY_TRANSFER_MOVES"; + if let Some(path) = std::env::var_os(DATABASE) { + let destination = PathBuf::from(path); + let mut stage = InitializationDirectory::begin(destination.clone()).unwrap(); + seed(&stage.database(), 17); + let identity = crate::db::sqlite_generation_identity(&stage.database()).unwrap(); + let files = owned_files(&stage); + stage.hold_interrupted(identity, [9; 32], files).unwrap(); + let family = stage.prepared.as_ref().unwrap(); + let recovery = family.retry.as_ref().unwrap().directory(&destination); + let moves: usize = std::env::var(MOVES).unwrap().parse().unwrap(); + for name in STAGED_MEMBERS.iter().take(moves) { + rename_noreplace(&recovery.join(name), &stage.directory.join(name)).unwrap(); + } + if moves > STAGED_MEMBERS.len() { + fs::remove_dir(&recovery).unwrap(); + } + std::process::exit(73); + } + let test = format!( + "{}::process_exit_during_interrupted_schema_transfer_preserves_or_restores_every_member", + module_path!().split_once("::").unwrap().1 + ); + for moves in [1, 3, 4, 5] { + let root = TempDir::new().unwrap(); + let destination = root.path().join("sessions.db"); + let child = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", &test]) + .env(DATABASE, &destination) + .env(MOVES, moves.to_string()) + .output() + .unwrap(); + assert_eq!(child.status.code(), Some(73), "{child:?}"); + let mut stage = InitializationDirectory::begin(destination.clone()).unwrap(); + let family = stage.prepared.as_ref().unwrap(); + let supported = std::iter::once(&family.main) + .chain(&family.sidecars) + .take(moves) + .all(|leaf| leaf.allocation.is_some()); + let recovery = family.retry.as_ref().unwrap().directory(&destination); + let before = STAGED_MEMBERS.map(|name| { + let in_stage = stage.directory.join(name); + let parked = recovery.join(name); + assert_ne!(in_stage.exists(), parked.exists()); + if in_stage.exists() { in_stage } else { parked } + }); + let bytes = before.each_ref().map(|file| fs::read(file).unwrap()); + let result = stage.resume_interrupted([9; 32]); + if supported { + let pins = result.unwrap().unwrap(); + for (name, contents) in STAGED_MEMBERS.into_iter().zip(bytes) { + assert_eq!(fs::read(stage.directory.join(name)).unwrap(), contents); + } + drop(pins); + assert!(!recovery.exists()); + } else { + assert_eq!(result.unwrap_err().kind(), io::ErrorKind::InvalidData); + for (file, contents) in before.iter().zip(bytes) { + assert_eq!(fs::read(file).unwrap(), contents); + } + } + assert!(!destination.exists()); + } + } + + #[test] + fn interrupted_install_requires_its_entire_recorded_family_before_restoring_any_member() { + let root = TempDir::new().unwrap(); + let destination = root.path().join("sessions.db"); + let mut stage = InitializationDirectory::begin(destination.clone()).unwrap(); + seed(&stage.database(), 7); + let files = owned_files(&stage); + let identity = crate::db::sqlite_generation_identity(&stage.database()).unwrap(); + stage.hold_interrupted(identity, [1; 32], files).unwrap(); + let recovery = stage + .prepared + .as_ref() + .unwrap() + .retry + .as_ref() + .unwrap() + .directory(&destination); + drop(stage); + let mut retry = InitializationDirectory::begin(destination.clone()).unwrap(); + assert_eq!( + retry.resume_interrupted([2; 32]).unwrap_err().kind(), + io::ErrorKind::InvalidData + ); + drop(retry); + let wal = recovery.join("schema.sqlite-wal"); + fs::write(&wal, b"independent WAL bytes").unwrap(); + let mut retry = InitializationDirectory::begin(destination.clone()).unwrap(); + assert_eq!( + retry.resume_interrupted([1; 32]).unwrap_err().kind(), + io::ErrorKind::InvalidData + ); + assert_eq!(fs::read(&wal).unwrap(), b"independent WAL bytes"); + assert!(recovery.join("schema.sqlite").is_file()); + assert!( + retry + .stage_cap() + .unwrap() + .entries() + .unwrap() + .next() + .is_none() + ); + assert!(!destination.exists()); + assert!(lock_offline_sqlite_family_for_reset(&destination).is_err()); + } + + #[test] + fn preparation_refuses_unclaimed_sidecar_handles_without_modifying_the_family() { + let root = TempDir::new().unwrap(); + let destination = root.path().join("sessions.db"); + let mut stage = InitializationDirectory::begin(destination.clone()).unwrap(); + seed(&stage.database(), 9); + let identity = crate::db::sqlite_generation_identity(&stage.database()).unwrap(); + let (main, mut files) = owned_files(&stage); + let separate = root.path().join("unrelated-empty-reservation"); + files[1] = create_private_file(&separate).unwrap(); + let shm = with_suffix(&stage.database(), "-shm"); + fs::write(&shm, b"preserve existing index").unwrap(); + let error = stage.prepare(identity, (main, files)).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert_eq!(fs::read(shm).unwrap(), b"preserve existing index"); + assert_eq!(fs::metadata(separate).unwrap().len(), 0); + assert!(!destination.exists()); + for suffix in SIDECARS { + assert!(!stage.reservation(suffix).exists()); + } + } + + /// A competing process may rename the private directory despite the + /// cooperative lease and install a symlink at the old path. Failed-start + /// cleanup must preserve occupied entries in the pinned directory and never + /// touch the distinct SQLite database beneath that replacement symlink. + #[cfg(unix)] + #[test] + fn failed_initialization_cleanup_uses_pinned_directory_after_swap() { + const SWAP_ROOT: &str = "TRACEDECAY_SQLITE_STAGE_SWAP_ROOT"; + const COMPLETE: &str = "tracedecay-sqlite-stage-swap-complete"; + if let Some(root) = std::env::var_os(SWAP_ROOT) { + let root = PathBuf::from(root); + let stage = root.join("sessions.db.tracedecay-init"); + fs::rename(&stage, root.join("retired-stage")).unwrap(); + symlink(root.join("foreign"), &stage).unwrap(); + println!("{COMPLETE}"); + return; + } + + let root = TempDir::new().unwrap(); + let destination = root.path().join("sessions.db"); + let stage = InitializationDirectory::begin(destination).unwrap(); + seed(&stage.database(), 7); + let foreign = root.path().join("foreign"); + fs::create_dir(&foreign).unwrap(); + let foreign_database = foreign.join("schema.sqlite"); + seed(&foreign_database, 99); + let foreign_main = fs::read(&foreign_database).unwrap(); + fs::write(foreign.join("prepared.sqlite"), b"foreign prepared").unwrap(); + for suffix in ["-wal", "-shm", "-journal"] { + fs::write( + with_suffix(&stage.database(), suffix), + b"owned staging sidecar", + ) + .unwrap(); + fs::write( + with_suffix(&foreign_database, suffix), + format!("foreign {suffix}"), + ) + .unwrap(); + } + + let (_, module) = module_path!().split_once("::").unwrap(); + let output = Command::new(std::env::current_exe().unwrap()) + .arg("--exact") + .arg(format!( + "{module}::failed_initialization_cleanup_uses_pinned_directory_after_swap" + )) + .arg("--nocapture") + .env(SWAP_ROOT, root.path()) + .output() + .unwrap(); + assert!( + output.status.success() && String::from_utf8_lossy(&output.stdout).contains(COMPLETE), + "cross-process stage swap did not execute: {output:?}" + ); + + // Dropping a failed schema initialization is the production cleanup + // caller. Its directory descriptor still points to retired-stage. + drop(stage); + let retired = root.path().join("retired-stage"); + assert!(retired.join("schema.sqlite").is_file()); + for suffix in ["-wal", "-shm", "-journal"] { + assert_eq!( + fs::read(with_suffix(&retired.join("schema.sqlite"), suffix)).unwrap(), + b"owned staging sidecar" + ); + assert_eq!( + fs::read(with_suffix(&foreign_database, suffix)).unwrap(), + format!("foreign {suffix}").as_bytes() + ); + } + assert_eq!(fs::read(&foreign_database).unwrap(), foreign_main); + assert_eq!( + fs::read(foreign.join("prepared.sqlite")).unwrap(), + b"foreign prepared" + ); + } + + #[test] + fn retiring_a_directory_requires_its_own_pin() { + let root = TempDir::new().unwrap(); + let destination = root.path().join("sessions.db"); + let mut stage = InitializationDirectory::begin(destination).unwrap(); + let other = root.path().join("another-empty-directory"); + create_private_directory(&other).unwrap(); + let original = std::mem::replace(&mut stage.directory, other.clone()); + let error = stage.retire().unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(other.is_dir()); + assert!(original.is_dir()); + stage.directory = original.clone(); + drop(stage); + assert!(!original.exists()); + assert!(other.is_dir()); + } + + #[test] + fn offline_reset_preserves_the_initialization_lease_and_refuses_unresolved_staging() { + let root = TempDir::new().unwrap(); + let destination = root.path().join("sessions.db"); + assert!( + lock_offline_sqlite_family_for_reset(&destination) + .unwrap() + .is_none() + ); + assert_eq!(fs::read_dir(root.path()).unwrap().count(), 0); + let publisher = InitializationDirectory::begin(destination.clone()).unwrap(); + assert!(lock_offline_sqlite_family_for_reset(&destination).is_err()); + drop(publisher); + let reset = lock_offline_sqlite_family_for_reset(&destination) + .unwrap() + .unwrap(); + assert!(InitializationDirectory::begin(destination.clone()).is_err()); + drop(reset); + let staging = InitializationDirectory::path(&destination); + create_private_directory(&staging).unwrap(); + let unrelated = staging.join("operator-data"); + fs::write(&unrelated, b"keep").unwrap(); + assert!(InitializationDirectory::verify_read_only_handoff(&destination).is_err()); + assert!(lock_offline_sqlite_family_for_reset(&destination).is_err()); + assert_eq!(fs::read(unrelated).unwrap(), b"keep"); + } + + #[test] + fn competing_initialization_leaves_owner_and_foreign_sidecars_intact() { + let root = TempDir::new().unwrap(); + let destination = root.path().join("sessions.db"); + let stage = InitializationDirectory::begin(destination.clone()).unwrap(); + seed(&stage.database(), 4); + assert!(InitializationDirectory::begin(destination.clone()).is_err()); + assert!(stage.database().exists()); + drop(stage); + let destination = root.path().join("other.db"); + for suffix in SIDECARS { + let foreign = with_suffix(&destination, suffix); + fs::write(&foreign, b"pre-existing sidecar").unwrap(); + assert!(InitializationDirectory::begin(destination.clone()).is_err()); + assert_eq!(fs::read(&foreign).unwrap(), b"pre-existing sidecar"); + assert!(!destination.exists()); + assert!(!InitializationDirectory::path(&destination).exists()); + fs::remove_file(foreign).unwrap(); + } + } +} diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization/interrupted.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization/interrupted.rs new file mode 100644 index 00000000000..f04c1b83fd6 --- /dev/null +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization/interrupted.rs @@ -0,0 +1,199 @@ +//! Retain a joined, interrupted schema for the next authorized installer. +use std::fs::File; +use std::io; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use tracedecay_private_fs::capability_dir::{self, Dir}; +use tracedecay_private_fs::{create_private_directory, open_private_directory}; + +use super::{ + InitializationDirectory, InitializationFiles, LeafIdentity, PreparedFamily, SIDECARS, + STAGED_MEMBERS, ambiguous, leaf, present, present_at, require_schema_members, sha256_file, + surviving_sidecars, verify_named_directory, +}; +use crate::sqlite_read_snapshot::with_suffix; + +/// Only a joined, handled failure can deposit the original members under +/// the lease's private recovery names. No alias or backup is created. +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub(super) struct InterruptedInstall { + locator: [u8; 32], + nonce: [u8; 16], + sidecar_digests: [[u8; 32]; 3], +} + +impl InterruptedInstall { + pub(super) fn directory(&self, destination: &Path) -> PathBuf { + destination.with_file_name(format!( + ".tracedecay-schema-{:032x}", + u128::from_le_bytes(self.nonce) + )) + } +} + +impl InitializationDirectory { + pub(crate) fn hold_interrupted( + &mut self, + runtime_identity: u64, + locator: [u8; 32], + (main, sidecars): InitializationFiles, + ) -> io::Result<()> { + self.verify_directory_name()?; + let stage = self.stage_cap()?; + require_schema_members(&stage)?; + let [wal, shm, journal] = surviving_sidecars(sidecars)?; + let member = |name: &str, original: Option| -> io::Result { + if present_at(&stage, name.as_ref())? { + let file = + original.ok_or_else(|| ambiguous("SQLite member lost its original handle"))?; + LeafIdentity::from_file(&file)?.pin_at(&stage, name.as_ref(), false)?; + Ok(file) + } else { + capability_dir::create_private_file(&stage, name.as_ref()) + } + }; + LeafIdentity::from_file(&main)?.pin_at(&stage, STAGED_MEMBERS[0].as_ref(), false)?; + let files = [ + main, + member(STAGED_MEMBERS[1], wal)?, + member(STAGED_MEMBERS[2], shm)?, + member(STAGED_MEMBERS[3], journal)?, + ]; + let [main, wal, shm, journal] = files.each_ref().map(LeafIdentity::from_file); + let mut nonce = [0; 16]; + getrandom::fill(&mut nonce).map_err(|error| io::Error::other(error.to_string()))?; + let [d0, d1, d2, d3] = files.each_ref().map(sha256_file); + let retry = InterruptedInstall { + locator, + nonce, + sidecar_digests: [d1?, d2?, d3?], + }; + let recovery = retry.directory(&self.destination); + if present(&recovery)? { + return Err(ambiguous("SQLite recovery directory is already occupied")); + } + let family = PreparedFamily { + retry: Some(retry), + runtime_identity, + main_sha256: d0?, + main: main?, + sidecars: [wal?, shm?, journal?], + }; + let mut pins = Vec::new(); + for (name, file) in STAGED_MEMBERS.into_iter().zip(&files) { + file.sync_all()?; + pins.push(LeafIdentity::from_file(file)?.pin_at(&stage, name.as_ref(), false)?); + } + // Windows keeps the live worker's main handle non-delete-sharing. + // Its successor pins are already open and match that original handle. + drop(files); + self.record(family)?; + create_private_directory(&recovery)?; + let recovery = Dir::from_std_file(open_private_directory(&recovery)?); + for (name, original) in STAGED_MEMBERS.into_iter().zip(&pins) { + capability_dir::rename_noreplace(&stage, name.as_ref(), &recovery, name.as_ref())?; + LeafIdentity::from_file(original)?.pin_at(&recovery, name.as_ref(), false)?; + } + capability_dir::sync_directory(&recovery)?; + capability_dir::sync_directory(&self.parent)?; + capability_dir::sync_directory(&stage) + } + + pub(crate) fn resume_interrupted( + &mut self, + locator: [u8; 32], + ) -> io::Result> { + let Some(family) = &self.prepared else { + return Ok(None); + }; + let Some(retry) = &family.retry else { + return Ok(None); + }; + if retry.locator != locator { + return Err(ambiguous( + "interrupted SQLite install belongs to a different store locator", + )); + } + self.verify_directory_name()?; + for suffix in SIDECARS { + if present_at(&self.parent, leaf(&with_suffix(&self.destination, suffix))?)? { + return Err(ambiguous( + "canonical SQLite sidecar is occupied during retry", + )); + } + } + let stage = self.stage_cap()?; + let identities = [ + &family.main, + &family.sidecars[0], + &family.sidecars[1], + &family.sidecars[2], + ]; + let recovery_path = retry.directory(&self.destination); + require_schema_members(&stage)?; + let recovery = match open_private_directory(&recovery_path) { + Ok(pin) => Some(Dir::from_std_file(pin)), + Err(error) if error.kind() == io::ErrorKind::NotFound => None, + Err(error) => return Err(error), + }; + if let Some(directory) = &recovery { + require_schema_members(directory)?; + } + // After a crash a file may still occupy the active SQLite namespace. + // A byte digest cannot prove an allocation (all empty WALs hash alike). + // Only the sheltered name, or a kernel generation in the active stage, + // can authorize restoration. Inspect the whole family before moving it. + let [wal, shm, journal] = &retry.sidecar_digests; + let digests = [&family.main_sha256, wal, shm, journal]; + let pin = |index: usize| -> io::Result<(File, bool)> { + let name = STAGED_MEMBERS[index].as_ref(); + let held = match &recovery { + Some(directory) if present_at(directory, name)? => Some(directory), + _ => None, + }; + let (directory, move_back) = match (present_at(&stage, name)?, held) { + (false, Some(directory)) => (directory, true), + (true, None) if identities[index].allocation.is_some() => (&stage, false), + _ => { + return Err(ambiguous( + "SQLite recovery member is missing, duplicated, or lacks its allocation generation", + )); + } + }; + let file = capability_dir::open_writable_regular_nofollow(directory, name)?; + if LeafIdentity::from_file(&file)? != *identities[index] + || sha256_file(&file)? != *digests[index] + { + return Err(ambiguous( + "interrupted SQLite member no longer matches its held contents", + )); + } + Ok((file, move_back)) + }; + let files = [pin(0)?, pin(1)?, pin(2)?, pin(3)?]; + for (index, (_, move_back)) in files.iter().enumerate() { + let name = STAGED_MEMBERS[index].as_ref(); + if *move_back { + let directory = recovery + .as_ref() + .ok_or_else(|| ambiguous("SQLite recovery directory disappeared"))?; + capability_dir::rename_noreplace(directory, name, &stage, name)?; + } + identities[index].pin_at(&stage, name, false)?; + } + if let Some(directory) = recovery { + capability_dir::sync_directory(&directory)?; + verify_named_directory(&directory.into_std_file(), &recovery_path)?; + self.parent.remove_dir(leaf(&recovery_path)?)?; + } + capability_dir::sync_directory(&stage)?; + capability_dir::sync_directory(&self.parent)?; + // Keep the receipt until the succeeding install replaces it. A crash + // after the empty directory was removed can still prove eligible files. + let files = files.map(|(file, _)| file); + let [main, wal, shm, journal] = files; + Ok(Some((main, [wal, shm, journal]))) + } +} diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization/tests/handoff.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization/tests/handoff.rs new file mode 100644 index 00000000000..298297760db --- /dev/null +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization/tests/handoff.rs @@ -0,0 +1,263 @@ +//! Admission at the publication event must not inherit any raw SQLite pins. +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use rusqlite::{Connection, Error, ErrorCode, OpenFlags}; +use tempfile::TempDir; +use tracing::{Event, Metadata, Subscriber, span}; + +use super::{InitializationDirectory, SIDECARS, owned_files, seed, with_suffix}; +use crate::logging::set_tracing_capture; + +const HANDOFF: &str = "sqlite.initialization.handoff"; +const DATABASE: &str = "TRACEDECAY_HANDOFF_DATABASE"; +const JOURNAL: &str = "TRACEDECAY_HANDOFF_JOURNAL"; +const BLOCKED: &str = "TRACEDECAY_HANDOFF_BLOCKED"; +const COMPLETE: &str = "tracedecay-handoff-probe-complete"; + +// Use a real connection at the completed-publication telemetry boundary. This +// deterministic schedule needs no sleeps or test-only production callbacks. +struct ConnectionAtHandoff { + path: PathBuf, + journal: &'static str, + writer: bool, + connection: Arc>>, +} + +impl Subscriber for ConnectionAtHandoff { + fn enabled(&self, metadata: &Metadata<'_>) -> bool { + metadata.name() == HANDOFF + } + + fn new_span(&self, _attributes: &span::Attributes<'_>) -> span::Id { + span::Id::from_u64(1) + } + + fn record(&self, _span: &span::Id, _values: &span::Record<'_>) {} + + fn record_follows_from(&self, _span: &span::Id, _follows: &span::Id) {} + + fn enter(&self, _span: &span::Id) {} + + fn exit(&self, _span: &span::Id) {} + + fn event(&self, event: &Event<'_>) { + if event.metadata().name() != HANDOFF { + return; + } + InitializationDirectory::verify_read_only_handoff(&self.path).unwrap(); + let flags = if self.writer { + OpenFlags::SQLITE_OPEN_READ_WRITE + } else { + OpenFlags::SQLITE_OPEN_READ_ONLY + }; + let connection = Connection::open_with_flags(&self.path, flags).unwrap(); + connection.busy_timeout(Duration::ZERO).unwrap(); + connection + .execute_batch(if self.writer { + "BEGIN IMMEDIATE" + } else { + "BEGIN DEFERRED" + }) + .unwrap(); + assert_eq!(value(&connection), 7); + if self.writer { + connection + .execute_batch("UPDATE data SET value = 17") + .unwrap(); + } + probe(&self.path, self.journal, self.writer, true); + let mut slot = self.connection.lock().unwrap(); + assert!(slot.is_none(), "handoff must occur exactly once"); + *slot = Some(connection); + } +} + +fn probe(path: &Path, journal: &str, writer: bool, blocked: bool) { + let (_, module) = module_path!().split_once("::").unwrap(); + let output = Command::new(env::current_exe().unwrap()) + .args([ + "--exact", + &format!("{module}::external_handoff_probe"), + "--nocapture", + ]) + .env(DATABASE, path) + .env(JOURNAL, if writer { "writer" } else { journal }) + .env(BLOCKED, blocked.to_string()) + .output() + .unwrap(); + assert!( + output.status.success() && String::from_utf8_lossy(&output.stdout).contains(COMPLETE), + "external SQLite {journal} probe did not complete: {output:?}" + ); +} + +#[test] +fn external_handoff_probe() { + let Some(path) = env::var_os(DATABASE) else { + return; + }; + let journal = env::var(JOURNAL).unwrap(); + let blocked: bool = env::var(BLOCKED).unwrap().parse().unwrap(); + let writer = + Connection::open_with_flags(PathBuf::from(path), OpenFlags::SQLITE_OPEN_READ_WRITE) + .unwrap(); + writer.busy_timeout(Duration::ZERO).unwrap(); + assert_eq!(value(&writer), 7); + let result = if journal == "writer" { + writer.execute_batch("BEGIN IMMEDIATE; ROLLBACK") + } else if journal == "WAL" { + writer + .query_row("PRAGMA journal_mode = DELETE", [], |row| { + row.get::<_, String>(0) + }) + .map(|mode| assert_eq!(mode, "delete")) + } else { + writer.execute_batch("BEGIN IMMEDIATE; UPDATE data SET value = 99; COMMIT") + }; + if blocked { + assert!( + matches!( + &result, + Err(Error::SqliteFailure(error, _)) if error.code == ErrorCode::DatabaseBusy + ), + "the admitted connection must retain its OS lock: {result:?}" + ); + } else { + result.expect("write after the connection releases its snapshot"); + } + println!("{COMPLETE}"); +} + +fn value(connection: &Connection) -> i64 { + connection + .query_row("SELECT value FROM data", [], |row| row.get(0)) + .unwrap() +} + +#[derive(Clone, Copy, Debug)] +enum Publication { + Fresh, + Prepared, + RecoverPublished, +} + +fn check_handoff(publication: Publication, writer: bool) { + let journals: &[&str] = if writer { &["WAL"] } else { &["DELETE", "WAL"] }; + for &journal in journals { + let root = TempDir::new().unwrap(); + let destination = root.path().join("sessions.db"); + let mut stage = InitializationDirectory::begin(destination.clone()).unwrap(); + seed(&stage.database(), 7); + let database = Connection::open(stage.database()).unwrap(); + let mode: String = database + .query_row(&format!("PRAGMA journal_mode = {journal}"), [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(mode, journal.to_ascii_lowercase()); + drop(database); + let identity = crate::db::sqlite_generation_identity(&stage.database()).unwrap(); + let files = owned_files(&stage); + let connection = Arc::new(Mutex::new(None)); + let capture = set_tracing_capture(ConnectionAtHandoff { + path: destination.clone(), + journal, + writer, + connection: Arc::clone(&connection), + }); + match publication { + Publication::Fresh => stage.publish(identity, files).unwrap(), + Publication::Prepared => { + // A prepared resume no longer has the creator's descriptors; + // its reopened verification pins must obey the same handoff. + drop(stage.prepare(identity, files).unwrap()); + assert!(InitializationDirectory::verify_read_only_handoff(&destination).is_err()); + assert_eq!(stage.publish_prepared(Vec::new()).unwrap(), identity); + drop(stage); + } + Publication::RecoverPublished => { + let pins = stage.prepare(identity, files).unwrap(); + for suffix in SIDECARS { + fs::rename(stage.reservation(suffix), with_suffix(&destination, suffix)) + .unwrap(); + } + fs::rename(stage.database(), &destination).unwrap(); + let durable = stage.prepared.as_ref().unwrap().has_durable_generations(); + drop(pins); + drop(stage); + let result = InitializationDirectory::recover_published(&destination); + if !durable { + // Do not fabricate generation support for APFS/overlayfs. + // Those filesystems must preserve the unresolved receipt. + assert!(result.is_err()); + assert!(connection.lock().unwrap().is_none()); + assert!( + InitializationDirectory::verify_read_only_handoff(&destination).is_err() + ); + let database = Connection::open(&destination).unwrap(); + assert_eq!(value(&database), 7); + continue; + } + result.unwrap(); + } + } + drop(capture); + let connection = connection + .lock() + .unwrap() + .take() + .expect("the publication event must admit a real read snapshot"); + // Old drop ordering loses this connection's process-owned main/SHM locks + // after announcing the handoff, even though its transaction is live. + probe(&destination, journal, writer, true); + assert_eq!(value(&connection), if writer { 17 } else { 7 }); + // Roll back so the child can prove ordinary acquisition after release + // against the original committed value in both access modes. + connection.execute_batch("ROLLBACK").unwrap(); + drop(connection); + probe(&destination, journal, writer, false); + let reopened = Connection::open(&destination).unwrap(); + reopened + .execute_batch("UPDATE data SET value = 23") + .unwrap(); + drop(reopened); + let reopened = + Connection::open_with_flags(&destination, OpenFlags::SQLITE_OPEN_READ_ONLY).unwrap(); + assert_eq!(value(&reopened), 23); + let integrity: String = reopened + .query_row("PRAGMA integrity_check", [], |row| row.get(0)) + .unwrap(); + assert_eq!(integrity, "ok"); + } +} + +#[test] +fn fresh_publication_handoff_preserves_read_locks() { + check_handoff(Publication::Fresh, false); +} + +#[test] +fn prepared_publication_handoff_preserves_read_locks() { + check_handoff(Publication::Prepared, false); +} + +#[test] +fn published_recovery_preserves_read_locks_or_refuses_unproven_generation() { + check_handoff(Publication::RecoverPublished, false); +} + +#[test] +fn publication_handoff_preserves_independent_wal_writer_lock() { + for publication in [ + Publication::Fresh, + Publication::Prepared, + Publication::RecoverPublished, + ] { + check_handoff(publication, true); + } +} diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs index 1a951d8b2c9..5e40672c251 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs @@ -1,10 +1,12 @@ use std::future::Future; -use std::path::PathBuf; +use std::ops::ControlFlow; +use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::Arc; use std::time::Duration; use asupersync::runtime::RuntimeHandle; +use sha2::{Digest, Sha256}; use tracedecay_rusqlite_runtime::reader::ReaderAcquireError; use tracedecay_rusqlite_runtime::repository::{ @@ -17,6 +19,7 @@ use tracedecay_store::{ StoreShardScopeV1, VerifiedStoreLocatorV1, }; +use super::initialization::InitializationDirectory; use super::{ PhysicalRuntimeAttachment, PhysicalRuntimeSnapshot, PhysicalWriterRuntimeSnapshot, PublishedShardRuntime, StoreRuntimeAccessMode, StoreRuntimeKey, StoreRuntimeOpenMode, @@ -141,36 +144,202 @@ impl ShardRuntimePublisher for LifecycleShardRuntimePublisher { operation: "publish physical registered runtime", }, )?; - // Attaching opens the file and parks its thread until the writer - // and readers have opened it on the storage pool. - let attach_request = request.clone(); - let physical = - crate::native_runtime::spawn_blocking(move || { - LifecycleShardRuntimeAttachment::new(RepositoryPhysicalAttachmentFactory) - .attach(&executor, &attach_request, AdmissionConfigV1::default()) - }) - .await - .map_err(|message| { - StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { - operation: "attach rusqlite repository runtime", - message, - } - })??; - publish_lifecycle_runtime(request, runtime, physical).await + let published_identity = if request.locator.is_prospective() { + Some( + initialize_staged_runtime(&executor, &request, AdmissionConfigV1::default()) + .await?, + ) + } else { + None + }; + let attachment = attach_runtime(&executor, &request, published_identity).await?; + publish_lifecycle_runtime(request, runtime, attachment).await }) } } +fn physical_failure( + operation: &'static str, + error: impl std::fmt::Display, +) -> StoreRuntimeRegistryFailure { + StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { + operation, + message: error.to_string(), + } +} + +async fn attach_runtime( + executor: &RuntimeHandle, + request: &ShardRuntimeBuildRequest, + published_identity: Option, +) -> Result { + let executor = executor.clone(); + let request = request.clone(); + crate::native_runtime::spawn_blocking(move || { + const OPERATION: &str = "attach published SQLite generation"; + if let Some(expected) = published_identity { + let current = crate::db::sqlite_generation_identity(request.locator.path()) + .map_err(|error| physical_failure(OPERATION, error))?; + if current != expected { + return Err(physical_failure( + OPERATION, + "published database identity changed", + )); + } + } + let attachment = LifecycleShardRuntimeAttachment::new(RepositoryPhysicalAttachmentFactory) + .attach(&executor, &request, AdmissionConfigV1::default())?; + if published_identity + .is_some_and(|expected| expected != attachment.0.opened_file_identity()) + { + attachment + .0 + .drain() + .and_then(|()| attachment.0.close_and_join()) + .map_err(|error| physical_failure(OPERATION, error))?; + return Err(physical_failure( + OPERATION, + "published database changed during attachment", + )); + } + Ok(attachment) + }) + .await + .map_err(|error| physical_failure("attach SQLite runtime", error))? +} + +pub(super) async fn initialize_staged_runtime( + executor: &RuntimeHandle, + request: &ShardRuntimeBuildRequest, + admission: AdmissionConfigV1, +) -> Result { + const OPERATION: &str = "publish initialized SQLite family"; + let authority = request + .database_authority + .as_ref() + .ok_or_else(|| physical_failure(OPERATION, "missing database authority"))?; + if request.mode != StoreRuntimeOpenMode::Initialize + || authority.canonical_database_path() != request.locator.path() + { + return Err(physical_failure( + OPERATION, + "initialization does not own this destination", + )); + } + authority + .require_active_write_scope(OPERATION) + .map_err(|error| physical_failure(OPERATION, error))?; + if request.open_cancellation.is_cancelled() { + return Err(StoreRuntimeRegistryFailure::OpenCancelled { + key: Box::new(StoreRuntimeKey::from_binding(&request.binding)), + }); + } + let destination = request.locator.path().to_owned(); + let locator_digest = Sha256::digest( + serde_json::to_vec(request.locator.verified()) + .map_err(|error| physical_failure(OPERATION, error))?, + ) + .into(); + let executor = executor.clone(); + let binding = request.binding.clone(); + let locator = request.locator.verified().clone(); + let started = crate::native_runtime::spawn_blocking(move || -> std::io::Result<_> { + let mut stage = InitializationDirectory::begin(destination)?; + if let Some(identity) = stage.resume_prepared()? { + return Ok(ControlFlow::Break(identity)); + } + let interrupted = stage.resume_interrupted(locator_digest)?; + let attachment = RepositoryPhysicalAttachmentFactory + .start_reserved_initialization( + &executor, + binding, + locator, + stage.database(), + admission, + interrupted, + ) + .map_err(|failure| { + if let Some((identity, files)) = failure.files + && let Err(hold) = stage.hold_interrupted(identity, locator_digest, files) + { + return std::io::Error::other(format!( + "{}; preserving failed startup: {hold}", + failure.error + )); + } + std::io::Error::other(failure.error) + })?; + Ok(ControlFlow::Continue((stage, attachment))) + }) + .await + .map_err(|error| physical_failure(OPERATION, error))? + .map_err(|error| physical_failure(OPERATION, error))?; + let (mut stage, attachment) = match started { + ControlFlow::Continue(started) => started, + ControlFlow::Break(already_published) => return Ok(already_published), + }; + let identity = attachment.opened_file_identity(); + let attachment = Arc::new(attachment); + let installation = install_final_schema_before_publication( + request, + attachment.as_ref(), + &stage.database(), + Some(Arc::clone(&attachment)), + ) + .await; + let files = + crate::native_runtime::spawn_blocking(move || attachment.close_pinned_initialization()) + .await + .and_then(|outcome| outcome) + .map_err(|error| physical_failure(OPERATION, error))?; + let installation = installation.and_then(|()| { + authority + .require_active_write_scope(OPERATION) + .map_err(|error| physical_failure(OPERATION, error))?; + if request.open_cancellation.is_cancelled() { + return Err(StoreRuntimeRegistryFailure::OpenCancelled { + key: Box::new(StoreRuntimeKey::from_binding(&request.binding)), + }); + } + Ok(()) + }); + crate::native_runtime::spawn_blocking(move || { + if let Err(error) = installation { + stage + .hold_interrupted(identity, locator_digest, files) + .map_err(|hold| { + physical_failure( + OPERATION, + format!("{error:?}; preserving interrupted schema: {hold}"), + ) + })?; + return Err(error); + } + stage + .publish(identity, files) + .map_err(|error| physical_failure(OPERATION, error)) + }) + .await + .map_err(|error| physical_failure(OPERATION, error))??; + Ok(identity) +} + async fn publish_lifecycle_runtime( request: ShardRuntimeBuildRequest, runtime: ShardRuntime, attachment: LifecyclePhysicalAttachment, ) -> Result { if request.mode == StoreRuntimeOpenMode::Initialize - && let Err(error) = - install_final_schema_before_publication(&request, attachment.as_physical()).await + && !request.locator.is_prospective() + && let Err(error) = install_final_schema_before_publication( + &request, + attachment.as_physical(), + request.locator.path(), + None, + ) + .await { - let _ = attachment.abort(request.locator.is_prospective()).await; + let _ = attachment.close().await; return Err(error); } if request.mode == StoreRuntimeOpenMode::Existing @@ -179,7 +348,7 @@ async fn publish_lifecycle_runtime( verify_final_schema_before_existing_publication(&request, attachment.as_physical()) .await { - let _ = attachment.abort(request.locator.is_prospective()).await; + let _ = attachment.close().await; return Err(error); } if request.mode == StoreRuntimeOpenMode::Existing @@ -190,30 +359,18 @@ async fn publish_lifecycle_runtime( { // This failed attachment must finish closing before the registry can // retry at a new epoch. - attachment - .abort(request.locator.is_prospective()) - .await - .map_err( - |message| StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { - operation: "close runtime after writer authority refusal", - message, - }, - )?; + attachment.close().await.map_err(|message| { + StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { + operation: "close runtime after writer authority refusal", + message, + } + })?; return Err(error); } if let Err(error) = runtime.transition(RuntimeMaintenanceStateV1::Ready) { - let _ = attachment.abort(request.locator.is_prospective()).await; + let _ = attachment.close().await; return Err(runtime_lifecycle_failure(error)); } - if request.locator.is_prospective() - && let Err(error) = attachment.commit_initialization() - { - let _ = attachment.abort(true).await; - return Err(StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { - operation: "commit initialized SQLite runtime", - message: error, - }); - } Ok(PublishedShardRuntime::new(runtime, attachment.into_box())) } @@ -314,21 +471,15 @@ impl LifecycleShardRuntimeAttachment { request: &ShardRuntimeBuildRequest, admission: AdmissionConfigV1, ) -> Result { - if request.locator.is_prospective() && request.mode != StoreRuntimeOpenMode::Initialize { - return Err(StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { - operation: "open prospective SQLite runtime", - message: "prospective locators require explicit initialization".to_owned(), - }); + let path = request.locator.path(); + match request.access { + StoreRuntimeAccessMode::ReadOnly => { + InitializationDirectory::verify_read_only_handoff(path) + } + StoreRuntimeAccessMode::ReadWrite => InitializationDirectory::recover_published(path), } - let attachment = if request.locator.is_prospective() { - self.repository.initialize( - runtime, - request.binding.clone(), - request.locator.verified().clone(), - request.locator.path().to_path_buf(), - admission, - ) - } else if request.access == StoreRuntimeAccessMode::ReadOnly { + .map_err(|error| physical_failure("verify initialized SQLite family", error))?; + let attachment = if request.access == StoreRuntimeAccessMode::ReadOnly { self.repository.attach_read_only( request.binding.clone(), request.locator.verified().clone(), @@ -357,20 +508,12 @@ impl LifecyclePhysicalAttachment { &self.0 } - fn commit_initialization(&self) -> Result<(), String> { - self.0.commit_initialization() - } - /// Closes an attachment that will not be published. Joining its writer /// and readers parks the calling thread. - async fn abort(self, created: bool) -> Result<(), String> { + async fn close(self) -> Result<(), String> { crate::native_runtime::spawn_blocking(move || { - if created { - self.0.abort_initialization() - } else { - self.0.drain()?; - self.0.close_and_join() - } + self.0.drain()?; + self.0.close_and_join() }) .await .and_then(|closed| closed) @@ -428,6 +571,7 @@ async fn authorized_schema_connection( request: &ShardRuntimeBuildRequest, attachment: &dyn PhysicalRuntimeAttachment, operation: &'static str, + physical_path: &Path, ) -> Result { let authority = request.database_authority.clone().ok_or_else(|| { StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { @@ -462,7 +606,7 @@ async fn authorized_schema_connection( }); } let authority = InitializingSchemaAuthority { - canonical_path: authority.canonical_database_path().to_path_buf(), + canonical_path: physical_path.to_path_buf(), authority, opened_file_identity, }; @@ -478,11 +622,14 @@ async fn authorized_schema_connection( async fn install_final_schema_before_publication( request: &ShardRuntimeBuildRequest, attachment: &dyn PhysicalRuntimeAttachment, + physical_path: &Path, + prospective_attachment: Option>, ) -> Result<(), StoreRuntimeRegistryFailure> { let connection = authorized_schema_connection( request, attachment, "install final schema for initialized SQLite runtime", + physical_path, ) .await?; match &request.binding.shard_id.scope { @@ -500,7 +647,7 @@ async fn install_final_schema_before_publication( | StoreShardScopeV1::ProfileSessions | StoreShardScopeV1::ProjectSessions { .. } => { crate::ports::registered_schema::install_from_authorized_connection( - connection, + connection.clone(), request.open_cancellation.clone(), ) .await @@ -527,6 +674,25 @@ async fn install_final_schema_before_publication( })?; } } + if request.locator.is_prospective() { + // Publication needs no live reader snapshots of the private staging + // database. Retire both SQLite reader lanes and join their workers + // before the still-open serialized writer checkpoints the WAL. + let prospective = prospective_attachment.ok_or_else(|| { + physical_failure( + "quiesce prospective SQLite readers", + "prospective attachment authority is missing", + ) + })?; + crate::native_runtime::spawn_blocking(move || prospective.close_initialization_readers()) + .await + .map_err(|error| physical_failure("quiesce prospective SQLite readers", error))? + .map_err(|error| physical_failure("quiesce prospective SQLite readers", error))?; + connection + .finish_initial_schema() + .await + .map_err(|error| physical_failure("finish prospective SQLite schema", error))?; + } Ok(()) } diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs index 7d3a10fd52e..7fe780a26c8 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs @@ -196,7 +196,10 @@ fn health_request( fn seed_db(root: &TempDir, name: &str) -> PathBuf { let path = root.path().join(name); - Connection::open(&path).unwrap(); + Connection::open(&path) + .unwrap() + .execute_batch("PRAGMA user_version = 0") + .unwrap(); path.canonicalize().unwrap() } @@ -290,7 +293,12 @@ async fn read_only_open_mounts_readers_without_acquiring_a_writer() { let root = TempDir::new().unwrap(); let resolver = Arc::new(FileResolver::default()); resolver.push(seed_final_graph_db(&root, "profile.db").await); - resolver.push(seed_final_graph_db(&root, "project.db").await); + let database = seed_final_graph_db(&root, "project.db").await; + let leftover = database.with_file_name("project.db.tracedecay-init"); + tracedecay_private_fs::create_private_directory(&leftover).unwrap(); + let owner = leftover.join("schema-installation.owner"); + drop(tracedecay_private_fs::create_private_file(&owner).unwrap()); + resolver.push(database); let registry = StoreRuntimeRegistry::new( asupersync::runtime::Runtime::current_handle().unwrap(), resolver, @@ -324,96 +332,101 @@ async fn read_only_open_mounts_readers_without_acquiring_a_writer() { assert!(!physical.writer_present); assert!(physical.reader_handles > 0); assert_health_route(&project, false).await; + assert!(owner.is_file()); + assert!( + !root + .path() + .join(".project.db.tracedecay-init.durable-directory.lock") + .exists() + ); } -/// The store file is a FIFO, so opening it waits until a writer opens the -/// other end. Only a task on the runtime's single worker does that while the -/// open waits; a fallback thread ends a wait that parked the worker, so the -/// failure is an assertion rather than a hang. #[cfg(unix)] #[asupersync::test] -async fn store_open_keeps_the_async_worker_serving_while_the_file_opens() { - use std::fs::OpenOptions; - use std::os::unix::fs::OpenOptionsExt; - use std::sync::atomic::AtomicBool; - +async fn store_open_refuses_a_fifo_before_opening_it() { let root = TempDir::new().unwrap(); - let held = root.path().join("held.db"); + let fifo = root.path().join("held.db"); assert!( std::process::Command::new("mkfifo") - .arg(&held) + .arg(&fifo) .status() .unwrap() .success() ); - let held = held.canonicalize().unwrap(); let resolver = Arc::new(FileResolver::default()); - resolver.push(held.clone()); + resolver.push(fifo.canonicalize().unwrap()); let registry = StoreRuntimeRegistry::new( asupersync::runtime::Runtime::current_handle().unwrap(), resolver, Arc::new(LifecycleShardRuntimePublisher), ) .unwrap(); + let outcome = registry + .open(StoreRuntimeOpenRequest::new( + profile_shard(), + incarnation(), + None, + )) + .await; + assert!(matches!( + outcome, + StoreRuntimeOpenResult::Failed(StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { + message, .. + }) if message.contains("not a regular file") + )); +} + +#[test] +fn store_open_keeps_the_async_worker_serving_while_attachment_waits() { + let executor = asupersync::runtime::RuntimeBuilder::current_thread() + .blocking_threads(1, 1) + .enable_time() + .build() + .unwrap(); + executor.block_on(async { + let root = TempDir::new().unwrap(); + let resolver = Arc::new(FileResolver::default()); + resolver.push(seed_final_graph_db(&root, "profile.db").await); + let registry = StoreRuntimeRegistry::new( + asupersync::runtime::Runtime::current_handle().unwrap(), + resolver, + Arc::new(LifecycleShardRuntimePublisher), + ) + .unwrap(); - let (released_by_worker, fallback_stop) = std::sync::mpsc::channel::<()>(); - let fallback = { - let held = held.clone(); - std::thread::spawn(move || { - let parked = fallback_stop.recv_timeout(Duration::from_secs(10)).is_err(); - if parked { - drop(OpenOptions::new().write(true).open(&held)); - } - parked - }) - }; - let settled = AtomicBool::new(false); - let open = async { - let outcome = registry - .open(StoreRuntimeOpenRequest::new( - profile_shard(), - incarnation(), - None, - )) - .await; - settled.store(true, Ordering::SeqCst); - outcome - }; - let release = async { let cx = asupersync::Cx::current().unwrap(); - while !settled.load(Ordering::SeqCst) { - match OpenOptions::new() - .write(true) - .custom_flags(libc::O_NONBLOCK) - .open(&held) - { - Ok(_writer) => { - released_by_worker.send(()).unwrap(); - return true; - } - Err(error) => assert_eq!(error.raw_os_error(), Some(libc::ENXIO)), - } - asupersync::time::sleep(cx.now(), Duration::from_millis(1)).await; - } - false - }; - let (outcome, released) = asupersync::join!(open, release); - drop(released_by_worker); - let parked = fallback.join().unwrap(); + let pool = cx.blocking_pool_handle().unwrap(); + let (entered, mut ready) = asupersync::channel::oneshot::channel(); + let (release, held) = std::sync::mpsc::channel(); + let mut blocker = cx + .spawn_blocking(move |_| { + entered.send_blocking(()).unwrap(); + held.recv_timeout(Duration::from_secs(10)).is_ok() + }) + .unwrap(); + ready.recv(&cx).await.unwrap(); - assert!( - released && !parked, - "the store open parked the runtime's only worker until the fallback thread released the file" - ); - assert!( - matches!( - outcome, - StoreRuntimeOpenResult::Failed( - StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { .. } - ) - ), - "a FIFO is not a SQLite store: {outcome:?}" - ); + let open = open_published( + ®istry, + StoreRuntimeOpenRequest::new(profile_shard(), incarnation(), None), + ); + let serve = async { + asupersync::time::timeout(cx.now(), Duration::from_secs(10), async { + while pool.pending_count() == 0 { + asupersync::time::sleep(cx.now(), Duration::from_millis(1)).await; + } + }) + .await + .expect("physical attachment must queue while the pool is occupied"); + release.send(()).unwrap(); + }; + let (opened, ()) = asupersync::join!(open, serve); + assert!( + blocker.join(&cx).await.unwrap(), + "the async worker must release the pending attachment" + ); + assert_health_route(&opened, true).await; + }); } #[asupersync::test] @@ -541,6 +554,229 @@ async fn writable_existing_open_refuses_tampered_final_shape_before_publication( ); } +#[asupersync::test] +async fn cancelled_schema_publication_retries_with_the_final_store() { + let root = TempDir::new().unwrap(); + let path = root.path().canonicalize().unwrap().join("remote.db"); + let authority = + crate::db::DatabaseAuthority::acquire_test(&path, "retry remote initialization").unwrap(); + let shard = StoreShardIdV1::remote_node( + id::("brain.registry"), + id::("profile.registry"), + id::("node.remote"), + ); + let binding = StoreRuntimeBindingV1::new( + shard.clone(), + incarnation(), + StoreAuthorityEpochV1::new(1).unwrap(), + ); + let verified = VerifiedStoreLocatorV1::new( + shard, + incarnation(), + canonical_store_locator_digest(&path).unwrap(), + ); + let locator = RuntimeLocatorRecord::new( + StoreRuntimeKey::from_binding(&binding), + ResolvedStoreLocator::prospective(verified, path.clone()), + ); + let request = |cancellation| { + ShardRuntimeBuildRequest::new( + binding.clone(), + locator.clone(), + StoreRuntimeOpenMode::Initialize, + StoreRuntimeAccessMode::ReadWrite, + Some(authority.clone()), + cancellation, + ) + }; + let cancelled = CancellationToken::new(); + cancelled.cancel(); + assert!(matches!( + LifecycleShardRuntimePublisher + .publish(request(cancelled)) + .await, + Err(StoreRuntimeRegistryFailure::OpenCancelled { .. }) + )); + assert!(!path.exists()); + let published = LifecycleShardRuntimePublisher + .publish(request(CancellationToken::new())) + .await + .unwrap(); + let reader = + Connection::open_with_flags(&path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY).unwrap(); + let count: i64 = reader + .query_row("SELECT COUNT(*) FROM remote_node_identity", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(count, 0); + assert_eq!( + reader + .query_row("PRAGMA integrity_check", [], |row| row.get::<_, String>(0)) + .unwrap(), + "ok" + ); + drop(reader); + drop(published); +} + +#[asupersync::test] +async fn stopped_schema_installation_resumes_committed_steps_in_a_new_process() { + use crate::ports::registered_schema; + use std::process::Command; + use tracedecay_domain::errors::TraceDecayError; + + const PATH: &str = "TRACEDECAY_TEST_SCHEMA_RETRY_DATABASE"; + const PHASE: &str = "TRACEDECAY_TEST_SCHEMA_RETRY_PHASE"; + let Some(path) = std::env::var_os(PATH) else { + let name = format!( + "{}::stopped_schema_installation_resumes_committed_steps_in_a_new_process", + module_path!().split_once("::").unwrap().1 + ); + for first in ["cancel", "failure"] { + let root = TempDir::new().unwrap(); + let path = root.path().canonicalize().unwrap().join("sessions.db"); + for phase in ["fresh_start_error", first, "resumed_start_error", "resume"] { + let output = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", &name, "--nocapture"]) + .env(PATH, &path) + .env(PHASE, phase) + .output() + .unwrap(); + assert!( + output.status.success() + && String::from_utf8_lossy(&output.stdout).contains("1 passed"), + "{phase}: {} {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + let residues = std::fs::read_dir(root.path()) + .unwrap() + .filter_map(|entry| { + let name = entry.unwrap().file_name(); + name.to_string_lossy() + .starts_with(".tracedecay-schema-") + .then_some(name) + }) + .collect::>(); + assert!(residues.is_empty(), "{residues:?}"); + } + return; + }; + registered_schema::register(|installation| { + Box::pin(async move { + let phase = std::env::var(PHASE).unwrap(); + if phase == "resume" { + installation.execute_batch("CREATE TABLE completed(value INTEGER); INSERT INTO completed SELECT value FROM saved;").await.unwrap(); + Ok(()) + } else { + installation + .execute_batch( + "CREATE TABLE saved(value INTEGER); INSERT INTO saved VALUES (17);", + ) + .await + .unwrap(); + if phase == "cancel" { + installation.cancellation().cancel(); + Err(TraceDecayError::store_open_cancelled("registered schema")) + } else { + let error = installation + .execute_batch("INSERT INTO missing_table VALUES (42);") + .await + .unwrap_err(); + Err(TraceDecayError::Database { + operation: "incomplete registered schema".to_owned(), + message: error.to_string(), + }) + } + } + }) + }); + let path = PathBuf::from(path); + let authority = + crate::db::DatabaseAuthority::acquire_test(&path, "retry registered schema").unwrap(); + let shard = profile_shard(); + let binding = StoreRuntimeBindingV1::new( + shard.clone(), + incarnation(), + StoreAuthorityEpochV1::new(1).unwrap(), + ); + let verified = VerifiedStoreLocatorV1::new( + shard, + incarnation(), + canonical_store_locator_digest(&path).unwrap(), + ); + let locator = RuntimeLocatorRecord::new( + StoreRuntimeKey::from_binding(&binding), + ResolvedStoreLocator::prospective(verified, path.clone()), + ); + let request = ShardRuntimeBuildRequest::new( + binding, + locator, + StoreRuntimeOpenMode::Initialize, + StoreRuntimeAccessMode::ReadWrite, + Some(authority), + CancellationToken::new(), + ); + if matches!( + std::env::var(PHASE).unwrap().as_str(), + "fresh_start_error" | "resumed_start_error" + ) { + let mut admission = tracedecay_store::AdmissionConfigV1::default(); + admission.per_shard_queue.max_operations = 0; + let runtime = crate::native_runtime::handle().unwrap(); + let error = super::super::ports::initialize_staged_runtime(&runtime, &request, admission) + .await + .unwrap_err(); + assert!( + matches!(error, StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { ref message, .. } if message.contains("invalid writer admission")), + "{error:?}" + ); + assert!(!path.exists()); + return; + } + let published = LifecycleShardRuntimePublisher.publish(request).await; + match std::env::var(PHASE).unwrap().as_str() { + "cancel" => assert!( + matches!( + published, + Err(StoreRuntimeRegistryFailure::OpenCancelled { .. }) + ), + "{:?}", + published.err() + ), + "failure" => assert!( + matches!(published, Err(StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { ref message, .. }) if message.contains("missing_table")), + "{:?}", + published.err() + ), + "resume" => { + let runtime = published.unwrap(); + let reader = + Connection::open_with_flags(&path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY) + .unwrap(); + assert_eq!( + reader + .query_row("SELECT value FROM completed", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 17 + ); + assert_eq!( + reader + .query_row("PRAGMA integrity_check", [], |row| row.get::<_, String>(0)) + .unwrap(), + "ok" + ); + drop(runtime); + return; + } + phase => panic!("unexpected schema test phase {phase}"), + } + assert!(!path.exists()); +} + #[asupersync::test] async fn remote_node_initialization_installs_only_the_final_registered_schema() { let root = TempDir::new().unwrap(); diff --git a/crates/tracedecay-runtime-core/src/sqlite_read_snapshot.rs b/crates/tracedecay-runtime-core/src/sqlite_read_snapshot.rs index 66b2124ba13..2fcc70ee3da 100644 --- a/crates/tracedecay-runtime-core/src/sqlite_read_snapshot.rs +++ b/crates/tracedecay-runtime-core/src/sqlite_read_snapshot.rs @@ -302,7 +302,7 @@ fn backup_offline_wal_family_without_shm( Ok(true) } -fn fold_staging_to_standalone( +pub(crate) fn fold_staging_to_standalone( staging: Connection, staging_path: &Path, checkpoint: impl Fn() -> io::Result<()>, diff --git a/crates/tracedecay-runtime-core/src/storage.rs b/crates/tracedecay-runtime-core/src/storage.rs index f44691d2f42..75cca308c1d 100644 --- a/crates/tracedecay-runtime-core/src/storage.rs +++ b/crates/tracedecay-runtime-core/src/storage.rs @@ -465,10 +465,15 @@ pub use paths_and_io::{ retry_transient_file_op, set_private_dir_permissions, try_acquire_sidecar_lock, validate_project_id, }; +pub(crate) use paths_and_io::{ + durable_directory_lock_has_data, try_lock_durable_directory, + try_lock_existing_durable_directory, +}; pub use profile_identity::{ PROFILE_IDENTITY_RECORD_NAME, PROFILE_IDENTITY_SCHEMA_VERSION, ProfileIdentityRecordV1, read_existing_profile_identity_record, }; +pub use tracedecay_graph_db::graph_store_data_paths; #[cfg(all(test, unix))] use paths_and_io::open_lock_file; diff --git a/crates/tracedecay-runtime-core/src/storage/paths_and_io.rs b/crates/tracedecay-runtime-core/src/storage/paths_and_io.rs index 5518d00b7ce..380c9f3ee92 100644 --- a/crates/tracedecay-runtime-core/src/storage/paths_and_io.rs +++ b/crates/tracedecay-runtime-core/src/storage/paths_and_io.rs @@ -762,6 +762,51 @@ pub(super) fn open_lock_file(lock_path: &Path, private: bool) -> io::Result io::Result { + let parent = path + .parent() + .ok_or_else(|| invalid_input("directory has no parent"))?; + let name = path + .file_name() + .ok_or_else(|| invalid_input("directory has no name"))?; + let lock_path = durable_directory_lock_path(&parent.canonicalize()?, name); + reject_symlink_components(&lock_path, "durable private store directory lock")?; + Ok(lock_path) +} + +fn try_lock_directory_file(file: fs::File) -> io::Result { + file.try_lock().map_err(io::Error::from)?; + Ok(FileLease::held(file, SIDECAR_LEASE_LABEL)) +} + +/// Keep the coordination inode outside a disposable private directory so +/// removal never splits contenders across two generations of the lock. +pub(crate) fn try_lock_durable_directory(path: &Path) -> io::Result { + try_lock_directory_file(open_lock_file(&private_directory_lock_path(path)?, true)?) +} + +/// Offline reset must preserve an existing lease inode and must not create +/// a new one in a legacy store's namespace. +pub(crate) fn try_lock_existing_durable_directory(path: &Path) -> io::Result> { + match tracedecay_private_fs::open_private_file(&private_directory_lock_path(path)?) { + Ok(file) => try_lock_directory_file(file).map(Some), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error), + } +} + +/// Read-only admission never opens the coordination file for writing. +pub(crate) fn durable_directory_lock_has_data(path: &Path) -> io::Result { + match fs::symlink_metadata(private_directory_lock_path(path)?) { + Ok(meta) if meta.is_file() => Ok(meta.len() != 0), + Ok(_) => Err(invalid_input( + "durable directory lock is not a regular file", + )), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } +} + /// Non-blocking sidecar lock acquisition. Returns the held lock file on /// success, or `None` when another process/thread already holds it (the caller /// then skips its critical section). See the sidecar-lock module note above for diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs index 74df7011a33..459198111ce 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs @@ -1,5 +1,9 @@ #[cfg(unix)] use std::ffi::CString; +#[cfg(any(target_os = "linux", target_os = "android"))] +use std::os::unix::fs::OpenOptionsExt; +#[cfg(windows)] +use std::os::windows::fs::OpenOptionsExt; use std::{ fmt, fs::{File, OpenOptions}, @@ -13,6 +17,7 @@ use std::{ use rusqlite::{ Connection, OpenFlags, Transaction, config::DbConfig, + ffi, hooks::{AuthAction, AuthContext, Authorization}, limits::Limit, }; @@ -65,22 +70,170 @@ const RETAINED_WAL_BYTES: i64 = WAL_SOFT_LIMIT_BYTES as i64; /// pathname stat. #[derive(Debug)] pub(crate) struct OpenedDatabaseFile { - /// One descriptor shared by every attachment worker. Duplicating this - /// descriptor and closing a clone while SQLite holds POSIX locks would - /// release locks belonging to other live connections to the same inode. - file: Arc>>, + /// Share one identity pin across workers. A private initialization file + /// stays pinned until workers exit. Existing stores use a lock-neutral + /// descriptor or give the SQLite VFS ownership of the descriptor. + file: Arc>>, + // Exclusive reservations for a *new* basename. Hold the descriptors until + // all SQLite workers exit: closing an unrelated fd during a live SQLite + // transaction can release process-owned POSIX locks. + created_sidecars: Option>>>, identity: u64, } +#[derive(Debug)] +enum DatabasePin { + File(File), + // The SQLite VFS defers the actual close while another local connection + // holds POSIX locks on this inode. A raw O_RDONLY fd cannot do that. + #[cfg(all(unix, not(any(target_os = "linux", target_os = "android"))))] + Sqlite(Connection), +} + impl OpenedDatabaseFile { + #[cfg(all(unix, not(any(target_os = "linux", target_os = "android"))))] + pub(crate) fn pin(path: &Path) -> Result { + let identity = live_path_identity(path)?; + // Opening the main file alone pins its inode without starting a read + // transaction, inspecting the schema, or opening unverified sidecars. + let connection = Connection::open_with_flags( + path, + OpenFlags::SQLITE_OPEN_READ_ONLY + | OpenFlags::SQLITE_OPEN_NO_MUTEX + | OpenFlags::SQLITE_OPEN_PRIVATE_CACHE, + ) + .map_err(|_| OpenedDatabaseFileError::Open)?; + match sqlite_connection_has_moved(&connection)? { + Some(false) => Self::verify_path_identity(path, identity)?, + Some(true) => return Err(OpenedDatabaseFileError::Replaced), + None => return Err(OpenedDatabaseFileError::Inspect), + } + Ok(Self { + file: Arc::new(Mutex::new(Some(DatabasePin::Sqlite(connection)))), + created_sidecars: None, + identity, + }) + } + + #[cfg(any(not(unix), target_os = "linux", target_os = "android"))] pub(crate) fn pin(path: &Path) -> Result { let file = open_pinned_database(path).map_err(|_| OpenedDatabaseFileError::Open)?; Self::adopt(file) } + /// Inspect the immutable main before a read-only or writable SQLite + /// worker can open any ambiguous sidecar of an interrupted initialization. + pub(crate) fn pin_initialized(path: &Path) -> Result { + // A failed header read would drop a freshly opened raw descriptor, + // releasing unrelated process-owned SQLite locks on Unix. Reject + // obviously truncated stores without opening another descriptor. + // All SQLite page sizes are multiples of 512, and a complete + // database contains an integral number of pages. This is only a + // necessary condition; it does not prove initialization completed. + { + let metadata = std::fs::metadata(path).map_err(|_| OpenedDatabaseFileError::Open)?; + if !metadata.is_file() { + return Err(OpenedDatabaseFileError::NotFile); + } + if metadata.len() < 512 || metadata.len() % 512 != 0 { + return Err(OpenedDatabaseFileError::InvalidHeader); + } + } + // Inspect the main SQLite file without opening and closing a separate + // raw descriptor on a rejected header: on Unix that close can release + // locks held by another live SQLite connection in this process. + // An immutable SQLite reader uses the VFS's lock-aware lifetime and + // never opens or mutates the unverified WAL/SHM/journal sidecars. + let pinned = Self::pin(path)?; + let inspection = + open_immutable_reader(path).map_err(|_| OpenedDatabaseFileError::InvalidHeader)?; + let initialized = + has_store_schema(&inspection).map_err(|_| OpenedDatabaseFileError::InvalidHeader)?; + pinned.verify_connection(&inspection, path)?; + drop(inspection); + if !initialized { + let sidecar = |suffix| match std::fs::symlink_metadata(sidecar_path(path, suffix)) { + Ok(meta) => Ok(Some(meta)), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None), + Err(_) => Err(OpenedDatabaseFileError::Inspect), + }; + match (sidecar("-wal")?, sidecar("-shm")?, sidecar("-journal")?) { + (None, None, None) => {} + (Some(wal), Some(shm), None) + if wal.is_file() + && wal.len() > 0 + && shm.is_file() + && shm.len() > 0 + && live_wal_has_store_schema(path) => {} + _ => return Err(OpenedDatabaseFileError::IncompleteInitialization), + } + } + pinned.verify_current_path(path)?; + Ok(pinned) + } + pub(crate) fn create_new(path: &Path) -> Result { let file = create_pinned_database(path).map_err(|_| OpenedDatabaseFileError::Create)?; - Self::adopt(file) + // O_EXCL on the database alone does not own SQLite's basename: + // SQLite may overwrite an orphan WAL/SHM/journal even on its first + // successful open. O_EXCL-reserve every sibling before handing the + // database to SQLite; never truncate an existing member or symlink. + // A failed reservation deliberately leaves prior creations untouched: + // pathname-based rollback could unlink an entry replaced by another + // process. An incomplete namespace needs explicit reconciliation. + let mut opened = Self::adopt(file)?; + let wal = reserve_new_sidecar(&sidecar_path(path, "-wal")) + .map_err(|_| OpenedDatabaseFileError::Reserve)?; + let shm = reserve_new_sidecar(&sidecar_path(path, "-shm")) + .map_err(|_| OpenedDatabaseFileError::Reserve)?; + let journal = reserve_new_sidecar(&sidecar_path(path, "-journal")) + .map_err(|_| OpenedDatabaseFileError::Reserve)?; + opened.created_sidecars = Some(Arc::new(Mutex::new(Some([wal, shm, journal])))); + Ok(opened) + } + + pub(crate) fn resume_initialization( + path: &Path, + files: &mut Option<(File, [File; 3])>, + ) -> Result { + let (original, _) = files.as_ref().ok_or(OpenedDatabaseFileError::Open)?; + let identity = opened_file_identity(original)?; + #[cfg(windows)] + let worker_pin = { + let pin = OpenOptions::new() + .read(true) + .write(true) + .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE) + .open(path) + .map_err(|_| OpenedDatabaseFileError::Open)?; + if opened_file_identity(&pin)? != identity { + return Err(OpenedDatabaseFileError::Replaced); + } + pin + }; + // Keep all input descriptors with the caller until every fallible + // path and worker-pin check succeeds, including Windows sharing. + Self::verify_path_identity(path, identity)?; + let (main, sidecars) = files.take().ok_or(OpenedDatabaseFileError::Open)?; + #[cfg(windows)] + let main = { + drop(main); + worker_pin + }; + Ok(Self { + file: Arc::new(Mutex::new(Some(DatabasePin::File(main)))), + created_sidecars: Some(Arc::new(Mutex::new(Some(sidecars)))), + identity, + }) + } + + pub(crate) fn take_idle_initialization_files(&self) -> Option<(File, [File; 3])> { + if Arc::strong_count(&self.file) != 1 + || Arc::strong_count(self.created_sidecars.as_ref()?) != 1 + { + return None; + } + self.take_completed_files() } /// Takes ownership of an already-open handle and records its identity. @@ -93,7 +246,8 @@ impl OpenedDatabaseFile { } let identity = opened_file_identity(&file)?; Ok(Self { - file: Arc::new(Mutex::new(Some(file))), + file: Arc::new(Mutex::new(Some(DatabasePin::File(file)))), + created_sidecars: None, identity, }) } @@ -105,21 +259,46 @@ impl OpenedDatabaseFile { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .take(); + if let Some(sidecars) = &self.created_sidecars { + sidecars + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + } + } + + pub(crate) fn take_completed_files(&self) -> Option<(File, [File; 3])> { + let sidecars = self + .created_sidecars + .as_ref()? + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take()?; + let main = self + .file + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take()?; + match main { + DatabasePin::File(main) => Some((main, sidecars)), + #[cfg(all(unix, not(any(target_os = "linux", target_os = "android"))))] + DatabasePin::Sqlite(_) => None, + } } pub(crate) const fn identity(&self) -> u64 { self.identity } - fn with_file( + fn with_pin( &self, - use_file: impl FnOnce(&File) -> T, + use_pin: impl FnOnce(&DatabasePin) -> T, ) -> Result { self.file .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .as_ref() - .map(use_file) + .map(use_pin) .ok_or(OpenedDatabaseFileError::Replaced) } @@ -127,9 +306,10 @@ impl OpenedDatabaseFile { // Share the *same* raw descriptor instead of dup(2): closing any // duplicate descriptor after a SQLite connection acquires POSIX locks // can silently release that connection's locks process-wide. - self.with_file(|_| ())?; + self.with_pin(|_| ())?; Ok(Self { file: Arc::clone(&self.file), + created_sidecars: self.created_sidecars.as_ref().map(Arc::clone), identity: self.identity, }) } @@ -141,7 +321,7 @@ impl OpenedDatabaseFile { ) -> Result { use std::os::unix::io::AsRawFd; - let descriptor = self.with_file(|file| file.as_raw_fd())?; + let descriptor = self.with_pin(|DatabasePin::File(file)| file.as_raw_fd())?; Ok(PathBuf::from(format!("/proc/self/fd/{descriptor}"))) } @@ -205,6 +385,10 @@ impl OpenedDatabaseFile { } pub(crate) fn verify_current_path(&self, path: &Path) -> Result<(), OpenedDatabaseFileError> { + Self::verify_path_identity(path, self.identity) + } + + fn verify_path_identity(path: &Path, identity: u64) -> Result<(), OpenedDatabaseFileError> { // POSIX record locks belong to the process: closing any descriptor // for this inode releases SQLite's locks, even on another connection. // Inspect the pathname without opening a temporary descriptor. Keep @@ -220,7 +404,7 @@ impl OpenedDatabaseFile { let current = File::open(path).map_err(|_| OpenedDatabaseFileError::Open)?; opened_file_identity(¤t)? }; - if current_identity != self.identity { + if current_identity != identity { return Err(OpenedDatabaseFileError::Replaced); } Ok(()) @@ -233,7 +417,13 @@ impl OpenedDatabaseFile { use std::os::unix::fs::MetadataExt; let unlinked = self - .with_file(|file| file.metadata().is_ok_and(|pinned| pinned.nlink() == 0)) + .with_pin(|pin| match pin { + DatabasePin::File(file) => file.metadata().is_ok_and(|pinned| pinned.nlink() == 0), + #[cfg(all(unix, not(any(target_os = "linux", target_os = "android"))))] + DatabasePin::Sqlite(connection) => { + sqlite_connection_has_moved(connection) == Ok(Some(true)) + } + }) .unwrap_or(true); if unlinked { return true; @@ -262,8 +452,10 @@ impl OpenedDatabaseFile { // HAS_MOVED below still observes the SQLite handle's different inode. self.verify_current_path(canonical_path)?; #[cfg(unix)] - if sqlite_connection_has_moved(_connection)?.unwrap_or(false) { - return Err(OpenedDatabaseFileError::Replaced); + match sqlite_connection_has_moved(_connection)? { + Some(false) => {} + Some(true) => return Err(OpenedDatabaseFileError::Replaced), + None => return Err(OpenedDatabaseFileError::Inspect), } #[cfg(unix)] { @@ -273,25 +465,6 @@ impl OpenedDatabaseFile { } Ok(()) } - - pub(crate) fn discard_created(self, path: &Path) -> Result<(), OpenedDatabaseFileError> { - self.verify_current_path(path)?; - let Self { file, .. } = self; - drop(file); - for candidate in [ - sidecar_path(path, "-wal"), - sidecar_path(path, "-shm"), - sidecar_path(path, "-journal"), - path.to_path_buf(), - ] { - match std::fs::remove_file(candidate) { - Ok(()) => {} - Err(error) if error.kind() == io::ErrorKind::NotFound => {} - Err(_) => return Err(OpenedDatabaseFileError::Remove), - } - } - Ok(()) - } } /// Check a live SQLite worker's physical file without opening and closing a @@ -316,13 +489,15 @@ pub(crate) fn live_path_identity(path: &Path) -> Result) -> fmt::Result { let message = match self { Self::Create => "could not create the canonical SQLite file", + Self::Reserve => "could not exclusively reserve the new SQLite sidecar namespace", + Self::IncompleteInitialization => { + "SQLite main has no published schema and its sidecar ownership is ambiguous" + } + Self::InvalidHeader => { + "existing SQLite file has no valid database header; refusing sidecar access" + } Self::Open => "could not open the verified SQLite file", Self::Inspect => "could not inspect the verified SQLite file descriptor", Self::NotFile => "verified SQLite locator is not a regular file", #[cfg(windows)] Self::Identify => "could not identify the verified SQLite file descriptor", Self::Replaced => "verified SQLite file was replaced while opening workers", - Self::Remove => "could not remove an uncommitted canonical SQLite file", #[cfg(not(any(unix, windows)))] Self::Unsupported => "SQLite file identity is unsupported on this platform", }; @@ -347,7 +528,18 @@ impl fmt::Display for OpenedDatabaseFileError { impl std::error::Error for OpenedDatabaseFileError {} -#[cfg(not(windows))] +/// An O_PATH descriptor pins the inode for /proc/self/fd based SQLite opens, +/// but its close does not release another connection's process-wide POSIX +/// record locks. O_RDONLY pins have that destructive close behavior on Linux. +#[cfg(any(target_os = "linux", target_os = "android"))] +fn open_pinned_database(path: &Path) -> io::Result { + OpenOptions::new() + .read(true) + .custom_flags(libc::O_PATH) + .open(path) +} + +#[cfg(not(any(windows, unix)))] fn open_pinned_database(path: &Path) -> io::Result { File::open(path) } @@ -383,6 +575,21 @@ fn create_pinned_database(path: &Path) -> io::Result { .open(path) } +/// SQLite is allowed to delete or recreate an owned journal/sidecar. Its +/// namespace reservation is exclusive at creation, but its file handle must +/// not prevent SQLite's own cleanup on Windows. +fn reserve_new_sidecar(path: &Path) -> io::Result { + let mut options = OpenOptions::new(); + options.read(true).write(true).create_new(true); + #[cfg(windows)] + { + options.share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE); + } + options.open(path) +} + +#[cfg(windows)] +const FILE_SHARE_DELETE: u32 = 0x0000_0004; #[cfg(windows)] const FILE_SHARE_READ: u32 = 0x0000_0001; #[cfg(windows)] @@ -395,7 +602,7 @@ fn sidecar_path(path: &Path, suffix: &str) -> PathBuf { } #[cfg(unix)] -fn opened_file_identity(file: &File) -> Result { +pub(crate) fn opened_file_identity(file: &File) -> Result { let metadata = file .metadata() .map_err(|_| OpenedDatabaseFileError::Inspect)?; @@ -434,15 +641,15 @@ fn sqlite_connection_has_moved( }; match result { rusqlite::ffi::SQLITE_OK => Ok(Some(moved != 0)), - // Workers with a retained raw pin may use the pinned identity when - // HAS_MOVED is unsupported. Unpinned verified readers must refuse it. + // Callers establishing identity through SQLite must have a VFS + // answer. A worker with an independent inode pin can retain it. rusqlite::ffi::SQLITE_NOTFOUND => Ok(None), _ => Err(OpenedDatabaseFileError::Inspect), } } #[cfg(windows)] -fn opened_file_identity(file: &File) -> Result { +pub(crate) fn opened_file_identity(file: &File) -> Result { use std::mem::MaybeUninit; use std::os::windows::io::AsRawHandle; @@ -498,7 +705,7 @@ unsafe extern "system" { } #[cfg(not(any(unix, windows)))] -fn opened_file_identity(_file: &File) -> Result { +pub(crate) fn opened_file_identity(_file: &File) -> Result { Err(OpenedDatabaseFileError::Unsupported) } @@ -832,7 +1039,7 @@ pub fn open_immutable_reader(path: &Path) -> Result Result { +fn has_store_schema(connection: &Connection) -> rusqlite::Result { + connection.query_row( + "SELECT (SELECT application_id FROM pragma_application_id) = ?1 OR EXISTS( + SELECT 1 FROM sqlite_schema WHERE type = 'table' + AND name NOT GLOB 'sqlite_*' AND name NOT GLOB 'td_runtime_*' + )", + [crate::schema_version::REGISTERED_STORE_APPLICATION_ID], + |row| row.get(0), + ) +} + +/// A legacy store's first schema commit can still be in its live WAL. +/// Ask SQLite to read that commit using the existing shared index. A read-only +/// VFS can also recover a disconnected WAL into a *heap* index, which cannot +/// establish that the adjacent SHM belongs to this active database. +fn live_wal_has_store_schema(path: &Path) -> bool { + let Ok(uri) = read_only_uri(&sqlite_host_path(path), "readonly_shm=1") else { + return false; + }; + let flags = OpenFlags::SQLITE_OPEN_READ_ONLY + | OpenFlags::SQLITE_OPEN_URI + | OpenFlags::SQLITE_OPEN_PRIVATE_CACHE; + let Ok(connection) = Connection::open_with_flags(uri, flags) else { + return false; + }; + if connection + .set_db_config(DbConfig::SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, true) + .is_err() + || connection.execute_batch("BEGIN").is_err() + || !matches!(has_store_schema(&connection), Ok(true)) + { + return false; + } + let mut file: *mut ffi::sqlite3_file = std::ptr::null_mut(); + // SAFETY: the read transaction keeps the main file and mapped WAL index + // live. FILE_POINTER borrows the connection's file; xShmMap inspects page + // zero with SQLite's WAL-index page size and extension disabled. + unsafe { + if ffi::sqlite3_file_control( + connection.handle(), + c"main".as_ptr(), + ffi::SQLITE_FCNTL_FILE_POINTER, + (&raw mut file).cast(), + ) != ffi::SQLITE_OK + { + return false; + } + let Some(methods) = file.as_ref().and_then(|file| file.pMethods.as_ref()) else { + return false; + }; + if methods.iVersion < 2 { + return false; + } + let Some(map) = methods.xShmMap else { + return false; + }; + let mut region = std::ptr::null_mut(); + let rc = map(file, 0, 32 * 1024, 0, &mut region); + matches!(rc, ffi::SQLITE_OK | ffi::SQLITE_READONLY) && !region.is_null() + } +} + +fn read_only_uri(path: &Path, options: &str) -> Result { #[cfg(unix)] let raw = { use std::os::unix::ffi::OsStrExt; @@ -946,7 +1215,7 @@ fn immutable_health_uri(path: &Path) -> Result { } } } - Ok(format!("file:{encoded}?immutable=1&mode=ro")) + Ok(format!("file:{encoded}?mode=ro&{options}")) } fn apply_pragmas( @@ -1349,7 +1618,7 @@ fn policy(stage: &'static str, source: rusqlite::Error) -> ConnectionPolicyError } #[cfg(all(test, unix))] -mod path_identity_tests; +pub(crate) mod path_identity_tests; #[cfg(test)] mod tests; diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs index 0f5b3a5eb91..6157803fa01 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs @@ -1,4 +1,11 @@ -use std::{env, fs, path::Path, process::Command, time::Duration}; +use std::{ + env, + fs::{self, OpenOptions}, + os::{fd::AsRawFd, unix::fs::FileExt}, + path::Path, + process::Command, + time::Duration, +}; use rusqlite::{Connection, Error, ErrorCode, OpenFlags}; use tempfile::TempDir; @@ -14,7 +21,7 @@ const PROBE_COMPLETE: &str = "tracedecay-path-identity-probe-complete"; // shares inode/lock bookkeeping within a process and can hide a lost OS lock. // Execute only the child test in a fresh process, with a marker to reject a // successful test harness invocation that accidentally selected zero tests. -fn probe_external_connection(path: &Path, operation: &str, blocked: bool) { +pub(crate) fn probe_external_connection(path: &Path, operation: &str, blocked: bool) { probe_external_connection_expectation( path, operation, @@ -157,6 +164,133 @@ fn worker_identity_checks_keep_cross_process_writer_exclusion() { ); } +/// A zero-byte existing SQLite file may belong to a live connection holding +/// a rollback writer reservation. Rejecting that unfinished attachment must +/// not open and close a second raw descriptor and unlock the other writer. +#[test] +fn rejecting_empty_attachment_does_not_release_another_writer_lock() { + let temp = TempDir::new().expect("tempdir"); + let path = temp.path().join("empty-locked.db"); + let writer = Connection::open(&path).expect("open SQLite connection"); + writer + .execute_batch("BEGIN IMMEDIATE") + .expect("reserve the SQLite writer lock without initializing pages"); + assert_eq!(fs::metadata(&path).unwrap().len(), 0); + probe_external_connection(&path, "write", true); + + assert!(matches!( + OpenedDatabaseFile::pin_initialized(&path), + Err(OpenedDatabaseFileError::InvalidHeader) + )); + probe_external_connection(&path, "write", true); + + writer + .execute_batch("ROLLBACK") + .expect("release writer reservation"); + probe_external_connection(&path, "write", false); +} + +/// SQLite's immutable header inspection during existing-store admission must +/// not release an unrelated live connection's process-owned POSIX locks. +/// A separate process is essential: same-process SQLite handles can mask loss. +/// Releasing an existing-store identity pin must not cancel locks owned by +/// a different live SQLite writer in this process. O_RDONLY descriptors do +/// exactly that on Linux even if they never acquired a lock themselves. +#[cfg(any(target_os = "linux", target_os = "android"))] +#[test] +fn releasing_existing_store_pin_preserves_unrelated_writer_locks() { + for journal_mode in ["DELETE", "WAL"] { + let temp = TempDir::new().expect("tempdir"); + let path = temp.path().join("sessions.db"); + let writer = Connection::open(&path).expect("initialize database"); + writer + .execute_batch("CREATE TABLE sample(value INTEGER); INSERT INTO sample VALUES (1);") + .expect("initialize committed schema"); + writer + .pragma_update(None, "journal_mode", journal_mode) + .expect("select journal mode"); + if journal_mode == "WAL" { + writer + .execute_batch("PRAGMA wal_checkpoint(TRUNCATE)") + .expect("checkpoint the initial schema"); + } + let pin = OpenedDatabaseFile::pin_initialized(&path) + .expect("pin fully initialized existing database"); + + writer + .execute_batch("BEGIN IMMEDIATE; UPDATE sample SET value = 2;") + .expect("hold an unrelated writer reservation"); + probe_external_connection(&path, "write", true); + pin.release_after_workers_exit(); + probe_external_connection(&path, "write", true); + + writer.execute_batch("COMMIT").expect("commit"); + drop(writer); + probe_external_connection(&path, "write", false); + assert_committed_data(&path); + } +} + +#[test] +fn admitting_initialized_store_keeps_existing_writer_locks() { + for journal_mode in ["DELETE", "WAL"] { + let temp = TempDir::new().expect("tempdir"); + let path = temp.path().join("live-existing.db"); + let writer = Connection::open(&path).expect("initialize existing SQLite file"); + writer + .execute_batch("CREATE TABLE sample(value INTEGER); INSERT INTO sample VALUES (1);") + .expect("commit initial sample"); + writer + .pragma_update(None, "journal_mode", journal_mode) + .expect("select journal mode"); + if journal_mode == "WAL" { + // Keep the committed schema on the main file for the immutable + // inspection: it deliberately does not consume live WAL frames. + writer + .execute_batch("PRAGMA wal_checkpoint(TRUNCATE)") + .expect("checkpoint initial schema"); + } + writer + .execute_batch("BEGIN IMMEDIATE; UPDATE sample SET value = 2;") + .expect("hold writer reservation"); + probe_external_connection(&path, "write", true); + + // This is the production existing-store admission path, not a + // substitute read-only connection opened by the test itself. + let opened = OpenedDatabaseFile::pin_initialized(&path) + .expect("inspect and pin a fully initialized main database"); + opened + .verify_current_path(&path) + .expect("original database remains bound"); + probe_external_connection(&path, "write", true); + probe_external_connection_expectation(&path, "read", "read:1"); + + writer.execute_batch("COMMIT").expect("commit update"); + // The pinned raw descriptor is intentionally closed only after the + // writer exits, so its own close cannot independently release locks. + drop(writer); + opened.release_after_workers_exit(); + probe_external_connection(&path, "write", false); + assert_committed_data(&path); + } +} + +#[test] +fn header_magic_without_a_complete_sqlite_page_is_refused() { + let temp = TempDir::new().expect("tempdir"); + for length in [16, 100, 511] { + let path = temp.path().join(format!("short-{length}.db")); + let mut bytes = vec![0_u8; length]; + bytes[..16].copy_from_slice(b"SQLite format 3\0"); + fs::write(&path, &bytes).expect("write incomplete SQLite header"); + assert!(matches!( + OpenedDatabaseFile::pin_initialized(&path), + Err(OpenedDatabaseFileError::InvalidHeader) + )); + assert_eq!(fs::read(&path).unwrap(), bytes); + } +} + #[test] fn path_checks_preserve_rollback_writer_lock() { let temp = TempDir::new().expect("tempdir"); @@ -378,6 +512,148 @@ fn path_checks_keep_pinned_identity_and_reject_missing_or_replaced_files() { ); } +const RESERVED_LOCK_PROBE_PATH: &str = "TRACEDECAY_SQLITE_RESERVED_LOCK_PROBE_PATH"; +const RESERVED_LOCK_PROBE_EXPECTATION: &str = "TRACEDECAY_SQLITE_RESERVED_LOCK_PROBE_EXPECTATION"; +const RESERVED_LOCK_PROBE_COMPLETE: &str = "tracedecay-sqlite-reserved-lock-probe-complete"; + +/// Inspect SQLite's POSIX RESERVED byte from a different process. This avoids +/// an SQLite schema read after we deliberately corrupt the fixture header. +#[test] +fn external_reserved_lock_probe() { + let Some(path) = env::var_os(RESERVED_LOCK_PROBE_PATH) else { + return; + }; + let file = OpenOptions::new() + .read(true) + .write(true) + .open(path) + .unwrap(); + let mut reservation = libc::flock { + l_type: libc::F_WRLCK as libc::c_short, + l_whence: libc::SEEK_SET as libc::c_short, + l_start: 0x4000_0001, + l_len: 1, + l_pid: 0, + }; + // SAFETY: the file owns a valid descriptor and reservation is initialized + // and writable for the F_SETLK request. + let result = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_SETLK, &mut reservation) }; + match env::var(RESERVED_LOCK_PROBE_EXPECTATION).unwrap().as_str() { + "blocked" => { + assert_eq!(result, -1, "SQLite writer's RESERVED lock was lost"); + let error = std::io::Error::last_os_error().raw_os_error(); + assert!( + error == Some(libc::EACCES) || error == Some(libc::EAGAIN), + "unexpected advisory-lock refusal: {error:?}" + ); + } + "allowed" => assert_eq!( + result, + 0, + "RESERVED byte remained locked after the writer closed: {}", + std::io::Error::last_os_error() + ), + other => panic!("unknown reserved lock expectation: {other}"), + } + println!("{RESERVED_LOCK_PROBE_COMPLETE}"); +} + +fn probe_reserved_lock(path: &Path, blocked: bool) { + let (_, module) = module_path!().split_once("::").unwrap(); + let output = Command::new(env::current_exe().unwrap()) + .args([ + "--exact", + &format!("{module}::external_reserved_lock_probe"), + "--nocapture", + ]) + .env(RESERVED_LOCK_PROBE_PATH, path) + .env( + RESERVED_LOCK_PROBE_EXPECTATION, + if blocked { "blocked" } else { "allowed" }, + ) + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success() + && stdout.contains("1 passed") + && stdout.contains(RESERVED_LOCK_PROBE_COMPLETE), + "cross-process lock probe failed: {stdout}\n{}", + String::from_utf8_lossy(&output.stderr) + ); +} + +/// Opening an already valid database for attachment must not close an +/// unrelated raw descriptor and discard a live writer's POSIX RESERVED lock. +/// Closing one attachment must preserve the independent writer’s transaction. +#[test] +fn valid_existing_store_admission_preserves_other_writer_locks() { + let temp = TempDir::new().unwrap(); + let path = temp.path().join("sessions.db"); + let writer = Connection::open(&path).unwrap(); + writer + .execute_batch( + "PRAGMA journal_mode = DELETE; + CREATE TABLE sample(value INTEGER); + INSERT INTO sample VALUES (1); + BEGIN IMMEDIATE; + UPDATE sample SET value = 2;", + ) + .unwrap(); + + probe_reserved_lock(&path, true); + let opened = OpenedDatabaseFile::pin_initialized(&path) + .expect("admit an initialized database while another writer is live"); + assert_eq!(opened.identity(), live_path_identity(&path).unwrap()); + probe_reserved_lock(&path, true); + + opened.release_after_workers_exit(); + probe_reserved_lock(&path, true); + writer.execute_batch("COMMIT").unwrap(); + drop(writer); + probe_reserved_lock(&path, false); + assert_committed_data(&path); +} + +#[test] +fn malformed_existing_store_refusal_keeps_other_sqlite_writer_locks() { + let temp = TempDir::new().unwrap(); + let path = temp.path().join("sessions.db"); + let writer = Connection::open(&path).unwrap(); + writer + .execute_batch( + "PRAGMA journal_mode = DELETE; + CREATE TABLE sample(value INTEGER); + INSERT INTO sample VALUES (1);", + ) + .unwrap(); + // Retain this descriptor through the original writer's entire lifetime: + // closing it early would itself discard the POSIX locks under test. + let mutation = OpenOptions::new() + .read(true) + .write(true) + .open(&path) + .unwrap(); + writer + .execute_batch("BEGIN IMMEDIATE; UPDATE sample SET value = 2;") + .unwrap(); + probe_reserved_lock(&path, true); + + // Corrupt only a disposable fixture, in place and without closing a raw + // descriptor. The main file remains page-aligned but is no longer SQLite. + mutation.write_all_at(b"Not SQLite data!", 0).unwrap(); + probe_reserved_lock(&path, true); + assert!(matches!( + OpenedDatabaseFile::pin_initialized(&path), + Err(OpenedDatabaseFileError::InvalidHeader) + )); + probe_reserved_lock(&path, true); + + drop(writer); + drop(mutation); + probe_reserved_lock(&path, false); +} + /// Closing a verified read snapshot must not release another SQLite /// connection's process-wide POSIX locks. A second connection in this process /// cannot prove this: run the lock/journal contender in a separate process. diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs index be04666e521..c9559dc27d1 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs @@ -374,16 +374,145 @@ fn policy_requires_an_existing_database() { } #[test] -fn create_new_pins_and_discards_the_exact_database() { +fn create_new_reserves_the_full_sqlite_basename_without_unverified_cleanup() { let directory = tempfile::tempdir().unwrap(); let path = directory.path().join("fresh.db"); - let created = OpenedDatabaseFile::create_new(&path).unwrap(); assert!(path.is_file()); assert_ne!(created.identity(), 0); - created.discard_created(&path).unwrap(); + for suffix in ["-wal", "-shm", "-journal"] { + let sidecar = sidecar_path(&path, suffix); + assert_eq!(std::fs::metadata(&sidecar).unwrap().len(), 0); + } + drop(created); + // Dropping a reservation must not unlink mutable names. In particular, a + // foreign process may have substituted an unrelated inode since creation. + assert!(path.is_file()); + for suffix in ["-wal", "-shm", "-journal"] { + assert!(sidecar_path(&path, suffix).is_file()); + } +} + +#[test] +fn create_new_refuses_existing_sidecar_members_without_touching_their_bytes() { + for suffix in ["-wal", "-shm", "-journal"] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("fresh.db"); + let foreign = sidecar_path(&path, suffix); + let bytes = format!("foreign SQLite {suffix} bytes"); + std::fs::write(&foreign, bytes.as_bytes()).unwrap(); + assert!(matches!( + OpenedDatabaseFile::create_new(&path), + Err(OpenedDatabaseFileError::Reserve) + )); + assert_eq!(std::fs::read(&foreign).unwrap(), bytes.as_bytes()); + } +} + +/// A reserved basename that failed on a foreign sidecar is left with an +/// empty main DB. A later existing-mode attachment must not let SQLite +/// initialize that empty file and destroy the foreign sidecar on retry. +#[test] +fn failed_sidecar_reservation_cannot_be_reopened_as_existing_sqlite() { + for suffix in ["-wal", "-shm", "-journal"] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("sessions.db"); + let foreign = sidecar_path(&path, suffix); + let bytes = format!("foreign occupied {suffix} bytes"); + std::fs::write(&foreign, bytes.as_bytes()).unwrap(); + + assert!(matches!( + OpenedDatabaseFile::create_new(&path), + Err(OpenedDatabaseFileError::Reserve) + )); + assert_eq!(std::fs::metadata(&path).unwrap().len(), 0); + assert!(matches!( + OpenedDatabaseFile::pin_initialized(&path), + Err(OpenedDatabaseFileError::InvalidHeader) + )); + assert_eq!(std::fs::read(&foreign).unwrap(), bytes.as_bytes()); + } +} + +#[test] +fn existing_sqlite_admission_requires_a_completed_header() { + let directory = tempfile::tempdir().unwrap(); + for size in [512_usize, 513, 1023, 1025, 4096] { + let path = directory.path().join(format!("incomplete-{size}.db")); + let mut bytes = vec![0_u8; size]; + bytes[..16].copy_from_slice(b"SQLite format 3\0"); + std::fs::write(&path, &bytes).unwrap(); + assert!(matches!( + OpenedDatabaseFile::pin_initialized(&path), + Err(OpenedDatabaseFileError::InvalidHeader) + )); + assert_eq!(std::fs::read(&path).unwrap(), bytes); + } + let path = directory.path().join("sessions.db"); + std::fs::write(&path, b"invalid database, not SQLite").unwrap(); + assert!(matches!( + OpenedDatabaseFile::pin_initialized(&path), + Err(OpenedDatabaseFileError::InvalidHeader) + )); + std::fs::remove_file(&path).unwrap(); + let database = Connection::open(&path).unwrap(); + database + .execute_batch("CREATE TABLE probe(value INTEGER)") + .unwrap(); + drop(database); + assert!(OpenedDatabaseFile::pin_initialized(&path).is_ok()); +} - assert!(!path.exists()); +#[test] +fn initialized_wal_with_uncheckpointed_schema_remains_attachable() { + const EXTERNAL_READER: &str = "TRACEDECAY_TEST_INSPECT_LIVE_FIRST_SCHEMA"; + if let Some(path) = std::env::var_os(EXTERNAL_READER) { + let path = PathBuf::from(path); + let shm = sidecar_path(&path, "-shm"); + let before = std::fs::read(&shm).unwrap(); + let pinned = OpenedDatabaseFile::pin_initialized(&path).unwrap(); + assert_eq!(std::fs::read(&shm).unwrap(), before); + pinned.release_after_workers_exit(); + return; + } + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("sessions.db"); + let writer = Connection::open(&path).unwrap(); + writer + .execute_batch( + "PRAGMA journal_mode = WAL; + CREATE TABLE probe(value INTEGER); + INSERT INTO probe VALUES (42);", + ) + .unwrap(); + let wal = sidecar_path(&path, "-wal"); + let before = std::fs::read(&wal).unwrap(); + assert!(!before.is_empty()); + // A separate process has no connection sharing the writer's VFS node. + // It must discover the live WAL index through SQLite without rewriting it. + let name = format!( + "{}::initialized_wal_with_uncheckpointed_schema_remains_attachable", + module_path!().split_once("::").unwrap().1 + ); + let external = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", &name, "--nocapture"]) + .env(EXTERNAL_READER, &path) + .output() + .unwrap(); + assert!( + external.status.success() && String::from_utf8_lossy(&external.stdout).contains("1 passed"), + "external WAL inspection failed: {external:?}" + ); + let pinned = OpenedDatabaseFile::pin_initialized(&path).unwrap(); + assert_ne!(pinned.identity(), 0); + assert_eq!(std::fs::read(&wal).unwrap(), before); + drop(writer); + pinned.release_after_workers_exit(); + let reader = Connection::open(&path).unwrap(); + assert_eq!( + reader.query_row("SELECT value FROM probe", [], |row| row.get::<_, i64>(0)), + Ok(42) + ); } #[test] @@ -396,6 +525,50 @@ fn create_new_refuses_to_replace_an_existing_database() { )); } +#[test] +fn legacy_first_schema_in_disconnected_wal_preserves_unclaimed_members() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("sessions.db"); + let wal = sidecar_path(&path, "-wal"); + let shm = sidecar_path(&path, "-shm"); + { + let writer = Connection::open(&path).unwrap(); + writer + .set_db_config(DbConfig::SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, true) + .unwrap(); + writer + .execute_batch( + "PRAGMA journal_mode=WAL; CREATE TABLE probe(value); INSERT INTO probe VALUES (7)", + ) + .unwrap(); + } + let main_bytes = std::fs::read(&path).unwrap(); + let wal_bytes = std::fs::read(&wal).unwrap(); + let shm_bytes = std::fs::read(&shm).unwrap(); + assert!(!wal_bytes.is_empty()); + assert!(matches!( + OpenedDatabaseFile::pin_initialized(&path), + Err(OpenedDatabaseFileError::IncompleteInitialization) + )); + assert_eq!(std::fs::read(&shm).unwrap(), shm_bytes); + std::fs::remove_file(&shm).unwrap(); + assert!(matches!( + OpenedDatabaseFile::pin_initialized(&path), + Err(OpenedDatabaseFileError::IncompleteInitialization) + )); + assert_eq!(std::fs::read(&path).unwrap(), main_bytes); + assert_eq!(std::fs::read(&wal).unwrap(), wal_bytes); + assert!(!shm.exists()); + // The owner can still run ordinary SQLite recovery on its own database. + let writer = Connection::open(&path).unwrap(); + assert_eq!( + writer.query_row("SELECT value FROM probe", [], |row| row.get::<_, i64>(0)), + Ok(7) + ); + drop(writer); + OpenedDatabaseFile::pin_initialized(&path).unwrap(); +} + #[cfg(all(unix, any(target_os = "linux", target_os = "android")))] #[test] fn worker_open_path_stays_on_the_pinned_file_across_an_a_b_a_swap() { @@ -479,6 +652,42 @@ fn writer_identity_fence_rejects_a_replacement_hidden_by_path_restore() { ); } +#[cfg(unix)] +#[test] +fn worker_identity_refuses_a_vfs_without_move_attestation() { + let file = database(); + let pinned = OpenedDatabaseFile::pin(file.path()).unwrap(); + // SQLite's built-in memdb VFS does not implement HAS_MOVED, although + // it accepts the same pathname as the pinned on-disk database. + let virtual_connection = Connection::open_with_flags_and_vfs( + file.path(), + rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE | rusqlite::OpenFlags::SQLITE_OPEN_CREATE, + "memdb", + ) + .expect("open bundled SQLite's memdb VFS"); + assert_eq!( + super::sqlite_connection_has_moved(&virtual_connection), + Ok(None), + "the regression must exercise an actual unsupported file control" + ); + assert_eq!( + pinned.verify_connection(&virtual_connection, file.path()), + Err(OpenedDatabaseFileError::Inspect), + "an unchanged filename cannot authenticate an unverified SQLite VFS handle" + ); + + // Rejecting that connection must leave the real database intact and + // permit a worker using the ordinary SQLite VFS. + let real_connection = Connection::open(file.path()).unwrap(); + pinned + .verify_connection(&real_connection, file.path()) + .unwrap(); + assert_eq!( + real_connection.query_row("SELECT value FROM items", [], |row| row.get::<_, i64>(0)), + Ok(1) + ); +} + #[cfg(unix)] #[test] fn verified_immutable_reader_never_binds_transient_b_to_restored_a_identity() { @@ -552,6 +761,32 @@ fn verified_immutable_reader_never_binds_transient_b_to_restored_a_identity() { } } +#[test] +fn header_only_existing_store_preserves_ambiguous_sidecars() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("existing.db"); + Connection::open(&path) + .unwrap() + .execute_batch("PRAGMA journal_mode=WAL") + .unwrap(); + let member = super::sidecar_path(&path, "-journal"); + std::fs::write(&member, b"unclaimed state").unwrap(); + assert!(matches!( + OpenedDatabaseFile::pin_initialized(&path), + Err(OpenedDatabaseFileError::IncompleteInitialization) + )); + assert_eq!(std::fs::read(&member).unwrap(), b"unclaimed state"); + std::fs::remove_file(&member).unwrap(); + let database = Connection::open(&path).unwrap(); + database + .execute_batch( + "CREATE TABLE existing_data(value INTEGER); INSERT INTO existing_data VALUES (7)", + ) + .unwrap(); + drop(database); + OpenedDatabaseFile::pin_initialized(&path).unwrap(); +} + #[cfg(windows)] #[test] fn windows_pinned_file_blocks_replacement_until_authority_closes() { @@ -582,7 +817,7 @@ fn windows_pinned_file_blocks_replacement_until_authority_closes() { #[cfg(windows)] #[test] -fn windows_discard_created_removes_the_complete_sqlite_family() { +fn windows_dropping_reservations_preserves_sidecars_written_after_creation() { let directory = tempfile::tempdir().unwrap(); let path = directory.path().join("fresh.db"); let created = OpenedDatabaseFile::create_new(&path).unwrap(); @@ -592,13 +827,13 @@ fn windows_discard_created_removes_the_complete_sqlite_family() { sidecar_path(&path, "-journal"), ]; for sidecar in &sidecars { - std::fs::write(sidecar, b"sidecar").unwrap(); + std::fs::write(sidecar, b"foreign sidecar").unwrap(); + } + drop(created); + assert!(path.is_file()); + for sidecar in &sidecars { + assert_eq!(std::fs::read(sidecar).unwrap(), b"foreign sidecar"); } - - created.discard_created(&path).unwrap(); - - assert!(!path.exists()); - assert!(sidecars.iter().all(|sidecar| !sidecar.exists())); } /// `sqlite_host_path` shortens a verbatim *disk* path so SQLite's win32 VFS diff --git a/crates/tracedecay-rusqlite-runtime/src/exact_sql/command.rs b/crates/tracedecay-rusqlite-runtime/src/exact_sql/command.rs index 55f6bb092a4..d2d375b8295 100644 --- a/crates/tracedecay-rusqlite-runtime/src/exact_sql/command.rs +++ b/crates/tracedecay-rusqlite-runtime/src/exact_sql/command.rs @@ -43,6 +43,10 @@ pub(crate) enum WriterCommand { lease: Arc, authority: Option>, }, + FinishInitialSchema { + reply: async_channel::Sender>, + authority: Arc, + }, CheckpointWalTruncate { reply: async_channel::Sender>, authority: Option>, @@ -553,6 +557,43 @@ pub(crate) async fn run_writer_command( } terminal } + WriterCommand::FinishInitialSchema { reply, authority } => { + let result = authority + .verify(ExactSqlWriteIntent::Execute) + .and_then(|()| { + with_exact_sql_guard( + connection, + ExactSqlAuthority::Canonical(ConnectionMode::Maintenance), + Some(Arc::clone(shutdown_requested)), + None, + true, + None, + || { + connection + .pragma_update( + None, + "application_id", + crate::schema_version::REGISTERED_STORE_APPLICATION_ID, + ) + .map_err(|error| { + sqlite_error("mark completed initial schema", error) + })?; + let checkpoint: (i64, i64, i64) = connection + .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |row| { + Ok((row.get(0)?, row.get(1)?, row.get(2)?)) + }) + .map_err(|error| { + sqlite_error("checkpoint completed initial schema", error) + })?; + if checkpoint != (0, 0, 0) { + return Err(ExactSqlError::Busy); + } + Ok(()) + }, + ) + }); + DeferredReply::new(reply, result) + } WriterCommand::CheckpointWalTruncate { reply, authority } => { if let Err(error) = verify_write_authority(authority.as_deref(), ExactSqlWriteIntent::Query) @@ -609,6 +650,9 @@ pub(crate) fn reject_writer_command(command: WriterCommand) { WriterCommand::BeginTransaction { reply, .. } => { let _ = reply.try_send(Err(ExactSqlError::WriterUnavailable)); } + WriterCommand::FinishInitialSchema { reply, .. } => { + let _ = reply.try_send(Err(ExactSqlError::WriterUnavailable)); + } WriterCommand::CheckpointWalTruncate { reply, .. } => { let _ = reply.try_send(Err(ExactSqlError::WriterUnavailable)); } diff --git a/crates/tracedecay-rusqlite-runtime/src/exact_sql/mod.rs b/crates/tracedecay-rusqlite-runtime/src/exact_sql/mod.rs index 419f7d6cffa..92a02a2736e 100644 --- a/crates/tracedecay-rusqlite-runtime/src/exact_sql/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/exact_sql/mod.rs @@ -615,6 +615,30 @@ impl ExactSqlHandle { .map_err(|_| ExactSqlError::WriterUnavailable) } + /// Marks the completed prospective store and requires a fully checkpointed + /// main while its original authorized SQLite writer still owns the family. + pub async fn finish_initial_schema(&self) -> Result<(), ExactSqlError> { + let authority = self + .write_authority + .clone() + .ok_or(ExactSqlError::AuthorityMismatch)?; + let (reply, response) = async_channel::bounded(1); + let cx = crate::native_runtime::cx().ok_or(ExactSqlError::WriterUnavailable)?; + // Schema publication is a serialized writer barrier, not an ordinary + // best-effort request. A busy bounded channel must not strand a + // prepared SQLite family when its existing work can still drain. + self.writer + .as_ref() + .ok_or(ExactSqlError::WriterUnavailable)? + .send(&cx, WriterCommand::FinishInitialSchema { reply, authority }) + .await + .map_err(map_writer_send_error)?; + response + .recv() + .await + .map_err(|_| ExactSqlError::WriterUnavailable)? + } + /// Checkpoints and truncates the WAL on the serialized writer connection. fn enqueue_checkpoint_wal_truncate( &self, diff --git a/crates/tracedecay-rusqlite-runtime/src/exact_sql/tests/async_dispatch.rs b/crates/tracedecay-rusqlite-runtime/src/exact_sql/tests/async_dispatch.rs index 2decbe4769e..5bea1d358e9 100644 --- a/crates/tracedecay-rusqlite-runtime/src/exact_sql/tests/async_dispatch.rs +++ b/crates/tracedecay-rusqlite-runtime/src/exact_sql/tests/async_dispatch.rs @@ -62,6 +62,86 @@ async fn poll_once(mut future: Pin<&mut F>) -> Poll { poll_fn(|context| Poll::Ready(future.as_mut().poll(context))).await } +#[test] +fn schema_finalization_waits_for_writer_capacity_without_losing_its_reply() { + current_thread().block_on(async { + let fixture = fixture('a', 'a'); + let mut handle = ExactSqlHandle::attach(&fixture.writer, &fixture.readers) + .unwrap() + .with_write_authority(Arc::new(AtomicWriteAuthority(Arc::new(AtomicBool::new( + true, + ))))) + .unwrap(); + // Occupy the only slot before polling the real finalization future. + // This reproduces a full exact-SQL channel without scheduling races. + let (sender, mut receiver) = actor_mpsc::channel(1); + let (first_reply, _first_response) = async_channel::bounded(1); + let authority = handle.write_authority.clone().unwrap(); + assert!( + sender + .try_send(WriterCommand::FinishInitialSchema { + reply: first_reply, + authority, + }) + .is_ok() + ); + handle.writer = Some(sender); + + let mut finishing = Box::pin(handle.finish_initial_schema()); + assert!(poll_once(finishing.as_mut()).await.is_pending()); + let first = receiver.try_recv().unwrap(); + match first { + WriterCommand::FinishInitialSchema { reply, .. } => { + assert!(reply.try_send(Ok(())).is_ok()); + } + _ => panic!("the occupied slot must hold the first schema barrier"), + } + + // Releasing capacity admits the queued finalization in FIFO order; + // success still requires the writer's own completion reply. + assert!(poll_once(finishing.as_mut()).await.is_pending()); + let second = receiver.try_recv().unwrap(); + match second { + WriterCommand::FinishInitialSchema { reply, .. } => { + assert!(reply.try_send(Ok(())).is_ok()); + } + _ => panic!("the deferred schema barrier was not admitted"), + } + finishing.await.unwrap(); + }); +} + +#[test] +fn schema_finalization_refuses_a_closed_writer_after_capacity_wait() { + current_thread().block_on(async { + let fixture = fixture('a', 'a'); + let mut handle = ExactSqlHandle::attach(&fixture.writer, &fixture.readers) + .unwrap() + .with_write_authority(Arc::new(AtomicWriteAuthority(Arc::new(AtomicBool::new( + true, + ))))) + .unwrap(); + let (sender, receiver) = actor_mpsc::channel(1); + let (reply, _response) = async_channel::bounded(1); + assert!( + sender + .try_send(WriterCommand::FinishInitialSchema { + reply, + authority: handle.write_authority.clone().unwrap(), + }) + .is_ok() + ); + handle.writer = Some(sender); + let mut finishing = Box::pin(handle.finish_initial_schema()); + assert!(poll_once(finishing.as_mut()).await.is_pending()); + drop(receiver); + assert!(matches!( + finishing.await, + Err(ExactSqlError::WriterUnavailable) + )); + }); +} + #[test] fn synchronous_adapters_remain_safe_inside_current_thread_tokio() { current_thread().block_on(async { diff --git a/crates/tracedecay-rusqlite-runtime/src/reader/pool/mod.rs b/crates/tracedecay-rusqlite-runtime/src/reader/pool/mod.rs index cddd0cc8c56..f44dbf21185 100644 --- a/crates/tracedecay-rusqlite-runtime/src/reader/pool/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/reader/pool/mod.rs @@ -29,6 +29,7 @@ use worker::SnapshotRevoker; mod lease; mod outcome; +mod shutdown; pub use lease::{ReaderLease, SnapshotLease}; pub use outcome::{ReaderAcquireError, ReaderPoolSnapshot, ReaderPoolState}; @@ -463,6 +464,9 @@ pub(super) struct PoolInner { /// registration. #[cfg(test)] after_spawn: Mutex>>, + /// Test-only: runs between the quiescence observation and ownership claim. + #[cfg(test)] + before_shutdown_claim: Mutex>>, } impl Drop for PoolInner { @@ -586,6 +590,8 @@ impl ReaderPool { admission: Arc::new(ReaderAdmissionRecorder::default()), #[cfg(test)] after_spawn: Mutex::new(None), + #[cfg(test)] + before_shutdown_claim: Mutex::new(None), }); let pool = Self { inner }; for _ in 0..pool.inner.budget.min_per_hot_shard { diff --git a/crates/tracedecay-rusqlite-runtime/src/reader/pool/shutdown.rs b/crates/tracedecay-rusqlite-runtime/src/reader/pool/shutdown.rs new file mode 100644 index 00000000000..c59434bc33f --- /dev/null +++ b/crates/tracedecay-rusqlite-runtime/src/reader/pool/shutdown.rs @@ -0,0 +1,91 @@ +//! Exclusive ownership is the shutdown boundary, not an observed Arc count. + +use std::sync::Arc; + +use super::{PoolInner, ReaderPool, ReaderQueryExecutor}; + +/// Owns the reader workers after every upgradeable pool reference is fenced. +/// The caller drops this outside its attachment lock so worker joins cannot +/// deadlock an attachment callback. +pub(crate) struct ReaderPoolShutdown { + inner: PoolInner, +} + +impl ReaderPoolShutdown { + pub(crate) fn join(self) { + drop(self.inner); + } +} + +impl ReaderPool { + /// Atomically retire the last pool reference before reporting shutdown. + /// + /// `is_quiescent` is only a hint: an exact-SQL handle can upgrade its weak + /// reference after the count check. A plain drop would then leave SQLite + /// connections alive past publication or release of the file identity pin. + /// A failed claim returns the same pool, retaining all workers and pins. + pub(crate) fn try_into_shutdown(self) -> Result, Self> { + #[cfg(test)] + if let Some(hook) = self + .inner + .before_shutdown_claim + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take() + { + hook(); + } + match Arc::try_unwrap(self.inner) { + Ok(inner) => Ok(ReaderPoolShutdown { inner }), + Err(inner) => Err(Self { inner }), + } + } + + #[cfg(test)] + pub(crate) fn run_once_before_shutdown_claim(&self, hook: impl FnOnce() + Send + 'static) { + *self + .inner + .before_shutdown_claim + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Box::new(hook)); + } +} + +#[cfg(test)] +mod tests { + use tracedecay_store::AdmissionConfigV1; + + use crate::reader::ReaderPool; + use crate::reader::tests::{CountExecutor, TestStore}; + + #[test] + fn a_late_weak_upgrade_cannot_outlive_an_acknowledged_reader_shutdown() { + let store = TestStore::new(); + let pool = ReaderPool::start( + store.locator(), + AdmissionConfigV1::default().readers, + CountExecutor, + ) + .unwrap(); + let weak = pool.downgrade(); + pool.begin_shutdown_drain(); + assert!(pool.is_quiescent()); + + // This is the interleaving between the attachment's advisory count + // check and retirement; no sleep or scheduler race is needed. + let late = weak.upgrade().unwrap(); + let pool = match pool.try_into_shutdown() { + Err(pool) => pool, + Ok(_) => panic!("a live late reference must prevent shutdown acknowledgment"), + }; + assert!(!pool.workers_exited()); + drop(late); + + let shutdown = match pool.try_into_shutdown() { + Ok(shutdown) => shutdown, + Err(_) => panic!("the last pool reference must be claimable"), + }; + assert!(weak.upgrade().is_none()); + shutdown.join(); + } +} diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs index 2ad9b08b1d7..9fc6a6a75bd 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs @@ -19,7 +19,7 @@ use tracedecay_store::{ use crate::{ CheckpointOutcome, CheckpointRequest, ExistingWriterLocator, PersistentWriter, RuntimeWriteAuthority, WriterActorError, WriterStartError, WriterState, - connection::{OpenedDatabaseFile, OpenedDatabaseFileError}, + connection::{OpenedDatabaseFile, OpenedDatabaseFileError, opened_file_identity}, exact_sql::{ExactSqlError, ExactSqlHandle}, ledger::{self, LedgerError}, reader::{ @@ -54,8 +54,8 @@ impl RepositoryPhysicalAttachmentFactory { path: PathBuf, admission: AdmissionConfigV1, ) -> Result { - let opened_database = - OpenedDatabaseFile::pin(&path).map_err(RepositoryAttachmentStartError::Identity)?; + let opened_database = OpenedDatabaseFile::pin_initialized(&path) + .map_err(RepositoryAttachmentStartError::Identity)?; let reader_locator = ExistingReaderLocator::new(binding.clone(), locator, path.clone()) .map_err(RepositoryAttachmentStartError::Reader)?; let reader_locator = reader_locator.with_opened_database( @@ -90,12 +90,15 @@ impl RepositoryPhysicalAttachmentFactory { readers: Some(readers), admission_open: true, drained: false, + workers_closing: false, closed: false, file_lost: false, lost_readers: None, lost_writer: None, #[cfg(test)] after_lost_notice: None, + #[cfg(test)] + before_worker_join: None, close_failure: None, retained_file_pin: Some(opened_database), })), @@ -123,8 +126,8 @@ impl RepositoryPhysicalAttachmentFactory { admission: AdmissionConfigV1, start_hook: &mut dyn FnMut(AttachmentWorkerStartStage), ) -> Result { - let opened_database = - OpenedDatabaseFile::pin(&path).map_err(RepositoryAttachmentStartError::Identity)?; + let opened_database = OpenedDatabaseFile::pin_initialized(&path) + .map_err(RepositoryAttachmentStartError::Identity)?; self.attach_opened( runtime, binding, @@ -137,27 +140,39 @@ impl RepositoryPhysicalAttachmentFactory { ) } + /// Initialization owns its sole set of raw descriptors across the entire + /// worker start. On a handled failure, only a fully released worker set + /// can return those original reservations to the external publisher. #[tracing::instrument(name = "rusqlite.attachment.initialize", level = "trace", skip_all)] - pub fn initialize( + pub fn start_reserved_initialization( &self, runtime: &RuntimeHandle, binding: StoreRuntimeBindingV1, locator: VerifiedStoreLocatorV1, path: PathBuf, admission: AdmissionConfigV1, - ) -> Result { - let opened_database = OpenedDatabaseFile::create_new(&path) - .map_err(RepositoryAttachmentStartError::Identity)?; - self.attach_opened( - runtime, - binding, - locator, - path, - admission, - opened_database, - true, - &mut |_| {}, - ) + mut resumed: Option, + ) -> Result { + let opened = if resumed.is_some() { + OpenedDatabaseFile::resume_initialization(&path, &mut resumed) + } else { + OpenedDatabaseFile::create_new(&path) + } + .map_err(|error| { + RepositoryInitializationStartFailure::before_workers(error, resumed.take()) + })?; + start_with_reserved_files(opened, |worker_pin| { + self.attach_opened( + runtime, + binding, + locator, + path, + admission, + worker_pin, + true, + &mut |_| {}, + ) + }) } #[allow(clippy::too_many_arguments)] @@ -310,12 +325,15 @@ impl RepositoryPhysicalAttachmentFactory { readers: Some(readers), admission_open: true, drained: false, + workers_closing: false, closed: false, file_lost: false, lost_readers: None, lost_writer: None, #[cfg(test)] after_lost_notice: None, + #[cfg(test)] + before_worker_join: None, close_failure: None, retained_file_pin, })); @@ -341,12 +359,67 @@ fn repository_start_failure( created: bool, failure: RepositoryAttachmentStartError, ) -> RepositoryAttachmentStartError { - if created && let Err(error) = opened_database.discard_created(database_path) { - return RepositoryAttachmentStartError::Identity(error); + if created { + // No portable syscall atomically unlinks a pathname *only if* it still + // identifies our inode. Even a second identity check followed by an + // unlink can destroy a concurrent replacement, including WAL/SHM and + // rollback journal files. Preserve the failed store for explicit + // reconciliation instead of deleting unknown operator bytes. + tracing::warn!( + path = %database_path.display(), + "failed SQLite initialization preserved its database and sidecars" + ); } + drop(opened_database); failure } +fn start_with_reserved_files( + opened: OpenedDatabaseFile, + start: impl FnOnce( + OpenedDatabaseFile, + ) + -> Result, +) -> Result { + let started = opened + .try_clone() + .map_err(RepositoryAttachmentStartError::Identity) + .and_then(start); + started.map_err(|error| RepositoryInitializationStartFailure { + error, + files: opened + .take_idle_initialization_files() + .map(|files| (opened.identity(), files)), + }) +} + +pub type InitializationFiles = (std::fs::File, [std::fs::File; 3]); + +#[derive(Debug)] +pub struct RepositoryInitializationStartFailure { + pub error: RepositoryAttachmentStartError, + pub files: Option<(u64, InitializationFiles)>, +} + +impl RepositoryInitializationStartFailure { + fn before_workers( + mut error: OpenedDatabaseFileError, + originals: Option, + ) -> Self { + let files = originals.and_then(|files| match opened_file_identity(&files.0) { + Ok(identity) => Some((identity, files)), + Err(inaccessible) => { + error = inaccessible; + None + } + }); + Self { + error: RepositoryAttachmentStartError::Identity(error), + files, + } + } +} + #[derive(Debug)] pub enum RepositoryAttachmentStartError { Identity(crate::connection::OpenedDatabaseFileError), @@ -390,6 +463,9 @@ struct RepositoryRuntimePhysicalState { readers: Option>, admission_open: bool, drained: bool, + /// Worker owners are temporarily outside the state lock while joining. + /// Empty live/lost slots do not prove their SQLite connections have closed. + workers_closing: bool, closed: bool, file_lost: bool, /// The reader pool of a lost file, kept until its workers have exited so @@ -403,6 +479,9 @@ struct RepositoryRuntimePhysicalState { /// connection open for as long as it blocks. #[cfg(test)] after_lost_notice: Option>, + /// Test-only: holds the extracted workers before the out-of-lock join. + #[cfg(test)] + before_worker_join: Option>, close_failure: Option, /// Kept until all SQLite workers exit: closing any raw fd for the inode /// while SQLite holds POSIX locks can release those locks process-wide. @@ -444,10 +523,13 @@ impl RepositoryRuntimePhysicalAttachment { } /// Whether a lost file has no open connection left: the writer task - /// finished and every reader worker exited. Also true for an attachment - /// whose file was never lost. + /// finished and every reader worker exited, including owners currently + /// joining outside the state lock. Otherwise true when no close is pending. pub fn lost_file_closed(&self) -> bool { let mut state = self.lock_state(); + if state.workers_closing { + return false; + } if state .lost_readers .as_ref() @@ -479,8 +561,80 @@ impl RepositoryRuntimePhysicalAttachment { self.lock_state().opened_file_identity } + /// Retire initialization's SQLite readers before the writer's final WAL + /// checkpoint. A live reader snapshot can make TRUNCATE report BUSY even + /// after every schema write has committed. Keep the writer and all original + /// file pins alive until the checkpoint has completed. + pub fn close_initialization_readers(&self) -> Result<(), String> { + if crate::storage_pool::on_pool() { + return Err("initialization readers cannot close on a storage pool thread".to_owned()); + } + let _serial = self + .drain_serial + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + { + let state = self.lock_state(); + if !state.admission_open || state.closed || state.initialization_file.is_none() { + return Err( + "only an unpublished live initialization may close its readers".to_owned(), + ); + } + let readers = state + .readers + .as_ref() + .ok_or_else(|| "initialization readers have already closed".to_owned())?; + // Fence both general and reserved health acquisitions before + // checking for the last retained snapshot. + readers.begin_shutdown_drain(); + } + let started = Instant::now(); + loop { + let mut state = self.lock_state(); + if state.file_lost { + return Err( + "SQLite initialization lost its database while draining readers".to_owned(), + ); + } + let quiescent = state.readers.as_ref().is_some_and(ReaderPool::is_quiescent); + if quiescent && let Some(readers) = state.readers.take() { + match readers.try_into_shutdown() { + Ok(readers) => { + state.workers_closing = true; + drop(state); + // Exclusive ownership fences late weak upgrades before + // joining. Keep every file pin until the joins finish. + readers.join(); + let mut state = self.lock_state(); + state.workers_closing = false; + if state.file_lost { + return Err( + "SQLite initialization lost its database while joining readers" + .to_owned(), + ); + } + return Ok(()); + } + Err(readers) => state.readers = Some(readers), + } + } + if started.elapsed() >= ATTACHMENT_DRAIN_TIMEOUT { + // Keep the draining readers owned by the attachment so its + // ordinary failure teardown can still join every worker. + return Err(format!( + "initialization readers did not quiesce within {ATTACHMENT_DRAIN_TIMEOUT:?}" + )); + } + drop(state); + thread::sleep(ATTACHMENT_DRAIN_POLL); + } + } + pub fn commit_initialization(&self) -> Result<(), String> { let mut state = self.lock_state(); + if !state.admission_open || state.closed { + return Err("repository initialization is no longer open".to_owned()); + } let opened = state .initialization_file .as_ref() @@ -494,6 +648,29 @@ impl RepositoryRuntimePhysicalAttachment { Ok(()) } + /// Hands the exclusive original reservations to the publisher after all + /// SQLite workers have exited. No raw descriptor is closed during a live + /// POSIX SQLite transaction. + pub fn close_pinned_initialization( + &self, + ) -> Result<(std::fs::File, [std::fs::File; 3]), String> { + self.commit_initialization()?; + self.drain()?; + let files = { + let state = self.lock_state(); + if state.file_lost { + return Err("initial schema lost its database generation".to_owned()); + } + state + .retained_file_pin + .as_ref() + .and_then(OpenedDatabaseFile::take_completed_files) + .ok_or_else(|| "initial schema has no original sidecar reservations".to_owned())? + }; + self.close_and_join()?; + Ok(files) + } + pub fn abort_initialization(&self) -> Result<(), String> { self.drain()?; self.close_and_join()?; @@ -501,9 +678,24 @@ impl RepositoryRuntimePhysicalAttachment { let Some(opened) = state.initialization_file.take() else { return Ok(()); }; - opened - .discard_created(&state.database_path) - .map_err(|error| error.to_string()) + // Aborting publication does not prove ownership of any current + // pathname. Preserve the database and every sidecar, just as on a + // worker-start failure; another process may have replaced them. + tracing::warn!( + path = %state.database_path.display(), + "aborted SQLite initialization preserved its database and sidecars" + ); + if state.file_lost { + // Drain can finish while a lost-file worker is still returning. + // Reuse the ordinary lost-worker teardown rather than closing a + // raw descriptor that could discard that worker's POSIX locks. + state.retained_file_pin = Some(opened); + drop(state); + let _ = self.lost_file_closed(); + } else { + opened.release_after_workers_exit(); + } + Ok(()) } pub fn exact_sql_handle(&self) -> Result { @@ -716,6 +908,9 @@ impl RepositoryRuntimePhysicalAttachment { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let mut state = self.lock_state(); + if let Some(message) = &state.close_failure { + return Err(message.clone()); + } if state.closed { return Ok(()); } @@ -732,15 +927,32 @@ impl RepositoryRuntimePhysicalAttachment { drop(state); let started = Instant::now(); - loop { - let state = self.lock_state(); + let (writer, readers) = loop { + let mut state = self.lock_state(); let writer_quiescent = state.writer.as_ref().is_none_or(|writer| { Arc::strong_count(writer) == 1 && writer.telemetry_snapshot().queue.queued_operations == 0 }); let readers_quiescent = state.readers.as_ref().is_none_or(ReaderPool::is_quiescent); if writer_quiescent && readers_quiescent { - break; + match state.writer.take().map(Arc::try_unwrap).transpose() { + Ok(writer) => match state + .readers + .take() + .map(ReaderPool::try_into_shutdown) + .transpose() + { + Ok(readers) => { + state.workers_closing = true; + break (writer, readers); + } + Err(readers) => { + state.writer = writer.map(Arc::new); + state.readers = Some(readers); + } + }, + Err(writer) => state.writer = Some(writer), + } } if started.elapsed() >= ATTACHMENT_DRAIN_TIMEOUT { let queued = state.writer.as_ref().map_or(0, |writer| { @@ -756,27 +968,29 @@ impl RepositoryRuntimePhysicalAttachment { } drop(state); thread::sleep(ATTACHMENT_DRAIN_POLL); - } - - let mut state = self.lock_state(); - let writer = match state.writer.take().map(Arc::try_unwrap).transpose() { - Ok(writer) => writer, - Err(writer) => { - state.writer = Some(writer); - return Err("repository writer is still serving a request".to_owned()); - } }; - let readers = state.readers.take(); - drop(state); - drop(readers); - if let Some(writer) = writer - && let Err(error) = writer.shutdown_and_join() + + #[cfg(test)] { + let hook = self.lock_state().before_worker_join.take(); + if let Some(hook) = hook { + hook(); + } + } + if let Some(readers) = readers { + readers.join(); + } + // Neither the joins nor the writer's lost-file callback run under the + // attachment lock. Health checks still see their in-flight ownership. + let joined = writer.map_or(Ok(()), PersistentWriter::shutdown_and_join); + let mut state = self.lock_state(); + state.workers_closing = false; + if let Err(error) = joined { let message = format!("join repository writer: {error}"); - self.lock_state().close_failure = Some(message.clone()); + state.close_failure = Some(message.clone()); return Err(message); } - self.lock_state().drained = true; + state.drained = true; Ok(()) } @@ -792,7 +1006,7 @@ impl RepositoryRuntimePhysicalAttachment { if state.admission_open { return Err("repository physical attachment must drain before close".to_owned()); } - if !state.drained { + if !state.drained || state.workers_closing { return Err("repository physical attachment has not completed drain".to_owned()); } if state.writer.is_some() || state.readers.is_some() { @@ -955,12 +1169,16 @@ fn infrastructure(operation: impl Into) -> StorageRuntimeErrorV1 { #[cfg(test)] mod tests { + mod shutdown; + use std::{fs, sync::mpsc, thread, time::Duration}; use tempfile::TempDir; use tracedecay_domain::LocatorDigest; use tracedecay_store::{AdmissionConfigV1, StoreIncarnationV1}; + #[cfg(unix)] + use crate::connection::path_identity_tests::probe_external_connection; use crate::exact_sql::{ExactSqlError, ExactSqlStatement, ExactSqlValue}; use asupersync::channel::watch; @@ -1026,6 +1244,590 @@ mod tests { .unwrap(); } + #[test] + fn initialization_reader_shutdown_releases_pinned_wal_snapshot_before_finalization() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("schema.sqlite3"); + let binding = binding(); + let attachment = Arc::new( + RepositoryPhysicalAttachmentFactory + .start_reserved_initialization( + &native_runtime(), + binding.clone(), + locator(&binding), + path.clone(), + AdmissionConfigV1::default(), + None, + ) + .unwrap(), + ); + let handle = attachment.exact_sql_handle().unwrap(); + handle + .execute_batch( + "CREATE TABLE durable (value INTEGER NOT NULL); INSERT INTO durable VALUES (1)" + .to_owned(), + ) + .unwrap(); + let snapshot = handle.begin_read_snapshot(Duration::from_secs(1)).unwrap(); + snapshot + .query(statement("SELECT COUNT(*) FROM durable", vec![])) + .unwrap(); + handle + .execute_batch("INSERT INTO durable VALUES (2)".to_owned()) + .unwrap(); + + let (started_tx, started_rx) = mpsc::sync_channel(1); + let (finished_tx, finished_rx) = mpsc::sync_channel(1); + let closing = Arc::clone(&attachment); + let join = thread::spawn(move || { + started_tx.send(()).unwrap(); + finished_tx + .send(closing.close_initialization_readers()) + .unwrap(); + }); + started_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + assert!( + matches!( + finished_rx.recv_timeout(Duration::from_millis(50)), + Err(mpsc::RecvTimeoutError::Timeout) + ), + "a live SQLite read snapshot must hold finalization's reader fence" + ); + drop(snapshot); + assert_eq!( + finished_rx.recv_timeout(Duration::from_secs(10)).unwrap(), + Ok(()) + ); + join.join().unwrap(); + + assert!( + handle + .begin_read_snapshot(Duration::from_millis(50)) + .is_err(), + "no reader may reopen between the final checkpoint and publication" + ); + assert!( + attachment.snapshot().writer_present, + "the serialized SQLite writer must outlive its readers" + ); + let files = attachment.close_pinned_initialization().unwrap(); + drop(files); + let reopened = rusqlite::Connection::open(&path).unwrap(); + let rows: i64 = reopened + .query_row("SELECT COUNT(*) FROM durable", [], |row| row.get(0)) + .unwrap(); + let integrity: String = reopened + .query_row("PRAGMA integrity_check", [], |row| row.get(0)) + .unwrap(); + assert_eq!(rows, 2); + assert_eq!(integrity, "ok"); + } + + #[test] + fn abort_initialization_preserves_committed_bytes_and_releases_shared_pins() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("aborted.sqlite3"); + let binding = binding(); + let attachment = RepositoryPhysicalAttachmentFactory + .start_reserved_initialization( + &native_runtime(), + binding.clone(), + locator(&binding), + path.clone(), + AdmissionConfigV1::default(), + None, + ) + .unwrap(); + let held_pin = attachment + .lock_state() + .initialization_file + .as_ref() + .unwrap() + .try_clone() + .unwrap(); + let handle = attachment.exact_sql_handle().unwrap(); + handle + .execute_batch("CREATE TABLE abort_probe (value INTEGER)".to_owned()) + .unwrap(); + handle + .execute(statement("INSERT INTO abort_probe VALUES (42)", vec![])) + .unwrap(); + + attachment.abort_initialization().unwrap(); + assert!( + path.is_file(), + "abort must not unlink the initialized store" + ); + assert!(held_pin.try_clone().is_err()); + { + let state = attachment.lock_state(); + assert!(state.closed && state.drained); + assert!(state.initialization_file.is_none()); + assert!(state.retained_file_pin.is_none()); + } + assert!(attachment.commit_initialization().is_err()); + assert_eq!( + handle + .execute(statement("INSERT INTO abort_probe VALUES (99)", vec![])) + .unwrap_err(), + ExactSqlError::WriterUnavailable + ); + let reopened = rusqlite::Connection::open_with_flags( + &path, + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .unwrap(); + assert_eq!( + reopened + .query_row("SELECT value FROM abort_probe", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(), + 42 + ); + assert_eq!( + reopened + .query_row("PRAGMA integrity_check", [], |row| row.get::<_, String>(0)) + .unwrap(), + "ok" + ); + drop(reopened); + let preserved = fs::read(&path).unwrap(); + attachment.abort_initialization().unwrap(); + drop(attachment); + assert_eq!(fs::read(&path).unwrap(), preserved); + } + + #[cfg(unix)] + #[test] + fn abort_initialization_preserves_foreign_cross_process_replacements() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("aborted.sqlite3"); + let binding = binding(); + let attachment = RepositoryPhysicalAttachmentFactory + .start_reserved_initialization( + &native_runtime(), + binding.clone(), + locator(&binding), + path.clone(), + AdmissionConfigV1::default(), + None, + ) + .unwrap(); + let held_pin = attachment + .lock_state() + .initialization_file + .as_ref() + .unwrap() + .try_clone() + .unwrap(); + attachment.drain().unwrap(); + attachment.close_and_join().unwrap(); + assert!(attachment.commit_initialization().is_err()); + let original = fs::read(&path).unwrap(); + // SQLite has finished with these fixture files. Populate all three + // names so the separate process replaces real inodes, not absences. + for suffix in ["-wal", "-shm", "-journal"] { + let mut sidecar = path.as_os_str().to_os_string(); + sidecar.push(suffix); + fs::write(std::path::Path::new(&sidecar), b"owned fixture").unwrap(); + } + let (_, module) = module_path!().split_once("::").unwrap(); + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + &format!("{module}::failed_initialization_foreign_replacement_child"), + "--nocapture", + ]) + .env("TRACEDECAY_FAILED_SQLITE_REPLACEMENT", &path) + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success() + && stdout.contains("1 passed") + && stdout.contains("foreign-sqlite-replacement-complete"), + "separate-process replacement failed: {stdout} {}", + String::from_utf8_lossy(&output.stderr) + ); + + attachment.abort_initialization().unwrap(); + assert!(held_pin.try_clone().is_err()); + attachment.abort_initialization().unwrap(); + drop(attachment); + assert_eq!(fs::read(&path).unwrap(), b"foreign database"); + assert_eq!(fs::read(path.with_extension("owned")).unwrap(), original); + for suffix in ["-wal", "-shm", "-journal"] { + let mut sidecar = path.as_os_str().to_os_string(); + sidecar.push(suffix); + assert_eq!( + fs::read(std::path::Path::new(&sidecar)).unwrap(), + format!("foreign {suffix}").as_bytes() + ); + sidecar.push(".owned"); + assert_eq!( + fs::read(std::path::Path::new(&sidecar)).unwrap(), + b"owned fixture" + ); + } + } + + #[cfg(unix)] + #[test] + fn abort_initialization_retains_the_pin_until_a_lost_writer_exits() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("lost-initialization.sqlite3"); + let binding = binding(); + let attachment = RepositoryPhysicalAttachmentFactory + .start_reserved_initialization( + &native_runtime(), + binding.clone(), + locator(&binding), + path.clone(), + AdmissionConfigV1::default(), + None, + ) + .unwrap(); + let held_pin = attachment + .lock_state() + .initialization_file + .as_ref() + .unwrap() + .try_clone() + .unwrap(); + let (hold, held) = mpsc::channel::<()>(); + attachment.lock_state().after_lost_notice = Some(Box::new(move || { + let _ = held.recv(); + })); + fs::remove_file(&path).unwrap(); + let deadline = Instant::now() + crate::FILE_PROBE_INTERVAL + Duration::from_secs(10); + while !attachment + .lock_state() + .lost_readers + .as_ref() + .is_some_and(ReaderPool::workers_exited) + { + assert!(Instant::now() < deadline, "lost readers never exited"); + thread::sleep(Duration::from_millis(20)); + } + assert!(attachment.file_lost()); + attachment.abort_initialization().unwrap(); + assert!(attachment.lock_state().initialization_file.is_none()); + assert!(attachment.lock_state().retained_file_pin.is_some()); + assert!(held_pin.try_clone().is_ok()); + assert!(!attachment.lost_file_closed()); + + hold.send(()).unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + while attachment.lock_state().retained_file_pin.is_some() { + let _ = attachment.snapshot(); + assert!(Instant::now() < deadline, "lost initialization pin leaked"); + thread::sleep(Duration::from_millis(5)); + } + assert!(attachment.lost_file_closed()); + assert!(held_pin.try_clone().is_err()); + } + + #[test] + fn malformed_existing_sqlite_refuses_before_touching_sidecars() { + let binding = binding(); + for size in [512_usize, 513, 1025, 4096] { + for suffix in ["-wal", "-shm", "-journal"] { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("sessions.db"); + let mut bytes = vec![0_u8; size]; + bytes[..16].copy_from_slice(b"SQLite format 3\0"); + fs::write(&path, &bytes).unwrap(); + + let mut sidecar = path.as_os_str().to_os_string(); + sidecar.push(suffix); + let sidecar = PathBuf::from(sidecar); + let sentinel = format!("foreign {suffix} bytes"); + fs::write(&sidecar, sentinel.as_bytes()).unwrap(); + + let result = RepositoryPhysicalAttachmentFactory.attach( + &native_runtime(), + binding.clone(), + locator(&binding), + path.clone(), + AdmissionConfigV1::default(), + ); + assert!(matches!( + result, + Err(RepositoryAttachmentStartError::Identity( + OpenedDatabaseFileError::InvalidHeader + )) + )); + assert_eq!(fs::read(&path).unwrap(), bytes); + assert_eq!(fs::read(&sidecar).unwrap(), sentinel.as_bytes()); + } + } + } + + /// The factory must refuse foreign sidecars before starting SQLite, and a + /// failed first creation must not become an existing database on retry. + /// Test the actual attachment entry points rather than just the pin helper. + #[test] + fn occupied_sidecar_refuses_initialization_and_existing_attachment_retries() { + let binding = binding(); + for suffix in ["-wal", "-shm", "-journal"] { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("sessions.db"); + let mut sidecar = path.as_os_str().to_os_string(); + sidecar.push(suffix); + let sidecar = PathBuf::from(sidecar); + let sentinel = format!("unowned {suffix} content"); + fs::write(&sidecar, sentinel.as_bytes()).unwrap(); + + let initialized = RepositoryPhysicalAttachmentFactory.start_reserved_initialization( + &native_runtime(), + binding.clone(), + locator(&binding), + path.clone(), + AdmissionConfigV1::default(), + None, + ); + assert!(matches!( + initialized, + Err(RepositoryInitializationStartFailure { + error: RepositoryAttachmentStartError::Identity( + OpenedDatabaseFileError::Reserve + ), + files: None, + }) + )); + assert_eq!(fs::metadata(&path).unwrap().len(), 0); + assert_eq!(fs::read(&sidecar).unwrap(), sentinel.as_bytes()); + + let existing = RepositoryPhysicalAttachmentFactory.attach( + &native_runtime(), + binding.clone(), + locator(&binding), + path.clone(), + AdmissionConfigV1::default(), + ); + assert!(matches!( + existing, + Err(RepositoryAttachmentStartError::Identity( + OpenedDatabaseFileError::InvalidHeader + )) + )); + let read_only = RepositoryPhysicalAttachmentFactory.attach_read_only( + binding.clone(), + locator(&binding), + path.clone(), + AdmissionConfigV1::default(), + ); + assert!(matches!( + read_only, + Err(RepositoryAttachmentStartError::Identity( + OpenedDatabaseFileError::InvalidHeader + )) + )); + assert_eq!(fs::metadata(&path).unwrap().len(), 0); + assert_eq!(fs::read(&sidecar).unwrap(), sentinel.as_bytes()); + } + } + + #[test] + fn reader_start_failure_returns_idle_initialization_files_for_retry() { + use std::process::Command; + + const CHILD: &str = "TRACEDECAY_TEST_SQLITE_READER_START_LIMIT"; + if std::env::var_os(CHILD).is_none() { + let name = format!( + "{}::reader_start_failure_returns_idle_initialization_files_for_retry", + module_path!().split_once("::").unwrap().1 + ); + let result = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", &name, "--nocapture"]) + .env(CHILD, "1") + .output() + .unwrap(); + assert!( + result.status.success() + && String::from_utf8_lossy(&result.stdout).contains("1 passed"), + "{} {}", + String::from_utf8_lossy(&result.stdout), + String::from_utf8_lossy(&result.stderr) + ); + return; + } + // Bundled SQLite may default process-wide allocation tracking off. + // This isolated child has not opened SQLite yet. + // SAFETY: SQLITE_CONFIG_MEMSTATUS consumes a single C int. + let configured = + unsafe { rusqlite::ffi::sqlite3_config(rusqlite::ffi::SQLITE_CONFIG_MEMSTATUS, 1_i32) }; + assert_eq!(configured, rusqlite::ffi::SQLITE_OK); + let directory = TempDir::new().unwrap(); + let path = directory.path().join("sessions.db"); + let binding = binding(); + let runtime = native_runtime(); + let files = OpenedDatabaseFile::create_new(&path).unwrap(); + let factory = RepositoryPhysicalAttachmentFactory; + let outcome = start_with_reserved_files(files, |worker_pin| { + factory.attach_opened( + &runtime, + binding.clone(), + locator(&binding), + path.clone(), + AdmissionConfigV1::default(), + worker_pin, + true, + &mut |stage| { + // SQLite's process-wide supported memory ceiling makes the + // next reader open report SQLITE_NOMEM. The writer has + // already reported successful startup in this isolated process. + if stage == AttachmentWorkerStartStage::BeforeReaders { + // SAFETY: sqlite3_hard_heap_limit64 takes no pointers. + unsafe { + rusqlite::ffi::sqlite3_hard_heap_limit64(1); + } + } else if stage == AttachmentWorkerStartStage::AfterReaders { + // SAFETY: zero disables the limit before worker shutdown. + unsafe { + rusqlite::ffi::sqlite3_hard_heap_limit64(0); + } + } + }, + ) + }); + let failure = match outcome { + Ok(_) => panic!("reader started under exhausted SQLite budget"), + Err(error) => error, + }; + assert!( + matches!(failure.error, RepositoryAttachmentStartError::Reader(_)), + "{}", + failure.error + ); + let mut cause: &(dyn Error + 'static) = &failure.error; + while let Some(next) = cause.source() { + if cause.is::() { + break; + } + cause = next; + } + assert_eq!( + cause + .downcast_ref::() + .and_then(rusqlite::Error::sqlite_error_code), + Some(rusqlite::ErrorCode::OutOfMemory) + ); + let (identity, files) = failure + .files + .expect("writer and readers returned all original file handles"); + let reopened = factory + .start_reserved_initialization( + &runtime, + binding.clone(), + locator(&binding), + path.clone(), + AdmissionConfigV1::default(), + Some(files), + ) + .unwrap(); + assert!(reopened.snapshot().healthy); + assert_eq!(reopened.opened_file_identity(), identity); + reopened.drain().unwrap(); + reopened.close_and_join().unwrap(); + } + + #[test] + fn failed_initialization_preserves_created_database_and_sidecars() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("failed.db"); + let opened = OpenedDatabaseFile::create_new(&path).unwrap(); + let error = repository_start_failure( + opened, + &path, + true, + RepositoryAttachmentStartError::Identity(OpenedDatabaseFileError::Inspect), + ); + assert!(matches!( + error, + RepositoryAttachmentStartError::Identity(OpenedDatabaseFileError::Inspect) + )); + assert!(path.is_file()); + for suffix in ["-wal", "-shm", "-journal"] { + let mut sidecar = path.as_os_str().to_os_string(); + sidecar.push(suffix); + assert!(std::path::Path::new(&sidecar).is_file()); + } + } + + #[cfg(unix)] + #[test] + fn failed_initialization_foreign_replacement_child() { + let Some(path) = std::env::var_os("TRACEDECAY_FAILED_SQLITE_REPLACEMENT") else { + return; + }; + let path = std::path::PathBuf::from(path); + fs::rename(&path, path.with_extension("owned")).unwrap(); + fs::write(&path, b"foreign database").unwrap(); + for suffix in ["-wal", "-shm", "-journal"] { + let mut sidecar = path.as_os_str().to_os_string(); + sidecar.push(suffix); + let sidecar = std::path::PathBuf::from(sidecar); + let mut displaced = sidecar.as_os_str().to_os_string(); + displaced.push(".owned"); + fs::rename(&sidecar, std::path::PathBuf::from(displaced)).unwrap(); + fs::write(&sidecar, format!("foreign {suffix}")).unwrap(); + } + println!("foreign-sqlite-replacement-complete"); + } + + #[cfg(unix)] + #[test] + fn failed_initialization_cannot_delete_foreign_cross_process_replacements() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("failed.db"); + let opened = OpenedDatabaseFile::create_new(&path).unwrap(); + let (_, module) = module_path!().split_once("::").unwrap(); + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + &format!("{module}::failed_initialization_foreign_replacement_child"), + "--nocapture", + ]) + .env("TRACEDECAY_FAILED_SQLITE_REPLACEMENT", &path) + .output() + .unwrap(); + assert!( + output.status.success() + && String::from_utf8_lossy(&output.stdout) + .contains("foreign-sqlite-replacement-complete"), + "separate-process filesystem replacement failed: {} {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + + let error = repository_start_failure( + opened, + &path, + true, + RepositoryAttachmentStartError::Identity(OpenedDatabaseFileError::Inspect), + ); + assert!(matches!( + error, + RepositoryAttachmentStartError::Identity(OpenedDatabaseFileError::Inspect) + )); + assert_eq!(fs::read(&path).unwrap(), b"foreign database"); + assert!(path.with_extension("owned").is_file()); + for suffix in ["-wal", "-shm", "-journal"] { + let mut sidecar = path.as_os_str().to_os_string(); + sidecar.push(suffix); + let sidecar = std::path::PathBuf::from(sidecar); + assert_eq!( + fs::read(&sidecar).unwrap(), + format!("foreign {suffix}").as_bytes() + ); + let mut displaced = sidecar.as_os_str().to_os_string(); + displaced.push(".owned"); + assert!(std::path::Path::new(&displaced).is_file()); + } + } + /// Unix-only: on Windows the writer's pin denies the swap-back rename /// outright, so this race cannot occur there; /// `connection::tests::windows_pinned_file_blocks_replacement_until_authority_closes` @@ -1122,6 +1924,99 @@ mod tests { .unwrap() } + #[cfg(unix)] + #[test] + fn existing_physical_attachments_preserve_an_independent_writers_locks() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("sessions.db"); + let writer = rusqlite::Connection::open(&path).unwrap(); + writer + .execute_batch( + "PRAGMA journal_mode=DELETE; + CREATE TABLE sample(value INTEGER); + INSERT INTO sample VALUES(3); + BEGIN IMMEDIATE; + UPDATE sample SET value=7", + ) + .unwrap(); + let binding = binding(); + let factory = RepositoryPhysicalAttachmentFactory; + let read_only = factory + .attach_read_only( + binding.clone(), + locator(&binding), + path.clone(), + AdmissionConfigV1::default(), + ) + .unwrap(); + probe_external_connection(&path, "write", true); + read_only.drain().unwrap(); + read_only.close_and_join().unwrap(); + probe_external_connection(&path, "write", true); + + let mut invalid = AdmissionConfigV1::default(); + invalid.readers.min_per_hot_shard = 0; + assert!(matches!( + factory.attach_read_only(binding.clone(), locator(&binding), path.clone(), invalid), + Err(RepositoryAttachmentStartError::Reader( + ReaderStartError::InvalidReaderBudget(_) + )) + )); + probe_external_connection(&path, "write", true); + writer.execute_batch("COMMIT").unwrap(); + probe_external_connection(&path, "write", false); + + // Dropping the route alias exercises the real writer's lost-file + // notification while an independent connection keeps the DB open + // through its physical pathname, with its WAL family intact. + let route = directory.path().join("route.db"); + std::os::unix::fs::symlink(&path, &route).unwrap(); + let writable = factory + .attach( + &native_runtime(), + binding.clone(), + locator(&binding), + route.clone(), + AdmissionConfigV1::default(), + ) + .unwrap(); + writer + .execute_batch("BEGIN IMMEDIATE; UPDATE sample SET value=11") + .unwrap(); + probe_external_connection(&path, "write", true); + fs::remove_file(route).unwrap(); + let deadline = + std::time::Instant::now() + crate::FILE_PROBE_INTERVAL + Duration::from_secs(10); + while !(writable.file_lost() && writable.lost_file_closed()) { + assert!( + std::time::Instant::now() < deadline, + "physical attachment did not close the lost route" + ); + thread::sleep(Duration::from_millis(10)); + } + writable.drain().unwrap(); + writable.close_and_join().unwrap(); + probe_external_connection(&path, "write", true); + probe_external_connection(&path, "journal", true); + writer.execute_batch("ROLLBACK").unwrap(); + drop(writer); + probe_external_connection(&path, "write", false); + probe_external_connection(&path, "journal", false); + let checked = rusqlite::Connection::open(&path).unwrap(); + assert_eq!( + checked + .query_row("SELECT value FROM sample", [], |row| row.get::<_, i64>(0)) + .unwrap(), + 7 + ); + assert_eq!( + checked + .query_row("PRAGMA integrity_check", [], |row| row.get::<_, String>(0)) + .unwrap(), + "ok" + ); + } + /// The attachment's original identity descriptor must not be dropped /// during startup or initialization commit while SQLite workers are live. /// On Unix, closing it can release the process's unrelated SQLite locks. @@ -1187,12 +2082,13 @@ mod tests { let created_path = directory.path().join("created.sqlite3"); let created = RepositoryPhysicalAttachmentFactory - .initialize( + .start_reserved_initialization( &native_runtime(), binding.clone(), locator(&binding), created_path.clone(), AdmissionConfigV1::default(), + None, ) .unwrap(); assert!( diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/attachment/tests/shutdown.rs b/crates/tracedecay-rusqlite-runtime/src/repository/attachment/tests/shutdown.rs new file mode 100644 index 00000000000..e9c91280958 --- /dev/null +++ b/crates/tracedecay-rusqlite-runtime/src/repository/attachment/tests/shutdown.rs @@ -0,0 +1,240 @@ +use std::{ + env, + path::Path, + process::Command, + sync::{Arc, mpsc}, + thread, + time::Duration, +}; + +use rusqlite::{Connection, OpenFlags}; +use tempfile::TempDir; +use tracedecay_store::AdmissionConfigV1; + +use super::super::RepositoryPhysicalAttachmentFactory; +#[cfg(unix)] +use super::super::close_lost_file; +use super::{binding, locator, native_runtime}; +#[cfg(unix)] +use crate::connection::path_identity_tests::probe_external_connection; + +const SNAPSHOT_PATH: &str = "TRACEDECAY_SHUTDOWN_SNAPSHOT_PATH"; +const SNAPSHOT_DONE: &str = "tracedecay-shutdown-snapshot-complete"; + +fn assert_external_snapshot(path: &Path) { + let (_, module) = module_path!().split_once("::").unwrap(); + let output = Command::new(env::current_exe().unwrap()) + .args([ + "--exact", + &format!("{module}::external_shutdown_snapshot"), + "--nocapture", + ]) + .env(SNAPSHOT_PATH, path) + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success() && stdout.contains("1 passed") && stdout.contains(SNAPSHOT_DONE), + "external snapshot failed: {stdout} {}", + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn external_shutdown_snapshot() { + let Some(path) = env::var_os(SNAPSHOT_PATH) else { + return; + }; + let connection = + Connection::open_with_flags(Path::new(&path), OpenFlags::SQLITE_OPEN_READ_ONLY).unwrap(); + assert_eq!( + connection + .query_row("SELECT value FROM sample", [], |row| row.get::<_, i64>(0)) + .unwrap(), + 41 + ); + assert_eq!( + connection + .query_row("PRAGMA integrity_check", [], |row| row.get::<_, String>(0)) + .unwrap(), + "ok" + ); + println!("{SNAPSHOT_DONE}"); +} + +#[derive(Clone, Copy)] +enum CloseMode { + InitializationReaders, + Attachment, +} + +fn exercise_late_reader_owner(mode: CloseMode) { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("shutdown.sqlite3"); + let binding = binding(); + let attachment = Arc::new( + RepositoryPhysicalAttachmentFactory + .start_reserved_initialization( + &native_runtime(), + binding.clone(), + locator(&binding), + path.clone(), + AdmissionConfigV1::default(), + None, + ) + .unwrap(), + ); + attachment + .exact_sql_handle() + .unwrap() + .execute_batch( + "CREATE TABLE sample(value INTEGER); INSERT INTO sample VALUES(41)".to_owned(), + ) + .unwrap(); + let pin = attachment + .lock_state() + .initialization_file + .as_ref() + .unwrap() + .try_clone() + .unwrap(); + if matches!(mode, CloseMode::Attachment) { + attachment.commit_initialization().unwrap(); + } + let independent_writer = Connection::open(&path).unwrap(); + independent_writer + .execute_batch("BEGIN IMMEDIATE; UPDATE sample SET value=99") + .unwrap(); + let (late_tx, late_rx) = mpsc::channel(); + let weak = { + let state = attachment.lock_state(); + let readers = state.readers.as_ref().unwrap(); + let weak = readers.downgrade(); + let upgrading = weak.clone(); + readers.run_once_before_shutdown_claim(move || { + late_tx.send(upgrading.upgrade().unwrap()).unwrap(); + }); + weak + }; + #[cfg(unix)] + probe_external_connection(&path, "write", true); + assert_external_snapshot(&path); + let (finished_tx, finished_rx) = mpsc::channel(); + let closing = Arc::clone(&attachment); + let join = thread::spawn(move || { + let result = match mode { + CloseMode::InitializationReaders => closing.close_initialization_readers(), + CloseMode::Attachment => closing.drain(), + }; + finished_tx.send(result).unwrap(); + }); + + // The hook upgrades only after the production caller has observed + // quiescence, forcing the count-check/retirement race without a sleep. + let late = late_rx.recv_timeout(Duration::from_secs(10)).unwrap(); + assert!(matches!( + finished_rx.recv_timeout(Duration::from_millis(50)), + Err(mpsc::RecvTimeoutError::Timeout) + )); + assert!(!late.workers_exited()); + assert!(pin.try_clone().is_ok()); + assert!(attachment.lock_state().readers.is_some()); + + independent_writer.execute_batch("ROLLBACK").unwrap(); + drop(late); + assert_eq!( + finished_rx.recv_timeout(Duration::from_secs(10)).unwrap(), + Ok(()) + ); + join.join().unwrap(); + assert!(weak.upgrade().is_none()); + match mode { + CloseMode::InitializationReaders => { + drop(attachment.close_pinned_initialization().unwrap()); + } + CloseMode::Attachment => attachment.close_and_join().unwrap(), + } + assert!(pin.try_clone().is_err()); + #[cfg(unix)] + probe_external_connection(&path, "write", false); + assert_external_snapshot(&path); +} + +#[test] +fn initialization_reader_close_waits_for_a_late_weak_upgrade() { + exercise_late_reader_owner(CloseMode::InitializationReaders); +} + +#[test] +fn attachment_drain_waits_for_a_late_weak_upgrade() { + exercise_late_reader_owner(CloseMode::Attachment); +} + +#[cfg(unix)] +#[test] +fn lost_notice_during_out_of_lock_join_preserves_pins_and_writer_exclusion() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("joining.sqlite3"); + let binding = binding(); + let attachment = Arc::new( + RepositoryPhysicalAttachmentFactory + .start_reserved_initialization( + &native_runtime(), + binding.clone(), + locator(&binding), + path.clone(), + AdmissionConfigV1::default(), + None, + ) + .unwrap(), + ); + attachment + .exact_sql_handle() + .unwrap() + .execute_batch( + "CREATE TABLE sample(value INTEGER); INSERT INTO sample VALUES(41)".to_owned(), + ) + .unwrap(); + attachment.commit_initialization().unwrap(); + let pin = attachment + .lock_state() + .retained_file_pin + .as_ref() + .unwrap() + .try_clone() + .unwrap(); + let independent_writer = Connection::open(&path).unwrap(); + independent_writer + .execute_batch("BEGIN IMMEDIATE; UPDATE sample SET value=99") + .unwrap(); + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + attachment.lock_state().before_worker_join = Some(Box::new(move || { + entered_tx.send(()).unwrap(); + // A failed parent assertion drops the sender and also releases this join. + let _ = release_rx.recv(); + })); + let closing = Arc::clone(&attachment); + let join = thread::spawn(move || closing.drain()); + entered_rx.recv_timeout(Duration::from_secs(10)).unwrap(); + + // Deliver the real writer callback at the exact interval where its owner + // has left the state slots but its SQLite connection has not been joined. + // Keep the files in place so an independent OS process can probe the locks. + close_lost_file(&attachment.state); + assert!(attachment.file_lost()); + assert!(!attachment.lost_file_closed()); + assert!(attachment.close_and_join().is_err()); + assert!(pin.try_clone().is_ok()); + probe_external_connection(&path, "write", true); + assert_external_snapshot(&path); + + independent_writer.execute_batch("ROLLBACK").unwrap(); + release_tx.send(()).unwrap(); + assert_eq!(join.join().unwrap(), Ok(())); + attachment.close_and_join().unwrap(); + assert!(attachment.lost_file_closed()); + assert!(pin.try_clone().is_err()); + probe_external_connection(&path, "write", false); + assert_external_snapshot(&path); +} diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/mod.rs b/crates/tracedecay-rusqlite-runtime/src/repository/mod.rs index fd0794fc386..6133e464cac 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/mod.rs @@ -46,7 +46,8 @@ use crate::StorageOperationExecutor; use crate::operation::StorageOperationError; pub use attachment::{ - RepositoryAttachmentStartError, RepositoryDispatchError, RepositoryPhysicalAttachmentFactory, + InitializationFiles, RepositoryAttachmentStartError, RepositoryDispatchError, + RepositoryInitializationStartFailure, RepositoryPhysicalAttachmentFactory, RepositoryRuntimePhysicalAttachment, RepositoryRuntimePhysicalSnapshot, RepositoryWriterRuntimeSnapshot, }; diff --git a/crates/tracedecay-rusqlite-runtime/src/schema_version.rs b/crates/tracedecay-rusqlite-runtime/src/schema_version.rs index 565e4f762e3..80b66b5592f 100644 --- a/crates/tracedecay-rusqlite-runtime/src/schema_version.rs +++ b/crates/tracedecay-rusqlite-runtime/src/schema_version.rs @@ -11,6 +11,10 @@ use std::task::{Context, Poll}; use crate::exact_sql::ExactSqlValue; +/// SQLite header application ID for a store whose complete registered schema +/// was installed in private staging before publication ("TDST"). +pub const REGISTERED_STORE_APPLICATION_ID: i32 = i32::from_be_bytes(*b"TDST"); + /// Ledger table. One row per component. pub const TABLE: &str = "tracedecay_schema_version"; diff --git a/crates/tracedecay-rusqlite-runtime/src/writer/worker/mod.rs b/crates/tracedecay-rusqlite-runtime/src/writer/worker/mod.rs index b18573ea65f..e7f850579f3 100644 --- a/crates/tracedecay-rusqlite-runtime/src/writer/worker/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/writer/worker/mod.rs @@ -423,14 +423,16 @@ impl Worker { return self.fail_start(WriterStartError::OpenedDatabaseIdentity(error)); } #[cfg(unix)] - if self.expected_file_identity.is_some() - && connection + if self.expected_file_identity.is_some() { + let actual = connection .path() - .and_then(|path| std::fs::canonicalize(path).ok()) - .as_deref() - != Some(self.canonical_path.as_path()) - { - return self.fail_start(WriterStartError::OpenedDatabasePathMismatch); + .and_then(|path| std::fs::canonicalize(path).ok()); + // Both paths may traverse the host's temporary-directory alias. + // File identity was checked against the caller's pathname above. + let expected = std::fs::canonicalize(&self.canonical_path); + if !matches!(expected, Ok(expected) if actual.as_ref() == Some(&expected)) { + return self.fail_start(WriterStartError::OpenedDatabasePathMismatch); + } } let opened_file_identity = match self.expected_file_identity { Some(expected) => { diff --git a/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/repository_attachment.rs b/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/repository_attachment.rs index ea8f1d287e0..226462a7acc 100644 --- a/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/repository_attachment.rs +++ b/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/repository_attachment.rs @@ -25,6 +25,16 @@ use tracedecay_store::{ use tracedecay_domain::test_fixtures::id; +/// Existing-attachment fixtures must contain a real SQLite header. Merely +/// opening and closing a new SQLite pathname leaves a zero-byte file, which +/// safe admission correctly treats as incomplete initialization. +fn initialized_existing_file(path: &std::path::Path) { + Connection::open(path) + .unwrap() + .execute_batch("PRAGMA user_version = 0") + .unwrap(); +} + fn binding() -> StoreRuntimeBindingV1 { serde_json::from_value(serde_json::json!({ "shard_id": StoreShardIdV1::project( @@ -110,7 +120,7 @@ fn health_request( fn repository_factory_attaches_writer_and_reserved_reader_runtime() { let directory = TempDir::new().unwrap(); let path = directory.path().join("project.db"); - Connection::open(&path).unwrap(); + initialized_existing_file(&path); let path = path.canonicalize().unwrap(); let binding = binding(); let locator = VerifiedStoreLocatorV1::new( @@ -148,7 +158,7 @@ fn repository_factory_attaches_writer_and_reserved_reader_runtime() { fn a_typed_read_awaited_on_a_native_worker_parks_no_native_thread() { let directory = TempDir::new().unwrap(); let path = directory.path().join("sessions.db"); - Connection::open(&path).unwrap(); + initialized_existing_file(&path); let path = path.canonicalize().unwrap(); let binding = binding(); let locator = VerifiedStoreLocatorV1::new( @@ -206,7 +216,7 @@ fn a_typed_read_awaited_on_a_native_worker_parks_no_native_thread() { fn temporal_health_dispatch_uses_the_reserved_reader_lane() { let directory = TempDir::new().unwrap(); let path = directory.path().join("sessions.db"); - Connection::open(&path).unwrap(); + initialized_existing_file(&path); let path = path.canonicalize().unwrap(); let binding = binding(); let locator = VerifiedStoreLocatorV1::new( @@ -239,7 +249,7 @@ fn temporal_health_dispatch_uses_the_reserved_reader_lane() { fn at_least_reads_refuse_coverage_without_a_committed_watermark() { let directory = TempDir::new().unwrap(); let path = directory.path().join("sessions.db"); - Connection::open(&path).unwrap(); + initialized_existing_file(&path); let path = path.canonicalize().unwrap(); let binding = binding(); let locator = VerifiedStoreLocatorV1::new( diff --git a/crates/tracedecay/src/daemon/production_harness/drop_shutdown_journey_test.rs b/crates/tracedecay/src/daemon/production_harness/drop_shutdown_journey_test.rs index 7b159cbcf6a..d45218c298a 100644 --- a/crates/tracedecay/src/daemon/production_harness/drop_shutdown_journey_test.rs +++ b/crates/tracedecay/src/daemon/production_harness/drop_shutdown_journey_test.rs @@ -198,8 +198,10 @@ async fn a_failed_later_mount_releases_the_whole_composition_before_returning() Err(error) => error, }; assert!( - format!("{error:?}").contains("file is not a database"), - "expected the second mount to refuse its graph store: {error:?}" + format!("{error:?}").contains( + "existing SQLite file has no valid database header; refusing sidecar access", + ), + "expected the second mount to refuse its graph store before sidecar access: {error:?}" ); let first_layout = default_profile_sharded_layout(&first, &profile) diff --git a/crates/tracedecay/src/daemon/tests/bootstrap.rs b/crates/tracedecay/src/daemon/tests/bootstrap.rs index 34f8b7dbcf1..a4a5e68ed10 100644 --- a/crates/tracedecay/src/daemon/tests/bootstrap.rs +++ b/crates/tracedecay/src/daemon/tests/bootstrap.rs @@ -1772,6 +1772,20 @@ async fn explicit_init_retries_after_joining_an_ordinary_missing_database_open() let client_identity = test_client_identity_for(profile_root.clone()); let layout = initialize_test_project(&project, &client_identity).await; std::fs::remove_file(&layout.graph_db_path).expect("remove generated project database"); + // The fixture models a genuinely absent SQLite basename, not an + // incomplete store with orphan WAL, SHM or journal entries. The fixture + // project was closed by initialize_test_project above, so its test-owned + // sidecars can be retired here. Production admission must instead refuse + // unknown sidecars rather than erase them during retry. + for suffix in ["-wal", "-shm", "-journal"] { + let mut name = layout.graph_db_path.as_os_str().to_os_string(); + name.push(suffix); + match std::fs::remove_file(std::path::PathBuf::from(name)) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => panic!("remove isolated SQLite fixture sidecar: {error}"), + } + } let missing_graph_db_path = layout.graph_db_path.clone(); let _database_scope = diff --git a/crates/tracedecay/tests/transcript_ingest_suite/projector_upgrade.rs b/crates/tracedecay/tests/transcript_ingest_suite/projector_upgrade.rs index a5c905f4a69..d77c053b177 100644 --- a/crates/tracedecay/tests/transcript_ingest_suite/projector_upgrade.rs +++ b/crates/tracedecay/tests/transcript_ingest_suite/projector_upgrade.rs @@ -101,7 +101,7 @@ fn shape_store_as_shipped(db_path: &Path) { /// so the next open recreates it empty. fn reset_store(db_path: &Path) { std::fs::remove_file(db_path).unwrap(); - for suffix in ["-wal", "-shm"] { + for suffix in ["-wal", "-shm", "-journal"] { let mut sidecar = db_path.as_os_str().to_owned(); sidecar.push(suffix); match std::fs::remove_file(sidecar) { diff --git a/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/stale_profile_authority_reset.rs b/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/stale_profile_authority_reset.rs index 741912ffad7..05f8370032c 100644 --- a/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/stale_profile_authority_reset.rs +++ b/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/stale_profile_authority_reset.rs @@ -201,6 +201,7 @@ fn released_profile_authority_resets_alone_and_projects_register_again() { removed, [ &PathBuf::from("global.db"), + &PathBuf::from("global.db-journal"), &PathBuf::from("global.db-shm"), &PathBuf::from("global.db-wal") ], diff --git a/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/stale_project_sessions_census.rs b/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/stale_project_sessions_census.rs index e34cd6c2303..74fb1033d84 100644 --- a/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/stale_project_sessions_census.rs +++ b/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/stale_project_sessions_census.rs @@ -114,6 +114,7 @@ fn one_scoped_reset_clears_the_profile_authority_and_every_project_sessions_stor .collect(); let mut expected_removed = vec![ PathBuf::from("global.db"), + PathBuf::from("global.db-journal"), PathBuf::from("global.db-shm"), PathBuf::from("global.db-wal"), ]; @@ -122,6 +123,7 @@ fn one_scoped_reset_clears_the_profile_authority_and_every_project_sessions_stor expected_removed.extend([ store.join(".sessions.db.host-admission/meta.json"), store.join("sessions.db"), + store.join("sessions.db-journal"), store.join("sessions.db-shm"), store.join("sessions.db-wal"), store.join("sessions.db.delivery-settlement-spool-v1/writer.v1.lock"), diff --git a/tests/storage_runtime_rusqlite_suite/repository_parity.rs b/tests/storage_runtime_rusqlite_suite/repository_parity.rs index c5a8f1f4f50..5103dee8bb0 100644 --- a/tests/storage_runtime_rusqlite_suite/repository_parity.rs +++ b/tests/storage_runtime_rusqlite_suite/repository_parity.rs @@ -146,7 +146,12 @@ fn profile_project_and_session_production_mounts_serve_health_data_ports() { for (family, shard_id) in families { let path = directory.path().join(format!("{family}.db")); - Connection::open(&path).unwrap(); + // Materialize a header without adding schema objects: a bare open + // leaves an incomplete zero-byte store, now correctly rejected. + Connection::open(&path) + .unwrap() + .execute_batch("PRAGMA user_version = 0") + .unwrap(); let path = path.canonicalize().unwrap(); assert_family_mount(family_binding(shard_id), &path); }