From 92d53ff3f0de91c5e6d9c15f58ab4b86307d91a3 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 18:22:42 -0800 Subject: [PATCH 01/60] fix(sqlite): reserve new sidecar namespace and preserve failed files Refs #4513 --- .../src/connection/mod.rs | 67 +++++++---- .../src/connection/tests.rs | 45 ++++++-- .../src/repository/attachment.rs | 105 +++++++++++++++++- 3 files changed, 183 insertions(+), 34 deletions(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs index fd4dae1b6ff..777b6db5445 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs @@ -69,6 +69,10 @@ pub(crate) struct OpenedDatabaseFile { /// descriptor and closing a clone while SQLite holds POSIX locks would /// release locks belonging to other live connections to the same inode. file: Arc>>, + // Exclusive reservations for a *new* basename. Hold the descriptors until + // all SQLite workers exit: closing an unrelated fd during a live SQLite + // transaction can release process-owned POSIX locks. + created_sidecars: Option>>>, identity: u64, } @@ -80,7 +84,22 @@ impl OpenedDatabaseFile { pub(crate) fn create_new(path: &Path) -> Result { let file = create_pinned_database(path).map_err(|_| OpenedDatabaseFileError::Create)?; - Self::adopt(file) + // O_EXCL on the database alone does not own SQLite's basename: + // SQLite may overwrite an orphan WAL/SHM/journal even on its first + // successful open. O_EXCL-reserve every sibling before handing the + // database to SQLite; never truncate an existing member or symlink. + // A failed reservation deliberately leaves prior creations untouched: + // pathname-based rollback could unlink an entry replaced by another + // process. An incomplete namespace needs explicit reconciliation. + let mut opened = Self::adopt(file)?; + let wal = reserve_new_sidecar(&sidecar_path(path, "-wal")) + .map_err(|_| OpenedDatabaseFileError::Reserve)?; + let shm = reserve_new_sidecar(&sidecar_path(path, "-shm")) + .map_err(|_| OpenedDatabaseFileError::Reserve)?; + let journal = reserve_new_sidecar(&sidecar_path(path, "-journal")) + .map_err(|_| OpenedDatabaseFileError::Reserve)?; + opened.created_sidecars = Some(Arc::new(Mutex::new(Some([wal, shm, journal])))); + Ok(opened) } /// Takes ownership of an already-open handle and records its identity. @@ -94,6 +113,7 @@ impl OpenedDatabaseFile { let identity = opened_file_identity(&file)?; Ok(Self { file: Arc::new(Mutex::new(Some(file))), + created_sidecars: None, identity, }) } @@ -105,6 +125,12 @@ impl OpenedDatabaseFile { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .take(); + if let Some(sidecars) = &self.created_sidecars { + sidecars + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + } } pub(crate) const fn identity(&self) -> u64 { @@ -130,6 +156,7 @@ impl OpenedDatabaseFile { self.with_file(|_| ())?; Ok(Self { file: Arc::clone(&self.file), + created_sidecars: self.created_sidecars.as_ref().map(Arc::clone), identity: self.identity, }) } @@ -274,24 +301,6 @@ impl OpenedDatabaseFile { Ok(()) } - pub(crate) fn discard_created(self, path: &Path) -> Result<(), OpenedDatabaseFileError> { - self.verify_current_path(path)?; - let Self { file, .. } = self; - drop(file); - for candidate in [ - sidecar_path(path, "-wal"), - sidecar_path(path, "-shm"), - sidecar_path(path, "-journal"), - path.to_path_buf(), - ] { - match std::fs::remove_file(candidate) { - Ok(()) => {} - Err(error) if error.kind() == io::ErrorKind::NotFound => {} - Err(_) => return Err(OpenedDatabaseFileError::Remove), - } - } - Ok(()) - } } /// Check a live SQLite worker's physical file without opening and closing a @@ -316,13 +325,13 @@ pub(crate) fn live_path_identity(path: &Path) -> Result) -> fmt::Result { let message = match self { Self::Create => "could not create the canonical SQLite file", + Self::Reserve => "could not exclusively reserve the new SQLite sidecar namespace", Self::Open => "could not open the verified SQLite file", Self::Inspect => "could not inspect the verified SQLite file descriptor", Self::NotFile => "verified SQLite locator is not a regular file", #[cfg(windows)] Self::Identify => "could not identify the verified SQLite file descriptor", Self::Replaced => "verified SQLite file was replaced while opening workers", - Self::Remove => "could not remove an uncommitted canonical SQLite file", #[cfg(not(any(unix, windows)))] Self::Unsupported => "SQLite file identity is unsupported on this platform", }; @@ -383,6 +392,22 @@ fn create_pinned_database(path: &Path) -> io::Result { .open(path) } +/// SQLite is allowed to delete or recreate an owned journal/sidecar. Its +/// namespace reservation is exclusive at creation, but its file handle must +/// not prevent SQLite's own cleanup on Windows. +fn reserve_new_sidecar(path: &Path) -> io::Result { + let mut options = OpenOptions::new(); + options.read(true).write(true).create_new(true); + #[cfg(windows)] + { + use std::os::windows::fs::OpenOptionsExt; + options.share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE); + } + options.open(path) +} + +#[cfg(windows)] +const FILE_SHARE_DELETE: u32 = 0x0000_0004; #[cfg(windows)] const FILE_SHARE_READ: u32 = 0x0000_0001; #[cfg(windows)] diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs index be04666e521..f3011b6934b 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs @@ -374,16 +374,39 @@ fn policy_requires_an_existing_database() { } #[test] -fn create_new_pins_and_discards_the_exact_database() { +fn create_new_reserves_the_full_sqlite_basename_without_unverified_cleanup() { let directory = tempfile::tempdir().unwrap(); let path = directory.path().join("fresh.db"); - let created = OpenedDatabaseFile::create_new(&path).unwrap(); assert!(path.is_file()); assert_ne!(created.identity(), 0); - created.discard_created(&path).unwrap(); + for suffix in ["-wal", "-shm", "-journal"] { + let sidecar = sidecar_path(&path, suffix); + assert_eq!(std::fs::metadata(&sidecar).unwrap().len(), 0); + } + drop(created); + // Dropping a reservation must not unlink mutable names. In particular, a + // foreign process may have substituted an unrelated inode since creation. + assert!(path.is_file()); + for suffix in ["-wal", "-shm", "-journal"] { + assert!(sidecar_path(&path, suffix).is_file()); + } +} - assert!(!path.exists()); +#[test] +fn create_new_refuses_existing_sidecar_members_without_touching_their_bytes() { + for suffix in ["-wal", "-shm", "-journal"] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("fresh.db"); + let foreign = sidecar_path(&path, suffix); + let bytes = format!("foreign SQLite {suffix} bytes"); + std::fs::write(&foreign, bytes.as_bytes()).unwrap(); + assert!(matches!( + OpenedDatabaseFile::create_new(&path), + Err(OpenedDatabaseFileError::Reserve) + )); + assert_eq!(std::fs::read(&foreign).unwrap(), bytes.as_bytes()); + } } #[test] @@ -582,7 +605,7 @@ fn windows_pinned_file_blocks_replacement_until_authority_closes() { #[cfg(windows)] #[test] -fn windows_discard_created_removes_the_complete_sqlite_family() { +fn windows_dropping_reservations_preserves_sidecars_written_after_creation() { let directory = tempfile::tempdir().unwrap(); let path = directory.path().join("fresh.db"); let created = OpenedDatabaseFile::create_new(&path).unwrap(); @@ -592,13 +615,13 @@ fn windows_discard_created_removes_the_complete_sqlite_family() { sidecar_path(&path, "-journal"), ]; for sidecar in &sidecars { - std::fs::write(sidecar, b"sidecar").unwrap(); + std::fs::write(sidecar, b"foreign sidecar").unwrap(); + } + drop(created); + assert!(path.is_file()); + for sidecar in &sidecars { + assert_eq!(std::fs::read(sidecar).unwrap(), b"foreign sidecar"); } - - created.discard_created(&path).unwrap(); - - assert!(!path.exists()); - assert!(sidecars.iter().all(|sidecar| !sidecar.exists())); } /// `sqlite_host_path` shortens a verbatim *disk* path so SQLite's win32 VFS diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs index 2ad9b08b1d7..932eda06feb 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs @@ -341,9 +341,18 @@ fn repository_start_failure( created: bool, failure: RepositoryAttachmentStartError, ) -> RepositoryAttachmentStartError { - if created && let Err(error) = opened_database.discard_created(database_path) { - return RepositoryAttachmentStartError::Identity(error); + if created { + // No portable syscall atomically unlinks a pathname *only if* it still + // identifies our inode. Even a second identity check followed by an + // unlink can destroy a concurrent replacement, including WAL/SHM and + // rollback journal files. Preserve the failed store for explicit + // reconciliation instead of deleting unknown operator bytes. + tracing::warn!( + path = %database_path.display(), + "failed SQLite initialization preserved its database and sidecars" + ); } + drop(opened_database); failure } @@ -1026,6 +1035,98 @@ mod tests { .unwrap(); } + #[test] + fn failed_initialization_preserves_created_database_and_sidecars() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("failed.db"); + let opened = OpenedDatabaseFile::create_new(&path).unwrap(); + let error = repository_start_failure( + opened, + &path, + true, + RepositoryAttachmentStartError::Identity(OpenedDatabaseFileError::Inspect), + ); + assert!(matches!( + error, + RepositoryAttachmentStartError::Identity(OpenedDatabaseFileError::Inspect) + )); + assert!(path.is_file()); + for suffix in ["-wal", "-shm", "-journal"] { + let mut sidecar = path.as_os_str().to_os_string(); + sidecar.push(suffix); + assert!(std::path::Path::new(&sidecar).is_file()); + } + } + + #[cfg(unix)] + #[test] + fn failed_initialization_foreign_replacement_child() { + let Some(path) = std::env::var_os("TRACEDECAY_FAILED_SQLITE_REPLACEMENT") else { + return; + }; + let path = std::path::PathBuf::from(path); + fs::rename(&path, path.with_extension("owned")).unwrap(); + fs::write(&path, b"foreign database").unwrap(); + for suffix in ["-wal", "-shm", "-journal"] { + let mut sidecar = path.as_os_str().to_os_string(); + sidecar.push(suffix); + let sidecar = std::path::PathBuf::from(sidecar); + let mut displaced = sidecar.as_os_str().to_os_string(); + displaced.push(".owned"); + fs::rename(&sidecar, std::path::PathBuf::from(displaced)).unwrap(); + fs::write(&sidecar, format!("foreign {suffix}")).unwrap(); + } + println!("foreign-sqlite-replacement-complete"); + } + + #[cfg(unix)] + #[test] + fn failed_initialization_cannot_delete_foreign_cross_process_replacements() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("failed.db"); + let opened = OpenedDatabaseFile::create_new(&path).unwrap(); + let (_, module) = module_path!().split_once("::").unwrap(); + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + &format!("{module}::failed_initialization_foreign_replacement_child"), + "--nocapture", + ]) + .env("TRACEDECAY_FAILED_SQLITE_REPLACEMENT", &path) + .output() + .unwrap(); + assert!( + output.status.success() + && String::from_utf8_lossy(&output.stdout) + .contains("foreign-sqlite-replacement-complete"), + "separate-process filesystem replacement failed: {} {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + + let error = repository_start_failure( + opened, + &path, + true, + RepositoryAttachmentStartError::Identity(OpenedDatabaseFileError::Inspect), + ); + assert!(matches!( + error, + RepositoryAttachmentStartError::Identity(OpenedDatabaseFileError::Inspect) + )); + assert_eq!(fs::read(&path).unwrap(), b"foreign database"); + assert!(path.with_extension("owned").is_file()); + for suffix in ["-wal", "-shm", "-journal"] { + let mut sidecar = path.as_os_str().to_os_string(); + sidecar.push(suffix); + let sidecar = std::path::PathBuf::from(sidecar); + assert_eq!(fs::read(&sidecar).unwrap(), format!("foreign {suffix}").as_bytes()); + let mut displaced = sidecar.as_os_str().to_os_string(); + displaced.push(".owned"); + assert!(std::path::Path::new(&displaced).is_file()); + } + } + /// Unix-only: on Windows the writer's pin denies the swap-back rename /// outright, so this race cannot occur there; /// `connection::tests::windows_pinned_file_blocks_replacement_until_authority_closes` From 1738c8e1de983e062053fb2544c62f3a9c222300 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 18:57:21 -0800 Subject: [PATCH 02/60] fix(sqlite): reject incomplete databases on existing attach --- .../src/connection/mod.rs | 27 ++++++++++++ .../src/connection/tests.rs | 41 +++++++++++++++++++ .../src/repository/attachment.rs | 4 +- 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs index 777b6db5445..25c95f39ecd 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs @@ -82,6 +82,31 @@ impl OpenedDatabaseFile { Self::adopt(file) } + /// Refuse unfinished stores before SQLite opens their WAL/SHM/journal. + /// A rejected new-basename reservation intentionally leaves the newly + /// created database in place so no pathname-based rollback can delete a + /// foreign replacement. That zero-length database must never become an + /// ordinary existing attachment on retry: SQLite would initialize it and + /// overwrite precisely the occupied sidecar that caused the refusal. + /// Read the magic through the existing pin; opening a second descriptor + /// here could discard another connection's POSIX locks on Unix. + pub(crate) fn pin_initialized(path: &Path) -> Result { + let opened = Self::pin(path)?; + let initialized = opened.with_file(|file| { + let mut reader = file; + let mut header = [0_u8; 16]; + match io::Read::read_exact(&mut reader, &mut header) { + Ok(()) => Ok(header == *b"SQLite format 3\0"), + Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => Ok(false), + Err(_) => Err(OpenedDatabaseFileError::Inspect), + } + })??; + if !initialized { + return Err(OpenedDatabaseFileError::InvalidHeader); + } + Ok(opened) + } + pub(crate) fn create_new(path: &Path) -> Result { let file = create_pinned_database(path).map_err(|_| OpenedDatabaseFileError::Create)?; // O_EXCL on the database alone does not own SQLite's basename: @@ -326,6 +351,7 @@ pub(crate) fn live_path_identity(path: &Path) -> Result "could not create the canonical SQLite file", Self::Reserve => "could not exclusively reserve the new SQLite sidecar namespace", + Self::InvalidHeader => "existing SQLite file has no valid database header; refusing sidecar access", Self::Open => "could not open the verified SQLite file", Self::Inspect => "could not inspect the verified SQLite file descriptor", Self::NotFile => "verified SQLite locator is not a regular file", diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs index f3011b6934b..d66446a0e94 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs @@ -409,6 +409,47 @@ fn create_new_refuses_existing_sidecar_members_without_touching_their_bytes() { } } +/// A reserved basename that failed on a foreign sidecar is left with an +/// empty main DB. A later existing-mode attachment must not let SQLite +/// initialize that empty file and destroy the foreign sidecar on retry. +#[test] +fn failed_sidecar_reservation_cannot_be_reopened_as_existing_sqlite() { + for suffix in ["-wal", "-shm", "-journal"] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("sessions.db"); + let foreign = sidecar_path(&path, suffix); + let bytes = format!("foreign occupied {suffix} bytes"); + std::fs::write(&foreign, bytes.as_bytes()).unwrap(); + + assert!(matches!( + OpenedDatabaseFile::create_new(&path), + Err(OpenedDatabaseFileError::Reserve) + )); + assert_eq!(std::fs::metadata(&path).unwrap().len(), 0); + assert!(matches!( + OpenedDatabaseFile::pin_initialized(&path), + Err(OpenedDatabaseFileError::InvalidHeader) + )); + assert_eq!(std::fs::read(&foreign).unwrap(), bytes.as_bytes()); + } +} + +#[test] +fn existing_sqlite_admission_requires_a_completed_header() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("sessions.db"); + std::fs::write(&path, b"invalid database, not SQLite").unwrap(); + assert!(matches!( + OpenedDatabaseFile::pin_initialized(&path), + Err(OpenedDatabaseFileError::InvalidHeader) + )); + std::fs::remove_file(&path).unwrap(); + let database = Connection::open(&path).unwrap(); + database.execute_batch("CREATE TABLE probe(value INTEGER)").unwrap(); + drop(database); + assert!(OpenedDatabaseFile::pin_initialized(&path).is_ok()); +} + #[test] fn create_new_refuses_to_replace_an_existing_database() { let file = NamedTempFile::new().unwrap(); diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs index 932eda06feb..97b7a4c565f 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs @@ -55,7 +55,7 @@ impl RepositoryPhysicalAttachmentFactory { admission: AdmissionConfigV1, ) -> Result { let opened_database = - OpenedDatabaseFile::pin(&path).map_err(RepositoryAttachmentStartError::Identity)?; + OpenedDatabaseFile::pin_initialized(&path).map_err(RepositoryAttachmentStartError::Identity)?; let reader_locator = ExistingReaderLocator::new(binding.clone(), locator, path.clone()) .map_err(RepositoryAttachmentStartError::Reader)?; let reader_locator = reader_locator.with_opened_database( @@ -124,7 +124,7 @@ impl RepositoryPhysicalAttachmentFactory { start_hook: &mut dyn FnMut(AttachmentWorkerStartStage), ) -> Result { let opened_database = - OpenedDatabaseFile::pin(&path).map_err(RepositoryAttachmentStartError::Identity)?; + OpenedDatabaseFile::pin_initialized(&path).map_err(RepositoryAttachmentStartError::Identity)?; self.attach_opened( runtime, binding, From a3c0a165a629db09a92c522065ebec91f0edcb3b Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 19:25:53 -0800 Subject: [PATCH 03/60] fix(sqlite): preserve aborted initialization and live worker pins Refs #4513 Remove the stale discard_created call left after unsafe pathname cleanup was deleted. Abort now preserves the database and all sidecars, releases shared descriptors only after normal worker teardown, and transfers a lost-file initialization pin to the existing deferred worker cleanup. Reject initialization publication once drain or close has begun. Add real runtime regressions for abort/read-back/stale handles, separate- process replacement of the database and WAL/SHM/journal after worker close, idempotent abort, and a lost writer held across initialization abort. These Rust tests still require nonzero Bazel and native macOS execution; no Cargo or direct Rust invocation was used. --- .../src/repository/attachment.rs | 236 +++++++++++++++++- 1 file changed, 228 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs index 97b7a4c565f..3aab9464f97 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs @@ -54,8 +54,8 @@ impl RepositoryPhysicalAttachmentFactory { path: PathBuf, admission: AdmissionConfigV1, ) -> Result { - let opened_database = - OpenedDatabaseFile::pin_initialized(&path).map_err(RepositoryAttachmentStartError::Identity)?; + let opened_database = OpenedDatabaseFile::pin_initialized(&path) + .map_err(RepositoryAttachmentStartError::Identity)?; let reader_locator = ExistingReaderLocator::new(binding.clone(), locator, path.clone()) .map_err(RepositoryAttachmentStartError::Reader)?; let reader_locator = reader_locator.with_opened_database( @@ -123,8 +123,8 @@ impl RepositoryPhysicalAttachmentFactory { admission: AdmissionConfigV1, start_hook: &mut dyn FnMut(AttachmentWorkerStartStage), ) -> Result { - let opened_database = - OpenedDatabaseFile::pin_initialized(&path).map_err(RepositoryAttachmentStartError::Identity)?; + let opened_database = OpenedDatabaseFile::pin_initialized(&path) + .map_err(RepositoryAttachmentStartError::Identity)?; self.attach_opened( runtime, binding, @@ -490,6 +490,9 @@ impl RepositoryRuntimePhysicalAttachment { pub fn commit_initialization(&self) -> Result<(), String> { let mut state = self.lock_state(); + if !state.admission_open || state.closed { + return Err("repository initialization is no longer open".to_owned()); + } let opened = state .initialization_file .as_ref() @@ -510,9 +513,24 @@ impl RepositoryRuntimePhysicalAttachment { let Some(opened) = state.initialization_file.take() else { return Ok(()); }; - opened - .discard_created(&state.database_path) - .map_err(|error| error.to_string()) + // Aborting publication does not prove ownership of any current + // pathname. Preserve the database and every sidecar, just as on a + // worker-start failure; another process may have replaced them. + tracing::warn!( + path = %state.database_path.display(), + "aborted SQLite initialization preserved its database and sidecars" + ); + if state.file_lost { + // Drain can finish while a lost-file worker is still returning. + // Reuse the ordinary lost-worker teardown rather than closing a + // raw descriptor that could discard that worker's POSIX locks. + state.retained_file_pin = Some(opened); + drop(state); + let _ = self.lost_file_closed(); + } else { + opened.release_after_workers_exit(); + } + Ok(()) } pub fn exact_sql_handle(&self) -> Result { @@ -1035,6 +1053,205 @@ mod tests { .unwrap(); } + #[test] + fn abort_initialization_preserves_committed_bytes_and_releases_shared_pins() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("aborted.sqlite3"); + let binding = binding(); + let attachment = RepositoryPhysicalAttachmentFactory + .initialize( + &native_runtime(), + binding.clone(), + locator(&binding), + path.clone(), + AdmissionConfigV1::default(), + ) + .unwrap(); + let held_pin = attachment + .lock_state() + .initialization_file + .as_ref() + .unwrap() + .try_clone() + .unwrap(); + let handle = attachment.exact_sql_handle().unwrap(); + handle + .execute_batch("CREATE TABLE abort_probe (value INTEGER)".to_owned()) + .unwrap(); + handle + .execute(statement("INSERT INTO abort_probe VALUES (42)", vec![])) + .unwrap(); + + attachment.abort_initialization().unwrap(); + assert!(path.is_file(), "abort must not unlink the initialized store"); + assert!(held_pin.try_clone().is_err()); + { + let state = attachment.lock_state(); + assert!(state.closed && state.drained); + assert!(state.initialization_file.is_none()); + assert!(state.retained_file_pin.is_none()); + } + assert!(attachment.commit_initialization().is_err()); + assert_eq!( + handle + .execute(statement("INSERT INTO abort_probe VALUES (99)", vec![])) + .unwrap_err(), + ExactSqlError::WriterUnavailable + ); + let reopened = rusqlite::Connection::open_with_flags( + &path, + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .unwrap(); + assert_eq!( + reopened + .query_row("SELECT value FROM abort_probe", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(), + 42 + ); + assert_eq!( + reopened + .query_row("PRAGMA integrity_check", [], |row| row.get::<_, String>(0)) + .unwrap(), + "ok" + ); + drop(reopened); + let preserved = fs::read(&path).unwrap(); + attachment.abort_initialization().unwrap(); + drop(attachment); + assert_eq!(fs::read(&path).unwrap(), preserved); + } + + #[cfg(unix)] + #[test] + fn abort_initialization_preserves_foreign_cross_process_replacements() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("aborted.sqlite3"); + let binding = binding(); + let attachment = RepositoryPhysicalAttachmentFactory + .initialize( + &native_runtime(), + binding.clone(), + locator(&binding), + path.clone(), + AdmissionConfigV1::default(), + ) + .unwrap(); + let held_pin = attachment + .lock_state() + .initialization_file + .as_ref() + .unwrap() + .try_clone() + .unwrap(); + attachment.drain().unwrap(); + attachment.close_and_join().unwrap(); + assert!(attachment.commit_initialization().is_err()); + let original = fs::read(&path).unwrap(); + // SQLite has finished with these fixture files. Populate all three + // names so the separate process replaces real inodes, not absences. + for suffix in ["-wal", "-shm", "-journal"] { + let mut sidecar = path.as_os_str().to_os_string(); + sidecar.push(suffix); + fs::write(std::path::Path::new(&sidecar), b"owned fixture").unwrap(); + } + let (_, module) = module_path!().split_once("::").unwrap(); + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + &format!("{module}::failed_initialization_foreign_replacement_child"), + "--nocapture", + ]) + .env("TRACEDECAY_FAILED_SQLITE_REPLACEMENT", &path) + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success() + && stdout.contains("1 passed") + && stdout.contains("foreign-sqlite-replacement-complete"), + "separate-process replacement failed: {stdout} {}", + String::from_utf8_lossy(&output.stderr) + ); + + attachment.abort_initialization().unwrap(); + assert!(held_pin.try_clone().is_err()); + attachment.abort_initialization().unwrap(); + drop(attachment); + assert_eq!(fs::read(&path).unwrap(), b"foreign database"); + assert_eq!(fs::read(path.with_extension("owned")).unwrap(), original); + for suffix in ["-wal", "-shm", "-journal"] { + let mut sidecar = path.as_os_str().to_os_string(); + sidecar.push(suffix); + assert_eq!( + fs::read(std::path::Path::new(&sidecar)).unwrap(), + format!("foreign {suffix}").as_bytes() + ); + sidecar.push(".owned"); + assert_eq!( + fs::read(std::path::Path::new(&sidecar)).unwrap(), + b"owned fixture" + ); + } + } + + #[cfg(unix)] + #[test] + fn abort_initialization_retains_the_pin_until_a_lost_writer_exits() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("lost-initialization.sqlite3"); + let binding = binding(); + let attachment = RepositoryPhysicalAttachmentFactory + .initialize( + &native_runtime(), + binding.clone(), + locator(&binding), + path.clone(), + AdmissionConfigV1::default(), + ) + .unwrap(); + let held_pin = attachment + .lock_state() + .initialization_file + .as_ref() + .unwrap() + .try_clone() + .unwrap(); + let (hold, held) = mpsc::channel::<()>(); + attachment.lock_state().after_lost_notice = Some(Box::new(move || { + let _ = held.recv(); + })); + fs::remove_file(&path).unwrap(); + let deadline = Instant::now() + crate::FILE_PROBE_INTERVAL + Duration::from_secs(10); + while !attachment + .lock_state() + .lost_readers + .as_ref() + .is_some_and(ReaderPool::workers_exited) + { + assert!(Instant::now() < deadline, "lost readers never exited"); + thread::sleep(Duration::from_millis(20)); + } + assert!(attachment.file_lost()); + attachment.abort_initialization().unwrap(); + assert!(attachment.lock_state().initialization_file.is_none()); + assert!(attachment.lock_state().retained_file_pin.is_some()); + assert!(held_pin.try_clone().is_ok()); + assert!(!attachment.lost_file_closed()); + + hold.send(()).unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + while attachment.lock_state().retained_file_pin.is_some() { + let _ = attachment.snapshot(); + assert!(Instant::now() < deadline, "lost initialization pin leaked"); + thread::sleep(Duration::from_millis(5)); + } + assert!(attachment.lost_file_closed()); + assert!(held_pin.try_clone().is_err()); + } + #[test] fn failed_initialization_preserves_created_database_and_sidecars() { let directory = TempDir::new().unwrap(); @@ -1120,7 +1337,10 @@ mod tests { let mut sidecar = path.as_os_str().to_os_string(); sidecar.push(suffix); let sidecar = std::path::PathBuf::from(sidecar); - assert_eq!(fs::read(&sidecar).unwrap(), format!("foreign {suffix}").as_bytes()); + assert_eq!( + fs::read(&sidecar).unwrap(), + format!("foreign {suffix}").as_bytes() + ); let mut displaced = sidecar.as_os_str().to_os_string(); displaced.push(".owned"); assert!(std::path::Path::new(&displaced).is_file()); From 2e9b650c4b75820ff912a3b0b22f02704b10303c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 20:11:48 -0800 Subject: [PATCH 04/60] style(sqlite): satisfy pinned formatting checks Apply the exact rustfmt changes reported by hosted Bazel Repository gates without touching SQLite behavior or other PR files. Refs #4513 --- crates/tracedecay-rusqlite-runtime/src/connection/mod.rs | 5 +++-- crates/tracedecay-rusqlite-runtime/src/connection/tests.rs | 4 +++- .../tracedecay-rusqlite-runtime/src/repository/attachment.rs | 5 ++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs index 25c95f39ecd..da892465af9 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs @@ -325,7 +325,6 @@ impl OpenedDatabaseFile { } Ok(()) } - } /// Check a live SQLite worker's physical file without opening and closing a @@ -367,7 +366,9 @@ impl fmt::Display for OpenedDatabaseFileError { let message = match self { Self::Create => "could not create the canonical SQLite file", Self::Reserve => "could not exclusively reserve the new SQLite sidecar namespace", - Self::InvalidHeader => "existing SQLite file has no valid database header; refusing sidecar access", + Self::InvalidHeader => { + "existing SQLite file has no valid database header; refusing sidecar access" + } Self::Open => "could not open the verified SQLite file", Self::Inspect => "could not inspect the verified SQLite file descriptor", Self::NotFile => "verified SQLite locator is not a regular file", diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs index d66446a0e94..321d675fa9e 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs @@ -445,7 +445,9 @@ fn existing_sqlite_admission_requires_a_completed_header() { )); std::fs::remove_file(&path).unwrap(); let database = Connection::open(&path).unwrap(); - database.execute_batch("CREATE TABLE probe(value INTEGER)").unwrap(); + database + .execute_batch("CREATE TABLE probe(value INTEGER)") + .unwrap(); drop(database); assert!(OpenedDatabaseFile::pin_initialized(&path).is_ok()); } diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs index 3aab9464f97..464e952cd00 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs @@ -1083,7 +1083,10 @@ mod tests { .unwrap(); attachment.abort_initialization().unwrap(); - assert!(path.is_file(), "abort must not unlink the initialized store"); + assert!( + path.is_file(), + "abort must not unlink the initialized store" + ); assert!(held_pin.try_clone().is_err()); { let state = attachment.lock_state(); From e6ea022992352c6c3b3ab79c2b5dd9af686a6cd1 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 21:44:09 -0800 Subject: [PATCH 05/60] fix(sqlite): refuse short attachments without dropping POSIX locks Avoid opening a raw database descriptor when an existing attachment cannot contain the SQLite header. Closing that descriptor could unlock another live SQLite writer; cover the refusal with a cross-process test. This does not fix header-bearing incomplete stores or sidecar swaps. Refs #4513 --- .../src/connection/mod.rs | 16 +++++++++++++ .../src/connection/path_identity_tests.rs | 24 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs index da892465af9..7afa43a6030 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs @@ -91,6 +91,22 @@ impl OpenedDatabaseFile { /// Read the magic through the existing pin; opening a second descriptor /// here could discard another connection's POSIX locks on Unix. pub(crate) fn pin_initialized(path: &Path) -> Result { + // A failed header read would drop a freshly opened raw descriptor. + // On Unix that close can release a different live SQLite connection's + // process-owned locks, even when that connection still has a transaction. + // Refuse an obviously unfinished empty/short database by metadata + // alone, before opening a descriptor. This does not establish that a + // longer, header-bearing store completed initialization. + #[cfg(unix)] + { + let metadata = std::fs::metadata(path).map_err(|_| OpenedDatabaseFileError::Open)?; + if !metadata.is_file() { + return Err(OpenedDatabaseFileError::NotFile); + } + if metadata.len() < 16 { + return Err(OpenedDatabaseFileError::InvalidHeader); + } + } let opened = Self::pin(path)?; let initialized = opened.with_file(|file| { let mut reader = file; diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs index a8c0ce5fcf0..2e6b92b7294 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs @@ -157,6 +157,30 @@ fn worker_identity_checks_keep_cross_process_writer_exclusion() { ); } +/// A zero-byte existing SQLite file may belong to a live connection holding +/// a rollback writer reservation. Rejecting that unfinished attachment must +/// not open and close a second raw descriptor and unlock the other writer. +#[test] +fn rejecting_empty_attachment_does_not_release_another_writer_lock() { + let temp = TempDir::new().expect("tempdir"); + let path = temp.path().join("empty-locked.db"); + let writer = Connection::open(&path).expect("open SQLite connection"); + writer + .execute_batch("BEGIN IMMEDIATE") + .expect("reserve the SQLite writer lock without initializing pages"); + assert_eq!(fs::metadata(&path).unwrap().len(), 0); + probe_external_connection(&path, "write", true); + + assert!(matches!( + OpenedDatabaseFile::pin_initialized(&path), + Err(OpenedDatabaseFileError::InvalidHeader) + )); + probe_external_connection(&path, "write", true); + + writer.execute_batch("ROLLBACK").expect("release writer reservation"); + probe_external_connection(&path, "write", false); +} + #[test] fn path_checks_preserve_rollback_writer_lock() { let temp = TempDir::new().expect("tempdir"); From 83d0221a14708abbade4e42396c10991b437304c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 21:45:15 -0800 Subject: [PATCH 06/60] fix(sqlite): reject partial header pages before descriptor pin SQLite's minimum initialized database occupies one 512-byte page. Refuse header-bearing shorter files without opening a raw Unix handle; otherwise they bypass magic-only admission and can release live locks. Refs #4513 --- .../src/connection/mod.rs | 4 +++- .../src/connection/path_identity_tests.rs | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs index 7afa43a6030..f2685c608fa 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs @@ -103,7 +103,9 @@ impl OpenedDatabaseFile { if !metadata.is_file() { return Err(OpenedDatabaseFileError::NotFile); } - if metadata.len() < 16 { + // SQLite's smallest initialized database occupies one 512-byte page. + // Header magic alone (16 bytes) is not a completed database. + if metadata.len() < 512 { return Err(OpenedDatabaseFileError::InvalidHeader); } } diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs index 2e6b92b7294..a3ad8d893d8 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs @@ -181,6 +181,22 @@ fn rejecting_empty_attachment_does_not_release_another_writer_lock() { probe_external_connection(&path, "write", false); } +#[test] +fn header_magic_without_a_complete_sqlite_page_is_refused() { + let temp = TempDir::new().expect("tempdir"); + for length in [16, 100, 511] { + let path = temp.path().join(format!("short-{length}.db")); + let mut bytes = vec![0_u8; length]; + bytes[..16].copy_from_slice(b"SQLite format 3\0"); + fs::write(&path, &bytes).expect("write incomplete SQLite header"); + assert!(matches!( + OpenedDatabaseFile::pin_initialized(&path), + Err(OpenedDatabaseFileError::InvalidHeader) + )); + assert_eq!(fs::read(&path).unwrap(), bytes); + } +} + #[test] fn path_checks_preserve_rollback_writer_lock() { let temp = TempDir::new().expect("tempdir"); From 761cc49cf04fd77940c4467de7f8bbcf17b8df6e Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 21:50:12 -0800 Subject: [PATCH 07/60] fix(sqlite): reject partial-page databases before SQLite opens Existing-mode admission must not open sidecars for databases whose file size cannot be a whole number of SQLite pages. Refuse those paths using metadata before opening a descriptor, preserving foreign journal entries and avoiding POSIX lock-loss on the refusal path. Add production attachment regressions for truncated header-bearing files and WAL, SHM, and journal sentinel preservation. Refs #4513 --- .../src/connection/mod.rs | 17 ++++----- .../src/connection/tests.rs | 11 ++++++ .../src/repository/attachment.rs | 36 +++++++++++++++++++ 3 files changed, 54 insertions(+), 10 deletions(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs index f2685c608fa..e0577179759 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs @@ -91,21 +91,18 @@ impl OpenedDatabaseFile { /// Read the magic through the existing pin; opening a second descriptor /// here could discard another connection's POSIX locks on Unix. pub(crate) fn pin_initialized(path: &Path) -> Result { - // A failed header read would drop a freshly opened raw descriptor. - // On Unix that close can release a different live SQLite connection's - // process-owned locks, even when that connection still has a transaction. - // Refuse an obviously unfinished empty/short database by metadata - // alone, before opening a descriptor. This does not establish that a - // longer, header-bearing store completed initialization. - #[cfg(unix)] + // A failed header read would drop a freshly opened raw descriptor, + // releasing unrelated process-owned SQLite locks on Unix. Reject + // obviously truncated stores without opening another descriptor. + // All SQLite page sizes are multiples of 512, and a complete + // database contains an integral number of pages. This is only a + // necessary condition; it does not prove initialization completed. { let metadata = std::fs::metadata(path).map_err(|_| OpenedDatabaseFileError::Open)?; if !metadata.is_file() { return Err(OpenedDatabaseFileError::NotFile); } - // SQLite's smallest initialized database occupies one 512-byte page. - // Header magic alone (16 bytes) is not a completed database. - if metadata.len() < 512 { + if metadata.len() < 512 || metadata.len() % 512 != 0 { return Err(OpenedDatabaseFileError::InvalidHeader); } } diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs index 321d675fa9e..3614abf46c2 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs @@ -437,6 +437,17 @@ fn failed_sidecar_reservation_cannot_be_reopened_as_existing_sqlite() { #[test] fn existing_sqlite_admission_requires_a_completed_header() { let directory = tempfile::tempdir().unwrap(); + for size in [513_usize, 1023, 1025] { + let path = directory.path().join(format!("truncated-{size}.db")); + let mut bytes = vec![0_u8; size]; + bytes[..16].copy_from_slice(b"SQLite format 3\0"); + std::fs::write(&path, &bytes).unwrap(); + assert!(matches!( + OpenedDatabaseFile::pin_initialized(&path), + Err(OpenedDatabaseFileError::InvalidHeader) + )); + assert_eq!(std::fs::read(&path).unwrap(), bytes); + } let path = directory.path().join("sessions.db"); std::fs::write(&path, b"invalid database, not SQLite").unwrap(); assert!(matches!( diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs index 464e952cd00..7796267b406 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs @@ -1255,6 +1255,42 @@ mod tests { assert!(held_pin.try_clone().is_err()); } + #[test] + fn truncated_existing_sqlite_refuses_before_touching_sidecars() { + let binding = binding(); + for size in [513_usize, 1025] { + for suffix in ["-wal", "-shm", "-journal"] { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("sessions.db"); + let mut bytes = vec![0_u8; size]; + bytes[..16].copy_from_slice(b"SQLite format 3\0"); + fs::write(&path, &bytes).unwrap(); + + let mut sidecar = path.as_os_str().to_os_string(); + sidecar.push(suffix); + let sidecar = PathBuf::from(sidecar); + let sentinel = format!("foreign {suffix} bytes"); + fs::write(&sidecar, sentinel.as_bytes()).unwrap(); + + let result = RepositoryPhysicalAttachmentFactory.attach( + &native_runtime(), + binding.clone(), + locator(&binding), + path.clone(), + AdmissionConfigV1::default(), + ); + assert!(matches!( + result, + Err(RepositoryAttachmentStartError::Identity( + OpenedDatabaseFileError::InvalidHeader + )) + )); + assert_eq!(fs::read(&path).unwrap(), bytes); + assert_eq!(fs::read(&sidecar).unwrap(), sentinel.as_bytes()); + } + } + } + #[test] fn failed_initialization_preserves_created_database_and_sidecars() { let directory = TempDir::new().unwrap(); From 9a6e93df66931392236cde9009b2c50941fa55ae Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 23:05:42 -0800 Subject: [PATCH 08/60] style(sqlite): format external lock regression Match the pinned Bazel rustfmt_check_2024 output for the refused empty-attachment writer-lock regression without changing its behavior. Refs #4513 --- .../src/connection/path_identity_tests.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs index a3ad8d893d8..e860a25622b 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs @@ -177,7 +177,9 @@ fn rejecting_empty_attachment_does_not_release_another_writer_lock() { )); probe_external_connection(&path, "write", true); - writer.execute_batch("ROLLBACK").expect("release writer reservation"); + writer + .execute_batch("ROLLBACK") + .expect("release writer reservation"); probe_external_connection(&path, "write", false); } From 45d0ed2f2b7ade588ef4b3ebd11936fef1185656 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 00:31:57 -0800 Subject: [PATCH 09/60] test(sqlite): cover factory refusal and retry sidecar integrity Refs #4513 --- .../src/repository/attachment.rs | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs index 7796267b406..2e2611daadc 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs @@ -1291,6 +1291,67 @@ mod tests { } } + /// The factory must refuse foreign sidecars before starting SQLite, and a + /// failed first creation must not become an existing database on retry. + /// Test the actual attachment entry points rather than just the pin helper. + #[test] + fn occupied_sidecar_refuses_initialization_and_existing_attachment_retries() { + let binding = binding(); + for suffix in ["-wal", "-shm", "-journal"] { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("sessions.db"); + let mut sidecar = path.as_os_str().to_os_string(); + sidecar.push(suffix); + let sidecar = PathBuf::from(sidecar); + let sentinel = format!("unowned {suffix} content"); + fs::write(&sidecar, sentinel.as_bytes()).unwrap(); + + let initialized = RepositoryPhysicalAttachmentFactory.initialize( + &native_runtime(), + binding.clone(), + locator(&binding), + path.clone(), + AdmissionConfigV1::default(), + ); + assert!(matches!( + initialized, + Err(RepositoryAttachmentStartError::Identity( + OpenedDatabaseFileError::Reserve + )) + )); + assert_eq!(fs::metadata(&path).unwrap().len(), 0); + assert_eq!(fs::read(&sidecar).unwrap(), sentinel.as_bytes()); + + let existing = RepositoryPhysicalAttachmentFactory.attach( + &native_runtime(), + binding.clone(), + locator(&binding), + path.clone(), + AdmissionConfigV1::default(), + ); + assert!(matches!( + existing, + Err(RepositoryAttachmentStartError::Identity( + OpenedDatabaseFileError::InvalidHeader + )) + )); + let read_only = RepositoryPhysicalAttachmentFactory.attach_read_only( + binding.clone(), + locator(&binding), + path.clone(), + AdmissionConfigV1::default(), + ); + assert!(matches!( + read_only, + Err(RepositoryAttachmentStartError::Identity( + OpenedDatabaseFileError::InvalidHeader + )) + )); + assert_eq!(fs::metadata(&path).unwrap().len(), 0); + assert_eq!(fs::read(&sidecar).unwrap(), sentinel.as_bytes()); + } + } + #[test] fn failed_initialization_preserves_created_database_and_sidecars() { let directory = TempDir::new().unwrap(); From b154e6b572dc2bbd3b6c96c8971dba8394407067 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:01:12 -0800 Subject: [PATCH 10/60] fix(sqlite): validate existing stores through SQLite VFS Refs #4513 --- .../src/connection/mod.rs | 27 ++++++++-------- .../src/connection/tests.rs | 31 +++++++++++++++++-- .../src/repository/attachment.rs | 4 +-- 3 files changed, 44 insertions(+), 18 deletions(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs index e0577179759..d1f1ce142c6 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs @@ -106,20 +106,19 @@ impl OpenedDatabaseFile { return Err(OpenedDatabaseFileError::InvalidHeader); } } - let opened = Self::pin(path)?; - let initialized = opened.with_file(|file| { - let mut reader = file; - let mut header = [0_u8; 16]; - match io::Read::read_exact(&mut reader, &mut header) { - Ok(()) => Ok(header == *b"SQLite format 3\0"), - Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => Ok(false), - Err(_) => Err(OpenedDatabaseFileError::Inspect), - } - })??; - if !initialized { - return Err(OpenedDatabaseFileError::InvalidHeader); - } - Ok(opened) + // Inspect the main SQLite file without opening and closing a separate + // raw descriptor on a rejected header: on Unix that close can release + // locks held by another live SQLite connection in this process. + // An immutable SQLite reader uses the VFS's lock-aware lifetime and + // never opens or mutates the unverified WAL/SHM/journal sidecars. + let inspection = open_immutable_reader(path) + .map_err(|_| OpenedDatabaseFileError::InvalidHeader)?; + let _: i64 = inspection + .query_row("SELECT count(*) FROM sqlite_schema", [], |row| row.get(0)) + .map_err(|_| OpenedDatabaseFileError::InvalidHeader)?; + drop(inspection); + // A valid main-file schema is necessary, not proof of sidecar ownership. + Self::pin(path) } pub(crate) fn create_new(path: &Path) -> Result { diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs index 3614abf46c2..cf179b3177c 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs @@ -437,8 +437,8 @@ fn failed_sidecar_reservation_cannot_be_reopened_as_existing_sqlite() { #[test] fn existing_sqlite_admission_requires_a_completed_header() { let directory = tempfile::tempdir().unwrap(); - for size in [513_usize, 1023, 1025] { - let path = directory.path().join(format!("truncated-{size}.db")); + for size in [512_usize, 513, 1023, 1025, 4096] { + let path = directory.path().join(format!("incomplete-{size}.db")); let mut bytes = vec![0_u8; size]; bytes[..16].copy_from_slice(b"SQLite format 3\0"); std::fs::write(&path, &bytes).unwrap(); @@ -463,6 +463,33 @@ fn existing_sqlite_admission_requires_a_completed_header() { assert!(OpenedDatabaseFile::pin_initialized(&path).is_ok()); } +#[test] +fn initialized_wal_with_uncheckpointed_schema_remains_attachable() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("sessions.db"); + let writer = Connection::open(&path).unwrap(); + writer + .execute_batch( + "PRAGMA journal_mode = WAL; + CREATE TABLE probe(value INTEGER); + INSERT INTO probe VALUES (42);", + ) + .unwrap(); + let wal = sidecar_path(&path, "-wal"); + let before = std::fs::read(&wal).unwrap(); + assert!(!before.is_empty()); + let pinned = OpenedDatabaseFile::pin_initialized(&path).unwrap(); + assert_ne!(pinned.identity(), 0); + assert_eq!(std::fs::read(&wal).unwrap(), before); + drop(writer); + pinned.release_after_workers_exit(); + let reader = Connection::open(&path).unwrap(); + assert_eq!( + reader.query_row("SELECT value FROM probe", [], |row| row.get::<_, i64>(0)), + Ok(42) + ); +} + #[test] fn create_new_refuses_to_replace_an_existing_database() { let file = NamedTempFile::new().unwrap(); diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs index 2e2611daadc..7b9b0950b68 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs @@ -1256,9 +1256,9 @@ mod tests { } #[test] - fn truncated_existing_sqlite_refuses_before_touching_sidecars() { + fn malformed_existing_sqlite_refuses_before_touching_sidecars() { let binding = binding(); - for size in [513_usize, 1025] { + for size in [512_usize, 513, 1025, 4096] { for suffix in ["-wal", "-shm", "-journal"] { let directory = TempDir::new().unwrap(); let path = directory.path().join("sessions.db"); From 02783ec8e9bbc3b651a0eb58d9b5e65d456577e2 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:03:53 -0800 Subject: [PATCH 11/60] test(sqlite): prove rejected headers retain POSIX writer locks Refs #4513 --- .../src/connection/path_identity_tests.rs | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs index e860a25622b..1c024c3b71b 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs @@ -1,4 +1,6 @@ use std::{env, fs, path::Path, process::Command, time::Duration}; +use std::fs::OpenOptions; +use std::os::{fd::AsRawFd, unix::fs::FileExt}; use rusqlite::{Connection, Error, ErrorCode, OpenFlags}; use tempfile::TempDir; @@ -419,3 +421,105 @@ fn path_checks_keep_pinned_identity_and_reject_missing_or_replaced_files() { Err(OpenedDatabaseFileError::Replaced), ); } + +const RESERVED_LOCK_PROBE_PATH: &str = "TRACEDECAY_SQLITE_RESERVED_LOCK_PROBE_PATH"; +const RESERVED_LOCK_PROBE_EXPECTATION: &str = "TRACEDECAY_SQLITE_RESERVED_LOCK_PROBE_EXPECTATION"; +const RESERVED_LOCK_PROBE_COMPLETE: &str = "tracedecay-sqlite-reserved-lock-probe-complete"; + +/// Inspect SQLite's POSIX RESERVED byte from a different process. This avoids +/// an SQLite schema read after we deliberately corrupt the fixture header. +#[test] +fn external_reserved_lock_probe() { + let Some(path) = env::var_os(RESERVED_LOCK_PROBE_PATH) else { + return; + }; + let file = OpenOptions::new().read(true).write(true).open(path).unwrap(); + let mut reservation = libc::flock { + l_type: libc::F_WRLCK as libc::c_short, + l_whence: libc::SEEK_SET as libc::c_short, + l_start: 0x4000_0001, + l_len: 1, + l_pid: 0, + }; + // SAFETY: the file owns a valid descriptor and reservation is initialized + // and writable for the F_SETLK request. + let result = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_SETLK, &mut reservation) }; + match env::var(RESERVED_LOCK_PROBE_EXPECTATION).unwrap().as_str() { + "blocked" => { + assert_eq!(result, -1, "SQLite writer's RESERVED lock was lost"); + let error = std::io::Error::last_os_error().raw_os_error(); + assert!( + error == Some(libc::EACCES) || error == Some(libc::EAGAIN), + "unexpected advisory-lock refusal: {error:?}" + ); + } + "allowed" => assert_eq!( + result, + 0, + "RESERVED byte remained locked after the writer closed: {}", + std::io::Error::last_os_error() + ), + other => panic!("unknown reserved lock expectation: {other}"), + } + println!("{RESERVED_LOCK_PROBE_COMPLETE}"); +} + +fn probe_reserved_lock(path: &Path, blocked: bool) { + let (_, module) = module_path!().split_once("::").unwrap(); + let output = Command::new(env::current_exe().unwrap()) + .args([ + "--exact", + &format!("{module}::external_reserved_lock_probe"), + "--nocapture", + ]) + .env(RESERVED_LOCK_PROBE_PATH, path) + .env( + RESERVED_LOCK_PROBE_EXPECTATION, + if blocked { "blocked" } else { "allowed" }, + ) + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success() + && stdout.contains("1 passed") + && stdout.contains(RESERVED_LOCK_PROBE_COMPLETE), + "cross-process lock probe failed: {stdout}\n{}", + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn malformed_existing_store_refusal_keeps_other_sqlite_writer_locks() { + let temp = TempDir::new().unwrap(); + let path = temp.path().join("sessions.db"); + let writer = Connection::open(&path).unwrap(); + writer + .execute_batch( + "PRAGMA journal_mode = DELETE; + CREATE TABLE sample(value INTEGER); + INSERT INTO sample VALUES (1);", + ) + .unwrap(); + // Retain this descriptor through the original writer's entire lifetime: + // closing it early would itself discard the POSIX locks under test. + let mutation = OpenOptions::new().read(true).write(true).open(&path).unwrap(); + writer + .execute_batch("BEGIN IMMEDIATE; UPDATE sample SET value = 2;") + .unwrap(); + probe_reserved_lock(&path, true); + + // Corrupt only a disposable fixture, in place and without closing a raw + // descriptor. The main file remains page-aligned but is no longer SQLite. + mutation.write_all_at(b"Not SQLite data!", 0).unwrap(); + probe_reserved_lock(&path, true); + assert!(matches!( + OpenedDatabaseFile::pin_initialized(&path), + Err(OpenedDatabaseFileError::InvalidHeader) + )); + probe_reserved_lock(&path, true); + + drop(writer); + drop(mutation); + probe_reserved_lock(&path, false); +} From 935d3d4d8733ab909b3adc77578c53ba26c2a0a2 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:05:13 -0800 Subject: [PATCH 12/60] style(sqlite): normalize lock regression imports Refs #4513 --- .../src/connection/path_identity_tests.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs index 1c024c3b71b..8334e3d6e99 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs @@ -1,6 +1,11 @@ -use std::{env, fs, path::Path, process::Command, time::Duration}; -use std::fs::OpenOptions; -use std::os::{fd::AsRawFd, unix::fs::FileExt}; +use std::{ + env, + fs::{self, OpenOptions}, + os::{fd::AsRawFd, unix::fs::FileExt}, + path::Path, + process::Command, + time::Duration, +}; use rusqlite::{Connection, Error, ErrorCode, OpenFlags}; use tempfile::TempDir; From fcd316b6ca0865784108bc1ac0e96da5e150d553 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:19:28 -0800 Subject: [PATCH 13/60] style(sqlite): satisfy pinned rustfmt for integrity tests Refs #4513 --- .../src/connection/mod.rs | 4 ++-- .../src/connection/path_identity_tests.rs | 12 ++++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs index d1f1ce142c6..c1ca4d472e9 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs @@ -111,8 +111,8 @@ impl OpenedDatabaseFile { // locks held by another live SQLite connection in this process. // An immutable SQLite reader uses the VFS's lock-aware lifetime and // never opens or mutates the unverified WAL/SHM/journal sidecars. - let inspection = open_immutable_reader(path) - .map_err(|_| OpenedDatabaseFileError::InvalidHeader)?; + let inspection = + open_immutable_reader(path).map_err(|_| OpenedDatabaseFileError::InvalidHeader)?; let _: i64 = inspection .query_row("SELECT count(*) FROM sqlite_schema", [], |row| row.get(0)) .map_err(|_| OpenedDatabaseFileError::InvalidHeader)?; diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs index 8334e3d6e99..6a350e16247 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs @@ -438,7 +438,11 @@ fn external_reserved_lock_probe() { let Some(path) = env::var_os(RESERVED_LOCK_PROBE_PATH) else { return; }; - let file = OpenOptions::new().read(true).write(true).open(path).unwrap(); + let file = OpenOptions::new() + .read(true) + .write(true) + .open(path) + .unwrap(); let mut reservation = libc::flock { l_type: libc::F_WRLCK as libc::c_short, l_whence: libc::SEEK_SET as libc::c_short, @@ -508,7 +512,11 @@ fn malformed_existing_store_refusal_keeps_other_sqlite_writer_locks() { .unwrap(); // Retain this descriptor through the original writer's entire lifetime: // closing it early would itself discard the POSIX locks under test. - let mutation = OpenOptions::new().read(true).write(true).open(&path).unwrap(); + let mutation = OpenOptions::new() + .read(true) + .write(true) + .open(&path) + .unwrap(); writer .execute_batch("BEGIN IMMEDIATE; UPDATE sample SET value = 2;") .unwrap(); From 58d49b8069a6464f94a99fcb6f6c78ca0a27bc3b Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 04:19:12 -0800 Subject: [PATCH 14/60] test(sqlite): preserve writer locks on valid-store admission Refs #4513 --- .../src/connection/path_identity_tests.rs | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs index 6a350e16247..e9721ca3b1d 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs @@ -498,6 +498,37 @@ fn probe_reserved_lock(path: &Path, blocked: bool) { ); } +/// Opening an already valid database for attachment must not close an +/// unrelated raw descriptor and discard a live writer's POSIX RESERVED lock. +/// The pin returned by admission outlives the writer, just as in production. +#[test] +fn valid_existing_store_admission_preserves_other_writer_locks() { + let temp = TempDir::new().unwrap(); + let path = temp.path().join("sessions.db"); + let writer = Connection::open(&path).unwrap(); + writer + .execute_batch( + "PRAGMA journal_mode = DELETE; + CREATE TABLE sample(value INTEGER); + INSERT INTO sample VALUES (1); + BEGIN IMMEDIATE; + UPDATE sample SET value = 2;", + ) + .unwrap(); + + probe_reserved_lock(&path, true); + let opened = OpenedDatabaseFile::pin_initialized(&path) + .expect("admit an initialized database while another writer is live"); + assert_eq!(opened.identity(), live_path_identity(&path).unwrap()); + probe_reserved_lock(&path, true); + + writer.execute_batch("COMMIT").unwrap(); + drop(writer); + opened.release_after_workers_exit(); + probe_reserved_lock(&path, false); + assert_committed_data(&path); +} + #[test] fn malformed_existing_store_refusal_keeps_other_sqlite_writer_locks() { let temp = TempDir::new().unwrap(); From 3e5fbc712f16488f6838a60a1c315c2005b2d21a Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:13:53 -0800 Subject: [PATCH 15/60] fix(sqlite): keep Windows sidecar imports at module scope --- crates/tracedecay-rusqlite-runtime/src/connection/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs index c1ca4d472e9..507d857d222 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs @@ -1,5 +1,7 @@ #[cfg(unix)] use std::ffi::CString; +#[cfg(windows)] +use std::os::windows::fs::OpenOptionsExt; use std::{ fmt, fs::{File, OpenOptions}, @@ -442,7 +444,6 @@ fn reserve_new_sidecar(path: &Path) -> io::Result { options.read(true).write(true).create_new(true); #[cfg(windows)] { - use std::os::windows::fs::OpenOptionsExt; options.share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE); } options.open(path) From 76ed787711eb5950ab63679e1f558f24c8a5183c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 07:46:22 -0800 Subject: [PATCH 16/60] test(sqlite): materialize existing-store fixtures before attachment Existing fixture files created by a bare SQLite open remain zero bytes; they are not completed SQLite stores and must be refused by admission. Use an empty-schema SQLite header instead, preserving fixture intent and the incomplete-initialization safety guard. Refs #4513 --- .../src/tests/schema_install_cancellation.rs | 7 ++++++- .../rusqlite_suite/repository_attachment.rs | 18 ++++++++++++++---- .../repository_parity.rs | 7 ++++++- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs b/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs index 92cf28bd455..ffc3ad9bc5c 100644 --- a/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs +++ b/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs @@ -42,7 +42,12 @@ impl EmptyStore { .unwrap(); let path = tracedecay_sessions::runtime::user_sessions_db_path(&profile_root); std::fs::create_dir_all(path.parent().unwrap()).unwrap(); - std::fs::File::create(&path).unwrap(); + // Exercise an initialized SQLite database with no application schema. + // A zero-byte placeholder is an incomplete store, not an attachable DB. + rusqlite::Connection::open(&path) + .unwrap() + .execute_batch("PRAGMA user_version = 0") + .unwrap(); Self { path, _scope: scope, diff --git a/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/repository_attachment.rs b/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/repository_attachment.rs index ea8f1d287e0..226462a7acc 100644 --- a/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/repository_attachment.rs +++ b/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/repository_attachment.rs @@ -25,6 +25,16 @@ use tracedecay_store::{ use tracedecay_domain::test_fixtures::id; +/// Existing-attachment fixtures must contain a real SQLite header. Merely +/// opening and closing a new SQLite pathname leaves a zero-byte file, which +/// safe admission correctly treats as incomplete initialization. +fn initialized_existing_file(path: &std::path::Path) { + Connection::open(path) + .unwrap() + .execute_batch("PRAGMA user_version = 0") + .unwrap(); +} + fn binding() -> StoreRuntimeBindingV1 { serde_json::from_value(serde_json::json!({ "shard_id": StoreShardIdV1::project( @@ -110,7 +120,7 @@ fn health_request( fn repository_factory_attaches_writer_and_reserved_reader_runtime() { let directory = TempDir::new().unwrap(); let path = directory.path().join("project.db"); - Connection::open(&path).unwrap(); + initialized_existing_file(&path); let path = path.canonicalize().unwrap(); let binding = binding(); let locator = VerifiedStoreLocatorV1::new( @@ -148,7 +158,7 @@ fn repository_factory_attaches_writer_and_reserved_reader_runtime() { fn a_typed_read_awaited_on_a_native_worker_parks_no_native_thread() { let directory = TempDir::new().unwrap(); let path = directory.path().join("sessions.db"); - Connection::open(&path).unwrap(); + initialized_existing_file(&path); let path = path.canonicalize().unwrap(); let binding = binding(); let locator = VerifiedStoreLocatorV1::new( @@ -206,7 +216,7 @@ fn a_typed_read_awaited_on_a_native_worker_parks_no_native_thread() { fn temporal_health_dispatch_uses_the_reserved_reader_lane() { let directory = TempDir::new().unwrap(); let path = directory.path().join("sessions.db"); - Connection::open(&path).unwrap(); + initialized_existing_file(&path); let path = path.canonicalize().unwrap(); let binding = binding(); let locator = VerifiedStoreLocatorV1::new( @@ -239,7 +249,7 @@ fn temporal_health_dispatch_uses_the_reserved_reader_lane() { fn at_least_reads_refuse_coverage_without_a_committed_watermark() { let directory = TempDir::new().unwrap(); let path = directory.path().join("sessions.db"); - Connection::open(&path).unwrap(); + initialized_existing_file(&path); let path = path.canonicalize().unwrap(); let binding = binding(); let locator = VerifiedStoreLocatorV1::new( diff --git a/tests/storage_runtime_rusqlite_suite/repository_parity.rs b/tests/storage_runtime_rusqlite_suite/repository_parity.rs index c5a8f1f4f50..5103dee8bb0 100644 --- a/tests/storage_runtime_rusqlite_suite/repository_parity.rs +++ b/tests/storage_runtime_rusqlite_suite/repository_parity.rs @@ -146,7 +146,12 @@ fn profile_project_and_session_production_mounts_serve_health_data_ports() { for (family, shard_id) in families { let path = directory.path().join(format!("{family}.db")); - Connection::open(&path).unwrap(); + // Materialize a header without adding schema objects: a bare open + // leaves an incomplete zero-byte store, now correctly rejected. + Connection::open(&path) + .unwrap() + .execute_batch("PRAGMA user_version = 0") + .unwrap(); let path = path.canonicalize().unwrap(); assert_family_mount(family_binding(shard_id), &path); } From a4d0871ecb46d9d2670ad61b52e3b55d0917e1aa Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:25:03 -0800 Subject: [PATCH 17/60] test(sqlite): initialize remaining existing-store test fixtures Materialize valid empty-schema databases before ordinary attachment in runtime-core registry and CLI integration fixtures. A bare SQLite open leaves a zero-byte file, which the #4513 admission guard must refuse. Refs #4513 --- .../core_cli_suite/cli_non_interactive_test.rs | 5 ++++- .../src/shard_runtime/registry/close.rs | 17 +++++++++++++---- .../registry/tests/production_routes.rs | 5 ++++- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-cli/tests/core_cli_suite/cli_non_interactive_test.rs b/crates/tracedecay-cli/tests/core_cli_suite/cli_non_interactive_test.rs index 3391fe3c5d6..ba4fa1591c5 100644 --- a/crates/tracedecay-cli/tests/core_cli_suite/cli_non_interactive_test.rs +++ b/crates/tracedecay-cli/tests/core_cli_suite/cli_non_interactive_test.rs @@ -1052,7 +1052,10 @@ fn write_empty_sqlite_fixture(path: &Path) { if let Some(parent) = path.parent() { std::fs::create_dir_all(parent).unwrap(); } - drop(rusqlite::Connection::open(path).expect("empty SQLite fixture")); + rusqlite::Connection::open(path) + .expect("empty SQLite fixture") + .execute_batch("PRAGMA user_version = 0") + .expect("materialize an initialized empty-schema SQLite fixture"); } async fn register_profile_sharded_store( diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/close.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/close.rs index d9a8501499a..0576c8ebe6d 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/close.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/close.rs @@ -704,6 +704,15 @@ mod tests { } } + /// An existing attachment needs an initialized SQLite header even when + /// the fixture intentionally has no tables. + fn seed_empty_sqlite(path: &Path) { + rusqlite::Connection::open(path) + .unwrap() + .execute_batch("PRAGMA user_version = 0") + .unwrap(); + } + async fn seed_final_graph_db(path: PathBuf) -> PathBuf { let connection = crate::db::engine::TestConnection::open(&path); crate::db::migrations::create_schema_connection(&connection) @@ -725,7 +734,7 @@ mod tests { let authority = DatabaseAuthority::for_runtime(&path, "mount exact-close code runtime").unwrap(); let profile_path = path.with_file_name("profile.db"); - rusqlite::Connection::open(&profile_path).unwrap(); + seed_empty_sqlite(&profile_path); let profile_path = profile_path.canonicalize().unwrap(); let registry = StoreRuntimeRegistry::new( asupersync::runtime::Runtime::current_handle().unwrap(), @@ -1052,7 +1061,7 @@ mod tests { let opened_identity = code.opened_file_identity().unwrap(); let detached = path.with_extension("detached.db"); let replacement = path.with_extension("replacement.db"); - rusqlite::Connection::open(&replacement).unwrap(); + seed_empty_sqlite(&replacement); std::fs::rename(&path, &detached).unwrap(); std::fs::rename(&replacement, &path).unwrap(); drop(code); @@ -1304,11 +1313,11 @@ mod tests { async fn failed_exact_close_retains_a_faulted_evicting_runtime() { let temporary = tempfile::tempdir().unwrap(); let path = temporary.path().join("runtime.db"); - rusqlite::Connection::open(&path).unwrap(); + seed_empty_sqlite(&path); let path = path.canonicalize().unwrap(); let authority = DatabaseAuthority::for_runtime(&path, "mount failing exact-close").unwrap(); let profile_path = path.with_file_name("profile.db"); - rusqlite::Connection::open(&profile_path).unwrap(); + seed_empty_sqlite(&profile_path); let profile_path = profile_path.canonicalize().unwrap(); let registry = StoreRuntimeRegistry::with_config( asupersync::runtime::Runtime::current_handle().unwrap(), diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs index 7d3a10fd52e..4f7c88b846d 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs @@ -196,7 +196,10 @@ fn health_request( fn seed_db(root: &TempDir, name: &str) -> PathBuf { let path = root.path().join(name); - Connection::open(&path).unwrap(); + Connection::open(&path) + .unwrap() + .execute_batch("PRAGMA user_version = 0") + .unwrap(); path.canonicalize().unwrap() } From 7506de73ab29be5b3b235f3da4e5402ace8bec16 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:29:56 -0800 Subject: [PATCH 18/60] test(sqlite): distinguish incomplete namespace from absent fixture Require the new typed invalid-header refusal for malformed graph stores. The isolated missing-database retry fixture now removes its own closed SQLite sidecars, so it models an absent basename without relaxing the production refusal of unknown WAL, SHM or journal files. Refs #4513 --- .../drop_shutdown_journey_test.rs | 6 ++++-- crates/tracedecay/src/daemon/tests/bootstrap.rs | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay/src/daemon/production_harness/drop_shutdown_journey_test.rs b/crates/tracedecay/src/daemon/production_harness/drop_shutdown_journey_test.rs index 7b159cbcf6a..d45218c298a 100644 --- a/crates/tracedecay/src/daemon/production_harness/drop_shutdown_journey_test.rs +++ b/crates/tracedecay/src/daemon/production_harness/drop_shutdown_journey_test.rs @@ -198,8 +198,10 @@ async fn a_failed_later_mount_releases_the_whole_composition_before_returning() Err(error) => error, }; assert!( - format!("{error:?}").contains("file is not a database"), - "expected the second mount to refuse its graph store: {error:?}" + format!("{error:?}").contains( + "existing SQLite file has no valid database header; refusing sidecar access", + ), + "expected the second mount to refuse its graph store before sidecar access: {error:?}" ); let first_layout = default_profile_sharded_layout(&first, &profile) diff --git a/crates/tracedecay/src/daemon/tests/bootstrap.rs b/crates/tracedecay/src/daemon/tests/bootstrap.rs index 34f8b7dbcf1..a4a5e68ed10 100644 --- a/crates/tracedecay/src/daemon/tests/bootstrap.rs +++ b/crates/tracedecay/src/daemon/tests/bootstrap.rs @@ -1772,6 +1772,20 @@ async fn explicit_init_retries_after_joining_an_ordinary_missing_database_open() let client_identity = test_client_identity_for(profile_root.clone()); let layout = initialize_test_project(&project, &client_identity).await; std::fs::remove_file(&layout.graph_db_path).expect("remove generated project database"); + // The fixture models a genuinely absent SQLite basename, not an + // incomplete store with orphan WAL, SHM or journal entries. The fixture + // project was closed by initialize_test_project above, so its test-owned + // sidecars can be retired here. Production admission must instead refuse + // unknown sidecars rather than erase them during retry. + for suffix in ["-wal", "-shm", "-journal"] { + let mut name = layout.graph_db_path.as_os_str().to_os_string(); + name.push(suffix); + match std::fs::remove_file(std::path::PathBuf::from(name)) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => panic!("remove isolated SQLite fixture sidecar: {error}"), + } + } let missing_graph_db_path = layout.graph_db_path.clone(); let _database_scope = From af51d02ce35d252e83e4f9ccb46c6da0787ae31c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 10:19:38 -0800 Subject: [PATCH 19/60] test(sqlite): preserve writer locks during existing-store admission Refs #4513. Add a non-vacuous cross-process rollback/WAL regression that admits a page-initialized live store, checks a separate process cannot steal BEGIN IMMEDIATE, confirms read-only committed visibility, then verifies final integrity after proper pin release. --- .../src/connection/path_identity_tests.rs | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs index e9721ca3b1d..9bba6bb8a5e 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs @@ -190,6 +190,53 @@ fn rejecting_empty_attachment_does_not_release_another_writer_lock() { probe_external_connection(&path, "write", false); } +/// SQLite's immutable header inspection during existing-store admission must +/// not release an unrelated live connection's process-owned POSIX locks. +/// A separate process is essential: same-process SQLite handles can mask loss. +#[test] +fn admitting_initialized_store_keeps_existing_writer_locks() { + for journal_mode in ["DELETE", "WAL"] { + let temp = TempDir::new().expect("tempdir"); + let path = temp.path().join("live-existing.db"); + let writer = Connection::open(&path).expect("initialize existing SQLite file"); + writer + .execute_batch("CREATE TABLE sample(value INTEGER); INSERT INTO sample VALUES (1);") + .expect("commit initial sample"); + writer + .pragma_update(None, "journal_mode", journal_mode) + .expect("select journal mode"); + if journal_mode == "WAL" { + // Keep the committed schema on the main file for the immutable + // inspection: it deliberately does not consume live WAL frames. + writer + .execute_batch("PRAGMA wal_checkpoint(TRUNCATE)") + .expect("checkpoint initial schema"); + } + writer + .execute_batch("BEGIN IMMEDIATE; UPDATE sample SET value = 2;") + .expect("hold writer reservation"); + probe_external_connection(&path, "write", true); + + // This is the production existing-store admission path, not a + // substitute read-only connection opened by the test itself. + let opened = OpenedDatabaseFile::pin_initialized(&path) + .expect("inspect and pin a fully initialized main database"); + opened + .verify_current_path(&path) + .expect("original database remains bound"); + probe_external_connection(&path, "write", true); + probe_external_connection_expectation(&path, "read", "read:1"); + + writer.execute_batch("COMMIT").expect("commit update"); + // The pinned raw descriptor is intentionally closed only after the + // writer exits, so its own close cannot independently release locks. + drop(writer); + opened.release_after_workers_exit(); + probe_external_connection(&path, "write", false); + assert_committed_data(&path); + } +} + #[test] fn header_magic_without_a_complete_sqlite_page_is_refused() { let temp = TempDir::new().expect("tempdir"); From 6fb6bbbecb25d05a377cf6b39974df2ce83e160a Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 11:17:42 -0800 Subject: [PATCH 20/60] test(sqlite): keep FIFO preflight regression nonblocking Refs #4513. Reject a nonregular store before opening its FIFO, without letting the test's fallback writer block after that early refusal. Preserve the original concurrent-open assertion if admission reaches its blocked FIFO read path. --- .../registry/tests/production_routes.rs | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs index 4f7c88b846d..6ab0a9737b8 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs @@ -331,8 +331,8 @@ async fn read_only_open_mounts_readers_without_acquiring_a_writer() { /// The store file is a FIFO, so opening it waits until a writer opens the /// other end. Only a task on the runtime's single worker does that while the -/// open waits; a fallback thread ends a wait that parked the worker, so the -/// failure is an assertion rather than a hang. +/// open waits. A nonregular-file preflight may also refuse the FIFO before +/// opening it; the fallback must not block waiting for a reader in that case. #[cfg(unix)] #[asupersync::test] async fn store_open_keeps_the_async_worker_serving_while_the_file_opens() { @@ -363,9 +363,20 @@ async fn store_open_keeps_the_async_worker_serving_while_the_file_opens() { let fallback = { let held = held.clone(); std::thread::spawn(move || { - let parked = fallback_stop.recv_timeout(Duration::from_secs(10)).is_err(); + let parked = matches!( + fallback_stop.recv_timeout(Duration::from_secs(10)), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) + ); if parked { - drop(OpenOptions::new().write(true).open(&held)); + // No FIFO reader exists if admission rejected the special + // file before SQLite could open it. The rescue itself must + // never introduce a second, unbounded blocking open. + drop( + OpenOptions::new() + .write(true) + .custom_flags(libc::O_NONBLOCK) + .open(&held), + ); } parked }) @@ -404,9 +415,15 @@ async fn store_open_keeps_the_async_worker_serving_while_the_file_opens() { drop(released_by_worker); let parked = fallback.join().unwrap(); + let refused_nonregular = matches!( + &outcome, + StoreRuntimeOpenResult::Failed( + StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { message, .. } + ) if message.contains("not a regular file") + ); assert!( - released && !parked, - "the store open parked the runtime's only worker until the fallback thread released the file" + !parked && (released || refused_nonregular), + "the worker must release a pending FIFO reader, or admission must refuse the nonregular file before opening it: {outcome:?}" ); assert!( matches!( From eb2573aed4593bb2bcd9ac98666677587fe39bfd Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:18:11 -0800 Subject: [PATCH 21/60] fix(sqlite): use path-only Linux pins to preserve writer locks Refs #4513. Closing an O_RDONLY identity pin can release a different SQLite writer's POSIX byte locks. Pin existing stores with O_PATH on Linux/Android so the /proc/self/fd worker route retains inode identity without that lock-loss side effect. Exercise pin release against a live separate-process write contender in rollback and WAL modes. --- .../src/connection/mod.rs | 15 +++++++- .../src/connection/path_identity_tests.rs | 38 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs index 507d857d222..0d6f843c7a1 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs @@ -1,5 +1,7 @@ #[cfg(unix)] use std::ffi::CString; +#[cfg(any(target_os = "linux", target_os = "android"))] +use std::os::unix::fs::OpenOptionsExt; #[cfg(windows)] use std::os::windows::fs::OpenOptionsExt; use std::{ @@ -400,7 +402,18 @@ impl fmt::Display for OpenedDatabaseFileError { impl std::error::Error for OpenedDatabaseFileError {} -#[cfg(not(windows))] +/// An O_PATH descriptor pins the inode for /proc/self/fd based SQLite opens, +/// but its close does not release another connection's process-wide POSIX +/// record locks. O_RDONLY pins have that destructive close behavior on Linux. +#[cfg(any(target_os = "linux", target_os = "android"))] +fn open_pinned_database(path: &Path) -> io::Result { + OpenOptions::new() + .read(true) + .custom_flags(libc::O_PATH) + .open(path) +} + +#[cfg(not(any(windows, target_os = "linux", target_os = "android")))] fn open_pinned_database(path: &Path) -> io::Result { File::open(path) } diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs index 9bba6bb8a5e..9ca712d87d0 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs @@ -193,6 +193,44 @@ fn rejecting_empty_attachment_does_not_release_another_writer_lock() { /// SQLite's immutable header inspection during existing-store admission must /// not release an unrelated live connection's process-owned POSIX locks. /// A separate process is essential: same-process SQLite handles can mask loss. +/// Releasing an existing-store identity pin must not cancel locks owned by +/// a different live SQLite writer in this process. O_RDONLY descriptors do +/// exactly that on Linux even if they never acquired a lock themselves. +#[cfg(any(target_os = "linux", target_os = "android"))] +#[test] +fn releasing_existing_store_pin_preserves_unrelated_writer_locks() { + for journal_mode in ["DELETE", "WAL"] { + let temp = TempDir::new().expect("tempdir"); + let path = temp.path().join("sessions.db"); + let writer = Connection::open(&path).expect("initialize database"); + writer + .execute_batch("CREATE TABLE sample(value INTEGER); INSERT INTO sample VALUES (1);") + .expect("initialize committed schema"); + writer + .pragma_update(None, "journal_mode", journal_mode) + .expect("select journal mode"); + if journal_mode == "WAL" { + writer + .execute_batch("PRAGMA wal_checkpoint(TRUNCATE)") + .expect("checkpoint the initial schema"); + } + let pin = OpenedDatabaseFile::pin_initialized(&path) + .expect("pin fully initialized existing database"); + + writer + .execute_batch("BEGIN IMMEDIATE; UPDATE sample SET value = 2;") + .expect("hold an unrelated writer reservation"); + probe_external_connection(&path, "write", true); + pin.release_after_workers_exit(); + probe_external_connection(&path, "write", true); + + writer.execute_batch("COMMIT").expect("commit"); + drop(writer); + probe_external_connection(&path, "write", false); + assert_committed_data(&path); + } +} + #[test] fn admitting_initialized_store_keeps_existing_writer_locks() { for journal_mode in ["DELETE", "WAL"] { From b931d06fa10d53aa5efe45c5af8673900a21f6cc Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:37:42 -0800 Subject: [PATCH 22/60] test(runtime): exercise queued attachment with a valid store --- .../registry/tests/production_routes.rs | 153 ++++++++---------- 1 file changed, 67 insertions(+), 86 deletions(-) diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs index 6ab0a9737b8..3a93b9414ed 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs @@ -329,111 +329,92 @@ async fn read_only_open_mounts_readers_without_acquiring_a_writer() { assert_health_route(&project, false).await; } -/// The store file is a FIFO, so opening it waits until a writer opens the -/// other end. Only a task on the runtime's single worker does that while the -/// open waits. A nonregular-file preflight may also refuse the FIFO before -/// opening it; the fallback must not block waiting for a reader in that case. #[cfg(unix)] #[asupersync::test] -async fn store_open_keeps_the_async_worker_serving_while_the_file_opens() { - use std::fs::OpenOptions; - use std::os::unix::fs::OpenOptionsExt; - use std::sync::atomic::AtomicBool; - +async fn store_open_refuses_a_fifo_before_opening_it() { let root = TempDir::new().unwrap(); - let held = root.path().join("held.db"); + let fifo = root.path().join("held.db"); assert!( std::process::Command::new("mkfifo") - .arg(&held) + .arg(&fifo) .status() .unwrap() .success() ); - let held = held.canonicalize().unwrap(); let resolver = Arc::new(FileResolver::default()); - resolver.push(held.clone()); + resolver.push(fifo.canonicalize().unwrap()); let registry = StoreRuntimeRegistry::new( asupersync::runtime::Runtime::current_handle().unwrap(), resolver, Arc::new(LifecycleShardRuntimePublisher), ) .unwrap(); + let outcome = registry + .open(StoreRuntimeOpenRequest::new( + profile_shard(), + incarnation(), + None, + )) + .await; + assert!(matches!( + outcome, + StoreRuntimeOpenResult::Failed(StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { + message, .. + }) if message.contains("not a regular file") + )); +} + +#[test] +fn store_open_keeps_the_async_worker_serving_while_attachment_waits() { + let executor = asupersync::runtime::RuntimeBuilder::current_thread() + .blocking_threads(1, 1) + .enable_time() + .build() + .unwrap(); + executor.block_on(async { + let root = TempDir::new().unwrap(); + let resolver = Arc::new(FileResolver::default()); + resolver.push(seed_final_graph_db(&root, "profile.db").await); + let registry = StoreRuntimeRegistry::new( + asupersync::runtime::Runtime::current_handle().unwrap(), + resolver, + Arc::new(LifecycleShardRuntimePublisher), + ) + .unwrap(); - let (released_by_worker, fallback_stop) = std::sync::mpsc::channel::<()>(); - let fallback = { - let held = held.clone(); - std::thread::spawn(move || { - let parked = matches!( - fallback_stop.recv_timeout(Duration::from_secs(10)), - Err(std::sync::mpsc::RecvTimeoutError::Timeout) - ); - if parked { - // No FIFO reader exists if admission rejected the special - // file before SQLite could open it. The rescue itself must - // never introduce a second, unbounded blocking open. - drop( - OpenOptions::new() - .write(true) - .custom_flags(libc::O_NONBLOCK) - .open(&held), - ); - } - parked - }) - }; - let settled = AtomicBool::new(false); - let open = async { - let outcome = registry - .open(StoreRuntimeOpenRequest::new( - profile_shard(), - incarnation(), - None, - )) - .await; - settled.store(true, Ordering::SeqCst); - outcome - }; - let release = async { let cx = asupersync::Cx::current().unwrap(); - while !settled.load(Ordering::SeqCst) { - match OpenOptions::new() - .write(true) - .custom_flags(libc::O_NONBLOCK) - .open(&held) - { - Ok(_writer) => { - released_by_worker.send(()).unwrap(); - return true; - } - Err(error) => assert_eq!(error.raw_os_error(), Some(libc::ENXIO)), - } - asupersync::time::sleep(cx.now(), Duration::from_millis(1)).await; - } - false - }; - let (outcome, released) = asupersync::join!(open, release); - drop(released_by_worker); - let parked = fallback.join().unwrap(); + let pool = cx.blocking_pool_handle().unwrap(); + let (entered, mut ready) = asupersync::channel::oneshot::channel(); + let (release, held) = std::sync::mpsc::channel(); + let mut blocker = cx + .spawn_blocking(move |_| { + entered.send_blocking(()).unwrap(); + held.recv_timeout(Duration::from_secs(10)).is_ok() + }) + .unwrap(); + ready.recv(&cx).await.unwrap(); - let refused_nonregular = matches!( - &outcome, - StoreRuntimeOpenResult::Failed( - StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { message, .. } - ) if message.contains("not a regular file") - ); - assert!( - !parked && (released || refused_nonregular), - "the worker must release a pending FIFO reader, or admission must refuse the nonregular file before opening it: {outcome:?}" - ); - assert!( - matches!( - outcome, - StoreRuntimeOpenResult::Failed( - StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { .. } - ) - ), - "a FIFO is not a SQLite store: {outcome:?}" - ); + let open = open_published( + ®istry, + StoreRuntimeOpenRequest::new(profile_shard(), incarnation(), None), + ); + let serve = async { + asupersync::time::timeout(cx.now(), Duration::from_secs(10), async { + while pool.pending_count() == 0 { + asupersync::time::sleep(cx.now(), Duration::from_millis(1)).await; + } + }) + .await + .expect("physical attachment must queue while the pool is occupied"); + release.send(()).unwrap(); + }; + let (opened, ()) = asupersync::join!(open, serve); + assert!( + blocker.join(&cx).await.unwrap(), + "the async worker must release the pending attachment" + ); + assert_health_route(&opened, true).await; + }); } #[asupersync::test] From ecfd8f6b0b098071a400944e1ad09653037187eb Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:52:34 -0800 Subject: [PATCH 23/60] test(automation): bind scheduler fixtures to backend identities Repair three hosted automation_runner_test failures by supplying the configured executable identity required by the production prefilter. Use the same identity for the prefilter and retained runner, preserving manual-schedule, exact-ledger, live-lock and lock-race assertions. Add a production-prefilter regression for unconfigured refusal, deduplication and same-occurrence recovery after binding, with zero backend calls and no effectful terminal. Do not relax backend admission, change timeouts, or alter the graph publication changes. Refs #4532 --- .../tests/automation_runner_test/jobs.rs | 124 ++++++++++++++++-- 1 file changed, 110 insertions(+), 14 deletions(-) diff --git a/crates/tracedecay/tests/automation_runner_test/jobs.rs b/crates/tracedecay/tests/automation_runner_test/jobs.rs index 740283d56e8..7342786381b 100644 --- a/crates/tracedecay/tests/automation_runner_test/jobs.rs +++ b/crates/tracedecay/tests/automation_runner_test/jobs.rs @@ -86,6 +86,7 @@ fn enabled_job_config() -> AutomationSettingsV1 { struct ContentBackend { calls: AtomicUsize, content: &'static str, + executable: Option, } impl ContentBackend { @@ -93,9 +94,15 @@ impl ContentBackend { Self { calls: AtomicUsize::new(0), content, + executable: None, } } + fn with_executable(mut self, executable: &Path) -> Self { + self.executable = Some(executable.to_path_buf()); + self + } + fn calls(&self) -> usize { self.calls.load(Ordering::SeqCst) } @@ -126,7 +133,7 @@ impl AgentTaskBackend for ContentBackend { } fn executable(&self) -> Option<&std::path::Path> { - None + self.executable.as_deref() } } @@ -988,6 +995,10 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let mut job = sample_job("sched-job"); job.schedule = Some("manual".to_string()); let config = enabled_job_config(); + // Reach the schedule gate with a bound identity, not the earlier + // backend-unconfigured refusal. This gate never invokes the backend. + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = ContentBackend::new("unused").with_executable(executable.path()); // Manual schedule: every scheduler tick of the same occurrence reaches // the prefilter, which records the skip once. @@ -996,7 +1007,7 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let run = evaluate_and_record_scheduler_skip( &dashboard_root, &config, - None, + backend.executable(), &job, "sched-occurrence", None, @@ -1019,7 +1030,7 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let repeated = evaluate_and_record_scheduler_skip( &dashboard_root, &config, - None, + backend.executable(), &job, "sched-occurrence", None, @@ -1031,6 +1042,76 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { repeated.ledger_record, records[0], "an immediate repeat skip must return the exact durable prior row" ); + assert_eq!(backend.calls(), 0); +} + +#[tokio::test] +async fn scheduler_prefilter_rejects_unconfigured_backend_and_recovers_without_effect() { + let temp = tempdir().unwrap(); + let dashboard_root = temp.path().join("dashboard"); + let job = sample_job("backend-recovery"); + let config = enabled_job_config(); + let occurrence = "backend-recovery-occurrence"; + let backend = ContentBackend::new("unused"); + + let first = evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .expect("a due job still requires a configured backend"); + assert_eq!(first.report["reason"], json!("backend_unconfigured")); + assert_eq!(first.ledger_record.status, AutomationRunStatus::Skipped); + assert_ne!(first.run_id, occurrence); + let repeated = evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(repeated.ledger_record, first.ledger_record); + + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = backend.with_executable(executable.path()); + assert!( + evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .is_none(), + "binding the backend must admit the same due occurrence without cooldown" + ); + assert_eq!(backend.calls(), 0); + assert_eq!( + load_run_records(&dashboard_root, 10).await.unwrap(), + vec![first.ledger_record], + "advisory admission must not append an effectful terminal" + ); + assert!( + tracedecay_automation_runtime::automation::run_ledger::find_run_record_exact_bounded( + &dashboard_root, + occurrence, + ) + .await + .unwrap() + .is_none() + ); } #[tokio::test] @@ -1206,15 +1287,22 @@ async fn scheduler_prefilter_live_lock_wins_over_not_due_summary() { let config = enabled_job_config(); let occurrence = "locked-prefilter-occurrence"; - let prefilter = - evaluate_and_record_scheduler_skip(&dashboard_root, &config, None, &job, occurrence, None) - .await - .unwrap() - .expect("the live job lock must produce a diagnostic"); + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = ContentBackend::new("unused").with_executable(executable.path()); + let prefilter = evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .expect("the live job lock must produce a diagnostic"); assert_eq!(prefilter.report["reason"], json!("scheduler_lock_active")); assert_ne!(prefilter.run_id, occurrence); - let backend = ContentBackend::new("unused"); let admitted_job = tracedecay_automation_runtime::automation::jobs::admit_job(&job).unwrap(); let retained = run_user_job_with_backend_for_retained_settlement( &dashboard_root, @@ -1262,12 +1350,21 @@ async fn retained_scheduler_runner_reacquires_after_due_prefilter() { job.schedule = Some("every 1h".to_owned()); let config = enabled_job_config(); let occurrence = "prefilter-gap-occurrence"; + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = ContentBackend::new("unused").with_executable(executable.path()); assert!( - evaluate_and_record_scheduler_skip(&dashboard_root, &config, None, &job, occurrence, None) - .await - .unwrap() - .is_none(), + evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .is_none(), "a new interval job is advisory-due before effect admission" ); let _raced_in = @@ -1276,7 +1373,6 @@ async fn retained_scheduler_runner_reacquires_after_due_prefilter() { .unwrap() .expect("the advisory prefilter must release its reservation lock"); - let backend = ContentBackend::new("unused"); let admitted_job = tracedecay_automation_runtime::automation::jobs::admit_job(&job).unwrap(); let retained = run_user_job_with_backend_for_retained_settlement( &dashboard_root, From 26311e57a49f54196395a2806f836b77b10556bf Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:59:00 -0800 Subject: [PATCH 24/60] fix(storage): publish complete SQLite stores after schema installation --- .../src/shard_runtime/registry.rs | 1 + .../shard_runtime/registry/initialization.rs | 455 ++++++++++++++++++ .../src/shard_runtime/registry/ports.rs | 212 +++++--- .../registry/tests/production_routes.rs | 66 +++ .../src/sqlite_read_snapshot.rs | 2 +- .../src/connection/mod.rs | 35 +- .../src/connection/tests.rs | 26 + .../src/schema_version.rs | 4 + 8 files changed, 729 insertions(+), 72 deletions(-) create mode 100644 crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry.rs index da1e1615004..29d7db89a91 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry.rs @@ -14,6 +14,7 @@ mod capacity; mod close; mod destructive; mod graph; +mod initialization; mod leases; mod open; mod ports; diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs new file mode 100644 index 00000000000..1a903bc34f1 --- /dev/null +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs @@ -0,0 +1,455 @@ +//! Prospective stores install their schema outside the SQLite basename used by +//! clients. A process death before the no-replace rename leaves only disposable +//! schema in the private work directory. Hard links identify the empty sidecar +//! reservations if the process dies partway through claiming the destination. +use std::fs::{self, File}; +use std::io; +use std::path::{Path, PathBuf}; + +use rusqlite::{Connection, OpenFlags}; +use tracedecay_private_fs::framed_log::{ + DirectorySyncPolicy, rename_noreplace, sync_directory, sync_parent_directory, +}; +use tracedecay_private_fs::{ + FileLease, create_private_directory, create_private_file, open_private_directory, + open_private_file, +}; + +use crate::sqlite_read_snapshot::{fold_staging_to_standalone, with_suffix}; + +const SIDECARS: [&str; 3] = ["-wal", "-shm", "-journal"]; +const SYNC: DirectorySyncPolicy = DirectorySyncPolicy::TolerateUnsupported; + +pub(super) struct InitializationDirectory { + destination: PathBuf, + directory: PathBuf, + _directory_pin: File, + _lease: FileLease, + owns_schema: bool, +} + +impl InitializationDirectory { + fn path(destination: &Path) -> PathBuf { + with_suffix(destination, ".tracedecay-init") + } + + fn acquire(destination: PathBuf, create: bool) -> io::Result { + let directory = Self::path(&destination); + if create { + match create_private_directory(&directory) { + Ok(()) => sync_parent_directory(&directory, SYNC)?, + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error), + } + } + let directory_pin = open_private_directory(&directory)?; + let directory = directory.canonicalize()?; + let path = directory.join("schema-installation.lock"); + let lock = match open_private_file(&path) { + Ok(file) => file, + Err(error) if create && error.kind() == io::ErrorKind::NotFound => { + if fs::read_dir(&directory)?.next().transpose()?.is_some() { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "unclaimed SQLite initialization directory is occupied", + )); + } + match create_private_file(&path) { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => { + open_private_file(&path)? + } + Err(error) => return Err(error), + } + } + Err(error) => return Err(error), + }; + lock.try_lock().map_err(io::Error::from)?; + Ok(Self { + destination, + directory, + _directory_pin: directory_pin, + owns_schema: false, + _lease: FileLease::held(lock, "sqlite_initialization.owner"), + }) + } + + pub(super) fn begin(destination: PathBuf) -> io::Result { + let mut stage = Self::acquire(destination, true)?; + if present(&stage.destination)? { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "SQLite destination exists", + )); + } + // Check every occupied canonical member before discarding even our + // private schema. An interrupted pre-protocol family has no witness. + stage.validate_sidecars(false)?; + if present(&stage.transferred())? + || (present(&stage.prepared())? + && (!present(&stage.database())? + || !same_regular_file(&stage.database(), &stage.prepared())?)) + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "published SQLite file is missing or ambiguous", + )); + } + stage.discard_schema()?; + stage.owns_schema = true; + Ok(stage) + } + + pub(super) fn database(&self) -> PathBuf { + self.directory.join("schema.sqlite") + } + + fn prepared(&self) -> PathBuf { + self.directory.join("prepared.sqlite") + } + + fn transferred(&self) -> PathBuf { + self.directory.join("transferred.sqlite") + } + + fn discard_schema(&self) -> io::Result<()> { + remove_if_present(&self.prepared())?; + for suffix in ["-wal", "-shm", "-journal", ""] { + remove_if_present(&with_suffix(&self.database(), suffix))?; + } + Ok(()) + } + + fn witness(&self, suffix: &str) -> PathBuf { + self.directory.join(format!("reservation{suffix}")) + } + + fn validate_sidecars(&self, required: bool) -> io::Result<()> { + for suffix in SIDECARS { + let name = with_suffix(&self.destination, suffix); + if !required && !present(&name)? { + continue; + } + if !empty_regular(&name)? + || !present(&self.witness(suffix))? + || !same_regular_file(&name, &self.witness(suffix))? + { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "occupied SQLite sidecar has no matching initialization reservation", + )); + } + } + Ok(()) + } + + fn reserve_sidecars(&self) -> io::Result<()> { + self.validate_sidecars(false)?; + for suffix in SIDECARS { + let witness = self.witness(suffix); + if !present(&witness)? { + create_private_file(&witness)?.sync_all()?; + } + if !empty_regular(&witness)? { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "invalid SQLite reservation", + )); + } + } + sync_directory(&self.directory, SYNC)?; + for suffix in SIDECARS { + let name = with_suffix(&self.destination, suffix); + if !present(&name)? { + fs::hard_link(self.witness(suffix), name)?; + } + } + sync_parent_directory(&self.destination, SYNC)?; + self.validate_sidecars(true) + } + + pub(super) fn publish(&self) -> io::Result<()> { + let path = self.database(); + let connection = Connection::open_with_flags( + &path, + OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_NOFOLLOW, + ) + .map_err(io::Error::other)?; + connection + .pragma_update( + None, + "application_id", + tracedecay_rusqlite_runtime::schema_version::REGISTERED_STORE_APPLICATION_ID, + ) + .map_err(io::Error::other)?; + fold_staging_to_standalone(connection, &path, || Ok(()))?; + File::options() + .read(true) + .write(true) + .open(&path)? + .sync_all()?; + self.claim_destination()?; + rename_noreplace(&path, &self.destination)?; + sync_parent_directory(&self.destination, SYNC)?; + self.finish_publication() + } + + fn claim_destination(&self) -> io::Result<()> { + // This alias witnesses the exact main inode across the no-replace + // rename. Remove it last, before the first runtime is returned. + fs::hard_link(self.database(), self.prepared())?; + self.reserve_sidecars() + } + + fn finish_publication(&self) -> io::Result<()> { + if present(&self.database())? { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "SQLite destination coexists with an unpublished initialization", + )); + } + if present(&self.prepared())? { + if !same_regular_file(&self.destination, &self.prepared())? { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "SQLite destination identity changed before handoff", + )); + } + self.validate_sidecars(true)?; + // All witnesses remain until this single namespace transition. + // Ordinary SQLite clients own the WAL, SHM and journal afterward. + rename_noreplace(&self.prepared(), &self.transferred())?; + sync_directory(&self.directory, SYNC)?; + } + if !present(&self.transferred())? + || !same_regular_file(&self.destination, &self.transferred())? + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "SQLite destination has no matching handoff", + )); + } + for suffix in SIDECARS { + remove_if_present(&self.witness(suffix))?; + } + remove_if_present(&self.transferred())?; + sync_directory(&self.directory, SYNC) + } + + // The normal existing/read-only route does not need to open an admission + // lease or write to the filesystem. Only an interrupted publication with + // outstanding private files requires reconciliation. + pub(super) fn recover_published(destination: &Path) -> io::Result<()> { + let directory = Self::path(destination); + if !present(&directory)? { + return Ok(()); + } + let _pin = open_private_directory(&directory)?; + if !Self::pending(&directory)? { + return Ok(()); + } + let stage = Self::acquire(destination.to_owned(), false)?; + if !Self::pending(&directory)? { + return Ok(()); + } + stage.finish_publication() + } + + fn pending(directory: &Path) -> io::Result { + for name in [ + "schema.sqlite", + "prepared.sqlite", + "transferred.sqlite", + "reservation-wal", + "reservation-shm", + "reservation-journal", + ] { + if present(&directory.join(name))? { + return Ok(true); + } + } + Ok(false) + } +} + +impl Drop for InitializationDirectory { + fn drop(&mut self) { + if !self.owns_schema { + return; + } + // A destination that appeared during this attempt is ambiguous until + // a new opener verifies the prepared-main witness. + match present(&self.destination) { + Ok(false) => { + if let Err(error) = self.discard_schema() { + tracing::warn!(%error, "could not retire unpublished SQLite schema"); + } + } + Err(error) => { + tracing::warn!(%error, "could not inspect initialized SQLite destination") + } + Ok(true) => {} + } + } +} + +fn present(path: &Path) -> io::Result { + match fs::symlink_metadata(path) { + Ok(_) => Ok(true), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } +} + +fn remove_if_present(path: &Path) -> io::Result<()> { + match fs::remove_file(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + } +} + +fn empty_regular(path: &Path) -> io::Result { + let meta = fs::symlink_metadata(path)?; + Ok(meta.is_file() && meta.len() == 0) +} + +fn same_regular_file(left: &Path, right: &Path) -> io::Result { + if !fs::symlink_metadata(left)?.is_file() || !fs::symlink_metadata(right)?.is_file() { + return Ok(false); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let a = fs::symlink_metadata(left)?; + let b = fs::symlink_metadata(right)?; + Ok(a.dev() == b.dev() && a.ino() == b.ino()) + } + #[cfg(windows)] + { + use tracedecay_private_fs::windows_file::information; + let a = information(&File::open(left)?)?; + let b = information(&File::open(right)?)?; + Ok(a.volume_serial_number == b.volume_serial_number && a.file_index == b.file_index) + } + #[cfg(not(any(unix, windows)))] + Err(io::Error::new( + io::ErrorKind::Unsupported, + "SQLite file identity unavailable", + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::process::Command; + use tempfile::TempDir; + + fn seed(path: &Path, value: i64) { + let db = Connection::open(path).unwrap(); + db.execute_batch("CREATE TABLE data(value INTEGER)") + .unwrap(); + db.execute("INSERT INTO data VALUES (?)", [value]).unwrap(); + } + + #[test] + fn process_exit_during_initialization_recovers_without_publishing_partial_schema() { + const CHILD: &str = "TRACEDECAY_TEST_EXIT_DURING_INITIALIZATION"; + if let Some(destination) = std::env::var_os(CHILD) { + let stage = InitializationDirectory::begin(destination.into()).unwrap(); + seed(&stage.database(), 1); + let state = std::env::var("TRACEDECAY_TEST_INITIALIZATION_STATE").unwrap(); + match state.as_str() { + "installing" => {} + "reserved" => stage.claim_destination().unwrap(), + "published" | "transferred" | "unproven" => { + stage.claim_destination().unwrap(); + rename_noreplace(&stage.database(), &stage.destination).unwrap(); + if state == "transferred" { + rename_noreplace(&stage.prepared(), &stage.transferred()).unwrap(); + } + if state != "published" { + remove_if_present(&stage.witness("-wal")).unwrap(); + } + } + state => panic!("unknown initialization state {state}"), + } + // Simulate ordinary process death with no Rust or SQLite destructors. + std::process::exit(73); + } + + let name = format!( + "{}::process_exit_during_initialization_recovers_without_publishing_partial_schema", + module_path!().split_once("::").unwrap().1 + ); + for state in [ + "installing", + "reserved", + "published", + "transferred", + "unproven", + ] { + let root = TempDir::new().unwrap(); + let destination = root.path().join("sessions.db"); + let output = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", &name]) + .env(CHILD, &destination) + .env("TRACEDECAY_TEST_INITIALIZATION_STATE", state) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(73), "{output:?}"); + if state == "unproven" { + assert!(InitializationDirectory::recover_published(&destination).is_err()); + for suffix in SIDECARS { + assert!(empty_regular(&with_suffix(&destination, suffix)).unwrap()); + } + continue; + } + let expected = if matches!(state, "published" | "transferred") { + InitializationDirectory::recover_published(&destination).unwrap(); + 1 + } else { + assert!(!destination.exists()); + let retry = InitializationDirectory::begin(destination.clone()).unwrap(); + assert!(!retry.database().exists()); + seed(&retry.database(), 2); + retry.publish().unwrap(); + 2 + }; + let db = Connection::open(&destination).unwrap(); + assert_eq!( + db.query_row("SELECT value FROM data", [], |row| row.get::<_, i64>(0)) + .unwrap(), + expected + ); + assert_eq!( + db.query_row("PRAGMA integrity_check", [], |row| row.get::<_, String>(0)) + .unwrap(), + "ok" + ); + assert!( + !InitializationDirectory::pending(&InitializationDirectory::path(&destination)) + .unwrap() + ); + } + } + + #[test] + fn competing_initialization_leaves_owner_and_foreign_sidecars_intact() { + let root = TempDir::new().unwrap(); + let destination = root.path().join("sessions.db"); + let stage = InitializationDirectory::begin(destination.clone()).unwrap(); + seed(&stage.database(), 4); + assert!(InitializationDirectory::begin(destination.clone()).is_err()); + assert!(stage.database().exists()); + drop(stage); + for suffix in SIDECARS { + let foreign = with_suffix(&destination, suffix); + fs::write(&foreign, b"pre-existing sidecar").unwrap(); + assert!(InitializationDirectory::begin(destination.clone()).is_err()); + assert_eq!(fs::read(&foreign).unwrap(), b"pre-existing sidecar"); + assert!(!destination.exists()); + fs::remove_file(foreign).unwrap(); + } + } +} diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs index 1a951d8b2c9..7f973397077 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs @@ -1,5 +1,5 @@ use std::future::Future; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::Arc; use std::time::Duration; @@ -17,6 +17,7 @@ use tracedecay_store::{ StoreShardScopeV1, VerifiedStoreLocatorV1, }; +use super::initialization::InitializationDirectory; use super::{ PhysicalRuntimeAttachment, PhysicalRuntimeSnapshot, PhysicalWriterRuntimeSnapshot, PublishedShardRuntime, StoreRuntimeAccessMode, StoreRuntimeKey, StoreRuntimeOpenMode, @@ -141,36 +142,135 @@ impl ShardRuntimePublisher for LifecycleShardRuntimePublisher { operation: "publish physical registered runtime", }, )?; - // Attaching opens the file and parks its thread until the writer - // and readers have opened it on the storage pool. - let attach_request = request.clone(); - let physical = - crate::native_runtime::spawn_blocking(move || { - LifecycleShardRuntimeAttachment::new(RepositoryPhysicalAttachmentFactory) - .attach(&executor, &attach_request, AdmissionConfigV1::default()) - }) - .await - .map_err(|message| { - StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { - operation: "attach rusqlite repository runtime", - message, - } - })??; - publish_lifecycle_runtime(request, runtime, physical).await + let published_identity = if request.locator.is_prospective() { + Some(initialize_staged_runtime(&executor, &request).await?) + } else { + None + }; + let attachment = attach_runtime(&executor, &request, None, published_identity).await?; + publish_lifecycle_runtime(request, runtime, attachment).await }) } } +fn physical_failure( + operation: &'static str, + error: impl std::fmt::Display, +) -> StoreRuntimeRegistryFailure { + StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { + operation, + message: error.to_string(), + } +} + +async fn attach_runtime( + executor: &RuntimeHandle, + request: &ShardRuntimeBuildRequest, + staging: Option>, + published_identity: Option, +) -> Result { + let executor = executor.clone(); + let request = request.clone(); + crate::native_runtime::spawn_blocking(move || { + const OPERATION: &str = "attach published SQLite generation"; + if let Some(expected) = published_identity { + let current = crate::db::sqlite_generation_identity(request.locator.path()) + .map_err(|error| physical_failure(OPERATION, error))?; + if current != expected { + return Err(physical_failure( + OPERATION, + "published database identity changed", + )); + } + } + let attachment = LifecycleShardRuntimeAttachment::new(RepositoryPhysicalAttachmentFactory) + .attach(&executor, &request, staging, AdmissionConfigV1::default())?; + if published_identity + .is_some_and(|expected| expected != attachment.0.opened_file_identity()) + { + attachment + .0 + .drain() + .and_then(|()| attachment.0.close_and_join()) + .map_err(|error| physical_failure(OPERATION, error))?; + return Err(physical_failure( + OPERATION, + "published database changed during attachment", + )); + } + Ok(attachment) + }) + .await + .map_err(|error| physical_failure("attach SQLite runtime", error))? +} + +async fn initialize_staged_runtime( + executor: &RuntimeHandle, + request: &ShardRuntimeBuildRequest, +) -> Result { + const OPERATION: &str = "publish initialized SQLite family"; + let authority = request + .database_authority + .as_ref() + .ok_or_else(|| physical_failure(OPERATION, "missing database authority"))?; + if request.mode != StoreRuntimeOpenMode::Initialize + || authority.canonical_database_path() != request.locator.path() + { + return Err(physical_failure( + OPERATION, + "initialization does not own this destination", + )); + } + authority + .require_active_write_scope(OPERATION) + .map_err(|error| physical_failure(OPERATION, error))?; + let destination = request.locator.path().to_owned(); + let stage = crate::native_runtime::spawn_blocking(move || { + InitializationDirectory::begin(destination).map(Arc::new) + }) + .await + .map_err(|error| physical_failure(OPERATION, error))? + .map_err(|error| physical_failure(OPERATION, error))?; + let path = stage.database(); + let attachment = attach_runtime(executor, request, Some(stage.clone()), None).await?; + let identity = attachment.0.opened_file_identity(); + let schema = + install_final_schema_before_publication(request, attachment.as_physical(), &path).await; + attachment + .close(schema.is_ok()) + .await + .map_err(|error| physical_failure(OPERATION, error))?; + schema?; + authority + .require_active_write_scope(OPERATION) + .map_err(|error| physical_failure(OPERATION, error))?; + if request.open_cancellation.is_cancelled() { + return Err(StoreRuntimeRegistryFailure::OpenCancelled { + key: Box::new(StoreRuntimeKey::from_binding(&request.binding)), + }); + } + crate::native_runtime::spawn_blocking(move || stage.publish()) + .await + .map_err(|error| physical_failure(OPERATION, error))? + .map_err(|error| physical_failure(OPERATION, error))?; + Ok(identity) +} + async fn publish_lifecycle_runtime( request: ShardRuntimeBuildRequest, runtime: ShardRuntime, attachment: LifecyclePhysicalAttachment, ) -> Result { if request.mode == StoreRuntimeOpenMode::Initialize - && let Err(error) = - install_final_schema_before_publication(&request, attachment.as_physical()).await + && !request.locator.is_prospective() + && let Err(error) = install_final_schema_before_publication( + &request, + attachment.as_physical(), + request.locator.path(), + ) + .await { - let _ = attachment.abort(request.locator.is_prospective()).await; + let _ = attachment.close(false).await; return Err(error); } if request.mode == StoreRuntimeOpenMode::Existing @@ -179,7 +279,7 @@ async fn publish_lifecycle_runtime( verify_final_schema_before_existing_publication(&request, attachment.as_physical()) .await { - let _ = attachment.abort(request.locator.is_prospective()).await; + let _ = attachment.close(false).await; return Err(error); } if request.mode == StoreRuntimeOpenMode::Existing @@ -190,30 +290,18 @@ async fn publish_lifecycle_runtime( { // This failed attachment must finish closing before the registry can // retry at a new epoch. - attachment - .abort(request.locator.is_prospective()) - .await - .map_err( - |message| StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { - operation: "close runtime after writer authority refusal", - message, - }, - )?; + attachment.close(false).await.map_err(|message| { + StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { + operation: "close runtime after writer authority refusal", + message, + } + })?; return Err(error); } if let Err(error) = runtime.transition(RuntimeMaintenanceStateV1::Ready) { - let _ = attachment.abort(request.locator.is_prospective()).await; + let _ = attachment.close(false).await; return Err(runtime_lifecycle_failure(error)); } - if request.locator.is_prospective() - && let Err(error) = attachment.commit_initialization() - { - let _ = attachment.abort(true).await; - return Err(StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { - operation: "commit initialized SQLite runtime", - message: error, - }); - } Ok(PublishedShardRuntime::new(runtime, attachment.into_box())) } @@ -297,7 +385,10 @@ async fn verify_final_schema_before_existing_publication( }) } -struct LifecyclePhysicalAttachment(RepositoryRuntimePhysicalAttachment); +struct LifecyclePhysicalAttachment( + RepositoryRuntimePhysicalAttachment, + Option>, +); struct LifecycleShardRuntimeAttachment { repository: RepositoryPhysicalAttachmentFactory, @@ -312,20 +403,19 @@ impl LifecycleShardRuntimeAttachment { &self, runtime: &RuntimeHandle, request: &ShardRuntimeBuildRequest, + staging: Option>, admission: AdmissionConfigV1, ) -> Result { - if request.locator.is_prospective() && request.mode != StoreRuntimeOpenMode::Initialize { - return Err(StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { - operation: "open prospective SQLite runtime", - message: "prospective locators require explicit initialization".to_owned(), - }); + if staging.is_none() { + InitializationDirectory::recover_published(request.locator.path()) + .map_err(|error| physical_failure("recover initialized SQLite family", error))?; } - let attachment = if request.locator.is_prospective() { + let attachment = if let Some(stage) = &staging { self.repository.initialize( runtime, request.binding.clone(), request.locator.verified().clone(), - request.locator.path().to_path_buf(), + stage.database(), admission, ) } else if request.access == StoreRuntimeAccessMode::ReadOnly { @@ -348,7 +438,7 @@ impl LifecycleShardRuntimeAttachment { operation: "attach rusqlite repository runtime", message: error.to_string(), })?; - Ok(LifecyclePhysicalAttachment(attachment)) + Ok(LifecyclePhysicalAttachment(attachment, staging)) } } @@ -357,20 +447,19 @@ impl LifecyclePhysicalAttachment { &self.0 } - fn commit_initialization(&self) -> Result<(), String> { - self.0.commit_initialization() - } - /// Closes an attachment that will not be published. Joining its writer /// and readers parks the calling thread. - async fn abort(self, created: bool) -> Result<(), String> { + async fn close(self, completed_schema: bool) -> Result<(), String> { crate::native_runtime::spawn_blocking(move || { - if created { - self.0.abort_initialization() - } else { - self.0.drain()?; - self.0.close_and_join() + let stage = self.1; + if stage.is_some() { + if !completed_schema { + return self.0.abort_initialization(); + } + self.0.commit_initialization()?; } + self.0.drain()?; + self.0.close_and_join() }) .await .and_then(|closed| closed) @@ -428,6 +517,7 @@ async fn authorized_schema_connection( request: &ShardRuntimeBuildRequest, attachment: &dyn PhysicalRuntimeAttachment, operation: &'static str, + physical_path: &Path, ) -> Result { let authority = request.database_authority.clone().ok_or_else(|| { StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { @@ -462,7 +552,7 @@ async fn authorized_schema_connection( }); } let authority = InitializingSchemaAuthority { - canonical_path: authority.canonical_database_path().to_path_buf(), + canonical_path: physical_path.to_path_buf(), authority, opened_file_identity, }; @@ -478,11 +568,13 @@ async fn authorized_schema_connection( async fn install_final_schema_before_publication( request: &ShardRuntimeBuildRequest, attachment: &dyn PhysicalRuntimeAttachment, + physical_path: &Path, ) -> Result<(), StoreRuntimeRegistryFailure> { let connection = authorized_schema_connection( request, attachment, "install final schema for initialized SQLite runtime", + physical_path, ) .await?; match &request.binding.shard_id.scope { diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs index 3a93b9414ed..7a57e162ec2 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs @@ -542,6 +542,72 @@ async fn writable_existing_open_refuses_tampered_final_shape_before_publication( ); } +#[asupersync::test] +async fn cancelled_schema_publication_retries_with_the_final_store() { + let root = TempDir::new().unwrap(); + let path = root.path().canonicalize().unwrap().join("remote.db"); + let authority = + crate::db::DatabaseAuthority::acquire_test(&path, "retry remote initialization").unwrap(); + let shard = StoreShardIdV1::remote_node( + id::("brain.registry"), + id::("profile.registry"), + id::("node.remote"), + ); + let binding = StoreRuntimeBindingV1::new( + shard.clone(), + incarnation(), + StoreAuthorityEpochV1::new(1).unwrap(), + ); + let verified = VerifiedStoreLocatorV1::new( + shard, + incarnation(), + canonical_store_locator_digest(&path).unwrap(), + ); + let locator = RuntimeLocatorRecord::new( + StoreRuntimeKey::from_binding(&binding), + ResolvedStoreLocator::prospective(verified, path.clone()), + ); + let request = |cancellation| { + ShardRuntimeBuildRequest::new( + binding.clone(), + locator.clone(), + StoreRuntimeOpenMode::Initialize, + StoreRuntimeAccessMode::ReadWrite, + Some(authority.clone()), + cancellation, + ) + }; + let cancelled = CancellationToken::new(); + cancelled.cancel(); + assert!(matches!( + LifecycleShardRuntimePublisher + .publish(request(cancelled)) + .await, + Err(StoreRuntimeRegistryFailure::OpenCancelled { .. }) + )); + assert!(!path.exists()); + let published = LifecycleShardRuntimePublisher + .publish(request(CancellationToken::new())) + .await + .unwrap(); + let reader = + Connection::open_with_flags(&path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY).unwrap(); + let count: i64 = reader + .query_row("SELECT COUNT(*) FROM remote_node_identity", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(count, 0); + assert_eq!( + reader + .query_row("PRAGMA integrity_check", [], |row| row.get::<_, String>(0)) + .unwrap(), + "ok" + ); + drop(reader); + drop(published); +} + #[asupersync::test] async fn remote_node_initialization_installs_only_the_final_registered_schema() { let root = TempDir::new().unwrap(); diff --git a/crates/tracedecay-runtime-core/src/sqlite_read_snapshot.rs b/crates/tracedecay-runtime-core/src/sqlite_read_snapshot.rs index 66b2124ba13..2fcc70ee3da 100644 --- a/crates/tracedecay-runtime-core/src/sqlite_read_snapshot.rs +++ b/crates/tracedecay-runtime-core/src/sqlite_read_snapshot.rs @@ -302,7 +302,7 @@ fn backup_offline_wal_family_without_shm( Ok(true) } -fn fold_staging_to_standalone( +pub(crate) fn fold_staging_to_standalone( staging: Connection, staging_path: &Path, checkpoint: impl Fn() -> io::Result<()>, diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs index 0d6f843c7a1..1fe158cf397 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs @@ -86,14 +86,8 @@ impl OpenedDatabaseFile { Self::adopt(file) } - /// Refuse unfinished stores before SQLite opens their WAL/SHM/journal. - /// A rejected new-basename reservation intentionally leaves the newly - /// created database in place so no pathname-based rollback can delete a - /// foreign replacement. That zero-length database must never become an - /// ordinary existing attachment on retry: SQLite would initialize it and - /// overwrite precisely the occupied sidecar that caused the refusal. - /// Read the magic through the existing pin; opening a second descriptor - /// here could discard another connection's POSIX locks on Unix. + /// Inspect the immutable main before a read-only or writable SQLite + /// worker can open any ambiguous sidecar of an interrupted initialization. pub(crate) fn pin_initialized(path: &Path) -> Result { // A failed header read would drop a freshly opened raw descriptor, // releasing unrelated process-owned SQLite locks on Unix. Reject @@ -117,11 +111,26 @@ impl OpenedDatabaseFile { // never opens or mutates the unverified WAL/SHM/journal sidecars. let inspection = open_immutable_reader(path).map_err(|_| OpenedDatabaseFileError::InvalidHeader)?; - let _: i64 = inspection - .query_row("SELECT count(*) FROM sqlite_schema", [], |row| row.get(0)) + let initialized: bool = inspection + .query_row( + "SELECT (SELECT application_id FROM pragma_application_id) = ?1 OR EXISTS( + SELECT 1 FROM sqlite_schema WHERE type = 'table' + AND name NOT GLOB 'sqlite_*' AND name NOT GLOB 'td_runtime_*' + )", + [crate::schema_version::REGISTERED_STORE_APPLICATION_ID], + |row| row.get(0), + ) .map_err(|_| OpenedDatabaseFileError::InvalidHeader)?; drop(inspection); - // A valid main-file schema is necessary, not proof of sidecar ownership. + if !initialized { + for suffix in ["-wal", "-shm", "-journal"] { + match std::fs::symlink_metadata(sidecar_path(path, suffix)) { + Ok(_) => return Err(OpenedDatabaseFileError::IncompleteInitialization), + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(_) => return Err(OpenedDatabaseFileError::Inspect), + } + } + } Self::pin(path) } @@ -369,6 +378,7 @@ pub enum OpenedDatabaseFileError { Create, Reserve, InvalidHeader, + IncompleteInitialization, Open, Inspect, NotFile, @@ -384,6 +394,9 @@ impl fmt::Display for OpenedDatabaseFileError { let message = match self { Self::Create => "could not create the canonical SQLite file", Self::Reserve => "could not exclusively reserve the new SQLite sidecar namespace", + Self::IncompleteInitialization => { + "SQLite main has no published schema and its sidecar ownership is ambiguous" + } Self::InvalidHeader => { "existing SQLite file has no valid database header; refusing sidecar access" } diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs index cf179b3177c..7388d481e6b 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs @@ -656,6 +656,32 @@ fn verified_immutable_reader_never_binds_transient_b_to_restored_a_identity() { } } +#[test] +fn header_only_existing_store_preserves_ambiguous_sidecars() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("existing.db"); + Connection::open(&path) + .unwrap() + .execute_batch("PRAGMA journal_mode=WAL") + .unwrap(); + let member = super::sidecar_path(&path, "-journal"); + std::fs::write(&member, b"unclaimed state").unwrap(); + assert!(matches!( + OpenedDatabaseFile::pin_initialized(&path), + Err(OpenedDatabaseFileError::IncompleteInitialization) + )); + assert_eq!(std::fs::read(&member).unwrap(), b"unclaimed state"); + std::fs::remove_file(&member).unwrap(); + let database = Connection::open(&path).unwrap(); + database + .execute_batch( + "CREATE TABLE existing_data(value INTEGER); INSERT INTO existing_data VALUES (7)", + ) + .unwrap(); + drop(database); + OpenedDatabaseFile::pin_initialized(&path).unwrap(); +} + #[cfg(windows)] #[test] fn windows_pinned_file_blocks_replacement_until_authority_closes() { diff --git a/crates/tracedecay-rusqlite-runtime/src/schema_version.rs b/crates/tracedecay-rusqlite-runtime/src/schema_version.rs index 565e4f762e3..80b66b5592f 100644 --- a/crates/tracedecay-rusqlite-runtime/src/schema_version.rs +++ b/crates/tracedecay-rusqlite-runtime/src/schema_version.rs @@ -11,6 +11,10 @@ use std::task::{Context, Poll}; use crate::exact_sql::ExactSqlValue; +/// SQLite header application ID for a store whose complete registered schema +/// was installed in private staging before publication ("TDST"). +pub const REGISTERED_STORE_APPLICATION_ID: i32 = i32::from_be_bytes(*b"TDST"); + /// Ledger table. One row per component. pub const TABLE: &str = "tracedecay_schema_version"; From b491915b040b5f6f7f88c061bd351f3cd3f319e1 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 15:09:30 -0800 Subject: [PATCH 25/60] fix(sqlite): admit first schema commits in active legacy WALs --- .../src/connection/mod.rs | 101 ++++++++++++++---- .../src/connection/tests.rs | 44 ++++++++ 2 files changed, 126 insertions(+), 19 deletions(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs index 1fe158cf397..416fad25bc4 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs @@ -111,24 +111,24 @@ impl OpenedDatabaseFile { // never opens or mutates the unverified WAL/SHM/journal sidecars. let inspection = open_immutable_reader(path).map_err(|_| OpenedDatabaseFileError::InvalidHeader)?; - let initialized: bool = inspection - .query_row( - "SELECT (SELECT application_id FROM pragma_application_id) = ?1 OR EXISTS( - SELECT 1 FROM sqlite_schema WHERE type = 'table' - AND name NOT GLOB 'sqlite_*' AND name NOT GLOB 'td_runtime_*' - )", - [crate::schema_version::REGISTERED_STORE_APPLICATION_ID], - |row| row.get(0), - ) - .map_err(|_| OpenedDatabaseFileError::InvalidHeader)?; + let initialized = + has_store_schema(&inspection).map_err(|_| OpenedDatabaseFileError::InvalidHeader)?; drop(inspection); if !initialized { - for suffix in ["-wal", "-shm", "-journal"] { - match std::fs::symlink_metadata(sidecar_path(path, suffix)) { - Ok(_) => return Err(OpenedDatabaseFileError::IncompleteInitialization), - Err(error) if error.kind() == io::ErrorKind::NotFound => {} - Err(_) => return Err(OpenedDatabaseFileError::Inspect), - } + let sidecar = |suffix| match std::fs::symlink_metadata(sidecar_path(path, suffix)) { + Ok(meta) => Ok(Some(meta)), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None), + Err(_) => Err(OpenedDatabaseFileError::Inspect), + }; + match (sidecar("-wal")?, sidecar("-shm")?, sidecar("-journal")?) { + (None, None, None) => {} + (Some(wal), Some(shm), None) + if wal.is_file() + && wal.len() > 0 + && shm.is_file() + && shm.len() > 0 + && live_wal_has_store_schema(path) => {} + _ => return Err(OpenedDatabaseFileError::IncompleteInitialization), } } Self::pin(path) @@ -923,7 +923,7 @@ pub fn open_immutable_reader(path: &Path) -> Result Result { +fn has_store_schema(connection: &Connection) -> rusqlite::Result { + connection.query_row( + "SELECT (SELECT application_id FROM pragma_application_id) = ?1 OR EXISTS( + SELECT 1 FROM sqlite_schema WHERE type = 'table' + AND name NOT GLOB 'sqlite_*' AND name NOT GLOB 'td_runtime_*' + )", + [crate::schema_version::REGISTERED_STORE_APPLICATION_ID], + |row| row.get(0), + ) +} + +/// A legacy store's first schema commit can still be in its live WAL. +/// Ask SQLite to read that commit using the existing shared index. A read-only +/// VFS can also recover a disconnected WAL into a *heap* index, which cannot +/// establish that the adjacent SHM belongs to this active database. +fn live_wal_has_store_schema(path: &Path) -> bool { + let Ok(uri) = read_only_uri(&sqlite_host_path(path), "readonly_shm=1") else { + return false; + }; + let flags = OpenFlags::SQLITE_OPEN_READ_ONLY + | OpenFlags::SQLITE_OPEN_URI + | OpenFlags::SQLITE_OPEN_PRIVATE_CACHE; + let Ok(connection) = Connection::open_with_flags(uri, flags) else { + return false; + }; + if connection + .set_db_config(DbConfig::SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, true) + .is_err() + || connection.execute_batch("BEGIN").is_err() + || !matches!(has_store_schema(&connection), Ok(true)) + { + return false; + } + use rusqlite::ffi; + let mut file: *mut ffi::sqlite3_file = std::ptr::null_mut(); + // SAFETY: the read transaction keeps the main file and mapped WAL index + // live. FILE_POINTER borrows the connection's file; xShmMap inspects page + // zero with SQLite's WAL-index page size and extension disabled. + unsafe { + if ffi::sqlite3_file_control( + connection.handle(), + c"main".as_ptr(), + ffi::SQLITE_FCNTL_FILE_POINTER, + (&raw mut file).cast(), + ) != ffi::SQLITE_OK + { + return false; + } + let Some(methods) = file.as_ref().and_then(|file| file.pMethods.as_ref()) else { + return false; + }; + if methods.iVersion < 2 { + return false; + } + let Some(map) = methods.xShmMap else { + return false; + }; + let mut region = std::ptr::null_mut(); + let rc = map(file, 0, 32 * 1024, 0, &mut region); + matches!(rc, ffi::SQLITE_OK | ffi::SQLITE_READONLY) && !region.is_null() + } +} + +fn read_only_uri(path: &Path, options: &str) -> Result { #[cfg(unix)] let raw = { use std::os::unix::ffi::OsStrExt; @@ -1003,7 +1066,7 @@ fn immutable_health_uri(path: &Path) -> Result { } } } - Ok(format!("file:{encoded}?immutable=1&mode=ro")) + Ok(format!("file:{encoded}?mode=ro&{options}")) } fn apply_pragmas( diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs index 7388d481e6b..8b852511ef7 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs @@ -500,6 +500,50 @@ fn create_new_refuses_to_replace_an_existing_database() { )); } +#[test] +fn legacy_first_schema_in_disconnected_wal_preserves_unclaimed_members() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("sessions.db"); + let wal = sidecar_path(&path, "-wal"); + let shm = sidecar_path(&path, "-shm"); + { + let writer = Connection::open(&path).unwrap(); + writer + .set_db_config(DbConfig::SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, true) + .unwrap(); + writer + .execute_batch( + "PRAGMA journal_mode=WAL; CREATE TABLE probe(value); INSERT INTO probe VALUES (7)", + ) + .unwrap(); + } + let main_bytes = std::fs::read(&path).unwrap(); + let wal_bytes = std::fs::read(&wal).unwrap(); + let shm_bytes = std::fs::read(&shm).unwrap(); + assert!(!wal_bytes.is_empty()); + assert!(matches!( + OpenedDatabaseFile::pin_initialized(&path), + Err(OpenedDatabaseFileError::IncompleteInitialization) + )); + assert_eq!(std::fs::read(&shm).unwrap(), shm_bytes); + std::fs::remove_file(&shm).unwrap(); + assert!(matches!( + OpenedDatabaseFile::pin_initialized(&path), + Err(OpenedDatabaseFileError::IncompleteInitialization) + )); + assert_eq!(std::fs::read(&path).unwrap(), main_bytes); + assert_eq!(std::fs::read(&wal).unwrap(), wal_bytes); + assert!(!shm.exists()); + // The owner can still run ordinary SQLite recovery on its own database. + let writer = Connection::open(&path).unwrap(); + assert_eq!( + writer.query_row("SELECT value FROM probe", [], |row| row.get::<_, i64>(0)), + Ok(7) + ); + drop(writer); + OpenedDatabaseFile::pin_initialized(&path).unwrap(); +} + #[cfg(all(unix, any(target_os = "linux", target_os = "android")))] #[test] fn worker_open_path_stays_on_the_pinned_file_across_an_a_b_a_swap() { From 4b399ed1df9e1a5802c4c464193c77a39845046b Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 15:09:31 -0800 Subject: [PATCH 26/60] fix(sqlite): compare resolved writer paths on Unix --- .../src/writer/worker/mod.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/writer/worker/mod.rs b/crates/tracedecay-rusqlite-runtime/src/writer/worker/mod.rs index b18573ea65f..e7f850579f3 100644 --- a/crates/tracedecay-rusqlite-runtime/src/writer/worker/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/writer/worker/mod.rs @@ -423,14 +423,16 @@ impl Worker { return self.fail_start(WriterStartError::OpenedDatabaseIdentity(error)); } #[cfg(unix)] - if self.expected_file_identity.is_some() - && connection + if self.expected_file_identity.is_some() { + let actual = connection .path() - .and_then(|path| std::fs::canonicalize(path).ok()) - .as_deref() - != Some(self.canonical_path.as_path()) - { - return self.fail_start(WriterStartError::OpenedDatabasePathMismatch); + .and_then(|path| std::fs::canonicalize(path).ok()); + // Both paths may traverse the host's temporary-directory alias. + // File identity was checked against the caller's pathname above. + let expected = std::fs::canonicalize(&self.canonical_path); + if !matches!(expected, Ok(expected) if actual.as_ref() == Some(&expected)) { + return self.fail_start(WriterStartError::OpenedDatabasePathMismatch); + } } let opened_file_identity = match self.expected_file_identity { Some(expected) => { From 2dc0f59892dab1b0421470e459f6af07aadeb2d5 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 15:09:32 -0800 Subject: [PATCH 27/60] fix(storage): retire SQLite staging under the durable directory lease --- .../shard_runtime/registry/initialization.rs | 100 ++++++++++++------ .../src/shard_runtime/registry/ports.rs | 12 ++- crates/tracedecay-runtime-core/src/storage.rs | 1 + .../src/storage/paths_and_io.rs | 16 +++ 4 files changed, 94 insertions(+), 35 deletions(-) diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs index 1a903bc34f1..2753b766aa8 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs @@ -18,12 +18,13 @@ use tracedecay_private_fs::{ use crate::sqlite_read_snapshot::{fold_staging_to_standalone, with_suffix}; const SIDECARS: [&str; 3] = ["-wal", "-shm", "-journal"]; +const OWNER: &str = "schema-installation.owner"; const SYNC: DirectorySyncPolicy = DirectorySyncPolicy::TolerateUnsupported; pub(super) struct InitializationDirectory { destination: PathBuf, directory: PathBuf, - _directory_pin: File, + directory_pin: Option, _lease: FileLease, owns_schema: bool, } @@ -35,6 +36,7 @@ impl InitializationDirectory { fn acquire(destination: PathBuf, create: bool) -> io::Result { let directory = Self::path(&destination); + let lease = crate::storage::try_lock_durable_directory(&directory)?; if create { match create_private_directory(&directory) { Ok(()) => sync_parent_directory(&directory, SYNC)?, @@ -44,33 +46,35 @@ impl InitializationDirectory { } let directory_pin = open_private_directory(&directory)?; let directory = directory.canonicalize()?; - let path = directory.join("schema-installation.lock"); - let lock = match open_private_file(&path) { - Ok(file) => file, - Err(error) if create && error.kind() == io::ErrorKind::NotFound => { + let marker = directory.join(OWNER); + match open_private_file(&marker) { + Ok(file) if file.metadata()?.len() == 0 => {} + Ok(_) => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "invalid SQLite initialization owner", + )); + } + Err(error) if error.kind() == io::ErrorKind::NotFound => { if fs::read_dir(&directory)?.next().transpose()?.is_some() { return Err(io::Error::new( io::ErrorKind::AlreadyExists, "unclaimed SQLite initialization directory is occupied", )); } - match create_private_file(&path) { - Ok(file) => file, - Err(error) if error.kind() == io::ErrorKind::AlreadyExists => { - open_private_file(&path)? - } - Err(error) => return Err(error), + if create { + create_private_file(&marker)?.sync_all()?; + sync_directory(&directory, SYNC)?; } } Err(error) => return Err(error), - }; - lock.try_lock().map_err(io::Error::from)?; + } Ok(Self { destination, directory, - _directory_pin: directory_pin, + directory_pin: Some(directory_pin), owns_schema: false, - _lease: FileLease::held(lock, "sqlite_initialization.owner"), + _lease: lease, }) } @@ -168,7 +172,7 @@ impl InitializationDirectory { self.validate_sidecars(true) } - pub(super) fn publish(&self) -> io::Result<()> { + pub(super) fn publish(mut self) -> io::Result<()> { let path = self.database(); let connection = Connection::open_with_flags( &path, @@ -201,7 +205,7 @@ impl InitializationDirectory { self.reserve_sidecars() } - fn finish_publication(&self) -> io::Result<()> { + fn finish_publication(&mut self) -> io::Result<()> { if present(&self.database())? { return Err(io::Error::new( io::ErrorKind::AlreadyExists, @@ -233,7 +237,23 @@ impl InitializationDirectory { remove_if_present(&self.witness(suffix))?; } remove_if_present(&self.transferred())?; - sync_directory(&self.directory, SYNC) + sync_directory(&self.directory, SYNC)?; + self.retire() + } + + fn retire(&mut self) -> io::Result<()> { + for entry in fs::read_dir(&self.directory)? { + if entry?.file_name() != OWNER { + return Err(io::Error::new( + io::ErrorKind::DirectoryNotEmpty, + "SQLite initialization directory still contains files", + )); + } + } + remove_if_present(&self.directory.join(OWNER))?; + drop(self.directory_pin.take()); + fs::remove_dir(&self.directory)?; + sync_parent_directory(&self.directory, SYNC) } // The normal existing/read-only route does not need to open an admission @@ -244,15 +264,18 @@ impl InitializationDirectory { if !present(&directory)? { return Ok(()); } - let _pin = open_private_directory(&directory)?; - if !Self::pending(&directory)? { - return Ok(()); - } - let stage = Self::acquire(destination.to_owned(), false)?; - if !Self::pending(&directory)? { - return Ok(()); + let mut stage = match Self::acquire(destination.to_owned(), false) { + Ok(stage) => stage, + Err(error) if error.kind() == io::ErrorKind::NotFound && !present(&directory)? => { + return Ok(()); + } + Err(error) => return Err(error), + }; + if Self::pending(&stage.directory)? { + stage.finish_publication() + } else { + stage.retire() } - stage.finish_publication() } fn pending(directory: &Path) -> io::Result { @@ -274,14 +297,31 @@ impl InitializationDirectory { impl Drop for InitializationDirectory { fn drop(&mut self) { + if self.directory_pin.is_none() { + return; + } if !self.owns_schema { + // Early refusals can leave our empty work directory. If a private + // schema or a reservation still exists, retain it for recovery. + if matches!(present(&self.directory.join(OWNER)), Ok(true)) + && matches!(Self::pending(&self.directory), Ok(false)) + { + if let Err(error) = self.retire() { + tracing::warn!(%error, "could not retire SQLite initialization directory"); + } + } return; } // A destination that appeared during this attempt is ambiguous until // a new opener verifies the prepared-main witness. match present(&self.destination) { Ok(false) => { - if let Err(error) = self.discard_schema() { + if let Err(error) = self.discard_schema().and_then(|()| { + if !Self::pending(&self.directory)? { + self.retire()?; + } + Ok(()) + }) { tracing::warn!(%error, "could not retire unpublished SQLite schema"); } } @@ -427,10 +467,7 @@ mod tests { .unwrap(), "ok" ); - assert!( - !InitializationDirectory::pending(&InitializationDirectory::path(&destination)) - .unwrap() - ); + assert!(!InitializationDirectory::path(&destination).exists()); } } @@ -449,6 +486,7 @@ mod tests { assert!(InitializationDirectory::begin(destination.clone()).is_err()); assert_eq!(fs::read(&foreign).unwrap(), b"pre-existing sidecar"); assert!(!destination.exists()); + assert!(!InitializationDirectory::path(&destination).exists()); fs::remove_file(foreign).unwrap(); } } diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs index 7f973397077..500745b8af3 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs @@ -249,10 +249,14 @@ async fn initialize_staged_runtime( key: Box::new(StoreRuntimeKey::from_binding(&request.binding)), }); } - crate::native_runtime::spawn_blocking(move || stage.publish()) - .await - .map_err(|error| physical_failure(OPERATION, error))? - .map_err(|error| physical_failure(OPERATION, error))?; + crate::native_runtime::spawn_blocking(move || { + Arc::try_unwrap(stage) + .map_err(|_| std::io::Error::other("SQLite schema workers still hold staging"))? + .publish() + }) + .await + .map_err(|error| physical_failure(OPERATION, error))? + .map_err(|error| physical_failure(OPERATION, error))?; Ok(identity) } diff --git a/crates/tracedecay-runtime-core/src/storage.rs b/crates/tracedecay-runtime-core/src/storage.rs index f44691d2f42..256667343f4 100644 --- a/crates/tracedecay-runtime-core/src/storage.rs +++ b/crates/tracedecay-runtime-core/src/storage.rs @@ -460,6 +460,7 @@ pub use layout::{ }; pub(crate) use layout::{has_path_local_profile_store, has_repository_profile_store}; pub use manifest::{read_store_manifest, write_store_manifest, write_store_manifest_to_path}; +pub(crate) use paths_and_io::try_lock_durable_directory; pub use paths_and_io::{ acquire_sidecar_lock_blocking, append_lock_path, reject_symlink_components, retry_transient_file_op, set_private_dir_permissions, try_acquire_sidecar_lock, diff --git a/crates/tracedecay-runtime-core/src/storage/paths_and_io.rs b/crates/tracedecay-runtime-core/src/storage/paths_and_io.rs index 5518d00b7ce..9e7ff208d40 100644 --- a/crates/tracedecay-runtime-core/src/storage/paths_and_io.rs +++ b/crates/tracedecay-runtime-core/src/storage/paths_and_io.rs @@ -762,6 +762,22 @@ pub(super) fn open_lock_file(lock_path: &Path, private: bool) -> io::Result io::Result { + let parent = path + .parent() + .ok_or_else(|| invalid_input("directory has no parent"))?; + let name = path + .file_name() + .ok_or_else(|| invalid_input("directory has no name"))?; + let lock_path = durable_directory_lock_path(&parent.canonicalize()?, name); + reject_symlink_components(&lock_path, "durable private store directory lock")?; + let file = open_lock_file(&lock_path, true)?; + file.try_lock().map_err(io::Error::from)?; + Ok(FileLease::held(file, SIDECAR_LEASE_LABEL)) +} + /// Non-blocking sidecar lock acquisition. Returns the held lock file on /// success, or `None` when another process/thread already holds it (the caller /// then skips its critical section). See the sidecar-lock module note above for From 06f75fb653e4f240df20de9693364d4b5da21978 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 15:15:40 -0800 Subject: [PATCH 28/60] style(sqlite): import the WAL probe dependency at module scope --- crates/tracedecay-rusqlite-runtime/src/connection/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs index 416fad25bc4..3b414422624 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs @@ -17,6 +17,7 @@ use std::{ use rusqlite::{ Connection, OpenFlags, Transaction, config::DbConfig, + ffi, hooks::{AuthAction, AuthContext, Authorization}, limits::Limit, }; @@ -1007,7 +1008,6 @@ fn live_wal_has_store_schema(path: &Path) -> bool { { return false; } - use rusqlite::ffi; let mut file: *mut ffi::sqlite3_file = std::ptr::null_mut(); // SAFETY: the read transaction keeps the main file and mapped WAL index // live. FILE_POINTER borrows the connection's file; xShmMap inspects page From 918efc110294727c26ac6b2df21f4efdc30cd03a Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:25:38 -0800 Subject: [PATCH 29/60] fix(sqlite): satisfy Clippy on initialization cleanup --- .../src/shard_runtime/registry/initialization.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs index 2753b766aa8..136a538e992 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs @@ -305,10 +305,9 @@ impl Drop for InitializationDirectory { // schema or a reservation still exists, retain it for recovery. if matches!(present(&self.directory.join(OWNER)), Ok(true)) && matches!(Self::pending(&self.directory), Ok(false)) + && let Err(error) = self.retire() { - if let Err(error) = self.retire() { - tracing::warn!(%error, "could not retire SQLite initialization directory"); - } + tracing::warn!(%error, "could not retire SQLite initialization directory"); } return; } @@ -326,7 +325,7 @@ impl Drop for InitializationDirectory { } } Err(error) => { - tracing::warn!(%error, "could not inspect initialized SQLite destination") + tracing::warn!(%error, "could not inspect initialized SQLite destination"); } Ok(true) => {} } From c70df88eebb22833896a01dea880653691210ed5 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:57:41 -0800 Subject: [PATCH 30/60] fix(sqlite): bind staging cleanup to pinned directory Refs #4513. Use unlinkat against the acquired private directory handle on Unix instead of following replaceable parent pathnames. Exercise a cross-process stage-directory swap with a foreign SQLite database and WAL/SHM/journal sentinels in a regression test. --- .../shard_runtime/registry/initialization.rs | 126 +++++++++++++++++- 1 file changed, 121 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs index 136a538e992..8986b110ee7 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs @@ -2,6 +2,8 @@ //! clients. A process death before the no-replace rename leaves only disposable //! schema in the private work directory. Hard links identify the empty sidecar //! reservations if the process dies partway through claiming the destination. +#[cfg(unix)] +use std::{ffi::CString, os::fd::AsRawFd}; use std::fs::{self, File}; use std::io; use std::path::{Path, PathBuf}; @@ -116,10 +118,52 @@ impl InitializationDirectory { self.directory.join("transferred.sqlite") } + /// Teardown is relative to the directory capability acquired with the + /// initialization lease. A noncooperating process can rename the directory + /// and replace its pathname with a symlink; unlinking through that pathname + /// would otherwise delete unrelated database files outside our staging area. + fn remove_private_member(&self, member: &str) -> io::Result<()> { + if member.is_empty() + || member == "." + || member == ".." + || member.contains('/') + || member.contains('\\') + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "invalid SQLite staging member", + )); + } + #[cfg(unix)] + { + let name = CString::new(member) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "invalid staging name"))?; + let directory = self + .directory_pin + .as_ref() + .ok_or_else(|| io::Error::other("SQLite staging directory is no longer pinned"))?; + // SAFETY: the directory descriptor remains open for this call, + // and name is a NUL-terminated, single-component filename. + let rc = unsafe { libc::unlinkat(directory.as_raw_fd(), name.as_ptr(), 0) }; + if rc == 0 { + return Ok(()); + } + let error = io::Error::last_os_error(); + if error.kind() == io::ErrorKind::NotFound { + return Ok(()); + } + Err(error) + } + #[cfg(not(unix))] + { + remove_if_present(&self.directory.join(member)) + } + } + fn discard_schema(&self) -> io::Result<()> { - remove_if_present(&self.prepared())?; + self.remove_private_member("prepared.sqlite")?; for suffix in ["-wal", "-shm", "-journal", ""] { - remove_if_present(&with_suffix(&self.database(), suffix))?; + self.remove_private_member(&format!("schema.sqlite{suffix}"))?; } Ok(()) } @@ -234,9 +278,9 @@ impl InitializationDirectory { )); } for suffix in SIDECARS { - remove_if_present(&self.witness(suffix))?; + self.remove_private_member(&format!("reservation{suffix}"))?; } - remove_if_present(&self.transferred())?; + self.remove_private_member("transferred.sqlite")?; sync_directory(&self.directory, SYNC)?; self.retire() } @@ -250,7 +294,7 @@ impl InitializationDirectory { )); } } - remove_if_present(&self.directory.join(OWNER))?; + self.remove_private_member(OWNER)?; drop(self.directory_pin.take()); fs::remove_dir(&self.directory)?; sync_parent_directory(&self.directory, SYNC) @@ -340,6 +384,7 @@ fn present(path: &Path) -> io::Result { } } +#[cfg(any(test, not(unix)))] fn remove_if_present(path: &Path) -> io::Result<()> { match fs::remove_file(path) { Ok(()) => Ok(()), @@ -381,6 +426,8 @@ fn same_regular_file(left: &Path, right: &Path) -> io::Result { #[cfg(test)] mod tests { use super::*; + #[cfg(unix)] + use std::os::unix::fs::symlink; use std::process::Command; use tempfile::TempDir; @@ -470,6 +517,75 @@ mod tests { } } + + /// A competing process may rename the private directory despite the + /// cooperative lease and install a symlink at the old path. Failed-start + /// cleanup must only remove entries from our *pinned* directory and never + /// the distinct valid SQLite database beneath that replacement symlink. + #[cfg(unix)] + #[test] + fn failed_initialization_cleanup_uses_pinned_directory_after_swap() { + const SWAP_ROOT: &str = "TRACEDECAY_SQLITE_STAGE_SWAP_ROOT"; + const COMPLETE: &str = "tracedecay-sqlite-stage-swap-complete"; + if let Some(root) = std::env::var_os(SWAP_ROOT) { + let root = PathBuf::from(root); + let stage = root.join("sessions.db.tracedecay-init"); + fs::rename(&stage, root.join("retired-stage")).unwrap(); + symlink(root.join("foreign"), &stage).unwrap(); + println!("{COMPLETE}"); + return; + } + + let root = TempDir::new().unwrap(); + let destination = root.path().join("sessions.db"); + let stage = InitializationDirectory::begin(destination).unwrap(); + seed(&stage.database(), 7); + let foreign = root.path().join("foreign"); + fs::create_dir(&foreign).unwrap(); + let foreign_database = foreign.join("schema.sqlite"); + seed(&foreign_database, 99); + let foreign_main = fs::read(&foreign_database).unwrap(); + fs::write(foreign.join("prepared.sqlite"), b"foreign prepared").unwrap(); + for suffix in ["-wal", "-shm", "-journal"] { + fs::write(with_suffix(&stage.database(), suffix), b"owned staging sidecar").unwrap(); + fs::write(with_suffix(&foreign_database, suffix), format!("foreign {suffix}")).unwrap(); + } + + let (_, module) = module_path!().split_once("::").unwrap(); + let output = Command::new(std::env::current_exe().unwrap()) + .arg("--exact") + .arg(format!( + "{module}::failed_initialization_cleanup_uses_pinned_directory_after_swap" + )) + .arg("--nocapture") + .env(SWAP_ROOT, root.path()) + .output() + .unwrap(); + assert!( + output.status.success() + && String::from_utf8_lossy(&output.stdout).contains(COMPLETE), + "cross-process stage swap did not execute: {output:?}" + ); + + // Dropping a failed schema initialization is the production cleanup + // caller. Its directory descriptor still points to retired-stage. + drop(stage); + let retired = root.path().join("retired-stage"); + assert!(!retired.join("schema.sqlite").exists()); + for suffix in ["-wal", "-shm", "-journal"] { + assert!(!with_suffix(&retired.join("schema.sqlite"), suffix).exists()); + assert_eq!( + fs::read(with_suffix(&foreign_database, suffix)).unwrap(), + format!("foreign {suffix}").as_bytes() + ); + } + assert_eq!(fs::read(&foreign_database).unwrap(), foreign_main); + assert_eq!( + fs::read(foreign.join("prepared.sqlite")).unwrap(), + b"foreign prepared" + ); + } + #[test] fn competing_initialization_leaves_owner_and_foreign_sidecars_intact() { let root = TempDir::new().unwrap(); From b9621a7a52ccf210aa294b339e0a84609358a55e Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 17:08:22 -0800 Subject: [PATCH 31/60] fix(storage): verify the staging directory before retirement --- .../shard_runtime/registry/initialization.rs | 90 +++++++++++++++---- 1 file changed, 75 insertions(+), 15 deletions(-) diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs index 8986b110ee7..0902ab1b2fc 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs @@ -2,11 +2,13 @@ //! clients. A process death before the no-replace rename leaves only disposable //! schema in the private work directory. Hard links identify the empty sidecar //! reservations if the process dies partway through claiming the destination. -#[cfg(unix)] -use std::{ffi::CString, os::fd::AsRawFd}; use std::fs::{self, File}; use std::io; +#[cfg(unix)] +use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; +#[cfg(unix)] +use std::{ffi::CString, os::fd::AsRawFd}; use rusqlite::{Connection, OpenFlags}; use tracedecay_private_fs::framed_log::{ @@ -285,7 +287,29 @@ impl InitializationDirectory { self.retire() } + fn verify_directory_name(&self) -> io::Result<()> { + let pinned = self + .directory_pin + .as_ref() + .ok_or_else(|| io::Error::other("SQLite staging directory is no longer pinned"))?; + let named = open_private_directory(&self.directory)?; + #[cfg(unix)] + let matches = same_unix_inode(&pinned.metadata()?, &named.metadata()?); + #[cfg(windows)] + let matches = same_windows_object(pinned, &named)?; + #[cfg(not(any(unix, windows)))] + let matches = false; + if !matches { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "SQLite staging directory no longer names the pinned directory", + )); + } + Ok(()) + } + fn retire(&mut self) -> io::Result<()> { + self.verify_directory_name()?; for entry in fs::read_dir(&self.directory)? { if entry?.file_name() != OWNER { return Err(io::Error::new( @@ -295,6 +319,9 @@ impl InitializationDirectory { } } self.remove_private_member(OWNER)?; + // rmdir is name-based. Refuse an already changed name as late as + // possible, while holding the lease and the original directory pin. + self.verify_directory_name()?; drop(self.directory_pin.take()); fs::remove_dir(&self.directory)?; sync_parent_directory(&self.directory, SYNC) @@ -398,23 +425,32 @@ fn empty_regular(path: &Path) -> io::Result { Ok(meta.is_file() && meta.len() == 0) } +#[cfg(unix)] +fn same_unix_inode(left: &fs::Metadata, right: &fs::Metadata) -> bool { + left.dev() == right.dev() && left.ino() == right.ino() +} + +#[cfg(windows)] +fn same_windows_object(left: &File, right: &File) -> io::Result { + let a = tracedecay_private_fs::windows_file::information(left)?; + let b = tracedecay_private_fs::windows_file::information(right)?; + Ok(a.volume_serial_number == b.volume_serial_number && a.file_index == b.file_index) +} + fn same_regular_file(left: &Path, right: &Path) -> io::Result { if !fs::symlink_metadata(left)?.is_file() || !fs::symlink_metadata(right)?.is_file() { return Ok(false); } #[cfg(unix)] { - use std::os::unix::fs::MetadataExt; - let a = fs::symlink_metadata(left)?; - let b = fs::symlink_metadata(right)?; - Ok(a.dev() == b.dev() && a.ino() == b.ino()) + Ok(same_unix_inode( + &fs::symlink_metadata(left)?, + &fs::symlink_metadata(right)?, + )) } #[cfg(windows)] { - use tracedecay_private_fs::windows_file::information; - let a = information(&File::open(left)?)?; - let b = information(&File::open(right)?)?; - Ok(a.volume_serial_number == b.volume_serial_number && a.file_index == b.file_index) + same_windows_object(&File::open(left)?, &File::open(right)?) } #[cfg(not(any(unix, windows)))] Err(io::Error::new( @@ -517,7 +553,6 @@ mod tests { } } - /// A competing process may rename the private directory despite the /// cooperative lease and install a symlink at the old path. Failed-start /// cleanup must only remove entries from our *pinned* directory and never @@ -547,8 +582,16 @@ mod tests { let foreign_main = fs::read(&foreign_database).unwrap(); fs::write(foreign.join("prepared.sqlite"), b"foreign prepared").unwrap(); for suffix in ["-wal", "-shm", "-journal"] { - fs::write(with_suffix(&stage.database(), suffix), b"owned staging sidecar").unwrap(); - fs::write(with_suffix(&foreign_database, suffix), format!("foreign {suffix}")).unwrap(); + fs::write( + with_suffix(&stage.database(), suffix), + b"owned staging sidecar", + ) + .unwrap(); + fs::write( + with_suffix(&foreign_database, suffix), + format!("foreign {suffix}"), + ) + .unwrap(); } let (_, module) = module_path!().split_once("::").unwrap(); @@ -562,8 +605,7 @@ mod tests { .output() .unwrap(); assert!( - output.status.success() - && String::from_utf8_lossy(&output.stdout).contains(COMPLETE), + output.status.success() && String::from_utf8_lossy(&output.stdout).contains(COMPLETE), "cross-process stage swap did not execute: {output:?}" ); @@ -586,6 +628,24 @@ mod tests { ); } + #[test] + fn retiring_a_directory_requires_its_own_pin() { + let root = TempDir::new().unwrap(); + let destination = root.path().join("sessions.db"); + let mut stage = InitializationDirectory::begin(destination).unwrap(); + let other = root.path().join("another-empty-directory"); + create_private_directory(&other).unwrap(); + let original = std::mem::replace(&mut stage.directory, other.clone()); + let error = stage.retire().unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(other.is_dir()); + assert!(original.join(OWNER).is_file()); + stage.directory = original.clone(); + drop(stage); + assert!(!original.exists()); + assert!(other.is_dir()); + } + #[test] fn competing_initialization_leaves_owner_and_foreign_sidecars_intact() { let root = TempDir::new().unwrap(); From 66560cf740955c22dcc067b71369963ea9093af0 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 17:48:48 -0800 Subject: [PATCH 32/60] test(sqlite): inspect live legacy WALs across processes --- .../src/connection/tests.rs | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs index 8b852511ef7..05dcaabac38 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs @@ -465,6 +465,16 @@ fn existing_sqlite_admission_requires_a_completed_header() { #[test] fn initialized_wal_with_uncheckpointed_schema_remains_attachable() { + const EXTERNAL_READER: &str = "TRACEDECAY_TEST_INSPECT_LIVE_FIRST_SCHEMA"; + if let Some(path) = std::env::var_os(EXTERNAL_READER) { + let path = PathBuf::from(path); + let shm = sidecar_path(&path, "-shm"); + let before = std::fs::read(&shm).unwrap(); + let pinned = OpenedDatabaseFile::pin_initialized(&path).unwrap(); + assert_eq!(std::fs::read(&shm).unwrap(), before); + pinned.release_after_workers_exit(); + return; + } let directory = tempfile::tempdir().unwrap(); let path = directory.path().join("sessions.db"); let writer = Connection::open(&path).unwrap(); @@ -478,6 +488,21 @@ fn initialized_wal_with_uncheckpointed_schema_remains_attachable() { let wal = sidecar_path(&path, "-wal"); let before = std::fs::read(&wal).unwrap(); assert!(!before.is_empty()); + // A separate process has no connection sharing the writer's VFS node. + // It must discover the live WAL index through SQLite without rewriting it. + let name = format!( + "{}::initialized_wal_with_uncheckpointed_schema_remains_attachable", + module_path!().split_once("::").unwrap().1 + ); + let external = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", &name, "--nocapture"]) + .env(EXTERNAL_READER, &path) + .output() + .unwrap(); + assert!( + external.status.success() && String::from_utf8_lossy(&external.stdout).contains("1 passed"), + "external WAL inspection failed: {external:?}" + ); let pinned = OpenedDatabaseFile::pin_initialized(&path).unwrap(); assert_ne!(pinned.identity(), 0); assert_eq!(std::fs::read(&wal).unwrap(), before); From 8f9b5db62bcef57ca3bf8e8c72ee2c541c6440c1 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 17:51:06 -0800 Subject: [PATCH 33/60] fix(sqlite): inspect completed handoffs without a writable lease --- .../shard_runtime/registry/initialization.rs | 25 ++++++++++++++++--- .../src/shard_runtime/registry/ports.rs | 12 +++++++-- .../registry/tests/production_routes.rs | 14 ++++++++++- 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs index 0902ab1b2fc..e00ff1b6110 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs @@ -327,9 +327,28 @@ impl InitializationDirectory { sync_parent_directory(&self.directory, SYNC) } - // The normal existing/read-only route does not need to open an admission - // lease or write to the filesystem. Only an interrupted publication with - // outstanding private files requires reconciliation. + /// A completed handoff can still have its empty directory and owner marker + /// after a process exit. Read-only callers need neither cleanup nor a lease. + /// Any other staged member requires writable reconciliation before attach. + pub(super) fn verify_read_only_handoff(destination: &Path) -> io::Result<()> { + let directory = Self::path(destination); + let _pin = match open_private_directory(&directory) { + Ok(pin) => pin, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error), + }; + for entry in fs::read_dir(&directory)? { + let member = entry?; + if member.file_name() != OWNER || !empty_regular(&member.path())? { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "SQLite staging requires writable reconciliation before read-only attachment", + )); + } + } + Ok(()) + } + pub(super) fn recover_published(destination: &Path) -> io::Result<()> { let directory = Self::path(destination); if !present(&directory)? { diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs index 500745b8af3..0bcac1b9e67 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs @@ -411,8 +411,16 @@ impl LifecycleShardRuntimeAttachment { admission: AdmissionConfigV1, ) -> Result { if staging.is_none() { - InitializationDirectory::recover_published(request.locator.path()) - .map_err(|error| physical_failure("recover initialized SQLite family", error))?; + let path = request.locator.path(); + match request.access { + StoreRuntimeAccessMode::ReadOnly => { + InitializationDirectory::verify_read_only_handoff(path) + } + StoreRuntimeAccessMode::ReadWrite => { + InitializationDirectory::recover_published(path) + } + } + .map_err(|error| physical_failure("verify initialized SQLite family", error))?; } let attachment = if let Some(stage) = &staging { self.repository.initialize( diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs index 7a57e162ec2..11197e726db 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs @@ -293,7 +293,12 @@ async fn read_only_open_mounts_readers_without_acquiring_a_writer() { let root = TempDir::new().unwrap(); let resolver = Arc::new(FileResolver::default()); resolver.push(seed_final_graph_db(&root, "profile.db").await); - resolver.push(seed_final_graph_db(&root, "project.db").await); + let database = seed_final_graph_db(&root, "project.db").await; + let leftover = database.with_file_name("project.db.tracedecay-init"); + tracedecay_private_fs::create_private_directory(&leftover).unwrap(); + let owner = leftover.join("schema-installation.owner"); + drop(tracedecay_private_fs::create_private_file(&owner).unwrap()); + resolver.push(database); let registry = StoreRuntimeRegistry::new( asupersync::runtime::Runtime::current_handle().unwrap(), resolver, @@ -327,6 +332,13 @@ async fn read_only_open_mounts_readers_without_acquiring_a_writer() { assert!(!physical.writer_present); assert!(physical.reader_handles > 0); assert_health_route(&project, false).await; + assert!(owner.is_file()); + assert!( + !root + .path() + .join(".project.db.tracedecay-init.durable-directory.lock") + .exists() + ); } #[cfg(unix)] From 2ff238544bb289f61ba6f6c3b27a8d775dfac454 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 17:51:09 -0800 Subject: [PATCH 34/60] fix(storage): preserve initialization leases during scoped reset --- .../src/commands/stale_store_reset.rs | 92 +++++++++++++++---- .../src/shard_runtime/registry.rs | 1 + .../shard_runtime/registry/initialization.rs | 42 +++++++++ crates/tracedecay-runtime-core/src/storage.rs | 2 +- .../src/storage/paths_and_io.rs | 25 ++++- 5 files changed, 139 insertions(+), 23 deletions(-) diff --git a/crates/tracedecay-cli/src/commands/stale_store_reset.rs b/crates/tracedecay-cli/src/commands/stale_store_reset.rs index 2d9287d1c31..69f43f36281 100644 --- a/crates/tracedecay-cli/src/commands/stale_store_reset.rs +++ b/crates/tracedecay-cli/src/commands/stale_store_reset.rs @@ -5,6 +5,7 @@ use tracedecay_domain::errors::{ResettableStoreV1, Result, StoreResetRequiredV1, use tracedecay_global_db::profile_registry_maintenance::verify_store_path_absent; use tracedecay_hooks::{PROFILE_HOOK_ADMISSIONS_DIR, PROJECT_HOOK_ADMISSIONS_DIR}; use tracedecay_runtime_core::config::{GLOBAL_DB_FILENAME, ProfileRoot}; +use tracedecay_runtime_core::shard_runtime::registry::lock_offline_sqlite_family_for_reset; use tracedecay_runtime_core::storage::{ SESSIONS_DB_FILENAME, STORE_MANIFEST_FILENAME, USER_SESSIONS_DB_FILENAME, profile_sharded_data_root, read_store_manifest, validate_project_id, @@ -189,7 +190,7 @@ fn print_project_registrations(profile_root: &Path) -> Result<()> { /// What a reset deletes inside a store's directory. enum StoreMembers { - /// A database and every entry named after it. + /// The SQLite family and its registered store artifacts. DatabaseFamily(&'static str), /// These exact directories. Directories(&'static [&'static str]), @@ -229,28 +230,66 @@ fn store_location( }) } -/// Removes `database` and every entry named after it in `directory`: its -/// SQLite sidecars and spools (`.db…`), its host-admission directory -/// (`..db…`), and its session relation graph (`.grafeo…`). +/// The offline lifecycle authority admits deletion of the stale store +/// itself. Similar names, private initialization data and stable coordination +/// locks are not members of its data family. fn remove_store_family(directory: &Path, database: &str) -> Result { - let stem = database.strip_suffix(".db").unwrap_or(database); - let owned_prefix = format!("{stem}."); - let hidden_prefix = format!(".{database}."); - let entries = std::fs::read_dir(directory).map_err(|error| TraceDecayError::Config { - message: format!("failed to list '{}': {error}", directory.display()), - })?; - let mut members = Vec::new(); - for entry in entries { - let entry = entry.map_err(|error| TraceDecayError::Config { - message: format!("failed to list '{}': {error}", directory.display()), + let _initialization = + lock_offline_sqlite_family_for_reset(&directory.join(database)).map_err(|error| { + TraceDecayError::Config { + message: format!("cannot reset SQLite family {database}: {error}"), + } + })?; + let mut members: Vec<(String, bool)> = ["", "-wal", "-shm", "-journal"] + .iter() + .map(|suffix| (format!("{database}{suffix}"), false)) + .collect(); + if database == SESSIONS_DB_FILENAME || database == USER_SESSIONS_DB_FILENAME { + let stem = database.strip_suffix(".db").unwrap_or(database); + let graph = format!("{stem}.grafeo"); + members.extend([ + (format!("{graph}.wal"), true), + (graph, false), + (format!(".{database}.host-admission"), true), + (format!("{database}.delivery-settlement-spool-v1"), true), + ]); + } + // Validate all exact names before deleting the first file. The exclusive + // offline lease then excludes other cooperating profile writers. + for (name, is_directory) in &members { + let path = directory.join(name); + tracedecay_runtime_core::storage::reject_symlink_components( + &path, + "stale store reset member", + ) + .map_err(|error| TraceDecayError::Config { + message: error.to_string(), })?; - let name = entry.file_name().to_string_lossy().into_owned(); - if name.starts_with(&owned_prefix) || name.starts_with(&hidden_prefix) { - members.push(name); + match std::fs::symlink_metadata(&path) { + Ok(metadata) + if (*is_directory && metadata.is_dir()) + || (!is_directory && metadata.is_file()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Ok(_) => { + return Err(TraceDecayError::Config { + message: format!( + "unexpected file type for stale store reset member '{}'", + path.display() + ), + }); + } + Err(error) => { + return Err(TraceDecayError::Config { + message: format!( + "cannot inspect stale store reset member '{}': {error}", + path.display() + ), + }); + } } } let mut removed = 0; - for name in &members { + for (name, _) in &members { removed += usize::from(remove_fixed_profile_path(directory, name)?); verify_store_path_absent(&directory.join(name))?; } @@ -279,6 +318,8 @@ mod tests { "global.db-wal", "global.db-shm", "profile-identity.json", + "sessions.unrelated", + ".sessions.db.notes", ] { std::fs::write(root.join(file), file).expect("write store file"); } @@ -302,8 +343,10 @@ mod tests { assert_eq!( kept, [ + ".sessions.db.notes", "code-index-v1", "profile-identity.json", + "sessions.unrelated", "store_manifest.json", "tracedecay.db", "tracedecay.db-wal", @@ -313,6 +356,19 @@ mod tests { ); } + #[test] + fn unexpected_sqlite_member_kind_preserves_the_whole_family() { + let directory = tempfile::TempDir::new().unwrap(); + let root = directory.path(); + std::fs::write(root.join("sessions.db"), b"main").unwrap(); + let foreign = root.join("sessions.db-wal"); + std::fs::create_dir(&foreign).unwrap(); + std::fs::write(foreign.join("entry"), b"keep").unwrap(); + assert!(remove_store_family(root, "sessions.db").is_err()); + assert_eq!(std::fs::read(root.join("sessions.db")).unwrap(), b"main"); + assert_eq!(std::fs::read(foreign.join("entry")).unwrap(), b"keep"); + } + #[test] fn a_store_that_is_not_resettable_on_its_own_refuses_the_whole_reset() { let store = |label: &str, remedy: &str| StoreResetRequiredV1 { diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry.rs index 29d7db89a91..76015075313 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry.rs @@ -15,6 +15,7 @@ mod close; mod destructive; mod graph; mod initialization; +pub use initialization::lock_offline_sqlite_family_for_reset; mod leases; mod open; mod ports; diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs index e00ff1b6110..2d744ea80aa 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs @@ -25,6 +25,21 @@ const SIDECARS: [&str; 3] = ["-wal", "-shm", "-journal"]; const OWNER: &str = "schema-installation.owner"; const SYNC: DirectorySyncPolicy = DirectorySyncPolicy::TolerateUnsupported; +/// Coordinates an authorized reset of an offline store with the existing +/// initializer. A staged or ambiguous family must finish reconciliation +/// before reset can discard any canonical members. +pub fn lock_offline_sqlite_family_for_reset(destination: &Path) -> io::Result> { + let staging = InitializationDirectory::path(destination); + let lease = crate::storage::try_lock_existing_durable_directory(&staging)?; + if present(&staging)? { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "SQLite initialization requires reconciliation before reset", + )); + } + Ok(lease) +} + pub(super) struct InitializationDirectory { destination: PathBuf, directory: PathBuf, @@ -665,6 +680,33 @@ mod tests { assert!(other.is_dir()); } + #[test] + fn offline_reset_preserves_the_initialization_lease_and_refuses_unresolved_staging() { + let root = TempDir::new().unwrap(); + let destination = root.path().join("sessions.db"); + assert!( + lock_offline_sqlite_family_for_reset(&destination) + .unwrap() + .is_none() + ); + assert_eq!(fs::read_dir(root.path()).unwrap().count(), 0); + let publisher = InitializationDirectory::begin(destination.clone()).unwrap(); + assert!(lock_offline_sqlite_family_for_reset(&destination).is_err()); + drop(publisher); + let reset = lock_offline_sqlite_family_for_reset(&destination) + .unwrap() + .unwrap(); + assert!(InitializationDirectory::begin(destination.clone()).is_err()); + drop(reset); + let staging = InitializationDirectory::path(&destination); + create_private_directory(&staging).unwrap(); + let unrelated = staging.join("operator-data"); + fs::write(&unrelated, b"keep").unwrap(); + assert!(InitializationDirectory::verify_read_only_handoff(&destination).is_err()); + assert!(lock_offline_sqlite_family_for_reset(&destination).is_err()); + assert_eq!(fs::read(unrelated).unwrap(), b"keep"); + } + #[test] fn competing_initialization_leaves_owner_and_foreign_sidecars_intact() { let root = TempDir::new().unwrap(); diff --git a/crates/tracedecay-runtime-core/src/storage.rs b/crates/tracedecay-runtime-core/src/storage.rs index 256667343f4..6f9aafbb583 100644 --- a/crates/tracedecay-runtime-core/src/storage.rs +++ b/crates/tracedecay-runtime-core/src/storage.rs @@ -460,12 +460,12 @@ pub use layout::{ }; pub(crate) use layout::{has_path_local_profile_store, has_repository_profile_store}; pub use manifest::{read_store_manifest, write_store_manifest, write_store_manifest_to_path}; -pub(crate) use paths_and_io::try_lock_durable_directory; pub use paths_and_io::{ acquire_sidecar_lock_blocking, append_lock_path, reject_symlink_components, retry_transient_file_op, set_private_dir_permissions, try_acquire_sidecar_lock, validate_project_id, }; +pub(crate) use paths_and_io::{try_lock_durable_directory, try_lock_existing_durable_directory}; pub use profile_identity::{ PROFILE_IDENTITY_RECORD_NAME, PROFILE_IDENTITY_SCHEMA_VERSION, ProfileIdentityRecordV1, read_existing_profile_identity_record, diff --git a/crates/tracedecay-runtime-core/src/storage/paths_and_io.rs b/crates/tracedecay-runtime-core/src/storage/paths_and_io.rs index 9e7ff208d40..2a3fdc2d403 100644 --- a/crates/tracedecay-runtime-core/src/storage/paths_and_io.rs +++ b/crates/tracedecay-runtime-core/src/storage/paths_and_io.rs @@ -762,9 +762,7 @@ pub(super) fn open_lock_file(lock_path: &Path, private: bool) -> io::Result io::Result { +fn private_directory_lock_path(path: &Path) -> io::Result { let parent = path .parent() .ok_or_else(|| invalid_input("directory has no parent"))?; @@ -773,11 +771,30 @@ pub(crate) fn try_lock_durable_directory(path: &Path) -> io::Result { .ok_or_else(|| invalid_input("directory has no name"))?; let lock_path = durable_directory_lock_path(&parent.canonicalize()?, name); reject_symlink_components(&lock_path, "durable private store directory lock")?; - let file = open_lock_file(&lock_path, true)?; + Ok(lock_path) +} + +fn try_lock_directory_file(file: fs::File) -> io::Result { file.try_lock().map_err(io::Error::from)?; Ok(FileLease::held(file, SIDECAR_LEASE_LABEL)) } +/// Keep the coordination inode outside a disposable private directory so +/// removal never splits contenders across two generations of the lock. +pub(crate) fn try_lock_durable_directory(path: &Path) -> io::Result { + try_lock_directory_file(open_lock_file(&private_directory_lock_path(path)?, true)?) +} + +/// Offline reset must preserve an existing lease inode and must not create +/// a new one in a legacy store's namespace. +pub(crate) fn try_lock_existing_durable_directory(path: &Path) -> io::Result> { + match tracedecay_private_fs::open_private_file(&private_directory_lock_path(path)?) { + Ok(file) => try_lock_directory_file(file).map(Some), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error), + } +} + /// Non-blocking sidecar lock acquisition. Returns the held lock file on /// success, or `None` when another process/thread already holds it (the caller /// then skips its critical section). See the sidecar-lock module note above for From 55cc61c5b3f238a759fecde2ef7a60b8a68d0161 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 18:30:48 -0800 Subject: [PATCH 35/60] fix(sqlite): retire exact graph verification marker on reset Refs #4513. Remove the session graph's .verified marker only for the corresponding store family under the offline initialization lease. Refuse unexpected marker types before deleting any family member; cover project/profile scoping and preservation of unrelated markers. --- .../src/commands/stale_store_reset.rs | 48 ++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-cli/src/commands/stale_store_reset.rs b/crates/tracedecay-cli/src/commands/stale_store_reset.rs index 69f43f36281..51b9b57576f 100644 --- a/crates/tracedecay-cli/src/commands/stale_store_reset.rs +++ b/crates/tracedecay-cli/src/commands/stale_store_reset.rs @@ -248,6 +248,9 @@ fn remove_store_family(directory: &Path, database: &str) -> Result { let stem = database.strip_suffix(".db").unwrap_or(database); let graph = format!("{stem}.grafeo"); members.extend([ + // A graph container owns its verification marker beside it: + // sessions.grafeo -> sessions.verified (also user-sessions). + (format!("{stem}.verified"), false), (format!("{graph}.wal"), true), (graph, false), (format!(".{database}.host-admission"), true), @@ -309,6 +312,9 @@ mod tests { "sessions.db-wal", "sessions.db-shm", "sessions.grafeo", + "sessions.verified", + "user-sessions.verified", + "tracedecay.verified", "tracedecay.db", "tracedecay.db-wal", "tracedecay.grafeo", @@ -332,7 +338,7 @@ mod tests { std::fs::write(root.join(dir).join("entry"), dir).expect("write store entry"); } - assert_eq!(remove_store_family(root, "sessions.db").unwrap(), 6); + assert_eq!(remove_store_family(root, "sessions.db").unwrap(), 7); assert_eq!(remove_store_family(root, GLOBAL_DB_FILENAME).unwrap(), 3); let mut kept: Vec = std::fs::read_dir(root) @@ -351,11 +357,51 @@ mod tests { "tracedecay.db", "tracedecay.db-wal", "tracedecay.grafeo", + "tracedecay.verified", "user-sessions.db", + "user-sessions.verified", ] ); } + + #[test] + fn profile_sessions_reset_removes_only_its_graph_verification_marker() { + let directory = tempfile::TempDir::new().unwrap(); + let root = directory.path(); + for name in [ + "sessions.db", + "sessions.verified", + "user-sessions.db", + "user-sessions.verified", + ] { + std::fs::write(root.join(name), name).unwrap(); + } + + assert_eq!(remove_store_family(root, USER_SESSIONS_DB_FILENAME).unwrap(), 2); + assert!(!root.join("user-sessions.db").exists()); + assert!(!root.join("user-sessions.verified").exists()); + assert_eq!(std::fs::read(root.join("sessions.db")).unwrap(), b"sessions.db"); + assert_eq!( + std::fs::read(root.join("sessions.verified")).unwrap(), + b"sessions.verified" + ); + } + + #[test] + fn unexpected_graph_verification_marker_kind_preserves_the_store() { + let directory = tempfile::TempDir::new().unwrap(); + let root = directory.path(); + std::fs::write(root.join("sessions.db"), b"store").unwrap(); + let marker = root.join("sessions.verified"); + std::fs::create_dir(&marker).unwrap(); + std::fs::write(marker.join("foreign"), b"preserve").unwrap(); + + assert!(remove_store_family(root, SESSIONS_DB_FILENAME).is_err()); + assert_eq!(std::fs::read(root.join("sessions.db")).unwrap(), b"store"); + assert_eq!(std::fs::read(marker.join("foreign")).unwrap(), b"preserve"); + } + #[test] fn unexpected_sqlite_member_kind_preserves_the_whole_family() { let directory = tempfile::TempDir::new().unwrap(); From 945dde873a3587be6764dae3b283dfe352b77765 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 18:32:18 -0800 Subject: [PATCH 36/60] fix(storage): retire complete graph data during offline wipes --- .../src/commands/stale_store_reset.rs | 92 ++-------- crates/tracedecay-cli/src/commands/storage.rs | 159 +++++++++++++----- .../cli_non_interactive_test.rs | 68 ++++++++ .../tracedecay-graph-db/src/corrupt_store.rs | 11 +- crates/tracedecay-graph-db/src/lib.rs | 1 + crates/tracedecay-graph-db/src/location.rs | 22 +++ .../tracedecay-graph-db/src/sealed_store.rs | 2 +- crates/tracedecay-runtime-core/src/storage.rs | 1 + 8 files changed, 229 insertions(+), 127 deletions(-) diff --git a/crates/tracedecay-cli/src/commands/stale_store_reset.rs b/crates/tracedecay-cli/src/commands/stale_store_reset.rs index 51b9b57576f..2b45f55d5ae 100644 --- a/crates/tracedecay-cli/src/commands/stale_store_reset.rs +++ b/crates/tracedecay-cli/src/commands/stale_store_reset.rs @@ -5,7 +5,6 @@ use tracedecay_domain::errors::{ResettableStoreV1, Result, StoreResetRequiredV1, use tracedecay_global_db::profile_registry_maintenance::verify_store_path_absent; use tracedecay_hooks::{PROFILE_HOOK_ADMISSIONS_DIR, PROJECT_HOOK_ADMISSIONS_DIR}; use tracedecay_runtime_core::config::{GLOBAL_DB_FILENAME, ProfileRoot}; -use tracedecay_runtime_core::shard_runtime::registry::lock_offline_sqlite_family_for_reset; use tracedecay_runtime_core::storage::{ SESSIONS_DB_FILENAME, STORE_MANIFEST_FILENAME, USER_SESSIONS_DB_FILENAME, profile_sharded_data_root, read_store_manifest, validate_project_id, @@ -13,7 +12,7 @@ use tracedecay_runtime_core::storage::{ use super::storage::{ ProfileOfflineAuthority, RestoreWording, join_outcome_and_restore, remove_fixed_profile_path, - take_profile_offline, + remove_store_family, take_profile_offline, }; const OPERATION: &str = "wipe --stale"; @@ -230,75 +229,6 @@ fn store_location( }) } -/// The offline lifecycle authority admits deletion of the stale store -/// itself. Similar names, private initialization data and stable coordination -/// locks are not members of its data family. -fn remove_store_family(directory: &Path, database: &str) -> Result { - let _initialization = - lock_offline_sqlite_family_for_reset(&directory.join(database)).map_err(|error| { - TraceDecayError::Config { - message: format!("cannot reset SQLite family {database}: {error}"), - } - })?; - let mut members: Vec<(String, bool)> = ["", "-wal", "-shm", "-journal"] - .iter() - .map(|suffix| (format!("{database}{suffix}"), false)) - .collect(); - if database == SESSIONS_DB_FILENAME || database == USER_SESSIONS_DB_FILENAME { - let stem = database.strip_suffix(".db").unwrap_or(database); - let graph = format!("{stem}.grafeo"); - members.extend([ - // A graph container owns its verification marker beside it: - // sessions.grafeo -> sessions.verified (also user-sessions). - (format!("{stem}.verified"), false), - (format!("{graph}.wal"), true), - (graph, false), - (format!(".{database}.host-admission"), true), - (format!("{database}.delivery-settlement-spool-v1"), true), - ]); - } - // Validate all exact names before deleting the first file. The exclusive - // offline lease then excludes other cooperating profile writers. - for (name, is_directory) in &members { - let path = directory.join(name); - tracedecay_runtime_core::storage::reject_symlink_components( - &path, - "stale store reset member", - ) - .map_err(|error| TraceDecayError::Config { - message: error.to_string(), - })?; - match std::fs::symlink_metadata(&path) { - Ok(metadata) - if (*is_directory && metadata.is_dir()) - || (!is_directory && metadata.is_file()) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Ok(_) => { - return Err(TraceDecayError::Config { - message: format!( - "unexpected file type for stale store reset member '{}'", - path.display() - ), - }); - } - Err(error) => { - return Err(TraceDecayError::Config { - message: format!( - "cannot inspect stale store reset member '{}': {error}", - path.display() - ), - }); - } - } - } - let mut removed = 0; - for (name, _) in &members { - removed += usize::from(remove_fixed_profile_path(directory, name)?); - verify_store_path_absent(&directory.join(name))?; - } - Ok(removed) -} - #[cfg(test)] mod tests { use super::*; @@ -315,6 +245,7 @@ mod tests { "sessions.verified", "user-sessions.verified", "tracedecay.verified", + "sessions.grafeo.corruption-lock", "tracedecay.db", "tracedecay.db-wal", "tracedecay.grafeo", @@ -332,13 +263,17 @@ mod tests { for dir in [ ".sessions.db.host-admission", "sessions.db.delivery-settlement-spool-v1", + "sessions.grafeo.wal", + "sessions.grafeo.spill", + "sessions.spill", + "sessions.sealed", "code-index-v1", ] { std::fs::create_dir(root.join(dir)).expect("create store directory"); std::fs::write(root.join(dir).join("entry"), dir).expect("write store entry"); } - assert_eq!(remove_store_family(root, "sessions.db").unwrap(), 7); + assert_eq!(remove_store_family(root, "sessions.db").unwrap(), 10); assert_eq!(remove_store_family(root, GLOBAL_DB_FILENAME).unwrap(), 3); let mut kept: Vec = std::fs::read_dir(root) @@ -352,6 +287,8 @@ mod tests { ".sessions.db.notes", "code-index-v1", "profile-identity.json", + "sessions.grafeo.corruption-lock", + "sessions.spill", "sessions.unrelated", "store_manifest.json", "tracedecay.db", @@ -364,7 +301,6 @@ mod tests { ); } - #[test] fn profile_sessions_reset_removes_only_its_graph_verification_marker() { let directory = tempfile::TempDir::new().unwrap(); @@ -378,10 +314,16 @@ mod tests { std::fs::write(root.join(name), name).unwrap(); } - assert_eq!(remove_store_family(root, USER_SESSIONS_DB_FILENAME).unwrap(), 2); + assert_eq!( + remove_store_family(root, USER_SESSIONS_DB_FILENAME).unwrap(), + 2 + ); assert!(!root.join("user-sessions.db").exists()); assert!(!root.join("user-sessions.verified").exists()); - assert_eq!(std::fs::read(root.join("sessions.db")).unwrap(), b"sessions.db"); + assert_eq!( + std::fs::read(root.join("sessions.db")).unwrap(), + b"sessions.db" + ); assert_eq!( std::fs::read(root.join("sessions.verified")).unwrap(), b"sessions.verified" diff --git a/crates/tracedecay-cli/src/commands/storage.rs b/crates/tracedecay-cli/src/commands/storage.rs index 42262122b4c..30729f891fc 100644 --- a/crates/tracedecay-cli/src/commands/storage.rs +++ b/crates/tracedecay-cli/src/commands/storage.rs @@ -7,7 +7,11 @@ use tracedecay_maintenance::retention::footprint::{ MAX_FOOTPRINT_ENTRIES, WalkTerminationV1, WalkedEntryV1, walk_regular_files_with_budget, }; use tracedecay_runtime_core::config::{GLOBAL_DB_FILENAME, ProfileRoot}; -use tracedecay_runtime_core::storage::{USER_MEMORY_DB_FILENAME, USER_SESSIONS_DB_FILENAME}; +use tracedecay_runtime_core::shard_runtime::registry::lock_offline_sqlite_family_for_reset; +use tracedecay_runtime_core::storage::{ + SESSIONS_DB_FILENAME, USER_MEMORY_DB_FILENAME, USER_SESSIONS_DB_FILENAME, + graph_store_data_paths, +}; use tracedecay_global_db::profile_registry_maintenance::{ ProfileRegistryMaintenanceRuntime, remove_store_directory, verify_store_path_absent, @@ -33,27 +37,109 @@ const PROFILE_SQLITE_DATABASES: [&str; 3] = [ USER_SESSIONS_DB_FILENAME, USER_MEMORY_DB_FILENAME, ]; -const PROFILE_DATABASE_PATHS: [&str; 14] = [ - "projects", - "remote", - "agent_managed", - "user-sessions.grafeo", - "user-sessions.grafeo.wal", - "user-memory.grafeo", - "user-memory.grafeo.wal", - ".user-sessions.db.host-admission", - tracedecay_hooks::PROFILE_HOOK_ADMISSIONS_DIR, - tracedecay_runtime_core::storage::RESPONSE_HANDLES_DIRECTORY, - "lcm-payloads", - "hook_analytics.jsonl", - "hook_analytics.jsonl.lock", - "maintenance", +#[derive(Clone, Copy)] +enum StoreMemberKind { + File, + Directory, +} + +use StoreMemberKind::{Directory, File}; + +const PROFILE_DATABASE_PATHS: [(&str, StoreMemberKind); 9] = [ + ("projects", Directory), + ("remote", Directory), + ("agent_managed", Directory), + (tracedecay_hooks::PROFILE_HOOK_ADMISSIONS_DIR, Directory), + ( + tracedecay_runtime_core::storage::RESPONSE_HANDLES_DIRECTORY, + Directory, + ), + ("lcm-payloads", Directory), + ("hook_analytics.jsonl", File), + ("hook_analytics.jsonl.lock", File), + ("maintenance", Directory), ]; -fn sqlite_family_member(database: &Path, suffix: &str) -> PathBuf { - let mut member = database.as_os_str().to_os_string(); - member.push(suffix); - member.into() +fn lock_offline_store_family( + directory: &Path, + database: &str, +) -> tracedecay_domain::errors::Result> { + let path = directory.join(database); + lock_offline_sqlite_family_for_reset(&path) + .map_err(|error| wipe_io("lock offline SQLite family", &path, &error)) +} + +/// Exact data files and directories belonging to the relational shard. +/// Sidecars and projections go before the SQLite main so an interrupted +/// reset can still rediscover its stale authority. +fn sqlite_store_data_members(database: &str) -> Vec<(PathBuf, StoreMemberKind)> { + let sessions = database == SESSIONS_DB_FILENAME || database == USER_SESSIONS_DB_FILENAME; + let mut members = Vec::new(); + if sessions || database == USER_MEMORY_DB_FILENAME { + let (files, directories) = + graph_store_data_paths(&Path::new(database).with_extension("grafeo")); + members.extend(directories.into_iter().map(|path| (path, Directory))); + members.extend(files.into_iter().map(|path| (path, File))); + } + if sessions { + members.push((format!(".{database}.host-admission").into(), Directory)); + members.push(( + format!("{database}.delivery-settlement-spool-v1").into(), + Directory, + )); + } + members.extend( + ["-wal", "-shm", "-journal", ""] + .map(|suffix| (PathBuf::from(format!("{database}{suffix}")), File)), + ); + members +} + +/// Under the exclusive offline lease, refuse every ambiguous member before +/// the first removal, including a directory in the place of a regular file. +fn remove_checked_profile_members( + directory: &Path, + members: &[(PathBuf, StoreMemberKind)], +) -> tracedecay_domain::errors::Result { + for (name, kind) in members { + let path = directory.join(name); + tracedecay_runtime_core::storage::reject_symlink_components(&path, "offline store data") + .map_err(|error| wipe_io("validate store data", &path, &error))?; + match std::fs::symlink_metadata(&path) { + Ok(meta) + if match kind { + File => meta.is_file(), + Directory => meta.is_dir(), + } => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Ok(_) => { + return Err(tracedecay_domain::errors::TraceDecayError::Config { + message: format!( + "unexpected file type for offline store data '{}'", + path.display() + ), + }); + } + Err(error) => return Err(wipe_io("inspect offline store data", &path, &error)), + } + } + let mut removed = 0; + for (name, _) in members { + removed += usize::from(remove_fixed_profile_path(directory, name)?); + verify_store_path_absent(&directory.join(name))?; + } + Ok(removed) +} + +/// The offline lifecycle authority admits deletion of the stale store +/// itself. Similar names, private initialization data and stable coordination +/// locks are not members of its data family. +pub(super) fn remove_store_family( + directory: &Path, + database: &str, +) -> tracedecay_domain::errors::Result { + let _initialization = lock_offline_store_family(directory, database)?; + remove_checked_profile_members(directory, &sqlite_store_data_members(database)) } fn wipe_io( @@ -121,7 +207,7 @@ fn validate_complete_wipe_profile_root( pub(super) fn remove_fixed_profile_path( profile_root: &Path, - name: &str, + name: impl AsRef, ) -> tracedecay_domain::errors::Result { use tracedecay_private_fs::framed_log::{DirectorySyncPolicy, sync_directory}; @@ -318,32 +404,15 @@ pub(crate) fn take_profile_offline( fn wipe_complete_profile_database_state( profile_root: &Path, ) -> tracedecay_domain::errors::Result { - let mut removed = 0usize; - for name in PROFILE_DATABASE_PATHS { - removed += usize::from(remove_fixed_profile_path(profile_root, name)?); - } + let mut leases = Vec::new(); + let mut members = PROFILE_DATABASE_PATHS + .map(|(name, is_directory)| (PathBuf::from(name), is_directory)) + .to_vec(); for database in PROFILE_SQLITE_DATABASES { - let database = profile_root.join(database); - for suffix in ["-wal", "-shm", "-journal", ""] { - let member = sqlite_family_member(&database, suffix); - removed += usize::from( - tracedecay_runtime_core::storage::PrivateStoreIo::remove_file_durable(&member) - .map_err(|error| { - wipe_io("remove profile SQLite family member", &member, &error) - })?, - ); - } + leases.push(lock_offline_store_family(profile_root, database)?); + members.extend(sqlite_store_data_members(database)); } - for name in PROFILE_DATABASE_PATHS { - verify_store_path_absent(&profile_root.join(name))?; - } - for database in PROFILE_SQLITE_DATABASES { - let database = profile_root.join(database); - for suffix in ["-wal", "-shm", "-journal", ""] { - verify_store_path_absent(&sqlite_family_member(&database, suffix))?; - } - } - Ok(removed) + remove_checked_profile_members(profile_root, &members) } /// Handles the `wipe` and `wipe --all` commands. diff --git a/crates/tracedecay-cli/tests/core_cli_suite/cli_non_interactive_test.rs b/crates/tracedecay-cli/tests/core_cli_suite/cli_non_interactive_test.rs index ba4fa1591c5..4e8b9247306 100644 --- a/crates/tracedecay-cli/tests/core_cli_suite/cli_non_interactive_test.rs +++ b/crates/tracedecay-cli/tests/core_cli_suite/cli_non_interactive_test.rs @@ -2647,7 +2647,19 @@ fn wipe_all_is_schema_independent_and_removes_every_profile_database_root() { let wal = PathBuf::from(wal); std::fs::create_dir(&wal).unwrap(); std::fs::write(wal.join("segment"), b"graph wal").unwrap(); + std::fs::write(path.with_extension("verified"), b"marker").unwrap(); + for directory in [ + path.with_extension("sealed"), + path.with_added_extension("spill"), + ] { + std::fs::create_dir(&directory).unwrap(); + std::fs::write(directory.join("member"), b"owned projection data").unwrap(); + } + std::fs::write(path.with_added_extension("corruption-lock"), b"keep lock").unwrap(); } + let unrelated = profile.join("user-sessions.spill"); + std::fs::create_dir(&unrelated).unwrap(); + std::fs::write(unrelated.join("member"), b"foreign directory").unwrap(); let host_admission = profile.join(".user-sessions.db.host-admission"); std::fs::create_dir(&host_admission).unwrap(); std::fs::write(host_admission.join("pending"), b"admission spool").unwrap(); @@ -2670,6 +2682,46 @@ fn wipe_all_is_schema_independent_and_removes_every_profile_database_root() { std::fs::write(&path, b"profile state").unwrap(); } + let lease_path = profile.join(".user-memory.db.tracedecay-init.durable-directory.lock"); + let initializer = tracedecay_private_fs::create_private_file(&lease_path).unwrap(); + initializer.try_lock().unwrap(); + let mut busy = tracedecay_command_without_daemon(home.path(), project.path()); + busy.args(["wipe", "--all", "--yes"]); + let refused = run_with_timeout(busy, cli_timeout()); + assert!(!refused.status.success(), "{refused:?}"); + assert!( + String::from_utf8_lossy(&refused.stderr).contains("lock offline SQLite family"), + "{refused:?}" + ); + for database in &database_paths { + assert!(database.is_file()); + } + drop(initializer); + + // A wrong-kind marker in the last top-level store must be discovered + // before even the earlier profile, remote or project data is touched. + let ambiguous = profile.join("user-memory.verified"); + std::fs::remove_file(&ambiguous).unwrap(); + std::fs::create_dir(&ambiguous).unwrap(); + let mut refusal = tracedecay_command_without_daemon(home.path(), project.path()); + refusal.args(["wipe", "--all", "--yes"]); + let refused = run_with_timeout(refusal, cli_timeout()); + assert!(!refused.status.success(), "{refused:?}"); + let stderr = String::from_utf8_lossy(&refused.stderr); + assert!( + stderr.contains("unexpected file type") && stderr.contains("user-memory.verified"), + "{stderr}" + ); + for database in &database_paths { + assert_eq!( + std::fs::read(database).unwrap(), + b"not a compatible SQLite schema" + ); + } + assert!(ambiguous.is_dir()); + std::fs::remove_dir(&ambiguous).unwrap(); + std::fs::write(ambiguous, b"marker").unwrap(); + let mut command = tracedecay_command_without_daemon(home.path(), project.path()); command.args(["wipe", "--all", "--yes"]); let output = run_with_timeout(command, cli_timeout()); @@ -2712,7 +2764,23 @@ fn wipe_all_is_schema_independent_and_removes_every_profile_database_root() { wal.push(".wal"); assert_namespace_absent(&path, "wipe --all left a Grafeo store"); assert_namespace_absent(&PathBuf::from(wal), "wipe --all left a Grafeo WAL"); + for member in [ + path.with_extension("verified"), + path.with_extension("sealed"), + path.with_added_extension("spill"), + ] { + assert_namespace_absent(&member, "wipe --all left graph data"); + } + assert_eq!( + std::fs::read(path.with_added_extension("corruption-lock")).unwrap(), + b"keep lock" + ); } + assert!(lease_path.is_file()); + assert_eq!( + std::fs::read(unrelated.join("member")).unwrap(), + b"foreign directory" + ); assert_namespace_absent( &host_admission, "wipe --all left the profile host-admission database companion", diff --git a/crates/tracedecay-graph-db/src/corrupt_store.rs b/crates/tracedecay-graph-db/src/corrupt_store.rs index 594535180fd..b7f6ecd513e 100644 --- a/crates/tracedecay-graph-db/src/corrupt_store.rs +++ b/crates/tracedecay-graph-db/src/corrupt_store.rs @@ -220,8 +220,8 @@ fn delete_container_family(container: &Path) -> Result<(), GraphDbError> { for sidecar in [ wal_sidecar_path(container), - container.with_extension("verified"), - container.with_extension("spill"), + crate::verified_marker::marker_path(container), + crate::location::spill_directory(container), ] { remove_family_member(&sidecar)?; } @@ -257,10 +257,8 @@ fn remove_family_member(member: &Path) -> Result<(), GraphDbError> { } /// `graph.grafeo` -> `graph.grafeo.wal`, matching Grafeo's sidecar layout. -fn wal_sidecar_path(container: &Path) -> PathBuf { - let mut sidecar = container.as_os_str().to_owned(); - sidecar.push(".wal"); - PathBuf::from(sidecar) +pub(crate) fn wal_sidecar_path(container: &Path) -> PathBuf { + container.with_added_extension("wal") } fn container_file_name(container: &Path) -> Result<&str, GraphDbError> { @@ -297,6 +295,7 @@ mod tests { ) .unwrap(); std::fs::write(container.with_extension("verified"), b"marker").unwrap(); + std::fs::create_dir(root.join("graph.grafeo.spill")).unwrap(); container } diff --git a/crates/tracedecay-graph-db/src/lib.rs b/crates/tracedecay-graph-db/src/lib.rs index c7aab2ccf1f..04e3efcec3c 100644 --- a/crates/tracedecay-graph-db/src/lib.rs +++ b/crates/tracedecay-graph-db/src/lib.rs @@ -51,6 +51,7 @@ pub use limits::{ MAX_VERIFIED_GENERATION_BATCH_LIVE_BYTES, MAX_VERIFIED_GENERATION_BATCH_MUTATIONS, MAX_VERIFIED_GENERATION_ENTITIES, MAX_VERIFIED_GENERATION_RELATIONS, }; +pub use location::graph_store_data_paths; #[cfg(any(feature = "test-helpers", feature = "eval-helpers"))] pub use location::{GraphDbLocation, GraphDbOpenOptions, GraphDurability, GraphFormatVersion}; #[cfg(not(any(feature = "test-helpers", feature = "eval-helpers")))] diff --git a/crates/tracedecay-graph-db/src/location.rs b/crates/tracedecay-graph-db/src/location.rs index 50e76a7ffe3..bbf8b6eb30e 100644 --- a/crates/tracedecay-graph-db/src/location.rs +++ b/crates/tracedecay-graph-db/src/location.rs @@ -7,6 +7,28 @@ use grafeo_engine::config::{DurabilityMode, StorageFormat}; use crate::{GraphCancellation, GraphDbError}; +/// Regular files and directories owned by one graph container. Offline store +/// reset discards its sealed projections along with the relational authority. +/// Persistent coordination locks are outside the data family. +pub fn graph_store_data_paths(container: &Path) -> ([PathBuf; 2], [PathBuf; 3]) { + ( + [ + crate::verified_marker::marker_path(container), + container.to_owned(), + ], + [ + crate::corrupt_store::wal_sidecar_path(container), + spill_directory(container), + crate::sealed_store::sealed_store_root(container), + ], + ) +} + +pub(crate) fn spill_directory(container: &Path) -> PathBuf { + // Grafeo derives the default spill directory from the complete filename. + container.with_added_extension("spill") +} + #[derive(Clone, Debug, PartialEq, Eq)] pub enum GraphDbLocation { #[cfg(any(test, feature = "test-helpers", feature = "eval-helpers"))] diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index e6441489824..d1d397a8507 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -496,7 +496,7 @@ fn sealed_store_disabled() -> bool { } /// `/var/db/graph.grafeo` -> `/var/db/graph.sealed`. -fn sealed_store_root(database_path: &Path) -> PathBuf { +pub(crate) fn sealed_store_root(database_path: &Path) -> PathBuf { database_path.with_extension("sealed") } diff --git a/crates/tracedecay-runtime-core/src/storage.rs b/crates/tracedecay-runtime-core/src/storage.rs index 6f9aafbb583..1d02748b870 100644 --- a/crates/tracedecay-runtime-core/src/storage.rs +++ b/crates/tracedecay-runtime-core/src/storage.rs @@ -470,6 +470,7 @@ pub use profile_identity::{ PROFILE_IDENTITY_RECORD_NAME, PROFILE_IDENTITY_SCHEMA_VERSION, ProfileIdentityRecordV1, read_existing_profile_identity_record, }; +pub use tracedecay_graph_db::graph_store_data_paths; #[cfg(all(test, unix))] use paths_and_io::open_lock_file; From 995cb201a7196a6c141216efb975808aabd0104a Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 20:20:07 -0800 Subject: [PATCH 37/60] fix(storage): publish only owned staged SQLite generations --- .../src/capability_dir.rs | 65 +- .../tracedecay-private-fs/src/windows_file.rs | 24 +- .../src/db/engine/connection.rs | 7 + .../shard_runtime/registry/initialization.rs | 875 ++++++++++-------- .../src/shard_runtime/registry/ports.rs | 62 +- crates/tracedecay-runtime-core/src/storage.rs | 5 +- .../src/storage/paths_and_io.rs | 12 + .../src/connection/mod.rs | 15 + .../src/exact_sql/command.rs | 44 + .../src/exact_sql/mod.rs | 19 + .../src/repository/attachment.rs | 23 + 11 files changed, 765 insertions(+), 386 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index c0c846fb8f4..58142bd4671 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -12,7 +12,70 @@ use std::io; use cap_fs_ext::DirExt; #[cfg(not(windows))] use cap_fs_ext::OpenOptionsMaybeDirExt; -use cap_std::fs::{Dir, File, OpenOptions}; +use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt}; +pub use cap_std::fs::Dir; +use cap_std::fs::{File, OpenOptions}; + +/// Pins the parent of a cross-directory publication. On Windows the no-delete +/// sharing mode also keeps its resolved Win32 pathname usable for rename. +pub fn pin_parent(path: &std::path::Path) -> io::Result { + #[cfg(windows)] + { + use std::os::windows::fs::OpenOptionsExt; + use windows_sys::Win32::Storage::FileSystem::{ + FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, FILE_SHARE_READ, + FILE_SHARE_WRITE, + }; + let file = std::fs::OpenOptions::new() + .read(true) + .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE) + .custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT) + .open(path)?; + if !file.metadata()?.is_dir() { + return Err(io::Error::new( + io::ErrorKind::NotADirectory, + "publication parent is not a directory", + )); + } + Ok(Dir::from_std_file(file)) + } + #[cfg(not(windows))] + { + Dir::open_ambient_dir(path, cap_std::ambient_authority()) + } +} + +pub fn open_regular_nofollow(directory: &Dir, name: &OsStr) -> io::Result { + let mut options = OpenOptions::new(); + options.read(true).follow(FollowSymlinks::No); + #[cfg(unix)] + { + use cap_std::fs::OpenOptionsExt; + options.custom_flags(libc::O_NONBLOCK); + } + let file = directory.open_with(name, &options)?.into_std(); + if !file.metadata()?.is_file() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "expected a regular file", + )); + } + Ok(file) +} + +pub fn create_private_file(directory: &Dir, name: &OsStr) -> io::Result { + #[cfg(windows)] + { + crate::create_private_file(&dir_entry_path(directory, name)?) + } + #[cfg(not(windows))] + { + use cap_std::fs::OpenOptionsExt; + let mut options = OpenOptions::new(); + options.read(true).write(true).create_new(true).mode(0o600); + Ok(directory.open_with(name, &options)?.into_std()) + } +} /// How many times a create-or-open that reports the entry missing is asked /// again before the answer is believed. xnu bounds its own retry of the same diff --git a/crates/tracedecay-private-fs/src/windows_file.rs b/crates/tracedecay-private-fs/src/windows_file.rs index 7de7ee9abc6..b88c838e88d 100644 --- a/crates/tracedecay-private-fs/src/windows_file.rs +++ b/crates/tracedecay-private-fs/src/windows_file.rs @@ -5,7 +5,8 @@ use std::mem::MaybeUninit; use std::os::windows::io::AsRawHandle; use windows_sys::Win32::Storage::FileSystem::{ - BY_HANDLE_FILE_INFORMATION, GetFileInformationByHandle, + BY_HANDLE_FILE_INFORMATION, FILE_STANDARD_INFO, FileStandardInfo, GetFileInformationByHandle, + GetFileInformationByHandleEx, }; #[derive(Clone, Copy)] @@ -22,6 +23,27 @@ pub fn information(file: &H) -> io::Result { identity_information(raw_information(file)?) } +/// A name that SQLite has marked for deletion cannot be reopened on Windows +/// until every remaining handle, including our creation reservation, closes. +pub fn deletion_pending(file: &impl AsRawHandle) -> io::Result { + let mut info = MaybeUninit::::uninit(); + // SAFETY: the live handle is borrowed, and FileStandardInfo writes exactly + // one FILE_STANDARD_INFO into the correctly sized aligned destination. + let ok = unsafe { + GetFileInformationByHandleEx( + file.as_raw_handle().cast(), + FileStandardInfo, + info.as_mut_ptr().cast(), + std::mem::size_of::() as u32, + ) + }; + if ok == 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: the successful call initialized the requested structure. + Ok(unsafe { info.assume_init() }.DeletePending) +} + /// Reads only the hard-link count from an open Windows file handle. /// /// Unlike [`information`], this does not require the volume to expose a diff --git a/crates/tracedecay-runtime-core/src/db/engine/connection.rs b/crates/tracedecay-runtime-core/src/db/engine/connection.rs index 3bd7c6f4eb5..62fec9cd718 100644 --- a/crates/tracedecay-runtime-core/src/db/engine/connection.rs +++ b/crates/tracedecay-runtime-core/src/db/engine/connection.rs @@ -186,6 +186,13 @@ impl Connection { Ok(Rows::from_exact(rows)) } + pub(crate) async fn finish_initial_schema(&self) -> Result<()> { + self.runtime + .finish_initial_schema() + .await + .map_err(Into::into) + } + pub async fn checkpoint_wal_truncate(&self) -> Result { let runtime = Arc::clone(&self.runtime); let rows = runtime.checkpoint_wal_truncate_async().await?; diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs index 2d744ea80aa..1a6025fd0c1 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs @@ -1,25 +1,20 @@ -//! Prospective stores install their schema outside the SQLite basename used by -//! clients. A process death before the no-replace rename leaves only disposable -//! schema in the private work directory. Hard links identify the empty sidecar -//! reservations if the process dies partway through claiming the destination. +//! A prospective SQLite store publishes its completed main last. Each staged +//! file moves once by no-replace rename; cleanup only removes an empty directory. +//! A failed schema install therefore cannot delete a file at a reused basename. +use std::ffi::OsStr; use std::fs::{self, File}; -use std::io; +use std::io::{self, Read, Seek, Write}; #[cfg(unix)] use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; -#[cfg(unix)] -use std::{ffi::CString, os::fd::AsRawFd}; +use std::time::SystemTime; -use rusqlite::{Connection, OpenFlags}; -use tracedecay_private_fs::framed_log::{ - DirectorySyncPolicy, rename_noreplace, sync_directory, sync_parent_directory, -}; -use tracedecay_private_fs::{ - FileLease, create_private_directory, create_private_file, open_private_directory, - open_private_file, -}; +use serde::{Deserialize, Serialize}; +use tracedecay_private_fs::capability_dir::{self, Dir}; +use tracedecay_private_fs::framed_log::{DirectorySyncPolicy, sync_parent_directory}; +use tracedecay_private_fs::{FileLease, create_private_directory, open_private_directory}; -use crate::sqlite_read_snapshot::{fold_staging_to_standalone, with_suffix}; +use crate::sqlite_read_snapshot::with_suffix; const SIDECARS: [&str; 3] = ["-wal", "-shm", "-journal"]; const OWNER: &str = "schema-installation.owner"; @@ -31,7 +26,13 @@ const SYNC: DirectorySyncPolicy = DirectorySyncPolicy::TolerateUnsupported; pub fn lock_offline_sqlite_family_for_reset(destination: &Path) -> io::Result> { let staging = InitializationDirectory::path(destination); let lease = crate::storage::try_lock_existing_durable_directory(&staging)?; - if present(&staging)? { + if present(&staging)? + || lease + .as_ref() + .map(|file| file.metadata()) + .transpose()? + .is_some_and(|meta| meta.len() != 0) + { return Err(io::Error::new( io::ErrorKind::AlreadyExists, "SQLite initialization requires reconciliation before reset", @@ -40,12 +41,108 @@ pub fn lock_offline_sqlite_family_for_reset(destination: &Path) -> io::Result bool { + self.main.allocation.is_some() + && self + .sidecars + .iter() + .all(|sidecar| sidecar.allocation.is_some()) + } + + fn load(file: &mut File) -> io::Result> { + let length = file.metadata()?.len(); + Ok(if length == 0 { + None + } else { + if length > 2048 { + return Err(ambiguous("invalid SQLite preparation receipt")); + } + let mut bytes = Vec::new(); + file.take(length).read_to_end(&mut bytes)?; + Some( + serde_json::from_slice(&bytes) + .map_err(|_| ambiguous("invalid SQLite preparation receipt"))?, + ) + }) + } +} + +#[derive(Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +struct LeafIdentity { + device: u64, + file_id: u64, + created: Option, + allocation: Option, +} + +impl LeafIdentity { + fn from_file(file: &File) -> io::Result { + let metadata = file.metadata()?; + #[cfg(unix)] + let (device, file_id) = (metadata.dev(), metadata.ino()); + #[cfg(windows)] + let (device, file_id) = { + let info = tracedecay_private_fs::windows_file::information(file)?; + (u64::from(info.volume_serial_number), info.file_index) + }; + #[cfg(target_os = "macos")] + let allocation = { + use std::os::macos::fs::MetadataExt; + let value = metadata.st_gen(); + (value != 0).then_some(u64::from(value)) + }; + #[cfg(not(target_os = "macos"))] + let allocation = tracedecay_private_fs::inode_generation(file)?; + #[cfg(any(unix, windows))] + { + Ok(Self { + device, + file_id, + created: metadata.created().ok(), + allocation, + }) + } + #[cfg(not(any(unix, windows)))] + { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "SQLite generation identity is unavailable", + )) + } + } + + fn require_at(&self, directory: &Dir, name: &OsStr, empty: bool) -> io::Result<()> { + let file = capability_dir::open_regular_nofollow(directory, name)?; + if *self != Self::from_file(&file)? || (empty && file.metadata()?.len() != 0) { + return Err(ambiguous( + "SQLite initialization member no longer matches its prepared generation", + )); + } + Ok(()) + } +} + +fn ambiguous(message: &'static str) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, message) +} + pub(super) struct InitializationDirectory { destination: PathBuf, directory: PathBuf, directory_pin: Option, - _lease: FileLease, - owns_schema: bool, + parent: Dir, + lease: FileLease, + prepared: Option, } impl InitializationDirectory { @@ -55,8 +152,19 @@ impl InitializationDirectory { fn acquire(destination: PathBuf, create: bool) -> io::Result { let directory = Self::path(&destination); - let lease = crate::storage::try_lock_durable_directory(&directory)?; + let parent = capability_dir::pin_parent( + destination + .parent() + .ok_or_else(|| ambiguous("SQLite destination has no parent"))?, + )?; + let mut lease = crate::storage::try_lock_durable_directory(&directory)?; + let prepared = PreparedFamily::load(&mut lease)?; if create { + if prepared.is_some() && !present(&directory)? { + return Err(ambiguous( + "prepared SQLite initialization directory is missing", + )); + } match create_private_directory(&directory) { Ok(()) => sync_parent_directory(&directory, SYNC)?, Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} @@ -65,61 +173,34 @@ impl InitializationDirectory { } let directory_pin = open_private_directory(&directory)?; let directory = directory.canonicalize()?; - let marker = directory.join(OWNER); - match open_private_file(&marker) { - Ok(file) if file.metadata()?.len() == 0 => {} - Ok(_) => { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "invalid SQLite initialization owner", - )); - } - Err(error) if error.kind() == io::ErrorKind::NotFound => { - if fs::read_dir(&directory)?.next().transpose()?.is_some() { - return Err(io::Error::new( - io::ErrorKind::AlreadyExists, - "unclaimed SQLite initialization directory is occupied", - )); - } - if create { - create_private_file(&marker)?.sync_all()?; - sync_directory(&directory, SYNC)?; - } - } - Err(error) => return Err(error), - } Ok(Self { destination, directory, directory_pin: Some(directory_pin), - owns_schema: false, - _lease: lease, + parent, + lease, + prepared, }) } pub(super) fn begin(destination: PathBuf) -> io::Result { - let mut stage = Self::acquire(destination, true)?; + let stage = Self::acquire(destination, true)?; if present(&stage.destination)? { return Err(io::Error::new( io::ErrorKind::AlreadyExists, "SQLite destination exists", )); } - // Check every occupied canonical member before discarding even our - // private schema. An interrupted pre-protocol family has no witness. - stage.validate_sidecars(false)?; - if present(&stage.transferred())? - || (present(&stage.prepared())? - && (!present(&stage.database())? - || !same_regular_file(&stage.database(), &stage.prepared())?)) - { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "published SQLite file is missing or ambiguous", - )); + if stage.prepared.is_none() { + stage.require_empty()?; + for suffix in SIDECARS { + if present(&with_suffix(&stage.destination, suffix))? { + return Err(ambiguous( + "occupied SQLite sidecar has no prepared generation", + )); + } + } } - stage.discard_schema()?; - stage.owns_schema = true; Ok(stage) } @@ -127,196 +208,232 @@ impl InitializationDirectory { self.directory.join("schema.sqlite") } - fn prepared(&self) -> PathBuf { - self.directory.join("prepared.sqlite") + #[cfg(test)] + fn reservation(&self, suffix: &str) -> PathBuf { + self.directory.join(format!("reservation{suffix}")) } - fn transferred(&self) -> PathBuf { - self.directory.join("transferred.sqlite") + fn stage_cap(&self) -> io::Result { + let pin = self + .directory_pin + .as_ref() + .ok_or_else(|| ambiguous("SQLite staging directory is no longer pinned"))?; + Ok(Dir::from_std_file(pin.try_clone()?)) } - /// Teardown is relative to the directory capability acquired with the - /// initialization lease. A noncooperating process can rename the directory - /// and replace its pathname with a symlink; unlinking through that pathname - /// would otherwise delete unrelated database files outside our staging area. - fn remove_private_member(&self, member: &str) -> io::Result<()> { - if member.is_empty() - || member == "." - || member == ".." - || member.contains('/') - || member.contains('\\') - { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "invalid SQLite staging member", + fn require_empty(&self) -> io::Result<()> { + self.verify_directory_name()?; + if self.stage_cap()?.entries()?.next().transpose()?.is_some() { + return Err(ambiguous( + "unproven SQLite staging contents require offline reconciliation", )); } - #[cfg(unix)] - { - let name = CString::new(member) - .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "invalid staging name"))?; - let directory = self - .directory_pin - .as_ref() - .ok_or_else(|| io::Error::other("SQLite staging directory is no longer pinned"))?; - // SAFETY: the directory descriptor remains open for this call, - // and name is a NUL-terminated, single-component filename. - let rc = unsafe { libc::unlinkat(directory.as_raw_fd(), name.as_ptr(), 0) }; - if rc == 0 { - return Ok(()); - } - let error = io::Error::last_os_error(); - if error.kind() == io::ErrorKind::NotFound { - return Ok(()); - } - Err(error) - } - #[cfg(not(unix))] - { - remove_if_present(&self.directory.join(member)) - } - } - - fn discard_schema(&self) -> io::Result<()> { - self.remove_private_member("prepared.sqlite")?; - for suffix in ["-wal", "-shm", "-journal", ""] { - self.remove_private_member(&format!("schema.sqlite{suffix}"))?; - } Ok(()) } - fn witness(&self, suffix: &str) -> PathBuf { - self.directory.join(format!("reservation{suffix}")) - } - - fn validate_sidecars(&self, required: bool) -> io::Result<()> { - for suffix in SIDECARS { - let name = with_suffix(&self.destination, suffix); - if !required && !present(&name)? { - continue; + fn prepare( + &mut self, + expected: u64, + (main_file, sidecars): (File, [File; 3]), + ) -> io::Result<()> { + self.verify_directory_name()?; + let stage = self.stage_cap()?; + #[cfg(windows)] + let sidecars = { + let mut retained = sidecars.map(Some); + for slot in &mut retained { + if let Some(file) = slot.as_ref() + && tracedecay_private_fs::windows_file::deletion_pending(file)? + { + drop(slot.take()); + } } - if !empty_regular(&name)? - || !present(&self.witness(suffix))? - || !same_regular_file(&name, &self.witness(suffix))? + retained + }; + #[cfg(not(windows))] + let sidecars = sidecars.map(Some); + let main = LeafIdentity::from_file(&main_file)?; + main.require_at(&stage, OsStr::new("schema.sqlite"), false)?; + for entry in stage.entries()? { + let name = entry?.file_name(); + if name != "schema.sqlite" + && !SIDECARS + .iter() + .any(|suffix| name == OsStr::new(&format!("schema.sqlite{suffix}"))) { - return Err(io::Error::new( - io::ErrorKind::AlreadyExists, - "occupied SQLite sidecar has no matching initialization reservation", + return Err(ambiguous( + "unexpected SQLite staging member requires offline reconciliation", )); } } + main_file.sync_all()?; + // WAL and rollback-journal bytes may be durable data. The original + // writer has checkpointed and joined; refuse any remaining frames. + // Only the original, now-unused SHM index may be cleared by descriptor. + for (index, suffix) in SIDECARS.iter().enumerate() { + let source = format!("schema.sqlite{suffix}"); + if present_at(&stage, source.as_ref())? { + let file = sidecars[index].as_ref().ok_or_else(|| { + ambiguous("SQLite sidecar has no surviving original reservation") + })?; + LeafIdentity::from_file(file)?.require_at( + &stage, + source.as_ref(), + *suffix != "-shm", + )?; + } + } + let reserve = |index: usize, suffix: &str| { + let source = format!("schema.sqlite{suffix}"); + let target = format!("reservation{suffix}"); + if present_at(&stage, source.as_ref())? { + let file = sidecars[index].as_ref().ok_or_else(|| { + ambiguous("SQLite sidecar has no surviving original reservation") + })?; + let identity = LeafIdentity::from_file(file)?; + identity.require_at(&stage, source.as_ref(), suffix != "-shm")?; + if suffix == "-shm" { + file.set_len(0)?; + } + file.sync_all()?; + capability_dir::rename_noreplace(&stage, source.as_ref(), &stage, target.as_ref())?; + identity.require_at(&stage, target.as_ref(), true)?; + Ok(identity) + } else { + let file = capability_dir::create_private_file(&stage, target.as_ref())?; + file.sync_all()?; + LeafIdentity::from_file(&file) + } + }; + let prepared = PreparedFamily { + runtime_identity: expected, + main, + sidecars: [ + reserve(0, "-wal")?, + reserve(1, "-shm")?, + reserve(2, "-journal")?, + ], + }; + capability_dir::sync_directory(&stage)?; + self.lease.rewind()?; + let bytes = serde_json::to_vec(&prepared).map_err(io::Error::other)?; + self.lease.write_all(&bytes)?; + self.lease.sync_all()?; + self.prepared = Some(prepared); Ok(()) } - fn reserve_sidecars(&self) -> io::Result<()> { - self.validate_sidecars(false)?; - for suffix in SIDECARS { - let witness = self.witness(suffix); - if !present(&witness)? { - create_private_file(&witness)?.sync_all()?; - } - if !empty_regular(&witness)? { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "invalid SQLite reservation", + fn check_members(&self, stage: &Dir, prepared: &PreparedFamily) -> io::Result<()> { + self.verify_directory_name()?; + for entry in stage.entries()? { + let name = entry?.file_name(); + if name != "schema.sqlite" + && !SIDECARS + .iter() + .any(|suffix| name == OsStr::new(&format!("reservation{suffix}"))) + { + return Err(ambiguous( + "unexpected SQLite staging member requires offline reconciliation", )); } } - sync_directory(&self.directory, SYNC)?; - for suffix in SIDECARS { - let name = with_suffix(&self.destination, suffix); - if !present(&name)? { - fs::hard_link(self.witness(suffix), name)?; + prepared + .main + .require_at(stage, OsStr::new("schema.sqlite"), false)?; + for (suffix, identity) in SIDECARS.into_iter().zip(&prepared.sidecars) { + let from = format!("reservation{suffix}"); + let to = with_suffix(&self.destination, suffix); + let to = leaf(&to)?; + match ( + present_at(stage, from.as_ref())?, + present_at(&self.parent, to)?, + ) { + (true, false) => identity.require_at(stage, from.as_ref(), true)?, + (false, true) => identity.require_at(&self.parent, to, true)?, + _ => return Err(ambiguous("SQLite reservation is missing or ambiguous")), } } - sync_parent_directory(&self.destination, SYNC)?; - self.validate_sidecars(true) + Ok(()) } - pub(super) fn publish(mut self) -> io::Result<()> { - let path = self.database(); - let connection = Connection::open_with_flags( - &path, - OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_NOFOLLOW, - ) - .map_err(io::Error::other)?; - connection - .pragma_update( - None, - "application_id", - tracedecay_rusqlite_runtime::schema_version::REGISTERED_STORE_APPLICATION_ID, - ) - .map_err(io::Error::other)?; - fold_staging_to_standalone(connection, &path, || Ok(()))?; - File::options() - .read(true) - .write(true) - .open(&path)? - .sync_all()?; - self.claim_destination()?; - rename_noreplace(&path, &self.destination)?; - sync_parent_directory(&self.destination, SYNC)?; - self.finish_publication() - } - - fn claim_destination(&self) -> io::Result<()> { - // This alias witnesses the exact main inode across the no-replace - // rename. Remove it last, before the first runtime is returned. - fs::hard_link(self.database(), self.prepared())?; - self.reserve_sidecars() - } - - fn finish_publication(&mut self) -> io::Result<()> { - if present(&self.database())? { - return Err(io::Error::new( - io::ErrorKind::AlreadyExists, - "SQLite destination coexists with an unpublished initialization", + pub(super) fn resume_prepared(&mut self) -> io::Result> { + let Some(prepared) = self.prepared.as_ref() else { + return Ok(None); + }; + // A persistent inode number alone cannot distinguish inode reuse after + // process death. A filesystem with no kernel allocation generation can publish in + // the original process, but must preserve interrupted preparation. + if !prepared.has_durable_generations() { + return Err(ambiguous( + "SQLite preparation has no durable file generation for recovery", )); } - if present(&self.prepared())? { - if !same_regular_file(&self.destination, &self.prepared())? { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "SQLite destination identity changed before handoff", - )); + self.publish_prepared().map(Some) + } + + fn publish_prepared(&mut self) -> io::Result { + let prepared = self + .prepared + .as_ref() + .ok_or_else(|| ambiguous("SQLite preparation receipt is missing"))?; + let stage = self.stage_cap()?; + self.check_members(&stage, prepared)?; + for (suffix, identity) in SIDECARS.into_iter().zip(&prepared.sidecars) { + let from = format!("reservation{suffix}"); + let to = with_suffix(&self.destination, suffix); + let to = leaf(&to)?; + if present_at(&stage, from.as_ref())? { + capability_dir::rename_noreplace(&stage, from.as_ref(), &self.parent, to)?; } - self.validate_sidecars(true)?; - // All witnesses remain until this single namespace transition. - // Ordinary SQLite clients own the WAL, SHM and journal afterward. - rename_noreplace(&self.prepared(), &self.transferred())?; - sync_directory(&self.directory, SYNC)?; - } - if !present(&self.transferred())? - || !same_regular_file(&self.destination, &self.transferred())? - { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "SQLite destination has no matching handoff", - )); + identity.require_at(&self.parent, to, true)?; } - for suffix in SIDECARS { - self.remove_private_member(&format!("reservation{suffix}"))?; - } - self.remove_private_member("transferred.sqlite")?; - sync_directory(&self.directory, SYNC)?; - self.retire() + capability_dir::sync_directory(&stage)?; + capability_dir::sync_directory(&self.parent)?; + self.check_members(&stage, prepared)?; + let expected = prepared.runtime_identity; + capability_dir::rename_noreplace( + &stage, + OsStr::new("schema.sqlite"), + &self.parent, + leaf(&self.destination)?, + )?; + capability_dir::sync_directory(&self.parent)?; + prepared + .main + .require_at(&self.parent, leaf(&self.destination)?, false)?; + drop(stage); + self.retire()?; + Ok(expected) + } + + pub(super) fn publish(mut self, expected: u64, files: (File, [File; 3])) -> io::Result<()> { + self.prepare(expected, files)?; + self.publish_prepared().map(|_| ()) } fn verify_directory_name(&self) -> io::Result<()> { let pinned = self .directory_pin .as_ref() - .ok_or_else(|| io::Error::other("SQLite staging directory is no longer pinned"))?; + .ok_or_else(|| ambiguous("SQLite staging directory is no longer pinned"))?; let named = open_private_directory(&self.directory)?; #[cfg(unix)] - let matches = same_unix_inode(&pinned.metadata()?, &named.metadata()?); + let matches = { + let left = pinned.metadata()?; + let right = named.metadata()?; + left.dev() == right.dev() && left.ino() == right.ino() + }; #[cfg(windows)] - let matches = same_windows_object(pinned, &named)?; + let matches = { + let left = tracedecay_private_fs::windows_file::information(pinned)?; + let right = tracedecay_private_fs::windows_file::information(&named)?; + left.volume_serial_number == right.volume_serial_number + && left.file_index == right.file_index + }; #[cfg(not(any(unix, windows)))] let matches = false; if !matches { - return Err(io::Error::new( - io::ErrorKind::InvalidData, + return Err(ambiguous( "SQLite staging directory no longer names the pinned directory", )); } @@ -324,27 +441,20 @@ impl InitializationDirectory { } fn retire(&mut self) -> io::Result<()> { - self.verify_directory_name()?; - for entry in fs::read_dir(&self.directory)? { - if entry?.file_name() != OWNER { - return Err(io::Error::new( - io::ErrorKind::DirectoryNotEmpty, - "SQLite initialization directory still contains files", - )); - } + self.require_empty()?; + if self.prepared.is_some() { + // If we crash after clearing the receipt, an empty directory is all + // that remains of the already-published preparation. + self.lease.set_len(0)?; + self.lease.sync_all()?; } - self.remove_private_member(OWNER)?; - // rmdir is name-based. Refuse an already changed name as late as - // possible, while holding the lease and the original directory pin. - self.verify_directory_name()?; drop(self.directory_pin.take()); - fs::remove_dir(&self.directory)?; - sync_parent_directory(&self.directory, SYNC) + // The kernel refuses a file or a nonempty directory even if a new + // entry appears after the read. No regular staging leaf is unlinked. + self.parent.remove_dir(leaf(&self.directory)?)?; + capability_dir::sync_directory(&self.parent) } - /// A completed handoff can still have its empty directory and owner marker - /// after a process exit. Read-only callers need neither cleanup nor a lease. - /// Any other staged member requires writable reconciliation before attach. pub(super) fn verify_read_only_handoff(destination: &Path) -> io::Result<()> { let directory = Self::path(destination); let _pin = match open_private_directory(&directory) { @@ -353,14 +463,23 @@ impl InitializationDirectory { Err(error) => return Err(error), }; for entry in fs::read_dir(&directory)? { - let member = entry?; - if member.file_name() != OWNER || !empty_regular(&member.path())? { - return Err(io::Error::new( - io::ErrorKind::InvalidData, + // An empty owner-only marker from the previous layout is harmless + // for read-only admission. Never retire it by mutable pathname. + let entry = entry?; + if entry.file_name() != OWNER + || !entry.file_type()?.is_file() + || entry.metadata()?.len() != 0 + { + return Err(ambiguous( "SQLite staging requires writable reconciliation before read-only attachment", )); } } + if crate::storage::durable_directory_lock_has_data(&directory)? { + return Err(ambiguous( + "SQLite preparation receipt requires writable reconciliation", + )); + } Ok(()) } @@ -376,130 +495,63 @@ impl InitializationDirectory { } Err(error) => return Err(error), }; - if Self::pending(&stage.directory)? { - stage.finish_publication() - } else { - stage.retire() - } - } - - fn pending(directory: &Path) -> io::Result { - for name in [ - "schema.sqlite", - "prepared.sqlite", - "transferred.sqlite", - "reservation-wal", - "reservation-shm", - "reservation-journal", - ] { - if present(&directory.join(name))? { - return Ok(true); + if let Some(prepared) = &stage.prepared { + if prepared.main.allocation.is_none() { + return Err(ambiguous( + "SQLite preparation has no durable file generation for recovery", + )); } + prepared + .main + .require_at(&stage.parent, leaf(destination)?, false)?; } - Ok(false) + stage.retire() } } impl Drop for InitializationDirectory { fn drop(&mut self) { - if self.directory_pin.is_none() { - return; - } - if !self.owns_schema { - // Early refusals can leave our empty work directory. If a private - // schema or a reservation still exists, retain it for recovery. - if matches!(present(&self.directory.join(OWNER)), Ok(true)) - && matches!(Self::pending(&self.directory), Ok(false)) - && let Err(error) = self.retire() - { - tracing::warn!(%error, "could not retire SQLite initialization directory"); - } - return; - } - // A destination that appeared during this attempt is ambiguous until - // a new opener verifies the prepared-main witness. - match present(&self.destination) { - Ok(false) => { - if let Err(error) = self.discard_schema().and_then(|()| { - if !Self::pending(&self.directory)? { - self.retire()?; - } - Ok(()) - }) { - tracing::warn!(%error, "could not retire unpublished SQLite schema"); - } - } - Err(error) => { - tracing::warn!(%error, "could not inspect initialized SQLite destination"); - } - Ok(true) => {} + if self.directory_pin.is_some() + && self.prepared.is_none() + && self.require_empty().is_ok() + && let Err(error) = self.retire() + { + tracing::warn!(%error, "could not retire empty SQLite initialization directory"); } } } -fn present(path: &Path) -> io::Result { - match fs::symlink_metadata(path) { +fn leaf(path: &Path) -> io::Result<&OsStr> { + path.file_name() + .ok_or_else(|| ambiguous("SQLite member has no basename")) +} + +fn present_at(directory: &Dir, name: &OsStr) -> io::Result { + match directory.symlink_metadata(name) { Ok(_) => Ok(true), Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false), Err(error) => Err(error), } } -#[cfg(any(test, not(unix)))] -fn remove_if_present(path: &Path) -> io::Result<()> { - match fs::remove_file(path) { - Ok(()) => Ok(()), - Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), +fn present(path: &Path) -> io::Result { + match fs::symlink_metadata(path) { + Ok(_) => Ok(true), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false), Err(error) => Err(error), } } -fn empty_regular(path: &Path) -> io::Result { - let meta = fs::symlink_metadata(path)?; - Ok(meta.is_file() && meta.len() == 0) -} - -#[cfg(unix)] -fn same_unix_inode(left: &fs::Metadata, right: &fs::Metadata) -> bool { - left.dev() == right.dev() && left.ino() == right.ino() -} - -#[cfg(windows)] -fn same_windows_object(left: &File, right: &File) -> io::Result { - let a = tracedecay_private_fs::windows_file::information(left)?; - let b = tracedecay_private_fs::windows_file::information(right)?; - Ok(a.volume_serial_number == b.volume_serial_number && a.file_index == b.file_index) -} - -fn same_regular_file(left: &Path, right: &Path) -> io::Result { - if !fs::symlink_metadata(left)?.is_file() || !fs::symlink_metadata(right)?.is_file() { - return Ok(false); - } - #[cfg(unix)] - { - Ok(same_unix_inode( - &fs::symlink_metadata(left)?, - &fs::symlink_metadata(right)?, - )) - } - #[cfg(windows)] - { - same_windows_object(&File::open(left)?, &File::open(right)?) - } - #[cfg(not(any(unix, windows)))] - Err(io::Error::new( - io::ErrorKind::Unsupported, - "SQLite file identity unavailable", - )) -} - #[cfg(test)] mod tests { use super::*; + use rusqlite::Connection; #[cfg(unix)] use std::os::unix::fs::symlink; use std::process::Command; use tempfile::TempDir; + use tracedecay_private_fs::create_private_file; + use tracedecay_private_fs::framed_log::rename_noreplace; fn seed(path: &Path, value: i64) { let db = Connection::open(path).unwrap(); @@ -508,41 +560,55 @@ mod tests { db.execute("INSERT INTO data VALUES (?)", [value]).unwrap(); } + fn owned_files(stage: &InitializationDirectory) -> (File, [File; 3]) { + let path = stage.database(); + let main = File::options().read(true).write(true).open(&path).unwrap(); + ( + main, + SIDECARS.map(|suffix| create_private_file(&with_suffix(&path, suffix)).unwrap()), + ) + } + #[test] fn process_exit_during_initialization_recovers_without_publishing_partial_schema() { const CHILD: &str = "TRACEDECAY_TEST_EXIT_DURING_INITIALIZATION"; if let Some(destination) = std::env::var_os(CHILD) { - let stage = InitializationDirectory::begin(destination.into()).unwrap(); + let mut stage = InitializationDirectory::begin(destination.into()).unwrap(); seed(&stage.database(), 1); let state = std::env::var("TRACEDECAY_TEST_INITIALIZATION_STATE").unwrap(); - match state.as_str() { - "installing" => {} - "reserved" => stage.claim_destination().unwrap(), - "published" | "transferred" | "unproven" => { - stage.claim_destination().unwrap(); - rename_noreplace(&stage.database(), &stage.destination).unwrap(); - if state == "transferred" { - rename_noreplace(&stage.prepared(), &stage.transferred()).unwrap(); + if state != "installing" { + let identity = crate::db::sqlite_generation_identity(&stage.database()).unwrap(); + let files = owned_files(&stage); + stage.prepare(identity, files).unwrap(); + if state != "prepared" { + let count = if state == "reserved" || state == "unproven" { + 1 + } else { + 3 + }; + for suffix in SIDECARS.into_iter().take(count) { + rename_noreplace( + &stage.reservation(suffix), + &with_suffix(&stage.destination, suffix), + ) + .unwrap(); } - if state != "published" { - remove_if_present(&stage.witness("-wal")).unwrap(); + if state == "published" { + rename_noreplace(&stage.database(), &stage.destination).unwrap(); } } - state => panic!("unknown initialization state {state}"), } - // Simulate ordinary process death with no Rust or SQLite destructors. std::process::exit(73); } - let name = format!( "{}::process_exit_during_initialization_recovers_without_publishing_partial_schema", module_path!().split_once("::").unwrap().1 ); for state in [ "installing", + "prepared", "reserved", "published", - "transferred", "unproven", ] { let root = TempDir::new().unwrap(); @@ -554,42 +620,119 @@ mod tests { .output() .unwrap(); assert_eq!(output.status.code(), Some(73), "{output:?}"); + let directory = InitializationDirectory::path(&destination); + if state == "installing" { + assert!(InitializationDirectory::begin(destination.clone()).is_err()); + assert_eq!(read_value(&directory.join("schema.sqlite")), 1); + assert!(!destination.exists()); + continue; + } if state == "unproven" { - assert!(InitializationDirectory::recover_published(&destination).is_err()); - for suffix in SIDECARS { - assert!(empty_regular(&with_suffix(&destination, suffix)).unwrap()); + let wal = with_suffix(&destination, "-wal"); + fs::write(&wal, b"unproven WAL").unwrap(); + let mut retry = InitializationDirectory::begin(destination.clone()).unwrap(); + assert!(retry.resume_prepared().is_err()); + assert_eq!(fs::read(wal).unwrap(), b"unproven WAL"); + assert!(!destination.exists()); + continue; + } + let durable = { + let stage = InitializationDirectory::acquire(destination.clone(), false).unwrap(); + stage.prepared.as_ref().unwrap().has_durable_generations() + }; + if !durable { + let published = state == "published"; + if published { + assert!(InitializationDirectory::recover_published(&destination).is_err()); + } else { + let mut retry = InitializationDirectory::begin(destination.clone()).unwrap(); + assert!(retry.resume_prepared().is_err()); } + assert_eq!(destination.exists(), published); + assert_eq!( + read_value( + if published { + destination.clone() + } else { + directory.join("schema.sqlite") + } + .as_path() + ), + 1 + ); + assert!(directory.is_dir()); continue; } - let expected = if matches!(state, "published" | "transferred") { + if state == "published" { InitializationDirectory::recover_published(&destination).unwrap(); - 1 } else { - assert!(!destination.exists()); - let retry = InitializationDirectory::begin(destination.clone()).unwrap(); - assert!(!retry.database().exists()); - seed(&retry.database(), 2); - retry.publish().unwrap(); - 2 - }; + let mut retry = InitializationDirectory::begin(destination.clone()).unwrap(); + assert!(retry.resume_prepared().unwrap().is_some()); + } + assert_eq!(read_value(&destination), 1); let db = Connection::open(&destination).unwrap(); - assert_eq!( - db.query_row("SELECT value FROM data", [], |row| row.get::<_, i64>(0)) - .unwrap(), - expected - ); assert_eq!( db.query_row("PRAGMA integrity_check", [], |row| row.get::<_, String>(0)) .unwrap(), "ok" ); - assert!(!InitializationDirectory::path(&destination).exists()); + assert!(!directory.exists()); + } + } + + fn read_value(path: &Path) -> i64 { + Connection::open(path) + .unwrap() + .query_row("SELECT value FROM data", [], |row| row.get(0)) + .unwrap() + } + + #[test] + fn unproven_private_members_survive_drop_and_retry() { + let root = TempDir::new().unwrap(); + for member in [ + "schema.sqlite", + "schema.sqlite-wal", + "schema.sqlite-shm", + "schema.sqlite-journal", + "prepared.sqlite", + ] { + let destination = root.path().join(member); + let stage = InitializationDirectory::begin(destination.clone()).unwrap(); + let path = stage.directory.join(member); + fs::write(&path, b"unclaimed data").unwrap(); + drop(stage); + assert!(InitializationDirectory::begin(destination.clone()).is_err()); + assert!(!destination.exists()); + assert_eq!(fs::read(path).unwrap(), b"unclaimed data"); + } + } + + #[test] + fn preparation_refuses_unclaimed_sidecar_handles_without_modifying_the_family() { + let root = TempDir::new().unwrap(); + let destination = root.path().join("sessions.db"); + let mut stage = InitializationDirectory::begin(destination.clone()).unwrap(); + seed(&stage.database(), 9); + let identity = crate::db::sqlite_generation_identity(&stage.database()).unwrap(); + let (main, mut files) = owned_files(&stage); + let separate = root.path().join("unrelated-empty-reservation"); + files[1] = create_private_file(&separate).unwrap(); + let shm = with_suffix(&stage.database(), "-shm"); + fs::write(&shm, b"preserve existing index").unwrap(); + let error = stage.prepare(identity, (main, files)).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert_eq!(fs::read(shm).unwrap(), b"preserve existing index"); + assert_eq!(fs::metadata(separate).unwrap().len(), 0); + assert!(!destination.exists()); + for suffix in SIDECARS { + assert!(!stage.reservation(suffix).exists()); } } /// A competing process may rename the private directory despite the /// cooperative lease and install a symlink at the old path. Failed-start - /// cleanup must only remove entries from our *pinned* directory and never + /// cleanup must preserve occupied entries in the pinned directory and never /// the distinct valid SQLite database beneath that replacement symlink. #[cfg(unix)] #[test] @@ -647,9 +790,12 @@ mod tests { // caller. Its directory descriptor still points to retired-stage. drop(stage); let retired = root.path().join("retired-stage"); - assert!(!retired.join("schema.sqlite").exists()); + assert!(retired.join("schema.sqlite").is_file()); for suffix in ["-wal", "-shm", "-journal"] { - assert!(!with_suffix(&retired.join("schema.sqlite"), suffix).exists()); + assert_eq!( + fs::read(with_suffix(&retired.join("schema.sqlite"), suffix)).unwrap(), + b"owned staging sidecar" + ); assert_eq!( fs::read(with_suffix(&foreign_database, suffix)).unwrap(), format!("foreign {suffix}").as_bytes() @@ -673,7 +819,7 @@ mod tests { let error = stage.retire().unwrap_err(); assert_eq!(error.kind(), io::ErrorKind::InvalidData); assert!(other.is_dir()); - assert!(original.join(OWNER).is_file()); + assert!(original.is_dir()); stage.directory = original.clone(); drop(stage); assert!(!original.exists()); @@ -716,6 +862,7 @@ mod tests { assert!(InitializationDirectory::begin(destination.clone()).is_err()); assert!(stage.database().exists()); drop(stage); + let destination = root.path().join("other.db"); for suffix in SIDECARS { let foreign = with_suffix(&destination, suffix); fs::write(&foreign, b"pre-existing sidecar").unwrap(); diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs index 0bcac1b9e67..2a8a884dc3a 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs @@ -224,23 +224,44 @@ async fn initialize_staged_runtime( authority .require_active_write_scope(OPERATION) .map_err(|error| physical_failure(OPERATION, error))?; + if request.open_cancellation.is_cancelled() { + return Err(StoreRuntimeRegistryFailure::OpenCancelled { + key: Box::new(StoreRuntimeKey::from_binding(&request.binding)), + }); + } let destination = request.locator.path().to_owned(); let stage = crate::native_runtime::spawn_blocking(move || { - InitializationDirectory::begin(destination).map(Arc::new) + let mut stage = InitializationDirectory::begin(destination)?; + let resumed = stage.resume_prepared()?; + Ok::<_, std::io::Error>((stage, resumed)) }) .await .map_err(|error| physical_failure(OPERATION, error))? .map_err(|error| physical_failure(OPERATION, error))?; + let (stage, resumed) = stage; + if let Some(identity) = resumed { + return Ok(identity); + } + let stage = Arc::new(stage); let path = stage.database(); let attachment = attach_runtime(executor, request, Some(stage.clone()), None).await?; let identity = attachment.0.opened_file_identity(); - let schema = - install_final_schema_before_publication(request, attachment.as_physical(), &path).await; - attachment - .close(schema.is_ok()) - .await - .map_err(|error| physical_failure(OPERATION, error))?; - schema?; + if let Err(error) = + install_final_schema_before_publication(request, attachment.as_physical(), &path).await + { + attachment + .close() + .await + .map_err(|error| physical_failure(OPERATION, error))?; + return Err(error); + } + let files = crate::native_runtime::spawn_blocking(move || { + let LifecyclePhysicalAttachment(physical, _stage) = attachment; + physical.close_completed_initialization() + }) + .await + .and_then(|result| result) + .map_err(|error| physical_failure(OPERATION, error))?; authority .require_active_write_scope(OPERATION) .map_err(|error| physical_failure(OPERATION, error))?; @@ -252,7 +273,7 @@ async fn initialize_staged_runtime( crate::native_runtime::spawn_blocking(move || { Arc::try_unwrap(stage) .map_err(|_| std::io::Error::other("SQLite schema workers still hold staging"))? - .publish() + .publish(identity, files) }) .await .map_err(|error| physical_failure(OPERATION, error))? @@ -274,7 +295,7 @@ async fn publish_lifecycle_runtime( ) .await { - let _ = attachment.close(false).await; + let _ = attachment.close().await; return Err(error); } if request.mode == StoreRuntimeOpenMode::Existing @@ -283,7 +304,7 @@ async fn publish_lifecycle_runtime( verify_final_schema_before_existing_publication(&request, attachment.as_physical()) .await { - let _ = attachment.close(false).await; + let _ = attachment.close().await; return Err(error); } if request.mode == StoreRuntimeOpenMode::Existing @@ -294,7 +315,7 @@ async fn publish_lifecycle_runtime( { // This failed attachment must finish closing before the registry can // retry at a new epoch. - attachment.close(false).await.map_err(|message| { + attachment.close().await.map_err(|message| { StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { operation: "close runtime after writer authority refusal", message, @@ -303,7 +324,7 @@ async fn publish_lifecycle_runtime( return Err(error); } if let Err(error) = runtime.transition(RuntimeMaintenanceStateV1::Ready) { - let _ = attachment.close(false).await; + let _ = attachment.close().await; return Err(runtime_lifecycle_failure(error)); } Ok(PublishedShardRuntime::new(runtime, attachment.into_box())) @@ -461,14 +482,11 @@ impl LifecyclePhysicalAttachment { /// Closes an attachment that will not be published. Joining its writer /// and readers parks the calling thread. - async fn close(self, completed_schema: bool) -> Result<(), String> { + async fn close(self) -> Result<(), String> { crate::native_runtime::spawn_blocking(move || { let stage = self.1; if stage.is_some() { - if !completed_schema { - return self.0.abort_initialization(); - } - self.0.commit_initialization()?; + return self.0.abort_initialization(); } self.0.drain()?; self.0.close_and_join() @@ -604,7 +622,7 @@ async fn install_final_schema_before_publication( | StoreShardScopeV1::ProfileSessions | StoreShardScopeV1::ProjectSessions { .. } => { crate::ports::registered_schema::install_from_authorized_connection( - connection, + connection.clone(), request.open_cancellation.clone(), ) .await @@ -631,6 +649,12 @@ async fn install_final_schema_before_publication( })?; } } + if request.locator.is_prospective() { + connection + .finish_initial_schema() + .await + .map_err(|error| physical_failure("finish prospective SQLite schema", error))?; + } Ok(()) } diff --git a/crates/tracedecay-runtime-core/src/storage.rs b/crates/tracedecay-runtime-core/src/storage.rs index 1d02748b870..75cca308c1d 100644 --- a/crates/tracedecay-runtime-core/src/storage.rs +++ b/crates/tracedecay-runtime-core/src/storage.rs @@ -465,7 +465,10 @@ pub use paths_and_io::{ retry_transient_file_op, set_private_dir_permissions, try_acquire_sidecar_lock, validate_project_id, }; -pub(crate) use paths_and_io::{try_lock_durable_directory, try_lock_existing_durable_directory}; +pub(crate) use paths_and_io::{ + durable_directory_lock_has_data, try_lock_durable_directory, + try_lock_existing_durable_directory, +}; pub use profile_identity::{ PROFILE_IDENTITY_RECORD_NAME, PROFILE_IDENTITY_SCHEMA_VERSION, ProfileIdentityRecordV1, read_existing_profile_identity_record, diff --git a/crates/tracedecay-runtime-core/src/storage/paths_and_io.rs b/crates/tracedecay-runtime-core/src/storage/paths_and_io.rs index 2a3fdc2d403..380c9f3ee92 100644 --- a/crates/tracedecay-runtime-core/src/storage/paths_and_io.rs +++ b/crates/tracedecay-runtime-core/src/storage/paths_and_io.rs @@ -795,6 +795,18 @@ pub(crate) fn try_lock_existing_durable_directory(path: &Path) -> io::Result io::Result { + match fs::symlink_metadata(private_directory_lock_path(path)?) { + Ok(meta) if meta.is_file() => Ok(meta.len() != 0), + Ok(_) => Err(invalid_input( + "durable directory lock is not a regular file", + )), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } +} + /// Non-blocking sidecar lock acquisition. Returns the held lock file on /// success, or `None` when another process/thread already holds it (the caller /// then skips its critical section). See the sidecar-lock module note above for diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs index 3b414422624..6304f54f12e 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs @@ -186,6 +186,21 @@ impl OpenedDatabaseFile { } } + pub(crate) fn take_completed_files(&self) -> Option<(File, [File; 3])> { + let sidecars = self + .created_sidecars + .as_ref()? + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take()?; + let main = self + .file + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take()?; + Some((main, sidecars)) + } + pub(crate) const fn identity(&self) -> u64 { self.identity } diff --git a/crates/tracedecay-rusqlite-runtime/src/exact_sql/command.rs b/crates/tracedecay-rusqlite-runtime/src/exact_sql/command.rs index 55f6bb092a4..d2d375b8295 100644 --- a/crates/tracedecay-rusqlite-runtime/src/exact_sql/command.rs +++ b/crates/tracedecay-rusqlite-runtime/src/exact_sql/command.rs @@ -43,6 +43,10 @@ pub(crate) enum WriterCommand { lease: Arc, authority: Option>, }, + FinishInitialSchema { + reply: async_channel::Sender>, + authority: Arc, + }, CheckpointWalTruncate { reply: async_channel::Sender>, authority: Option>, @@ -553,6 +557,43 @@ pub(crate) async fn run_writer_command( } terminal } + WriterCommand::FinishInitialSchema { reply, authority } => { + let result = authority + .verify(ExactSqlWriteIntent::Execute) + .and_then(|()| { + with_exact_sql_guard( + connection, + ExactSqlAuthority::Canonical(ConnectionMode::Maintenance), + Some(Arc::clone(shutdown_requested)), + None, + true, + None, + || { + connection + .pragma_update( + None, + "application_id", + crate::schema_version::REGISTERED_STORE_APPLICATION_ID, + ) + .map_err(|error| { + sqlite_error("mark completed initial schema", error) + })?; + let checkpoint: (i64, i64, i64) = connection + .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |row| { + Ok((row.get(0)?, row.get(1)?, row.get(2)?)) + }) + .map_err(|error| { + sqlite_error("checkpoint completed initial schema", error) + })?; + if checkpoint != (0, 0, 0) { + return Err(ExactSqlError::Busy); + } + Ok(()) + }, + ) + }); + DeferredReply::new(reply, result) + } WriterCommand::CheckpointWalTruncate { reply, authority } => { if let Err(error) = verify_write_authority(authority.as_deref(), ExactSqlWriteIntent::Query) @@ -609,6 +650,9 @@ pub(crate) fn reject_writer_command(command: WriterCommand) { WriterCommand::BeginTransaction { reply, .. } => { let _ = reply.try_send(Err(ExactSqlError::WriterUnavailable)); } + WriterCommand::FinishInitialSchema { reply, .. } => { + let _ = reply.try_send(Err(ExactSqlError::WriterUnavailable)); + } WriterCommand::CheckpointWalTruncate { reply, .. } => { let _ = reply.try_send(Err(ExactSqlError::WriterUnavailable)); } diff --git a/crates/tracedecay-rusqlite-runtime/src/exact_sql/mod.rs b/crates/tracedecay-rusqlite-runtime/src/exact_sql/mod.rs index 419f7d6cffa..45f051b1138 100644 --- a/crates/tracedecay-rusqlite-runtime/src/exact_sql/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/exact_sql/mod.rs @@ -615,6 +615,25 @@ impl ExactSqlHandle { .map_err(|_| ExactSqlError::WriterUnavailable) } + /// Marks the completed prospective store and requires a fully checkpointed + /// main while its original authorized SQLite writer still owns the family. + pub async fn finish_initial_schema(&self) -> Result<(), ExactSqlError> { + let authority = self + .write_authority + .clone() + .ok_or(ExactSqlError::AuthorityMismatch)?; + let (reply, response) = async_channel::bounded(1); + self.writer + .as_ref() + .ok_or(ExactSqlError::WriterUnavailable)? + .try_send(WriterCommand::FinishInitialSchema { reply, authority }) + .map_err(map_writer_send_error)?; + response + .recv() + .await + .map_err(|_| ExactSqlError::WriterUnavailable)? + } + /// Checkpoints and truncates the WAL on the serialized writer connection. fn enqueue_checkpoint_wal_truncate( &self, diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs index 7b9b0950b68..b73ce704579 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs @@ -506,6 +506,29 @@ impl RepositoryRuntimePhysicalAttachment { Ok(()) } + /// Hands the exclusive original reservations to the publisher after all + /// SQLite workers have exited. No raw descriptor is closed during a live + /// POSIX SQLite transaction. + pub fn close_completed_initialization( + &self, + ) -> Result<(std::fs::File, [std::fs::File; 3]), String> { + self.commit_initialization()?; + self.drain()?; + let files = { + let state = self.lock_state(); + if state.file_lost { + return Err("initial schema lost its database generation".to_owned()); + } + state + .retained_file_pin + .as_ref() + .and_then(OpenedDatabaseFile::take_completed_files) + .ok_or_else(|| "initial schema has no original sidecar reservations".to_owned())? + }; + self.close_and_join()?; + Ok(files) + } + pub fn abort_initialization(&self) -> Result<(), String> { self.drain()?; self.close_and_join()?; From e3972642b8d0329e297487beccc5d7ea03050bea Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 20:24:34 -0800 Subject: [PATCH 38/60] fix(storage): retain SQLite leaf pins through publication --- .../src/capability_dir.rs | 8 ++ .../shard_runtime/registry/initialization.rs | 77 ++++++++++--------- 2 files changed, 49 insertions(+), 36 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 58142bd4671..774d8c3dc7b 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -48,6 +48,14 @@ pub fn pin_parent(path: &std::path::Path) -> io::Result { pub fn open_regular_nofollow(directory: &Dir, name: &OsStr) -> io::Result { let mut options = OpenOptions::new(); options.read(true).follow(FollowSymlinks::No); + #[cfg(windows)] + { + use cap_std::fs::OpenOptionsExt; + use windows_sys::Win32::Storage::FileSystem::{ + FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, + }; + options.share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE); + } #[cfg(unix)] { use cap_std::fs::OpenOptionsExt; diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs index 1a6025fd0c1..cdb552aaff7 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs @@ -121,14 +121,14 @@ impl LeafIdentity { } } - fn require_at(&self, directory: &Dir, name: &OsStr, empty: bool) -> io::Result<()> { + fn pin_at(&self, directory: &Dir, name: &OsStr, empty: bool) -> io::Result { let file = capability_dir::open_regular_nofollow(directory, name)?; if *self != Self::from_file(&file)? || (empty && file.metadata()?.len() != 0) { return Err(ambiguous( "SQLite initialization member no longer matches its prepared generation", )); } - Ok(()) + Ok(file) } } @@ -235,7 +235,7 @@ impl InitializationDirectory { &mut self, expected: u64, (main_file, sidecars): (File, [File; 3]), - ) -> io::Result<()> { + ) -> io::Result<[File; 4]> { self.verify_directory_name()?; let stage = self.stage_cap()?; #[cfg(windows)] @@ -253,7 +253,7 @@ impl InitializationDirectory { #[cfg(not(windows))] let sidecars = sidecars.map(Some); let main = LeafIdentity::from_file(&main_file)?; - main.require_at(&stage, OsStr::new("schema.sqlite"), false)?; + let main_pin = main.pin_at(&stage, OsStr::new("schema.sqlite"), false)?; for entry in stage.entries()? { let name = entry?.file_name(); if name != "schema.sqlite" @@ -276,7 +276,7 @@ impl InitializationDirectory { let file = sidecars[index].as_ref().ok_or_else(|| { ambiguous("SQLite sidecar has no surviving original reservation") })?; - LeafIdentity::from_file(file)?.require_at( + LeafIdentity::from_file(file)?.pin_at( &stage, source.as_ref(), *suffix != "-shm", @@ -291,28 +291,29 @@ impl InitializationDirectory { ambiguous("SQLite sidecar has no surviving original reservation") })?; let identity = LeafIdentity::from_file(file)?; - identity.require_at(&stage, source.as_ref(), suffix != "-shm")?; + identity.pin_at(&stage, source.as_ref(), suffix != "-shm")?; if suffix == "-shm" { file.set_len(0)?; } file.sync_all()?; capability_dir::rename_noreplace(&stage, source.as_ref(), &stage, target.as_ref())?; - identity.require_at(&stage, target.as_ref(), true)?; - Ok(identity) + let pin = identity.pin_at(&stage, target.as_ref(), true)?; + Ok::<_, io::Error>((identity, pin)) } else { let file = capability_dir::create_private_file(&stage, target.as_ref())?; file.sync_all()?; - LeafIdentity::from_file(&file) + let identity = LeafIdentity::from_file(&file)?; + let pin = identity.pin_at(&stage, target.as_ref(), true)?; + Ok((identity, pin)) } }; + let (wal, wal_pin) = reserve(0, "-wal")?; + let (shm, shm_pin) = reserve(1, "-shm")?; + let (journal, journal_pin) = reserve(2, "-journal")?; let prepared = PreparedFamily { runtime_identity: expected, main, - sidecars: [ - reserve(0, "-wal")?, - reserve(1, "-shm")?, - reserve(2, "-journal")?, - ], + sidecars: [wal, shm, journal], }; capability_dir::sync_directory(&stage)?; self.lease.rewind()?; @@ -320,10 +321,10 @@ impl InitializationDirectory { self.lease.write_all(&bytes)?; self.lease.sync_all()?; self.prepared = Some(prepared); - Ok(()) + Ok([main_pin, wal_pin, shm_pin, journal_pin]) } - fn check_members(&self, stage: &Dir, prepared: &PreparedFamily) -> io::Result<()> { + fn pin_members(&self, stage: &Dir, prepared: &PreparedFamily) -> io::Result> { self.verify_directory_name()?; for entry in stage.entries()? { let name = entry?.file_name(); @@ -337,23 +338,27 @@ impl InitializationDirectory { )); } } - prepared - .main - .require_at(stage, OsStr::new("schema.sqlite"), false)?; + let mut pins = vec![ + prepared + .main + .pin_at(stage, OsStr::new("schema.sqlite"), false)?, + ]; for (suffix, identity) in SIDECARS.into_iter().zip(&prepared.sidecars) { let from = format!("reservation{suffix}"); let to = with_suffix(&self.destination, suffix); let to = leaf(&to)?; - match ( - present_at(stage, from.as_ref())?, - present_at(&self.parent, to)?, - ) { - (true, false) => identity.require_at(stage, from.as_ref(), true)?, - (false, true) => identity.require_at(&self.parent, to, true)?, - _ => return Err(ambiguous("SQLite reservation is missing or ambiguous")), - } + pins.push( + match ( + present_at(stage, from.as_ref())?, + present_at(&self.parent, to)?, + ) { + (true, false) => identity.pin_at(stage, from.as_ref(), true)?, + (false, true) => identity.pin_at(&self.parent, to, true)?, + _ => return Err(ambiguous("SQLite reservation is missing or ambiguous")), + }, + ); } - Ok(()) + Ok(pins) } pub(super) fn resume_prepared(&mut self) -> io::Result> { @@ -377,7 +382,7 @@ impl InitializationDirectory { .as_ref() .ok_or_else(|| ambiguous("SQLite preparation receipt is missing"))?; let stage = self.stage_cap()?; - self.check_members(&stage, prepared)?; + let _pins = self.pin_members(&stage, prepared)?; for (suffix, identity) in SIDECARS.into_iter().zip(&prepared.sidecars) { let from = format!("reservation{suffix}"); let to = with_suffix(&self.destination, suffix); @@ -385,11 +390,11 @@ impl InitializationDirectory { if present_at(&stage, from.as_ref())? { capability_dir::rename_noreplace(&stage, from.as_ref(), &self.parent, to)?; } - identity.require_at(&self.parent, to, true)?; + identity.pin_at(&self.parent, to, true)?; } capability_dir::sync_directory(&stage)?; capability_dir::sync_directory(&self.parent)?; - self.check_members(&stage, prepared)?; + self.pin_members(&stage, prepared)?; let expected = prepared.runtime_identity; capability_dir::rename_noreplace( &stage, @@ -400,14 +405,14 @@ impl InitializationDirectory { capability_dir::sync_directory(&self.parent)?; prepared .main - .require_at(&self.parent, leaf(&self.destination)?, false)?; + .pin_at(&self.parent, leaf(&self.destination)?, false)?; drop(stage); self.retire()?; Ok(expected) } pub(super) fn publish(mut self, expected: u64, files: (File, [File; 3])) -> io::Result<()> { - self.prepare(expected, files)?; + let _creation_pins = self.prepare(expected, files)?; self.publish_prepared().map(|_| ()) } @@ -457,12 +462,12 @@ impl InitializationDirectory { pub(super) fn verify_read_only_handoff(destination: &Path) -> io::Result<()> { let directory = Self::path(destination); - let _pin = match open_private_directory(&directory) { + let pin = match open_private_directory(&directory) { Ok(pin) => pin, Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), Err(error) => return Err(error), }; - for entry in fs::read_dir(&directory)? { + for entry in Dir::from_std_file(pin).entries()? { // An empty owner-only marker from the previous layout is harmless // for read-only admission. Never retire it by mutable pathname. let entry = entry?; @@ -503,7 +508,7 @@ impl InitializationDirectory { } prepared .main - .require_at(&stage.parent, leaf(destination)?, false)?; + .pin_at(&stage.parent, leaf(destination)?, false)?; } stage.retire() } From 22538251a5b88b17a1158da87d01f986e6caf435 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 20:29:05 -0800 Subject: [PATCH 39/60] fix(storage): verify the complete pending SQLite handoff --- .../shard_runtime/registry/initialization.rs | 92 ++++++++++++++++--- 1 file changed, 78 insertions(+), 14 deletions(-) diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs index cdb552aaff7..521eabef32c 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs @@ -366,8 +366,8 @@ impl InitializationDirectory { return Ok(None); }; // A persistent inode number alone cannot distinguish inode reuse after - // process death. A filesystem with no kernel allocation generation can publish in - // the original process, but must preserve interrupted preparation. + // process death. A filesystem without allocation generations can publish + // in the original process, but must preserve interrupted preparation. if !prepared.has_durable_generations() { return Err(ambiguous( "SQLite preparation has no durable file generation for recovery", @@ -460,8 +460,18 @@ impl InitializationDirectory { capability_dir::sync_directory(&self.parent) } + fn require_released_receipt(directory: &Path) -> io::Result<()> { + if crate::storage::durable_directory_lock_has_data(directory)? { + return Err(ambiguous( + "SQLite preparation receipt requires writable reconciliation", + )); + } + Ok(()) + } + pub(super) fn verify_read_only_handoff(destination: &Path) -> io::Result<()> { let directory = Self::path(destination); + Self::require_released_receipt(&directory)?; let pin = match open_private_directory(&directory) { Ok(pin) => pin, Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), @@ -480,35 +490,40 @@ impl InitializationDirectory { )); } } - if crate::storage::durable_directory_lock_has_data(&directory)? { - return Err(ambiguous( - "SQLite preparation receipt requires writable reconciliation", - )); - } Ok(()) } pub(super) fn recover_published(destination: &Path) -> io::Result<()> { let directory = Self::path(destination); if !present(&directory)? { - return Ok(()); + return Self::require_released_receipt(&directory); } let mut stage = match Self::acquire(destination.to_owned(), false) { Ok(stage) => stage, Err(error) if error.kind() == io::ErrorKind::NotFound && !present(&directory)? => { - return Ok(()); + return Self::require_released_receipt(&directory); } Err(error) => return Err(error), }; + let mut pins = Vec::new(); if let Some(prepared) = &stage.prepared { - if prepared.main.allocation.is_none() { + if !prepared.has_durable_generations() { return Err(ambiguous( "SQLite preparation has no durable file generation for recovery", )); } - prepared - .main - .pin_at(&stage.parent, leaf(destination)?, false)?; + pins.push( + prepared + .main + .pin_at(&stage.parent, leaf(destination)?, false)?, + ); + for (suffix, identity) in SIDECARS.into_iter().zip(&prepared.sidecars) { + pins.push(identity.pin_at( + &stage.parent, + leaf(&with_suffix(destination, suffix))?, + true, + )?); + } } stage.retire() } @@ -713,6 +728,55 @@ mod tests { } } + #[test] + fn publication_requires_its_original_directory_and_schema() { + let root = TempDir::new().unwrap(); + let destination = root.path().join("sessions.db"); + let mut stage = InitializationDirectory::begin(destination.clone()).unwrap(); + let schema = stage.database(); + seed(&schema, 7); + let identity = crate::db::sqlite_generation_identity(&schema).unwrap(); + let files = owned_files(&stage); + let other = root.path().join("independent-directory"); + create_private_directory(&other).unwrap(); + let independent = other.join("schema.sqlite"); + seed(&independent, 99); + let bytes = fs::read(&independent).unwrap(); + stage.directory = other; + assert_eq!( + stage.publish(identity, files).unwrap_err().kind(), + io::ErrorKind::InvalidData + ); + assert!(!destination.exists()); + for suffix in SIDECARS { + assert!(!with_suffix(&destination, suffix).exists()); + } + assert_eq!(fs::read(&independent).unwrap(), bytes); + assert_eq!(read_value(&schema), 7); + } + + #[test] + fn missing_stage_with_an_outstanding_receipt_denies_handoff_and_reset() { + let root = TempDir::new().unwrap(); + let destination = root.path().join("sessions.db"); + seed(&destination, 7); + let directory = InitializationDirectory::path(&destination); + let mut lease = crate::storage::try_lock_durable_directory(&directory).unwrap(); + lease.write_all(b"unresolved preparation").unwrap(); + drop(lease); + assert!(InitializationDirectory::verify_read_only_handoff(&destination).is_err()); + assert!(InitializationDirectory::recover_published(&destination).is_err()); + assert!(lock_offline_sqlite_family_for_reset(&destination).is_err()); + assert!(!directory.exists()); + let mut lease = crate::storage::try_lock_existing_durable_directory(&directory) + .unwrap() + .unwrap(); + let mut receipt = String::new(); + lease.read_to_string(&mut receipt).unwrap(); + assert_eq!(receipt, "unresolved preparation"); + assert_eq!(read_value(&destination), 7); + } + #[test] fn preparation_refuses_unclaimed_sidecar_handles_without_modifying_the_family() { let root = TempDir::new().unwrap(); @@ -738,7 +802,7 @@ mod tests { /// A competing process may rename the private directory despite the /// cooperative lease and install a symlink at the old path. Failed-start /// cleanup must preserve occupied entries in the pinned directory and never - /// the distinct valid SQLite database beneath that replacement symlink. + /// touch the distinct SQLite database beneath that replacement symlink. #[cfg(unix)] #[test] fn failed_initialization_cleanup_uses_pinned_directory_after_swap() { From 140531b756a9b3c36a9614cc54fbd2f26c037311 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 20:56:18 -0800 Subject: [PATCH 40/60] fix(storage): resume a stopped SQLite schema installation --- .../src/capability_dir.rs | 13 +- .../shard_runtime/registry/initialization.rs | 186 +++++++++++++----- .../registry/initialization/interrupted.rs | 181 +++++++++++++++++ .../src/shard_runtime/registry/ports.rs | 101 ++++++---- .../registry/tests/production_routes.rs | 141 +++++++++++++ .../src/connection/mod.rs | 10 + .../src/repository/attachment.rs | 27 ++- 7 files changed, 568 insertions(+), 91 deletions(-) create mode 100644 crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization/interrupted.rs diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 774d8c3dc7b..4beb40576e2 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -46,8 +46,19 @@ pub fn pin_parent(path: &std::path::Path) -> io::Result { } pub fn open_regular_nofollow(directory: &Dir, name: &OsStr) -> io::Result { + open_regular(directory, name, false) +} + +pub fn open_writable_regular_nofollow(directory: &Dir, name: &OsStr) -> io::Result { + open_regular(directory, name, true) +} + +fn open_regular(directory: &Dir, name: &OsStr, writable: bool) -> io::Result { let mut options = OpenOptions::new(); - options.read(true).follow(FollowSymlinks::No); + options + .read(true) + .write(writable) + .follow(FollowSymlinks::No); #[cfg(windows)] { use cap_std::fs::OpenOptionsExt; diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs index 521eabef32c..a1b4bc5c881 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs @@ -1,5 +1,5 @@ -//! A prospective SQLite store publishes its completed main last. Each staged -//! file moves once by no-replace rename; cleanup only removes an empty directory. +//! A prospective SQLite store publishes its completed main last. It moves staged +//! files by no-replace rename and retires only empty directories. //! A failed schema install therefore cannot delete a file at a reused basename. use std::ffi::OsStr; use std::fs::{self, File}; @@ -10,6 +10,9 @@ use std::path::{Path, PathBuf}; use std::time::SystemTime; use serde::{Deserialize, Serialize}; +mod interrupted; + +use interrupted::InterruptedInstall; use tracedecay_private_fs::capability_dir::{self, Dir}; use tracedecay_private_fs::framed_log::{DirectorySyncPolicy, sync_parent_directory}; use tracedecay_private_fs::{FileLease, create_private_directory, open_private_directory}; @@ -17,6 +20,14 @@ use tracedecay_private_fs::{FileLease, create_private_directory, open_private_di use crate::sqlite_read_snapshot::with_suffix; const SIDECARS: [&str; 3] = ["-wal", "-shm", "-journal"]; +const STAGED_MEMBERS: [&str; 4] = [ + "schema.sqlite", + "schema.sqlite-wal", + "schema.sqlite-shm", + "schema.sqlite-journal", +]; +pub(super) type InitializationFiles = (File, [File; 3]); + const OWNER: &str = "schema-installation.owner"; const SYNC: DirectorySyncPolicy = DirectorySyncPolicy::TolerateUnsupported; @@ -44,11 +55,28 @@ pub fn lock_offline_sqlite_family_for_reset(destination: &Path) -> io::Result, runtime_identity: u64, main: LeafIdentity, sidecars: [LeafIdentity; 3], } +fn surviving_sidecars(sidecars: [File; 3]) -> io::Result<[Option; 3]> { + let retained = sidecars.map(Some); + #[cfg(windows)] + let mut retained = retained; + #[cfg(windows)] + for slot in &mut retained { + if let Some(file) = slot.as_ref() + && tracedecay_private_fs::windows_file::deletion_pending(file)? + { + drop(slot.take()); + } + } + Ok(retained) +} + impl PreparedFamily { fn has_durable_generations(&self) -> bool { self.main.allocation.is_some() @@ -238,34 +266,10 @@ impl InitializationDirectory { ) -> io::Result<[File; 4]> { self.verify_directory_name()?; let stage = self.stage_cap()?; - #[cfg(windows)] - let sidecars = { - let mut retained = sidecars.map(Some); - for slot in &mut retained { - if let Some(file) = slot.as_ref() - && tracedecay_private_fs::windows_file::deletion_pending(file)? - { - drop(slot.take()); - } - } - retained - }; - #[cfg(not(windows))] - let sidecars = sidecars.map(Some); + let sidecars = surviving_sidecars(sidecars)?; let main = LeafIdentity::from_file(&main_file)?; let main_pin = main.pin_at(&stage, OsStr::new("schema.sqlite"), false)?; - for entry in stage.entries()? { - let name = entry?.file_name(); - if name != "schema.sqlite" - && !SIDECARS - .iter() - .any(|suffix| name == OsStr::new(&format!("schema.sqlite{suffix}"))) - { - return Err(ambiguous( - "unexpected SQLite staging member requires offline reconciliation", - )); - } - } + require_schema_members(&stage)?; main_file.sync_all()?; // WAL and rollback-journal bytes may be durable data. The original // writer has checkpointed and joined; refuse any remaining frames. @@ -311,17 +315,24 @@ impl InitializationDirectory { let (shm, shm_pin) = reserve(1, "-shm")?; let (journal, journal_pin) = reserve(2, "-journal")?; let prepared = PreparedFamily { + retry: None, runtime_identity: expected, main, sidecars: [wal, shm, journal], }; capability_dir::sync_directory(&stage)?; - self.lease.rewind()?; + self.record(prepared)?; + Ok([main_pin, wal_pin, shm_pin, journal_pin]) + } + + fn record(&mut self, prepared: PreparedFamily) -> io::Result<()> { let bytes = serde_json::to_vec(&prepared).map_err(io::Error::other)?; + self.lease.rewind()?; self.lease.write_all(&bytes)?; + self.lease.set_len(bytes.len() as u64)?; self.lease.sync_all()?; self.prepared = Some(prepared); - Ok([main_pin, wal_pin, shm_pin, journal_pin]) + Ok(()) } fn pin_members(&self, stage: &Dir, prepared: &PreparedFamily) -> io::Result> { @@ -365,6 +376,9 @@ impl InitializationDirectory { let Some(prepared) = self.prepared.as_ref() else { return Ok(None); }; + if prepared.retry.is_some() { + return Ok(None); + } // A persistent inode number alone cannot distinguish inode reuse after // process death. A filesystem without allocation generations can publish // in the original process, but must preserve interrupted preparation. @@ -421,28 +435,7 @@ impl InitializationDirectory { .directory_pin .as_ref() .ok_or_else(|| ambiguous("SQLite staging directory is no longer pinned"))?; - let named = open_private_directory(&self.directory)?; - #[cfg(unix)] - let matches = { - let left = pinned.metadata()?; - let right = named.metadata()?; - left.dev() == right.dev() && left.ino() == right.ino() - }; - #[cfg(windows)] - let matches = { - let left = tracedecay_private_fs::windows_file::information(pinned)?; - let right = tracedecay_private_fs::windows_file::information(&named)?; - left.volume_serial_number == right.volume_serial_number - && left.file_index == right.file_index - }; - #[cfg(not(any(unix, windows)))] - let matches = false; - if !matches { - return Err(ambiguous( - "SQLite staging directory no longer names the pinned directory", - )); - } - Ok(()) + verify_named_directory(pinned, &self.directory) } fn retire(&mut self) -> io::Result<()> { @@ -507,6 +500,11 @@ impl InitializationDirectory { }; let mut pins = Vec::new(); if let Some(prepared) = &stage.prepared { + if prepared.retry.is_some() { + return Err(ambiguous( + "canonical SQLite database appeared during an interrupted schema installation", + )); + } if !prepared.has_durable_generations() { return Err(ambiguous( "SQLite preparation has no durable file generation for recovery", @@ -541,6 +539,44 @@ impl Drop for InitializationDirectory { } } +fn require_schema_members(directory: &Dir) -> io::Result<()> { + for entry in directory.entries()? { + let name = entry?.file_name(); + if !STAGED_MEMBERS + .iter() + .any(|member| name == OsStr::new(member)) + { + return Err(ambiguous("unproven member in SQLite schema directory")); + } + } + Ok(()) +} + +fn verify_named_directory(pinned: &File, path: &Path) -> io::Result<()> { + let named = open_private_directory(path)?; + #[cfg(unix)] + let matches = { + let left = pinned.metadata()?; + let right = named.metadata()?; + left.dev() == right.dev() && left.ino() == right.ino() + }; + #[cfg(windows)] + let matches = { + let left = tracedecay_private_fs::windows_file::information(pinned)?; + let right = tracedecay_private_fs::windows_file::information(&named)?; + left.volume_serial_number == right.volume_serial_number + && left.file_index == right.file_index + }; + #[cfg(not(any(unix, windows)))] + let matches = false; + if !matches { + return Err(ambiguous( + "SQLite staging directory no longer names the pinned directory", + )); + } + Ok(()) +} + fn leaf(path: &Path) -> io::Result<&OsStr> { path.file_name() .ok_or_else(|| ambiguous("SQLite member has no basename")) @@ -777,6 +813,52 @@ mod tests { assert_eq!(read_value(&destination), 7); } + #[test] + fn interrupted_install_requires_its_entire_recorded_family_before_restoring_any_member() { + let root = TempDir::new().unwrap(); + let destination = root.path().join("sessions.db"); + let mut stage = InitializationDirectory::begin(destination.clone()).unwrap(); + seed(&stage.database(), 7); + let files = owned_files(&stage); + let identity = crate::db::sqlite_generation_identity(&stage.database()).unwrap(); + stage.hold_interrupted(identity, [1; 32], files).unwrap(); + let recovery = stage + .prepared + .as_ref() + .unwrap() + .retry + .as_ref() + .unwrap() + .directory(&destination); + drop(stage); + let mut retry = InitializationDirectory::begin(destination.clone()).unwrap(); + assert_eq!( + retry.resume_interrupted([2; 32]).unwrap_err().kind(), + io::ErrorKind::InvalidData + ); + drop(retry); + let wal = recovery.join("schema.sqlite-wal"); + fs::write(&wal, b"independent WAL bytes").unwrap(); + let mut retry = InitializationDirectory::begin(destination.clone()).unwrap(); + assert_eq!( + retry.resume_interrupted([1; 32]).unwrap_err().kind(), + io::ErrorKind::InvalidData + ); + assert_eq!(fs::read(&wal).unwrap(), b"independent WAL bytes"); + assert!(recovery.join("schema.sqlite").is_file()); + assert!( + retry + .stage_cap() + .unwrap() + .entries() + .unwrap() + .next() + .is_none() + ); + assert!(!destination.exists()); + assert!(lock_offline_sqlite_family_for_reset(&destination).is_err()); + } + #[test] fn preparation_refuses_unclaimed_sidecar_handles_without_modifying_the_family() { let root = TempDir::new().unwrap(); diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization/interrupted.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization/interrupted.rs new file mode 100644 index 00000000000..df6c06fffb3 --- /dev/null +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization/interrupted.rs @@ -0,0 +1,181 @@ +//! Retain a joined, interrupted schema for the next authorized installer. +use std::fs::File; +use std::io::{self, Read, Seek}; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use tracedecay_private_fs::capability_dir::{self, Dir}; +use tracedecay_private_fs::{create_private_directory, open_private_directory}; + +use super::{ + InitializationDirectory, InitializationFiles, LeafIdentity, PreparedFamily, SIDECARS, + STAGED_MEMBERS, ambiguous, leaf, present, present_at, require_schema_members, + surviving_sidecars, verify_named_directory, +}; +use crate::sqlite_read_snapshot::with_suffix; + +/// Only a joined, handled failure can deposit the original members under +/// the lease's private recovery names. No alias or backup is created. +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub(super) struct InterruptedInstall { + locator: [u8; 32], + nonce: [u8; 16], + digests: [[u8; 32]; 4], +} + +impl InterruptedInstall { + pub(super) fn directory(&self, destination: &Path) -> PathBuf { + destination.with_file_name(format!( + ".tracedecay-schema-{:032x}", + u128::from_le_bytes(self.nonce) + )) + } +} + +fn digest(mut file: &File) -> io::Result<[u8; 32]> { + file.rewind()?; + let length = file.metadata()?.len(); + let mut input = file.take(length); + let mut digest = Sha256::new(); + let mut buffer = [0u8; 8192]; + loop { + let n = input.read(&mut buffer)?; + if n == 0 { + break; + } + digest.update(&buffer[..n]); + } + if input.limit() != 0 || file.metadata()?.len() != length { + return Err(ambiguous( + "SQLite recovery member changed length during verification", + )); + } + Ok(digest.finalize().into()) +} + +impl InitializationDirectory { + pub(crate) fn hold_interrupted( + &mut self, + runtime_identity: u64, + locator: [u8; 32], + (main, sidecars): InitializationFiles, + ) -> io::Result<()> { + self.verify_directory_name()?; + let stage = self.stage_cap()?; + require_schema_members(&stage)?; + let [wal, shm, journal] = surviving_sidecars(sidecars)?; + let member = |name: &str, original: Option| -> io::Result { + if present_at(&stage, name.as_ref())? { + let file = + original.ok_or_else(|| ambiguous("SQLite member lost its original handle"))?; + LeafIdentity::from_file(&file)?.pin_at(&stage, name.as_ref(), false)?; + Ok(file) + } else { + capability_dir::create_private_file(&stage, name.as_ref()) + } + }; + LeafIdentity::from_file(&main)?.pin_at(&stage, STAGED_MEMBERS[0].as_ref(), false)?; + let files = [ + main, + member(STAGED_MEMBERS[1], wal)?, + member(STAGED_MEMBERS[2], shm)?, + member(STAGED_MEMBERS[3], journal)?, + ]; + let [main, wal, shm, journal] = files.each_ref().map(LeafIdentity::from_file); + let mut nonce = [0; 16]; + getrandom::fill(&mut nonce).map_err(|error| io::Error::other(error.to_string()))?; + let [d0, d1, d2, d3] = files.each_ref().map(digest); + let retry = InterruptedInstall { + locator, + nonce, + digests: [d0?, d1?, d2?, d3?], + }; + let recovery = retry.directory(&self.destination); + if present(&recovery)? { + return Err(ambiguous("SQLite recovery directory is already occupied")); + } + let family = PreparedFamily { + retry: Some(retry), + runtime_identity, + main: main?, + sidecars: [wal?, shm?, journal?], + }; + for file in &files { + file.sync_all()?; + } + self.record(family)?; + create_private_directory(&recovery)?; + let recovery = Dir::from_std_file(open_private_directory(&recovery)?); + for (name, original) in STAGED_MEMBERS.into_iter().zip(&files) { + capability_dir::rename_noreplace(&stage, name.as_ref(), &recovery, name.as_ref())?; + LeafIdentity::from_file(original)?.pin_at(&recovery, name.as_ref(), false)?; + } + capability_dir::sync_directory(&recovery)?; + capability_dir::sync_directory(&self.parent)?; + capability_dir::sync_directory(&stage) + } + + pub(crate) fn resume_interrupted( + &mut self, + locator: [u8; 32], + ) -> io::Result> { + let Some(family) = &self.prepared else { + return Ok(None); + }; + let Some(retry) = &family.retry else { + return Ok(None); + }; + if retry.locator != locator { + return Err(ambiguous( + "interrupted SQLite install belongs to a different store locator", + )); + } + self.require_empty()?; + for suffix in SIDECARS { + if present_at(&self.parent, leaf(&with_suffix(&self.destination, suffix))?)? { + return Err(ambiguous( + "canonical SQLite sidecar is occupied during retry", + )); + } + } + let stage = self.stage_cap()?; + let identities = [ + &family.main, + &family.sidecars[0], + &family.sidecars[1], + &family.sidecars[2], + ]; + let recovery_path = retry.directory(&self.destination); + let recovery = Dir::from_std_file(open_private_directory(&recovery_path)?); + require_schema_members(&recovery)?; + let pin = |index: usize| -> io::Result { + let name = STAGED_MEMBERS[index]; + let file = capability_dir::open_writable_regular_nofollow(&recovery, name.as_ref())?; + if LeafIdentity::from_file(&file)? != *identities[index] + || digest(&file)? != retry.digests[index] + { + return Err(ambiguous( + "interrupted SQLite member no longer matches its held contents", + )); + } + Ok(file) + }; + let files = [pin(0)?, pin(1)?, pin(2)?, pin(3)?]; + for (index, source) in STAGED_MEMBERS.into_iter().enumerate() { + capability_dir::rename_noreplace(&recovery, source.as_ref(), &stage, source.as_ref())?; + identities[index].pin_at(&stage, source.as_ref(), false)?; + } + capability_dir::sync_directory(&recovery)?; + verify_named_directory(&recovery.into_std_file(), &recovery_path)?; + self.parent.remove_dir(leaf(&recovery_path)?)?; + capability_dir::sync_directory(&stage)?; + capability_dir::sync_directory(&self.parent)?; + self.lease.set_len(0)?; + self.lease.sync_all()?; + self.prepared = None; + let [main, wal, shm, journal] = files; + Ok(Some((main, [wal, shm, journal]))) + } +} diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs index 2a8a884dc3a..1006b319851 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs @@ -5,6 +5,7 @@ use std::sync::Arc; use std::time::Duration; use asupersync::runtime::RuntimeHandle; +use sha2::{Digest, Sha256}; use tracedecay_rusqlite_runtime::reader::ReaderAcquireError; use tracedecay_rusqlite_runtime::repository::{ @@ -17,7 +18,7 @@ use tracedecay_store::{ StoreShardScopeV1, VerifiedStoreLocatorV1, }; -use super::initialization::InitializationDirectory; +use super::initialization::{InitializationDirectory, InitializationFiles}; use super::{ PhysicalRuntimeAttachment, PhysicalRuntimeSnapshot, PhysicalWriterRuntimeSnapshot, PublishedShardRuntime, StoreRuntimeAccessMode, StoreRuntimeKey, StoreRuntimeOpenMode, @@ -166,7 +167,7 @@ fn physical_failure( async fn attach_runtime( executor: &RuntimeHandle, request: &ShardRuntimeBuildRequest, - staging: Option>, + staging: Option<(Arc, Option)>, published_identity: Option, ) -> Result { let executor = executor.clone(); @@ -230,54 +231,69 @@ async fn initialize_staged_runtime( }); } let destination = request.locator.path().to_owned(); + let locator_digest = Sha256::digest( + serde_json::to_vec(request.locator.verified()) + .map_err(|error| physical_failure(OPERATION, error))?, + ) + .into(); let stage = crate::native_runtime::spawn_blocking(move || { let mut stage = InitializationDirectory::begin(destination)?; let resumed = stage.resume_prepared()?; - Ok::<_, std::io::Error>((stage, resumed)) + let interrupted = stage.resume_interrupted(locator_digest)?; + Ok::<_, std::io::Error>((stage, resumed, interrupted)) }) .await .map_err(|error| physical_failure(OPERATION, error))? .map_err(|error| physical_failure(OPERATION, error))?; - let (stage, resumed) = stage; + let (stage, resumed, interrupted) = stage; if let Some(identity) = resumed { return Ok(identity); } let stage = Arc::new(stage); let path = stage.database(); - let attachment = attach_runtime(executor, request, Some(stage.clone()), None).await?; + let attachment = + attach_runtime(executor, request, Some((stage.clone(), interrupted)), None).await?; let identity = attachment.0.opened_file_identity(); - if let Err(error) = - install_final_schema_before_publication(request, attachment.as_physical(), &path).await - { - attachment - .close() - .await - .map_err(|error| physical_failure(OPERATION, error))?; - return Err(error); - } + let installation = + install_final_schema_before_publication(request, attachment.as_physical(), &path).await; let files = crate::native_runtime::spawn_blocking(move || { let LifecyclePhysicalAttachment(physical, _stage) = attachment; - physical.close_completed_initialization() + physical.close_pinned_initialization() }) .await .and_then(|result| result) .map_err(|error| physical_failure(OPERATION, error))?; - authority - .require_active_write_scope(OPERATION) - .map_err(|error| physical_failure(OPERATION, error))?; - if request.open_cancellation.is_cancelled() { - return Err(StoreRuntimeRegistryFailure::OpenCancelled { - key: Box::new(StoreRuntimeKey::from_binding(&request.binding)), - }); - } + let installation = installation.and_then(|()| { + authority + .require_active_write_scope(OPERATION) + .map_err(|error| physical_failure(OPERATION, error))?; + if request.open_cancellation.is_cancelled() { + return Err(StoreRuntimeRegistryFailure::OpenCancelled { + key: Box::new(StoreRuntimeKey::from_binding(&request.binding)), + }); + } + Ok(()) + }); crate::native_runtime::spawn_blocking(move || { - Arc::try_unwrap(stage) - .map_err(|_| std::io::Error::other("SQLite schema workers still hold staging"))? + let mut stage = Arc::try_unwrap(stage) + .map_err(|_| physical_failure(OPERATION, "SQLite schema workers still hold staging"))?; + if let Err(error) = installation { + stage + .hold_interrupted(identity, locator_digest, files) + .map_err(|hold| { + physical_failure( + OPERATION, + format!("{error:?}; preserving interrupted schema: {hold}"), + ) + })?; + return Err(error); + } + stage .publish(identity, files) + .map_err(|error| physical_failure(OPERATION, error)) }) .await - .map_err(|error| physical_failure(OPERATION, error))? - .map_err(|error| physical_failure(OPERATION, error))?; + .map_err(|error| physical_failure(OPERATION, error))??; Ok(identity) } @@ -428,7 +444,7 @@ impl LifecycleShardRuntimeAttachment { &self, runtime: &RuntimeHandle, request: &ShardRuntimeBuildRequest, - staging: Option>, + staging: Option<(Arc, Option)>, admission: AdmissionConfigV1, ) -> Result { if staging.is_none() { @@ -443,14 +459,25 @@ impl LifecycleShardRuntimeAttachment { } .map_err(|error| physical_failure("verify initialized SQLite family", error))?; } - let attachment = if let Some(stage) = &staging { - self.repository.initialize( - runtime, - request.binding.clone(), - request.locator.verified().clone(), - stage.database(), - admission, - ) + let (stage, interrupted) = staging.unzip(); + let attachment = if let Some(stage) = &stage { + match interrupted.flatten() { + Some(files) => self.repository.resume_initialization( + runtime, + request.binding.clone(), + request.locator.verified().clone(), + stage.database(), + admission, + files, + ), + None => self.repository.initialize( + runtime, + request.binding.clone(), + request.locator.verified().clone(), + stage.database(), + admission, + ), + } } else if request.access == StoreRuntimeAccessMode::ReadOnly { self.repository.attach_read_only( request.binding.clone(), @@ -471,7 +498,7 @@ impl LifecycleShardRuntimeAttachment { operation: "attach rusqlite repository runtime", message: error.to_string(), })?; - Ok(LifecyclePhysicalAttachment(attachment, staging)) + Ok(LifecyclePhysicalAttachment(attachment, stage)) } } diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs index 11197e726db..949af24cbe9 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs @@ -620,6 +620,147 @@ async fn cancelled_schema_publication_retries_with_the_final_store() { drop(published); } +#[asupersync::test] +async fn stopped_schema_installation_resumes_committed_steps_in_a_new_process() { + use crate::ports::registered_schema; + use std::process::Command; + use tracedecay_domain::errors::TraceDecayError; + + const PATH: &str = "TRACEDECAY_TEST_SCHEMA_RETRY_DATABASE"; + const PHASE: &str = "TRACEDECAY_TEST_SCHEMA_RETRY_PHASE"; + let Some(path) = std::env::var_os(PATH) else { + let name = format!( + "{}::stopped_schema_installation_resumes_committed_steps_in_a_new_process", + module_path!().split_once("::").unwrap().1 + ); + for first in ["cancel", "failure"] { + let root = TempDir::new().unwrap(); + let path = root.path().canonicalize().unwrap().join("sessions.db"); + for phase in [first, "resume"] { + let output = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", &name, "--nocapture"]) + .env(PATH, &path) + .env(PHASE, phase) + .output() + .unwrap(); + assert!( + output.status.success() + && String::from_utf8_lossy(&output.stdout).contains("1 passed"), + "{phase}: {} {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + let residues = std::fs::read_dir(root.path()) + .unwrap() + .filter_map(|entry| { + let name = entry.unwrap().file_name(); + name.to_string_lossy() + .starts_with(".tracedecay-schema-") + .then_some(name) + }) + .collect::>(); + assert!(residues.is_empty(), "{residues:?}"); + } + return; + }; + registered_schema::register(|installation| { + Box::pin(async move { + let phase = std::env::var(PHASE).unwrap(); + if phase == "resume" { + installation.execute_batch("CREATE TABLE completed(value INTEGER); INSERT INTO completed SELECT value FROM saved;").await.unwrap(); + Ok(()) + } else { + installation + .execute_batch( + "CREATE TABLE saved(value INTEGER); INSERT INTO saved VALUES (17);", + ) + .await + .unwrap(); + if phase == "cancel" { + installation.cancellation().cancel(); + Err(TraceDecayError::store_open_cancelled("registered schema")) + } else { + let error = installation + .execute_batch("INSERT INTO missing_table VALUES (42);") + .await + .unwrap_err(); + Err(TraceDecayError::Database { + operation: "incomplete registered schema".to_owned(), + message: error.to_string(), + }) + } + } + }) + }); + let path = PathBuf::from(path); + let authority = + crate::db::DatabaseAuthority::acquire_test(&path, "retry registered schema").unwrap(); + let shard = profile_shard(); + let binding = StoreRuntimeBindingV1::new( + shard.clone(), + incarnation(), + StoreAuthorityEpochV1::new(1).unwrap(), + ); + let verified = VerifiedStoreLocatorV1::new( + shard, + incarnation(), + canonical_store_locator_digest(&path).unwrap(), + ); + let locator = RuntimeLocatorRecord::new( + StoreRuntimeKey::from_binding(&binding), + ResolvedStoreLocator::prospective(verified, path.clone()), + ); + let published = LifecycleShardRuntimePublisher + .publish(ShardRuntimeBuildRequest::new( + binding, + locator, + StoreRuntimeOpenMode::Initialize, + StoreRuntimeAccessMode::ReadWrite, + Some(authority), + CancellationToken::new(), + )) + .await; + match std::env::var(PHASE).unwrap().as_str() { + "cancel" => assert!( + matches!( + published, + Err(StoreRuntimeRegistryFailure::OpenCancelled { .. }) + ), + "{:?}", + published.err() + ), + "failure" => assert!( + matches!(published, Err(StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { ref message, .. }) if message.contains("missing_table")), + "{:?}", + published.err() + ), + "resume" => { + let runtime = published.unwrap(); + let reader = + Connection::open_with_flags(&path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY) + .unwrap(); + assert_eq!( + reader + .query_row("SELECT value FROM completed", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 17 + ); + assert_eq!( + reader + .query_row("PRAGMA integrity_check", [], |row| row.get::<_, String>(0)) + .unwrap(), + "ok" + ); + drop(runtime); + return; + } + phase => panic!("unexpected schema test phase {phase}"), + } + assert!(!path.exists()); +} + #[asupersync::test] async fn remote_node_initialization_installs_only_the_final_registered_schema() { let root = TempDir::new().unwrap(); diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs index 6304f54f12e..5d5bc531bbc 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs @@ -155,6 +155,16 @@ impl OpenedDatabaseFile { Ok(opened) } + pub(crate) fn resume_initialization( + path: &Path, + (main, sidecars): (File, [File; 3]), + ) -> Result { + let mut opened = Self::adopt(main)?; + opened.verify_current_path(path)?; + opened.created_sidecars = Some(Arc::new(Mutex::new(Some(sidecars)))); + Ok(opened) + } + /// Takes ownership of an already-open handle and records its identity. fn adopt(file: File) -> Result { let metadata = file diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs index b73ce704579..fdd94f4bb0a 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs @@ -160,6 +160,31 @@ impl RepositoryPhysicalAttachmentFactory { ) } + /// Rejoins a private, unfinished schema through the initializer's retained + /// file generations. The ordinary worker startup and admission checks apply. + pub fn resume_initialization( + &self, + runtime: &RuntimeHandle, + binding: StoreRuntimeBindingV1, + locator: VerifiedStoreLocatorV1, + path: PathBuf, + admission: AdmissionConfigV1, + files: (std::fs::File, [std::fs::File; 3]), + ) -> Result { + let opened = OpenedDatabaseFile::resume_initialization(&path, files) + .map_err(RepositoryAttachmentStartError::Identity)?; + self.attach_opened( + runtime, + binding, + locator, + path, + admission, + opened, + true, + &mut |_| {}, + ) + } + #[allow(clippy::too_many_arguments)] #[tracing::instrument(name = "rusqlite.attachment.attach", level = "trace", skip_all)] fn attach_opened( @@ -509,7 +534,7 @@ impl RepositoryRuntimePhysicalAttachment { /// Hands the exclusive original reservations to the publisher after all /// SQLite workers have exited. No raw descriptor is closed during a live /// POSIX SQLite transaction. - pub fn close_completed_initialization( + pub fn close_pinned_initialization( &self, ) -> Result<(std::fs::File, [std::fs::File; 3]), String> { self.commit_initialization()?; From df991d4cf9ec6b976a90b8b3ad7953d34ae44ed0 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 20:58:14 -0800 Subject: [PATCH 41/60] fix(sqlite): verify prepared contents before recovery publication Refs #4513. Bind recovered SQLite main bytes to the durable receipt, including same-inode foreign replacement, with a cross-process regression. --- .../shard_runtime/registry/initialization.rs | 96 ++++++++++++++++--- 1 file changed, 82 insertions(+), 14 deletions(-) diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs index 521eabef32c..d0e3c915a9d 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs @@ -10,6 +10,7 @@ use std::path::{Path, PathBuf}; use std::time::SystemTime; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use tracedecay_private_fs::capability_dir::{self, Dir}; use tracedecay_private_fs::framed_log::{DirectorySyncPolicy, sync_parent_directory}; use tracedecay_private_fs::{FileLease, create_private_directory, open_private_directory}; @@ -46,6 +47,7 @@ pub fn lock_offline_sqlite_family_for_reset(destination: &Path) -> io::Result io::Result { + let mut file = self.main.pin_at(directory, name, false)?; + if sha256_file(&mut file)? != self.main_sha256 { + return Err(ambiguous("prepared SQLite main file content changed")); + } + Ok(file) + } + fn load(file: &mut File) -> io::Result> { let length = file.metadata()?.len(); Ok(if length == 0 { @@ -132,6 +142,20 @@ impl LeafIdentity { } } +fn sha256_file(file: &mut File) -> io::Result<[u8; 32]> { + file.rewind()?; + let mut hasher = Sha256::new(); + let mut buffer = [0_u8; 16 * 1024]; + loop { + let length = file.read(&mut buffer)?; + if length == 0 { + break; + } + hasher.update(&buffer[..length]); + } + Ok(hasher.finalize().into()) +} + fn ambiguous(message: &'static str) -> io::Error { io::Error::new(io::ErrorKind::InvalidData, message) } @@ -253,7 +277,7 @@ impl InitializationDirectory { #[cfg(not(windows))] let sidecars = sidecars.map(Some); let main = LeafIdentity::from_file(&main_file)?; - let main_pin = main.pin_at(&stage, OsStr::new("schema.sqlite"), false)?; + let mut main_pin = main.pin_at(&stage, OsStr::new("schema.sqlite"), false)?; for entry in stage.entries()? { let name = entry?.file_name(); if name != "schema.sqlite" @@ -267,6 +291,7 @@ impl InitializationDirectory { } } main_file.sync_all()?; + let main_sha256 = sha256_file(&mut main_pin)?; // WAL and rollback-journal bytes may be durable data. The original // writer has checkpointed and joined; refuse any remaining frames. // Only the original, now-unused SHM index may be cleared by descriptor. @@ -313,6 +338,7 @@ impl InitializationDirectory { let prepared = PreparedFamily { runtime_identity: expected, main, + main_sha256, sidecars: [wal, shm, journal], }; capability_dir::sync_directory(&stage)?; @@ -338,11 +364,7 @@ impl InitializationDirectory { )); } } - let mut pins = vec![ - prepared - .main - .pin_at(stage, OsStr::new("schema.sqlite"), false)?, - ]; + let mut pins = vec![prepared.pin_main(stage, OsStr::new("schema.sqlite"))?]; for (suffix, identity) in SIDECARS.into_iter().zip(&prepared.sidecars) { let from = format!("reservation{suffix}"); let to = with_suffix(&self.destination, suffix); @@ -403,9 +425,7 @@ impl InitializationDirectory { leaf(&self.destination)?, )?; capability_dir::sync_directory(&self.parent)?; - prepared - .main - .pin_at(&self.parent, leaf(&self.destination)?, false)?; + prepared.pin_main(&self.parent, leaf(&self.destination)?)?; drop(stage); self.retire()?; Ok(expected) @@ -512,11 +532,7 @@ impl InitializationDirectory { "SQLite preparation has no durable file generation for recovery", )); } - pins.push( - prepared - .main - .pin_at(&stage.parent, leaf(destination)?, false)?, - ); + pins.push(prepared.pin_main(&stage.parent, leaf(destination)?)?); for (suffix, identity) in SIDECARS.into_iter().zip(&prepared.sidecars) { pins.push(identity.pin_at( &stage.parent, @@ -589,6 +605,58 @@ mod tests { ) } + #[test] + fn changed_prepared_sqlite_bytes_are_preserved_and_never_published() { + const CHILD: &str = "TRACEDECAY_SQLITE_PREPARED_SWAP"; + const FOREIGN: &str = "TRACEDECAY_SQLITE_PREPARED_FOREIGN"; + if let Some(target) = std::env::var_os(CHILD) { + let bytes = fs::read(std::env::var_os(FOREIGN).unwrap()).unwrap(); + let mut file = fs::OpenOptions::new().write(true).open(target).unwrap(); + assert_eq!(file.metadata().unwrap().len(), bytes.len() as u64); + file.rewind().unwrap(); + file.write_all(&bytes).unwrap(); + file.sync_all().unwrap(); + return; + } + + let root = TempDir::new().unwrap(); + let destination = root.path().join("sessions.db"); + let mut stage = InitializationDirectory::begin(destination.clone()).unwrap(); + seed(&stage.database(), 7); + let foreign = root.path().join("foreign.sqlite"); + seed(&foreign, 99); + let before = fs::read(stage.database()).unwrap(); + assert_eq!(before.len(), fs::metadata(&foreign).unwrap().len() as usize); + let identity = crate::db::sqlite_generation_identity(&stage.database()).unwrap(); + let files = owned_files(&stage); + stage.prepare(identity, files).unwrap(); + + let (_, module) = module_path!().split_once("::").unwrap(); + let test = format!("{module}::changed_prepared_sqlite_bytes_are_preserved_and_never_published"); + let child = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", &test]) + .env(CHILD, stage.database()) + .env(FOREIGN, &foreign) + .output() + .unwrap(); + assert!(child.status.success(), "cross-process mutation failed: {child:?}"); + assert!(LeafIdentity::from_file(&File::open(stage.database()).unwrap()).unwrap() + == stage.prepared.as_ref().unwrap().main); + assert_ne!(fs::read(stage.database()).unwrap(), before); + assert_eq!(read_value(&stage.database()), 99); + assert_eq!( + stage.publish_prepared().unwrap_err().kind(), + io::ErrorKind::InvalidData + ); + assert!(!destination.exists()); + for suffix in SIDECARS { + assert!(!with_suffix(&destination, suffix).exists()); + assert!(stage.reservation(suffix).exists()); + } + assert_eq!(read_value(&stage.database()), 99); + assert_eq!(read_value(&foreign), 99); + } + #[test] fn process_exit_during_initialization_recovers_without_publishing_partial_schema() { const CHILD: &str = "TRACEDECAY_TEST_EXIT_DURING_INITIALIZATION"; From a7f6305fd91e96f276405e9643e701de30cc2a76 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 20:59:19 -0800 Subject: [PATCH 42/60] style(sqlite): clarify prepared-content generation witness Refs #4513. Keep the new cross-process regression rustfmt-aligned. --- .../src/shard_runtime/registry/initialization.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs index d0e3c915a9d..df0d4f85a2a 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs @@ -47,6 +47,7 @@ pub fn lock_offline_sqlite_family_for_reset(destination: &Path) -> io::Result Date: Sun, 8 Nov 2026 21:00:08 -0800 Subject: [PATCH 43/60] fix(storage): recover provable interrupted SQLite transfers --- .../shard_runtime/registry/initialization.rs | 68 +++++++++++++++++ .../registry/initialization/interrupted.rs | 73 ++++++++++++++----- .../src/connection/mod.rs | 15 ++++ 3 files changed, 138 insertions(+), 18 deletions(-) diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs index a1b4bc5c881..a3d9add2b53 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs @@ -813,6 +813,74 @@ mod tests { assert_eq!(read_value(&destination), 7); } + #[test] + fn process_exit_during_interrupted_schema_transfer_preserves_or_restores_every_member() { + const DATABASE: &str = "TRACEDECAY_TEST_EXIT_DURING_RETRY_TRANSFER"; + const MOVES: &str = "TRACEDECAY_TEST_RETRY_TRANSFER_MOVES"; + if let Some(path) = std::env::var_os(DATABASE) { + let destination = PathBuf::from(path); + let mut stage = InitializationDirectory::begin(destination.clone()).unwrap(); + seed(&stage.database(), 17); + let identity = crate::db::sqlite_generation_identity(&stage.database()).unwrap(); + let files = owned_files(&stage); + stage.hold_interrupted(identity, [9; 32], files).unwrap(); + let family = stage.prepared.as_ref().unwrap(); + let recovery = family.retry.as_ref().unwrap().directory(&destination); + let moves: usize = std::env::var(MOVES).unwrap().parse().unwrap(); + for name in STAGED_MEMBERS.iter().take(moves) { + rename_noreplace(&recovery.join(name), &stage.directory.join(name)).unwrap(); + } + if moves > STAGED_MEMBERS.len() { + fs::remove_dir(&recovery).unwrap(); + } + std::process::exit(73); + } + let test = format!( + "{}::process_exit_during_interrupted_schema_transfer_preserves_or_restores_every_member", + module_path!().split_once("::").unwrap().1 + ); + for moves in [1, 3, 4, 5] { + let root = TempDir::new().unwrap(); + let destination = root.path().join("sessions.db"); + let child = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", &test]) + .env(DATABASE, &destination) + .env(MOVES, moves.to_string()) + .output() + .unwrap(); + assert_eq!(child.status.code(), Some(73), "{child:?}"); + let mut stage = InitializationDirectory::begin(destination.clone()).unwrap(); + let family = stage.prepared.as_ref().unwrap(); + let supported = std::iter::once(&family.main) + .chain(&family.sidecars) + .take(moves) + .all(|leaf| leaf.allocation.is_some()); + let recovery = family.retry.as_ref().unwrap().directory(&destination); + let before = STAGED_MEMBERS.map(|name| { + let in_stage = stage.directory.join(name); + let parked = recovery.join(name); + assert_ne!(in_stage.exists(), parked.exists()); + if in_stage.exists() { in_stage } else { parked } + }); + let bytes = before.each_ref().map(|file| fs::read(file).unwrap()); + let result = stage.resume_interrupted([9; 32]); + if supported { + let pins = result.unwrap().unwrap(); + for (name, contents) in STAGED_MEMBERS.into_iter().zip(bytes) { + assert_eq!(fs::read(stage.directory.join(name)).unwrap(), contents); + } + drop(pins); + assert!(!recovery.exists()); + } else { + assert_eq!(result.unwrap_err().kind(), io::ErrorKind::InvalidData); + for (file, contents) in before.iter().zip(bytes) { + assert_eq!(fs::read(file).unwrap(), contents); + } + } + assert!(!destination.exists()); + } + } + #[test] fn interrupted_install_requires_its_entire_recorded_family_before_restoring_any_member() { let root = TempDir::new().unwrap(); diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization/interrupted.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization/interrupted.rs index df6c06fffb3..005c18f2514 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization/interrupted.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization/interrupted.rs @@ -102,13 +102,18 @@ impl InitializationDirectory { main: main?, sidecars: [wal?, shm?, journal?], }; - for file in &files { + let mut pins = Vec::new(); + for (name, file) in STAGED_MEMBERS.into_iter().zip(&files) { file.sync_all()?; + pins.push(LeafIdentity::from_file(file)?.pin_at(&stage, name.as_ref(), false)?); } + // Windows keeps the live worker's main handle non-delete-sharing. + // Its successor pins are already open and match that original handle. + drop(files); self.record(family)?; create_private_directory(&recovery)?; let recovery = Dir::from_std_file(open_private_directory(&recovery)?); - for (name, original) in STAGED_MEMBERS.into_iter().zip(&files) { + for (name, original) in STAGED_MEMBERS.into_iter().zip(&pins) { capability_dir::rename_noreplace(&stage, name.as_ref(), &recovery, name.as_ref())?; LeafIdentity::from_file(original)?.pin_at(&recovery, name.as_ref(), false)?; } @@ -132,7 +137,7 @@ impl InitializationDirectory { "interrupted SQLite install belongs to a different store locator", )); } - self.require_empty()?; + self.verify_directory_name()?; for suffix in SIDECARS { if present_at(&self.parent, leaf(&with_suffix(&self.destination, suffix))?)? { return Err(ambiguous( @@ -148,11 +153,35 @@ impl InitializationDirectory { &family.sidecars[2], ]; let recovery_path = retry.directory(&self.destination); - let recovery = Dir::from_std_file(open_private_directory(&recovery_path)?); - require_schema_members(&recovery)?; - let pin = |index: usize| -> io::Result { - let name = STAGED_MEMBERS[index]; - let file = capability_dir::open_writable_regular_nofollow(&recovery, name.as_ref())?; + require_schema_members(&stage)?; + let recovery = match open_private_directory(&recovery_path) { + Ok(pin) => Some(Dir::from_std_file(pin)), + Err(error) if error.kind() == io::ErrorKind::NotFound => None, + Err(error) => return Err(error), + }; + if let Some(directory) = &recovery { + require_schema_members(directory)?; + } + // After a crash a file may still occupy the active SQLite namespace. + // A byte digest cannot prove an allocation (all empty WALs hash alike). + // Only the sheltered name, or a kernel generation in the active stage, + // can authorize restoration. Inspect the whole family before moving it. + let pin = |index: usize| -> io::Result<(File, bool)> { + let name = STAGED_MEMBERS[index].as_ref(); + let held = match &recovery { + Some(directory) if present_at(directory, name)? => Some(directory), + _ => None, + }; + let (directory, move_back) = match (present_at(&stage, name)?, held) { + (false, Some(directory)) => (directory, true), + (true, None) if identities[index].allocation.is_some() => (&stage, false), + _ => { + return Err(ambiguous( + "SQLite recovery member is missing, duplicated, or lacks its allocation generation", + )); + } + }; + let file = capability_dir::open_writable_regular_nofollow(directory, name)?; if LeafIdentity::from_file(&file)? != *identities[index] || digest(&file)? != retry.digests[index] { @@ -160,21 +189,29 @@ impl InitializationDirectory { "interrupted SQLite member no longer matches its held contents", )); } - Ok(file) + Ok((file, move_back)) }; let files = [pin(0)?, pin(1)?, pin(2)?, pin(3)?]; - for (index, source) in STAGED_MEMBERS.into_iter().enumerate() { - capability_dir::rename_noreplace(&recovery, source.as_ref(), &stage, source.as_ref())?; - identities[index].pin_at(&stage, source.as_ref(), false)?; + for (index, (_, move_back)) in files.iter().enumerate() { + let name = STAGED_MEMBERS[index].as_ref(); + if *move_back { + let directory = recovery + .as_ref() + .ok_or_else(|| ambiguous("SQLite recovery directory disappeared"))?; + capability_dir::rename_noreplace(directory, name, &stage, name)?; + } + identities[index].pin_at(&stage, name, false)?; + } + if let Some(directory) = recovery { + capability_dir::sync_directory(&directory)?; + verify_named_directory(&directory.into_std_file(), &recovery_path)?; + self.parent.remove_dir(leaf(&recovery_path)?)?; } - capability_dir::sync_directory(&recovery)?; - verify_named_directory(&recovery.into_std_file(), &recovery_path)?; - self.parent.remove_dir(leaf(&recovery_path)?)?; capability_dir::sync_directory(&stage)?; capability_dir::sync_directory(&self.parent)?; - self.lease.set_len(0)?; - self.lease.sync_all()?; - self.prepared = None; + // Keep the receipt until the succeeding install replaces it. A crash + // after the empty directory was removed can still prove eligible files. + let files = files.map(|(file, _)| file); let [main, wal, shm, journal] = files; Ok(Some((main, [wal, shm, journal]))) } diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs index 5d5bc531bbc..8ef8164e096 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs @@ -159,6 +159,21 @@ impl OpenedDatabaseFile { path: &Path, (main, sidecars): (File, [File; 3]), ) -> Result { + #[cfg(windows)] + let main = { + // Returning to ordinary worker semantics denies main-file renames + // for the entire active SQLite attachment. + let worker_pin = OpenOptions::new() + .read(true) + .write(true) + .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE) + .open(path) + .map_err(|_| OpenedDatabaseFileError::Open)?; + if opened_file_identity(&worker_pin)? != opened_file_identity(&main)? { + return Err(OpenedDatabaseFileError::Replaced); + } + worker_pin + }; let mut opened = Self::adopt(main)?; opened.verify_current_path(path)?; opened.created_sidecars = Some(Arc::new(Mutex::new(Some(sidecars)))); From ef03ad636afb26142ceff82e13045ab44e132aae Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:05:26 -0800 Subject: [PATCH 44/60] test(fs): retain the inode pin when allocation generations are absent --- .../src/inode_generation.rs | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/crates/tracedecay-private-fs/src/inode_generation.rs b/crates/tracedecay-private-fs/src/inode_generation.rs index b9896492923..93eac0177ba 100644 --- a/crates/tracedecay-private-fs/src/inode_generation.rs +++ b/crates/tracedecay-private-fs/src/inode_generation.rs @@ -61,44 +61,45 @@ fn generation_bits(generation: libc::c_long) -> u64 { #[cfg(test)] mod tests { + #[cfg(target_os = "linux")] + use std::fs::File; #[cfg(target_os = "linux")] use std::os::unix::ffi::OsStrExt; #[cfg(target_os = "linux")] use std::os::unix::fs::MetadataExt; - #[cfg(target_os = "linux")] - use std::path::Path; #[cfg(target_os = "linux")] use super::inode_generation; - /// Writes `path` and reads back the allocation identity the kernel keeps - /// unique: the inode number with the generation, `None` where the - /// filesystem answers `FS_IOC_GETVERSION` with no generation at all. #[cfg(target_os = "linux")] - fn write_and_identify(path: &Path) -> (u64, Option) { - std::fs::write(path, b"{}\n").unwrap(); - let file = std::fs::File::open(path).unwrap(); + fn allocation_identity(file: &File) -> (u64, Option) { ( file.metadata().unwrap().ino(), - inode_generation(&file).unwrap(), + inode_generation(file).unwrap(), ) } - /// Ext4 recycles the inode number and advances the generation; ZFS keeps - /// the generation and allocates a new inode number; tmpfs reports no - /// generation and allocates a new inode number. The pair differs on all - /// of them, and neither field is required to on its own. #[cfg(target_os = "linux")] #[test] fn replacing_a_file_changes_its_inode_allocation_identity() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("rollout.jsonl"); - let first = write_and_identify(&path); + let original = File::create(&path).unwrap(); + let first = allocation_identity(&original); + + // A reported generation survives close. Without it, filesystems such + // as overlayfs can immediately reuse a closed file's inode number. + let pin = first.1.is_none().then_some(original); std::fs::remove_file(&path).unwrap(); - let second = write_and_identify(&path); + let replacement = File::create(&path).unwrap(); + let second = allocation_identity(&replacement); + if let Some(unlinked) = pin { + assert_eq!(unlinked.metadata().unwrap().nlink(), 0); + assert_eq!(allocation_identity(&unlinked), first); + } assert_ne!( first, second, - "a replacement must not share the predecessor's identity" + "a replacement must differ from a pinned or generation-backed predecessor" ); if is_ext4(temp.path()) { assert!( From d7fe73d9586599d73b97d4673b2991025192c42e Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:29:03 -0800 Subject: [PATCH 45/60] fix(storage): retain SQLite reservations through worker startup --- .../shard_runtime/registry/initialization.rs | 2 +- .../src/shard_runtime/registry/ports.rs | 126 +++++------ .../registry/tests/production_routes.rs | 38 +++- .../src/connection/mod.rs | 51 +++-- .../src/repository/attachment.rs | 209 +++++++++++++++--- .../src/repository/mod.rs | 3 +- 6 files changed, 304 insertions(+), 125 deletions(-) diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs index ba224fce886..9933b1d56d5 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs @@ -27,7 +27,7 @@ const STAGED_MEMBERS: [&str; 4] = [ "schema.sqlite-shm", "schema.sqlite-journal", ]; -pub(super) type InitializationFiles = (File, [File; 3]); +use tracedecay_rusqlite_runtime::repository::InitializationFiles; const OWNER: &str = "schema-installation.owner"; const SYNC: DirectorySyncPolicy = DirectorySyncPolicy::TolerateUnsupported; diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs index 1006b319851..e12524afa4b 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs @@ -1,4 +1,5 @@ use std::future::Future; +use std::ops::ControlFlow; use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::Arc; @@ -18,7 +19,7 @@ use tracedecay_store::{ StoreShardScopeV1, VerifiedStoreLocatorV1, }; -use super::initialization::{InitializationDirectory, InitializationFiles}; +use super::initialization::InitializationDirectory; use super::{ PhysicalRuntimeAttachment, PhysicalRuntimeSnapshot, PhysicalWriterRuntimeSnapshot, PublishedShardRuntime, StoreRuntimeAccessMode, StoreRuntimeKey, StoreRuntimeOpenMode, @@ -144,11 +145,14 @@ impl ShardRuntimePublisher for LifecycleShardRuntimePublisher { }, )?; let published_identity = if request.locator.is_prospective() { - Some(initialize_staged_runtime(&executor, &request).await?) + Some( + initialize_staged_runtime(&executor, &request, AdmissionConfigV1::default()) + .await?, + ) } else { None }; - let attachment = attach_runtime(&executor, &request, None, published_identity).await?; + let attachment = attach_runtime(&executor, &request, published_identity).await?; publish_lifecycle_runtime(request, runtime, attachment).await }) } @@ -167,7 +171,6 @@ fn physical_failure( async fn attach_runtime( executor: &RuntimeHandle, request: &ShardRuntimeBuildRequest, - staging: Option<(Arc, Option)>, published_identity: Option, ) -> Result { let executor = executor.clone(); @@ -185,7 +188,7 @@ async fn attach_runtime( } } let attachment = LifecycleShardRuntimeAttachment::new(RepositoryPhysicalAttachmentFactory) - .attach(&executor, &request, staging, AdmissionConfigV1::default())?; + .attach(&executor, &request, AdmissionConfigV1::default())?; if published_identity .is_some_and(|expected| expected != attachment.0.opened_file_identity()) { @@ -205,9 +208,10 @@ async fn attach_runtime( .map_err(|error| physical_failure("attach SQLite runtime", error))? } -async fn initialize_staged_runtime( +pub(super) async fn initialize_staged_runtime( executor: &RuntimeHandle, request: &ShardRuntimeBuildRequest, + admission: AdmissionConfigV1, ) -> Result { const OPERATION: &str = "publish initialized SQLite family"; let authority = request @@ -236,33 +240,52 @@ async fn initialize_staged_runtime( .map_err(|error| physical_failure(OPERATION, error))?, ) .into(); - let stage = crate::native_runtime::spawn_blocking(move || { + let executor = executor.clone(); + let binding = request.binding.clone(); + let locator = request.locator.verified().clone(); + let started = crate::native_runtime::spawn_blocking(move || -> std::io::Result<_> { let mut stage = InitializationDirectory::begin(destination)?; - let resumed = stage.resume_prepared()?; + if let Some(identity) = stage.resume_prepared()? { + return Ok(ControlFlow::Break(identity)); + } let interrupted = stage.resume_interrupted(locator_digest)?; - Ok::<_, std::io::Error>((stage, resumed, interrupted)) + let attachment = RepositoryPhysicalAttachmentFactory + .start_reserved_initialization( + &executor, + binding, + locator, + stage.database(), + admission, + interrupted, + ) + .map_err(|failure| { + if let Some((identity, files)) = failure.files + && let Err(hold) = stage.hold_interrupted(identity, locator_digest, files) + { + return std::io::Error::other(format!( + "{}; preserving failed startup: {hold}", + failure.error + )); + } + std::io::Error::other(failure.error) + })?; + Ok(ControlFlow::Continue((stage, attachment))) }) .await .map_err(|error| physical_failure(OPERATION, error))? .map_err(|error| physical_failure(OPERATION, error))?; - let (stage, resumed, interrupted) = stage; - if let Some(identity) = resumed { - return Ok(identity); - } - let stage = Arc::new(stage); - let path = stage.database(); - let attachment = - attach_runtime(executor, request, Some((stage.clone(), interrupted)), None).await?; - let identity = attachment.0.opened_file_identity(); + let (mut stage, attachment) = match started { + ControlFlow::Continue(started) => started, + ControlFlow::Break(already_published) => return Ok(already_published), + }; + let identity = attachment.opened_file_identity(); let installation = - install_final_schema_before_publication(request, attachment.as_physical(), &path).await; - let files = crate::native_runtime::spawn_blocking(move || { - let LifecyclePhysicalAttachment(physical, _stage) = attachment; - physical.close_pinned_initialization() - }) - .await - .and_then(|result| result) - .map_err(|error| physical_failure(OPERATION, error))?; + install_final_schema_before_publication(request, &attachment, &stage.database()).await; + let files = + crate::native_runtime::spawn_blocking(move || attachment.close_pinned_initialization()) + .await + .and_then(|outcome| outcome) + .map_err(|error| physical_failure(OPERATION, error))?; let installation = installation.and_then(|()| { authority .require_active_write_scope(OPERATION) @@ -275,8 +298,6 @@ async fn initialize_staged_runtime( Ok(()) }); crate::native_runtime::spawn_blocking(move || { - let mut stage = Arc::try_unwrap(stage) - .map_err(|_| physical_failure(OPERATION, "SQLite schema workers still hold staging"))?; if let Err(error) = installation { stage .hold_interrupted(identity, locator_digest, files) @@ -426,10 +447,7 @@ async fn verify_final_schema_before_existing_publication( }) } -struct LifecyclePhysicalAttachment( - RepositoryRuntimePhysicalAttachment, - Option>, -); +struct LifecyclePhysicalAttachment(RepositoryRuntimePhysicalAttachment); struct LifecycleShardRuntimeAttachment { repository: RepositoryPhysicalAttachmentFactory, @@ -444,41 +462,17 @@ impl LifecycleShardRuntimeAttachment { &self, runtime: &RuntimeHandle, request: &ShardRuntimeBuildRequest, - staging: Option<(Arc, Option)>, admission: AdmissionConfigV1, ) -> Result { - if staging.is_none() { - let path = request.locator.path(); - match request.access { - StoreRuntimeAccessMode::ReadOnly => { - InitializationDirectory::verify_read_only_handoff(path) - } - StoreRuntimeAccessMode::ReadWrite => { - InitializationDirectory::recover_published(path) - } + let path = request.locator.path(); + match request.access { + StoreRuntimeAccessMode::ReadOnly => { + InitializationDirectory::verify_read_only_handoff(path) } - .map_err(|error| physical_failure("verify initialized SQLite family", error))?; + StoreRuntimeAccessMode::ReadWrite => InitializationDirectory::recover_published(path), } - let (stage, interrupted) = staging.unzip(); - let attachment = if let Some(stage) = &stage { - match interrupted.flatten() { - Some(files) => self.repository.resume_initialization( - runtime, - request.binding.clone(), - request.locator.verified().clone(), - stage.database(), - admission, - files, - ), - None => self.repository.initialize( - runtime, - request.binding.clone(), - request.locator.verified().clone(), - stage.database(), - admission, - ), - } - } else if request.access == StoreRuntimeAccessMode::ReadOnly { + .map_err(|error| physical_failure("verify initialized SQLite family", error))?; + let attachment = if request.access == StoreRuntimeAccessMode::ReadOnly { self.repository.attach_read_only( request.binding.clone(), request.locator.verified().clone(), @@ -498,7 +492,7 @@ impl LifecycleShardRuntimeAttachment { operation: "attach rusqlite repository runtime", message: error.to_string(), })?; - Ok(LifecyclePhysicalAttachment(attachment, stage)) + Ok(LifecyclePhysicalAttachment(attachment)) } } @@ -511,10 +505,6 @@ impl LifecyclePhysicalAttachment { /// and readers parks the calling thread. async fn close(self) -> Result<(), String> { crate::native_runtime::spawn_blocking(move || { - let stage = self.1; - if stage.is_some() { - return self.0.abort_initialization(); - } self.0.drain()?; self.0.close_and_join() }) diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs index 949af24cbe9..7fe780a26c8 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs @@ -636,7 +636,7 @@ async fn stopped_schema_installation_resumes_committed_steps_in_a_new_process() for first in ["cancel", "failure"] { let root = TempDir::new().unwrap(); let path = root.path().canonicalize().unwrap().join("sessions.db"); - for phase in [first, "resume"] { + for phase in ["fresh_start_error", first, "resumed_start_error", "resume"] { let output = Command::new(std::env::current_exe().unwrap()) .args(["--exact", &name, "--nocapture"]) .env(PATH, &path) @@ -711,16 +711,32 @@ async fn stopped_schema_installation_resumes_committed_steps_in_a_new_process() StoreRuntimeKey::from_binding(&binding), ResolvedStoreLocator::prospective(verified, path.clone()), ); - let published = LifecycleShardRuntimePublisher - .publish(ShardRuntimeBuildRequest::new( - binding, - locator, - StoreRuntimeOpenMode::Initialize, - StoreRuntimeAccessMode::ReadWrite, - Some(authority), - CancellationToken::new(), - )) - .await; + let request = ShardRuntimeBuildRequest::new( + binding, + locator, + StoreRuntimeOpenMode::Initialize, + StoreRuntimeAccessMode::ReadWrite, + Some(authority), + CancellationToken::new(), + ); + if matches!( + std::env::var(PHASE).unwrap().as_str(), + "fresh_start_error" | "resumed_start_error" + ) { + let mut admission = tracedecay_store::AdmissionConfigV1::default(); + admission.per_shard_queue.max_operations = 0; + let runtime = crate::native_runtime::handle().unwrap(); + let error = super::super::ports::initialize_staged_runtime(&runtime, &request, admission) + .await + .unwrap_err(); + assert!( + matches!(error, StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { ref message, .. } if message.contains("invalid writer admission")), + "{error:?}" + ); + assert!(!path.exists()); + return; + } + let published = LifecycleShardRuntimePublisher.publish(request).await; match std::env::var(PHASE).unwrap().as_str() { "cancel" => assert!( matches!( diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs index 8ef8164e096..7adc96cac24 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs @@ -157,27 +157,46 @@ impl OpenedDatabaseFile { pub(crate) fn resume_initialization( path: &Path, - (main, sidecars): (File, [File; 3]), + files: &mut Option<(File, [File; 3])>, ) -> Result { + let (original, _) = files.as_ref().ok_or(OpenedDatabaseFileError::Open)?; + let identity = opened_file_identity(original)?; #[cfg(windows)] - let main = { - // Returning to ordinary worker semantics denies main-file renames - // for the entire active SQLite attachment. - let worker_pin = OpenOptions::new() + let worker_pin = { + let pin = OpenOptions::new() .read(true) .write(true) .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE) .open(path) .map_err(|_| OpenedDatabaseFileError::Open)?; - if opened_file_identity(&worker_pin)? != opened_file_identity(&main)? { + if opened_file_identity(&pin)? != identity { return Err(OpenedDatabaseFileError::Replaced); } + pin + }; + // Keep all input descriptors with the caller until every fallible + // path and worker-pin check succeeds, including Windows sharing. + Self::verify_path_identity(path, identity)?; + let (main, sidecars) = files.take().ok_or(OpenedDatabaseFileError::Open)?; + #[cfg(windows)] + let main = { + drop(main); worker_pin }; - let mut opened = Self::adopt(main)?; - opened.verify_current_path(path)?; - opened.created_sidecars = Some(Arc::new(Mutex::new(Some(sidecars)))); - Ok(opened) + Ok(Self { + file: Arc::new(Mutex::new(Some(main))), + created_sidecars: Some(Arc::new(Mutex::new(Some(sidecars)))), + identity, + }) + } + + pub(crate) fn take_idle_initialization_files(&self) -> Option<(File, [File; 3])> { + if Arc::strong_count(&self.file) != 1 + || Arc::strong_count(self.created_sidecars.as_ref()?) != 1 + { + return None; + } + self.take_completed_files() } /// Takes ownership of an already-open handle and records its identity. @@ -325,6 +344,10 @@ impl OpenedDatabaseFile { } pub(crate) fn verify_current_path(&self, path: &Path) -> Result<(), OpenedDatabaseFileError> { + Self::verify_path_identity(path, self.identity) + } + + fn verify_path_identity(path: &Path, identity: u64) -> Result<(), OpenedDatabaseFileError> { // POSIX record locks belong to the process: closing any descriptor // for this inode releases SQLite's locks, even on another connection. // Inspect the pathname without opening a temporary descriptor. Keep @@ -340,7 +363,7 @@ impl OpenedDatabaseFile { let current = File::open(path).map_err(|_| OpenedDatabaseFileError::Open)?; opened_file_identity(¤t)? }; - if current_identity != self.identity { + if current_identity != identity { return Err(OpenedDatabaseFileError::Replaced); } Ok(()) @@ -530,7 +553,7 @@ fn sidecar_path(path: &Path, suffix: &str) -> PathBuf { } #[cfg(unix)] -fn opened_file_identity(file: &File) -> Result { +pub(crate) fn opened_file_identity(file: &File) -> Result { let metadata = file .metadata() .map_err(|_| OpenedDatabaseFileError::Inspect)?; @@ -575,7 +598,7 @@ fn sqlite_connection_has_moved(connection: &Connection) -> Result Result { +pub(crate) fn opened_file_identity(file: &File) -> Result { use std::mem::MaybeUninit; use std::os::windows::io::AsRawHandle; @@ -631,7 +654,7 @@ unsafe extern "system" { } #[cfg(not(any(unix, windows)))] -fn opened_file_identity(_file: &File) -> Result { +pub(crate) fn opened_file_identity(_file: &File) -> Result { Err(OpenedDatabaseFileError::Unsupported) } diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs index fdd94f4bb0a..65a576fb7f3 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs @@ -19,7 +19,7 @@ use tracedecay_store::{ use crate::{ CheckpointOutcome, CheckpointRequest, ExistingWriterLocator, PersistentWriter, RuntimeWriteAuthority, WriterActorError, WriterStartError, WriterState, - connection::{OpenedDatabaseFile, OpenedDatabaseFileError}, + connection::{OpenedDatabaseFile, OpenedDatabaseFileError, opened_file_identity}, exact_sql::{ExactSqlError, ExactSqlHandle}, ledger::{self, LedgerError}, reader::{ @@ -146,43 +146,42 @@ impl RepositoryPhysicalAttachmentFactory { path: PathBuf, admission: AdmissionConfigV1, ) -> Result { - let opened_database = OpenedDatabaseFile::create_new(&path) - .map_err(RepositoryAttachmentStartError::Identity)?; - self.attach_opened( - runtime, - binding, - locator, - path, - admission, - opened_database, - true, - &mut |_| {}, - ) + self.start_reserved_initialization(runtime, binding, locator, path, admission, None) + .map_err(|failure| failure.error) } - /// Rejoins a private, unfinished schema through the initializer's retained - /// file generations. The ordinary worker startup and admission checks apply. - pub fn resume_initialization( + /// Initialization owns its sole set of raw descriptors across the entire + /// worker start. On a handled failure, only a fully released worker set + /// can return those original reservations to the external publisher. + pub fn start_reserved_initialization( &self, runtime: &RuntimeHandle, binding: StoreRuntimeBindingV1, locator: VerifiedStoreLocatorV1, path: PathBuf, admission: AdmissionConfigV1, - files: (std::fs::File, [std::fs::File; 3]), - ) -> Result { - let opened = OpenedDatabaseFile::resume_initialization(&path, files) - .map_err(RepositoryAttachmentStartError::Identity)?; - self.attach_opened( - runtime, - binding, - locator, - path, - admission, - opened, - true, - &mut |_| {}, - ) + mut resumed: Option, + ) -> Result { + let opened = if resumed.is_some() { + OpenedDatabaseFile::resume_initialization(&path, &mut resumed) + } else { + OpenedDatabaseFile::create_new(&path) + } + .map_err(|error| { + RepositoryInitializationStartFailure::before_workers(error, resumed.take()) + })?; + start_with_reserved_files(opened, |worker_pin| { + self.attach_opened( + runtime, + binding, + locator, + path, + admission, + worker_pin, + true, + &mut |_| {}, + ) + }) } #[allow(clippy::too_many_arguments)] @@ -381,6 +380,52 @@ fn repository_start_failure( failure } +fn start_with_reserved_files( + opened: OpenedDatabaseFile, + start: impl FnOnce( + OpenedDatabaseFile, + ) + -> Result, +) -> Result { + let started = opened + .try_clone() + .map_err(RepositoryAttachmentStartError::Identity) + .and_then(start); + started.map_err(|error| RepositoryInitializationStartFailure { + error, + files: opened + .take_idle_initialization_files() + .map(|files| (opened.identity(), files)), + }) +} + +pub type InitializationFiles = (std::fs::File, [std::fs::File; 3]); + +#[derive(Debug)] +pub struct RepositoryInitializationStartFailure { + pub error: RepositoryAttachmentStartError, + pub files: Option<(u64, InitializationFiles)>, +} + +impl RepositoryInitializationStartFailure { + fn before_workers( + mut error: OpenedDatabaseFileError, + originals: Option, + ) -> Self { + let files = originals.and_then(|files| match opened_file_identity(&files.0) { + Ok(identity) => Some((identity, files)), + Err(inaccessible) => { + error = inaccessible; + None + } + }); + Self { + error: RepositoryAttachmentStartError::Identity(error), + files, + } + } +} + #[derive(Debug)] pub enum RepositoryAttachmentStartError { Identity(crate::connection::OpenedDatabaseFileError), @@ -1400,6 +1445,110 @@ mod tests { } } + #[test] + fn reader_start_failure_returns_idle_initialization_files_for_retry() { + use std::process::Command; + + const CHILD: &str = "TRACEDECAY_TEST_SQLITE_READER_START_LIMIT"; + if std::env::var_os(CHILD).is_none() { + let name = format!( + "{}::reader_start_failure_returns_idle_initialization_files_for_retry", + module_path!().split_once("::").unwrap().1 + ); + let result = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", &name, "--nocapture"]) + .env(CHILD, "1") + .output() + .unwrap(); + assert!( + result.status.success() + && String::from_utf8_lossy(&result.stdout).contains("1 passed"), + "{} {}", + String::from_utf8_lossy(&result.stdout), + String::from_utf8_lossy(&result.stderr) + ); + return; + } + // Bundled SQLite may default process-wide allocation tracking off. + // This isolated child has not opened SQLite yet. + // SAFETY: SQLITE_CONFIG_MEMSTATUS consumes a single C int. + let configured = + unsafe { rusqlite::ffi::sqlite3_config(rusqlite::ffi::SQLITE_CONFIG_MEMSTATUS, 1_i32) }; + assert_eq!(configured, rusqlite::ffi::SQLITE_OK); + let directory = TempDir::new().unwrap(); + let path = directory.path().join("sessions.db"); + let binding = binding(); + let runtime = native_runtime(); + let files = OpenedDatabaseFile::create_new(&path).unwrap(); + let factory = RepositoryPhysicalAttachmentFactory; + let outcome = start_with_reserved_files(files, |worker_pin| { + factory.attach_opened( + &runtime, + binding.clone(), + locator(&binding), + path.clone(), + AdmissionConfigV1::default(), + worker_pin, + true, + &mut |stage| { + // SQLite's process-wide supported memory ceiling makes the + // next reader open report SQLITE_NOMEM. The writer has + // already reported successful startup in this isolated process. + if stage == AttachmentWorkerStartStage::BeforeReaders { + // SAFETY: sqlite3_hard_heap_limit64 takes no pointers. + unsafe { + rusqlite::ffi::sqlite3_hard_heap_limit64(1); + } + } else if stage == AttachmentWorkerStartStage::AfterReaders { + // SAFETY: zero disables the limit before worker shutdown. + unsafe { + rusqlite::ffi::sqlite3_hard_heap_limit64(0); + } + } + }, + ) + }); + let failure = match outcome { + Ok(_) => panic!("reader started under exhausted SQLite budget"), + Err(error) => error, + }; + assert!( + matches!(failure.error, RepositoryAttachmentStartError::Reader(_)), + "{}", + failure.error + ); + let mut cause: &(dyn Error + 'static) = &failure.error; + while let Some(next) = cause.source() { + if cause.is::() { + break; + } + cause = next; + } + assert_eq!( + cause + .downcast_ref::() + .and_then(rusqlite::Error::sqlite_error_code), + Some(rusqlite::ErrorCode::OutOfMemory) + ); + let (identity, files) = failure + .files + .expect("writer and readers returned all original file handles"); + let reopened = factory + .start_reserved_initialization( + &runtime, + binding.clone(), + locator(&binding), + path.clone(), + AdmissionConfigV1::default(), + Some(files), + ) + .unwrap(); + assert!(reopened.snapshot().healthy); + assert_eq!(reopened.opened_file_identity(), identity); + reopened.drain().unwrap(); + reopened.close_and_join().unwrap(); + } + #[test] fn failed_initialization_preserves_created_database_and_sidecars() { let directory = TempDir::new().unwrap(); diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/mod.rs b/crates/tracedecay-rusqlite-runtime/src/repository/mod.rs index fd0794fc386..6133e464cac 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/mod.rs @@ -46,7 +46,8 @@ use crate::StorageOperationExecutor; use crate::operation::StorageOperationError; pub use attachment::{ - RepositoryAttachmentStartError, RepositoryDispatchError, RepositoryPhysicalAttachmentFactory, + InitializationFiles, RepositoryAttachmentStartError, RepositoryDispatchError, + RepositoryInitializationStartFailure, RepositoryPhysicalAttachmentFactory, RepositoryRuntimePhysicalAttachment, RepositoryRuntimePhysicalSnapshot, RepositoryWriterRuntimeSnapshot, }; From 622aa10ec60061ecd5309001997b17d866716673 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:38:20 -0800 Subject: [PATCH 46/60] simplify(storage): use one physical initialization entry point --- .../src/repository/attachment.rs | 38 +++++++++---------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs index 65a576fb7f3..a4168dac5c5 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs @@ -137,22 +137,10 @@ impl RepositoryPhysicalAttachmentFactory { ) } - #[tracing::instrument(name = "rusqlite.attachment.initialize", level = "trace", skip_all)] - pub fn initialize( - &self, - runtime: &RuntimeHandle, - binding: StoreRuntimeBindingV1, - locator: VerifiedStoreLocatorV1, - path: PathBuf, - admission: AdmissionConfigV1, - ) -> Result { - self.start_reserved_initialization(runtime, binding, locator, path, admission, None) - .map_err(|failure| failure.error) - } - /// Initialization owns its sole set of raw descriptors across the entire /// worker start. On a handled failure, only a fully released worker set /// can return those original reservations to the external publisher. + #[tracing::instrument(name = "rusqlite.attachment.initialize", level = "trace", skip_all)] pub fn start_reserved_initialization( &self, runtime: &RuntimeHandle, @@ -1152,12 +1140,13 @@ mod tests { let path = directory.path().join("aborted.sqlite3"); let binding = binding(); let attachment = RepositoryPhysicalAttachmentFactory - .initialize( + .start_reserved_initialization( &native_runtime(), binding.clone(), locator(&binding), path.clone(), AdmissionConfigV1::default(), + None, ) .unwrap(); let held_pin = attachment @@ -1227,12 +1216,13 @@ mod tests { let path = directory.path().join("aborted.sqlite3"); let binding = binding(); let attachment = RepositoryPhysicalAttachmentFactory - .initialize( + .start_reserved_initialization( &native_runtime(), binding.clone(), locator(&binding), path.clone(), AdmissionConfigV1::default(), + None, ) .unwrap(); let held_pin = attachment @@ -1300,12 +1290,13 @@ mod tests { let path = directory.path().join("lost-initialization.sqlite3"); let binding = binding(); let attachment = RepositoryPhysicalAttachmentFactory - .initialize( + .start_reserved_initialization( &native_runtime(), binding.clone(), locator(&binding), path.clone(), AdmissionConfigV1::default(), + None, ) .unwrap(); let held_pin = attachment @@ -1399,18 +1390,22 @@ mod tests { let sentinel = format!("unowned {suffix} content"); fs::write(&sidecar, sentinel.as_bytes()).unwrap(); - let initialized = RepositoryPhysicalAttachmentFactory.initialize( + let initialized = RepositoryPhysicalAttachmentFactory.start_reserved_initialization( &native_runtime(), binding.clone(), locator(&binding), path.clone(), AdmissionConfigV1::default(), + None, ); assert!(matches!( initialized, - Err(RepositoryAttachmentStartError::Identity( - OpenedDatabaseFileError::Reserve - )) + Err(RepositoryInitializationStartFailure { + error: RepositoryAttachmentStartError::Identity( + OpenedDatabaseFileError::Reserve + ), + files: None, + }) )); assert_eq!(fs::metadata(&path).unwrap().len(), 0); assert_eq!(fs::read(&sidecar).unwrap(), sentinel.as_bytes()); @@ -1805,12 +1800,13 @@ mod tests { let created_path = directory.path().join("created.sqlite3"); let created = RepositoryPhysicalAttachmentFactory - .initialize( + .start_reserved_initialization( &native_runtime(), binding.clone(), locator(&binding), created_path.clone(), AdmissionConfigV1::default(), + None, ) .unwrap(); assert!( From 90f5e7a39c65f62a4e1df276ca055d366ec680ad Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:18:28 -0800 Subject: [PATCH 47/60] fix(storage): preserve other writers when Darwin attachments close --- .../src/connection/mod.rs | 93 +++++++++++++----- .../src/connection/path_identity_tests.rs | 7 +- .../src/repository/attachment.rs | 95 +++++++++++++++++++ 3 files changed, 170 insertions(+), 25 deletions(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs index 7adc96cac24..b8b36a52926 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs @@ -70,10 +70,10 @@ const RETAINED_WAL_BYTES: i64 = WAL_SOFT_LIMIT_BYTES as i64; /// pathname stat. #[derive(Debug)] pub(crate) struct OpenedDatabaseFile { - /// One descriptor shared by every attachment worker. Duplicating this - /// descriptor and closing a clone while SQLite holds POSIX locks would - /// release locks belonging to other live connections to the same inode. - file: Arc>>, + /// Share one identity pin across workers. A private initialization file + /// stays pinned until workers exit. Existing stores use a lock-neutral + /// descriptor or give the SQLite VFS ownership of the descriptor. + file: Arc>>, // Exclusive reservations for a *new* basename. Hold the descriptors until // all SQLite workers exit: closing an unrelated fd during a live SQLite // transaction can release process-owned POSIX locks. @@ -81,7 +81,41 @@ pub(crate) struct OpenedDatabaseFile { identity: u64, } +#[derive(Debug)] +enum DatabasePin { + File(File), + // The SQLite VFS defers the actual close while another local connection + // holds POSIX locks on this inode. A raw O_RDONLY fd cannot do that. + #[cfg(all(unix, not(any(target_os = "linux", target_os = "android"))))] + Sqlite(Connection), +} + impl OpenedDatabaseFile { + #[cfg(all(unix, not(any(target_os = "linux", target_os = "android"))))] + pub(crate) fn pin(path: &Path) -> Result { + let identity = live_path_identity(path)?; + // Opening the main file alone pins its inode without starting a read + // transaction, inspecting the schema, or opening unverified sidecars. + let connection = Connection::open_with_flags( + path, + OpenFlags::SQLITE_OPEN_READ_ONLY + | OpenFlags::SQLITE_OPEN_NO_MUTEX + | OpenFlags::SQLITE_OPEN_PRIVATE_CACHE, + ) + .map_err(|_| OpenedDatabaseFileError::Open)?; + match sqlite_connection_has_moved(&connection)? { + Some(false) => Self::verify_path_identity(path, identity)?, + Some(true) => return Err(OpenedDatabaseFileError::Replaced), + None => return Err(OpenedDatabaseFileError::Inspect), + } + Ok(Self { + file: Arc::new(Mutex::new(Some(DatabasePin::Sqlite(connection)))), + created_sidecars: None, + identity, + }) + } + + #[cfg(any(not(unix), target_os = "linux", target_os = "android"))] pub(crate) fn pin(path: &Path) -> Result { let file = open_pinned_database(path).map_err(|_| OpenedDatabaseFileError::Open)?; Self::adopt(file) @@ -110,10 +144,12 @@ impl OpenedDatabaseFile { // locks held by another live SQLite connection in this process. // An immutable SQLite reader uses the VFS's lock-aware lifetime and // never opens or mutates the unverified WAL/SHM/journal sidecars. + let pinned = Self::pin(path)?; let inspection = open_immutable_reader(path).map_err(|_| OpenedDatabaseFileError::InvalidHeader)?; let initialized = has_store_schema(&inspection).map_err(|_| OpenedDatabaseFileError::InvalidHeader)?; + pinned.verify_connection(&inspection, path)?; drop(inspection); if !initialized { let sidecar = |suffix| match std::fs::symlink_metadata(sidecar_path(path, suffix)) { @@ -132,7 +168,8 @@ impl OpenedDatabaseFile { _ => return Err(OpenedDatabaseFileError::IncompleteInitialization), } } - Self::pin(path) + pinned.verify_current_path(path)?; + Ok(pinned) } pub(crate) fn create_new(path: &Path) -> Result { @@ -184,7 +221,7 @@ impl OpenedDatabaseFile { worker_pin }; Ok(Self { - file: Arc::new(Mutex::new(Some(main))), + file: Arc::new(Mutex::new(Some(DatabasePin::File(main)))), created_sidecars: Some(Arc::new(Mutex::new(Some(sidecars)))), identity, }) @@ -209,7 +246,7 @@ impl OpenedDatabaseFile { } let identity = opened_file_identity(&file)?; Ok(Self { - file: Arc::new(Mutex::new(Some(file))), + file: Arc::new(Mutex::new(Some(DatabasePin::File(file)))), created_sidecars: None, identity, }) @@ -242,22 +279,26 @@ impl OpenedDatabaseFile { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .take()?; - Some((main, sidecars)) + match main { + DatabasePin::File(main) => Some((main, sidecars)), + #[cfg(all(unix, not(any(target_os = "linux", target_os = "android"))))] + DatabasePin::Sqlite(_) => None, + } } pub(crate) const fn identity(&self) -> u64 { self.identity } - fn with_file( + fn with_pin( &self, - use_file: impl FnOnce(&File) -> T, + use_pin: impl FnOnce(&DatabasePin) -> T, ) -> Result { self.file .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .as_ref() - .map(use_file) + .map(use_pin) .ok_or(OpenedDatabaseFileError::Replaced) } @@ -265,7 +306,7 @@ impl OpenedDatabaseFile { // Share the *same* raw descriptor instead of dup(2): closing any // duplicate descriptor after a SQLite connection acquires POSIX locks // can silently release that connection's locks process-wide. - self.with_file(|_| ())?; + self.with_pin(|_| ())?; Ok(Self { file: Arc::clone(&self.file), created_sidecars: self.created_sidecars.as_ref().map(Arc::clone), @@ -280,7 +321,7 @@ impl OpenedDatabaseFile { ) -> Result { use std::os::unix::io::AsRawFd; - let descriptor = self.with_file(|file| file.as_raw_fd())?; + let descriptor = self.with_pin(|DatabasePin::File(file)| file.as_raw_fd())?; Ok(PathBuf::from(format!("/proc/self/fd/{descriptor}"))) } @@ -376,7 +417,13 @@ impl OpenedDatabaseFile { use std::os::unix::fs::MetadataExt; let unlinked = self - .with_file(|file| file.metadata().is_ok_and(|pinned| pinned.nlink() == 0)) + .with_pin(|pin| match pin { + DatabasePin::File(file) => file.metadata().is_ok_and(|pinned| pinned.nlink() == 0), + #[cfg(all(unix, not(any(target_os = "linux", target_os = "android"))))] + DatabasePin::Sqlite(connection) => { + sqlite_connection_has_moved(connection) == Ok(Some(true)) + } + }) .unwrap_or(true); if unlinked { return true; @@ -405,7 +452,7 @@ impl OpenedDatabaseFile { // HAS_MOVED below still observes the SQLite handle's different inode. self.verify_current_path(canonical_path)?; #[cfg(unix)] - if sqlite_connection_has_moved(_connection)? { + if sqlite_connection_has_moved(_connection)? == Some(true) { return Err(OpenedDatabaseFileError::Replaced); } #[cfg(unix)] @@ -490,7 +537,7 @@ fn open_pinned_database(path: &Path) -> io::Result { .open(path) } -#[cfg(not(any(windows, target_os = "linux", target_os = "android")))] +#[cfg(not(any(windows, unix)))] fn open_pinned_database(path: &Path) -> io::Result { File::open(path) } @@ -574,7 +621,9 @@ fn metadata_identity(metadata: &std::fs::Metadata) -> u64 { } #[cfg(unix)] -fn sqlite_connection_has_moved(connection: &Connection) -> Result { +fn sqlite_connection_has_moved( + connection: &Connection, +) -> Result, OpenedDatabaseFileError> { let database_name = CString::new("main").expect("static SQLite database name"); let mut moved = 0_i32; // SAFETY: `connection` owns a live SQLite handle, `database_name` is a @@ -589,10 +638,10 @@ fn sqlite_connection_has_moved(connection: &Connection) -> Result Ok(moved != 0), - // VFS implementations predating HAS_MOVED report NOTFOUND. The - // pinned dev/inode check remains authoritative in that case. - rusqlite::ffi::SQLITE_NOTFOUND => Ok(false), + rusqlite::ffi::SQLITE_OK => Ok(Some(moved != 0)), + // Callers establishing identity through SQLite must have a VFS + // answer. A worker with an independent inode pin can retain it. + rusqlite::ffi::SQLITE_NOTFOUND => Ok(None), _ => Err(OpenedDatabaseFileError::Inspect), } } @@ -1532,7 +1581,7 @@ fn policy(stage: &'static str, source: rusqlite::Error) -> ConnectionPolicyError } #[cfg(all(test, unix))] -mod path_identity_tests; +pub(crate) mod path_identity_tests; #[cfg(test)] mod tests; diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs index 9ca712d87d0..6f21308841e 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs @@ -21,7 +21,7 @@ const PROBE_COMPLETE: &str = "tracedecay-path-identity-probe-complete"; // shares inode/lock bookkeeping within a process and can hide a lost OS lock. // Execute only the child test in a fresh process, with a marker to reject a // successful test harness invocation that accidentally selected zero tests. -fn probe_external_connection(path: &Path, operation: &str, blocked: bool) { +pub(crate) fn probe_external_connection(path: &Path, operation: &str, blocked: bool) { probe_external_connection_expectation( path, operation, @@ -585,7 +585,7 @@ fn probe_reserved_lock(path: &Path, blocked: bool) { /// Opening an already valid database for attachment must not close an /// unrelated raw descriptor and discard a live writer's POSIX RESERVED lock. -/// The pin returned by admission outlives the writer, just as in production. +/// Closing one attachment must preserve the independent writer’s transaction. #[test] fn valid_existing_store_admission_preserves_other_writer_locks() { let temp = TempDir::new().unwrap(); @@ -607,9 +607,10 @@ fn valid_existing_store_admission_preserves_other_writer_locks() { assert_eq!(opened.identity(), live_path_identity(&path).unwrap()); probe_reserved_lock(&path, true); + opened.release_after_workers_exit(); + probe_reserved_lock(&path, true); writer.execute_batch("COMMIT").unwrap(); drop(writer); - opened.release_after_workers_exit(); probe_reserved_lock(&path, false); assert_committed_data(&path); } diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs index a4168dac5c5..a90e6ee2d3d 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs @@ -1735,6 +1735,101 @@ mod tests { .unwrap() } + #[cfg(unix)] + #[test] + fn existing_physical_attachments_preserve_an_independent_writers_locks() { + use crate::connection::path_identity_tests::probe_external_connection; + + let directory = TempDir::new().unwrap(); + let path = directory.path().join("sessions.db"); + let writer = rusqlite::Connection::open(&path).unwrap(); + writer + .execute_batch( + "PRAGMA journal_mode=DELETE; + CREATE TABLE sample(value INTEGER); + INSERT INTO sample VALUES(3); + BEGIN IMMEDIATE; + UPDATE sample SET value=7", + ) + .unwrap(); + let binding = binding(); + let factory = RepositoryPhysicalAttachmentFactory; + let read_only = factory + .attach_read_only( + binding.clone(), + locator(&binding), + path.clone(), + AdmissionConfigV1::default(), + ) + .unwrap(); + probe_external_connection(&path, "write", true); + read_only.drain().unwrap(); + read_only.close_and_join().unwrap(); + probe_external_connection(&path, "write", true); + + let mut invalid = AdmissionConfigV1::default(); + invalid.readers.min_per_hot_shard = 0; + assert!(matches!( + factory.attach_read_only(binding.clone(), locator(&binding), path.clone(), invalid), + Err(RepositoryAttachmentStartError::Reader( + ReaderStartError::InvalidReaderBudget(_) + )) + )); + probe_external_connection(&path, "write", true); + writer.execute_batch("COMMIT").unwrap(); + probe_external_connection(&path, "write", false); + + // Dropping the route alias exercises the real writer's lost-file + // notification while an independent connection keeps the DB open + // through its physical pathname, with its WAL family intact. + let route = directory.path().join("route.db"); + std::os::unix::fs::symlink(&path, &route).unwrap(); + let writable = factory + .attach( + &native_runtime(), + binding.clone(), + locator(&binding), + route.clone(), + AdmissionConfigV1::default(), + ) + .unwrap(); + writer + .execute_batch("BEGIN IMMEDIATE; UPDATE sample SET value=11") + .unwrap(); + probe_external_connection(&path, "write", true); + fs::remove_file(route).unwrap(); + let deadline = + std::time::Instant::now() + crate::FILE_PROBE_INTERVAL + Duration::from_secs(10); + while !(writable.file_lost() && writable.lost_file_closed()) { + assert!( + std::time::Instant::now() < deadline, + "physical attachment did not close the lost route" + ); + thread::sleep(Duration::from_millis(10)); + } + writable.drain().unwrap(); + writable.close_and_join().unwrap(); + probe_external_connection(&path, "write", true); + probe_external_connection(&path, "journal", true); + writer.execute_batch("ROLLBACK").unwrap(); + drop(writer); + probe_external_connection(&path, "write", false); + probe_external_connection(&path, "journal", false); + let checked = rusqlite::Connection::open(&path).unwrap(); + assert_eq!( + checked + .query_row("SELECT value FROM sample", [], |row| row.get::<_, i64>(0)) + .unwrap(), + 7 + ); + assert_eq!( + checked + .query_row("PRAGMA integrity_check", [], |row| row.get::<_, String>(0)) + .unwrap(), + "ok" + ); + } + /// The attachment's original identity descriptor must not be dropped /// during startup or initialization commit while SQLite workers are live. /// On Unix, closing it can release the process's unrelated SQLite locks. From 10e09e65f1f4b389cbed4aecc4197245712d2045 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:23:30 -0800 Subject: [PATCH 48/60] style(storage): group the Unix attachment test import --- .../tracedecay-rusqlite-runtime/src/repository/attachment.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs index a90e6ee2d3d..18e52563f48 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs @@ -1069,6 +1069,8 @@ mod tests { use tracedecay_domain::LocatorDigest; use tracedecay_store::{AdmissionConfigV1, StoreIncarnationV1}; + #[cfg(unix)] + use crate::connection::path_identity_tests::probe_external_connection; use crate::exact_sql::{ExactSqlError, ExactSqlStatement, ExactSqlValue}; use asupersync::channel::watch; @@ -1738,8 +1740,6 @@ mod tests { #[cfg(unix)] #[test] fn existing_physical_attachments_preserve_an_independent_writers_locks() { - use crate::connection::path_identity_tests::probe_external_connection; - let directory = TempDir::new().unwrap(); let path = directory.path().join("sessions.db"); let writer = rusqlite::Connection::open(&path).unwrap(); From b3c9629564ce1aa81e0a96cbd7c76def57a35785 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 04:28:49 -0800 Subject: [PATCH 49/60] test(sqlite): include reserved journals in reset journeys Refs #4513. The prospective SQLite initializer now owns empty -journal reservations as well as WAL/SHM. Require the scoped reset tests to account for those exact owned files and retire the journal in the shipped-store fixture before attempting a new installation. The hosted Linux fleet at b6c7cd3 reproduced the old assertions and leftover journal in the test fixture; these assertions remain strict. --- .../tests/transcript_ingest_suite/projector_upgrade.rs | 2 +- .../stale_profile_authority_reset.rs | 1 + .../stale_project_sessions_census.rs | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay/tests/transcript_ingest_suite/projector_upgrade.rs b/crates/tracedecay/tests/transcript_ingest_suite/projector_upgrade.rs index a5c905f4a69..d77c053b177 100644 --- a/crates/tracedecay/tests/transcript_ingest_suite/projector_upgrade.rs +++ b/crates/tracedecay/tests/transcript_ingest_suite/projector_upgrade.rs @@ -101,7 +101,7 @@ fn shape_store_as_shipped(db_path: &Path) { /// so the next open recreates it empty. fn reset_store(db_path: &Path) { std::fs::remove_file(db_path).unwrap(); - for suffix in ["-wal", "-shm"] { + for suffix in ["-wal", "-shm", "-journal"] { let mut sidecar = db_path.as_os_str().to_owned(); sidecar.push(suffix); match std::fs::remove_file(sidecar) { diff --git a/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/stale_profile_authority_reset.rs b/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/stale_profile_authority_reset.rs index 741912ffad7..05f8370032c 100644 --- a/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/stale_profile_authority_reset.rs +++ b/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/stale_profile_authority_reset.rs @@ -201,6 +201,7 @@ fn released_profile_authority_resets_alone_and_projects_register_again() { removed, [ &PathBuf::from("global.db"), + &PathBuf::from("global.db-journal"), &PathBuf::from("global.db-shm"), &PathBuf::from("global.db-wal") ], diff --git a/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/stale_project_sessions_census.rs b/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/stale_project_sessions_census.rs index e34cd6c2303..74fb1033d84 100644 --- a/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/stale_project_sessions_census.rs +++ b/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/stale_project_sessions_census.rs @@ -114,6 +114,7 @@ fn one_scoped_reset_clears_the_profile_authority_and_every_project_sessions_stor .collect(); let mut expected_removed = vec![ PathBuf::from("global.db"), + PathBuf::from("global.db-journal"), PathBuf::from("global.db-shm"), PathBuf::from("global.db-wal"), ]; @@ -122,6 +123,7 @@ fn one_scoped_reset_clears_the_profile_authority_and_every_project_sessions_stor expected_removed.extend([ store.join(".sessions.db.host-admission/meta.json"), store.join("sessions.db"), + store.join("sessions.db-journal"), store.join("sessions.db-shm"), store.join("sessions.db-wal"), store.join("sessions.db.delivery-settlement-spool-v1/writer.v1.lock"), From 185181fb2b78eb91905afe47034b91d83c197b7f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 05:29:25 -0800 Subject: [PATCH 50/60] fix(sqlite): await writer capacity to finalize prospective schema Refs #4513. Treat prospective schema finalization as a serialized writer barrier rather than a best-effort try_send. Deterministic bounded-channel regressions cover backpressure, FIFO admission, completion replies, and closed-writer refusal without relaxing ordinary SQL admission. --- .../src/exact_sql/mod.rs | 7 +- .../src/exact_sql/tests/async_dispatch.rs | 80 +++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/exact_sql/mod.rs b/crates/tracedecay-rusqlite-runtime/src/exact_sql/mod.rs index 45f051b1138..92a02a2736e 100644 --- a/crates/tracedecay-rusqlite-runtime/src/exact_sql/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/exact_sql/mod.rs @@ -623,10 +623,15 @@ impl ExactSqlHandle { .clone() .ok_or(ExactSqlError::AuthorityMismatch)?; let (reply, response) = async_channel::bounded(1); + let cx = crate::native_runtime::cx().ok_or(ExactSqlError::WriterUnavailable)?; + // Schema publication is a serialized writer barrier, not an ordinary + // best-effort request. A busy bounded channel must not strand a + // prepared SQLite family when its existing work can still drain. self.writer .as_ref() .ok_or(ExactSqlError::WriterUnavailable)? - .try_send(WriterCommand::FinishInitialSchema { reply, authority }) + .send(&cx, WriterCommand::FinishInitialSchema { reply, authority }) + .await .map_err(map_writer_send_error)?; response .recv() diff --git a/crates/tracedecay-rusqlite-runtime/src/exact_sql/tests/async_dispatch.rs b/crates/tracedecay-rusqlite-runtime/src/exact_sql/tests/async_dispatch.rs index 2decbe4769e..5bea1d358e9 100644 --- a/crates/tracedecay-rusqlite-runtime/src/exact_sql/tests/async_dispatch.rs +++ b/crates/tracedecay-rusqlite-runtime/src/exact_sql/tests/async_dispatch.rs @@ -62,6 +62,86 @@ async fn poll_once(mut future: Pin<&mut F>) -> Poll { poll_fn(|context| Poll::Ready(future.as_mut().poll(context))).await } +#[test] +fn schema_finalization_waits_for_writer_capacity_without_losing_its_reply() { + current_thread().block_on(async { + let fixture = fixture('a', 'a'); + let mut handle = ExactSqlHandle::attach(&fixture.writer, &fixture.readers) + .unwrap() + .with_write_authority(Arc::new(AtomicWriteAuthority(Arc::new(AtomicBool::new( + true, + ))))) + .unwrap(); + // Occupy the only slot before polling the real finalization future. + // This reproduces a full exact-SQL channel without scheduling races. + let (sender, mut receiver) = actor_mpsc::channel(1); + let (first_reply, _first_response) = async_channel::bounded(1); + let authority = handle.write_authority.clone().unwrap(); + assert!( + sender + .try_send(WriterCommand::FinishInitialSchema { + reply: first_reply, + authority, + }) + .is_ok() + ); + handle.writer = Some(sender); + + let mut finishing = Box::pin(handle.finish_initial_schema()); + assert!(poll_once(finishing.as_mut()).await.is_pending()); + let first = receiver.try_recv().unwrap(); + match first { + WriterCommand::FinishInitialSchema { reply, .. } => { + assert!(reply.try_send(Ok(())).is_ok()); + } + _ => panic!("the occupied slot must hold the first schema barrier"), + } + + // Releasing capacity admits the queued finalization in FIFO order; + // success still requires the writer's own completion reply. + assert!(poll_once(finishing.as_mut()).await.is_pending()); + let second = receiver.try_recv().unwrap(); + match second { + WriterCommand::FinishInitialSchema { reply, .. } => { + assert!(reply.try_send(Ok(())).is_ok()); + } + _ => panic!("the deferred schema barrier was not admitted"), + } + finishing.await.unwrap(); + }); +} + +#[test] +fn schema_finalization_refuses_a_closed_writer_after_capacity_wait() { + current_thread().block_on(async { + let fixture = fixture('a', 'a'); + let mut handle = ExactSqlHandle::attach(&fixture.writer, &fixture.readers) + .unwrap() + .with_write_authority(Arc::new(AtomicWriteAuthority(Arc::new(AtomicBool::new( + true, + ))))) + .unwrap(); + let (sender, receiver) = actor_mpsc::channel(1); + let (reply, _response) = async_channel::bounded(1); + assert!( + sender + .try_send(WriterCommand::FinishInitialSchema { + reply, + authority: handle.write_authority.clone().unwrap(), + }) + .is_ok() + ); + handle.writer = Some(sender); + let mut finishing = Box::pin(handle.finish_initial_schema()); + assert!(poll_once(finishing.as_mut()).await.is_pending()); + drop(receiver); + assert!(matches!( + finishing.await, + Err(ExactSqlError::WriterUnavailable) + )); + }); +} + #[test] fn synchronous_adapters_remain_safe_inside_current_thread_tokio() { current_thread().block_on(async { From 45358967af30f76b252ed834435fd109d502bbf7 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:24:21 -0800 Subject: [PATCH 51/60] test(sqlite): qualify unpublished stores through real admission --- .../lcm_schema/temporal_catalog/admission.rs | 55 +++++++++-- .../src/tests/schema_install_cancellation.rs | 96 ++++++++++++------- 2 files changed, 105 insertions(+), 46 deletions(-) diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs index af0c127c645..41756174d9d 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs @@ -103,15 +103,31 @@ async fn temporal_schema_accepts_only_fresh_or_exact_final_stores() { drop(conn); drop(raw_db); - let db = open_global_db(&db_path) + // Removing an application table does not make a previously opened SQLite + // namespace freshly owned. Existing sidecars remain unclaimed. + let error = match open_global_db(&db_path).await { + Ok(_) => panic!("unpublished existing SQLite must not be reinitialized"), + Err(error) => error, + }; + assert!( + error.to_string().contains("sidecar ownership is ambiguous"), + "unpublished SQLite sidecars must remain unclaimed: {error}" + ); + assert!( + !table_exists(&db_path, "tracedecay_schema_version").await, + "refusal must not publish a schema marker into an unowned database" + ); + + let fresh_path = tmp.path().join(".tracedecay").join("fresh.db"); + let db = open_global_db(&fresh_path) .await - .expect("a fresh store should receive the final temporal schema"); + .expect("a new prospective store should receive the final temporal schema"); drop(db); - let initial_catalog = temporal_schema_object_catalog(&db_path).await; - let initial_version = temporal_schema_version(&db_path).await; + let initial_catalog = temporal_schema_object_catalog(&fresh_path).await; + let initial_version = temporal_schema_version(&fresh_path).await; let restart_path = tmp.path().join(".tracedecay").join("restart.db"); - copy_database_for_temporal_restart(&db_path, &restart_path).await; + copy_database_for_temporal_restart(&fresh_path, &restart_path).await; let reopened = open_global_db(&restart_path) .await .expect("idempotent temporal reopen should succeed"); @@ -460,7 +476,7 @@ async fn temporal_schema_is_not_installed_into_a_nonempty_store() { } #[asupersync::test] -async fn writer_ledger_alone_does_not_refuse_a_fresh_store() { +async fn unpublished_writer_ledger_is_preserved_and_fresh_publication_still_works() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); seed_writer_ledger(&db_path).await; @@ -469,13 +485,32 @@ async fn writer_ledger_alone_does_not_refuse_a_fresh_store() { "the seeded store must hold writer ledger tables" ); - let db = open_global_db(&db_path) + let error = match open_global_db(&db_path).await { + Ok(_) => panic!("an unpublished writer-ledger database is not a fresh store"), + Err(error) => error, + }; + assert!( + error.to_string().contains("sidecar ownership is ambiguous"), + "unpublished writer-ledger sidecars must not be claimed: {error}" + ); + assert!( + table_exists(&db_path, "td_runtime_writer_idempotency_v2").await, + "refusal must preserve the existing writer ledger" + ); + assert!( + !table_exists(&db_path, "tracedecay_schema_version").await, + "refusal must not install a schema into an unowned store" + ); + + let fresh_path = tmp.path().join("fresh").join("sessions.db"); + let db = open_global_db(&fresh_path) .await - .expect("a store that holds only the writer ledger must receive the final temporal schema"); + .expect("new prospective publication must install the final schema and writer ledger"); drop(db); assert!( - table_exists(&db_path, "tracedecay_schema_version").await, - "writer ledger metadata must not block the final schema marker" + table_exists(&fresh_path, "tracedecay_schema_version").await + && table_exists(&fresh_path, "td_runtime_writer_idempotency_v2").await, + "a freshly owned family must publish the full schema and writer ledger" ); } diff --git a/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs b/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs index ffc3ad9bc5c..5711f53f64e 100644 --- a/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs +++ b/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs @@ -1,4 +1,4 @@ -//! Daemon shutdown cancelling a registered store attach mid schema install. +//! Cancellation of registered daemon admission after prospective schema publication. use std::future::Future; use std::path::PathBuf; @@ -16,19 +16,16 @@ use tracedecay_runtime_core::db::{ use super::harness::TEST_RUNTIME_NONCE; use crate::RegisteredGlobalDbOwnerV1; -/// The operation a cancel reports once it rolled back an open schema -/// transaction, rather than stopping before the transaction began. -const INSTALL_ROLLBACK: &str = "roll back attached registered global schema"; - -/// An existing profile-sessions store file with no schema objects, the shape -/// daemon attach installs the full registered schema into. -struct EmptyStore { +/// A newly owned profile-sessions store. Prospective publication installs +/// the schema before daemon admission; an unmarked existing SQLite file +/// cannot be used as a freshly owned initialization fixture. +struct RegisteredStore { path: PathBuf, _scope: DaemonDatabaseScope, _directory: TempDir, } -impl EmptyStore { +impl RegisteredStore { fn create() -> Self { crate::register_registered_schema_installer(); let directory = tempfile::tempdir().unwrap(); @@ -42,12 +39,6 @@ impl EmptyStore { .unwrap(); let path = tracedecay_sessions::runtime::user_sessions_db_path(&profile_root); std::fs::create_dir_all(path.parent().unwrap()).unwrap(); - // Exercise an initialized SQLite database with no application schema. - // A zero-byte placeholder is an incomplete store, not an attachable DB. - rusqlite::Connection::open(&path) - .unwrap() - .execute_batch("PRAGMA user_version = 0") - .unwrap(); Self { path, _scope: scope, @@ -59,10 +50,15 @@ impl EmptyStore { let authority = DatabaseAuthority::for_owned_runtime(&self.path, "schema install cancellation test") .unwrap(); + let mode = if self.path.try_exists().unwrap() { + TestDatabaseRuntimeMode::Existing + } else { + TestDatabaseRuntimeMode::Initialize + }; Database::publish_registered_daemon_test_runtime_with_retirement_control( &self.path, &authority, - TestDatabaseRuntimeMode::Existing, + mode, TestDatabaseRuntimeScope::ProfileSessions, ) .await @@ -115,9 +111,45 @@ impl EmptyStore { } } +#[asupersync::test] +async fn unpublished_existing_wal_is_refused_without_deleting_foreign_bytes() { + let store = RegisteredStore::create(); + let raw = rusqlite::Connection::open(&store.path).unwrap(); + raw.execute_batch("CREATE TABLE td_runtime_unpublished_probe(value INTEGER);") + .unwrap(); + drop(raw); + let original_main = std::fs::read(&store.path).unwrap(); + let wal = store.path.with_file_name(format!( + "{}-wal", + store.path.file_name().unwrap().to_string_lossy() + )); + let foreign_wal = b"unclaimed foreign WAL bytes"; + std::fs::write(&wal, foreign_wal).unwrap(); + + let authority = + DatabaseAuthority::for_owned_runtime(&store.path, "unpublished sidecar admission").unwrap(); + let attempted = Database::publish_registered_daemon_test_runtime_with_retirement_control( + &store.path, + &authority, + TestDatabaseRuntimeMode::Existing, + TestDatabaseRuntimeScope::ProfileSessions, + ) + .await; + let error = match attempted { + Ok(_) => panic!("an unmarked SQLite namespace must not claim a foreign WAL"), + Err(error) => error, + }; + assert!( + error.to_string().contains("sidecar ownership is ambiguous"), + "unexpected refusal: {error}" + ); + assert_eq!(std::fs::read(&store.path).unwrap(), original_main); + assert_eq!(std::fs::read(&wal).unwrap(), foreign_wal); +} + #[asupersync::test(flavor = "multi_thread")] -async fn cancelled_daemon_attach_rolls_back_schema_and_reopens() { - let reference = EmptyStore::create(); +async fn cancelled_daemon_attach_preserves_published_schema_and_reopens() { + let reference = RegisteredStore::create(); let (outcome, total_polls) = reference.attach_cancelled_at(usize::MAX).await; outcome.unwrap(); let installed = reference.schema_objects().await; @@ -128,13 +160,16 @@ async fn cancelled_daemon_attach_rolls_back_schema_and_reopens() { let stride = (total_polls / 20).max(1); let mut cancelled = Vec::new(); - let mut rolled_back_installs = Vec::new(); let mut completed = Vec::new(); - for cancel_at in (1..total_polls).step_by(stride).chain([total_polls + 10]) { - let store = EmptyStore::create(); - // Publishing the runtime installs its writer ledger before admission. - // Cancellation must preserve exactly that pre-admission schema. + for cancel_at in std::iter::once(1) + .chain((2..total_polls).step_by(stride)) + .chain([total_polls + 10]) + { + let store = RegisteredStore::create(); + // Cancellation cannot modify the already-published schema or + // prevent the next admission from reopening the same family. let before_admission = store.schema_objects().await; + assert_eq!(before_admission, installed); let (outcome, _) = store.attach_cancelled_at(cancel_at).await; let objects = store.schema_objects().await; match outcome { @@ -145,20 +180,14 @@ async fn cancelled_daemon_attach_rolls_back_schema_and_reopens() { ); assert_eq!( objects, before_admission, - "cancel at poll {cancel_at}/{total_polls} must roll the install back" + "cancel at poll {cancel_at}/{total_polls} must preserve the published schema" ); - if error - .project_route_context() - .is_some_and(|(_, _, detail)| detail.starts_with(INSTALL_ROLLBACK)) - { - rolled_back_installs.push(cancel_at); - } cancelled.push(cancel_at); } Ok(()) => { assert_eq!( objects, installed, - "an attach that commits before observing cancel at poll {cancel_at}/{total_polls} installs everything" + "an admission completed before cancel at poll {cancel_at}/{total_polls} preserves the published schema" ); completed.push(cancel_at); } @@ -173,10 +202,5 @@ async fn cancelled_daemon_attach_rolls_back_schema_and_reopens() { cancelled.contains(&1), "a cancel before admission starts must stop it; cancelled at {cancelled:?}" ); - assert!( - !rolled_back_installs.is_empty(), - "a cancel inside the schema transaction must roll the install back; \ - cancelled at {cancelled:?}, completed at {completed:?} of {total_polls} polls" - ); assert!(completed.contains(&(total_polls + 10))); } From ebbf98285ce47816dcd6adf4a37d83c62c47ca03 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 09:46:12 -0800 Subject: [PATCH 52/60] fix(sqlite): close staging readers before final WAL checkpoint The schema writer must retain its original reservations while all staging reader workers exit. Retire both reader lanes before the TRUNCATE barrier, so live snapshots cannot turn a complete schema into a false BUSY refusal. Add a real repository attachment regression for a pinned WAL snapshot, reader admission fencing, retained writer, and committed SQLite integrity. Refs #4513 --- .../src/shard_runtime/registry/ports.rs | 25 +++- .../src/repository/attachment.rs | 130 ++++++++++++++++++ 2 files changed, 153 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs index e12524afa4b..5e40672c251 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs @@ -279,8 +279,14 @@ pub(super) async fn initialize_staged_runtime( ControlFlow::Break(already_published) => return Ok(already_published), }; let identity = attachment.opened_file_identity(); - let installation = - install_final_schema_before_publication(request, &attachment, &stage.database()).await; + let attachment = Arc::new(attachment); + let installation = install_final_schema_before_publication( + request, + attachment.as_ref(), + &stage.database(), + Some(Arc::clone(&attachment)), + ) + .await; let files = crate::native_runtime::spawn_blocking(move || attachment.close_pinned_initialization()) .await @@ -329,6 +335,7 @@ async fn publish_lifecycle_runtime( &request, attachment.as_physical(), request.locator.path(), + None, ) .await { @@ -616,6 +623,7 @@ async fn install_final_schema_before_publication( request: &ShardRuntimeBuildRequest, attachment: &dyn PhysicalRuntimeAttachment, physical_path: &Path, + prospective_attachment: Option>, ) -> Result<(), StoreRuntimeRegistryFailure> { let connection = authorized_schema_connection( request, @@ -667,6 +675,19 @@ async fn install_final_schema_before_publication( } } if request.locator.is_prospective() { + // Publication needs no live reader snapshots of the private staging + // database. Retire both SQLite reader lanes and join their workers + // before the still-open serialized writer checkpoints the WAL. + let prospective = prospective_attachment.ok_or_else(|| { + physical_failure( + "quiesce prospective SQLite readers", + "prospective attachment authority is missing", + ) + })?; + crate::native_runtime::spawn_blocking(move || prospective.close_initialization_readers()) + .await + .map_err(|error| physical_failure("quiesce prospective SQLite readers", error))? + .map_err(|error| physical_failure("quiesce prospective SQLite readers", error))?; connection .finish_initial_schema() .await diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs index 18e52563f48..04629e7a65c 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs @@ -546,6 +546,61 @@ impl RepositoryRuntimePhysicalAttachment { self.lock_state().opened_file_identity } + /// Retire initialization's SQLite readers before the writer's final WAL + /// checkpoint. A live reader snapshot can make TRUNCATE report BUSY even + /// after every schema write has committed. Keep the writer and all original + /// file pins alive until the checkpoint has completed. + pub fn close_initialization_readers(&self) -> Result<(), String> { + if crate::storage_pool::on_pool() { + return Err("initialization readers cannot close on a storage pool thread".to_owned()); + } + let _serial = self + .drain_serial + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + { + let state = self.lock_state(); + if !state.admission_open || state.closed || state.initialization_file.is_none() { + return Err("only an unpublished live initialization may close its readers".to_owned()); + } + let readers = state + .readers + .as_ref() + .ok_or_else(|| "initialization readers have already closed".to_owned())?; + // Fence both general and reserved health acquisitions before + // checking for the last retained snapshot. + readers.begin_shutdown_drain(); + } + let started = Instant::now(); + loop { + let mut state = self.lock_state(); + if state.file_lost { + return Err("SQLite initialization lost its database while draining readers".to_owned()); + } + let quiescent = state + .readers + .as_ref() + .is_some_and(ReaderPool::is_quiescent); + if quiescent { + let readers = state.readers.take(); + drop(state); + // Drop joins the idle SQLite workers. No independent raw pin + // closes before the writer's final schema barrier. + drop(readers); + return Ok(()); + } + if started.elapsed() >= ATTACHMENT_DRAIN_TIMEOUT { + // Keep the draining readers owned by the attachment so its + // ordinary failure teardown can still join every worker. + return Err(format!( + "initialization readers did not quiesce within {ATTACHMENT_DRAIN_TIMEOUT:?}" + )); + } + drop(state); + thread::sleep(ATTACHMENT_DRAIN_POLL); + } + } + pub fn commit_initialization(&self) -> Result<(), String> { let mut state = self.lock_state(); if !state.admission_open || state.closed { @@ -1136,6 +1191,81 @@ mod tests { .unwrap(); } + #[test] + fn initialization_reader_shutdown_releases_pinned_wal_snapshot_before_finalization() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("schema.sqlite3"); + let binding = binding(); + let attachment = Arc::new( + RepositoryPhysicalAttachmentFactory + .start_reserved_initialization( + &native_runtime(), + binding.clone(), + locator(&binding), + path.clone(), + AdmissionConfigV1::default(), + None, + ) + .unwrap(), + ); + let handle = attachment.exact_sql_handle().unwrap(); + handle + .execute_batch( + "CREATE TABLE durable (value INTEGER NOT NULL); INSERT INTO durable VALUES (1)" + .to_owned(), + ) + .unwrap(); + let snapshot = handle.begin_read_snapshot(Duration::from_secs(1)).unwrap(); + snapshot + .query(statement("SELECT COUNT(*) FROM durable", vec![])) + .unwrap(); + handle + .execute_batch("INSERT INTO durable VALUES (2)".to_owned()) + .unwrap(); + + let (started_tx, started_rx) = mpsc::sync_channel(1); + let (finished_tx, finished_rx) = mpsc::sync_channel(1); + let closing = Arc::clone(&attachment); + let join = thread::spawn(move || { + started_tx.send(()).unwrap(); + finished_tx.send(closing.close_initialization_readers()).unwrap(); + }); + started_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + assert!( + matches!( + finished_rx.recv_timeout(Duration::from_millis(50)), + Err(mpsc::RecvTimeoutError::Timeout) + ), + "a live SQLite read snapshot must hold finalization's reader fence" + ); + drop(snapshot); + assert_eq!( + finished_rx.recv_timeout(Duration::from_secs(10)).unwrap(), + Ok(()) + ); + join.join().unwrap(); + + assert!( + handle.begin_read_snapshot(Duration::from_millis(50)).is_err(), + "no reader may reopen between the final checkpoint and publication" + ); + assert!( + attachment.snapshot().writer_present, + "the serialized SQLite writer must outlive its readers" + ); + let files = attachment.close_pinned_initialization().unwrap(); + drop(files); + let reopened = rusqlite::Connection::open(&path).unwrap(); + let rows: i64 = reopened + .query_row("SELECT COUNT(*) FROM durable", [], |row| row.get(0)) + .unwrap(); + let integrity: String = reopened + .query_row("PRAGMA integrity_check", [], |row| row.get(0)) + .unwrap(); + assert_eq!(rows, 2); + assert_eq!(integrity, "ok"); + } + #[test] fn abort_initialization_preserves_committed_bytes_and_releases_shared_pins() { let directory = TempDir::new().unwrap(); From 77e765888ad1bd1f0f9ba4745d24e9ebb1097de7 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 10:49:14 -0800 Subject: [PATCH 53/60] style(sqlite): format initialization reader shutdown barrier Match Bazel's Rust 2024 rustfmt gate for the reader-drain repair. Refs #4513 --- .../src/repository/attachment.rs | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs index 04629e7a65c..35f7ef8fa1a 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs @@ -561,7 +561,9 @@ impl RepositoryRuntimePhysicalAttachment { { let state = self.lock_state(); if !state.admission_open || state.closed || state.initialization_file.is_none() { - return Err("only an unpublished live initialization may close its readers".to_owned()); + return Err( + "only an unpublished live initialization may close its readers".to_owned(), + ); } let readers = state .readers @@ -575,12 +577,11 @@ impl RepositoryRuntimePhysicalAttachment { loop { let mut state = self.lock_state(); if state.file_lost { - return Err("SQLite initialization lost its database while draining readers".to_owned()); + return Err( + "SQLite initialization lost its database while draining readers".to_owned(), + ); } - let quiescent = state - .readers - .as_ref() - .is_some_and(ReaderPool::is_quiescent); + let quiescent = state.readers.as_ref().is_some_and(ReaderPool::is_quiescent); if quiescent { let readers = state.readers.take(); drop(state); @@ -1228,7 +1229,9 @@ mod tests { let closing = Arc::clone(&attachment); let join = thread::spawn(move || { started_tx.send(()).unwrap(); - finished_tx.send(closing.close_initialization_readers()).unwrap(); + finished_tx + .send(closing.close_initialization_readers()) + .unwrap(); }); started_rx.recv_timeout(Duration::from_secs(5)).unwrap(); assert!( @@ -1246,7 +1249,9 @@ mod tests { join.join().unwrap(); assert!( - handle.begin_read_snapshot(Duration::from_millis(50)).is_err(), + handle + .begin_read_snapshot(Duration::from_millis(50)) + .is_err(), "no reader may reopen between the final checkpoint and publication" ); assert!( From bc5cbc1fc5ad78e05748d1e73628f0e92296e74f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:34:30 -0800 Subject: [PATCH 54/60] fix(sqlite): close publication pins before releasing handoff Transfer creation pins into publication and close all raw main/sidecar verification handles before clearing the admission receipt. Apply the same ordering to prepared publication and published-family recovery. Add deterministic Unix subprocess regressions that admit a real SQLite connection at the completed-handoff event and verify rollback/WAL read locks, independent WAL writer exclusion, release, and restart integrity. Preserve refusal when cold recovery lacks durable allocation generations. Standalone Linux SQLite 3.46.1 mechanism checks passed; repository Bazel and native macOS qualification remain required on this exact commit. Refs #4513 --- .../shard_runtime/registry/initialization.rs | 32 ++- .../registry/initialization/tests/handoff.rs | 258 ++++++++++++++++++ 2 files changed, 283 insertions(+), 7 deletions(-) create mode 100644 crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization/tests/handoff.rs diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs index 9933b1d56d5..30d25426f56 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization.rs @@ -417,16 +417,16 @@ impl InitializationDirectory { "SQLite preparation has no durable file generation for recovery", )); } - self.publish_prepared().map(Some) + self.publish_prepared(Vec::new()).map(Some) } - fn publish_prepared(&mut self) -> io::Result { + fn publish_prepared(&mut self, mut pins: Vec) -> io::Result { let prepared = self .prepared .as_ref() .ok_or_else(|| ambiguous("SQLite preparation receipt is missing"))?; let stage = self.stage_cap()?; - let _pins = self.pin_members(&stage, prepared)?; + pins.extend(self.pin_members(&stage, prepared)?); for (suffix, identity) in SIDECARS.into_iter().zip(&prepared.sidecars) { let from = format!("reservation{suffix}"); let to = with_suffix(&self.destination, suffix); @@ -449,13 +449,17 @@ impl InitializationDirectory { capability_dir::sync_directory(&self.parent)?; prepared.pin_main(&self.parent, leaf(&self.destination)?)?; drop(stage); + // Read-only admission observes the receipt without taking the writer's + // lease. Close every raw main/SHM pin before clearing that receipt: + // a later close could release the new reader's process-wide POSIX locks. + drop(pins); self.retire()?; Ok(expected) } pub(super) fn publish(mut self, expected: u64, files: (File, [File; 3])) -> io::Result<()> { - let _creation_pins = self.prepare(expected, files)?; - self.publish_prepared().map(|_| ()) + let pins = self.prepare(expected, files)?; + self.publish_prepared(Vec::from(pins)).map(|_| ()) } fn verify_directory_name(&self) -> io::Result<()> { @@ -478,7 +482,15 @@ impl InitializationDirectory { // The kernel refuses a file or a nonempty directory even if a new // entry appears after the read. No regular staging leaf is unlinked. self.parent.remove_dir(leaf(&self.directory)?)?; - capability_dir::sync_directory(&self.parent) + capability_dir::sync_directory(&self.parent)?; + if self.prepared.is_some() { + tracing::debug!( + name: "sqlite.initialization.handoff", + path = %self.destination.display(), + "completed SQLite initialization handed off" + ); + } + Ok(()) } fn require_released_receipt(directory: &Path) -> io::Result<()> { @@ -547,6 +559,9 @@ impl InitializationDirectory { )?); } } + // Recovery also reopens raw evidence descriptors. They must not outlive + // the receipt that excludes read-only attachments from the family. + drop(pins); stage.retire() } } @@ -633,6 +648,9 @@ mod tests { use tracedecay_private_fs::create_private_file; use tracedecay_private_fs::framed_log::rename_noreplace; + #[cfg(all(unix, feature = "test-helpers"))] + mod handoff; + fn seed(path: &Path, value: i64) { let db = Connection::open(path).unwrap(); db.execute_batch("CREATE TABLE data(value INTEGER)") @@ -695,7 +713,7 @@ mod tests { assert_ne!(fs::read(stage.database()).unwrap(), before); assert_eq!(read_value(&stage.database()), 99); assert_eq!( - stage.publish_prepared().unwrap_err().kind(), + stage.publish_prepared(Vec::new()).unwrap_err().kind(), io::ErrorKind::InvalidData ); assert!(!destination.exists()); diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization/tests/handoff.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization/tests/handoff.rs new file mode 100644 index 00000000000..4ccdf29f1de --- /dev/null +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization/tests/handoff.rs @@ -0,0 +1,258 @@ +//! Admission at the publication event must not inherit any raw SQLite pins. +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use rusqlite::{Connection, Error, ErrorCode, OpenFlags}; +use tempfile::TempDir; +use tracing::{Event, Metadata, Subscriber, span}; + +use super::{InitializationDirectory, SIDECARS, owned_files, seed, with_suffix}; +use crate::logging::set_tracing_capture; + +const HANDOFF: &str = "sqlite.initialization.handoff"; +const DATABASE: &str = "TRACEDECAY_HANDOFF_DATABASE"; +const JOURNAL: &str = "TRACEDECAY_HANDOFF_JOURNAL"; +const BLOCKED: &str = "TRACEDECAY_HANDOFF_BLOCKED"; +const COMPLETE: &str = "tracedecay-handoff-probe-complete"; + +// Use a real connection at the completed-publication telemetry boundary. This +// deterministic schedule needs no sleeps or test-only production callbacks. +struct ConnectionAtHandoff { + path: PathBuf, + journal: &'static str, + writer: bool, + connection: Arc>>, +} + +impl Subscriber for ConnectionAtHandoff { + fn enabled(&self, metadata: &Metadata<'_>) -> bool { + metadata.name() == HANDOFF + } + + fn new_span(&self, _attributes: &span::Attributes<'_>) -> span::Id { + span::Id::from_u64(1) + } + + fn record(&self, _span: &span::Id, _values: &span::Record<'_>) {} + + fn record_follows_from(&self, _span: &span::Id, _follows: &span::Id) {} + + fn enter(&self, _span: &span::Id) {} + + fn exit(&self, _span: &span::Id) {} + + fn event(&self, event: &Event<'_>) { + if event.metadata().name() != HANDOFF { + return; + } + InitializationDirectory::verify_read_only_handoff(&self.path).unwrap(); + let flags = if self.writer { + OpenFlags::SQLITE_OPEN_READ_WRITE + } else { + OpenFlags::SQLITE_OPEN_READ_ONLY + }; + let connection = Connection::open_with_flags(&self.path, flags).unwrap(); + connection.busy_timeout(Duration::ZERO).unwrap(); + connection + .execute_batch(if self.writer { + "BEGIN IMMEDIATE" + } else { + "BEGIN DEFERRED" + }) + .unwrap(); + assert_eq!(value(&connection), 7); + if self.writer { + connection.execute_batch("UPDATE data SET value = 17").unwrap(); + } + probe(&self.path, self.journal, self.writer, true); + let mut slot = self.connection.lock().unwrap(); + assert!(slot.is_none(), "handoff must occur exactly once"); + *slot = Some(connection); + } +} + +fn probe(path: &Path, journal: &str, writer: bool, blocked: bool) { + let (_, module) = module_path!().split_once("::").unwrap(); + let output = Command::new(env::current_exe().unwrap()) + .args([ + "--exact", + &format!("{module}::external_handoff_probe"), + "--nocapture", + ]) + .env(DATABASE, path) + .env(JOURNAL, if writer { "writer" } else { journal }) + .env(BLOCKED, blocked.to_string()) + .output() + .unwrap(); + assert!( + output.status.success() && String::from_utf8_lossy(&output.stdout).contains(COMPLETE), + "external SQLite {journal} probe did not complete: {output:?}" + ); +} + +#[test] +fn external_handoff_probe() { + let Some(path) = env::var_os(DATABASE) else { + return; + }; + let journal = env::var(JOURNAL).unwrap(); + let blocked: bool = env::var(BLOCKED).unwrap().parse().unwrap(); + let writer = + Connection::open_with_flags(PathBuf::from(path), OpenFlags::SQLITE_OPEN_READ_WRITE).unwrap(); + writer.busy_timeout(Duration::ZERO).unwrap(); + assert_eq!(value(&writer), 7); + let result = if journal == "writer" { + writer.execute_batch("BEGIN IMMEDIATE; ROLLBACK") + } else if journal == "WAL" { + writer + .query_row("PRAGMA journal_mode = DELETE", [], |row| { + row.get::<_, String>(0) + }) + .map(|mode| assert_eq!(mode, "delete")) + } else { + writer.execute_batch("BEGIN IMMEDIATE; UPDATE data SET value = 99; COMMIT") + }; + if blocked { + assert!( + matches!( + &result, + Err(Error::SqliteFailure(error, _)) if error.code == ErrorCode::DatabaseBusy + ), + "the admitted connection must retain its OS lock: {result:?}" + ); + } else { + result.expect("write after the connection releases its snapshot"); + } + println!("{COMPLETE}"); +} + +fn value(connection: &Connection) -> i64 { + connection + .query_row("SELECT value FROM data", [], |row| row.get(0)) + .unwrap() +} + +#[derive(Clone, Copy, Debug)] +enum Publication { + Fresh, + Prepared, + RecoverPublished, +} + +fn check_handoff(publication: Publication, writer: bool) { + let journals: &[&str] = if writer { &["WAL"] } else { &["DELETE", "WAL"] }; + for &journal in journals { + let root = TempDir::new().unwrap(); + let destination = root.path().join("sessions.db"); + let mut stage = InitializationDirectory::begin(destination.clone()).unwrap(); + seed(&stage.database(), 7); + let database = Connection::open(stage.database()).unwrap(); + let mode: String = database + .query_row(&format!("PRAGMA journal_mode = {journal}"), [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(mode, journal.to_ascii_lowercase()); + drop(database); + let identity = crate::db::sqlite_generation_identity(&stage.database()).unwrap(); + let files = owned_files(&stage); + let connection = Arc::new(Mutex::new(None)); + let capture = set_tracing_capture(ConnectionAtHandoff { + path: destination.clone(), + journal, + writer, + connection: Arc::clone(&connection), + }); + match publication { + Publication::Fresh => stage.publish(identity, files).unwrap(), + Publication::Prepared => { + // A prepared resume no longer has the creator's descriptors; + // its reopened verification pins must obey the same handoff. + drop(stage.prepare(identity, files).unwrap()); + assert!(InitializationDirectory::verify_read_only_handoff(&destination).is_err()); + assert_eq!(stage.publish_prepared(Vec::new()).unwrap(), identity); + drop(stage); + } + Publication::RecoverPublished => { + let pins = stage.prepare(identity, files).unwrap(); + for suffix in SIDECARS { + fs::rename(stage.reservation(suffix), with_suffix(&destination, suffix)) + .unwrap(); + } + fs::rename(stage.database(), &destination).unwrap(); + let durable = stage.prepared.as_ref().unwrap().has_durable_generations(); + drop(pins); + drop(stage); + let result = InitializationDirectory::recover_published(&destination); + if !durable { + // Do not fabricate generation support for APFS/overlayfs. + // Those filesystems must preserve the unresolved receipt. + assert!(result.is_err()); + assert!(connection.lock().unwrap().is_none()); + assert!( + InitializationDirectory::verify_read_only_handoff(&destination).is_err() + ); + let database = Connection::open(&destination).unwrap(); + assert_eq!(value(&database), 7); + continue; + } + result.unwrap(); + } + } + drop(capture); + let connection = connection + .lock() + .unwrap() + .take() + .expect("the publication event must admit a real read snapshot"); + // Old drop ordering loses this connection's process-owned main/SHM locks + // after announcing the handoff, even though its transaction is live. + probe(&destination, journal, writer, true); + assert_eq!(value(&connection), if writer { 17 } else { 7 }); + // Roll back so the child can prove ordinary acquisition after release + // against the original committed value in both access modes. + connection.execute_batch("ROLLBACK").unwrap(); + drop(connection); + probe(&destination, journal, writer, false); + let reopened = Connection::open(&destination).unwrap(); + reopened.execute_batch("UPDATE data SET value = 23").unwrap(); + drop(reopened); + let reopened = + Connection::open_with_flags(&destination, OpenFlags::SQLITE_OPEN_READ_ONLY).unwrap(); + assert_eq!(value(&reopened), 23); + let integrity: String = reopened + .query_row("PRAGMA integrity_check", [], |row| row.get(0)) + .unwrap(); + assert_eq!(integrity, "ok"); + } +} + +#[test] +fn fresh_publication_handoff_preserves_read_locks() { + check_handoff(Publication::Fresh, false); +} + +#[test] +fn prepared_publication_handoff_preserves_read_locks() { + check_handoff(Publication::Prepared, false); +} + +#[test] +fn published_recovery_preserves_read_locks_or_refuses_unproven_generation() { + check_handoff(Publication::RecoverPublished, false); +} + +#[test] +fn publication_handoff_preserves_independent_wal_writer_lock() { + for publication in [ + Publication::Fresh, + Publication::Prepared, + Publication::RecoverPublished, + ] { + check_handoff(publication, true); + } +} From 714474f9dc8ed0362c7088d83cc36c143574795d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:23:30 -0800 Subject: [PATCH 55/60] style(sqlite): format initialization handoff regressions Refs #4513. Apply the exact three rustfmt 2024 hunks reported by hosted Bazel repository gates for bc5cbc1; no behavioral edits. --- .../registry/initialization/tests/handoff.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization/tests/handoff.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization/tests/handoff.rs index 4ccdf29f1de..298297760db 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization/tests/handoff.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/initialization/tests/handoff.rs @@ -66,7 +66,9 @@ impl Subscriber for ConnectionAtHandoff { .unwrap(); assert_eq!(value(&connection), 7); if self.writer { - connection.execute_batch("UPDATE data SET value = 17").unwrap(); + connection + .execute_batch("UPDATE data SET value = 17") + .unwrap(); } probe(&self.path, self.journal, self.writer, true); let mut slot = self.connection.lock().unwrap(); @@ -102,7 +104,8 @@ fn external_handoff_probe() { let journal = env::var(JOURNAL).unwrap(); let blocked: bool = env::var(BLOCKED).unwrap().parse().unwrap(); let writer = - Connection::open_with_flags(PathBuf::from(path), OpenFlags::SQLITE_OPEN_READ_WRITE).unwrap(); + Connection::open_with_flags(PathBuf::from(path), OpenFlags::SQLITE_OPEN_READ_WRITE) + .unwrap(); writer.busy_timeout(Duration::ZERO).unwrap(); assert_eq!(value(&writer), 7); let result = if journal == "writer" { @@ -219,7 +222,9 @@ fn check_handoff(publication: Publication, writer: bool) { drop(connection); probe(&destination, journal, writer, false); let reopened = Connection::open(&destination).unwrap(); - reopened.execute_batch("UPDATE data SET value = 23").unwrap(); + reopened + .execute_batch("UPDATE data SET value = 23") + .unwrap(); drop(reopened); let reopened = Connection::open_with_flags(&destination, OpenFlags::SQLITE_OPEN_READ_ONLY).unwrap(); From 89f84bd672c9884355c091aef1c6a633db67fe4b Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:28:43 -0800 Subject: [PATCH 56/60] fix(sqlite): claim reader shutdown ownership atomically Refs #4513. Use Arc::try_unwrap to fence late WeakReaderPool upgrades before acknowledging initialization-reader or attachment shutdown. Restore the same pool and retry within the existing deadline when another owner won. Track worker owners extracted for out-of-lock joins so a concurrent lost file callback cannot report closure and release main/sidecar pins while SQLite connections are still alive. Preserve a failed join on retries. Add deterministic production-path races and independent SQLite subprocess checks for writer exclusion, committed data and integrity. Rust/Bazel and native macOS execution remain pending; local standalone Linux SQLite mechanism probes are not repository test results. --- .../src/reader/pool/mod.rs | 6 + .../src/reader/pool/shutdown.rs | 91 +++++++ .../src/repository/attachment.rs | 112 +++++--- .../repository/attachment/tests/shutdown.rs | 240 ++++++++++++++++++ 4 files changed, 420 insertions(+), 29 deletions(-) create mode 100644 crates/tracedecay-rusqlite-runtime/src/reader/pool/shutdown.rs create mode 100644 crates/tracedecay-rusqlite-runtime/src/repository/attachment/tests/shutdown.rs diff --git a/crates/tracedecay-rusqlite-runtime/src/reader/pool/mod.rs b/crates/tracedecay-rusqlite-runtime/src/reader/pool/mod.rs index cddd0cc8c56..f44dbf21185 100644 --- a/crates/tracedecay-rusqlite-runtime/src/reader/pool/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/reader/pool/mod.rs @@ -29,6 +29,7 @@ use worker::SnapshotRevoker; mod lease; mod outcome; +mod shutdown; pub use lease::{ReaderLease, SnapshotLease}; pub use outcome::{ReaderAcquireError, ReaderPoolSnapshot, ReaderPoolState}; @@ -463,6 +464,9 @@ pub(super) struct PoolInner { /// registration. #[cfg(test)] after_spawn: Mutex>>, + /// Test-only: runs between the quiescence observation and ownership claim. + #[cfg(test)] + before_shutdown_claim: Mutex>>, } impl Drop for PoolInner { @@ -586,6 +590,8 @@ impl ReaderPool { admission: Arc::new(ReaderAdmissionRecorder::default()), #[cfg(test)] after_spawn: Mutex::new(None), + #[cfg(test)] + before_shutdown_claim: Mutex::new(None), }); let pool = Self { inner }; for _ in 0..pool.inner.budget.min_per_hot_shard { diff --git a/crates/tracedecay-rusqlite-runtime/src/reader/pool/shutdown.rs b/crates/tracedecay-rusqlite-runtime/src/reader/pool/shutdown.rs new file mode 100644 index 00000000000..c59434bc33f --- /dev/null +++ b/crates/tracedecay-rusqlite-runtime/src/reader/pool/shutdown.rs @@ -0,0 +1,91 @@ +//! Exclusive ownership is the shutdown boundary, not an observed Arc count. + +use std::sync::Arc; + +use super::{PoolInner, ReaderPool, ReaderQueryExecutor}; + +/// Owns the reader workers after every upgradeable pool reference is fenced. +/// The caller drops this outside its attachment lock so worker joins cannot +/// deadlock an attachment callback. +pub(crate) struct ReaderPoolShutdown { + inner: PoolInner, +} + +impl ReaderPoolShutdown { + pub(crate) fn join(self) { + drop(self.inner); + } +} + +impl ReaderPool { + /// Atomically retire the last pool reference before reporting shutdown. + /// + /// `is_quiescent` is only a hint: an exact-SQL handle can upgrade its weak + /// reference after the count check. A plain drop would then leave SQLite + /// connections alive past publication or release of the file identity pin. + /// A failed claim returns the same pool, retaining all workers and pins. + pub(crate) fn try_into_shutdown(self) -> Result, Self> { + #[cfg(test)] + if let Some(hook) = self + .inner + .before_shutdown_claim + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take() + { + hook(); + } + match Arc::try_unwrap(self.inner) { + Ok(inner) => Ok(ReaderPoolShutdown { inner }), + Err(inner) => Err(Self { inner }), + } + } + + #[cfg(test)] + pub(crate) fn run_once_before_shutdown_claim(&self, hook: impl FnOnce() + Send + 'static) { + *self + .inner + .before_shutdown_claim + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Box::new(hook)); + } +} + +#[cfg(test)] +mod tests { + use tracedecay_store::AdmissionConfigV1; + + use crate::reader::ReaderPool; + use crate::reader::tests::{CountExecutor, TestStore}; + + #[test] + fn a_late_weak_upgrade_cannot_outlive_an_acknowledged_reader_shutdown() { + let store = TestStore::new(); + let pool = ReaderPool::start( + store.locator(), + AdmissionConfigV1::default().readers, + CountExecutor, + ) + .unwrap(); + let weak = pool.downgrade(); + pool.begin_shutdown_drain(); + assert!(pool.is_quiescent()); + + // This is the interleaving between the attachment's advisory count + // check and retirement; no sleep or scheduler race is needed. + let late = weak.upgrade().unwrap(); + let pool = match pool.try_into_shutdown() { + Err(pool) => pool, + Ok(_) => panic!("a live late reference must prevent shutdown acknowledgment"), + }; + assert!(!pool.workers_exited()); + drop(late); + + let shutdown = match pool.try_into_shutdown() { + Ok(shutdown) => shutdown, + Err(_) => panic!("the last pool reference must be claimable"), + }; + assert!(weak.upgrade().is_none()); + shutdown.join(); + } +} diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs index 35f7ef8fa1a..21b8f5a157f 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs @@ -90,12 +90,15 @@ impl RepositoryPhysicalAttachmentFactory { readers: Some(readers), admission_open: true, drained: false, + workers_closing: false, closed: false, file_lost: false, lost_readers: None, lost_writer: None, #[cfg(test)] after_lost_notice: None, + #[cfg(test)] + before_worker_join: None, close_failure: None, retained_file_pin: Some(opened_database), })), @@ -322,12 +325,15 @@ impl RepositoryPhysicalAttachmentFactory { readers: Some(readers), admission_open: true, drained: false, + workers_closing: false, closed: false, file_lost: false, lost_readers: None, lost_writer: None, #[cfg(test)] after_lost_notice: None, + #[cfg(test)] + before_worker_join: None, close_failure: None, retained_file_pin, })); @@ -457,6 +463,9 @@ struct RepositoryRuntimePhysicalState { readers: Option>, admission_open: bool, drained: bool, + /// Worker owners are temporarily outside the state lock while joining. + /// Empty live/lost slots do not prove their SQLite connections have closed. + workers_closing: bool, closed: bool, file_lost: bool, /// The reader pool of a lost file, kept until its workers have exited so @@ -470,6 +479,9 @@ struct RepositoryRuntimePhysicalState { /// connection open for as long as it blocks. #[cfg(test)] after_lost_notice: Option>, + /// Test-only: holds the extracted workers before the out-of-lock join. + #[cfg(test)] + before_worker_join: Option>, close_failure: Option, /// Kept until all SQLite workers exit: closing any raw fd for the inode /// while SQLite holds POSIX locks can release those locks process-wide. @@ -511,10 +523,13 @@ impl RepositoryRuntimePhysicalAttachment { } /// Whether a lost file has no open connection left: the writer task - /// finished and every reader worker exited. Also true for an attachment - /// whose file was never lost. + /// finished and every reader worker exited, including owners currently + /// joining outside the state lock. Otherwise true when no close is pending. pub fn lost_file_closed(&self) -> bool { let mut state = self.lock_state(); + if state.workers_closing { + return false; + } if state .lost_readers .as_ref() @@ -582,13 +597,28 @@ impl RepositoryRuntimePhysicalAttachment { ); } let quiescent = state.readers.as_ref().is_some_and(ReaderPool::is_quiescent); - if quiescent { - let readers = state.readers.take(); - drop(state); - // Drop joins the idle SQLite workers. No independent raw pin - // closes before the writer's final schema barrier. - drop(readers); - return Ok(()); + if quiescent + && let Some(readers) = state.readers.take() + { + match readers.try_into_shutdown() { + Ok(readers) => { + state.workers_closing = true; + drop(state); + // Exclusive ownership fences late weak upgrades before + // joining. Keep every file pin until the joins finish. + readers.join(); + let mut state = self.lock_state(); + state.workers_closing = false; + if state.file_lost { + return Err( + "SQLite initialization lost its database while joining readers" + .to_owned(), + ); + } + return Ok(()); + } + Err(readers) => state.readers = Some(readers), + } } if started.elapsed() >= ATTACHMENT_DRAIN_TIMEOUT { // Keep the draining readers owned by the attachment so its @@ -880,6 +910,9 @@ impl RepositoryRuntimePhysicalAttachment { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let mut state = self.lock_state(); + if let Some(message) = &state.close_failure { + return Err(message.clone()); + } if state.closed { return Ok(()); } @@ -896,15 +929,32 @@ impl RepositoryRuntimePhysicalAttachment { drop(state); let started = Instant::now(); - loop { - let state = self.lock_state(); + let (writer, readers) = loop { + let mut state = self.lock_state(); let writer_quiescent = state.writer.as_ref().is_none_or(|writer| { Arc::strong_count(writer) == 1 && writer.telemetry_snapshot().queue.queued_operations == 0 }); let readers_quiescent = state.readers.as_ref().is_none_or(ReaderPool::is_quiescent); if writer_quiescent && readers_quiescent { - break; + match state.writer.take().map(Arc::try_unwrap).transpose() { + Ok(writer) => match state + .readers + .take() + .map(ReaderPool::try_into_shutdown) + .transpose() + { + Ok(readers) => { + state.workers_closing = true; + break (writer, readers); + } + Err(readers) => { + state.writer = writer.map(Arc::new); + state.readers = Some(readers); + } + }, + Err(writer) => state.writer = Some(writer), + } } if started.elapsed() >= ATTACHMENT_DRAIN_TIMEOUT { let queued = state.writer.as_ref().map_or(0, |writer| { @@ -920,27 +970,29 @@ impl RepositoryRuntimePhysicalAttachment { } drop(state); thread::sleep(ATTACHMENT_DRAIN_POLL); - } - - let mut state = self.lock_state(); - let writer = match state.writer.take().map(Arc::try_unwrap).transpose() { - Ok(writer) => writer, - Err(writer) => { - state.writer = Some(writer); - return Err("repository writer is still serving a request".to_owned()); - } }; - let readers = state.readers.take(); - drop(state); - drop(readers); - if let Some(writer) = writer - && let Err(error) = writer.shutdown_and_join() + + #[cfg(test)] { + let hook = self.lock_state().before_worker_join.take(); + if let Some(hook) = hook { + hook(); + } + } + if let Some(readers) = readers { + readers.join(); + } + // Neither the joins nor the writer's lost-file callback run under the + // attachment lock. Health checks still see their in-flight ownership. + let joined = writer.map_or(Ok(()), PersistentWriter::shutdown_and_join); + let mut state = self.lock_state(); + state.workers_closing = false; + if let Err(error) = joined { let message = format!("join repository writer: {error}"); - self.lock_state().close_failure = Some(message.clone()); + state.close_failure = Some(message.clone()); return Err(message); } - self.lock_state().drained = true; + state.drained = true; Ok(()) } @@ -956,7 +1008,7 @@ impl RepositoryRuntimePhysicalAttachment { if state.admission_open { return Err("repository physical attachment must drain before close".to_owned()); } - if !state.drained { + if !state.drained || state.workers_closing { return Err("repository physical attachment has not completed drain".to_owned()); } if state.writer.is_some() || state.readers.is_some() { @@ -1119,6 +1171,8 @@ fn infrastructure(operation: impl Into) -> StorageRuntimeErrorV1 { #[cfg(test)] mod tests { + mod shutdown; + use std::{fs, sync::mpsc, thread, time::Duration}; use tempfile::TempDir; diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/attachment/tests/shutdown.rs b/crates/tracedecay-rusqlite-runtime/src/repository/attachment/tests/shutdown.rs new file mode 100644 index 00000000000..e9c91280958 --- /dev/null +++ b/crates/tracedecay-rusqlite-runtime/src/repository/attachment/tests/shutdown.rs @@ -0,0 +1,240 @@ +use std::{ + env, + path::Path, + process::Command, + sync::{Arc, mpsc}, + thread, + time::Duration, +}; + +use rusqlite::{Connection, OpenFlags}; +use tempfile::TempDir; +use tracedecay_store::AdmissionConfigV1; + +use super::super::RepositoryPhysicalAttachmentFactory; +#[cfg(unix)] +use super::super::close_lost_file; +use super::{binding, locator, native_runtime}; +#[cfg(unix)] +use crate::connection::path_identity_tests::probe_external_connection; + +const SNAPSHOT_PATH: &str = "TRACEDECAY_SHUTDOWN_SNAPSHOT_PATH"; +const SNAPSHOT_DONE: &str = "tracedecay-shutdown-snapshot-complete"; + +fn assert_external_snapshot(path: &Path) { + let (_, module) = module_path!().split_once("::").unwrap(); + let output = Command::new(env::current_exe().unwrap()) + .args([ + "--exact", + &format!("{module}::external_shutdown_snapshot"), + "--nocapture", + ]) + .env(SNAPSHOT_PATH, path) + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success() && stdout.contains("1 passed") && stdout.contains(SNAPSHOT_DONE), + "external snapshot failed: {stdout} {}", + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn external_shutdown_snapshot() { + let Some(path) = env::var_os(SNAPSHOT_PATH) else { + return; + }; + let connection = + Connection::open_with_flags(Path::new(&path), OpenFlags::SQLITE_OPEN_READ_ONLY).unwrap(); + assert_eq!( + connection + .query_row("SELECT value FROM sample", [], |row| row.get::<_, i64>(0)) + .unwrap(), + 41 + ); + assert_eq!( + connection + .query_row("PRAGMA integrity_check", [], |row| row.get::<_, String>(0)) + .unwrap(), + "ok" + ); + println!("{SNAPSHOT_DONE}"); +} + +#[derive(Clone, Copy)] +enum CloseMode { + InitializationReaders, + Attachment, +} + +fn exercise_late_reader_owner(mode: CloseMode) { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("shutdown.sqlite3"); + let binding = binding(); + let attachment = Arc::new( + RepositoryPhysicalAttachmentFactory + .start_reserved_initialization( + &native_runtime(), + binding.clone(), + locator(&binding), + path.clone(), + AdmissionConfigV1::default(), + None, + ) + .unwrap(), + ); + attachment + .exact_sql_handle() + .unwrap() + .execute_batch( + "CREATE TABLE sample(value INTEGER); INSERT INTO sample VALUES(41)".to_owned(), + ) + .unwrap(); + let pin = attachment + .lock_state() + .initialization_file + .as_ref() + .unwrap() + .try_clone() + .unwrap(); + if matches!(mode, CloseMode::Attachment) { + attachment.commit_initialization().unwrap(); + } + let independent_writer = Connection::open(&path).unwrap(); + independent_writer + .execute_batch("BEGIN IMMEDIATE; UPDATE sample SET value=99") + .unwrap(); + let (late_tx, late_rx) = mpsc::channel(); + let weak = { + let state = attachment.lock_state(); + let readers = state.readers.as_ref().unwrap(); + let weak = readers.downgrade(); + let upgrading = weak.clone(); + readers.run_once_before_shutdown_claim(move || { + late_tx.send(upgrading.upgrade().unwrap()).unwrap(); + }); + weak + }; + #[cfg(unix)] + probe_external_connection(&path, "write", true); + assert_external_snapshot(&path); + let (finished_tx, finished_rx) = mpsc::channel(); + let closing = Arc::clone(&attachment); + let join = thread::spawn(move || { + let result = match mode { + CloseMode::InitializationReaders => closing.close_initialization_readers(), + CloseMode::Attachment => closing.drain(), + }; + finished_tx.send(result).unwrap(); + }); + + // The hook upgrades only after the production caller has observed + // quiescence, forcing the count-check/retirement race without a sleep. + let late = late_rx.recv_timeout(Duration::from_secs(10)).unwrap(); + assert!(matches!( + finished_rx.recv_timeout(Duration::from_millis(50)), + Err(mpsc::RecvTimeoutError::Timeout) + )); + assert!(!late.workers_exited()); + assert!(pin.try_clone().is_ok()); + assert!(attachment.lock_state().readers.is_some()); + + independent_writer.execute_batch("ROLLBACK").unwrap(); + drop(late); + assert_eq!( + finished_rx.recv_timeout(Duration::from_secs(10)).unwrap(), + Ok(()) + ); + join.join().unwrap(); + assert!(weak.upgrade().is_none()); + match mode { + CloseMode::InitializationReaders => { + drop(attachment.close_pinned_initialization().unwrap()); + } + CloseMode::Attachment => attachment.close_and_join().unwrap(), + } + assert!(pin.try_clone().is_err()); + #[cfg(unix)] + probe_external_connection(&path, "write", false); + assert_external_snapshot(&path); +} + +#[test] +fn initialization_reader_close_waits_for_a_late_weak_upgrade() { + exercise_late_reader_owner(CloseMode::InitializationReaders); +} + +#[test] +fn attachment_drain_waits_for_a_late_weak_upgrade() { + exercise_late_reader_owner(CloseMode::Attachment); +} + +#[cfg(unix)] +#[test] +fn lost_notice_during_out_of_lock_join_preserves_pins_and_writer_exclusion() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("joining.sqlite3"); + let binding = binding(); + let attachment = Arc::new( + RepositoryPhysicalAttachmentFactory + .start_reserved_initialization( + &native_runtime(), + binding.clone(), + locator(&binding), + path.clone(), + AdmissionConfigV1::default(), + None, + ) + .unwrap(), + ); + attachment + .exact_sql_handle() + .unwrap() + .execute_batch( + "CREATE TABLE sample(value INTEGER); INSERT INTO sample VALUES(41)".to_owned(), + ) + .unwrap(); + attachment.commit_initialization().unwrap(); + let pin = attachment + .lock_state() + .retained_file_pin + .as_ref() + .unwrap() + .try_clone() + .unwrap(); + let independent_writer = Connection::open(&path).unwrap(); + independent_writer + .execute_batch("BEGIN IMMEDIATE; UPDATE sample SET value=99") + .unwrap(); + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + attachment.lock_state().before_worker_join = Some(Box::new(move || { + entered_tx.send(()).unwrap(); + // A failed parent assertion drops the sender and also releases this join. + let _ = release_rx.recv(); + })); + let closing = Arc::clone(&attachment); + let join = thread::spawn(move || closing.drain()); + entered_rx.recv_timeout(Duration::from_secs(10)).unwrap(); + + // Deliver the real writer callback at the exact interval where its owner + // has left the state slots but its SQLite connection has not been joined. + // Keep the files in place so an independent OS process can probe the locks. + close_lost_file(&attachment.state); + assert!(attachment.file_lost()); + assert!(!attachment.lost_file_closed()); + assert!(attachment.close_and_join().is_err()); + assert!(pin.try_clone().is_ok()); + probe_external_connection(&path, "write", true); + assert_external_snapshot(&path); + + independent_writer.execute_batch("ROLLBACK").unwrap(); + release_tx.send(()).unwrap(); + assert_eq!(join.join().unwrap(), Ok(())); + attachment.close_and_join().unwrap(); + assert!(attachment.lost_file_closed()); + assert!(pin.try_clone().is_err()); + probe_external_connection(&path, "write", false); + assert_external_snapshot(&path); +} From d80c1b617abd13b2698a61f15e9d1758a79ef81d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:16:40 -0800 Subject: [PATCH 57/60] style(sqlite): format initialization reader shutdown check Refs #4513 Apply the exact Rust 2024 rustfmt hunk from hosted repository gates without changing shutdown ownership, SQLite behavior or test assertions. --- .../tracedecay-rusqlite-runtime/src/repository/attachment.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs index 21b8f5a157f..9fc6a6a75bd 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs @@ -597,9 +597,7 @@ impl RepositoryRuntimePhysicalAttachment { ); } let quiescent = state.readers.as_ref().is_some_and(ReaderPool::is_quiescent); - if quiescent - && let Some(readers) = state.readers.take() - { + if quiescent && let Some(readers) = state.readers.take() { match readers.try_into_shutdown() { Ok(readers) => { state.workers_closing = true; From 73b01ef17b856fef9e8a25f1a82980a9427642fb Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:36:25 -0800 Subject: [PATCH 58/60] fix(storage): confine publication renames to pinned parents Refs #4513. Require exact single-entry names before either native no-replace rename or sibling replacement. A dirfd does not confine absolute paths, parent traversal or intermediate symlinks. Reject normalized aliases and NUL without opening any SQLite descriptor or mutating either namespace. Add filesystem preservation regressions for the database and all three sidecars, plus an independent-process intermediate-directory replacement. Keep the separate full-path framed-log API and existing positive tests. Native Linux renameat2 reproduces all three escapes with a committed SQLite row. Repository Bazel and native macOS qualification are pending; this does not resolve the remaining same-UID canonical leaf race. --- .../src/capability_dir.rs | 38 +++-- .../src/capability_dir/namespace_tests.rs | 161 ++++++++++++++++++ 2 files changed, 188 insertions(+), 11 deletions(-) create mode 100644 crates/tracedecay-private-fs/src/capability_dir/namespace_tests.rs diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 4beb40576e2..f1da81003f5 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -144,6 +144,7 @@ fn look_again_on_not_found(mut attempt: impl FnMut() -> io::Result) -> io: /// Atomically renames one directory entry between already-open parent /// capabilities without allowing an occupied destination to be replaced. +/// Both names must be single, unnormalized directory entries, not paths. /// Platforms without a suitable primitive fail closed: retaining bytes is /// always preferable to risking a replacement. pub fn rename_noreplace( @@ -152,6 +153,8 @@ pub fn rename_noreplace( to_parent: &Dir, to: &OsStr, ) -> io::Result<()> { + require_entry_name(from)?; + require_entry_name(to)?; #[cfg(all(target_os = "linux", target_env = "gnu"))] { use std::os::fd::AsRawFd; @@ -203,17 +206,8 @@ fn dir_entry_path(parent: &Dir, name: &OsStr) -> io::Result /// Windows uses the shared namespace-durable rename authority, including its /// NTFS fallback for replacement while delete-sharing readers remain open. pub fn replace_file(parent: &Dir, source: &OsStr, destination: &OsStr) -> io::Result<()> { - for name in [source, destination] { - let mut components = std::path::Path::new(name).components(); - if !matches!(components.next(), Some(std::path::Component::Normal(_))) - || components.next().is_some() - { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "file replacement requires sibling entry names", - )); - } - } + require_entry_name(source)?; + require_entry_name(destination)?; #[cfg(windows)] { let source = dir_entry_path(parent, source)?; @@ -228,6 +222,25 @@ pub fn replace_file(parent: &Dir, source: &OsStr, destination: &OsStr) -> io::Re parent.rename(source, parent, destination) } +/// Native *at renames still resolve absolute paths, parent traversal and +/// intermediate symlinks. A held parent only confines a single entry name. +/// Compare the original spelling as well: Path components normalize trailing +/// separators and interior dots, but the native syscall receives those bytes. +fn require_entry_name(name: &OsStr) -> io::Result<()> { + let mut components = std::path::Path::new(name).components(); + if matches!(components.next(), Some(std::path::Component::Normal(entry)) if entry == name) + && components.next().is_none() + && !name.as_encoded_bytes().contains(&0) + { + Ok(()) + } else { + Err(io::Error::new( + io::ErrorKind::InvalidInput, + "publication requires single directory entry names", + )) + } +} + /// Resolves an open directory capability back to its live filesystem path so /// path-based Win32 primitives can address entries below it. #[cfg(windows)] @@ -535,3 +548,6 @@ mod tests { } } } + +#[cfg(test)] +mod namespace_tests; diff --git a/crates/tracedecay-private-fs/src/capability_dir/namespace_tests.rs b/crates/tracedecay-private-fs/src/capability_dir/namespace_tests.rs new file mode 100644 index 00000000000..ffd6ed74cab --- /dev/null +++ b/crates/tracedecay-private-fs/src/capability_dir/namespace_tests.rs @@ -0,0 +1,161 @@ +use std::{ffi::OsStr, fs, io, path::Path}; + +use super::{Dir, rename_noreplace, replace_file}; + +const FAMILY: [&str; 4] = [ + "sessions.db", + "sessions.db-wal", + "sessions.db-shm", + "sessions.db-journal", +]; + +fn open(path: &Path) -> Dir { + Dir::open_ambient_dir(path, cap_std::ambient_authority()).unwrap() +} + +#[test] +fn publication_refuses_escaped_sources_and_destinations_without_moving_store_files() { + let root = tempfile::tempdir().unwrap(); + let stage_path = root.path().join("stage"); + let target_path = root.path().join("target"); + let foreign_path = root.path().join("foreign"); + for path in [&stage_path, &target_path, &foreign_path] { + fs::create_dir(path).unwrap(); + } + let stage = open(&stage_path); + let target = open(&target_path); + + for name in FAMILY { + let owned = format!("owned {name}"); + let foreign = format!("foreign {name}"); + fs::write(stage_path.join(name), owned.as_bytes()).unwrap(); + fs::write(foreign_path.join(name), foreign.as_bytes()).unwrap(); + for source in [Path::new("../foreign").join(name), foreign_path.join(name)] { + let error = rename_noreplace(&stage, source.as_os_str(), &target, OsStr::new(name)) + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert_eq!(fs::read(foreign_path.join(name)).unwrap(), foreign.as_bytes()); + assert!(!target_path.join(name).exists()); + } + let destination_name = format!("published-{name}"); + for destination in [ + Path::new("../foreign").join(&destination_name), + foreign_path.join(&destination_name), + ] { + let error = + rename_noreplace(&stage, OsStr::new(name), &target, destination.as_os_str()) + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert_eq!(fs::read(stage_path.join(name)).unwrap(), owned.as_bytes()); + assert_eq!(fs::read(foreign_path.join(name)).unwrap(), foreign.as_bytes()); + assert!(!foreign_path.join(&destination_name).exists()); + } + } +} + +#[test] +fn publication_rejects_normalized_aliases_and_nul_before_either_rename() { + let root = tempfile::tempdir().unwrap(); + let parent = open(root.path()); + fs::write(root.path().join("payload"), b"owned").unwrap(); + fs::write(root.path().join("published"), b"foreign").unwrap(); + + for name in [ + "", + ".", + "..", + "./payload", + "payload/", + "payload/.", + "child/payload", + "payload\0ignored", + ] { + let invalid = OsStr::new(name); + for result in [ + rename_noreplace(&parent, invalid, &parent, OsStr::new("absent")), + rename_noreplace(&parent, OsStr::new("payload"), &parent, invalid), + replace_file(&parent, invalid, OsStr::new("published")), + replace_file(&parent, OsStr::new("payload"), invalid), + ] { + assert_eq!(result.unwrap_err().kind(), io::ErrorKind::InvalidInput); + } + assert_eq!(fs::read(root.path().join("payload")).unwrap(), b"owned"); + assert_eq!(fs::read(root.path().join("published")).unwrap(), b"foreign"); + assert!(!root.path().join("absent").exists()); + } +} + +#[cfg(unix)] +mod process_replacement { + use std::{env, os::unix::fs::symlink, process::Command}; + + use super::{FAMILY, OsStr, Path, fs, io, open, rename_noreplace}; + + const ROOT: &str = "TRACEDECAY_CAPABILITY_RENAME_TEST_ROOT"; + const COMPLETE: &str = "capability-rename-intermediate-replaced"; + + #[test] + fn replace_intermediate_directory_in_child() { + let Some(root) = env::var_os(ROOT) else { + return; + }; + let root = Path::new(&root); + fs::remove_dir(root.join("stage/bridge")).unwrap(); + symlink(root.join("foreign"), root.join("stage/bridge")).unwrap(); + println!("{COMPLETE}"); + } + + #[test] + fn child_replacing_an_intermediate_directory_cannot_redirect_publication() { + let root = tempfile::tempdir().unwrap(); + let stage_path = root.path().join("stage"); + let foreign_path = root.path().join("foreign"); + fs::create_dir_all(stage_path.join("bridge")).unwrap(); + fs::create_dir(&foreign_path).unwrap(); + for name in FAMILY { + fs::write(foreign_path.join(name), name.as_bytes()).unwrap(); + } + fs::write(stage_path.join("owned.sqlite"), b"owned").unwrap(); + let stage = open(&stage_path); + + // The child changes the namespace after the parent capability is held. + let (_, module) = module_path!().split_once("::").unwrap(); + let output = Command::new(env::current_exe().unwrap()) + .args([ + "--exact", + &format!("{module}::replace_intermediate_directory_in_child"), + "--nocapture", + ]) + .env(ROOT, root.path()) + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success() && stdout.contains("1 passed") && stdout.contains(COMPLETE), + "namespace replacement child failed: {stdout} {}", + String::from_utf8_lossy(&output.stderr) + ); + let rebound = fs::symlink_metadata(stage_path.join("bridge")).unwrap(); + assert!(rebound.is_symlink()); + + for (source, destination) in [ + ("bridge/sessions.db", "published.sqlite"), + ("owned.sqlite", "bridge/published.sqlite"), + ] { + let error = rename_noreplace( + &stage, + OsStr::new(source), + &stage, + OsStr::new(destination), + ) + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + } + for name in FAMILY { + assert_eq!(fs::read(foreign_path.join(name)).unwrap(), name.as_bytes()); + } + assert_eq!(fs::read(stage_path.join("owned.sqlite")).unwrap(), b"owned"); + assert!(!stage_path.join("published.sqlite").exists()); + assert!(!foreign_path.join("published.sqlite").exists()); + } +} From 12f5b305b6b9a190e3794116c4a6bfad0e49edb8 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:53:23 -0800 Subject: [PATCH 59/60] fix(sqlite): refuse unverifiable worker VFS identity Refs #4513. Require HAS_MOVED attestation when verifying a live SQLite worker. An unchanged path cannot establish the identity of a VFS handle when file control is unsupported. Exercise the actual SQLite memdb VFS and confirm a normal disk worker still attaches to the unmodified store. --- .../src/connection/mod.rs | 6 ++-- .../src/connection/tests.rs | 34 +++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs index 6b155753677..459198111ce 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs @@ -452,8 +452,10 @@ impl OpenedDatabaseFile { // HAS_MOVED below still observes the SQLite handle's different inode. self.verify_current_path(canonical_path)?; #[cfg(unix)] - if sqlite_connection_has_moved(_connection)? == Some(true) { - return Err(OpenedDatabaseFileError::Replaced); + match sqlite_connection_has_moved(_connection)? { + Some(false) => {} + Some(true) => return Err(OpenedDatabaseFileError::Replaced), + None => return Err(OpenedDatabaseFileError::Inspect), } #[cfg(unix)] { diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs index 05dcaabac38..b85a16a7ae2 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs @@ -652,6 +652,40 @@ fn writer_identity_fence_rejects_a_replacement_hidden_by_path_restore() { ); } +#[cfg(unix)] +#[test] +fn worker_identity_refuses_a_vfs_without_move_attestation() { + let file = database(); + let pinned = OpenedDatabaseFile::pin(file.path()).unwrap(); + // SQLite's built-in memdb VFS does not implement HAS_MOVED, although + // it accepts the same pathname as the pinned on-disk database. + let virtual_connection = Connection::open_with_flags_and_vfs( + file.path(), + rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE | rusqlite::OpenFlags::SQLITE_OPEN_CREATE, + "memdb", + ) + .expect("open bundled SQLite's memdb VFS"); + assert_eq!( + super::sqlite_connection_has_moved(&virtual_connection), + Ok(None), + "the regression must exercise an actual unsupported file control" + ); + assert_eq!( + pinned.verify_connection(&virtual_connection, file.path()), + Err(OpenedDatabaseFileError::Inspect), + "an unchanged filename cannot authenticate an unverified SQLite VFS handle" + ); + + // Rejecting that connection must leave the real database intact and + // permit a worker using the ordinary SQLite VFS. + let real_connection = Connection::open(file.path()).unwrap(); + pinned.verify_connection(&real_connection, file.path()).unwrap(); + assert_eq!( + real_connection.query_row("SELECT value FROM items", [], |row| row.get::<_, i64>(0)), + Ok(1) + ); +} + #[cfg(unix)] #[test] fn verified_immutable_reader_never_binds_transient_b_to_restored_a_identity() { From b1f093e8966e0e65c1e31f7c3c6b0077717730f9 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:10:29 -0800 Subject: [PATCH 60/60] style(sqlite): satisfy exact-head Bazel rustfmt gates Refs #4513 Apply the four mechanical Rust 2024 formatting changes reported by hosted Bazel run 42764933700. Preserve all assertions and behavior. --- .../src/capability_dir/namespace_tests.rs | 20 ++++++++++--------- .../src/connection/tests.rs | 4 +++- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir/namespace_tests.rs b/crates/tracedecay-private-fs/src/capability_dir/namespace_tests.rs index ffd6ed74cab..c4b7c0c604c 100644 --- a/crates/tracedecay-private-fs/src/capability_dir/namespace_tests.rs +++ b/crates/tracedecay-private-fs/src/capability_dir/namespace_tests.rs @@ -34,7 +34,10 @@ fn publication_refuses_escaped_sources_and_destinations_without_moving_store_fil let error = rename_noreplace(&stage, source.as_os_str(), &target, OsStr::new(name)) .unwrap_err(); assert_eq!(error.kind(), io::ErrorKind::InvalidInput); - assert_eq!(fs::read(foreign_path.join(name)).unwrap(), foreign.as_bytes()); + assert_eq!( + fs::read(foreign_path.join(name)).unwrap(), + foreign.as_bytes() + ); assert!(!target_path.join(name).exists()); } let destination_name = format!("published-{name}"); @@ -47,7 +50,10 @@ fn publication_refuses_escaped_sources_and_destinations_without_moving_store_fil .unwrap_err(); assert_eq!(error.kind(), io::ErrorKind::InvalidInput); assert_eq!(fs::read(stage_path.join(name)).unwrap(), owned.as_bytes()); - assert_eq!(fs::read(foreign_path.join(name)).unwrap(), foreign.as_bytes()); + assert_eq!( + fs::read(foreign_path.join(name)).unwrap(), + foreign.as_bytes() + ); assert!(!foreign_path.join(&destination_name).exists()); } } @@ -142,13 +148,9 @@ mod process_replacement { ("bridge/sessions.db", "published.sqlite"), ("owned.sqlite", "bridge/published.sqlite"), ] { - let error = rename_noreplace( - &stage, - OsStr::new(source), - &stage, - OsStr::new(destination), - ) - .unwrap_err(); + let error = + rename_noreplace(&stage, OsStr::new(source), &stage, OsStr::new(destination)) + .unwrap_err(); assert_eq!(error.kind(), io::ErrorKind::InvalidInput); } for name in FAMILY { diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs index b85a16a7ae2..c9559dc27d1 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs @@ -679,7 +679,9 @@ fn worker_identity_refuses_a_vfs_without_move_attestation() { // Rejecting that connection must leave the real database intact and // permit a worker using the ordinary SQLite VFS. let real_connection = Connection::open(file.path()).unwrap(); - pinned.verify_connection(&real_connection, file.path()).unwrap(); + pinned + .verify_connection(&real_connection, file.path()) + .unwrap(); assert_eq!( real_connection.query_row("SELECT value FROM items", [], |row| row.get::<_, i64>(0)), Ok(1)