From 8d707da366004711cd21e8b1a803fa567eefc505 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:35:06 -0800 Subject: [PATCH 1/4] fix(sqlite): reclaim dropped TEMP pages without resetting live state Refs #4152 Configure FULL auto-vacuum on writer and maintenance TEMP databases before schema creation; preserve live TEMP and durable main objects. Add a real FILE-backed page reclamation, survival and integrity regression. --- .../src/connection/mod.rs | 22 ++++ .../src/connection/tests.rs | 104 ++++++++++++++++++ 2 files changed, 126 insertions(+) diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs index fd4dae1b6ff..08ab76133a1 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs @@ -941,6 +941,28 @@ fn apply_pragmas( .pragma_update(None, "trusted_schema", false) .map_err(|source| policy("trusted schema", source))?; + // A dropped TEMP b-tree normally retains its pages in SQLite's unlinked + // TEMP file until this connection closes. FULL auto-vacuum truncates that + // file at commit without resetting unrelated live TEMP tables. Apply this + // before any TEMP schema is created; changing temp_store later destroys + // the TEMP schema and is not a safe way to reclaim it. + if mode != ConnectionMode::Reader { + connection + .execute_batch("PRAGMA temp.auto_vacuum = FULL") + .map_err(|source| policy("TEMP auto-vacuum", source))?; + let temp_auto_vacuum: i64 = connection + .query_row("PRAGMA temp.auto_vacuum", [], |row| row.get(0)) + .map_err(|source| policy("TEMP auto-vacuum verification", source))?; + if temp_auto_vacuum != 1 { + return Err(policy( + "TEMP auto-vacuum verification", + rusqlite::Error::InvalidParameterName(format!( + "temp.auto_vacuum={temp_auto_vacuum}, expected 1" + )), + )); + } + } + if mode == ConnectionMode::Writer { if fresh_writer { connection diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs index be04666e521..d82d55ffaf1 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs @@ -247,6 +247,110 @@ fn limits_and_authorizer_reject_oversized_or_unsafe_sql() { assert!(connection.prepare(&oversized).is_err()); } +/// Reclaim the actual FILE-backed SQLite TEMP pager while leaving both the +/// main database and an independent live TEMP table intact. Clearing temp_store +/// would also free the file, but would silently delete the survivor's rows. +#[test] +fn temp_auto_vacuum_reclaims_pages_without_clearing_live_tables() { + let file = database(); + let connection = Connection::open(file.path()).expect("open FILE-backed test connection"); + connection + .execute_batch("PRAGMA temp_store = FILE") + .expect("select filesystem TEMP storage before connection policy"); + super::apply_pragmas( + &connection, + ConnectionMode::Maintenance, + false, + Durability::Normal, + ) + .expect("apply canonical connection policy"); + assert_eq!( + connection + .query_row("PRAGMA temp.auto_vacuum", [], |row| row.get::<_, i64>(0)) + .unwrap(), + 1, + "TEMP must use FULL auto-vacuum to reclaim dropped b-trees" + ); + connection + .execute_batch( + "CREATE TEMP TABLE survivor(value TEXT NOT NULL); + INSERT INTO temp.survivor VALUES ('live row'); + CREATE TEMP TABLE disposable(body BLOB NOT NULL); + BEGIN IMMEDIATE;", + ) + .expect("create a live and a disposable TEMP relation"); + let payload = vec![0x53_u8; 8192]; + { + let mut insert = connection + .prepare("INSERT INTO temp.disposable(body) VALUES (?1)") + .unwrap(); + for _ in 0..1024 { + insert.execute([payload.as_slice()]).unwrap(); + } + } + connection.execute_batch("COMMIT").unwrap(); + let before: i64 = connection + .query_row("PRAGMA temp.page_count", [], |row| row.get(0)) + .unwrap(); + assert!( + before > 1024, + "fixture must allocate actual FILE TEMP pages: {before}" + ); + connection + .execute_batch("DROP TABLE temp.disposable") + .expect("drop disposable TEMP b-tree"); + let after: i64 = connection + .query_row("PRAGMA temp.page_count", [], |row| row.get(0)) + .unwrap(); + assert!( + after * 8 < before, + "dropping a TEMP table must shrink its pages: {before} -> {after}" + ); + assert_eq!( + connection + .query_row("SELECT value FROM temp.survivor", [], |row| { + row.get::<_, String>(0) + }) + .unwrap(), + "live row", + "reclamation must not reset active TEMP objects" + ); + assert_eq!( + connection + .query_row("SELECT value FROM main.items", [], |row| row.get::<_, i64>(0)) + .unwrap(), + 1, + "main database rows must survive" + ); + for schema in ["temp", "main"] { + assert_eq!( + connection + .query_row(&format!("PRAGMA {schema}.integrity_check"), [], |row| { + row.get::<_, String>(0) + }) + .unwrap(), + "ok", + "{schema} corruption after reclaiming TEMP pages" + ); + } + drop(connection); + + // Public opens must install the policy before the first TEMP allocation. + let reopened = open(file.path(), ConnectionMode::Maintenance).unwrap(); + assert_eq!( + reopened + .query_row("PRAGMA temp.auto_vacuum", [], |row| row.get::<_, i64>(0)) + .unwrap(), + 1 + ); + assert_eq!( + reopened + .query_row("SELECT value FROM main.items", [], |row| row.get::<_, i64>(0)) + .unwrap(), + 1 + ); +} + #[test] fn writer_bootstraps_fresh_incremental_auto_vacuum_before_wal() { let directory = tempfile::tempdir().expect("temporary directory"); From a274f1ead9583606c8d90029439b142f55e0b906 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:37:15 -0800 Subject: [PATCH 2/4] test(sqlite): verify writer TEMP reclamation policy Refs #4152 Assert the normal writer connection, not only maintenance setup, enables FULL TEMP auto-vacuum before the first TEMP schema object. --- crates/tracedecay-rusqlite-runtime/src/connection/tests.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs index d82d55ffaf1..b2b7ff63d38 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs @@ -49,6 +49,13 @@ fn writer_mode_applies_wal_integrity_and_write_policy() { assert_eq!(pragma_i64(&connection, "synchronous"), 1); assert_eq!(pragma_i64(&connection, "foreign_keys"), 1); assert_eq!(pragma_i64(&connection, "trusted_schema"), 0); + assert_eq!( + connection + .query_row("PRAGMA temp.auto_vacuum", [], |row| row.get::<_, i64>(0)) + .unwrap(), + 1, + "production writers must install TEMP reclamation before using TEMP" + ); assert!( connection .db_config(DbConfig::SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE) From 9ca0cf1bb3461ca9fbabd07d7b2d75f1c4c5ffba Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:38:12 -0800 Subject: [PATCH 3/4] test(sqlite): prove rolled-back TEMP drops preserve pages Refs #4152 A failed TEMP table removal must restore all committed rows and page count, then a committed removal must shrink pages while an unrelated table lives. --- .../src/connection/tests.rs | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs index b2b7ff63d38..500fa3b7c65 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs @@ -303,6 +303,27 @@ fn temp_auto_vacuum_reclaims_pages_without_clearing_live_tables() { before > 1024, "fixture must allocate actual FILE TEMP pages: {before}" ); + // Rollback must restore a dropped TEMP b-tree and every committed row. + // Reclaiming pages on a failed transaction would silently lose data. + connection + .execute_batch("BEGIN IMMEDIATE; DROP TABLE temp.disposable; ROLLBACK;") + .expect("undo a tentative TEMP reclamation"); + assert_eq!( + connection + .query_row("SELECT COUNT(*) FROM temp.disposable", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(), + 1024, + "rolled-back TEMP drop must preserve every committed row" + ); + assert_eq!( + connection + .query_row("PRAGMA temp.page_count", [], |row| row.get::<_, i64>(0)) + .unwrap(), + before, + "rollback must restore the original TEMP page allocation" + ); connection .execute_batch("DROP TABLE temp.disposable") .expect("drop disposable TEMP b-tree"); From 3a3ecf40eb9712487ab787bbc05adf7a75e27c09 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:52:03 -0800 Subject: [PATCH 4/4] fix(storage): preserve durable vacuum modes during maintenance --- .../src/connection/mod.rs | 47 +++++------- .../src/connection/tests.rs | 71 +++++++++++++++++-- .../src/exact_sql/tests/authority.rs | 7 +- 3 files changed, 87 insertions(+), 38 deletions(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs index 08ab76133a1..0e9a5d72113 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs @@ -941,34 +941,20 @@ fn apply_pragmas( .pragma_update(None, "trusted_schema", false) .map_err(|source| policy("trusted schema", source))?; - // A dropped TEMP b-tree normally retains its pages in SQLite's unlinked - // TEMP file until this connection closes. FULL auto-vacuum truncates that - // file at commit without resetting unrelated live TEMP tables. Apply this - // before any TEMP schema is created; changing temp_store later destroys - // the TEMP schema and is not a safe way to reclaim it. - if mode != ConnectionMode::Reader { + if mode == ConnectionMode::Writer { + // SQLite also remembers the last auto_vacuum assignment for a later + // full VACUUM of *any* database on this connection. Only repository + // writers deny full VACUUM. Maintenance allows it on main and attached + // stores, so its startup must leave their independent modes alone. connection - .execute_batch("PRAGMA temp.auto_vacuum = FULL") + .pragma_update(Some("temp"), "auto_vacuum", "FULL") .map_err(|source| policy("TEMP auto-vacuum", source))?; - let temp_auto_vacuum: i64 = connection - .query_row("PRAGMA temp.auto_vacuum", [], |row| row.get(0)) - .map_err(|source| policy("TEMP auto-vacuum verification", source))?; - if temp_auto_vacuum != 1 { - return Err(policy( - "TEMP auto-vacuum verification", - rusqlite::Error::InvalidParameterName(format!( - "temp.auto_vacuum={temp_auto_vacuum}, expected 1" - )), - )); - } - } - - if mode == ConnectionMode::Writer { + verify_pragma_i64(connection, Some("temp"), "auto_vacuum", 1)?; if fresh_writer { connection .pragma_update(None, "auto_vacuum", "INCREMENTAL") .map_err(|source| policy("fresh auto-vacuum", source))?; - verify_pragma_i64(connection, "auto_vacuum", 2)?; + verify_pragma_i64(connection, None, "auto_vacuum", 2)?; } connection .pragma_update(None, "journal_mode", "WAL") @@ -995,8 +981,8 @@ fn apply_pragmas( .map_err(|source| policy("query-only reader", source))?; } - verify_pragma_i64(connection, "foreign_keys", 1)?; - verify_pragma_i64(connection, "trusted_schema", 0)?; + verify_pragma_i64(connection, None, "foreign_keys", 1)?; + verify_pragma_i64(connection, None, "trusted_schema", 0)?; if !connection .db_config(DbConfig::SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE) .map_err(|source| policy("checkpoint-on-close verification", source))? @@ -1011,13 +997,13 @@ fn apply_pragmas( match mode { ConnectionMode::Writer => { verify_pragma_text(connection, "journal_mode", "wal")?; - verify_pragma_i64(connection, "wal_autocheckpoint", 0)?; - verify_pragma_i64(connection, "synchronous", synchronous_value)?; - verify_pragma_i64(connection, "journal_size_limit", RETAINED_WAL_BYTES)?; + verify_pragma_i64(connection, None, "wal_autocheckpoint", 0)?; + verify_pragma_i64(connection, None, "synchronous", synchronous_value)?; + verify_pragma_i64(connection, None, "journal_size_limit", RETAINED_WAL_BYTES)?; } - ConnectionMode::Reader => verify_pragma_i64(connection, "query_only", 1)?, + ConnectionMode::Reader => verify_pragma_i64(connection, None, "query_only", 1)?, ConnectionMode::Maintenance => { - verify_pragma_i64(connection, "journal_size_limit", RETAINED_WAL_BYTES)?; + verify_pragma_i64(connection, None, "journal_size_limit", RETAINED_WAL_BYTES)?; } } Ok(()) @@ -1097,11 +1083,12 @@ impl WriterPageCache { fn verify_pragma_i64( connection: &Connection, + schema: Option<&str>, name: &'static str, expected: i64, ) -> Result<(), ConnectionPolicyError> { let actual: i64 = connection - .pragma_query_value(None, name, |row| row.get(0)) + .pragma_query_value(schema, name, |row| row.get(0)) .map_err(|source| policy("pragma verification", source))?; if actual != expected { return Err(policy( diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs index 500fa3b7c65..7050571229c 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/tests.rs @@ -1,6 +1,6 @@ use std::path::{Path, PathBuf}; -use rusqlite::{Connection, ErrorCode, config::DbConfig, limits::Limit}; +use rusqlite::{Connection, ErrorCode, config::DbConfig, ffi, limits::Limit}; use tempfile::NamedTempFile; use tracedecay_store::WAL_SOFT_LIMIT_BYTES; @@ -254,6 +254,27 @@ fn limits_and_authorizer_reject_oversized_or_unsafe_sql() { assert!(connection.prepare(&oversized).is_err()); } +fn temp_file_bytes(connection: &Connection) -> i64 { + let mut file: *mut ffi::sqlite3_file = std::ptr::null_mut(); + let mut bytes = 0; + // The connection exclusively owns its TEMP pager. FILE_POINTER borrows + // the open VFS file, including on systems where its name was unlinked. + unsafe { + assert_eq!( + ffi::sqlite3_file_control( + connection.handle(), + c"temp".as_ptr(), + ffi::SQLITE_FCNTL_FILE_POINTER, + (&raw mut file).cast(), + ), + ffi::SQLITE_OK + ); + let methods = file.as_ref().unwrap().pMethods.as_ref().unwrap(); + assert_eq!(methods.xFileSize.unwrap()(file, &mut bytes), ffi::SQLITE_OK); + } + bytes +} + /// Reclaim the actual FILE-backed SQLite TEMP pager while leaving both the /// main database and an independent live TEMP table intact. Clearing temp_store /// would also free the file, but would silently delete the survivor's rows. @@ -266,7 +287,7 @@ fn temp_auto_vacuum_reclaims_pages_without_clearing_live_tables() { .expect("select filesystem TEMP storage before connection policy"); super::apply_pragmas( &connection, - ConnectionMode::Maintenance, + ConnectionMode::Writer, false, Durability::Normal, ) @@ -303,6 +324,11 @@ fn temp_auto_vacuum_reclaims_pages_without_clearing_live_tables() { before > 1024, "fixture must allocate actual FILE TEMP pages: {before}" ); + let bytes_before = temp_file_bytes(&connection); + assert!( + bytes_before >= 8 * 1024 * 1024, + "TEMP data must spill to disk" + ); // Rollback must restore a dropped TEMP b-tree and every committed row. // Reclaiming pages on a failed transaction would silently lose data. connection @@ -327,6 +353,11 @@ fn temp_auto_vacuum_reclaims_pages_without_clearing_live_tables() { connection .execute_batch("DROP TABLE temp.disposable") .expect("drop disposable TEMP b-tree"); + let bytes_after = temp_file_bytes(&connection); + assert!( + bytes_after > 0 && bytes_after * 8 < bytes_before, + "the still-open TEMP file must shrink: {bytes_before} -> {bytes_after}" + ); let after: i64 = connection .query_row("PRAGMA temp.page_count", [], |row| row.get(0)) .unwrap(); @@ -345,7 +376,8 @@ fn temp_auto_vacuum_reclaims_pages_without_clearing_live_tables() { ); assert_eq!( connection - .query_row("SELECT value FROM main.items", [], |row| row.get::<_, i64>(0)) + .query_row("SELECT value FROM main.items", [], |row| row + .get::<_, i64>(0)) .unwrap(), 1, "main database rows must survive" @@ -364,7 +396,7 @@ fn temp_auto_vacuum_reclaims_pages_without_clearing_live_tables() { drop(connection); // Public opens must install the policy before the first TEMP allocation. - let reopened = open(file.path(), ConnectionMode::Maintenance).unwrap(); + let reopened = open(file.path(), ConnectionMode::Writer).unwrap(); assert_eq!( reopened .query_row("PRAGMA temp.auto_vacuum", [], |row| row.get::<_, i64>(0)) @@ -373,12 +405,41 @@ fn temp_auto_vacuum_reclaims_pages_without_clearing_live_tables() { ); assert_eq!( reopened - .query_row("SELECT value FROM main.items", [], |row| row.get::<_, i64>(0)) + .query_row("SELECT value FROM main.items", [], |row| row + .get::<_, i64>(0)) .unwrap(), 1 ); } +#[test] +fn maintenance_vacuum_preserves_each_stores_auto_vacuum_mode() { + let root = tempfile::tempdir().unwrap(); + let main = root.path().join("main.db"); + let attached = root.path().join("attached.db"); + for (path, mode) in [(&main, "INCREMENTAL"), (&attached, "NONE")] { + let db = Connection::open(path).unwrap(); + db.pragma_update(None, "auto_vacuum", mode).unwrap(); + db.execute_batch("CREATE TABLE sample(value INTEGER); INSERT INTO sample VALUES(7)") + .unwrap(); + } + + let db = open(&main, ConnectionMode::Maintenance).unwrap(); + db.execute("ATTACH ?1 AS auxiliary", [attached.to_str().unwrap()]) + .unwrap(); + db.execute_batch("VACUUM main; VACUUM auxiliary").unwrap(); + drop(db); + for (path, expected) in [(&main, 2), (&attached, 0)] { + let db = Connection::open(path).unwrap(); + assert_eq!(pragma_i64(&db, "auto_vacuum"), expected); + assert_eq!( + db.query_row("SELECT value FROM sample", [], |row| row.get::<_, i64>(0)) + .unwrap(), + 7 + ); + } +} + #[test] fn writer_bootstraps_fresh_incremental_auto_vacuum_before_wal() { let directory = tempfile::tempdir().expect("temporary directory"); diff --git a/crates/tracedecay-rusqlite-runtime/src/exact_sql/tests/authority.rs b/crates/tracedecay-rusqlite-runtime/src/exact_sql/tests/authority.rs index 1f6ba5c7f75..2850ef8361d 100644 --- a/crates/tracedecay-rusqlite-runtime/src/exact_sql/tests/authority.rs +++ b/crates/tracedecay-rusqlite-runtime/src/exact_sql/tests/authority.rs @@ -61,16 +61,17 @@ fn writer_actor_allows_only_product_schema_pragmas() { .execute_batch(pragma.to_owned()) .unwrap_or_else(|error| panic!("{pragma} must remain available: {error}")); } - for pragma in [ + for sql in [ + "VACUUM main", "PRAGMA auto_vacuum = NONE", "PRAGMA foreign_keys = OFF", "PRAGMA secure_delete = OFF", "PRAGMA writable_schema = ON", ] { - let error = channel.execute_batch(pragma.to_owned()).unwrap_err(); + let error = channel.execute_batch(sql.to_owned()).unwrap_err(); assert!( matches!(error, ExactSqlError::Sqlite { .. }), - "{pragma}: {error}" + "{sql}: {error}" ); } }