From 4a05a0c8efa2839197a5b1653e9f71f2f4542e92 Mon Sep 17 00:00:00 2001 From: Elise Ong Date: Tue, 10 Nov 2026 12:25:22 -0500 Subject: [PATCH 1/6] feat(storage): add catalog migration 10, point_in_time_recovery A table with point-in-time recovery enabled on the service can be restored to any second of its window, items written before the enable included. That needs the table's recovery setting and a history of its committed writes. Migration 10 creates the two tables that hold them: pitr_state, one row per table with recovery enabled (the retention in days, the enable instant, the cutoff of the latest prune, and the time a reconstruct in progress holds against the retention sweep, with its expiry), and item_history, one row per committed write (the key as the data table stores it, the commit time in microseconds, and the item after the write or NULL for a delete). On SQLite it is a file body in the catalog, and both tables are removed with the table's catalog row. On PostgreSQL it is a data database body, beside the item data, so a history row can commit in the write's own transaction; item_history's sequence has a cache of 1, as stream_pending's has. It does not apply on MongoDB or Cassandra. The name continuous_backups is not reused, since migration 5 drops it again under --reconverge. The MongoDB migration tests count the new id, the SQLite golden test pins what it creates on a released catalog, and the released-catalog helper of the resource tag test removes the two tables as it removes migration 9's columns. docs/schema/sqlite.txt and docs/schema/postgres.txt are regenerated from fresh installs. The upgrade manual lists migration 10 and states each migration's declared kind per backend, the guides' sample output reads migration 10 as this build prints it, and CONTRIBUTING's worked example becomes migration 11. Verified: the SQLite migration tests (81 pass); the MongoDB fresh, lock and released-catalog tests on a mongo:7 replica set (24 pass); the PostgreSQL migration tests on PostgreSQL 16; a fresh install's schema dumps differ from the committed ones by exactly the two tables, their indexes, constraints and sequence; the upgrade manual's kinds against each backend's migration table. --- CONTRIBUTING.md | 47 ++++++++--------- .../storage-cassandra/src/migrations/mod.rs | 4 ++ crates/storage-mongodb/src/migrations/mod.rs | 4 ++ .../storage-mongodb/tests/migrations_fresh.rs | 16 +++--- .../storage-mongodb/tests/migrations_locks.rs | 2 +- .../tests/migrations_released.rs | 2 +- .../0010_point_in_time_recovery.sql | 50 +++++++++++++++++++ crates/storage-postgres/src/migrations/mod.rs | 13 ++++- .../0010_point_in_time_recovery.sql | 40 +++++++++++++++ .../src/migrations/golden_tests.rs | 32 ++++++++++++ crates/storage-sqlite/src/migrations/mod.rs | 9 ++++ .../golden/0010_point_in_time_recovery.txt | 24 +++++++++ crates/storage/src/migrations/registry.rs | 6 +++ docs/getting-started.md | 6 +-- docs/manuals/04-quickstart-setup-guide.md | 4 +- docs/manuals/07-upgrade-manual.md | 6 ++- docs/manuals/08-install-linux.md | 2 +- docs/manuals/09-install-macos.md | 2 +- docs/schema/postgres.txt | 21 ++++++++ docs/schema/sqlite.txt | 4 ++ tests/test_resource_tag_migration.py | 15 ++++-- 21 files changed, 263 insertions(+), 46 deletions(-) create mode 100644 crates/storage-postgres/data_migrations/0010_point_in_time_recovery.sql create mode 100644 crates/storage-sqlite/migrations/0010_point_in_time_recovery.sql create mode 100644 crates/storage-sqlite/tests/fixtures/golden/0010_point_in_time_recovery.txt diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e321d02e0..3b7e5a495 100755 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -359,28 +359,29 @@ SQLite, PostgreSQL, Cassandra and MongoDB migrate through this framework. ### Example: a nullable column on `tables` -Migration 10, `table_note` (10 being the next free id at the time of writing; -migrations 6, 7, 8 and 9 are `backup_catalog_index`, `stream_pending`, -`restore_progress` and `warm_throughput`), adds a nullable `note TEXT` column -to the catalog's `tables` table on SQLite and PostgreSQL. +Migration 11, `table_note` (11 being the next free id at the time of writing; +migrations 6 to 10 are `backup_catalog_index`, `stream_pending`, +`restore_progress`, `warm_throughput` and `point_in_time_recovery`), adds a +nullable `note TEXT` column to the catalog's `tables` table on SQLite and +PostgreSQL. In `crates/storage/src/migrations/registry.rs`, at the end of the `migration_ids!` list: ```rust /// Adds a nullable `note` column to the `tables` catalog table. - TableNote = 10, "table_note", + TableNote = 11, "table_note", "add a nullable note column to the tables catalog"; ``` PostgreSQL has `ADD COLUMN IF NOT EXISTS`, so its body is a file, -`crates/storage-postgres/migrations/0010_table_note.sql`: +`crates/storage-postgres/migrations/0011_table_note.sql`: ```sql -- Copyright 2026 ExtendDB contributors -- SPDX-License-Identifier: Apache-2.0 -- --- Migration 10, table_note, PostgreSQL catalog database: a nullable note +-- Migration 11, table_note, PostgreSQL catalog database: a nullable note -- column on tables. IF NOT EXISTS makes a second run change nothing. ALTER TABLE tables ADD COLUMN IF NOT EXISTS note TEXT; @@ -395,7 +396,7 @@ In `crates/storage-postgres/src/migrations/mod.rs`, the arm in `migration()`: bodies: vec![file_body!( StoreKind::Catalog, "migrations", - "0010_table_note.sql" + "0011_table_note.sql" )], }, ``` @@ -417,13 +418,13 @@ and the id in the transactional arm of `runs_in()`: SQLite has no `ADD COLUMN IF NOT EXISTS`, and a file with a plain `ADD COLUMN` fails when it runs a second time, so the SQLite body is code that checks for the column first. -`crates/storage-sqlite/src/migrations/m0010_table_note.rs`: +`crates/storage-sqlite/src/migrations/m0011_table_note.rs`: ```rust // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 -//! Migration 10, `table_note`, SQLite: a nullable `note` column on `tables`. +//! Migration 11, `table_note`, SQLite: a nullable `note` column on `tables`. //! //! SQLite has no `ADD COLUMN IF NOT EXISTS`, so the body reads the table's //! columns first and adds the column only when it is missing. A second run @@ -435,10 +436,10 @@ use extenddb_storage::migrations::{CodeBody, Done, MigrationError}; use super::SqliteStores; /// This file, whose SHA-256 the checksum test pins. -pub(crate) const SOURCE: &str = include_str!("m0010_table_note.rs"); +pub(crate) const SOURCE: &str = include_str!("m0011_table_note.rs"); /// Recorded as this migration's checksum. Changes only with a new migration. -const FINGERPRINT: &str = "0010_table_note/sqlite/1"; +const FINGERPRINT: &str = "0011_table_note/sqlite/1"; pub(crate) struct TableNote; @@ -477,7 +478,7 @@ beside the others: ```rust #[rustfmt::skip] #[allow(clippy::all, deprecated)] -mod m0010_table_note; +mod m0011_table_note; ``` and the arm in `migration()`: @@ -488,8 +489,8 @@ and the arm in `migration()`: convergent: true, bodies: vec![StoreBody::code( StoreKind::Catalog, - &m0010_table_note::TableNote, - m0010_table_note::SOURCE, + &m0011_table_note::TableNote, + m0011_table_note::SOURCE, )], }, ``` @@ -501,8 +502,8 @@ rows of `tables`: ```rust #[tokio::test] -async fn migration_10_adds_the_note_column_to_tables() { - let db = temp_db("m10"); +async fn migration_11_adds_the_note_column_to_tables() { + let db = temp_db("m11"); load_dump(&db, DEV_0_1_14).await; run(&db).await.expect("migrate"); let mut text = dump( @@ -512,12 +513,12 @@ async fn migration_10_adds_the_note_column_to_tables() { ) .await; text.push_str(&dump(&db, "SELECT table_name, note FROM tables ORDER BY table_name").await); - golden("0010_table_note", &text); + golden("0011_table_note", &text); } ``` -`EXTENDDB_BLESS_EXPECTED=1 cargo test -p extenddb-storage-sqlite --lib migration_10` -writes `crates/storage-sqlite/tests/fixtures/golden/0010_table_note.txt`, one +`EXTENDDB_BLESS_EXPECTED=1 cargo test -p extenddb-storage-sqlite --lib migration_11` +writes `crates/storage-sqlite/tests/fixtures/golden/0011_table_note.txt`, one tab-separated line per row under each query: ``` @@ -532,15 +533,15 @@ The PostgreSQL body is a file, so it has no golden test. The example declares `Kind::Breaking`. A nullable column can be additive by the rule in step 4, but declaring it additive needs the old-binary job, which this example does not add. As a breaking migration it makes a 1.x binary -without migration 10 refuse a catalog that has it, and a rollback restores the +without migration 11 refuse a catalog that has it, and a rollback restores the backup. The same PR changes the code that reads and writes `tables.note` in each backend's catalog store, and commits the `docs/schema/sqlite.txt` and `docs/schema/postgres.txt` that the integration jobs regenerate. It changes no per-table DDL, because `tables` is a catalog table, and adds no `RELEASED` -entry. `extenddb migrate --list` then shows migration 10, and on a catalog at -migration 9 `extenddb migrate` lists migration 10 as pending and exits 3. +entry. `extenddb migrate --list` then shows migration 11, and on a catalog at +migration 10 `extenddb migrate` lists migration 11 as pending and exits 3. ### Running the migration tests diff --git a/crates/storage-cassandra/src/migrations/mod.rs b/crates/storage-cassandra/src/migrations/mod.rs index 85d10a1b4..c5bf7a344 100644 --- a/crates/storage-cassandra/src/migrations/mod.rs +++ b/crates/storage-cassandra/src/migrations/mod.rs @@ -168,6 +168,10 @@ pub fn migration(id: MigrationId) -> Migration { "0009_warm_throughput.cql" )], }, + MigrationId::PointInTimeRecovery => Migration::NotApplicable( + "point-in-time recovery is not available on Cassandra; its restore answers \ + PointInTimeRecoveryUnavailableException", + ), } } diff --git a/crates/storage-mongodb/src/migrations/mod.rs b/crates/storage-mongodb/src/migrations/mod.rs index 0c97c8972..159798479 100644 --- a/crates/storage-mongodb/src/migrations/mod.rs +++ b/crates/storage-mongodb/src/migrations/mod.rs @@ -173,6 +173,10 @@ pub fn migration(id: MigrationId) -> Migration { "the catalog collections have no validator, so a table or index document carries \ a warm_throughput field without a schema change", ), + MigrationId::PointInTimeRecovery => Migration::NotApplicable( + "point-in-time recovery is not available on MongoDB; its restore answers \ + PointInTimeRecoveryUnavailableException", + ), } } diff --git a/crates/storage-mongodb/tests/migrations_fresh.rs b/crates/storage-mongodb/tests/migrations_fresh.rs index 7ce27b921..286f022b1 100644 --- a/crates/storage-mongodb/tests/migrations_fresh.rs +++ b/crates/storage-mongodb/tests/migrations_fresh.rs @@ -325,6 +325,7 @@ async fn a_fresh_store_gets_every_migration_and_the_expected_definitions() { (7, d.target.catalog.clone()), (8, d.target.catalog.clone()), (9, d.target.catalog.clone()), + (10, d.target.catalog.clone()), ] ); let not_applicable: Vec = report @@ -333,7 +334,7 @@ async fn a_fresh_store_gets_every_migration_and_the_expected_definitions() { .filter(|s| s.outcome == Outcome::NotApplicable) .map(|s| s.id) .collect(); - assert_eq!(not_applicable, [3, 4, 7, 8, 9], "{report:?}"); + assert_eq!(not_applicable, [3, 4, 7, 8, 9, 10], "{report:?}"); assert_eq!( d.status().await, CatalogStatus::Current { @@ -354,6 +355,7 @@ async fn a_fresh_store_gets_every_migration_and_the_expected_definitions() { (7, Outcome::NotApplicable), (8, Outcome::NotApplicable), (9, Outcome::NotApplicable), + (10, Outcome::NotApplicable), ] ); assert!( @@ -449,7 +451,7 @@ async fn a_second_run_applies_nothing_and_reconverge_keeps_the_rows() { let reconverged = d.reconverge().await.expect("reconverge"); // Migration 1's steps in the catalog and the data database, and // migrations 5's and 6's in the catalog: 2 is not convergent, and 3, 4, - // 7, 8 and 9 do not apply on MongoDB. + // 7 to 10 do not apply on MongoDB. let ran: Vec<(u32, String)> = reconverged .applied .iter() @@ -540,24 +542,24 @@ async fn a_changed_checksum_a_newer_catalog_and_an_older_release_are_reported() "notes": "", "applied_at": bson::DateTime::now(), "applied_by": "9.9.9" } }; ledger - .insert_one(row(10, "breaking")) + .insert_one(row(11, "breaking")) .await .expect("insert"); let status = d.status().await; assert_eq!(status.label(), "AheadIncompatible"); assert_eq!(status.exit_code(), exit::AHEAD_INCOMPATIBLE); ledger - .delete_one(doc! { "_id": 10_i64 }) + .delete_one(doc! { "_id": 11_i64 }) .await .expect("delete"); ledger - .insert_one(row(10, "additive")) + .insert_one(row(11, "additive")) .await .expect("insert"); assert_eq!(d.status().await.label(), "AheadCompatible"); assert!(d.migrate().await.expect("migrate").applied.is_empty()); ledger - .delete_one(doc! { "_id": 10_i64 }) + .delete_one(doc! { "_id": 11_i64 }) .await .expect("delete"); @@ -638,6 +640,6 @@ async fn a_ledger_document_this_binary_cannot_read_stops_the_status() { .expect_err("an error"); migrations.close().await; assert!(error.to_string().contains("\"sideways\""), "{error}"); - assert_eq!(MigrationId::LATEST.number(), 9); + assert_eq!(MigrationId::LATEST.number(), 10); d.drop_databases().await; } diff --git a/crates/storage-mongodb/tests/migrations_locks.rs b/crates/storage-mongodb/tests/migrations_locks.rs index bf8d5d911..934916533 100644 --- a/crates/storage-mongodb/tests/migrations_locks.rs +++ b/crates/storage-mongodb/tests/migrations_locks.rs @@ -53,7 +53,7 @@ async fn two_migrators_started_together_apply_once_and_the_other_finds_nothing() applied.sort_unstable(); assert_eq!( applied, - [0, 10], + [0, 11], "one applies every step and the other finds nothing" ); assert_eq!( diff --git a/crates/storage-mongodb/tests/migrations_released.rs b/crates/storage-mongodb/tests/migrations_released.rs index fda836c1d..3777620e9 100644 --- a/crates/storage-mongodb/tests/migrations_released.rs +++ b/crates/storage-mongodb/tests/migrations_released.rs @@ -161,7 +161,7 @@ async fn every_released_catalog_upgrades_to_the_definitions_of_a_fresh_install() panic!("{state} is not behind"); }; assert_eq!(version, None, "{state} has no ledger"); - assert_eq!(pending.len(), 10, "{state}"); + assert_eq!(pending.len(), 11, "{state}"); let report = d.migrate().await.expect("migrate"); assert_eq!( (report.from, report.to), diff --git a/crates/storage-postgres/data_migrations/0010_point_in_time_recovery.sql b/crates/storage-postgres/data_migrations/0010_point_in_time_recovery.sql new file mode 100644 index 000000000..6d9ae4190 --- /dev/null +++ b/crates/storage-postgres/data_migrations/0010_point_in_time_recovery.sql @@ -0,0 +1,50 @@ +-- Copyright 2026 ExtendDB contributors +-- SPDX-License-Identifier: Apache-2.0 +-- +-- Migration 10, point_in_time_recovery, PostgreSQL data database. Creates the +-- two tables point-in-time recovery keeps, beside the item data, so a write's +-- history row commits in the write's transaction. No foreign key reaches the +-- catalog, which is another database. IF NOT EXISTS makes a second run change +-- nothing. The name continuous_backups is not reused: migration 5 drops it +-- again under --reconverge. +-- +-- pitr_state: one row for each table whose recovery is enabled, holding the +-- retention in days (1 to 35) and the enable instant in microseconds since +-- the Unix epoch. No row means recovery is disabled. pruned_to_us is the +-- cutoff of the latest prune that removed a row (0 before one): no state +-- before it is held. pinned_at_us is the earliest time a reconstruct in +-- progress reads, which no prune passes until pin_expires_us; both are NULL +-- until the first reconstruct, and a pin past pin_expires_us holds nothing. +CREATE TABLE IF NOT EXISTS pitr_state ( + table_id TEXT PRIMARY KEY, + recovery_period_days INTEGER NOT NULL, + enabled_at_us BIGINT NOT NULL, + pruned_to_us BIGINT NOT NULL DEFAULT 0, + pinned_at_us BIGINT, + pin_expires_us BIGINT +); + +-- item_history: one row for each committed mutation of a table in +-- pitr_state. pk is the data table's pk; sk is the sort key as text (S as +-- stored, N normalized, B hex) and '' for a table without one; item_data +-- is the item after the mutation, or NULL for a delete. +-- +-- seq orders the rows of one key in commit order: two writes to one item +-- take the item's row lock in turn, and each draws its seq after its data +-- write, so the second draws the higher value. That needs the sequence to +-- hand out values in the order of the calls across sessions, which a cache +-- of more than one value per session breaks; the ALTER sets the cache to 1 +-- again on a database where it was changed. +CREATE TABLE IF NOT EXISTS item_history ( + seq BIGSERIAL PRIMARY KEY, + table_id TEXT NOT NULL, + pk TEXT COLLATE "C" NOT NULL, + sk TEXT COLLATE "C" NOT NULL, + commit_time_us BIGINT NOT NULL, + item_data JSONB +); + +ALTER SEQUENCE item_history_seq_seq CACHE 1; + +CREATE INDEX IF NOT EXISTS idx_item_history_key ON item_history (table_id, pk, sk, seq); +CREATE INDEX IF NOT EXISTS idx_item_history_table_seq ON item_history (table_id, seq); diff --git a/crates/storage-postgres/src/migrations/mod.rs b/crates/storage-postgres/src/migrations/mod.rs index 0f360ac1d..9f1768aa0 100644 --- a/crates/storage-postgres/src/migrations/mod.rs +++ b/crates/storage-postgres/src/migrations/mod.rs @@ -167,6 +167,16 @@ pub fn migration(id: MigrationId) -> Migration { "0009_warm_throughput.sql" )], }, + MigrationId::PointInTimeRecovery => Migration::Applies { + // Declared `Kind::Breaking` for the reason migration 8 gives. + kind: Kind::Breaking, + convergent: true, + bodies: vec![file_body!( + StoreKind::Data, + "data_migrations", + "0010_point_in_time_recovery.sql" + )], + }, } } @@ -192,7 +202,8 @@ pub(crate) const fn runs_in(id: MigrationId) -> RunsIn { | MigrationId::BackupCatalogIndex | MigrationId::StreamPending | MigrationId::RestoreProgress - | MigrationId::WarmThroughput => RunsIn::Transaction, + | MigrationId::WarmThroughput + | MigrationId::PointInTimeRecovery => RunsIn::Transaction, MigrationId::IndexTableBaseKey => RunsIn::Autocommit, } } diff --git a/crates/storage-sqlite/migrations/0010_point_in_time_recovery.sql b/crates/storage-sqlite/migrations/0010_point_in_time_recovery.sql new file mode 100644 index 000000000..5089bf993 --- /dev/null +++ b/crates/storage-sqlite/migrations/0010_point_in_time_recovery.sql @@ -0,0 +1,40 @@ +-- Copyright 2026 ExtendDB contributors +-- SPDX-License-Identifier: Apache-2.0 +-- +-- Migration 10, point_in_time_recovery, SQLite. Creates the two tables +-- point-in-time recovery keeps. IF NOT EXISTS makes a second run change +-- nothing. The name continuous_backups is not reused: migration 5 drops it +-- again under --reconverge. +-- +-- pitr_state: one row for each table whose recovery is enabled, holding the +-- retention in days (1 to 35) and the enable instant in microseconds since +-- the Unix epoch. No row means recovery is disabled. pruned_to_us is the +-- cutoff of the latest prune that removed a row (0 before one): no state +-- before it is held. pinned_at_us is the earliest time a reconstruct in +-- progress reads, which no prune passes until pin_expires_us; both are NULL +-- until the first reconstruct, and a pin past pin_expires_us holds nothing. +CREATE TABLE IF NOT EXISTS pitr_state ( + table_id TEXT PRIMARY KEY REFERENCES tables(table_id) ON DELETE CASCADE, + recovery_period_days INTEGER NOT NULL, + enabled_at_us INTEGER NOT NULL, + pruned_to_us INTEGER NOT NULL DEFAULT 0, + pinned_at_us INTEGER, + pin_expires_us INTEGER +); + +-- item_history: one row for each committed mutation of a table in +-- pitr_state, written in the mutation's transaction. seq orders the rows of +-- the database in commit order. pk and sk hold the item's key as the table's +-- data table stores it (sk is NULL for a table without a sort key); +-- item_data is the item after the mutation as JSON, or NULL for a delete. +CREATE TABLE IF NOT EXISTS item_history ( + seq INTEGER PRIMARY KEY, + table_id TEXT NOT NULL REFERENCES tables(table_id) ON DELETE CASCADE, + pk TEXT NOT NULL, + sk, + commit_time_us INTEGER NOT NULL, + item_data TEXT +); + +CREATE INDEX IF NOT EXISTS idx_item_history_key ON item_history (table_id, pk, sk, seq); +CREATE INDEX IF NOT EXISTS idx_item_history_table_seq ON item_history (table_id, seq); diff --git a/crates/storage-sqlite/src/migrations/golden_tests.rs b/crates/storage-sqlite/src/migrations/golden_tests.rs index d0a2d135b..36d038bc5 100644 --- a/crates/storage-sqlite/src/migrations/golden_tests.rs +++ b/crates/storage-sqlite/src/migrations/golden_tests.rs @@ -780,3 +780,35 @@ async fn migration_9_adds_the_warm_throughput_columns() { ); golden("0009_warm_throughput", &text); } + +/// Migration 10 creates `pitr_state` and `item_history` on a released +/// catalog, both empty, and running it again changes nothing. +#[tokio::test] +async fn migration_10_creates_the_point_in_time_recovery_tables() { + let db = temp_db("m10"); + load_dump(&db, DEV_0_1_14).await; + let tables = "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' \ + AND name IN ('pitr_state', 'item_history')"; + assert_eq!( + count(&db, tables).await, + 0, + "the 0.1.14 catalog predates them" + ); + run(&db).await.expect("migrate"); + let report = reconverge(&db).await.expect("reconverge"); + assert!(report.applied.iter().any(|s| s.id == 10), "{report:?}"); + assert_eq!( + status(&db).await, + CatalogStatus::Current { version: LATEST } + ); + let mut text = String::new(); + for sql in [ + "SELECT type, name, tbl_name, sql FROM sqlite_master \ + WHERE tbl_name IN ('pitr_state', 'item_history') ORDER BY type, name", + "SELECT COUNT(*) FROM pitr_state", + "SELECT COUNT(*) FROM item_history", + ] { + text.push_str(&dump(&db, sql).await); + } + golden("0010_point_in_time_recovery", &text); +} diff --git a/crates/storage-sqlite/src/migrations/mod.rs b/crates/storage-sqlite/src/migrations/mod.rs index 62211a0c3..2f30cd017 100644 --- a/crates/storage-sqlite/src/migrations/mod.rs +++ b/crates/storage-sqlite/src/migrations/mod.rs @@ -138,6 +138,15 @@ pub fn migration(id: MigrationId) -> Migration { m0009_warm_throughput::SOURCE, )], }, + MigrationId::PointInTimeRecovery => Migration::Applies { + kind: Kind::Breaking, + convergent: true, + bodies: vec![file_body!( + StoreKind::Catalog, + "migrations", + "0010_point_in_time_recovery.sql" + )], + }, } } diff --git a/crates/storage-sqlite/tests/fixtures/golden/0010_point_in_time_recovery.txt b/crates/storage-sqlite/tests/fixtures/golden/0010_point_in_time_recovery.txt new file mode 100644 index 000000000..b2c3a8dfc --- /dev/null +++ b/crates/storage-sqlite/tests/fixtures/golden/0010_point_in_time_recovery.txt @@ -0,0 +1,24 @@ +-- SELECT type, name, tbl_name, sql FROM sqlite_master WHERE tbl_name IN ('pitr_state', 'item_history') ORDER BY type, name +index idx_item_history_key item_history CREATE INDEX idx_item_history_key ON item_history (table_id, pk, sk, seq) +index idx_item_history_table_seq item_history CREATE INDEX idx_item_history_table_seq ON item_history (table_id, seq) +index sqlite_autoindex_pitr_state_1 pitr_state NULL +table item_history item_history CREATE TABLE item_history ( + seq INTEGER PRIMARY KEY, + table_id TEXT NOT NULL REFERENCES tables(table_id) ON DELETE CASCADE, + pk TEXT NOT NULL, + sk, + commit_time_us INTEGER NOT NULL, + item_data TEXT +) +table pitr_state pitr_state CREATE TABLE pitr_state ( + table_id TEXT PRIMARY KEY REFERENCES tables(table_id) ON DELETE CASCADE, + recovery_period_days INTEGER NOT NULL, + enabled_at_us INTEGER NOT NULL, + pruned_to_us INTEGER NOT NULL DEFAULT 0, + pinned_at_us INTEGER, + pin_expires_us INTEGER +) +-- SELECT COUNT(*) FROM pitr_state +0 +-- SELECT COUNT(*) FROM item_history +0 diff --git a/crates/storage/src/migrations/registry.rs b/crates/storage/src/migrations/registry.rs index 95ff9be2e..30ffe7af4 100644 --- a/crates/storage/src/migrations/registry.rs +++ b/crates/storage/src/migrations/registry.rs @@ -93,6 +93,12 @@ migration_ids! { /// a global secondary index is stored. WarmThroughput = 9, "warm_throughput", "add a nullable warm_throughput column to the tables and indexes catalogs"; + /// Creates `pitr_state`, a table's point-in-time recovery setting, and + /// `item_history`, one row for each committed mutation of a table whose + /// recovery is enabled, which a restore reads. + PointInTimeRecovery = 10, "point_in_time_recovery", + "create pitr_state and item_history, the point-in-time recovery setting and item \ + history of each table"; } /// Highest id that has shipped in a release. Bodies at or below it are frozen: diff --git a/docs/getting-started.md b/docs/getting-started.md index 1978740ba..60910c579 100755 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -269,7 +269,7 @@ Config: extenddb.toml Catalog database: extenddb_catalog --- Checking catalog migrations... - OK: Current: the catalog is at migration 9 + OK: Current: the catalog is at migration 10 --- Checking catalog connection... OK: Connected to catalog. --- Checking data database... @@ -287,7 +287,7 @@ extenddb runs as a daemon (background process) and logs to syslog. On startup it ```bash ./target/release/extenddb serve --config extenddb.toml -# extenddb 0.1.14 (catalog migration 9) starting on 127.0.0.1:18443 +# extenddb 0.1.14 (catalog migration 10) starting on 127.0.0.1:18443 # storage: postgres (postgresql://extenddb:***@localhost:5432/extenddb_catalog) ``` @@ -1386,7 +1386,7 @@ Each runner requires its tools to be installed. The runner checks prerequisites ```bash ./target/release/extenddb version # extenddb 0.1.14 -# catalog migration 9 (postgres) +# catalog migration 10 (postgres) # commit abc1234 # built 2026-04-17T12:00:00Z ``` diff --git a/docs/manuals/04-quickstart-setup-guide.md b/docs/manuals/04-quickstart-setup-guide.md index d62fabfb2..54ecbbaee 100755 --- a/docs/manuals/04-quickstart-setup-guide.md +++ b/docs/manuals/04-quickstart-setup-guide.md @@ -130,7 +130,7 @@ Check the version: ```bash ./target/release/extenddb version # extenddb 0.1.14 -# catalog migration 9 (postgres) +# catalog migration 10 (postgres) # commit abc1234 # built 2026-04-17T12:00:00Z ``` @@ -172,7 +172,7 @@ Config: extenddb.toml Catalog database: extenddb_catalog --- Checking catalog migrations... - OK: Current: the catalog is at migration 9 + OK: Current: the catalog is at migration 10 --- Checking catalog connection... OK: Connected to catalog. --- Checking data database... diff --git a/docs/manuals/07-upgrade-manual.md b/docs/manuals/07-upgrade-manual.md index 7f244409f..0cf8a6c7c 100755 --- a/docs/manuals/07-upgrade-manual.md +++ b/docs/manuals/07-upgrade-manual.md @@ -4,10 +4,12 @@ ## Current Status -This release versions the PostgreSQL, SQLite, Cassandra and MongoDB catalogs by a migration ledger and carries catalog migrations 1 to 9, which `extenddb migrate --list` prints. **Every deployment created by a 0.1.x release runs `extenddb migrate --yes` once** before a server of this release starts on it: the server refuses a catalog with pending migrations. Migration 1 brings the catalog of every 0.1.x release to the v1.0 schema, migration 5 drops `continuous_backups`, a table and on MongoDB a collection (see [Catalog 0.0.5](#catalog-005)), and migration 6 adds the backup catalog index and the restore job records ([below](#migration-6-the-backup-catalog-index)). Migration 7 adds the `stream_pending` table to the PostgreSQL data database, where a write stores its stream record until the record gets its sequence number after the write commits; it does not apply to the other backends. Migration 8 adds the `restore_progress` table to the PostgreSQL data database, where a store-backed restore records each data file it applied; it does not apply on the other backends either. +This release versions the PostgreSQL, SQLite, Cassandra and MongoDB catalogs by a migration ledger and carries catalog migrations 1 to 10, which `extenddb migrate --list` prints. **Every deployment created by a 0.1.x release runs `extenddb migrate --yes` once** before a server of this release starts on it: the server refuses a catalog with pending migrations. Migration 1 brings the catalog of every 0.1.x release to the v1.0 schema, migration 5 drops `continuous_backups`, a table and on MongoDB a collection (see [Catalog 0.0.5](#catalog-005)), and migration 6 adds the backup catalog index and the restore job records ([below](#migration-6-the-backup-catalog-index)). Migration 7 adds the `stream_pending` table to the PostgreSQL data database, where a write stores its stream record until the record gets its sequence number after the write commits; it does not apply to the other backends. Migration 8 adds the `restore_progress` table to the PostgreSQL data database, where a store-backed restore records each data file it applied; it does not apply on the other backends either. Migration 9, `warm_throughput`, adds a nullable `warm_throughput` column to the `tables` and `indexes` catalog tables on PostgreSQL, SQLite and Cassandra. It holds the `WarmThroughput` a client sets on a table or a global secondary index, which `DescribeTable` reports; nothing acts on the value (see [Differences from DynamoDB](../differences-from-dynamodb.md)). Tables created before the migration report the defaults for their billing mode. On MongoDB it changes nothing, since the catalog collections have no validator, and the ledger records it as not applicable. Migration 9 is breaking: to go back to a release without it, restore the backup taken before the upgrade (see [Rollback Procedure](#rollback-procedure)). +Migration 10, `point_in_time_recovery`, adds two empty tables, `pitr_state` and `item_history`, to the SQLite catalog and to the PostgreSQL data database, for point-in-time recovery state; it does not apply on MongoDB or Cassandra. Migration 10 is breaking: to go back to a release without it, restore the backup taken before the upgrade (see [Rollback Procedure](#rollback-procedure)). + Configure `[backup.store]` before the release that stops creating in-catalog backups. From this release CreateBackup writes to the configured store on PostgreSQL, SQLite and MongoDB, and is asynchronous there as on the service: the response is `CREATING` and the backup becomes `AVAILABLE` once the upload ends, so a client that deletes a backup it just created has to poll DescribeBackup first (the previous release returned `AVAILABLE`). A store-backed backup is listed, described, deleted and restored like any other. `RestoreTableFromBackup` of a backup in the configured store runs the restore job from the store on every backend (PostgreSQL, SQLite, MongoDB and Cassandra); in-catalog backups, those of the previous release included, restore as before. An S3 store needs `spool_path` set (the restore's download area) on PostgreSQL, MongoDB and Cassandra, and that must be a directory of its own, not the database directory; a filesystem store needs no spool. Migration 8 adds the `restore_progress` table to the PostgreSQL data database; run `extenddb migrate --yes` before starting the new binary. The progress rows of a restore in flight live in the data database, so a data database restored from a dump taken before migration 8 has none of them and no `restore_progress` table: the new binary refuses to serve it until `extenddb migrate --yes` has run again, and its reconciler then finds no progress for the restore and resumes it from its first data file into the (empty) target, or `FAILED`s it as `target_not_empty` if the target already holds items, in which case `extenddb backup remove-target --account ` (the admin guide's `backup` section) removes the target and its record and the backup restores again. The same `target_not_empty` state is what a server stopped inside a restore's first data file on MongoDB, or inside its first batch of 10,000 items or 32 MiB on Cassandra, leaves on restart, since both record progress after the write; the command is the remedy there too. `backup remove-target` writes the catalog, so on SQLite the server is stopped for it, as for `purge`. The previous binary resumes no restore (a store-backed restore in flight at a rollback is lost with the catalog rows, as the rollback paragraph below says). Quiesce restores before a downgrade. `kind = "disabled"` is no longer needed to keep a deployment able to restore the backups it takes. Without a store CreateBackup writes the in-catalog format and logs a deprecation warning naming `[backup.store]` on each call; a later release's CreateBackup refuses without one. Cassandra writes the in-catalog format with or without a store. `kind = "disabled"` under `[backup.store]` is the opt-out that keeps a deployment on the in-catalog format on purpose and silences the warning. On file-backed SQLite an absent section is a filesystem store at `backups/` beside the database file, created `0700` at startup; that root must be owned by the extenddb user with mode `0700`, under directories owned by root or that user which others cannot write to. A default root that breaks the rule (a database file in a group-writable directory, say), or whose `extenddb-store.json` names another catalog (a catalog re-created by `destroy` and `init` under the old root), is logged at startup as a warning (`backup store: the default root ... failed its check`) and leaves the store unconfigured, not unhealthy, so the upgrade never stops a server that never configured backups and CreateBackup keeps working on it; the same root configured explicitly is a startup error or an unhealthy store. The warning applies only while the catalog has no backup rows for the default root (an upgrade, a fresh deployment, a catalog re-created by `destroy` and `init`): once a server has written rows for it, the default root is handled as a configured one, so its backups never vanish behind a warning (a failed identity check is an unhealthy store, reported on `/health`; a root that can no longer be opened is a startup error naming it). Instances that share a catalog must share one store root, an S3 bucket and prefix or a shared mount: a second instance pointed at a host-local directory under the same configured path starts with the store unhealthy (`/health` reports `backup_store` unhealthy; the admin guide's `[backup.store]` section says what the identity file and the manifest probe check). `extenddb destroy` leaves the store; `extenddb backup purge --confirm` is what deletes its backups, and a catalog re-created with `init` needs the old root's `extenddb-store.json` removed or a new root, since the file names the old catalog. A backup that a 0.1.x release took can restore without its global and local secondary indexes: a SQLite backup taken by 0.1.14 records only the table's key schema and attribute definitions, and its restore creates the table with its items and no index. Take a new backup of each table after upgrading to v1.0, and keep the 0.1.x backups for their items only. @@ -43,7 +45,7 @@ Exit status 6 means the migration lock, or a running server, was still held when `verify` also checks the connection to the data database. When that check fails, `verify` exits 1 on a Behind catalog instead of 3, because `migrate --yes` cannot apply the pending migrations while the data database cannot be reached. It still exits 4 or 5 for the statuses that need a person. -A migration is additive when the previous release serves the catalog correctly with it applied, and breaking otherwise. Migrations 1 to 7 and 9 are breaking. Migration 8 creates a table in the PostgreSQL data database that the previous release never reads; on PostgreSQL it is declared breaking only because the job that proves an additive declaration against the old binary does not yet exist (CONTRIBUTING, step 4 of adding a migration), and on SQLite, MongoDB and Cassandra it does not apply (`migrate --list` prints it as not applicable, and `migrate` counts it as additive). Do not run a 0.1.x server against a migrated catalog. A 0.1.x binary that writes to a migrated catalog (its `migrate --yes`, or a SQLite dev server) leaves it WrittenByOlderRelease. `migrate --yes --reconverge` repairs it by running the convergent migrations again. Migration 2, which rewrites stored policies, is not convergent: the repair skips it and prints a line that says so (see the v0.1.14 notes below). +A migration is additive when the previous release serves the catalog correctly with it applied, and breaking otherwise. Every migration is declared breaking on each backend it applies to: migrations 1, 2, 5 and 6 on all four backends, 3, 4 and 10 on PostgreSQL and SQLite, 7 and 8 on PostgreSQL, and 9 on PostgreSQL, SQLite and Cassandra. On a backend where a migration does not apply, `migrate --list` prints it as not applicable and `migrate` counts it as additive. Migrations 8 and 10 create tables in the PostgreSQL data database that the previous release never reads; on PostgreSQL they are declared breaking only because the job that proves an additive declaration against the old binary does not yet exist (CONTRIBUTING, step 4 of adding a migration). Do not run a 0.1.x server against a migrated catalog. A 0.1.x binary that writes to a migrated catalog (its `migrate --yes`, or a SQLite dev server) leaves it WrittenByOlderRelease. `migrate --yes --reconverge` repairs it by running the convergent migrations again. Migration 2, which rewrites stored policies, is not convergent: the repair skips it and prints a line that says so (see the v0.1.14 notes below). ### Migration 6: the backup catalog index diff --git a/docs/manuals/08-install-linux.md b/docs/manuals/08-install-linux.md index cf6a977c8..a4980368f 100755 --- a/docs/manuals/08-install-linux.md +++ b/docs/manuals/08-install-linux.md @@ -118,7 +118,7 @@ Expected: ``` === extenddb verify === ... - OK: Current: the catalog is at migration 9 + OK: Current: the catalog is at migration 10 ... === HEALTHY: All checks passed === ``` diff --git a/docs/manuals/09-install-macos.md b/docs/manuals/09-install-macos.md index a87f6c685..8421d506c 100755 --- a/docs/manuals/09-install-macos.md +++ b/docs/manuals/09-install-macos.md @@ -96,7 +96,7 @@ Expected: ``` === extenddb verify === ... - OK: Current: the catalog is at migration 9 + OK: Current: the catalog is at migration 10 ... === HEALTHY: All checks passed === ``` diff --git a/docs/schema/postgres.txt b/docs/schema/postgres.txt index 0054b32ed..bcfefa84e 100644 --- a/docs/schema/postgres.txt +++ b/docs/schema/postgres.txt @@ -348,6 +348,8 @@ sequence stream_seq: bigint START 1 INCREMENT 1 table extenddb_migrations table gsi_pending table idempotency_tokens +table item_history +table pitr_state table restore_progress table schema_history table stream_pending @@ -373,6 +375,18 @@ column idempotency_tokens.account_id: text NOT NULL column idempotency_tokens.token: text NOT NULL column idempotency_tokens.fingerprint: text NOT NULL column idempotency_tokens.created_at: timestamp with time zone NOT NULL DEFAULT now() +column item_history.seq: bigint NOT NULL DEFAULT nextval('item_history_seq_seq'::regclass) +column item_history.table_id: text NOT NULL +column item_history.pk: text NOT NULL COLLATE "C" +column item_history.sk: text NOT NULL COLLATE "C" +column item_history.commit_time_us: bigint NOT NULL +column item_history.item_data: jsonb +column pitr_state.table_id: text NOT NULL +column pitr_state.recovery_period_days: integer NOT NULL +column pitr_state.enabled_at_us: bigint NOT NULL +column pitr_state.pruned_to_us: bigint NOT NULL DEFAULT 0 +column pitr_state.pinned_at_us: bigint +column pitr_state.pin_expires_us: bigint column restore_progress.table_id: text NOT NULL column restore_progress.file_index: bigint NOT NULL column restore_progress.items_applied: bigint NOT NULL @@ -405,9 +419,13 @@ index gsi_pending_pkey: CREATE UNIQUE INDEX gsi_pending_pkey ON public.gsi_pendi index idempotency_tokens_pkey: CREATE UNIQUE INDEX idempotency_tokens_pkey ON public.idempotency_tokens USING btree (account_id, token) index idx_gsi_pending_claim: CREATE INDEX idx_gsi_pending_claim ON public.gsi_pending USING btree (worker_partition, ready_at, id) index idx_idempotency_tokens_created: CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at) +index idx_item_history_key: CREATE INDEX idx_item_history_key ON public.item_history USING btree (table_id, pk, sk, seq) +index idx_item_history_table_seq: CREATE INDEX idx_item_history_table_seq ON public.item_history USING btree (table_id, seq) index idx_stream_records_created: CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at) index idx_stream_shards_table: CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id) index idx_vector_index_holds_table: CREATE INDEX idx_vector_index_holds_table ON public.vector_index_holds USING btree (table_id) +index item_history_pkey: CREATE UNIQUE INDEX item_history_pkey ON public.item_history USING btree (seq) +index pitr_state_pkey: CREATE UNIQUE INDEX pitr_state_pkey ON public.pitr_state USING btree (table_id) index restore_progress_pkey: CREATE UNIQUE INDEX restore_progress_pkey ON public.restore_progress USING btree (table_id, file_index) index schema_history_pkey: CREATE UNIQUE INDEX schema_history_pkey ON public.schema_history USING btree (filename) index stream_pending_pkey: CREATE UNIQUE INDEX stream_pending_pkey ON public.stream_pending USING btree (shard_id, id) @@ -417,6 +435,8 @@ index vector_index_holds_pkey: CREATE UNIQUE INDEX vector_index_holds_pkey ON pu constraint extenddb_migrations.extenddb_migrations_pkey: PRIMARY KEY (id) constraint gsi_pending.gsi_pending_pkey: PRIMARY KEY (id) constraint idempotency_tokens.idempotency_tokens_pkey: PRIMARY KEY (account_id, token) +constraint item_history.item_history_pkey: PRIMARY KEY (seq) +constraint pitr_state.pitr_state_pkey: PRIMARY KEY (table_id) constraint restore_progress.restore_progress_pkey: PRIMARY KEY (table_id, file_index) constraint schema_history.schema_history_pkey: PRIMARY KEY (filename) constraint stream_pending.stream_pending_pkey: PRIMARY KEY (shard_id, id) @@ -426,5 +446,6 @@ constraint stream_records.stream_records_shard_id_fkey: FOREIGN KEY (shard_id) R constraint stream_shards.stream_shards_pkey: PRIMARY KEY (shard_id) constraint vector_index_holds.vector_index_holds_pkey: PRIMARY KEY (table_id, index_id) sequence gsi_pending_id_seq: bigint START 1 INCREMENT 1 +sequence item_history_seq_seq: bigint START 1 INCREMENT 1 sequence stream_pending_id_seq: bigint START 1 INCREMENT 1 sequence stream_seq: bigint START 1 INCREMENT 1 diff --git a/docs/schema/sqlite.txt b/docs/schema/sqlite.txt index 33a97d696..28fdae830 100644 --- a/docs/schema/sqlite.txt +++ b/docs/schema/sqlite.txt @@ -5,6 +5,8 @@ index idx_backup_items_arn on backup_items: CREATE INDEX idx_backup_items_arn ON index idx_backups_table on backups: CREATE INDEX idx_backups_table ON backups (account_id, table_name) index idx_gsi_pending_claim on gsi_pending: CREATE INDEX idx_gsi_pending_claim ON gsi_pending (worker_partition, ready_at, id) index idx_idempotency_tokens_created on idempotency_tokens: CREATE INDEX idx_idempotency_tokens_created ON idempotency_tokens (created_at) +index idx_item_history_key on item_history: CREATE INDEX idx_item_history_key ON item_history (table_id, pk, sk, seq) +index idx_item_history_table_seq on item_history: CREATE INDEX idx_item_history_table_seq ON item_history (table_id, seq) index idx_login_attempts_principal_time on login_attempts: CREATE INDEX idx_login_attempts_principal_time ON login_attempts (principal, attempted_at DESC) index idx_login_attempts_source_ip_time on login_attempts: CREATE INDEX idx_login_attempts_source_ip_time ON login_attempts (source_ip, attempted_at DESC) WHERE source_ip IS NOT NULL index idx_metrics_bucket on metrics: CREATE INDEX idx_metrics_bucket ON metrics (bucket) @@ -35,8 +37,10 @@ table iam_user_tags on iam_user_tags: CREATE TABLE iam_user_tags ( account_id TE table iam_users on iam_users: CREATE TABLE iam_users ( account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, user_name TEXT NOT NULL, user_arn TEXT NOT NULL UNIQUE, password_hash TEXT, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), PRIMARY KEY (account_id, user_name) ) table idempotency_tokens on idempotency_tokens: CREATE TABLE idempotency_tokens ( account_id TEXT NOT NULL, token TEXT NOT NULL, fingerprint TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), PRIMARY KEY (account_id, token) ) table indexes on indexes: CREATE TABLE indexes ( table_id TEXT NOT NULL, index_id TEXT NOT NULL, index_name TEXT NOT NULL, index_type TEXT NOT NULL, key_schema TEXT NOT NULL, projection TEXT NOT NULL, index_status TEXT NOT NULL DEFAULT 'ACTIVE', provisioned_throughput TEXT, propagation_delay_ms INTEGER, warm_throughput TEXT, PRIMARY KEY (table_id, index_name), CONSTRAINT indexes_table_id_fkey FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, CONSTRAINT chk_propagation_delay_ms_non_negative CHECK (propagation_delay_ms IS NULL OR propagation_delay_ms >= 0) ) +table item_history on item_history: CREATE TABLE item_history ( seq INTEGER PRIMARY KEY, table_id TEXT NOT NULL REFERENCES tables(table_id) ON DELETE CASCADE, pk TEXT NOT NULL, sk, commit_time_us INTEGER NOT NULL, item_data TEXT ) table login_attempts on login_attempts: CREATE TABLE login_attempts ( principal TEXT NOT NULL, attempted_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), success INTEGER NOT NULL, source_ip TEXT ) table metrics on metrics: CREATE TABLE metrics ( bucket TEXT NOT NULL, metric TEXT NOT NULL, table_name TEXT NOT NULL DEFAULT '', index_name TEXT NOT NULL DEFAULT '', operation TEXT NOT NULL DEFAULT '', sum REAL NOT NULL DEFAULT 0, count INTEGER NOT NULL DEFAULT 0, min REAL NOT NULL DEFAULT 9e999, max REAL NOT NULL DEFAULT -9e999, PRIMARY KEY (bucket, metric, table_name, index_name, operation) ) +table pitr_state on pitr_state: CREATE TABLE pitr_state ( table_id TEXT PRIMARY KEY REFERENCES tables(table_id) ON DELETE CASCADE, recovery_period_days INTEGER NOT NULL, enabled_at_us INTEGER NOT NULL, pruned_to_us INTEGER NOT NULL DEFAULT 0, pinned_at_us INTEGER, pin_expires_us INTEGER ) table schema_history on schema_history: CREATE TABLE schema_history ( filename TEXT PRIMARY KEY, applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) ) table seq_counters on seq_counters: CREATE TABLE seq_counters ( name TEXT PRIMARY KEY, value INTEGER NOT NULL DEFAULT 0 ) table settings on settings: CREATE TABLE settings ( key TEXT PRIMARY KEY, value TEXT NOT NULL ) diff --git a/tests/test_resource_tag_migration.py b/tests/test_resource_tag_migration.py index 6ae02872b..67e1a179b 100644 --- a/tests/test_resource_tag_migration.py +++ b/tests/test_resource_tag_migration.py @@ -305,10 +305,14 @@ def _as_released_catalog(sqlite_path: str) -> None: left: no ledger rows (the runner reads an empty ledger as it reads a missing one), a dotted catalog_version, the marker row every released baseline seeded, and the baseline's shape for every table migration 1 - checks: the objects migration 6 (backup_catalog_index) and migration 9 - (warm_throughput) add are removed, since no release before 1.0 had them - and the check after migration 1 refuses a `table_restores` carrying its - twelve job columns and a `tables` or `indexes` carrying warm_throughput. + checks: the objects migration 6 (backup_catalog_index), migration 9 + (warm_throughput) and migration 10 (point_in_time_recovery) add are + removed, since no release before 1.0 had them and the check after + migration 1 refuses a `table_restores` carrying its twelve job columns and + a `tables` or `indexes` carrying warm_throughput. Migration 10's tables + are dropped so that the run creates them, as it does on a released + catalog; the check after migration 1 does not read them, and no + assertion here does. The runner then applies every migration, the condition-key rewrite included, as it does on a released catalog.""" conn = sqlite3.connect(sqlite_path) @@ -322,6 +326,9 @@ def _as_released_catalog(sqlite_path: str) -> None: # Migration 9 adds warm_throughput to tables and indexes. for table in ("tables", "indexes"): conn.execute(f"ALTER TABLE {table} DROP COLUMN warm_throughput") + # Migration 10 adds pitr_state and item_history. + conn.execute("DROP TABLE IF EXISTS item_history") + conn.execute("DROP TABLE IF EXISTS pitr_state") conn.execute("DELETE FROM extenddb_migrations") conn.execute( "UPDATE settings SET value = '0.0.3' WHERE key = 'catalog_version'" From 205f620f73b6a3b99b4cdb509f3d5ed9043209f0 Mon Sep 17 00:00:00 2001 From: Elise Ong Date: Tue, 10 Nov 2026 12:25:22 -0500 Subject: [PATCH 2/6] feat(storage): item history on SQLite and PostgreSQL On the service, a restore to a time T holds the latest image of each key committed at or before T, and a key deleted by T is absent (a restore between the third and fourth steps of a four-step write timeline returned the one surviving item, two samples). Items that exist at the enable are in every restore after it, at their image then; a disable drops the history and a re-enable starts a new window; the window is RecoveryPeriodInDays, 1 to 35 days, 35 by default (two samples of each). extenddb_storage::pitr (behind a new pitr feature) is the per-backend store the engine reads: pitr_state, enable_pitr, disable_pitr, reconstruct_page, prune_before, pitr_tables and remove_orphaned_history, with the shared retention sweep and its worker. SQLite and PostgreSQL implement it behind their own pitr features, and DataEngine::as_pitr returns the store. - Capture: PutItem, UpdateItem, DeleteItem and TransactWriteItems record the mutation in the write's own transaction, after the data write, and only for a table with a pitr_state row, read in that transaction. A write that changed nothing (a failed condition, a delete of a missing item, a REMOVE-only update of a missing item, a condition check) records nothing; on PostgreSQL a delete of a missing key holds no row lock, so a recorded one could order after a concurrent first put of the key. A transaction's members share one commit time. A PostgreSQL put or delete with no condition, index or stream stays one autocommit statement, which writes the history row in the same statement, so the feature costs a table without recovery nothing measurable and adds no create races. - Enable: records the setting and copies every item into the history at the enable instant; an enable on an enabled table changes only a given period. Enable and disable exclude the table's writers and each other while they run (SQLite's write lock and BEGIN IMMEDIATE, PostgreSQL LOCK TABLE IN SHARE ROW EXCLUSIVE MODE), so no write is missing from both the baseline and the history, and two enables of one table both succeed. An enable or disable of a table a DeleteTable removed is TableNotFound. - Reconstruct: one page of the rows committed at or before T that no later row of the key at or before T supersedes, deletes omitted, by an anti-join on (table_id, pk, sk, seq) paged on seq. PostgreSQL keys the sort key as text (N plain decimal, B hex, empty without one) so the baseline computes the same key from the typed columns. Each page pins its time against the sweep for 15 minutes; a page whose state a prune has passed answers SnapshotExpired rather than a partial state. - Retention: an hourly sweep prunes each table to the floor of its window, in batches under a deadline, keeping every state from the floor on and never passing a live pin. The cutoff of the latest prune is recorded, and the floor is never before it, so a period raised after a prune reports only a window the history holds. A stored period outside 1 to 35 is an error. On PostgreSQL it also removes the history and setting of tables the catalog no longer has, since the catalog is another database. The worker records the PitrRetention metric. Verified: the shared checks of extenddb_storage::pitr::conformance run on both backends, with the PostgreSQL cases for concurrent writers, an uncommitted first put, an enable or disable with a write in flight, two enables at once, a sweep between reconstruct pages and deleted tables in a separate catalog database (the PostgreSQL file passed 20 runs in a row); a negative control for each rule of the stores, the write hooks, the writer exclusion, the pins and the sweep; release-build put throughput on a table without recovery within 1.2 percent of a build without the feature (sequential and 8 writers, 6 runs each). --- crates/core/src/metrics/types.rs | 3 + crates/storage-postgres/Cargo.toml | 3 + .../storage-postgres/src/data/data_engine.rs | 5 + .../storage-postgres/src/data/delete_item.rs | 89 ++- crates/storage-postgres/src/data/put_item.rs | 84 ++- .../storage-postgres/src/data/transactions.rs | 22 + .../storage-postgres/src/data/update_item.rs | 9 + crates/storage-postgres/src/lib.rs | 21 +- crates/storage-postgres/src/pitr.rs | 575 ++++++++++++++++++ .../storage-postgres/src/pitr/autocommit.rs | 132 ++++ .../tests/pitr_item_history.rs | 555 +++++++++++++++++ .../tests/pitr_support/mod.rs | 381 ++++++++++++ crates/storage-sqlite/Cargo.toml | 3 + crates/storage-sqlite/src/data/data_engine.rs | 5 + crates/storage-sqlite/src/data/delete_item.rs | 9 + crates/storage-sqlite/src/data/put_item.rs | 9 + .../storage-sqlite/src/data/transactions.rs | 17 + crates/storage-sqlite/src/data/update_item.rs | 9 + crates/storage-sqlite/src/hooks.rs | 18 +- crates/storage-sqlite/src/lib.rs | 2 + crates/storage-sqlite/src/pitr.rs | 448 ++++++++++++++ crates/storage-sqlite/src/pitr_tests.rs | 400 ++++++++++++ crates/storage/Cargo.toml | 8 + crates/storage/src/lib.rs | 9 + crates/storage/src/pitr.rs | 340 +++++++++++ crates/storage/src/pitr/conformance.rs | 419 +++++++++++++ crates/storage/src/pitr/conformance/enable.rs | 236 +++++++ crates/storage/src/pitr/conformance/pins.rs | 109 ++++ .../storage/src/pitr/conformance/retention.rs | 158 +++++ crates/storage/src/pitr/conformance/writes.rs | 262 ++++++++ crates/storage/src/pitr/worker_tests.rs | 145 +++++ 31 files changed, 4437 insertions(+), 48 deletions(-) create mode 100644 crates/storage-postgres/src/pitr.rs create mode 100644 crates/storage-postgres/src/pitr/autocommit.rs create mode 100644 crates/storage-postgres/tests/pitr_item_history.rs create mode 100644 crates/storage-postgres/tests/pitr_support/mod.rs create mode 100644 crates/storage-sqlite/src/pitr.rs create mode 100644 crates/storage-sqlite/src/pitr_tests.rs create mode 100644 crates/storage/src/pitr.rs create mode 100644 crates/storage/src/pitr/conformance.rs create mode 100644 crates/storage/src/pitr/conformance/enable.rs create mode 100644 crates/storage/src/pitr/conformance/pins.rs create mode 100644 crates/storage/src/pitr/conformance/retention.rs create mode 100644 crates/storage/src/pitr/conformance/writes.rs create mode 100644 crates/storage/src/pitr/worker_tests.rs diff --git a/crates/core/src/metrics/types.rs b/crates/core/src/metrics/types.rs index 0030ce1b2..f49e08c91 100755 --- a/crates/core/src/metrics/types.rs +++ b/crates/core/src/metrics/types.rs @@ -193,6 +193,8 @@ pub enum QuerySource { TtlSweeper, /// Background: stream record cleanup. StreamCleanup, + /// Background: point-in-time recovery history retention. + PitrRetention, /// Background: idempotency token cleanup. IdempotencyCleanup, /// Background: metrics flush. @@ -228,6 +230,7 @@ impl std::fmt::Display for QuerySource { Self::PollGsiDelay => f.write_str("PollGsiDelay"), Self::TtlSweeper => f.write_str("TtlSweeper"), Self::StreamCleanup => f.write_str("StreamCleanup"), + Self::PitrRetention => f.write_str("PitrRetention"), Self::IdempotencyCleanup => f.write_str("IdempotencyCleanup"), Self::MetricsFlush => f.write_str("MetricsFlush"), Self::MetricsPrune => f.write_str("MetricsPrune"), diff --git a/crates/storage-postgres/Cargo.toml b/crates/storage-postgres/Cargo.toml index b2c47b590..256795da6 100755 --- a/crates/storage-postgres/Cargo.toml +++ b/crates/storage-postgres/Cargo.toml @@ -11,6 +11,9 @@ license.workspace = true # Compiles the snapshot page-size setter the consistency test uses and the # shared conformance check. Production builds do not compile them. test-hooks = ["extenddb-storage/test-hooks"] +# Point-in-time recovery item history: the store and the write hook. Off by +# default; the migration that creates the tables applies either way. +pitr = ["extenddb-storage/pitr"] # Test hooks: the runner without a lock and its catalog connection, the lock # and the registration with another keepalive interval, the serving # session's settings, and the post-check's definitions. diff --git a/crates/storage-postgres/src/data/data_engine.rs b/crates/storage-postgres/src/data/data_engine.rs index 88dd03794..1fce2f7a8 100755 --- a/crates/storage-postgres/src/data/data_engine.rs +++ b/crates/storage-postgres/src/data/data_engine.rs @@ -31,6 +31,11 @@ impl DataEngine for PostgresEngine { self.vector_capable.then_some(self) } + #[cfg(feature = "pitr")] + fn as_pitr(&self) -> Option<&dyn extenddb_storage::pitr::PitrCaptureStore> { + Some(self) + } + fn put_item( &self, key_info: &TableKeyInfo, diff --git a/crates/storage-postgres/src/data/delete_item.rs b/crates/storage-postgres/src/data/delete_item.rs index 9a6dfda93..6ff44b17c 100755 --- a/crates/storage-postgres/src/data/delete_item.rs +++ b/crates/storage-postgres/src/data/delete_item.rs @@ -184,6 +184,16 @@ impl PostgresEngine { ) .await?; } + // Reached only with a row deleted: a missing item returned above. + #[cfg(feature = "pitr")] + crate::pitr::record_item_history_in_tx( + &mut tx, + key_info, + key, + None, + std::time::SystemTime::now(), + ) + .await?; // Persist async GSI work inside the same transaction — one row // per async index, each honoring its own propagation delay. let mut async_enqueued = enqueue_async_indexes( @@ -236,32 +246,46 @@ impl PostgresEngine { Ok(None) } } else { - let delete_sql = format!("DELETE FROM {ddb_table} WHERE pk = $1 AND {sk_col} = $2"); super::gsi_build::pause_before_autocommit_statement().await; - match &sk { - SortKeyValue::S(s) => { - sqlx::query(&delete_sql) - .bind(pk_text.as_ref()) - .bind(s) - .execute(&self.data_pool) - .await - } - SortKeyValue::N(n) => { - sqlx::query(&delete_sql) - .bind(pk_text.as_ref()) - .bind(n) - .execute(&self.data_pool) - .await - } - SortKeyValue::B(b) => { - sqlx::query(&delete_sql) - .bind(pk_text.as_ref()) - .bind(b) - .execute(&self.data_pool) - .await + // Item history, when the table records it and an item was + // deleted, in the same statement as the write. + #[cfg(feature = "pitr")] + crate::pitr::autocommit_delete( + &self.data_pool, + key_info, + pk_text.as_ref(), + Some(&sk), + ) + .await?; + #[cfg(not(feature = "pitr"))] + { + let delete_sql = + format!("DELETE FROM {ddb_table} WHERE pk = $1 AND {sk_col} = $2"); + match &sk { + SortKeyValue::S(s) => { + sqlx::query(&delete_sql) + .bind(pk_text.as_ref()) + .bind(s) + .execute(&self.data_pool) + .await + } + SortKeyValue::N(n) => { + sqlx::query(&delete_sql) + .bind(pk_text.as_ref()) + .bind(n) + .execute(&self.data_pool) + .await + } + SortKeyValue::B(b) => { + sqlx::query(&delete_sql) + .bind(pk_text.as_ref()) + .bind(b) + .execute(&self.data_pool) + .await + } } + .map_err(|e| StorageError::Internal(e.to_string()))?; } - .map_err(|e| StorageError::Internal(e.to_string()))?; self.fix_up_autocommit(stamp, key_info, key).await; Ok(None) } @@ -346,6 +370,16 @@ impl PostgresEngine { ) .await?; } + // Reached only with a row deleted: a missing item returned above. + #[cfg(feature = "pitr")] + crate::pitr::record_item_history_in_tx( + &mut tx, + key_info, + key, + None, + std::time::SystemTime::now(), + ) + .await?; // Persist async GSI work inside the same transaction — one row // per async index, each honoring its own propagation delay. let mut async_enqueued = enqueue_async_indexes( @@ -398,9 +432,14 @@ impl PostgresEngine { Ok(None) } } else { - let delete_sql = format!("DELETE FROM {ddb_table} WHERE pk = $1"); super::gsi_build::pause_before_autocommit_statement().await; - sqlx::query(&delete_sql) + // Item history, when the table records it and an item was + // deleted, in the same statement as the write. + #[cfg(feature = "pitr")] + crate::pitr::autocommit_delete(&self.data_pool, key_info, pk_text.as_ref(), None) + .await?; + #[cfg(not(feature = "pitr"))] + sqlx::query(&format!("DELETE FROM {ddb_table} WHERE pk = $1")) .bind(pk_text.as_ref()) .execute(&self.data_pool) .await diff --git a/crates/storage-postgres/src/data/put_item.rs b/crates/storage-postgres/src/data/put_item.rs index 2fbc3102b..c2d1edf22 100755 --- a/crates/storage-postgres/src/data/put_item.rs +++ b/crates/storage-postgres/src/data/put_item.rs @@ -203,6 +203,15 @@ impl PostgresEngine { ) .await?; } + #[cfg(feature = "pitr")] + crate::pitr::record_item_history_in_tx( + &mut tx, + key_info, + &item, + Some(&item), + std::time::SystemTime::now(), + ) + .await?; // Persist async GSI work inside the same transaction — one row // per async index, each honoring its own propagation delay. let mut async_enqueued = enqueue_async_indexes( @@ -259,18 +268,32 @@ impl PostgresEngine { Ok(None) } } else { - let upsert_sql = format!( - "INSERT INTO {ddb_table} (pk, {sk_col}, item_data) VALUES ($1, $2, $3) \ - ON CONFLICT (pk, {sk_col}) DO UPDATE SET item_data = EXCLUDED.item_data" - ); super::gsi_build::pause_before_autocommit_statement().await; - bind_sk_execute!( - &upsert_sql, + // Item history, when the table records it, in the same + // statement as the write. + #[cfg(feature = "pitr")] + crate::pitr::autocommit_put( + &self.data_pool, + key_info, pk_text.as_str(), - &sk, + Some(&sk), &item_json, - &self.data_pool - )?; + ) + .await?; + #[cfg(not(feature = "pitr"))] + { + let upsert_sql = format!( + "INSERT INTO {ddb_table} (pk, {sk_col}, item_data) VALUES ($1, $2, $3) \ + ON CONFLICT (pk, {sk_col}) DO UPDATE SET item_data = EXCLUDED.item_data" + ); + bind_sk_execute!( + &upsert_sql, + pk_text.as_str(), + &sk, + &item_json, + &self.data_pool + )?; + } self.fix_up_autocommit(stamp, key_info, &item).await; Ok(None) } @@ -381,6 +404,15 @@ impl PostgresEngine { ) .await?; } + #[cfg(feature = "pitr")] + crate::pitr::record_item_history_in_tx( + &mut tx, + key_info, + &item, + Some(&item), + std::time::SystemTime::now(), + ) + .await?; // Persist async GSI work inside the same transaction — one row // per async index, each honoring its own propagation delay. let mut async_enqueued = enqueue_async_indexes( @@ -437,17 +469,31 @@ impl PostgresEngine { Ok(None) } } else { - let upsert_sql = format!( - "INSERT INTO {ddb_table} (pk, item_data) VALUES ($1, $2) \ - ON CONFLICT (pk) DO UPDATE SET item_data = EXCLUDED.item_data" - ); super::gsi_build::pause_before_autocommit_statement().await; - sqlx::query(&upsert_sql) - .bind(pk_text.as_str()) - .bind(&item_json) - .execute(&self.data_pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + // Item history, when the table records it, in the same + // statement as the write. + #[cfg(feature = "pitr")] + crate::pitr::autocommit_put( + &self.data_pool, + key_info, + pk_text.as_str(), + None, + &item_json, + ) + .await?; + #[cfg(not(feature = "pitr"))] + { + let upsert_sql = format!( + "INSERT INTO {ddb_table} (pk, item_data) VALUES ($1, $2) \ + ON CONFLICT (pk) DO UPDATE SET item_data = EXCLUDED.item_data" + ); + sqlx::query(&upsert_sql) + .bind(pk_text.as_str()) + .bind(&item_json) + .execute(&self.data_pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } self.fix_up_autocommit(stamp, key_info, &item).await; Ok(None) } diff --git a/crates/storage-postgres/src/data/transactions.rs b/crates/storage-postgres/src/data/transactions.rs index 6748ed00d..c983d9ed4 100644 --- a/crates/storage-postgres/src/data/transactions.rs +++ b/crates/storage-postgres/src/data/transactions.rs @@ -230,6 +230,28 @@ impl PostgresEngine { }); } + // Item history for every member at one instant, so a reconstruct + // holds all of the transaction or none of it. + #[cfg(feature = "pitr")] + { + let commit_time = std::time::SystemTime::now(); + for (op, (old_item, new_item)) in ops.iter().zip(op_items.iter()) { + let (TransactWriteOp::Put { key_info, .. } + | TransactWriteOp::Delete { key_info, .. } + | TransactWriteOp::Update { key_info, .. } + | TransactWriteOp::ConditionCheck { key_info, .. }) = op; + crate::pitr::capture_in_tx( + &mut tx, + key_info, + old_item.as_ref(), + new_item.as_ref(), + commit_time, + ) + .await + .map_err(cancel_all)?; + } + } + // Write stream records atomically within the transaction (BLOCKER #1 fix). for (op, (old_item, new_item)) in ops.iter().zip(op_items.iter()) { let capture = match op { diff --git a/crates/storage-postgres/src/data/update_item.rs b/crates/storage-postgres/src/data/update_item.rs index a92ae9660..b32080a91 100755 --- a/crates/storage-postgres/src/data/update_item.rs +++ b/crates/storage-postgres/src/data/update_item.rs @@ -329,6 +329,15 @@ impl PostgresEngine { ) .await?; } + #[cfg(feature = "pitr")] + crate::pitr::record_item_history_in_tx( + &mut tx, + key_info, + &item, + Some(&item), + std::time::SystemTime::now(), + ) + .await?; // Persist async GSI work inside the same transaction — one row per // async index, each honoring its own propagation delay. let mut async_enqueued = enqueue_async_indexes( diff --git a/crates/storage-postgres/src/lib.rs b/crates/storage-postgres/src/lib.rs index 6a8948405..e4dc0bccc 100755 --- a/crates/storage-postgres/src/lib.rs +++ b/crates/storage-postgres/src/lib.rs @@ -26,6 +26,8 @@ pub mod migrations; mod operations; mod password_hash_store; mod pg_util; +#[cfg(feature = "pitr")] +mod pitr; mod restore_engine; mod snapshot_engine; #[doc(hidden)] @@ -716,6 +718,17 @@ impl ServerRuntimeHooks for PostgresRuntimeHooks { workers::stream_record_cleanup_worker(storage_for_stream, metrics, token).await; }); + // 3a. Point-in-time recovery retention sweep + #[cfg(feature = "pitr")] + let pitr_retention = { + let storage = self.engine.clone(); + let metrics = ctx.metrics.clone(); + let token = ctx.shutdown.clone(); + tokio::spawn(async move { + extenddb_storage::pitr::retention_worker(storage, metrics, token).await; + }) + }; + // 3b. Stream record numbering worker let storage_for_numbering = self.engine.clone(); let token = ctx.shutdown.clone(); @@ -771,7 +784,8 @@ impl ServerRuntimeHooks for PostgresRuntimeHooks { workers::poll_gsi_delay(catalog_store_for_gsi, gsi_delay, token).await; }); - vec![ + #[allow(unused_mut)] + let mut tasks = vec![ control_plane, table_size, stream_cleanup, @@ -782,7 +796,10 @@ impl ServerRuntimeHooks for PostgresRuntimeHooks { gsi_builds, pool_metrics, gsi_poller, - ] + ]; + #[cfg(feature = "pitr")] + tasks.push(pitr_retention); + tasks } fn backend_info(&self) -> Option { diff --git a/crates/storage-postgres/src/pitr.rs b/crates/storage-postgres/src/pitr.rs new file mode 100644 index 000000000..7b9f4f69d --- /dev/null +++ b/crates/storage-postgres/src/pitr.rs @@ -0,0 +1,575 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Point-in-time recovery item history on PostgreSQL (`pitr_state`, +//! `item_history`, migration 10, both in the data database). +//! +//! [`record_item_history_in_tx`] runs inside a write's data transaction and +//! adds a row only when the table has a `pitr_state` row. A caller records +//! after its data write, so the row's `seq` is drawn under the item's row +//! lock and follows commit order for that key; the reconstruct and the prune +//! only compare `seq` within one key. The reconstruct is the anti-join the +//! SQLite store runs, paged on `seq`, over `idx_item_history_key`. +//! +//! A put or a delete that needs no transaction records in the same statement +//! as its data write ([`autocommit_put`], [`autocommit_delete`]): the +//! statement takes the data table's lock before it takes its snapshot, so a +//! write held by an enable reads the `pitr_state` row the enable committed. +//! +//! A page first pins its time in `pitr_state` and a prune lowers its cutoff +//! to a pin that holds; both lock the table's `pitr_state` row, so a prune +//! either sees the pin or has already moved `pruned_to_us` past the time and +//! the page refuses. + +use std::time::SystemTime; + +use extenddb_core::types::ScalarAttributeType; +use extenddb_core::types::{Item, TableKeyInfo}; +use extenddb_storage::error::StorageError; +use extenddb_storage::pitr::{ + DEFAULT_RECOVERY_PERIOD_DAYS, PitrCaptureStore, PitrState, RECONSTRUCT_PIN_LEASE, + ReconstructPage, from_unix_micros, pinned_cutoff, reconstruct_held, stored_period, unix_micros, +}; +use extenddb_storage::util::{SortKeyValue, parse_sk, pk_to_text, sk_info}; +use futures::future::BoxFuture; + +use crate::PostgresEngine; +use crate::data::{data_table_name, json_to_item}; + +mod autocommit; +pub(crate) use autocommit::{autocommit_delete, autocommit_put}; + +/// Superseded: a later row of the same key committed at or before `$2`. +const SUPERSEDED: &str = "EXISTS (SELECT 1 FROM item_history n \ + WHERE n.table_id = h.table_id AND n.pk = h.pk AND n.sk = h.sk \ + AND n.seq > h.seq AND n.commit_time_us <= $2)"; + +/// An earlier row of the same key committed at or before `$2`. +const HAS_EARLIER: &str = "EXISTS (SELECT 1 FROM item_history e \ + WHERE e.table_id = h.table_id AND e.pk = h.pk AND e.sk = h.sk \ + AND e.seq < h.seq AND e.commit_time_us <= $2)"; + +/// One reconstruct page: `$1` table id, `$2` time, `$3` the cursor, `$4` the +/// limit. +pub(crate) fn reconstruct_sql() -> String { + format!( + "SELECT h.seq, h.item_data FROM item_history h \ + WHERE h.table_id = $1 AND h.commit_time_us <= $2 AND h.seq > $3 \ + AND h.item_data IS NOT NULL AND NOT {SUPERSEDED} \ + ORDER BY h.seq LIMIT $4" + ) +} + +fn store_err(what: &str, e: sqlx::Error) -> StorageError { + StorageError::Internal(format!("item history {what}: {e}")) +} + +/// The sort key as `item_history.sk` holds it: equal for two values the data +/// table's typed column holds equal, and equal to what [`sk_sql`] computes +/// from that column, so a baseline row and a later write's row of one item +/// carry one key. +fn sk_text(sk: &SortKeyValue) -> String { + match sk { + SortKeyValue::S(s) => s.clone(), + SortKeyValue::N(n) => n.normalized().to_plain_string(), + SortKeyValue::B(b) => b.iter().map(|byte| format!("{byte:02x}")).collect(), + } +} + +/// [`sk_text`] computed from the data table's typed sort key column. +fn sk_sql(sk: Option) -> &'static str { + match sk { + None => "''", + Some(ScalarAttributeType::S) => "sk_s", + Some(ScalarAttributeType::N) => "trim_scale(sk_n)::text", + Some(ScalarAttributeType::B) => "encode(sk_b, 'hex')", + } +} + +/// Record one committed mutation of the table `key_info` names, in the +/// mutation's data transaction, after its data write. `key` is any image that +/// carries the primary key; `image` is the item after the mutation, `None` +/// for a delete. Returns whether a row was added, which is when the table's +/// recovery is enabled. +pub(crate) async fn record_item_history_in_tx( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + key_info: &TableKeyInfo, + key: &Item, + image: Option<&Item>, + commit_time: SystemTime, +) -> Result { + let pk_name = &key_info.key_schema[0].attribute_name; + let pk_value = key + .get(pk_name) + .ok_or_else(|| StorageError::Internal("missing partition key".to_owned()))?; + let pk_text = pk_to_text(pk_value)?; + let sk = match sk_info(&key_info.key_schema, &key_info.attribute_definitions) { + Some((sk_name, sk_type)) => { + let sk_value = key + .get(sk_name) + .ok_or_else(|| StorageError::Internal("missing sort key".to_owned()))?; + sk_text(&parse_sk(sk_value, sk_type)?) + } + None => String::new(), + }; + let item_json = image + .map(serde_json::to_value) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + let done = sqlx::query( + "INSERT INTO item_history (table_id, pk, sk, commit_time_us, item_data) \ + SELECT $1, $2, $3, $4, $5 WHERE EXISTS (SELECT 1 FROM pitr_state WHERE table_id = $1)", + ) + .bind(&key_info.table_id) + .bind(pk_text.as_ref()) + .bind(sk) + .bind(unix_micros(commit_time)?) + .bind(item_json) + .execute(&mut **tx) + .await + .map_err(|e| store_err("record", e))?; + Ok(done.rows_affected() == 1) +} + +/// The write paths' hook: record a mutation from its before and after images, +/// in the mutation's data transaction after its data write. A mutation with +/// neither image changed nothing (a delete of a missing item, a condition +/// check) and records nothing: a delete of a missing key holds no row lock, so +/// its row could take a higher `seq` than a concurrent first put of the key +/// that commits after it. Every mutation of one transaction passes the same +/// `commit_time`, so a reconstruct never splits a transaction. +pub(crate) async fn capture_in_tx( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + key_info: &TableKeyInfo, + old: Option<&Item>, + new: Option<&Item>, + commit_time: SystemTime, +) -> Result<(), StorageError> { + let Some(key) = new.or(old) else { + return Ok(()); + }; + record_item_history_in_tx(tx, key_info, key, new, commit_time).await?; + Ok(()) +} + +/// The columns [`state_of`] reads, in its order. +const STATE_COLUMNS: &str = "recovery_period_days, enabled_at_us, pruned_to_us"; + +fn state_of( + table_id: &str, + (days, at, pruned): (i32, i64, i64), +) -> Result { + Ok(PitrState { + pruned_to: from_unix_micros(pruned), + ..PitrState::new( + stored_period(table_id, i64::from(days))?, + from_unix_micros(at), + ) + }) +} + +async fn state_in( + conn: &mut sqlx::PgConnection, + table_id: &str, +) -> Result, StorageError> { + let row: Option<(i32, i64, i64)> = sqlx::query_as(&format!( + "SELECT {STATE_COLUMNS} FROM pitr_state WHERE table_id = $1" + )) + .bind(table_id) + .fetch_optional(conn) + .await + .map_err(|e| store_err("state", e))?; + row.map(|r| state_of(table_id, r)).transpose() +} + +/// Exclude the table's writers, and every other enable and disable of it, +/// until the transaction ends. `SHARE ROW EXCLUSIVE` conflicts with the `ROW +/// EXCLUSIVE` every data write takes, so the lock waits for the writes in +/// flight to commit and holds new ones until this commits; a write held here +/// then reads the state as this transaction left it. It also conflicts with +/// itself, so a second enable reads the state the first committed. A data +/// table a DeleteTable dropped is [`StorageError::TableNotFound`]. +async fn exclude_writers( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + key_info: &TableKeyInfo, +) -> Result<(), StorageError> { + sqlx::query(&format!( + "LOCK TABLE {} IN SHARE ROW EXCLUSIVE MODE", + data_table_name(&key_info.table_id) + )) + .execute(&mut **tx) + .await + .map_err(|e| { + if e.as_database_error().and_then(|d| d.code()).as_deref() == Some("42P01") { + StorageError::TableNotFound(key_info.table_name.clone()) + } else { + store_err("lock", e) + } + })?; + Ok(()) +} + +fn micros_from_now(d: std::time::Duration) -> Result<(i64, i64), StorageError> { + let now = SystemTime::now(); + Ok((unix_micros(now)?, unix_micros(now + d)?)) +} + +/// Pin `$2` on table `$1` until `$4`, at `$3` now: a pin that has expired is +/// replaced, one that holds keeps the earlier time and the later expiry. +const PIN_SQL: &str = "UPDATE pitr_state SET \ + pinned_at_us = CASE WHEN pin_expires_us IS NULL OR pin_expires_us < $3 THEN $2 \ + ELSE LEAST(pinned_at_us, $2) END, \ + pin_expires_us = CASE WHEN pin_expires_us IS NULL OR pin_expires_us < $3 THEN $4 \ + ELSE GREATEST(pin_expires_us, $4) END \ + WHERE table_id = $1"; + +impl PitrCaptureStore for PostgresEngine { + fn pitr_state<'a>( + &'a self, + table_id: &'a str, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let mut conn = self + .data_pool + .acquire() + .await + .map_err(|e| store_err("state", e))?; + state_in(&mut conn, table_id).await + }) + } + + fn enable_pitr<'a>( + &'a self, + key_info: &'a TableKeyInfo, + period: Option, + at: SystemTime, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let table_id = key_info.table_id.as_str(); + let mut tx = self + .data_pool + .begin() + .await + .map_err(|e| store_err("enable", e))?; + exclude_writers(&mut tx, key_info).await?; + let state = match state_in(&mut tx, table_id).await? { + Some(mut state) => { + if let Some(days) = period { + sqlx::query( + "UPDATE pitr_state SET recovery_period_days = $1 WHERE table_id = $2", + ) + .bind(i32::from(days)) + .bind(table_id) + .execute(&mut *tx) + .await + .map_err(|e| store_err("enable", e))?; + state.recovery_period_days = days; + } + state + } + None => { + let state = PitrState::new(period.unwrap_or(DEFAULT_RECOVERY_PERIOD_DAYS), at); + sqlx::query( + "INSERT INTO pitr_state (table_id, recovery_period_days, enabled_at_us) \ + VALUES ($1, $2, $3)", + ) + .bind(table_id) + .bind(i32::from(state.recovery_period_days)) + .bind(unix_micros(at)?) + .execute(&mut *tx) + .await + .map_err(|e| store_err("enable", e))?; + let sk = sk_info(&key_info.key_schema, &key_info.attribute_definitions) + .map(|(_, t)| t); + sqlx::query(&format!( + "INSERT INTO item_history (table_id, pk, sk, commit_time_us, item_data) \ + SELECT $1, pk, {}, $2, item_data FROM {}", + sk_sql(sk), + data_table_name(table_id) + )) + .bind(table_id) + .bind(unix_micros(at)?) + .execute(&mut *tx) + .await + .map_err(|e| store_err("baseline", e))?; + state + } + }; + tx.commit().await.map_err(|e| store_err("enable", e))?; + Ok(state) + }) + } + + fn pitr_tables(&self) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + let rows: Vec<(String, i32, i64, i64)> = sqlx::query_as(&format!( + "SELECT table_id, {STATE_COLUMNS} FROM pitr_state ORDER BY table_id" + )) + .fetch_all(&self.data_pool) + .await + .map_err(|e| store_err("tables", e))?; + rows.into_iter() + .map(|(id, days, at, pruned)| { + let state = state_of(&id, (days, at, pruned))?; + Ok((id, state)) + }) + .collect() + }) + } + + /// The catalog is another database, so DeleteTable leaves the data + /// database's history and setting: tables with history or a setting that + /// the catalog does not have are removed here, a batch of rows at a time, + /// the setting once no row is left. + fn remove_orphaned_history(&self, limit: u32) -> BoxFuture<'_, Result> { + Box::pin(async move { + let candidates: Vec = sqlx::query_scalar( + "SELECT table_id FROM pitr_state UNION \ + SELECT DISTINCT table_id FROM item_history", + ) + .fetch_all(&self.data_pool) + .await + .map_err(|e| store_err("orphans", e))?; + if candidates.is_empty() { + return Ok(0); + } + let live: Vec = + sqlx::query_scalar("SELECT table_id FROM tables WHERE table_id = ANY($1)") + .bind(&candidates) + .fetch_all(&self.pool) + .await + .map_err(|e| store_err("orphans", e))?; + let gone: Vec = candidates + .into_iter() + .filter(|id| !live.contains(id)) + .collect(); + if gone.is_empty() { + return Ok(0); + } + let mut tx = self + .data_pool + .begin() + .await + .map_err(|e| store_err("orphans", e))?; + // The setting rows first, in one order, as a prune locks its + // table's row before it deletes history. + sqlx::query( + "SELECT table_id FROM pitr_state WHERE table_id = ANY($1) \ + ORDER BY table_id FOR UPDATE", + ) + .bind(&gone) + .execute(&mut *tx) + .await + .map_err(|e| store_err("orphans", e))?; + let removed = sqlx::query( + "DELETE FROM item_history WHERE seq IN (SELECT seq FROM item_history \ + WHERE table_id = ANY($1) ORDER BY seq LIMIT $2)", + ) + .bind(&gone) + .bind(i64::from(limit)) + .execute(&mut *tx) + .await + .map_err(|e| store_err("orphans", e))? + .rows_affected(); + sqlx::query( + "DELETE FROM pitr_state s WHERE s.table_id = ANY($1) \ + AND NOT EXISTS (SELECT 1 FROM item_history h WHERE h.table_id = s.table_id)", + ) + .bind(&gone) + .execute(&mut *tx) + .await + .map_err(|e| store_err("orphans", e))?; + tx.commit().await.map_err(|e| store_err("orphans", e))?; + Ok(removed) + }) + } + + fn disable_pitr<'a>( + &'a self, + key_info: &'a TableKeyInfo, + ) -> BoxFuture<'a, Result<(), StorageError>> { + Box::pin(async move { + let mut tx = self + .data_pool + .begin() + .await + .map_err(|e| store_err("disable", e))?; + exclude_writers(&mut tx, key_info).await?; + // The setting row first, as a prune locks it before it deletes + // history. + for sql in [ + "DELETE FROM pitr_state WHERE table_id = $1", + "DELETE FROM item_history WHERE table_id = $1", + ] { + sqlx::query(sql) + .bind(&key_info.table_id) + .execute(&mut *tx) + .await + .map_err(|e| store_err("disable", e))?; + } + tx.commit().await.map_err(|e| store_err("disable", e)) + }) + } + + fn reconstruct_page<'a>( + &'a self, + table_id: &'a str, + at: SystemTime, + after: Option, + limit: u32, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let at_us = unix_micros(at)?; + let (now_us, expires_us) = micros_from_now(RECONSTRUCT_PIN_LEASE)?; + let mut tx = self + .data_pool + .begin() + .await + .map_err(|e| store_err("pin", e))?; + let pruned: Option = sqlx::query_scalar( + "SELECT pruned_to_us FROM pitr_state WHERE table_id = $1 FOR UPDATE", + ) + .bind(table_id) + .fetch_optional(&mut *tx) + .await + .map_err(|e| store_err("pin", e))?; + reconstruct_held(table_id, at_us, pruned)?; + sqlx::query(PIN_SQL) + .bind(table_id) + .bind(at_us) + .bind(now_us) + .bind(expires_us) + .execute(&mut *tx) + .await + .map_err(|e| store_err("pin", e))?; + tx.commit().await.map_err(|e| store_err("pin", e))?; + let rows: Vec<(i64, serde_json::Value)> = sqlx::query_as(&reconstruct_sql()) + .bind(table_id) + .bind(at_us) + .bind(after.unwrap_or(i64::MIN)) + .bind(i64::from(limit)) + .fetch_all(&self.data_pool) + .await + .map_err(|e| store_err("reconstruct", e))?; + let next = if rows.len() == limit as usize { + rows.last().map(|(seq, _)| *seq) + } else { + None + }; + let items = rows + .into_iter() + .map(|(_, v)| json_to_item(v)) + .collect::>()?; + Ok(ReconstructPage { items, next }) + }) + } + + fn prune_before<'a>( + &'a self, + table_id: &'a str, + cutoff: SystemTime, + limit: u32, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let (now_us, _) = micros_from_now(std::time::Duration::ZERO)?; + let mut tx = self + .data_pool + .begin() + .await + .map_err(|e| store_err("prune", e))?; + let pin: Option<(Option, Option)> = sqlx::query_as( + "SELECT pinned_at_us, pin_expires_us FROM pitr_state WHERE table_id = $1 \ + FOR UPDATE", + ) + .bind(table_id) + .fetch_optional(&mut *tx) + .await + .map_err(|e| store_err("prune", e))?; + let cutoff_us = + pinned_cutoff(unix_micros(cutoff)?, pin.unwrap_or((None, None)), now_us); + let superseded = format!( + "DELETE FROM item_history WHERE seq IN (SELECT h.seq FROM item_history h \ + WHERE h.table_id = $1 AND h.commit_time_us <= $2 AND {SUPERSEDED} \ + ORDER BY h.seq LIMIT $3)" + ); + let mut removed = sqlx::query(&superseded) + .bind(table_id) + .bind(cutoff_us) + .bind(i64::from(limit)) + .execute(&mut *tx) + .await + .map_err(|e| store_err("prune", e))? + .rows_affected(); + // A delete is dropped only once it is the only row of its key at + // or before the cutoff: with an older image still present, the + // image would become the latest row and bring the item back. + if removed < u64::from(limit) { + let deletes = format!( + "DELETE FROM item_history WHERE seq IN (SELECT h.seq FROM item_history h \ + WHERE h.table_id = $1 AND h.commit_time_us <= $2 \ + AND h.item_data IS NULL AND NOT {SUPERSEDED} AND NOT {HAS_EARLIER} \ + ORDER BY h.seq LIMIT $3)" + ); + removed += sqlx::query(&deletes) + .bind(table_id) + .bind(cutoff_us) + .bind(i64::try_from(u64::from(limit) - removed).unwrap_or(i64::MAX)) + .execute(&mut *tx) + .await + .map_err(|e| store_err("prune", e))? + .rows_affected(); + } + if removed > 0 { + sqlx::query( + "UPDATE pitr_state SET pruned_to_us = GREATEST(pruned_to_us, $1) \ + WHERE table_id = $2", + ) + .bind(cutoff_us) + .bind(table_id) + .execute(&mut *tx) + .await + .map_err(|e| store_err("prune", e))?; + } + tx.commit().await.map_err(|e| store_err("prune", e))?; + Ok(removed) + }) + } +} + +#[cfg(feature = "test-hooks")] +impl PostgresEngine { + /// [`record_item_history_in_tx`] in a data transaction of its own, + /// committed or rolled back as `commit` says, for the shared checks of + /// `extenddb_storage::pitr::conformance`. + /// + /// # Errors + /// + /// The record or the transaction fails. + pub async fn record_item_history_for_tests( + &self, + key_info: &TableKeyInfo, + key: &Item, + image: Option<&Item>, + commit_time: SystemTime, + commit: bool, + ) -> Result { + let mut tx = self + .data_pool + .begin() + .await + .map_err(|e| store_err("begin", e))?; + let added = record_item_history_in_tx(&mut tx, key_info, key, image, commit_time).await?; + if commit { + tx.commit().await.map_err(|e| store_err("commit", e))?; + } else { + tx.rollback().await.map_err(|e| store_err("rollback", e))?; + } + Ok(added) + } + + /// The reconstruct statement, for a test of its plan. + #[must_use] + pub fn item_history_reconstruct_sql_for_tests() -> String { + reconstruct_sql() + } +} diff --git a/crates/storage-postgres/src/pitr/autocommit.rs b/crates/storage-postgres/src/pitr/autocommit.rs new file mode 100644 index 000000000..0c10979b8 --- /dev/null +++ b/crates/storage-postgres/src/pitr/autocommit.rs @@ -0,0 +1,132 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The put and delete that need no transaction, each one autocommit +//! statement that also records the write's history row when the table's +//! recovery is enabled. + +use std::time::SystemTime; + +use extenddb_core::types::{ScalarAttributeType, TableKeyInfo}; +use extenddb_storage::error::StorageError; +use extenddb_storage::pitr::unix_micros; +use extenddb_storage::util::{SortKeyValue, sk_column}; + +use super::sk_text; +use crate::data::data_table_name; +use crate::data::index::db_error; + +/// The history row of a write that is its statement's only data row, added +/// when the table has a `pitr_state` row. `$1` is the data pk, then +/// `$table`, `$sk`, `$time` and `$image` name the statement's parameters. +fn history_from(w: &str, table: &str, sk: &str, time: &str, image: &str) -> String { + format!( + "INSERT INTO item_history (table_id, pk, sk, commit_time_us, item_data) \ + SELECT {table}, $1, {sk}, {time}, {image} FROM {w} \ + WHERE EXISTS (SELECT 1 FROM pitr_state WHERE table_id = {table})" + ) +} + +/// A put with no condition, index or stream, as one autocommit statement: +/// the upsert, and the history row when the table's recovery is enabled. +pub(crate) async fn autocommit_put( + pool: &sqlx::PgPool, + key_info: &TableKeyInfo, + pk_text: &str, + sk: Option<&SortKeyValue>, + item_json: &serde_json::Value, +) -> Result<(), StorageError> { + let ddb_table = data_table_name(&key_info.table_id); + let commit_time = unix_micros(SystemTime::now())?; + let Some(sk) = sk else { + let sql = format!( + "WITH w AS (INSERT INTO {ddb_table} (pk, item_data) VALUES ($1, $2) \ + ON CONFLICT (pk) DO UPDATE SET item_data = EXCLUDED.item_data RETURNING 1) {}", + history_from("w", "$3", "''", "$4", "$2") + ); + sqlx::query(&sql) + .bind(pk_text) + .bind(item_json) + .bind(&key_info.table_id) + .bind(commit_time) + .execute(pool) + .await + .map_err(db_error)?; + return Ok(()); + }; + let sk_col = sk_column(sk_type(sk)); + let sql = format!( + "WITH w AS (INSERT INTO {ddb_table} (pk, {sk_col}, item_data) VALUES ($1, $2, $3) \ + ON CONFLICT (pk, {sk_col}) DO UPDATE SET item_data = EXCLUDED.item_data RETURNING 1) {}", + history_from("w", "$4", "$5", "$6", "$3") + ); + let query = bind_typed_sk(sqlx::query(&sql).bind(pk_text), sk); + query + .bind(item_json) + .bind(&key_info.table_id) + .bind(sk_text(sk)) + .bind(commit_time) + .execute(pool) + .await + .map_err(db_error)?; + Ok(()) +} + +/// A delete with no condition, index or stream, as one autocommit +/// statement: the delete, and the history row when it removed an item and +/// the table's recovery is enabled. +pub(crate) async fn autocommit_delete( + pool: &sqlx::PgPool, + key_info: &TableKeyInfo, + pk_text: &str, + sk: Option<&SortKeyValue>, +) -> Result<(), StorageError> { + let ddb_table = data_table_name(&key_info.table_id); + let commit_time = unix_micros(SystemTime::now())?; + let Some(sk) = sk else { + let sql = format!( + "WITH w AS (DELETE FROM {ddb_table} WHERE pk = $1 RETURNING 1) {}", + history_from("w", "$2", "''", "$3", "NULL") + ); + sqlx::query(&sql) + .bind(pk_text) + .bind(&key_info.table_id) + .bind(commit_time) + .execute(pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + return Ok(()); + }; + let sk_col = sk_column(sk_type(sk)); + let sql = format!( + "WITH w AS (DELETE FROM {ddb_table} WHERE pk = $1 AND {sk_col} = $2 RETURNING 1) {}", + history_from("w", "$3", "$4", "$5", "NULL") + ); + bind_typed_sk(sqlx::query(&sql).bind(pk_text), sk) + .bind(&key_info.table_id) + .bind(sk_text(sk)) + .bind(commit_time) + .execute(pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) +} + +fn sk_type(sk: &SortKeyValue) -> ScalarAttributeType { + match sk { + SortKeyValue::S(_) => ScalarAttributeType::S, + SortKeyValue::N(_) => ScalarAttributeType::N, + SortKeyValue::B(_) => ScalarAttributeType::B, + } +} + +fn bind_typed_sk<'q>( + query: sqlx::query::Query<'q, sqlx::Postgres, sqlx::postgres::PgArguments>, + sk: &'q SortKeyValue, +) -> sqlx::query::Query<'q, sqlx::Postgres, sqlx::postgres::PgArguments> { + match sk { + SortKeyValue::S(s) => query.bind(s), + SortKeyValue::N(n) => query.bind(n), + SortKeyValue::B(b) => query.bind(b), + } +} diff --git a/crates/storage-postgres/tests/pitr_item_history.rs b/crates/storage-postgres/tests/pitr_item_history.rs new file mode 100644 index 000000000..25cb52ed1 --- /dev/null +++ b/crates/storage-postgres/tests/pitr_item_history.rs @@ -0,0 +1,555 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The PostgreSQL item-history store: the shared checks of +//! `extenddb_storage::pitr::conformance`, then the query plan. +//! +//! Needs `EXTENDDB_TEST_PG_CONNECTION_STRING` (host-only, e.g. +//! `postgres://user:pass@127.0.0.1:5432`); skips otherwise. Each test builds +//! its own scratch databases and drops them afterwards, on a panic too. + +#![cfg(feature = "pitr")] + +mod pitr_support; + +use std::time::SystemTime; + +use extenddb_core::types::{AttributeValue, Item, ScalarAttributeType}; +use extenddb_storage::DataEngine; +use extenddb_storage::pitr::PitrCaptureStore; +use extenddb_storage::pitr::conformance::{HistoryFixture, at}; +use extenddb_storage_postgres::PostgresEngine; +use pitr_support::{close, open, open_two_databases}; + +extenddb_storage::pitr_conformance_tests!(open, close); + +/// The reconstruct reads through the key index, with no sequential scan, on +/// a history large enough that the planner would pick one if it could. +#[tokio::test] +async fn reconstruct_reads_through_the_key_index() { + let Some(s) = open().await else { return }; + let info = s.table("t", None, true).await; + for i in 0..2_000 { + let k = Item::from([("pk".to_owned(), AttributeValue::S(format!("k{}", i % 50)))]); + s.record(&info, &k, Some(&k), at(i), true).await; + } + sqlx::query("ANALYZE item_history") + .execute(&s.db) + .await + .expect("analyze"); + let sql = format!( + "EXPLAIN {}", + PostgresEngine::item_history_reconstruct_sql_for_tests() + ); + let plan: Vec = sqlx::query_scalar(&sql) + .bind(&info.table_id) + .bind(i64::MAX) + .bind(i64::MIN) + .bind(10_i64) + .fetch_all(&s.db) + .await + .expect("plan"); + assert!( + plan.iter().any(|l| l.contains("idx_item_history_key")), + "{plan:#?}" + ); + assert!(plan.iter().all(|l| !l.contains("Seq Scan")), "{plan:#?}"); + close(s).await; +} + +/// Concurrent writers on a few keys: once they finish, the state now equals +/// the live table. Two writes to one item take its row lock in turn and each +/// draws its `seq` after its data write, so a key's latest history row is its +/// last committed write. Deletes of missing keys race first puts of the same +/// keys here, the case where a recorded delete could order wrongly. +#[tokio::test] +async fn concurrent_writers_leave_the_state_equal_to_the_live_table() { + use extenddb_core::expression::ExpressionMaps; + use extenddb_storage::pitr::conformance::state; + let Some(s) = open().await else { return }; + let info = s.table("t", None, true).await; + let mut tasks = Vec::new(); + let engine = std::sync::Arc::new(s); + for task in 0..8_u64 { + let engine = engine.clone(); + let info = info.clone(); + tasks.push(tokio::spawn(async move { + let maps = ExpressionMaps::default(); + let mut x = task.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1); + for i in 0..60_u64 { + x = x + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + let key = Item::from([( + "pk".to_owned(), + AttributeValue::S(format!("k{}", (x >> 33) % 3)), + )]); + if (x >> 40) % 3 == 0 { + engine + .engine + .delete_item(&info, &key, false, None, &maps, None) + .await + .expect("delete"); + } else { + let mut item = key.clone(); + item.insert("v".to_owned(), AttributeValue::S(format!("{task}-{i}"))); + engine + .engine + .put_item(&info, item, false, None, &maps, None) + .await + .expect("put"); + } + } + })); + } + for t in tasks { + t.await.expect("writer"); + } + let s = std::sync::Arc::into_inner(engine).expect("sole owner"); + let live: Vec = { + let mut v: Vec = s + .live_items(&info) + .await + .iter() + .map(|i| serde_json::to_string(i).expect("json")) + .collect(); + v.sort(); + v + }; + assert_eq!(state(&s, &info, SystemTime::now(), 2).await, live); + close(s).await; +} + +/// A delete of a key whose first put is still uncommitted deletes nothing and +/// records nothing. The delete takes no row lock, so a recorded delete would +/// draw a higher `seq` than the put that commits after it, and the reconstruct +/// would drop the item the table holds. +#[tokio::test] +async fn a_delete_racing_an_uncommitted_first_put_records_nothing() { + use extenddb_core::expression::ExpressionMaps; + use extenddb_storage::pitr::conformance::state; + let Some(s) = open().await else { return }; + let info = s.table("t", None, true).await; + let item = Item::from([ + ("pk".to_owned(), AttributeValue::S("k".to_owned())), + ("v".to_owned(), AttributeValue::S("a".to_owned())), + ]); + let mut put = s.db.begin().await.expect("begin"); + sqlx::query(&format!( + "INSERT INTO \"_ddb_{}\" (pk, item_data) VALUES ('k', $1)", + info.table_id + )) + .bind(serde_json::to_value(&item).expect("json")) + .execute(&mut *put) + .await + .expect("insert the item"); + sqlx::query( + "INSERT INTO item_history (table_id, pk, sk, commit_time_us, item_data) \ + VALUES ($1, 'k', '', $2, $3)", + ) + .bind(&info.table_id) + .bind(extenddb_storage::pitr::unix_micros(SystemTime::now()).expect("time")) + .bind(serde_json::to_value(&item).expect("json")) + .execute(&mut *put) + .await + .expect("record the put"); + let key = Item::from([("pk".to_owned(), AttributeValue::S("k".to_owned()))]); + let maps = ExpressionMaps::default(); + tokio::time::timeout( + std::time::Duration::from_secs(10), + s.engine.delete_item(&info, &key, false, None, &maps, None), + ) + .await + .expect("the delete does not wait on the uncommitted put") + .expect("delete"); + put.commit().await.expect("commit the put"); + assert_eq!(s.live_items(&info).await.len(), 1); + assert_eq!(s.history_rows(&info.table_id).await, 1); + assert_eq!( + state(&s, &info, SystemTime::now(), 10).await, + vec![serde_json::to_string(&item).expect("json")] + ); + close(s).await; +} + +/// Writers keep writing while recovery is enabled: once they finish, the +/// state now equals the live table. The enable excludes the table's writers +/// while it takes the baseline, so a write lands in the baseline or in the +/// history, never in neither. +#[tokio::test] +async fn an_enable_during_writes_loses_no_write() { + use extenddb_core::expression::ExpressionMaps; + use extenddb_storage::pitr::conformance::state; + let Some(s) = open().await else { return }; + let info = s.table("t", None, false).await; + let engine = std::sync::Arc::new(s); + let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let mut tasks = Vec::new(); + for task in 0..6_u32 { + let (engine, info, stop) = (engine.clone(), info.clone(), stop.clone()); + tasks.push(tokio::spawn(async move { + let maps = ExpressionMaps::default(); + let mut i = 0_u32; + while !stop.load(std::sync::atomic::Ordering::Relaxed) { + let item = Item::from([ + ( + "pk".to_owned(), + AttributeValue::S(format!("k{}", (task + i) % 5)), + ), + ("v".to_owned(), AttributeValue::S(format!("{task}-{i}"))), + ]); + engine + .engine + .put_item(&info, item, false, None, &maps, None) + .await + .expect("put"); + i += 1; + } + })); + } + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + engine + .engine + .enable_pitr(&info, None, SystemTime::now()) + .await + .expect("enable"); + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + stop.store(true, std::sync::atomic::Ordering::Relaxed); + for t in tasks { + t.await.expect("writer"); + } + let s = std::sync::Arc::into_inner(engine).expect("sole owner"); + let mut live: Vec = s + .live_items(&info) + .await + .iter() + .map(|i| serde_json::to_string(i).expect("json")) + .collect(); + live.sort(); + assert_eq!(state(&s, &info, SystemTime::now(), 3).await, live); + close(s).await; +} + +/// A write whose transaction is open when the enable starts: its history +/// statement ran before the enable, so it recorded nothing, and it commits +/// after. The enable waits for it, so the baseline holds the write; an enable +/// that did not wait would take its baseline without the write, and the +/// write would be in neither. +#[tokio::test] +async fn an_enable_waits_for_a_write_in_flight() { + use extenddb_storage::pitr::conformance::state; + let Some(s) = open().await else { return }; + let info = s.table("t", None, false).await; + let old = Item::from([ + ("pk".to_owned(), AttributeValue::S("k".to_owned())), + ("v".to_owned(), AttributeValue::S("old".to_owned())), + ]); + let mut new = old.clone(); + new.insert("v".to_owned(), AttributeValue::S("new".to_owned())); + s.engine + .put_item(&info, old, false, None, &Default::default(), None) + .await + .expect("put"); + let mut write = s.db.begin().await.expect("begin"); + sqlx::query(&format!( + "UPDATE \"_ddb_{}\" SET item_data = $1 WHERE pk = 'k'", + info.table_id + )) + .bind(serde_json::to_value(&new).expect("json")) + .execute(&mut *write) + .await + .expect("write"); + let recorded = sqlx::query( + "INSERT INTO item_history (table_id, pk, sk, commit_time_us, item_data) \ + SELECT $1, 'k', '', 0, $2 WHERE EXISTS (SELECT 1 FROM pitr_state WHERE table_id = $1)", + ) + .bind(&info.table_id) + .bind(serde_json::to_value(&new).expect("json")) + .execute(&mut *write) + .await + .expect("history statement"); + assert_eq!(recorded.rows_affected(), 0); + let s = std::sync::Arc::new(s); + let enabling = { + let (s, info) = (s.clone(), info.clone()); + tokio::spawn(async move { + s.engine + .enable_pitr(&info, None, SystemTime::now()) + .await + .expect("enable") + }) + }; + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + write.commit().await.expect("commit the write"); + tokio::time::timeout(std::time::Duration::from_secs(10), enabling) + .await + .expect("the enable finishes once the write commits") + .expect("enable task"); + let s = std::sync::Arc::into_inner(s).expect("sole owner"); + assert_eq!( + state(&s, &info, SystemTime::now(), 10).await, + vec![serde_json::to_string(&new).expect("json")] + ); + close(s).await; +} + +/// The catalog is another database, so a deleted table's history and setting +/// stay in the data database until the retention sweep removes them, a batch +/// at a time, the setting last; a live table's history is left. +#[tokio::test] +async fn the_sweep_removes_the_history_of_deleted_tables() { + let Some(s) = open().await else { return }; + let gone = s.table("gone", None, true).await; + let kept = s.table("kept", None, true).await; + for i in 0..5_u64 { + let k = Item::from([("pk".to_owned(), AttributeValue::S(format!("k{i}")))]); + s.record(&gone, &k, Some(&k), at(i), true).await; + s.record(&kept, &k, Some(&k), at(i), true).await; + } + sqlx::query("DELETE FROM tables WHERE table_id = $1") + .bind(&gone.table_id) + .execute(&s.catalog) + .await + .expect("delete the catalog row"); + let report = extenddb_storage::pitr::sweep( + &s.engine, + at(10), + 2, + std::time::Instant::now() + std::time::Duration::from_secs(60), + ) + .await + .expect("sweep"); + assert_eq!(report.orphaned, 5); + assert_eq!(s.history_rows(&gone.table_id).await, 0); + assert_eq!(s.history_rows(&kept.table_id).await, 5); + let states: Vec = sqlx::query_scalar("SELECT table_id FROM pitr_state") + .fetch_all(&s.db) + .await + .expect("states"); + assert_eq!(states, vec![kept.table_id.clone()]); + close(s).await; +} + +/// The orphan sweep with the catalog and the data in two databases, as a +/// deployment runs them: the live table ids come from the catalog, which the +/// data database does not hold. +#[tokio::test] +async fn the_sweep_reads_live_tables_from_the_catalog_database() { + let Some(s) = open_two_databases().await else { + return; + }; + let gone = s.table("gone", None, true).await; + let kept = s.table("kept", None, true).await; + for i in 0..3_u64 { + let k = Item::from([("pk".to_owned(), AttributeValue::S(format!("k{i}")))]); + s.record(&gone, &k, Some(&k), at(i), true).await; + s.record(&kept, &k, Some(&k), at(i), true).await; + } + let catalog_tables: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM information_schema.tables \ + WHERE table_schema = 'public' AND table_name = 'tables'", + ) + .fetch_one(&s.db) + .await + .expect("information schema"); + assert_eq!(catalog_tables, 0, "the data database holds no catalog"); + sqlx::query("DELETE FROM tables WHERE table_id = $1") + .bind(&gone.table_id) + .execute(&s.catalog) + .await + .expect("delete the catalog row"); + let report = extenddb_storage::pitr::sweep( + &s.engine, + at(10), + 100, + std::time::Instant::now() + std::time::Duration::from_secs(60), + ) + .await + .expect("sweep"); + assert_eq!(report.orphaned, 3); + assert_eq!(s.history_rows(&gone.table_id).await, 0); + assert_eq!(s.history_rows(&kept.table_id).await, 3); + close(s).await; +} + +/// A disable that starts while a write of the table is open waits for it, +/// so it removes the write's history row too. A disable that did not wait +/// would delete the rows it sees, and the write's row would commit after it +/// and survive the disable. +#[tokio::test] +async fn a_disable_waits_for_a_write_in_flight() { + let Some(s) = open().await else { return }; + let info = s.table("t", None, false).await; + let old = Item::from([ + ("pk".to_owned(), AttributeValue::S("k".to_owned())), + ("v".to_owned(), AttributeValue::S("old".to_owned())), + ]); + let mut new = old.clone(); + new.insert("v".to_owned(), AttributeValue::S("new".to_owned())); + s.engine + .put_item(&info, old, false, None, &Default::default(), None) + .await + .expect("put"); + s.engine + .enable_pitr(&info, None, SystemTime::now()) + .await + .expect("enable"); + let mut write = s.db.begin().await.expect("begin"); + sqlx::query(&format!( + "UPDATE \"_ddb_{}\" SET item_data = $1 WHERE pk = 'k'", + info.table_id + )) + .bind(serde_json::to_value(&new).expect("json")) + .execute(&mut *write) + .await + .expect("write"); + let recorded = sqlx::query( + "INSERT INTO item_history (table_id, pk, sk, commit_time_us, item_data) \ + SELECT $1, 'k', '', 0, $2 WHERE EXISTS (SELECT 1 FROM pitr_state WHERE table_id = $1)", + ) + .bind(&info.table_id) + .bind(serde_json::to_value(&new).expect("json")) + .execute(&mut *write) + .await + .expect("history statement"); + assert_eq!(recorded.rows_affected(), 1); + let s = std::sync::Arc::new(s); + let disabling = { + let (s, info) = (s.clone(), info.clone()); + tokio::spawn(async move { s.engine.disable_pitr(&info).await.expect("disable") }) + }; + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + assert!( + !disabling.is_finished(), + "the disable ran past the open write" + ); + write.commit().await.expect("commit the write"); + tokio::time::timeout(std::time::Duration::from_secs(10), disabling) + .await + .expect("the disable finishes once the write commits") + .expect("disable task"); + let s = std::sync::Arc::into_inner(s).expect("sole owner"); + assert_eq!(s.history_rows(&info.table_id).await, 0); + assert_eq!( + s.engine.pitr_state(&info.table_id).await.expect("state"), + None + ); + close(s).await; +} + +/// A put and a delete that need no transaction, held by an enable's lock: +/// each runs once the enable commits and records its history row, because +/// the statement that writes the item also reads the state, after the lock. +#[tokio::test] +async fn a_write_held_by_an_enable_records_its_history() { + use extenddb_core::expression::ExpressionMaps; + let Some(s) = open().await else { return }; + let plain = s.table("plain", None, false).await; + let sorted_table = s.table("sorted", Some(ScalarAttributeType::N), false).await; + let keyed = Item::from([ + ("pk".to_owned(), AttributeValue::S("k".to_owned())), + ("sk".to_owned(), AttributeValue::N("1.50".to_owned())), + ]); + s.engine + .put_item( + &sorted_table, + keyed.clone(), + false, + None, + &ExpressionMaps::default(), + None, + ) + .await + .expect("put"); + let s = std::sync::Arc::new(s); + for (info, delete) in [(plain.clone(), false), (sorted_table.clone(), true)] { + let mut enable = s.db.begin().await.expect("begin"); + sqlx::query(&format!( + "LOCK TABLE \"_ddb_{}\" IN SHARE ROW EXCLUSIVE MODE", + info.table_id + )) + .execute(&mut *enable) + .await + .expect("lock"); + let writing = { + let (s, info) = (s.clone(), info.clone()); + tokio::spawn(async move { + let maps = ExpressionMaps::default(); + if delete { + let key = Item::from([ + ("pk".to_owned(), AttributeValue::S("k".to_owned())), + ("sk".to_owned(), AttributeValue::N("1.5".to_owned())), + ]); + s.engine + .delete_item(&info, &key, false, None, &maps, None) + .await + .map(|_| ()) + } else { + let item = Item::from([("pk".to_owned(), AttributeValue::S("p".to_owned()))]); + s.engine + .put_item(&info, item, false, None, &maps, None) + .await + .map(|_| ()) + } + }) + }; + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + assert!( + !writing.is_finished(), + "the write ran past the enable's lock" + ); + sqlx::query( + "INSERT INTO pitr_state (table_id, recovery_period_days, enabled_at_us) \ + VALUES ($1, 35, 0)", + ) + .bind(&info.table_id) + .execute(&mut *enable) + .await + .expect("state"); + enable.commit().await.expect("commit the enable"); + tokio::time::timeout(std::time::Duration::from_secs(10), writing) + .await + .expect("the write finishes once the enable commits") + .expect("write task") + .expect("write"); + assert_eq!( + s.history_rows(&info.table_id).await, + 1, + "{}", + info.table_name + ); + } + let rows: Vec<(String, Option)> = + sqlx::query_as("SELECT sk, item_data FROM item_history WHERE table_id = $1") + .bind(&sorted_table.table_id) + .fetch_all(&s.db) + .await + .expect("rows"); + assert_eq!(rows, vec![("1.5".to_owned(), None)]); + let s = std::sync::Arc::into_inner(s).expect("sole owner"); + close(s).await; +} + +/// An enable or a disable that runs after a DeleteTable dropped the data +/// table names the table as not found. +#[tokio::test] +async fn an_enable_or_disable_after_the_drop_is_table_not_found() { + use extenddb_storage::error::StorageError; + let Some(s) = open().await else { return }; + let info = s.table("gone", None, false).await; + sqlx::query(&format!("DROP TABLE \"_ddb_{}\"", info.table_id)) + .execute(&s.db) + .await + .expect("drop the data table"); + let enabled = s.engine.enable_pitr(&info, None, SystemTime::now()).await; + assert!( + matches!(enabled, Err(StorageError::TableNotFound(ref n)) if n == "gone"), + "{enabled:?}" + ); + let disabled = s.engine.disable_pitr(&info).await; + assert!( + matches!(disabled, Err(StorageError::TableNotFound(ref n)) if n == "gone"), + "{disabled:?}" + ); + close(s).await; +} diff --git a/crates/storage-postgres/tests/pitr_support/mod.rs b/crates/storage-postgres/tests/pitr_support/mod.rs new file mode 100644 index 000000000..3c549d7cc --- /dev/null +++ b/crates/storage-postgres/tests/pitr_support/mod.rs @@ -0,0 +1,381 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The scratch PostgreSQL fixture of the item-history tests: one database, +//! or a catalog and a data database, built from the shipped migrations and +//! dropped afterwards, on a panic too. + +use std::time::SystemTime; + +use extenddb_core::types::{ + AttributeDefinition, BillingMode, CreateTableInput, Item, KeySchemaElement, KeyType, + ScalarAttributeType, TableKeyInfo, +}; +use extenddb_storage::pitr::PitrCaptureStore; +use extenddb_storage::pitr::conformance::HistoryFixture; +use extenddb_storage::{DataEngine, TableEngine}; +use extenddb_storage_postgres::{PostgresConfig, PostgresEngine}; +use futures::future::BoxFuture; +use sqlx::PgPool; +use sqlx::postgres::PgPoolOptions; + +pub const ACCOUNT: &str = "123456789012"; +const REGION: &str = "us-east-1"; + +pub struct Scratch { + pub engine: PostgresEngine, + /// The data database, which holds `pitr_state` and `item_history`. + pub db: PgPool, + /// The catalog database, the same as `db` unless the scratch has two. + pub catalog: PgPool, + admin: PgPool, + guard: DropDatabases, + _slot: tokio::sync::OwnedSemaphorePermit, +} + +/// Drops the scratch databases when dropped while armed: on a panic, when +/// [`close`] never runs. A thread of its own runs the drop, since a panic +/// unwinds inside the test's runtime. +struct DropDatabases { + base: String, + names: Vec, + armed: bool, +} + +impl Drop for DropDatabases { + fn drop(&mut self) { + if !self.armed { + return; + } + let (base, names) = (self.base.clone(), std::mem::take(&mut self.names)); + let _ = std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime"); + runtime.block_on(async { + let Ok(admin) = PgPoolOptions::new() + .max_connections(1) + .connect(&format!("{base}/postgres")) + .await + else { + return; + }; + for name in names { + let _ = + sqlx::query(&format!("DROP DATABASE IF EXISTS \"{name}\" WITH (FORCE)")) + .execute(&admin) + .await; + } + admin.close().await; + }); + }) + .join(); + } +} + +/// Scratch databases open at once. Each holds about fifteen connections, and +/// the server's default limit is 100. +const SCRATCH_SLOTS: usize = 4; + +async fn slot() -> tokio::sync::OwnedSemaphorePermit { + static SLOTS: std::sync::OnceLock> = + std::sync::OnceLock::new(); + SLOTS + .get_or_init(|| std::sync::Arc::new(tokio::sync::Semaphore::new(SCRATCH_SLOTS))) + .clone() + .acquire_owned() + .await + .expect("slot") +} + +/// The server's base URL, or `None` to skip. With `EXTENDDB_TEST_PG_REQUIRED` +/// set to any value but `0`, a job that must reach a server fails here +/// instead. +fn base_conn() -> Option { + let conn = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING").unwrap_or_default(); + if conn.trim().is_empty() { + let required = + std::env::var("EXTENDDB_TEST_PG_REQUIRED").is_ok_and(|v| !v.is_empty() && v != "0"); + assert!( + !required, + "EXTENDDB_TEST_PG_CONNECTION_STRING is not set, and EXTENDDB_TEST_PG_REQUIRED says \ + this job must reach a PostgreSQL server" + ); + return None; + } + Some(conn.trim_end_matches('/').to_owned()) +} + +fn shipped_migrations(dirs: &[&str]) -> Vec { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); + let mut out = Vec::new(); + for dir in dirs { + let mut files: Vec<_> = std::fs::read_dir(root.join(dir)) + .expect("read the migrations directory") + .map(|e| e.expect("directory entry").path()) + .filter(|p| p.extension().is_some_and(|e| e == "sql")) + .collect(); + files.sort(); + for f in files { + out.push(std::fs::read_to_string(&f).expect("read a migration file")); + } + } + out +} + +pub async fn open() -> Option { + open_scratch(false).await +} + +/// A scratch whose catalog and data are two databases, as a deployment +/// runs them. +pub async fn open_two_databases() -> Option { + open_scratch(true).await +} + +async fn create_database(admin: &PgPool, base: &str, guard: &mut DropDatabases) -> PgPool { + let name = format!("eddb_pitr_{}", uuid::Uuid::new_v4().simple())[..24].to_owned(); + sqlx::query(&format!("CREATE DATABASE \"{name}\"")) + .execute(admin) + .await + .expect("create a scratch database"); + guard.names.push(name.clone()); + PgPoolOptions::new() + .max_connections(3) + .connect(&format!("{base}/{name}")) + .await + .expect("connect to the scratch database") +} + +async fn apply(pool: &PgPool, dirs: &[&str]) { + for sql in shipped_migrations(dirs) { + sqlx::raw_sql(&sql) + .execute(pool) + .await + .expect("apply a shipped migration"); + } +} + +async fn open_scratch(two_databases: bool) -> Option { + let Some(base) = base_conn() else { + eprintln!( + "SKIP: EXTENDDB_TEST_PG_CONNECTION_STRING is not set, so there is no PostgreSQL \ + to build a scratch catalog in." + ); + return None; + }; + let slot = slot().await; + let admin = PgPoolOptions::new() + .max_connections(1) + .connect(&format!("{base}/postgres")) + .await + .expect("connect to the postgres maintenance database"); + let mut guard = DropDatabases { + base: base.clone(), + names: Vec::new(), + armed: true, + }; + let catalog = create_database(&admin, &base, &mut guard).await; + let catalog_name = guard.names[0].clone(); + let db = if two_databases { + apply(&catalog, &["migrations"]).await; + let data = create_database(&admin, &base, &mut guard).await; + apply(&data, &["data_migrations"]).await; + sqlx::query( + "INSERT INTO settings (key, value) VALUES ('data_database_connection_string', $1) \ + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", + ) + .bind(format!("{base}/{}", guard.names[1])) + .execute(&catalog) + .await + .expect("name the data database"); + data + } else { + apply(&catalog, &["migrations", "data_migrations"]).await; + sqlx::query( + "ALTER TABLE stream_shards DROP CONSTRAINT IF EXISTS stream_shards_table_id_fkey", + ) + .execute(&catalog) + .await + .expect("match the two-database layout"); + catalog.clone() + }; + sqlx::query( + "INSERT INTO settings (key, value) VALUES ('control_plane_delay_seconds', '0') \ + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", + ) + .execute(&catalog) + .await + .expect("pin the control-plane delay to zero"); + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES ($1, $2)") + .bind(ACCOUNT) + .bind(format!("acct-{catalog_name}")) + .execute(&catalog) + .await + .expect("seed the account row"); + let engine = PostgresEngine::new( + &PostgresConfig { + connection_string: format!("{base}/{catalog_name}"), + pool_size: 10, + max_item_size_bytes: 400_000, + }, + REGION, + ) + .await + .expect("open a PostgresEngine on the scratch database"); + Some(Scratch { + engine, + db, + catalog, + admin, + guard, + _slot: slot, + }) +} + +pub async fn close(s: Scratch) { + let Scratch { + engine, + db, + catalog, + admin, + mut guard, + _slot, + } = s; + drop(engine); + db.close().await; + catalog.close().await; + for name in &guard.names { + sqlx::query(&format!("DROP DATABASE IF EXISTS \"{name}\" WITH (FORCE)")) + .execute(&admin) + .await + .expect("drop the scratch database"); + } + guard.armed = false; + admin.close().await; +} + +impl HistoryFixture for Scratch { + fn store(&self) -> &dyn PitrCaptureStore { + &self.engine + } + + fn data(&self) -> &dyn DataEngine { + &self.engine + } + + fn table<'a>( + &'a self, + name: &'a str, + sk: Option, + enabled: bool, + ) -> BoxFuture<'a, TableKeyInfo> { + Box::pin(async move { + let mut key_schema = vec![KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }]; + let mut attribute_definitions = vec![AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }]; + if let Some(t) = sk { + key_schema.push(KeySchemaElement { + attribute_name: "sk".to_owned(), + key_type: KeyType::Range, + }); + attribute_definitions.push(AttributeDefinition { + attribute_name: "sk".to_owned(), + attribute_type: t, + }); + } + self.engine + .create_table( + ACCOUNT, + CreateTableInput { + table_name: name.to_owned(), + key_schema, + attribute_definitions, + billing_mode: Some(BillingMode::PayPerRequest), + ..Default::default() + }, + ) + .await + .expect("create the table"); + let info = self + .engine + .table_key_info(ACCOUNT, name) + .await + .expect("read the key info"); + if enabled { + sqlx::query( + "INSERT INTO pitr_state (table_id, recovery_period_days, enabled_at_us) \ + VALUES ($1, 35, 0)", + ) + .bind(&info.table_id) + .execute(&self.db) + .await + .expect("pitr_state"); + } + info + }) + } + + fn record<'a>( + &'a self, + info: &'a TableKeyInfo, + key: &'a Item, + image: Option<&'a Item>, + at: SystemTime, + commit: bool, + ) -> BoxFuture<'a, bool> { + Box::pin(async move { + self.engine + .record_item_history_for_tests(info, key, image, at, commit) + .await + .expect("record") + }) + } + + fn commit_times<'a>(&'a self, table_id: &'a str) -> BoxFuture<'a, Vec> { + Box::pin(async move { + sqlx::query_scalar( + "SELECT commit_time_us FROM item_history WHERE table_id = $1 ORDER BY seq", + ) + .bind(table_id) + .fetch_all(&self.db) + .await + .expect("commit times") + }) + } + + fn live_items<'a>(&'a self, info: &'a TableKeyInfo) -> BoxFuture<'a, Vec> { + Box::pin(async move { + let sql = format!("SELECT item_data FROM \"_ddb_{}\"", info.table_id); + let rows: Vec = sqlx::query_scalar(&sql) + .fetch_all(&self.db) + .await + .expect("live items"); + rows.into_iter() + .map(|v| serde_json::from_value(v).expect("item")) + .collect() + }) + } + + fn history_rows<'a>(&'a self, table_id: &'a str) -> BoxFuture<'a, i64> { + Box::pin(async move { + sqlx::query_scalar("SELECT COUNT(*) FROM item_history WHERE table_id = $1") + .bind(table_id) + .fetch_one(&self.db) + .await + .expect("count") + }) + } + + fn execute<'a>(&'a self, sql: &'a str) -> BoxFuture<'a, ()> { + Box::pin(async move { + sqlx::query(sql).execute(&self.db).await.expect("execute"); + }) + } +} diff --git a/crates/storage-sqlite/Cargo.toml b/crates/storage-sqlite/Cargo.toml index 96a432a1a..33e58215d 100644 --- a/crates/storage-sqlite/Cargo.toml +++ b/crates/storage-sqlite/Cargo.toml @@ -16,6 +16,9 @@ memory = [] # Compiles the snapshot page-size setter the consistency test uses and the # shared conformance check. Production builds do not compile them. test-hooks = ["extenddb-storage/test-hooks"] +# Point-in-time recovery item history: the store and the write hook. Off by +# default; the migration that creates the tables applies either way. +pitr = ["extenddb-storage/pitr"] [dependencies] extenddb-core = { workspace = true } diff --git a/crates/storage-sqlite/src/data/data_engine.rs b/crates/storage-sqlite/src/data/data_engine.rs index 964ea5f96..75ae7ce62 100644 --- a/crates/storage-sqlite/src/data/data_engine.rs +++ b/crates/storage-sqlite/src/data/data_engine.rs @@ -24,6 +24,11 @@ impl DataEngine for SqliteEngine { Some(self) } + #[cfg(feature = "pitr")] + fn as_pitr(&self) -> Option<&dyn extenddb_storage::pitr::PitrCaptureStore> { + Some(self) + } + fn put_item( &self, key_info: &TableKeyInfo, diff --git a/crates/storage-sqlite/src/data/delete_item.rs b/crates/storage-sqlite/src/data/delete_item.rs index 91c0d8ee2..b7708ea13 100644 --- a/crates/storage-sqlite/src/data/delete_item.rs +++ b/crates/storage-sqlite/src/data/delete_item.rs @@ -105,6 +105,15 @@ impl SqliteEngine { if let Some(capture) = stream { write_stream_record_in_tx(&mut tx, key_info, capture, old.as_ref(), None).await?; } + #[cfg(feature = "pitr")] + crate::pitr::capture_in_tx( + &mut tx, + key_info, + old.as_ref(), + None, + std::time::SystemTime::now(), + ) + .await?; } tx.commit() diff --git a/crates/storage-sqlite/src/data/put_item.rs b/crates/storage-sqlite/src/data/put_item.rs index 7ad7a9c21..5d6c008d7 100644 --- a/crates/storage-sqlite/src/data/put_item.rs +++ b/crates/storage-sqlite/src/data/put_item.rs @@ -138,6 +138,15 @@ impl SqliteEngine { write_stream_record_in_tx(&mut tx, key_info, capture, old.as_ref(), Some(&item)) .await?; } + #[cfg(feature = "pitr")] + crate::pitr::capture_in_tx( + &mut tx, + key_info, + old.as_ref(), + Some(&item), + std::time::SystemTime::now(), + ) + .await?; tx.commit() .await diff --git a/crates/storage-sqlite/src/data/transactions.rs b/crates/storage-sqlite/src/data/transactions.rs index f3192c7e5..a85e88b67 100644 --- a/crates/storage-sqlite/src/data/transactions.rs +++ b/crates/storage-sqlite/src/data/transactions.rs @@ -162,6 +162,23 @@ impl SqliteEngine { }); } + // Item history for every member at one instant, so a reconstruct + // holds all of the transaction or none of it. + #[cfg(feature = "pitr")] + { + let commit_time = std::time::SystemTime::now(); + for (op, (old_item, new_item)) in ops.iter().zip(op_items.iter()) { + crate::pitr::capture_in_tx( + &mut tx, + op_key_info(op), + old_item.as_ref(), + new_item.as_ref(), + commit_time, + ) + .await?; + } + } + // Capture stream records after all writes are staged. for (op, (old_item, new_item)) in ops.iter().zip(op_items.iter()) { let capture = match op { diff --git a/crates/storage-sqlite/src/data/update_item.rs b/crates/storage-sqlite/src/data/update_item.rs index 0eabdde21..db3c99690 100644 --- a/crates/storage-sqlite/src/data/update_item.rs +++ b/crates/storage-sqlite/src/data/update_item.rs @@ -173,6 +173,15 @@ impl SqliteEngine { write_stream_record_in_tx(&mut tx, key_info, capture, old.as_ref(), Some(&item)) .await?; } + #[cfg(feature = "pitr")] + crate::pitr::capture_in_tx( + &mut tx, + key_info, + old.as_ref(), + Some(&item), + std::time::SystemTime::now(), + ) + .await?; tx.commit() .await diff --git a/crates/storage-sqlite/src/hooks.rs b/crates/storage-sqlite/src/hooks.rs index 0b8a634cc..fd586a579 100644 --- a/crates/storage-sqlite/src/hooks.rs +++ b/crates/storage-sqlite/src/hooks.rs @@ -44,6 +44,16 @@ impl ServerRuntimeHooks for SqliteRuntimeHooks { workers::stream_record_cleanup_worker(engine, metrics, token).await; }); + #[cfg(feature = "pitr")] + let pitr_retention = { + let engine = self.engine.clone(); + let metrics = ctx.metrics.clone(); + let token = ctx.shutdown.clone(); + tokio::spawn(async move { + extenddb_storage::pitr::retention_worker(engine, metrics, token).await; + }) + }; + let engine = self.engine.clone(); let metrics = ctx.metrics.clone(); let token = ctx.shutdown.clone(); @@ -73,7 +83,8 @@ impl ServerRuntimeHooks for SqliteRuntimeHooks { workers::poll_index_propagation_delay(catalog_store, index_delay_cache, token).await; }); - vec![ + #[allow(unused_mut)] + let mut tasks = vec![ control_plane, table_size, stream_cleanup, @@ -81,7 +92,10 @@ impl ServerRuntimeHooks for SqliteRuntimeHooks { ttl_cleanup, gsi_propagation, gsi_delay, - ] + ]; + #[cfg(feature = "pitr")] + tasks.push(pitr_retention); + tasks } fn backend_info(&self) -> Option { diff --git a/crates/storage-sqlite/src/lib.rs b/crates/storage-sqlite/src/lib.rs index 13acdf4b7..0005fe01f 100644 --- a/crates/storage-sqlite/src/lib.rs +++ b/crates/storage-sqlite/src/lib.rs @@ -38,6 +38,8 @@ pub mod migrations; mod number_key; mod operations; mod password_hash_store; +#[cfg(feature = "pitr")] +mod pitr; mod restore_engine; #[cfg(test)] mod restore_job_tests; diff --git a/crates/storage-sqlite/src/pitr.rs b/crates/storage-sqlite/src/pitr.rs new file mode 100644 index 000000000..a751fa7af --- /dev/null +++ b/crates/storage-sqlite/src/pitr.rs @@ -0,0 +1,448 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Point-in-time recovery item history on SQLite (`pitr_state`, +//! `item_history`, migration 10). +//! +//! [`record_item_history_in_tx`] runs inside a write's transaction and adds a +//! row only when the table has a `pitr_state` row, so the decision to capture +//! and the capture commit with the write. SQLite runs one writer at a time, so +//! `seq` (the row id) follows commit order. +//! +//! A reconstruct page selects the rows committed at or before the time that no +//! later row of the same key committed at or before it supersedes, by an +//! anti-join on `idx_item_history_key`, paged on `seq`. Each history row is +//! read once over all pages, where a window function over the history would +//! read the whole history again for every page. +//! +//! A page first pins its time in `pitr_state` and a prune lowers its cutoff +//! to a pin that holds; both run under the engine write lock, so a prune +//! either sees the pin or has already moved `pruned_to_us` past the time and +//! the page refuses. + +use std::time::SystemTime; + +use extenddb_core::types::{Item, TableKeyInfo}; +use extenddb_storage::error::StorageError; +use extenddb_storage::pitr::{ + DEFAULT_RECOVERY_PERIOD_DAYS, PitrCaptureStore, PitrState, RECONSTRUCT_PIN_LEASE, + ReconstructPage, from_unix_micros, pinned_cutoff, reconstruct_held, stored_period, unix_micros, +}; +use extenddb_storage::util::{parse_sk, pk_to_text, sk_column, sk_info}; +use futures::future::BoxFuture; + +use crate::data::{BoundValue, data_table_name, json_to_item, sk_bound}; +use crate::store::SqliteEngine; + +/// Superseded: a later row of the same key committed at or before `?2`. +const SUPERSEDED: &str = "EXISTS (SELECT 1 FROM item_history n \ + WHERE n.table_id = h.table_id AND n.pk = h.pk AND n.sk IS h.sk \ + AND n.seq > h.seq AND n.commit_time_us <= ?2)"; + +/// An earlier row of the same key committed at or before `?2`. +const HAS_EARLIER: &str = "EXISTS (SELECT 1 FROM item_history e \ + WHERE e.table_id = h.table_id AND e.pk = h.pk AND e.sk IS h.sk \ + AND e.seq < h.seq AND e.commit_time_us <= ?2)"; + +/// One reconstruct page: `?1` table id, `?2` time, `?3` the cursor, `?4` the +/// limit. +fn reconstruct_sql() -> String { + format!( + "SELECT h.seq, h.item_data FROM item_history h \ + WHERE h.table_id = ?1 AND h.commit_time_us <= ?2 AND h.seq > ?3 \ + AND h.item_data IS NOT NULL AND NOT {SUPERSEDED} \ + ORDER BY h.seq LIMIT ?4" + ) +} + +fn store_err(what: &str, e: sqlx::Error) -> StorageError { + StorageError::Internal(format!("item history {what}: {e}")) +} + +/// Pin `?2` on table `?1` until `?4`, at `?3` now: a pin that has expired is +/// replaced, one that holds keeps the earlier time and the later expiry. +const PIN_SQL: &str = "UPDATE pitr_state SET \ + pinned_at_us = CASE WHEN pin_expires_us IS NULL OR pin_expires_us < ?3 THEN ?2 \ + ELSE MIN(pinned_at_us, ?2) END, \ + pin_expires_us = CASE WHEN pin_expires_us IS NULL OR pin_expires_us < ?3 THEN ?4 \ + ELSE MAX(pin_expires_us, ?4) END \ + WHERE table_id = ?1"; + +/// Record one committed mutation of the table `key_info` names, in the +/// mutation's transaction. `key` is any image that carries the primary key; +/// `image` is the item after the mutation, `None` for a delete. Returns +/// whether a row was added, which is when the table's recovery is enabled. +pub(crate) async fn record_item_history_in_tx( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + key_info: &TableKeyInfo, + key: &Item, + image: Option<&Item>, + commit_time: SystemTime, +) -> Result { + let pk_name = &key_info.key_schema[0].attribute_name; + let pk_value = key + .get(pk_name) + .ok_or_else(|| StorageError::Internal("missing partition key".to_owned()))?; + let pk_text = pk_to_text(pk_value)?; + let sk = match sk_info(&key_info.key_schema, &key_info.attribute_definitions) { + Some((sk_name, sk_type)) => { + let sk_value = key + .get(sk_name) + .ok_or_else(|| StorageError::Internal("missing sort key".to_owned()))?; + Some(sk_bound(&parse_sk(sk_value, sk_type)?)) + } + None => None, + }; + let item_json = image + .map(serde_json::to_string) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + let query = sqlx::query( + "INSERT INTO item_history (table_id, pk, sk, commit_time_us, item_data) \ + SELECT ?1, ?2, ?3, ?4, ?5 WHERE EXISTS (SELECT 1 FROM pitr_state WHERE table_id = ?1)", + ) + .bind(&key_info.table_id) + .bind(pk_text.as_ref()); + let query = match sk { + Some(BoundValue::Text(s)) => query.bind(s), + Some(BoundValue::Blob(b)) => query.bind(b), + None => query.bind(None::), + }; + let done = query + .bind(unix_micros(commit_time)?) + .bind(item_json) + .execute(&mut **tx) + .await + .map_err(|e| store_err("record", e))?; + Ok(done.rows_affected() == 1) +} + +/// The write paths' hook: record a mutation from its before and after images, +/// in the mutation's transaction after its data write. A mutation with +/// neither image changed nothing (a delete of a missing item, a condition +/// check) and records nothing. Every mutation of one transaction passes the +/// same `commit_time`, so a reconstruct never splits a transaction. +pub(crate) async fn capture_in_tx( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + key_info: &TableKeyInfo, + old: Option<&Item>, + new: Option<&Item>, + commit_time: SystemTime, +) -> Result<(), StorageError> { + let Some(key) = new.or(old) else { + return Ok(()); + }; + record_item_history_in_tx(tx, key_info, key, new, commit_time).await?; + Ok(()) +} + +/// The columns [`state_of`] reads, in its order. +const STATE_COLUMNS: &str = "recovery_period_days, enabled_at_us, pruned_to_us"; + +fn state_of( + table_id: &str, + (days, at, pruned): (i64, i64, i64), +) -> Result { + Ok(PitrState { + pruned_to: from_unix_micros(pruned), + ..PitrState::new(stored_period(table_id, days)?, from_unix_micros(at)) + }) +} + +async fn state_in( + conn: &mut sqlx::SqliteConnection, + table_id: &str, +) -> Result, StorageError> { + let row: Option<(i64, i64, i64)> = sqlx::query_as(&format!( + "SELECT {STATE_COLUMNS} FROM pitr_state WHERE table_id = ?" + )) + .bind(table_id) + .fetch_optional(conn) + .await + .map_err(|e| store_err("state", e))?; + row.map(|r| state_of(table_id, r)).transpose() +} + +/// Whether the catalog still has the table. A DeleteTable removes the row +/// and drops the data table in one transaction under the write lock. +async fn table_exists( + conn: &mut sqlx::SqliteConnection, + table_id: &str, +) -> Result { + sqlx::query_scalar("SELECT EXISTS (SELECT 1 FROM tables WHERE table_id = ?)") + .bind(table_id) + .fetch_one(conn) + .await + .map_err(|e| store_err("table", e)) +} + +fn micros_from_now(d: std::time::Duration) -> Result<(i64, i64), StorageError> { + let now = SystemTime::now(); + Ok((unix_micros(now)?, unix_micros(now + d)?)) +} + +impl PitrCaptureStore for SqliteEngine { + fn pitr_state<'a>( + &'a self, + table_id: &'a str, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let mut conn = self + .pool + .acquire() + .await + .map_err(|e| store_err("state", e))?; + state_in(&mut conn, table_id).await + }) + } + + /// Under the write lock, which every write takes before its transaction, + /// so the baseline and the state commit with no write between them. + fn enable_pitr<'a>( + &'a self, + key_info: &'a TableKeyInfo, + period: Option, + at: SystemTime, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let table_id = key_info.table_id.as_str(); + let _writer = self.write_lock.lock().await; + let mut tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(|e| store_err("enable", e))?; + if !table_exists(&mut tx, table_id).await? { + return Err(StorageError::TableNotFound(key_info.table_name.clone())); + } + let state = match state_in(&mut tx, table_id).await? { + Some(mut state) => { + if let Some(days) = period { + sqlx::query( + "UPDATE pitr_state SET recovery_period_days = ? WHERE table_id = ?", + ) + .bind(i64::from(days)) + .bind(table_id) + .execute(&mut *tx) + .await + .map_err(|e| store_err("enable", e))?; + state.recovery_period_days = days; + } + state + } + None => { + let state = PitrState::new(period.unwrap_or(DEFAULT_RECOVERY_PERIOD_DAYS), at); + sqlx::query( + "INSERT INTO pitr_state (table_id, recovery_period_days, enabled_at_us) \ + VALUES (?, ?, ?)", + ) + .bind(table_id) + .bind(i64::from(state.recovery_period_days)) + .bind(unix_micros(at)?) + .execute(&mut *tx) + .await + .map_err(|e| store_err("enable", e))?; + // The data table's key columns are the history's: the + // same pk text and the same typed sort key value. + let sk = sk_info(&key_info.key_schema, &key_info.attribute_definitions) + .map_or("NULL", |(_, t)| sk_column(t)); + sqlx::query(&format!( + "INSERT INTO item_history (table_id, pk, sk, commit_time_us, item_data) \ + SELECT ?, pk, {sk}, ?, item_data FROM {}", + data_table_name(table_id) + )) + .bind(table_id) + .bind(unix_micros(at)?) + .execute(&mut *tx) + .await + .map_err(|e| store_err("baseline", e))?; + state + } + }; + tx.commit().await.map_err(|e| store_err("enable", e))?; + Ok(state) + }) + } + + fn pitr_tables(&self) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + let rows: Vec<(String, i64, i64, i64)> = sqlx::query_as(&format!( + "SELECT table_id, {STATE_COLUMNS} FROM pitr_state ORDER BY table_id" + )) + .fetch_all(&self.pool) + .await + .map_err(|e| store_err("tables", e))?; + rows.into_iter() + .map(|(id, days, at, pruned)| { + let state = state_of(&id, (days, at, pruned))?; + Ok((id, state)) + }) + .collect() + }) + } + + /// The catalog's foreign keys remove a table's history and setting with + /// the table, so there is never an orphan here. + fn remove_orphaned_history(&self, _limit: u32) -> BoxFuture<'_, Result> { + Box::pin(async { Ok(0) }) + } + + fn disable_pitr<'a>( + &'a self, + key_info: &'a TableKeyInfo, + ) -> BoxFuture<'a, Result<(), StorageError>> { + Box::pin(async move { + let _writer = self.write_lock.lock().await; + let mut tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(|e| store_err("disable", e))?; + if !table_exists(&mut tx, &key_info.table_id).await? { + return Err(StorageError::TableNotFound(key_info.table_name.clone())); + } + for sql in [ + "DELETE FROM item_history WHERE table_id = ?", + "DELETE FROM pitr_state WHERE table_id = ?", + ] { + sqlx::query(sql) + .bind(&key_info.table_id) + .execute(&mut *tx) + .await + .map_err(|e| store_err("disable", e))?; + } + tx.commit().await.map_err(|e| store_err("disable", e)) + }) + } + + fn reconstruct_page<'a>( + &'a self, + table_id: &'a str, + at: SystemTime, + after: Option, + limit: u32, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let at_us = unix_micros(at)?; + { + let (now_us, expires_us) = micros_from_now(RECONSTRUCT_PIN_LEASE)?; + let _writer = self.write_lock.lock().await; + let mut tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(|e| store_err("pin", e))?; + let pruned: Option = + sqlx::query_scalar("SELECT pruned_to_us FROM pitr_state WHERE table_id = ?") + .bind(table_id) + .fetch_optional(&mut *tx) + .await + .map_err(|e| store_err("pin", e))?; + reconstruct_held(table_id, at_us, pruned)?; + sqlx::query(PIN_SQL) + .bind(table_id) + .bind(at_us) + .bind(now_us) + .bind(expires_us) + .execute(&mut *tx) + .await + .map_err(|e| store_err("pin", e))?; + tx.commit().await.map_err(|e| store_err("pin", e))?; + } + let rows: Vec<(i64, serde_json::Value)> = sqlx::query_as(&reconstruct_sql()) + .bind(table_id) + .bind(at_us) + .bind(after.unwrap_or(i64::MIN)) + .bind(i64::from(limit)) + .fetch_all(&self.pool) + .await + .map_err(|e| store_err("reconstruct", e))?; + let next = if rows.len() == limit as usize { + rows.last().map(|(seq, _)| *seq) + } else { + None + }; + let items = rows + .into_iter() + .map(|(_, v)| json_to_item(v)) + .collect::>()?; + Ok(ReconstructPage { items, next }) + }) + } + + fn prune_before<'a>( + &'a self, + table_id: &'a str, + cutoff: SystemTime, + limit: u32, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let (now_us, _) = micros_from_now(std::time::Duration::ZERO)?; + // The write lock orders this with every write, enable, disable + // and reconstruct pin of the engine. + let _writer = self.write_lock.lock().await; + let mut tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(|e| store_err("prune", e))?; + let pin: Option<(Option, Option)> = sqlx::query_as( + "SELECT pinned_at_us, pin_expires_us FROM pitr_state WHERE table_id = ?", + ) + .bind(table_id) + .fetch_optional(&mut *tx) + .await + .map_err(|e| store_err("prune", e))?; + let cutoff_us = + pinned_cutoff(unix_micros(cutoff)?, pin.unwrap_or((None, None)), now_us); + let superseded = format!( + "DELETE FROM item_history WHERE seq IN (SELECT h.seq FROM item_history h \ + WHERE h.table_id = ?1 AND h.commit_time_us <= ?2 AND {SUPERSEDED} \ + ORDER BY h.seq LIMIT ?3)" + ); + let mut removed = sqlx::query(&superseded) + .bind(table_id) + .bind(cutoff_us) + .bind(i64::from(limit)) + .execute(&mut *tx) + .await + .map_err(|e| store_err("prune", e))? + .rows_affected(); + // A delete is dropped only once it is the only row of its key at + // or before the cutoff: with an older image still present, the + // image would become the latest row and bring the item back. + if removed < u64::from(limit) { + let deletes = format!( + "DELETE FROM item_history WHERE seq IN (SELECT h.seq FROM item_history h \ + WHERE h.table_id = ?1 AND h.commit_time_us <= ?2 \ + AND h.item_data IS NULL AND NOT {SUPERSEDED} AND NOT {HAS_EARLIER} \ + ORDER BY h.seq LIMIT ?3)" + ); + removed += sqlx::query(&deletes) + .bind(table_id) + .bind(cutoff_us) + .bind(i64::try_from(u64::from(limit) - removed).unwrap_or(i64::MAX)) + .execute(&mut *tx) + .await + .map_err(|e| store_err("prune", e))? + .rows_affected(); + } + if removed > 0 { + sqlx::query( + "UPDATE pitr_state SET pruned_to_us = MAX(pruned_to_us, ?) WHERE table_id = ?", + ) + .bind(cutoff_us) + .bind(table_id) + .execute(&mut *tx) + .await + .map_err(|e| store_err("prune", e))?; + } + tx.commit().await.map_err(|e| store_err("prune", e))?; + Ok(removed) + }) + } +} + +#[cfg(test)] +#[path = "pitr_tests.rs"] +mod tests; diff --git a/crates/storage-sqlite/src/pitr_tests.rs b/crates/storage-sqlite/src/pitr_tests.rs new file mode 100644 index 000000000..4a8c7f1e9 --- /dev/null +++ b/crates/storage-sqlite/src/pitr_tests.rs @@ -0,0 +1,400 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The SQLite item-history store: the shared checks of +//! `extenddb_storage::pitr::conformance`, then what is SQLite's own (the +//! catalog cascade and the query plan). + +use std::time::SystemTime; + +use extenddb_core::types::{ + AttributeDefinition, BillingMode, CreateTableInput, Item, KeySchemaElement, KeyType, + ScalarAttributeType, TableKeyInfo, +}; +use extenddb_storage::DataEngine; +use extenddb_storage::pitr::PitrCaptureStore; +use extenddb_storage::pitr::conformance::{HistoryFixture, at}; +use futures::future::BoxFuture; + +use super::record_item_history_in_tx; +use crate::store::SqliteEngine; + +const ACCOUNT: &str = "111111111111"; + +async fn open() -> Option { + let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::migrations::apply_for_tests(&engine.pool).await; + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, ?)") + .bind(ACCOUNT) + .bind(ACCOUNT) + .execute(&engine.pool) + .await + .expect("account"); + Some(engine) +} + +async fn close(_engine: SqliteEngine) {} + +impl HistoryFixture for SqliteEngine { + fn store(&self) -> &dyn PitrCaptureStore { + self + } + + fn data(&self) -> &dyn DataEngine { + self + } + + fn table<'a>( + &'a self, + name: &'a str, + sk: Option, + enabled: bool, + ) -> BoxFuture<'a, TableKeyInfo> { + Box::pin(async move { + let mut key_schema = vec![KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }]; + let mut attribute_definitions = vec![AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }]; + if let Some(t) = sk { + key_schema.push(KeySchemaElement { + attribute_name: "sk".to_owned(), + key_type: KeyType::Range, + }); + attribute_definitions.push(AttributeDefinition { + attribute_name: "sk".to_owned(), + attribute_type: t, + }); + } + let desc = self + .create_table_impl( + ACCOUNT, + CreateTableInput { + table_name: name.to_owned(), + key_schema: key_schema.clone(), + attribute_definitions: attribute_definitions.clone(), + billing_mode: Some(BillingMode::PayPerRequest), + ..Default::default() + }, + false, + ) + .await + .expect("create the table"); + let table_id = desc.table_id; + if enabled { + sqlx::query( + "INSERT INTO pitr_state (table_id, recovery_period_days, enabled_at_us) \ + VALUES (?, 35, 0)", + ) + .bind(&table_id) + .execute(&self.pool) + .await + .expect("pitr_state"); + } + TableKeyInfo { + table_name: name.to_owned(), + account_id: ACCOUNT.to_owned(), + table_id, + base_key_schema: key_schema.clone(), + key_schema, + attribute_definitions, + ..Default::default() + } + }) + } + + fn record<'a>( + &'a self, + info: &'a TableKeyInfo, + key: &'a Item, + image: Option<&'a Item>, + at: SystemTime, + commit: bool, + ) -> BoxFuture<'a, bool> { + Box::pin(async move { + let mut tx = self.pool.begin().await.expect("begin"); + let added = record_item_history_in_tx(&mut tx, info, key, image, at) + .await + .expect("record"); + if commit { + tx.commit().await.expect("commit"); + } else { + tx.rollback().await.expect("rollback"); + } + added + }) + } + + fn commit_times<'a>(&'a self, table_id: &'a str) -> BoxFuture<'a, Vec> { + Box::pin(async move { + sqlx::query_scalar( + "SELECT commit_time_us FROM item_history WHERE table_id = ? ORDER BY seq", + ) + .bind(table_id) + .fetch_all(&self.pool) + .await + .expect("commit times") + }) + } + + fn live_items<'a>(&'a self, info: &'a TableKeyInfo) -> BoxFuture<'a, Vec> { + Box::pin(async move { + let sql = format!( + "SELECT item_data FROM {}", + crate::data::data_table_name(&info.table_id) + ); + let rows: Vec<(serde_json::Value,)> = sqlx::query_as(&sql) + .fetch_all(&self.pool) + .await + .expect("live items"); + rows.into_iter() + .map(|(v,)| serde_json::from_value(v).expect("item")) + .collect() + }) + } + + fn history_rows<'a>(&'a self, table_id: &'a str) -> BoxFuture<'a, i64> { + Box::pin(async move { + sqlx::query_scalar("SELECT COUNT(*) FROM item_history WHERE table_id = ?") + .bind(table_id) + .fetch_one(&self.pool) + .await + .expect("count") + }) + } + + fn execute<'a>(&'a self, sql: &'a str) -> BoxFuture<'a, ()> { + Box::pin(async move { + sqlx::query(sql).execute(&self.pool).await.expect("execute"); + }) + } +} + +extenddb_storage::pitr_conformance_tests!(open, close); + +fn item(pk: &str) -> Item { + Item::from([( + "pk".to_owned(), + extenddb_core::types::AttributeValue::S(pk.to_owned()), + )]) +} + +/// The history and the recovery setting go with the table's catalog row. +#[tokio::test] +async fn dropping_the_catalog_row_drops_its_history_and_state() { + let engine = open().await.expect("engine"); + let info = engine.table("t", None, true).await; + let k = item("k"); + assert!(engine.record(&info, &k, Some(&k), at(1), true).await); + sqlx::query("DELETE FROM tables WHERE table_id = ?") + .bind(&info.table_id) + .execute(&engine.pool) + .await + .expect("delete"); + assert_eq!(engine.history_rows(&info.table_id).await, 0); + let state_rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM pitr_state") + .fetch_one(&engine.pool) + .await + .expect("count"); + assert_eq!(state_rows, 0); +} + +/// The reconstruct reads through the key index, with no table scan. +#[tokio::test] +async fn reconstruct_reads_through_the_key_index() { + let engine = open().await.expect("engine"); + let info = engine.table("t", None, true).await; + let k = item("k"); + engine.record(&info, &k, Some(&k), at(1), true).await; + let sql = format!("EXPLAIN QUERY PLAN {}", super::reconstruct_sql()); + let plan: Vec<(i64, i64, i64, String)> = sqlx::query_as(&sql) + .bind(&info.table_id) + .bind(0_i64) + .bind(0_i64) + .bind(10_i64) + .fetch_all(&engine.pool) + .await + .expect("plan"); + let text: Vec<&str> = plan.iter().map(|r| r.3.as_str()).collect(); + assert!( + text.iter().any(|l| l.contains("idx_item_history_key")), + "{text:?}" + ); + assert!(text.iter().all(|l| !l.starts_with("SCAN")), "{text:?}"); +} + +/// A file database with a pool of four connections, where a write and an +/// enable can run on two connections at once. +async fn open_pooled() -> (std::sync::Arc, std::path::PathBuf) { + let path = std::env::temp_dir().join(format!("eddb-pitr-{}.db", uuid::Uuid::new_v4())); + let engine = SqliteEngine::new(&path.to_string_lossy(), 4, "us-east-1", 409_600) + .await + .expect("engine"); + crate::migrations::apply_for_tests(&engine.pool).await; + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, ?)") + .bind(ACCOUNT) + .bind(ACCOUNT) + .execute(&engine.pool) + .await + .expect("account"); + (std::sync::Arc::new(engine), path) +} + +async fn close_pooled(engine: std::sync::Arc, path: std::path::PathBuf) { + let engine = std::sync::Arc::into_inner(engine).expect("sole owner"); + engine.pool.close().await; + for suffix in ["", "-wal", "-shm"] { + let _ = std::fs::remove_file(format!("{}{suffix}", path.to_string_lossy())); + } +} + +fn valued(v: &str) -> Item { + let mut i = item("k"); + i.insert( + "v".to_owned(), + extenddb_core::types::AttributeValue::S(v.to_owned()), + ); + i +} + +/// A write as the data path runs it, held open: the engine write lock, then +/// an immediate transaction with the data write and its history statement. +/// Returns what the history statement returned. +async fn write_in_flight<'a>( + engine: &'a SqliteEngine, + info: &TableKeyInfo, + new: &Item, +) -> ( + tokio::sync::OwnedMutexGuard<()>, + sqlx::Transaction<'a, sqlx::Sqlite>, + bool, +) { + let writer = engine.write_lock.clone().lock_owned().await; + let mut tx = engine + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .expect("begin"); + sqlx::query(&format!( + "UPDATE {} SET item_data = ? WHERE pk = 'k'", + crate::data::data_table_name(&info.table_id) + )) + .bind(serde_json::to_string(new).expect("json")) + .execute(&mut *tx) + .await + .expect("write"); + let recorded = record_item_history_in_tx(&mut tx, info, new, Some(new), SystemTime::now()) + .await + .expect("history statement"); + (writer, tx, recorded) +} + +/// An enable that starts while a write is open waits for it, so its +/// baseline holds the write: the write's history statement ran before the +/// enable and recorded nothing. +#[tokio::test] +async fn an_enable_waits_for_a_write_in_flight() { + use extenddb_storage::pitr::conformance::state; + let (engine, path) = open_pooled().await; + let info = engine.table("t", None, false).await; + engine + .put_item(&info, valued("old"), false, None, &Default::default(), None) + .await + .expect("put"); + let new = valued("new"); + let (writer, tx, recorded) = write_in_flight(&engine, &info, &new).await; + assert!(!recorded); + let enabling = { + let (engine, info) = (engine.clone(), info.clone()); + tokio::spawn(async move { + engine + .enable_pitr(&info, None, SystemTime::now()) + .await + .expect("enable") + }) + }; + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + assert!( + !enabling.is_finished(), + "the enable ran past the open write" + ); + tx.commit().await.expect("commit the write"); + drop(writer); + tokio::time::timeout(std::time::Duration::from_secs(10), enabling) + .await + .expect("the enable finishes once the write commits") + .expect("enable task"); + assert_eq!( + state(&*engine, &info, SystemTime::now(), 10).await, + vec![serde_json::to_string(&new).expect("json")] + ); + close_pooled(engine, path).await; +} + +/// A disable that starts while a write is open waits for it, so it removes +/// the write's history row too: no row of the write survives the disable. +#[tokio::test] +async fn a_disable_waits_for_a_write_in_flight() { + let (engine, path) = open_pooled().await; + let info = engine.table("t", None, false).await; + engine + .put_item(&info, valued("old"), false, None, &Default::default(), None) + .await + .expect("put"); + engine + .enable_pitr(&info, None, SystemTime::now()) + .await + .expect("enable"); + let new = valued("new"); + let (writer, tx, recorded) = write_in_flight(&engine, &info, &new).await; + assert!(recorded); + let disabling = { + let (engine, info) = (engine.clone(), info.clone()); + tokio::spawn(async move { engine.disable_pitr(&info).await.expect("disable") }) + }; + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + assert!( + !disabling.is_finished(), + "the disable ran past the open write" + ); + tx.commit().await.expect("commit the write"); + drop(writer); + tokio::time::timeout(std::time::Duration::from_secs(10), disabling) + .await + .expect("the disable finishes once the write commits") + .expect("disable task"); + assert_eq!(engine.history_rows(&info.table_id).await, 0); + assert_eq!( + engine.pitr_state(&info.table_id).await.expect("state"), + None + ); + close_pooled(engine, path).await; +} + +/// An enable or a disable of a table a DeleteTable removed names the table +/// as not found. +#[tokio::test] +async fn an_enable_or_disable_of_a_deleted_table_is_table_not_found() { + let engine = open().await.expect("engine"); + let info = engine.table("gone", None, false).await; + sqlx::query("DELETE FROM tables WHERE table_id = ?") + .bind(&info.table_id) + .execute(&engine.pool) + .await + .expect("delete the catalog row"); + let enabled = engine.enable_pitr(&info, None, SystemTime::now()).await; + assert!( + matches!(enabled, Err(extenddb_storage::error::StorageError::TableNotFound(ref n)) if n == "gone"), + "{enabled:?}" + ); + let disabled = engine.disable_pitr(&info).await; + assert!( + matches!(disabled, Err(extenddb_storage::error::StorageError::TableNotFound(ref n)) if n == "gone"), + "{disabled:?}" + ); +} diff --git a/crates/storage/Cargo.toml b/crates/storage/Cargo.toml index a024143df..c2578dcfe 100755 --- a/crates/storage/Cargo.toml +++ b/crates/storage/Cargo.toml @@ -17,6 +17,9 @@ test-util = [] # Compiles the `snapshot_conformance` check that each backend's tests run # against a real database. Production builds do not compile it. test-hooks = [] +# The item-history store trait of point-in-time recovery. Off by default; the +# engine and the SQL backends enable it with their own `pitr` features. +pitr = [] [dependencies] async-trait = { workspace = true } @@ -39,3 +42,8 @@ toml = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } zeroize = { workspace = true } + +[dev-dependencies] +# `start_paused` for the retention worker's test, whose first sweep runs an +# hour after start. +tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index 96bed67b0..47e702ec4 100755 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -22,6 +22,8 @@ pub mod management_store_factory; pub mod migrations; pub mod operations; pub mod password; +#[cfg(feature = "pitr")] +pub mod pitr; pub mod policy_migration; pub mod restore_engine; pub mod secret_cipher; @@ -454,6 +456,13 @@ pub trait DataEngine: Send + Sync { None } + /// The backend's point-in-time recovery store, `None` on a backend + /// without one, whose recovery operations keep the unavailable answer. + #[cfg(feature = "pitr")] + fn as_pitr(&self) -> Option<&dyn pitr::PitrCaptureStore> { + None + } + /// Execute multiple get operations in a single consistent snapshot. /// /// Returns one `Option` per request, in the same order as `ops`. diff --git a/crates/storage/src/pitr.rs b/crates/storage/src/pitr.rs new file mode 100644 index 000000000..d2cec8ab5 --- /dev/null +++ b/crates/storage/src/pitr.rs @@ -0,0 +1,340 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Point-in-time recovery: the item-history store a backend keeps for a table +//! whose recovery is enabled, so a restore can rebuild the table's state at a +//! chosen time. +//! +//! A backend records one history row for every committed mutation of such a +//! table, inside the write's own transaction, so a mutation and its row commit +//! together. That write is internal to each backend's data path. This trait is +//! the read and retention side the engine calls. +//! +//! Measured on the live service: a restore to a time T holds the latest image +//! of each key committed at or before T, and keys deleted by T are absent. The +//! service lands a few seconds before T; the store reconstructs exactly at the +//! time it is given, and the engine decides which time to pass. + +#[cfg(feature = "test-hooks")] +pub mod conformance; + +use std::time::{SystemTime, UNIX_EPOCH}; + +use extenddb_core::types::{Item, TableKeyInfo}; +use futures::future::BoxFuture; + +use crate::error::StorageError; + +/// One page of a reconstruct. `next` is the cursor for the following page, or +/// `None` after the last. +#[derive(Debug, Default)] +pub struct ReconstructPage { + /// The items of this page, in history order. + pub items: Vec, + /// Pass as `after` to read the next page. + pub next: Option, +} + +/// The retention a fresh enable takes when the request gives none (measured). +pub const DEFAULT_RECOVERY_PERIOD_DAYS: u8 = 35; + +/// How long a reconstruct keeps the history it reads from pruning after it +/// reads a page. A page read later than this after the previous one fails +/// with [`StorageError::SnapshotExpired`] when a prune has passed its time +/// meanwhile. +pub const RECONSTRUCT_PIN_LEASE: std::time::Duration = std::time::Duration::from_secs(900); + +/// A table's recovery setting: present while recovery is enabled. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PitrState { + /// Retention, 1 to 35 days. + pub recovery_period_days: u8, + /// When recovery was last enabled. + pub enabled_at: SystemTime, + /// The cutoff of the latest prune that removed a row: the history holds + /// every state from it on, and may not hold an earlier one. The Unix + /// epoch until a prune removes a row. + pub pruned_to: SystemTime, +} + +impl PitrState { + /// The state of a fresh enable: `recovery_period_days` of retention from + /// `enabled_at`, nothing pruned. + #[must_use] + pub const fn new(recovery_period_days: u8, enabled_at: SystemTime) -> Self { + Self { + recovery_period_days, + enabled_at, + pruned_to: UNIX_EPOCH, + } + } +} + +/// The per-backend item-history store. +pub trait PitrCaptureStore: Send + Sync { + /// The table's recovery setting, `None` while recovery is disabled. + fn pitr_state<'a>( + &'a self, + table_id: &'a str, + ) -> BoxFuture<'a, Result, StorageError>>; + + /// Enable recovery on the table `key_info` names, as the service does + /// (measured). On a table without recovery: record the state with + /// `period` or [`DEFAULT_RECOVERY_PERIOD_DAYS`] and `at` as the enable + /// instant, and record every item the table holds as a history row at + /// `at`, the baseline a restore starts from. On a table with recovery: + /// change only the period, and only when `period` is given. No write is + /// left out of both the baseline and the history: the backend excludes + /// the table's writers while it takes the baseline. Two enables of one + /// table at once both succeed, one after the other. Returns the state. + /// A table that is gone is [`StorageError::TableNotFound`] with its name. + fn enable_pitr<'a>( + &'a self, + key_info: &'a TableKeyInfo, + period: Option, + at: SystemTime, + ) -> BoxFuture<'a, Result>; + + /// Every table with recovery enabled and its setting, for the retention + /// sweep. + fn pitr_tables(&self) -> BoxFuture<'_, Result, StorageError>>; + + /// Remove at most `limit` history rows, and then the setting, of tables + /// the catalog no longer has. Returns the rows removed. A backend whose + /// catalog removes them with the table returns 0. + fn remove_orphaned_history(&self, limit: u32) -> BoxFuture<'_, Result>; + + /// Disable recovery: remove the state and every history row of the + /// table, excluding its writers while it does, so no row of a write in + /// flight survives the disable. A disabled table is left as it is. A + /// table that is gone is [`StorageError::TableNotFound`] with its name. + fn disable_pitr<'a>( + &'a self, + key_info: &'a TableKeyInfo, + ) -> BoxFuture<'a, Result<(), StorageError>>; + + /// One page of the table's state at `at`: the latest image of each key + /// committed at or before `at`, with keys whose latest record by then is a + /// delete omitted. `after` is the previous page's `next`, `None` for the + /// first page. At most `limit` items. + /// + /// Each page pins `at` for [`RECONSTRUCT_PIN_LEASE`]: no prune removes a + /// row the state at `at` reads while the pin holds, so the pages of one + /// reconstruct return one state even when a sweep runs between them. + /// When the history no longer holds the state at `at` (a prune passed it + /// while no pin held it, or recovery was disabled), the page is + /// [`StorageError::SnapshotExpired`] and the reconstruct must start again. + fn reconstruct_page<'a>( + &'a self, + table_id: &'a str, + at: SystemTime, + after: Option, + limit: u32, + ) -> BoxFuture<'a, Result>; + + /// Remove at most `limit` history rows that no reconstruct at or after + /// `cutoff` reads: rows superseded by a later row committed at or before + /// `cutoff`, and deletes that are the only row of their key at or before + /// `cutoff`. The state at `cutoff` is unchanged. A reconstruct's pin + /// earlier than `cutoff` lowers the cutoff to it. When a row goes, the + /// state's `pruned_to` moves to the cutoff used. Returns the number of + /// rows removed; fewer than `limit` means nothing removable is left. + fn prune_before<'a>( + &'a self, + table_id: &'a str, + cutoff: SystemTime, + limit: u32, + ) -> BoxFuture<'a, Result>; +} + +/// The time `micros` microseconds after the Unix epoch. +#[must_use] +pub fn from_unix_micros(micros: i64) -> SystemTime { + UNIX_EPOCH + std::time::Duration::from_micros(u64::try_from(micros).unwrap_or(0)) +} + +/// Microseconds since the Unix epoch, the unit history rows store. +/// +/// # Errors +/// +/// A time before the epoch or past the `i64` range. +pub fn unix_micros(t: SystemTime) -> Result { + t.duration_since(UNIX_EPOCH) + .ok() + .and_then(|d| i64::try_from(d.as_micros()).ok()) + .ok_or_else(|| StorageError::Internal(format!("time out of range: {t:?}"))) +} + +/// The floor of a table's restorable window at `now`: the latest of the +/// enable instant, `now` less the retention, and the cutoff of the latest +/// prune, so the floor is never earlier than a state the history holds after +/// the period is raised. A sweep prunes to it, so every state a restore +/// inside the window reads stays. +#[must_use] +pub fn retention_floor(state: &PitrState, now: SystemTime) -> SystemTime { + let retention = std::time::Duration::from_secs(u64::from(state.recovery_period_days) * 86_400); + let floor = now + .checked_sub(retention) + .map_or(state.enabled_at, |t| t.max(state.enabled_at)); + floor.max(state.pruned_to) +} + +/// A stored retention, checked: a value outside 1 to 35 is a corrupt row, +/// reported rather than read as some other period. +/// +/// # Errors +/// +/// `days` is outside 1 to 35. +pub fn stored_period(table_id: &str, days: i64) -> Result { + u8::try_from(days) + .ok() + .filter(|d| (1..=35).contains(d)) + .ok_or_else(|| { + StorageError::Internal(format!( + "pitr_state of table {table_id} holds recovery_period_days {days}, outside 1 to 35" + )) + }) +} + +/// A page at `at_us` reads a state the history holds: recovery is enabled +/// and no prune has passed `at_us`. `pruned` is the table's `pruned_to` +/// in microseconds, `None` without a recovery setting. +/// +/// # Errors +/// +/// [`StorageError::SnapshotExpired`] when the state is no longer held. +pub fn reconstruct_held( + table_id: &str, + at_us: i64, + pruned: Option, +) -> Result<(), StorageError> { + match pruned { + None => Err(StorageError::SnapshotExpired(format!( + "point-in-time recovery of table {table_id} was disabled during the reconstruct" + ))), + Some(p) if at_us < p => Err(StorageError::SnapshotExpired(format!( + "the history of table {table_id} was pruned past the reconstruct time" + ))), + Some(_) => Ok(()), + } +} + +/// The cutoff a prune uses: `cutoff_us`, or a pin that holds at `now_us` +/// when that is earlier. `pin` is the pinned time and its expiry, in +/// microseconds. +#[must_use] +pub fn pinned_cutoff( + cutoff_us: i64, + (pinned, expires): (Option, Option), + now_us: i64, +) -> i64 { + match (pinned, expires) { + (Some(p), Some(e)) if e >= now_us => cutoff_us.min(p), + _ => cutoff_us, + } +} + +/// What one retention sweep removed. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct SweepReport { + /// History rows pruned below the floors. + pub pruned: u64, + /// History rows of deleted tables removed. + pub orphaned: u64, + /// Whether the deadline stopped the sweep before it finished. + pub stopped_at_deadline: bool, +} + +/// One retention sweep: every enabled table pruned to its floor at `now`, in +/// batches of `batch`, then the history of deleted tables, until done or +/// past `deadline`. A batch that removes fewer rows than `batch` ends that +/// table's work. +/// +/// # Errors +/// +/// A store read or delete fails. +pub async fn sweep( + store: &dyn PitrCaptureStore, + now: SystemTime, + batch: u32, + deadline: std::time::Instant, +) -> Result { + let mut report = SweepReport::default(); + for (table_id, state) in store.pitr_tables().await? { + let floor = retention_floor(&state, now); + loop { + if std::time::Instant::now() >= deadline { + report.stopped_at_deadline = true; + return Ok(report); + } + let n = store.prune_before(&table_id, floor, batch).await?; + report.pruned += n; + if n < u64::from(batch) { + break; + } + } + } + loop { + if std::time::Instant::now() >= deadline { + report.stopped_at_deadline = true; + return Ok(report); + } + let n = store.remove_orphaned_history(batch).await?; + report.orphaned += n; + if n < u64::from(batch) { + return Ok(report); + } + } +} + +/// How often the retention sweep runs, how many rows one delete removes, and +/// how long one sweep may run. +const SWEEP_INTERVAL: std::time::Duration = std::time::Duration::from_secs(3600); +const SWEEP_BATCH: u32 = 5_000; +const SWEEP_DEADLINE: std::time::Duration = std::time::Duration::from_secs(300); + +/// The background retention sweep a backend with a store spawns: [`sweep`] +/// once an hour, under a deadline, until shutdown. A sweep the deadline +/// stopped resumes at the next run. +pub async fn retention_worker( + store: std::sync::Arc, + metrics: std::sync::Arc, + token: crate::hooks::CancellationToken, +) { + use extenddb_core::metrics::QuerySource; + while crate::hooks::sleep_or_shutdown(&token, SWEEP_INTERVAL).await { + let start = std::time::Instant::now(); + match sweep( + &*store, + SystemTime::now(), + SWEEP_BATCH, + start + SWEEP_DEADLINE, + ) + .await + { + Ok(report) => { + if report.pruned + report.orphaned > 0 || report.stopped_at_deadline { + tracing::info!( + pruned = report.pruned, + orphaned = report.orphaned, + stopped_at_deadline = report.stopped_at_deadline, + "point-in-time recovery retention sweep" + ); + } + #[allow(clippy::cast_precision_loss)] + metrics.record_worker_success( + QuerySource::PitrRetention, + start.elapsed().as_micros() as f64, + ); + } + Err(e) => { + tracing::error!("point-in-time recovery retention sweep failed: {e}"); + metrics.record_worker_error(QuerySource::PitrRetention); + } + } + } +} + +#[cfg(test)] +#[path = "pitr/worker_tests.rs"] +mod worker_tests; diff --git a/crates/storage/src/pitr/conformance.rs b/crates/storage/src/pitr/conformance.rs new file mode 100644 index 000000000..83eb7bacf --- /dev/null +++ b/crates/storage/src/pitr/conformance.rs @@ -0,0 +1,419 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Backend-independent checks of a [`PitrCaptureStore`], run by each backend's +//! own tests against a real database through a [`HistoryFixture`]. +//! +//! The reconstruct rule is the one measured on the live service: a restore to +//! T holds the latest image of each key committed at or before T, and a key +//! deleted by T is absent (a restore between the third and fourth steps of +//! the four-step timeline below returned the one surviving item, two runs). + +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use extenddb_core::types::{AttributeValue, Item, ScalarAttributeType, TableKeyInfo}; +use futures::future::BoxFuture; + +use super::PitrCaptureStore; +use crate::DataEngine; + +mod enable; +mod pins; +mod retention; +mod writes; +pub use enable::*; +pub use pins::*; +pub use retention::*; +pub use writes::*; + +/// What a backend's test supplies: tables, the in-transaction record, and a +/// row count of the history. +pub trait HistoryFixture { + /// The store under test. + fn store(&self) -> &dyn PitrCaptureStore; + + /// The backend's data path, whose writes the hook records. + fn data(&self) -> &dyn DataEngine; + + /// A table keyed on `pk` (S) and, when `sk` is given, a sort key `sk` of + /// that type, with recovery enabled when `enabled`. Names are unique + /// within one fixture. + fn table<'a>( + &'a self, + name: &'a str, + sk: Option, + enabled: bool, + ) -> BoxFuture<'a, TableKeyInfo>; + + /// Record one mutation through the backend's write-transaction hook, + /// then commit the transaction, or roll it back when `commit` is false. + /// Returns what the hook returned. + fn record<'a>( + &'a self, + info: &'a TableKeyInfo, + key: &'a Item, + image: Option<&'a Item>, + at: SystemTime, + commit: bool, + ) -> BoxFuture<'a, bool>; + + /// The number of history rows of `table_id`. + fn history_rows<'a>(&'a self, table_id: &'a str) -> BoxFuture<'a, i64>; + + /// The commit times of `table_id`'s history rows, in `seq` order. + fn commit_times<'a>(&'a self, table_id: &'a str) -> BoxFuture<'a, Vec>; + + /// Every item the table's data table holds now. + fn live_items<'a>(&'a self, info: &'a TableKeyInfo) -> BoxFuture<'a, Vec>; + + /// Run one SQL statement on the database that holds `pitr_state`, to put + /// a row in a state the store's own calls do not reach. + fn execute<'a>(&'a self, sql: &'a str) -> BoxFuture<'a, ()>; +} + +/// A fixed instant `secs` seconds after an arbitrary epoch. +#[must_use] +pub fn at(secs: u64) -> SystemTime { + UNIX_EPOCH + Duration::from_secs(1_800_000_000 + secs) +} + +pub(super) fn s(pk: &str, v: &str) -> Item { + Item::from([ + ("pk".to_owned(), AttributeValue::S(pk.to_owned())), + ("v".to_owned(), AttributeValue::S(v.to_owned())), + ]) +} + +fn keyed(pk: &str, sk: AttributeValue, v: &str) -> Item { + Item::from([ + ("pk".to_owned(), AttributeValue::S(pk.to_owned())), + ("sk".to_owned(), sk), + ("v".to_owned(), AttributeValue::S(v.to_owned())), + ]) +} + +/// An item as a backend reads it back: parsed again from its JSON, which +/// normalizes a number's spelling. +fn as_read(item: &Item) -> Item { + serde_json::from_value(serde_json::to_value(item).expect("json")).expect("item") +} + +pub(super) fn sorted(items: &[Item]) -> Vec { + let mut v: Vec = items + .iter() + .map(|i| serde_json::to_string(&as_read(i)).expect("json")) + .collect(); + v.sort(); + v +} + +async fn commit( + f: &F, + info: &TableKeyInfo, + key: &Item, + image: Option<&Item>, + t: SystemTime, +) -> bool { + f.record(info, key, image, t, true).await +} + +/// Every item of the state at `t`, read in pages of `limit`, sorted. +pub async fn state( + f: &F, + info: &TableKeyInfo, + t: SystemTime, + limit: u32, +) -> Vec { + let mut out = Vec::new(); + let mut after = None; + loop { + let page = f + .store() + .reconstruct_page(&info.table_id, t, after, limit) + .await + .expect("reconstruct"); + assert!(page.items.len() <= limit as usize); + out.extend( + page.items + .iter() + .map(|i| serde_json::to_string(i).expect("json")), + ); + match page.next { + Some(n) => after = Some(n), + None => break, + } + } + out.sort(); + out +} + +/// The four-step timeline of the live measurement, 10 s apart: t1 put k1=v1; +/// t2 update k1=v2 and put k2=x1; t3 delete k1; t4 put k1=v3 and update k2=x2. +async fn timeline(f: &F, info: &TableKeyInfo) { + let k1 = s("k1", ""); + let k2 = s("k2", ""); + commit(f, info, &k1, Some(&s("k1", "v1")), at(10)).await; + commit(f, info, &k1, Some(&s("k1", "v2")), at(20)).await; + commit(f, info, &k2, Some(&s("k2", "x1")), at(20)).await; + commit(f, info, &k1, None, at(30)).await; + commit(f, info, &k1, Some(&s("k1", "v3")), at(40)).await; + commit(f, info, &k2, Some(&s("k2", "x2")), at(40)).await; +} + +/// A row is added only for a table with recovery enabled. +pub async fn records_only_while_recovery_is_enabled(f: &F) { + let on = f.table("on", None, true).await; + let off = f.table("off", None, false).await; + let k = s("k", "a"); + assert!(commit(f, &on, &k, Some(&k), at(1)).await); + assert!(!commit(f, &off, &k, Some(&k), at(1)).await); + assert_eq!(f.history_rows(&on.table_id).await, 1); + assert_eq!(f.history_rows(&off.table_id).await, 0); +} + +/// A row rolls back with the write's transaction. +pub async fn a_record_rolls_back_with_its_transaction(f: &F) { + let info = f.table("t", None, true).await; + let k = s("k", "a"); + assert!(f.record(&info, &k, Some(&k), at(1), false).await); + assert_eq!(f.history_rows(&info.table_id).await, 0); +} + +/// The timeline reconstructs at each step and between steps, in pages of 1, +/// 2 and 100. +pub async fn reconstructs_the_timeline_at_each_point(f: &F) { + let info = f.table("t", None, true).await; + timeline(f, &info).await; + let cases: [(u64, Vec); 7] = [ + (5, vec![]), + (10, vec![s("k1", "v1")]), + (20, vec![s("k1", "v2"), s("k2", "x1")]), + (25, vec![s("k1", "v2"), s("k2", "x1")]), + (30, vec![s("k2", "x1")]), + (35, vec![s("k2", "x1")]), + (40, vec![s("k1", "v3"), s("k2", "x2")]), + ]; + for (secs, expected) in cases { + for limit in [1, 2, 100] { + assert_eq!( + state(f, &info, at(secs), limit).await, + sorted(&expected), + "at +{secs}s, page {limit}" + ); + } + } +} + +/// A commit one microsecond after T is not in the state at T. +pub async fn sub_second_commits_count_at_their_instant(f: &F) { + let info = f.table("t", None, true).await; + let k = s("k", ""); + commit(f, &info, &k, Some(&s("k", "a")), at(10)).await; + let later = at(10) + Duration::from_micros(1); + commit(f, &info, &k, Some(&s("k", "b")), later).await; + assert_eq!(state(f, &info, at(10), 10).await, sorted(&[s("k", "a")])); + assert_eq!(state(f, &info, later, 10).await, sorted(&[s("k", "b")])); +} + +/// Commits of one key at one instant resolve in commit order. +pub async fn commits_at_one_instant_resolve_in_commit_order(f: &F) { + let info = f.table("t", None, true).await; + let k = s("k", ""); + commit(f, &info, &k, Some(&s("k", "a")), at(10)).await; + commit(f, &info, &k, None, at(10)).await; + commit(f, &info, &k, Some(&s("k", "c")), at(10)).await; + assert_eq!(state(f, &info, at(10), 10).await, sorted(&[s("k", "c")])); +} + +/// A sort key of each type identifies an item as the data table does: a +/// number by its value, whatever its spelling, and two sort keys under one +/// partition key as two items. +pub async fn sort_keys_of_every_type_identify_items(f: &F) { + let cases = [ + ( + ScalarAttributeType::S, + AttributeValue::S("a".to_owned()), + AttributeValue::S("a".to_owned()), + AttributeValue::S("b".to_owned()), + ), + ( + ScalarAttributeType::N, + AttributeValue::N("1".to_owned()), + AttributeValue::N("1.0".to_owned()), + AttributeValue::N("2".to_owned()), + ), + ( + ScalarAttributeType::B, + AttributeValue::B(vec![0, 1]), + AttributeValue::B(vec![0, 1]), + AttributeValue::B(vec![0, 2]), + ), + ]; + for (i, (ty, first, same, sibling_sk)) in cases.into_iter().enumerate() { + let info = f.table(&format!("t{i}"), Some(ty), true).await; + let a = keyed("p", first.clone(), "1"); + let b = keyed("p", same, "2"); + let sibling = keyed("p", sibling_sk, "4"); + let other = keyed("q", first, "3"); + commit(f, &info, &a, Some(&a), at(1)).await; + commit(f, &info, &sibling, Some(&sibling), at(1)).await; + commit(f, &info, &b, Some(&b), at(2)).await; + commit(f, &info, &other, Some(&other), at(3)).await; + assert_eq!( + state(f, &info, at(3), 10).await, + sorted(&[b.clone(), sibling.clone(), other.clone()]), + "{ty:?}" + ); + commit(f, &info, &b, None, at(4)).await; + assert_eq!( + state(f, &info, at(4), 10).await, + sorted(&[sibling, other]), + "{ty:?}" + ); + } +} + +/// A table reads and prunes only its own history. +pub async fn tables_do_not_see_each_others_history(f: &F) { + let a = f.table("a", None, true).await; + let b = f.table("b", None, true).await; + commit(f, &a, &s("k", ""), Some(&s("k", "a")), at(1)).await; + commit(f, &b, &s("k", ""), None, at(2)).await; + assert_eq!(state(f, &a, at(5), 10).await, sorted(&[s("k", "a")])); + assert!(state(f, &b, at(5), 10).await.is_empty()); + assert_eq!( + f.store() + .prune_before(&b.table_id, at(5), 10) + .await + .expect("prune"), + 1 + ); + assert_eq!(f.history_rows(&a.table_id).await, 1); +} + +/// Pruning keeps the state at and after the cutoff, including an item written +/// once before it and never changed. +pub async fn pruning_keeps_every_state_from_the_cutoff_on(f: &F) { + let info = f.table("t", None, true).await; + timeline(f, &info).await; + let old = s("old", "kept"); + commit(f, &info, &old, Some(&old), at(1)).await; + let mut before = Vec::new(); + for t in [30, 35, 40, 50] { + before.push(state(f, &info, at(t), 10).await); + } + // At +30 s: k1's v1 and v2 are superseded by its delete, and the delete is + // then k1's only row; k2's x1 and the old item are the state. + assert_eq!( + f.store() + .prune_before(&info.table_id, at(30), 100) + .await + .expect("prune"), + 3 + ); + assert_eq!( + f.store() + .prune_before(&info.table_id, at(30), 100) + .await + .expect("prune"), + 0 + ); + for (t, expected) in [30, 35, 40, 50].into_iter().zip(before) { + assert_eq!(state(f, &info, at(t), 10).await, expected, "at +{t}s"); + } + assert_eq!(f.history_rows(&info.table_id).await, 4); +} + +/// A prune one row at a time never brings a deleted item back. +pub async fn a_bounded_prune_never_brings_a_deleted_item_back(f: &F) { + let info = f.table("t", None, true).await; + let k = s("k", ""); + commit(f, &info, &k, Some(&s("k", "a")), at(1)).await; + commit(f, &info, &k, Some(&s("k", "b")), at(2)).await; + commit(f, &info, &k, None, at(3)).await; + let mut total = 0; + loop { + let removed = f + .store() + .prune_before(&info.table_id, at(10), 1) + .await + .expect("prune"); + assert!(removed <= 1); + assert!( + state(f, &info, at(10), 10).await.is_empty(), + "after {total} rows" + ); + if removed == 0 { + break; + } + total += removed; + } + assert_eq!(total, 3); + assert_eq!(f.history_rows(&info.table_id).await, 0); +} + +/// Rows committed after the cutoff are not pruned. +pub async fn rows_after_the_cutoff_are_not_pruned(f: &F) { + let info = f.table("t", None, true).await; + let k = s("k", ""); + commit(f, &info, &k, Some(&s("k", "a")), at(1)).await; + commit(f, &info, &k, None, at(20)).await; + assert_eq!( + f.store() + .prune_before(&info.table_id, at(10), 100) + .await + .expect("prune"), + 0 + ); + assert_eq!(state(f, &info, at(10), 10).await, sorted(&[s("k", "a")])); + assert!(state(f, &info, at(20), 10).await.is_empty()); +} + +/// One `#[tokio::test]` per check. `$open` is an async fn returning +/// `Option` (`None` skips, for a backend without a database to test on); +/// `$close` is an async fn taking the fixture once the check is done. +#[macro_export] +macro_rules! pitr_conformance_tests { + ($open:path, $close:path) => { + $crate::pitr_conformance_tests!(@each $open, $close, + records_only_while_recovery_is_enabled, + a_record_rolls_back_with_its_transaction, + reconstructs_the_timeline_at_each_point, + sub_second_commits_count_at_their_instant, + commits_at_one_instant_resolve_in_commit_order, + sort_keys_of_every_type_identify_items, + tables_do_not_see_each_others_history, + pruning_keeps_every_state_from_the_cutoff_on, + a_bounded_prune_never_brings_a_deleted_item_back, + rows_after_the_cutoff_are_not_pruned, + every_data_path_write_is_recorded, + writes_that_change_nothing_record_nothing, + a_transaction_records_its_members_at_one_instant, + a_cancelled_transaction_records_nothing, + writes_to_a_table_without_recovery_record_nothing, + the_state_now_is_the_live_table, + the_enable_takes_a_baseline_the_hook_supersedes, + an_enable_on_an_enabled_table_changes_only_a_given_period, + a_disable_drops_the_history_and_a_re_enable_starts_again, + the_sweep_prunes_to_the_retention_floor, + a_sweep_stops_at_its_deadline, + the_floor_is_the_enable_until_the_retention_passes, + a_raised_period_keeps_the_floor_at_the_pruned_cutoff, + a_prune_between_pages_keeps_the_state_of_the_reconstruct, + a_page_after_a_prune_passed_its_time_refuses, + a_page_after_a_disable_refuses, + two_enables_at_once_both_succeed, + a_stored_period_outside_1_to_35_is_an_error + ); + }; + (@each $open:path, $close:path, $($check:ident),+) => { + $( + #[tokio::test] + async fn $check() { + let Some(fixture) = $open().await else { return }; + $crate::pitr::conformance::$check(&fixture).await; + $close(fixture).await; + } + )+ + }; +} diff --git a/crates/storage/src/pitr/conformance/enable.rs b/crates/storage/src/pitr/conformance/enable.rs new file mode 100644 index 000000000..5f96e46dd --- /dev/null +++ b/crates/storage/src/pitr/conformance/enable.rs @@ -0,0 +1,236 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The checks of enabling and disabling recovery. Measured on the live +//! service: items that exist at the enable are in every restore after it, at +//! their image then; a disable drops the history, and a re-enable takes a new +//! baseline; an enable on an enabled table changes only a given period. + +use std::time::{Duration, SystemTime}; + +use extenddb_core::expression::ExpressionMaps; +use extenddb_core::types::{AttributeValue, Item, ScalarAttributeType, TableKeyInfo}; + +use super::super::{DEFAULT_RECOVERY_PERIOD_DAYS, PitrState}; +use super::{HistoryFixture, at, sorted, state}; + +async fn put(f: &F, info: &TableKeyInfo, item: &Item) { + f.data() + .put_item( + info, + item.clone(), + false, + None, + &ExpressionMaps::default(), + None, + ) + .await + .expect("put"); +} + +async fn enable( + f: &F, + info: &TableKeyInfo, + period: Option, + t: SystemTime, +) -> PitrState { + f.store() + .enable_pitr(info, period, t) + .await + .expect("enable") +} + +fn keyed(pk: &str, sk: Option, v: &str) -> Item { + let mut item = Item::from([ + ("pk".to_owned(), AttributeValue::S(pk.to_owned())), + ("v".to_owned(), AttributeValue::S(v.to_owned())), + ]); + if let Some(sk) = sk { + item.insert("sk".to_owned(), sk); + } + item +} + +/// Sort key values that stress the key text: numbers that differ in +/// spelling, scale (a trailing zero) and magnitude, and binary values longer +/// than a line of base64. +fn sort_keys(ty: ScalarAttributeType) -> Vec { + match ty { + ScalarAttributeType::S => ["a", "a b", "\u{e9}", "z"] + .iter() + .map(|s| AttributeValue::S((*s).to_owned())) + .collect(), + ScalarAttributeType::N => [ + "0", + "100", + "1.5", + "2.50", + "-0.25", + "1E-130", + "9.9999E+125", + "12345678901234567890.123", + ] + .iter() + .map(|s| AttributeValue::N((*s).to_owned())) + .collect(), + ScalarAttributeType::B => vec![ + AttributeValue::B(vec![0]), + AttributeValue::B(vec![0xff; 3]), + AttributeValue::B((0..=255).collect()), + ], + } +} + +/// Items that exist at the enable are the state at the enable, and a later +/// write of each one supersedes its baseline row: the baseline and the write +/// hook key an item the same way, for every key shape. +pub async fn the_enable_takes_a_baseline_the_hook_supersedes(f: &F) { + let shapes = [ + None, + Some(ScalarAttributeType::S), + Some(ScalarAttributeType::N), + Some(ScalarAttributeType::B), + ]; + for (n, ty) in shapes.into_iter().enumerate() { + let info = f.table(&format!("b{n}"), ty, false).await; + let sks: Vec> = + ty.map_or(vec![None], |t| sort_keys(t).into_iter().map(Some).collect()); + let mut before = Vec::new(); + for pk in ["p", "q"] { + for sk in &sks { + let item = keyed(pk, sk.clone(), "before"); + put(f, &info, &item).await; + before.push(item); + } + } + let enabled = SystemTime::now(); + enable(f, &info, None, enabled).await; + assert_eq!( + f.history_rows(&info.table_id).await, + i64::try_from(before.len()).expect("count"), + "{ty:?}: one baseline row per item" + ); + assert_eq!(state(f, &info, enabled, 5).await, sorted(&before), "{ty:?}"); + let mut after = Vec::new(); + for item in &before { + let mut changed = item.clone(); + changed.insert("v".to_owned(), AttributeValue::S("after".to_owned())); + put(f, &info, &changed).await; + after.push(changed); + } + let now = SystemTime::now(); + assert_eq!(state(f, &info, now, 5).await, sorted(&after), "{ty:?}"); + assert_eq!( + state(f, &info, now, 5).await, + sorted(&f.live_items(&info).await), + "{ty:?}" + ); + assert_eq!(state(f, &info, enabled, 5).await, sorted(&before), "{ty:?}"); + } +} + +/// A fresh enable takes the default period, or the given one; an enable on an +/// enabled table changes only a given period and takes no second baseline. +pub async fn an_enable_on_an_enabled_table_changes_only_a_given_period(f: &F) { + let info = f.table("t", None, false).await; + put(f, &info, &keyed("k", None, "a")).await; + let first = enable(f, &info, None, at(10)).await; + assert_eq!(first.recovery_period_days, DEFAULT_RECOVERY_PERIOD_DAYS); + assert_eq!(first.enabled_at, at(10)); + let changed = enable(f, &info, Some(20), at(20)).await; + assert_eq!( + (changed.recovery_period_days, changed.enabled_at), + (20, at(10)) + ); + let kept = enable(f, &info, None, at(30)).await; + assert_eq!((kept.recovery_period_days, kept.enabled_at), (20, at(10))); + assert_eq!( + f.store().pitr_state(&info.table_id).await.expect("state"), + Some(kept) + ); + assert_eq!(f.history_rows(&info.table_id).await, 1); + + let other = f.table("u", None, false).await; + let given = enable(f, &other, Some(7), at(10)).await; + assert_eq!(given.recovery_period_days, 7); +} + +/// A disable removes the state and the history; writes while disabled record +/// nothing; a re-enable is a fresh enable whose baseline holds them. +pub async fn a_disable_drops_the_history_and_a_re_enable_starts_again(f: &F) { + let info = f.table("t", None, false).await; + put(f, &info, &keyed("b1", None, "first")).await; + enable(f, &info, Some(7), SystemTime::now()).await; + put(f, &info, &keyed("b1", None, "first-window")).await; + f.store().disable_pitr(&info).await.expect("disable"); + assert_eq!( + f.store().pitr_state(&info.table_id).await.expect("state"), + None + ); + assert_eq!(f.history_rows(&info.table_id).await, 0); + put(f, &info, &keyed("b2", None, "while-disabled")).await; + assert_eq!(f.history_rows(&info.table_id).await, 0); + f.store() + .disable_pitr(&info) + .await + .expect("a second disable"); + let again = SystemTime::now() + Duration::from_millis(1); + let state_now = enable(f, &info, None, again).await; + assert_eq!(state_now.recovery_period_days, DEFAULT_RECOVERY_PERIOD_DAYS); + assert_eq!(state_now.enabled_at, again); + assert_eq!( + state(f, &info, again, 10).await, + sorted(&[ + keyed("b1", None, "first-window"), + keyed("b2", None, "while-disabled") + ]) + ); +} + +/// Two enables of one table at once both succeed, and the table holds one +/// baseline: measured on the live service, both concurrent requests answer +/// 200 and the table then reports the period of the later one. +pub async fn two_enables_at_once_both_succeed(f: &F) { + for round in 0..10 { + let info = f.table(&format!("t{round}"), None, false).await; + for n in 0..300 { + put(f, &info, &keyed(&format!("k{n}"), None, "v")).await; + } + let now = SystemTime::now(); + let (a, b) = futures::join!( + f.store().enable_pitr(&info, None, now), + f.store().enable_pitr(&info, Some(7), now) + ); + let (a, b) = (a.expect("first enable"), b.expect("second enable")); + assert!( + [a.recovery_period_days, b.recovery_period_days].contains(&7), + "round {round}: {a:?} {b:?}" + ); + assert_eq!(f.history_rows(&info.table_id).await, 300, "round {round}"); + let stored = f.store().pitr_state(&info.table_id).await.expect("state"); + assert_eq!( + stored.map(|s| s.recovery_period_days), + Some(7), + "round {round}" + ); + } +} + +/// A stored period outside 1 to 35 is a corrupt row: reading it is an +/// error, not some other period. +pub async fn a_stored_period_outside_1_to_35_is_an_error(f: &F) { + let info = f.table("t", None, true).await; + for days in [0, 36, 300] { + f.execute(&format!( + "UPDATE pitr_state SET recovery_period_days = {days} WHERE table_id = '{}'", + info.table_id + )) + .await; + let read = f.store().pitr_state(&info.table_id).await; + assert!( + matches!(read, Err(crate::error::StorageError::Internal(ref m)) if m.contains(&days.to_string())), + "{days}: {read:?}" + ); + assert!(f.store().pitr_tables().await.is_err(), "{days}"); + } +} diff --git a/crates/storage/src/pitr/conformance/pins.rs b/crates/storage/src/pitr/conformance/pins.rs new file mode 100644 index 000000000..b49cb626a --- /dev/null +++ b/crates/storage/src/pitr/conformance/pins.rs @@ -0,0 +1,109 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The checks of a reconstruct read in pages while a sweep runs: the pages +//! of one reconstruct return one state, and a page whose state the history +//! no longer holds refuses instead of returning part of it. + +use std::time::SystemTime; + +use extenddb_core::types::{Item, TableKeyInfo}; + +use super::super::ReconstructPage; +use super::{HistoryFixture, at, s, sorted, state}; +use crate::error::StorageError; + +async fn page( + f: &F, + info: &TableKeyInfo, + t: SystemTime, + after: Option, +) -> Result { + f.store() + .reconstruct_page(&info.table_id, t, after, 1) + .await +} + +/// `a` at +1 s, `b`=1 at +2 s, `b`=2 at +4 s: the state at +3 s is `a` and +/// `b`=1, and a prune to +4 s alone would remove `b`=1. +async fn history(f: &F) -> TableKeyInfo { + let info = f.table("t", None, true).await; + for (item, t) in [(s("a", "1"), 1), (s("b", "1"), 2), (s("b", "2"), 4)] { + assert!(f.record(&info, &item, Some(&item), at(t), true).await); + } + info +} + +fn items(pages: &[ReconstructPage]) -> Vec { + pages.iter().flat_map(|p| p.items.clone()).collect() +} + +/// A sweep between two pages of a reconstruct at T, to a cutoff past T, +/// leaves the rest of the state at T for the next page. +pub async fn a_prune_between_pages_keeps_the_state_of_the_reconstruct(f: &F) { + let info = history(f).await; + let first = page(f, &info, at(3), None).await.expect("page 1"); + assert_eq!(first.items.len(), 1); + f.store() + .prune_before(&info.table_id, at(4), 100) + .await + .expect("prune"); + let second = page(f, &info, at(3), first.next).await.expect("page 2"); + let mut pages = vec![first, second]; + while let Some(next) = pages.last().and_then(|p| p.next) { + pages.push(page(f, &info, at(3), Some(next)).await.expect("page")); + } + assert_eq!(sorted(&items(&pages)), sorted(&[s("a", "1"), s("b", "1")])); + let state_now = f + .store() + .pitr_state(&info.table_id) + .await + .expect("state") + .expect("enabled"); + assert!(state_now.pruned_to <= at(3), "{state_now:?}"); +} + +/// Once the pin has lapsed, a prune past T goes through, and the next page +/// of the reconstruct at T refuses: the history no longer holds its state. +pub async fn a_page_after_a_prune_passed_its_time_refuses(f: &F) { + let info = history(f).await; + let first = page(f, &info, at(3), None).await.expect("page 1"); + f.execute(&format!( + "UPDATE pitr_state SET pin_expires_us = 0 WHERE table_id = '{}'", + info.table_id + )) + .await; + assert_eq!( + f.store() + .prune_before(&info.table_id, at(4), 100) + .await + .expect("prune"), + 1 + ); + let state_now = f + .store() + .pitr_state(&info.table_id) + .await + .expect("state") + .expect("enabled"); + assert_eq!(state_now.pruned_to, at(4)); + match page(f, &info, at(3), first.next).await { + Err(StorageError::SnapshotExpired(_)) => {} + other => panic!("{other:?}"), + } + assert_eq!( + state(f, &info, at(4), 10).await, + sorted(&[s("a", "1"), s("b", "2")]) + ); +} + +/// A disable between two pages: the next page refuses. +pub async fn a_page_after_a_disable_refuses(f: &F) { + let info = history(f).await; + let first = page(f, &info, at(3), None).await.expect("page 1"); + f.store().disable_pitr(&info).await.expect("disable"); + match page(f, &info, at(3), first.next).await { + Err(StorageError::SnapshotExpired(_)) => {} + other => panic!("{other:?}"), + } +} diff --git a/crates/storage/src/pitr/conformance/retention.rs b/crates/storage/src/pitr/conformance/retention.rs new file mode 100644 index 000000000..f2766ebd8 --- /dev/null +++ b/crates/storage/src/pitr/conformance/retention.rs @@ -0,0 +1,158 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The checks of the retention sweep: every enabled table is pruned to the +//! floor of its window, `now` less the retention once that is past the +//! enable, and every state a restore inside the window reads stays. + +use std::time::{Duration, Instant}; + +use extenddb_core::types::{AttributeValue, Item}; + +use super::super::{retention_floor, sweep}; +use super::{HistoryFixture, at, sorted, state}; + +fn s(pk: &str, v: &str) -> Item { + Item::from([ + ("pk".to_owned(), AttributeValue::S(pk.to_owned())), + ("v".to_owned(), AttributeValue::S(v.to_owned())), + ]) +} + +const DAY: u64 = 86_400; + +/// A sweep two days after an enable with a one-day retention prunes what no +/// restore after the floor reads, and keeps every state from the floor on. +pub async fn the_sweep_prunes_to_the_retention_floor(f: &F) { + let info = f.table("t", None, false).await; + let other = f.table("u", None, false).await; + f.store() + .enable_pitr(&info, Some(1), at(0)) + .await + .expect("enable"); + let k = s("k", ""); + for (v, t) in [("a", 10), ("b", 20)] { + f.record(&info, &k, Some(&s("k", v)), at(t), true).await; + } + f.record(&info, &k, None, at(30), true).await; + f.record(&info, &s("j", ""), Some(&s("j", "x")), at(40), true) + .await; + f.record(&info, &k, Some(&s("k", "c")), at(DAY + 100), true) + .await; + let tables = f.store().pitr_tables().await.expect("tables"); + assert_eq!( + tables.iter().map(|(id, _)| id.as_str()).collect::>(), + vec![info.table_id.as_str()] + ); + let floor = at(DAY); + let before = ( + state(f, &info, floor, 10).await, + state(f, &info, at(DAY + 100), 10).await, + ); + let report = sweep( + f.store(), + at(2 * DAY), + 2, + Instant::now() + Duration::from_secs(60), + ) + .await + .expect("sweep"); + assert_eq!((report.pruned, report.stopped_at_deadline), (3, false)); + assert_eq!(state(f, &info, floor, 10).await, before.0); + assert_eq!(state(f, &info, at(DAY + 100), 10).await, before.1); + assert_eq!(before.0, sorted(&[s("j", "x")])); + assert_eq!(f.history_rows(&info.table_id).await, 2); + assert_eq!(f.history_rows(&other.table_id).await, 0); +} + +/// A sweep past its deadline stops before it removes anything and says so. +pub async fn a_sweep_stops_at_its_deadline(f: &F) { + let info = f.table("t", None, false).await; + f.store() + .enable_pitr(&info, Some(1), at(0)) + .await + .expect("enable"); + let k = s("k", ""); + f.record(&info, &k, Some(&s("k", "a")), at(10), true).await; + f.record(&info, &k, Some(&s("k", "b")), at(20), true).await; + let report = sweep(f.store(), at(2 * DAY), 10, Instant::now()) + .await + .expect("sweep"); + assert!(report.stopped_at_deadline); + assert_eq!(report.pruned, 0); + assert_eq!(f.history_rows(&info.table_id).await, 2); +} + +/// The floor is the enable instant until the retention has passed. +pub async fn the_floor_is_the_enable_until_the_retention_passes(f: &F) { + let info = f.table("t", None, false).await; + f.store() + .enable_pitr(&info, Some(1), at(DAY)) + .await + .expect("enable"); + let k = s("k", ""); + f.record(&info, &k, Some(&s("k", "a")), at(DAY + 10), true) + .await; + f.record(&info, &k, Some(&s("k", "b")), at(DAY + 20), true) + .await; + let state_now = f + .store() + .pitr_state(&info.table_id) + .await + .expect("state") + .expect("enabled"); + assert_eq!(retention_floor(&state_now, at(DAY + 3_600)), at(DAY)); + assert_eq!(retention_floor(&state_now, at(3 * DAY)), at(2 * DAY)); + let report = sweep( + f.store(), + at(DAY + 3_600), + 10, + Instant::now() + Duration::from_secs(60), + ) + .await + .expect("sweep"); + assert_eq!(report.pruned, 0); + assert_eq!(f.history_rows(&info.table_id).await, 2); +} + +/// A period raised after a sweep keeps the floor at the sweep's cutoff: the +/// history no longer holds the states before it, so the window never +/// reaches them. +pub async fn a_raised_period_keeps_the_floor_at_the_pruned_cutoff(f: &F) { + let info = f.table("t", None, false).await; + f.store() + .enable_pitr(&info, Some(1), at(0)) + .await + .expect("enable"); + let k = s("k", ""); + for (v, t) in [("v1", DAY), ("v2", 5 * DAY), ("v3", 9 * DAY + DAY / 2)] { + f.record(&info, &k, Some(&s("k", v)), at(t), true).await; + } + let before = state(f, &info, at(9 * DAY), 10).await; + let report = sweep( + f.store(), + at(10 * DAY), + 10, + Instant::now() + Duration::from_secs(60), + ) + .await + .expect("sweep"); + assert_eq!(report.pruned, 1); + let raised = f + .store() + .enable_pitr(&info, Some(35), at(10 * DAY)) + .await + .expect("raise the period"); + assert_eq!(raised.recovery_period_days, 35); + let state_now = f + .store() + .pitr_state(&info.table_id) + .await + .expect("state") + .expect("enabled"); + assert_eq!(state_now.pruned_to, at(9 * DAY)); + let floor = retention_floor(&state_now, at(10 * DAY)); + assert_eq!(floor, at(9 * DAY)); + assert_eq!(state(f, &info, floor, 10).await, before); + assert_eq!(before, sorted(&[s("k", "v2")])); +} diff --git a/crates/storage/src/pitr/conformance/writes.rs b/crates/storage/src/pitr/conformance/writes.rs new file mode 100644 index 000000000..b0fba76d2 --- /dev/null +++ b/crates/storage/src/pitr/conformance/writes.rs @@ -0,0 +1,262 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The checks that drive a backend's data paths and read what the write hook +//! recorded. + +use std::time::SystemTime; + +use extenddb_core::expression::{Expr, ExpressionMaps, PathElement, UpdateAction}; +use extenddb_core::types::{ + AttributeValue, Item, ReturnValuesOnConditionCheckFailure, ScalarAttributeType, TableKeyInfo, +}; + +use super::{HistoryFixture, s, sorted, state}; +use crate::error::StorageError; +use crate::transact::TransactWriteOp; + +/// A key-only item. +fn k(pk: &str) -> Item { + Item::from([("pk".to_owned(), AttributeValue::S(pk.to_owned()))]) +} + +fn path(name: &str) -> Vec { + vec![PathElement::Attribute(name.to_owned())] +} + +fn function(name: &str, attr: &str) -> Expr { + Expr::Function { + name: name.to_owned(), + args: vec![Expr::Path(path(attr))], + } +} + +fn set_v(value: &str) -> (Vec, ExpressionMaps) { + let mut maps = ExpressionMaps::default(); + maps.values + .insert(":v".to_owned(), AttributeValue::S(value.to_owned())); + ( + vec![UpdateAction::Set { + path: path("v"), + value: Expr::Placeholder(":v".to_owned()), + }], + maps, + ) +} + +async fn put( + f: &F, + info: &TableKeyInfo, + item: &Item, + condition: Option<&Expr>, +) -> Result<(), StorageError> { + let maps = ExpressionMaps::default(); + f.data() + .put_item(info, item.clone(), false, condition, &maps, None) + .await + .map(|_| ()) +} + +async fn delete(f: &F, info: &TableKeyInfo, key: &Item) { + let maps = ExpressionMaps::default(); + f.data() + .delete_item(info, key, false, None, &maps, None) + .await + .expect("delete"); +} + +async fn update( + f: &F, + info: &TableKeyInfo, + key: &Item, + actions: &[UpdateAction], + maps: &ExpressionMaps, +) { + f.data() + .update_item(info, key, actions, false, false, None, maps, None) + .await + .expect("update"); +} + +fn now() -> SystemTime { + SystemTime::now() +} + +/// PutItem, UpdateItem and DeleteItem each add one row, which reconstructs +/// the item as the write left it. +pub async fn every_data_path_write_is_recorded(f: &F) { + let info = f.table("t", None, true).await; + let key = k("k"); + put(f, &info, &s("k", "a"), None).await.expect("put"); + assert_eq!(state(f, &info, now(), 10).await, sorted(&[s("k", "a")])); + let (actions, maps) = set_v("b"); + update(f, &info, &key, &actions, &maps).await; + assert_eq!(state(f, &info, now(), 10).await, sorted(&[s("k", "b")])); + delete(f, &info, &key).await; + assert!(state(f, &info, now(), 10).await.is_empty()); + assert_eq!(f.history_rows(&info.table_id).await, 3); +} + +/// A failed condition, a delete of a missing item and a REMOVE-only update +/// of a missing item change nothing and record nothing. +pub async fn writes_that_change_nothing_record_nothing(f: &F) { + let info = f.table("t", None, true).await; + let exists = function("attribute_exists", "pk"); + let err = put(f, &info, &s("k", "a"), Some(&exists)).await; + assert!( + matches!(err, Err(StorageError::ConditionFailed(_))), + "{err:?}" + ); + delete(f, &info, &k("missing")).await; + let remove = [UpdateAction::Remove { path: path("v") }]; + update(f, &info, &k("missing"), &remove, &ExpressionMaps::default()).await; + assert_eq!(f.history_rows(&info.table_id).await, 0); +} + +fn ops<'a>( + a: &'a TableKeyInfo, + b: &'a TableKeyInfo, + items: &'a [Item; 4], + actions: &'a [UpdateAction], + maps: &'a ExpressionMaps, + check: &'a Expr, +) -> Vec> { + let none = ReturnValuesOnConditionCheckFailure::None; + vec![ + TransactWriteOp::Put { + key_info: a, + item: &items[0], + condition: None, + maps, + return_values_on_ccf: none, + stream: None, + }, + TransactWriteOp::Delete { + key_info: a, + key: &items[1], + condition: None, + maps, + return_values_on_ccf: none, + stream: None, + }, + TransactWriteOp::Update { + key_info: b, + key: &items[2], + actions, + condition: None, + maps, + return_values_on_ccf: none, + stream: None, + }, + TransactWriteOp::ConditionCheck { + key_info: b, + key: &items[3], + condition: check, + maps, + return_values_on_ccf: none, + }, + ] +} + +/// TransactWriteItems over two tables records each changed member with one +/// commit time, and the condition check records nothing. +pub async fn a_transaction_records_its_members_at_one_instant(f: &F) { + let a = f.table("a", None, true).await; + let b = f.table("b", None, true).await; + put(f, &a, &s("gone", "x"), None).await.expect("put"); + put(f, &b, &s("checked", "x"), None).await.expect("put"); + let (actions, maps) = set_v("u"); + let exists = function("attribute_exists", "pk"); + let items = [s("new", "n"), k("gone"), k("upd"), k("checked")]; + f.data() + .transact_write_items(&ops(&a, &b, &items, &actions, &maps, &exists), None) + .await + .expect("transact"); + let times_a = f.commit_times(&a.table_id).await; + let times_b = f.commit_times(&b.table_id).await; + assert_eq!( + times_a.len(), + 3, + "a: the earlier put, then the put and the delete" + ); + assert_eq!(times_b.len(), 2, "b: the earlier put, then the update"); + assert_eq!(times_a[1], times_a[2]); + assert_eq!(times_a[1], times_b[1]); + assert_eq!(state(f, &a, now(), 10).await, sorted(&[s("new", "n")])); + assert_eq!( + state(f, &b, now(), 10).await, + sorted(&[s("checked", "x"), s("upd", "u")]) + ); +} + +/// A transaction cancelled by a failed condition check records nothing. +pub async fn a_cancelled_transaction_records_nothing(f: &F) { + let a = f.table("a", None, true).await; + let b = f.table("b", None, true).await; + let (actions, maps) = set_v("u"); + let exists = function("attribute_exists", "pk"); + let items = [s("new", "n"), k("gone"), k("upd"), k("checked")]; + let err = f + .data() + .transact_write_items(&ops(&a, &b, &items, &actions, &maps, &exists), None) + .await; + assert!( + matches!(err, Err(StorageError::TransactionCanceled(_))), + "{err:?}" + ); + assert_eq!(f.history_rows(&a.table_id).await, 0); + assert_eq!(f.history_rows(&b.table_id).await, 0); +} + +/// Writes to a table whose recovery is not enabled record nothing. +pub async fn writes_to_a_table_without_recovery_record_nothing(f: &F) { + let info = f.table("t", None, false).await; + put(f, &info, &s("k", "a"), None).await.expect("put"); + delete(f, &info, &k("k")).await; + assert_eq!(f.history_rows(&info.table_id).await, 0); +} + +/// After a mix of writes over every key shape, the state now is exactly what +/// the data table holds. +pub async fn the_state_now_is_the_live_table(f: &F) { + type SortKey = fn(u32) -> AttributeValue; + let shapes: [(Option, Option); 4] = [ + (None, None), + ( + Some(ScalarAttributeType::S), + Some(|i| AttributeValue::S(format!("s{}", i % 3))), + ), + ( + Some(ScalarAttributeType::N), + Some(|i| AttributeValue::N(format!("{}.0", i % 3))), + ), + ( + Some(ScalarAttributeType::B), + Some(|i| AttributeValue::B(vec![0, (i % 3) as u8])), + ), + ]; + for (n, (ty, sk)) in shapes.into_iter().enumerate() { + let info = f.table(&format!("t{n}"), ty, true).await; + for i in 0..40_u32 { + let mut item = s(&format!("p{}", i % 4), &format!("v{i}")); + if let Some(sk) = sk { + item.insert("sk".to_owned(), sk(i / 4)); + } + let mut key = item.clone(); + key.remove("v"); + match i % 5 { + 0..=2 => put(f, &info, &item, None).await.expect("put"), + 3 => { + let (actions, maps) = set_v(&format!("u{i}")); + update(f, &info, &key, &actions, &maps).await; + } + _ => delete(f, &info, &key).await, + } + } + assert_eq!( + state(f, &info, now(), 7).await, + sorted(&f.live_items(&info).await), + "{ty:?}" + ); + } +} diff --git a/crates/storage/src/pitr/worker_tests.rs b/crates/storage/src/pitr/worker_tests.rs new file mode 100644 index 000000000..5b21562de --- /dev/null +++ b/crates/storage/src/pitr/worker_tests.rs @@ -0,0 +1,145 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The retention worker: its first sweep an hour after start, and the +//! `PitrRetention` worker metric of each sweep. + +use std::sync::Arc; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::time::{Duration, SystemTime}; + +use extenddb_core::metrics::{Dimension, MetricName, MetricsCollector, MetricsQuery, QuerySource}; +use extenddb_core::types::TableKeyInfo; +use futures::future::BoxFuture; + +use super::{PitrCaptureStore, PitrState, ReconstructPage, retention_worker}; +use crate::error::StorageError; +use crate::hooks::CancellationToken; + +/// A store with one enabled table whose prune removes 3 rows, or fails when +/// `fail` is set, counting the sweeps (`pitr_tables` calls). +struct Store { + sweeps: AtomicU32, + fail: bool, +} + +impl PitrCaptureStore for Store { + fn pitr_state<'a>( + &'a self, + _: &'a str, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async { Ok(None) }) + } + + fn enable_pitr<'a>( + &'a self, + _: &'a TableKeyInfo, + _: Option, + _: SystemTime, + ) -> BoxFuture<'a, Result> { + unreachable!("the worker never enables") + } + + fn pitr_tables(&self) -> BoxFuture<'_, Result, StorageError>> { + self.sweeps.fetch_add(1, Ordering::SeqCst); + Box::pin(async { + Ok(vec![( + "t".to_owned(), + PitrState::new(1, SystemTime::UNIX_EPOCH), + )]) + }) + } + + fn remove_orphaned_history(&self, _: u32) -> BoxFuture<'_, Result> { + Box::pin(async { Ok(0) }) + } + + fn disable_pitr<'a>(&'a self, _: &'a TableKeyInfo) -> BoxFuture<'a, Result<(), StorageError>> { + unreachable!("the worker never disables") + } + + fn reconstruct_page<'a>( + &'a self, + _: &'a str, + _: SystemTime, + _: Option, + _: u32, + ) -> BoxFuture<'a, Result> { + unreachable!("the worker never reconstructs") + } + + fn prune_before<'a>( + &'a self, + _: &'a str, + _: SystemTime, + _: u32, + ) -> BoxFuture<'a, Result> { + let fail = self.fail; + Box::pin(async move { + if fail { + Err(StorageError::Internal("prune failed".to_owned())) + } else { + Ok(3) + } + }) + } +} + +/// The worker metric `metric` recorded under `PitrRetention`, as its count. +fn recorded(metrics: &MetricsCollector, metric: MetricName) -> u64 { + metrics + .query(&MetricsQuery { + metric: Some(metric), + ..Default::default() + }) + .iter() + .filter(|m| { + m.dimensions.contains(&Dimension::Operation( + QuerySource::PitrRetention.to_string(), + )) + }) + .map(|m| m.count) + .sum() +} + +async fn run(fail: bool) -> (Arc, Arc, Vec) { + let store = Arc::new(Store { + sweeps: AtomicU32::new(0), + fail, + }); + let metrics = Arc::new(MetricsCollector::new()); + let token = CancellationToken::new(); + let worker = tokio::spawn(retention_worker( + store.clone(), + metrics.clone(), + token.clone(), + )); + let mut seen = Vec::new(); + for wait in [3_599, 2, 3_600] { + tokio::time::sleep(Duration::from_secs(wait)).await; + seen.push(store.sweeps.load(Ordering::SeqCst)); + } + token.cancel(); + worker.await.expect("the worker stops at shutdown"); + (store, metrics, seen) +} + +/// The first sweep runs an hour after start and one more each hour; each +/// success records `WorkerLastSuccess` for `PitrRetention`. +#[tokio::test(start_paused = true)] +async fn the_worker_sweeps_hourly_and_records_each_success() { + let (_, metrics, seen) = run(false).await; + assert_eq!(seen, vec![0, 1, 2]); + assert_eq!(recorded(&metrics, MetricName::WorkerLastSuccess), 2); + assert_eq!(recorded(&metrics, MetricName::WorkerErrorCount), 0); +} + +/// A sweep that fails records `WorkerErrorCount` for `PitrRetention`, and +/// the worker keeps running. +#[tokio::test(start_paused = true)] +async fn a_failed_sweep_records_a_worker_error() { + let (_, metrics, seen) = run(true).await; + assert_eq!(seen, vec![0, 1, 2]); + assert_eq!(recorded(&metrics, MetricName::WorkerErrorCount), 2); + assert_eq!(recorded(&metrics, MetricName::WorkerLastSuccess), 0); +} From 080c59693049f671653b46d327b71e2ae766e12a Mon Sep 17 00:00:00 2001 From: Elise Ong Date: Tue, 10 Nov 2026 12:25:22 -0500 Subject: [PATCH 3/6] feat(engine): enable, disable and describe point-in-time recovery On the service, UpdateContinuousBackups answers ENABLED at once. RecoveryPeriodInDays is 1 to 35, checked before the table on an enable and on a disable ("Member must have value greater than or equal to 1", "less than or equal to 35"), and a disable that gives one is refused ("Invalid Request: Cannot specify RecoveryPeriodInDays when disabling point-in-time recovery."). The description carries RecoveryPeriodInDays and an undocumented PointInTimeRecoveryWindowDays equal to it; right after an enable EarliestRestorableDateTime and LatestRestorableDateTime are both the enable instant in whole seconds, and LatestRestorableDateTime stays there for the first minutes and then runs about five minutes behind the request. A disable answers DISABLED with no window. RecoveryPeriodInDays is read as an Integer: a fraction is truncated toward zero, a value outside 32 bits is clamped to it in the message ('2147483647', '-2147483648'), another JSON type is a SerializationException ("STRING_VALUE cannot be converted to Integer", "TRUE_VALUE cannot be converted to Integer"), checked before the TableName constraints; a bad period and a missing PointInTimeRecoveryEnabled are one message, "2 validation errors detected", the period first. An enable of a table being created is TableNotFoundException, and of a restore target being loaded ContinuousBackupsUnavailableException "Backups are being enabled for the table: . Please retry later" (two samples of each). ExtendDB refused every enable with ContinuousBackupsUnavailableException. With the engine's new pitr feature and a backend that has an item-history store, DescribeContinuousBackups and UpdateContinuousBackups read and write the store as the service answers: the period read and checked before the table (its type before the TableName check, through check_member_types, which the server calls first), the CREATING refusals, the fresh enable at 35 days, the period changed on an enabled table only when given, the window from the enable second (or now less the period once that is later) to now less 300 s and never before it, and the two period members. RestoreTableToPointInTime answers as before. Without the feature, or on MongoDB and Cassandra, nothing changes. The server, app and binary crates gain a pitr feature that turns on the engine's and the selected backend's. tests/test_pitr_enable.py runs against DynamoDB, and against a server built with pitr when EXTENDDB_TEST_PITR is set; test_continuous_backups.py skips there only the cells whose answer the feature changes. Verified: the two files pass on DynamoDB (42 pass), on a SQLite pitr build (54 pass, 1 skip) and on a PostgreSQL pitr build, and test_continuous_backups.py passes on a build without pitr (13 pass); engine unit tests; negative controls on the window-days member, the floor, the period's reading and check order, the disable refusal and the CREATING refusals, each run against a server or the engine. --- crates/app/Cargo.toml | 1 + crates/bin/Cargo.toml | 8 + crates/core/src/types/backup.rs | 7 + crates/engine/Cargo.toml | 6 + crates/engine/src/backup.rs | 1 + crates/engine/src/backup/pitr.rs | 195 ++++++-- crates/engine/src/backup/pitr/capture.rs | 526 ++++++++++++++++++++++ crates/engine/src/lib.rs | 3 +- crates/server/Cargo.toml | 2 + crates/server/src/request_helpers.rs | 3 + tests/test_continuous_backups.py | 11 +- tests/test_pitr_enable.py | 537 +++++++++++++++++++++++ 12 files changed, 1256 insertions(+), 44 deletions(-) create mode 100644 crates/engine/src/backup/pitr/capture.rs create mode 100644 tests/test_pitr_enable.py diff --git a/crates/app/Cargo.toml b/crates/app/Cargo.toml index fef29af1e..46836eca2 100644 --- a/crates/app/Cargo.toml +++ b/crates/app/Cargo.toml @@ -14,6 +14,7 @@ license.workspace = true dev-mode = ["extenddb-config/dev-mode"] mongodb-test-hooks = ["extenddb-server/mongodb-test-hooks"] backup-test-hooks = ["extenddb-server/backup-test-hooks"] +pitr = ["extenddb-server/pitr"] [dependencies] extenddb-auth = { workspace = true } diff --git a/crates/bin/Cargo.toml b/crates/bin/Cargo.toml index df58be389..d6db28f9e 100755 --- a/crates/bin/Cargo.toml +++ b/crates/bin/Cargo.toml @@ -30,6 +30,14 @@ mongodb-test-hooks = [ # requires a dev backend (`sqlite`/`sqlite-memory`) and fails to build with a # production backend like `postgres` (enforced at compile time in `main.rs`). dev-mode = ["extenddb-app/dev-mode"] +# Point-in-time recovery, off by default: the engine's operations and the +# item-history store of the backend this build selects. A backend without a +# store (Cassandra, MongoDB) keeps the unavailable answer. +pitr = [ + "extenddb-app/pitr", + "extenddb-storage-sqlite?/pitr", + "extenddb-storage-postgres?/pitr", +] [dependencies] extenddb-app = { workspace = true } diff --git a/crates/core/src/types/backup.rs b/crates/core/src/types/backup.rs index 583588227..c44cbf546 100755 --- a/crates/core/src/types/backup.rs +++ b/crates/core/src/types/backup.rs @@ -177,6 +177,13 @@ pub struct PointInTimeRecoveryDescription { pub earliest_restorable_date_time: Option, #[serde(skip_serializing_if = "Option::is_none")] pub latest_restorable_date_time: Option, + /// The retention in days, on a table whose recovery is enabled. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub recovery_period_in_days: Option, + /// The service sends this member, undocumented, beside + /// `RecoveryPeriodInDays` and equal to it (measured). + #[serde(skip_serializing_if = "Option::is_none", default)] + pub point_in_time_recovery_window_days: Option, } #[cfg(test)] diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index bc33e7ff3..735375a06 100755 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -17,6 +17,12 @@ backup-test-hooks = [] # off, the three operations are not known and the server answers them with # UnknownOperationException exactly as before. partiql = ["extenddb-core/partiql"] +# Point-in-time recovery capture and restore, off by default and incomplete. +# With this off, the continuous-backup operations answer as they do now: +# DescribeContinuousBackups reports recovery disabled, UpdateContinuousBackups +# refuses an enable, and RestoreTableToPointInTime answers unavailable. The +# item-history capture store and the restore reconstruct are built behind this gate. +pitr = ["extenddb-storage/pitr"] [dependencies] extenddb-auth = { workspace = true } diff --git a/crates/engine/src/backup.rs b/crates/engine/src/backup.rs index 213d3be53..f43dd7fa9 100755 --- a/crates/engine/src/backup.rs +++ b/crates/engine/src/backup.rs @@ -2024,6 +2024,7 @@ pub mod sync; #[cfg(test)] mod tests; +pub use pitr::check_member_types; pub(crate) use pitr::{ handle_describe_continuous_backups, handle_restore_table_to_point_in_time, handle_update_continuous_backups, diff --git a/crates/engine/src/backup/pitr.rs b/crates/engine/src/backup/pitr.rs index cf8c45de0..5859e686c 100644 --- a/crates/engine/src/backup/pitr.rs +++ b/crates/engine/src/backup/pitr.rs @@ -4,8 +4,12 @@ //! `DescribeContinuousBackups`, `UpdateContinuousBackups` and //! `RestoreTableToPointInTime`. //! -//! No storage backend implements point-in-time recovery, so the engine answers -//! the three operations itself and consults storage only to look the table up. +//! Without the `pitr` feature, or on a backend without an item-history store, +//! the engine answers the three operations itself and consults storage only +//! to look the table up. With both, DescribeContinuousBackups and +//! UpdateContinuousBackups read and write the store (`capture`), and +//! RestoreTableToPointInTime still answers as below: it refuses every +//! restore, including one from a table whose recovery is enabled. //! The checks run in the service's order (measured on DynamoDB, two samples of //! each request): //! @@ -14,6 +18,9 @@ //! - UpdateContinuousBackups: the `TableName` member and its constraints, //! then `PointInTimeRecoverySpecification` and its //! `PointInTimeRecoveryEnabled`, then the table, then the enable request. +//! With an item-history store, the JSON type of `RecoveryPeriodInDays` +//! comes first of all, and its bounds are reported with a missing +//! `PointInTimeRecoveryEnabled` in one message (`capture`). //! - RestoreTableToPointInTime: the source member (`SourceTableName` or //! `SourceTableArn`) and `TargetTableName`, then the source ARN's kind or //! the `SourceTableName` constraints, then the source table, then the time @@ -23,11 +30,13 @@ //! //! A table that does not exist is `TableNotFoundException` with the service's //! text, `Table not found: `, on all three. A table in any status -//! resolves, CREATING and DELETING included. +//! resolves, CREATING and DELETING included, except that an enable with an +//! item-history store refuses a CREATING table. use extenddb_core::error::DynamoDbError; use extenddb_core::types::{ ContinuousBackupsDescription, DescribeTableInput, PointInTimeRecoveryDescription, + TableDescription, }; use extenddb_storage::error::StorageError; use serde_json::{Value, json}; @@ -137,9 +146,10 @@ fn pitr_unsupported_error() -> DynamoDbError { /// The typed refusal for `RestoreTableToPointInTime`. /// /// The service models `PointInTimeRecoveryUnavailableException` on this -/// operation and returns it whenever recovery is not enabled on the source -/// table, which for ExtendDB is always. The message is the service's -/// ("Point in time recovery is not enabled for table ''"). +/// operation and returns it when recovery is not enabled on the source table, +/// with this message ("Point in time recovery is not enabled for table +/// ''"). ExtendDB returns it for every restore, so with an item-history +/// store the message is also returned for a table whose recovery is enabled. fn pitr_restore_unavailable_error(table_name: &str) -> DynamoDbError { DynamoDbError::PointInTimeRecoveryUnavailableException(format!( "Point in time recovery is not enabled for table '{table_name}'" @@ -149,12 +159,10 @@ fn pitr_restore_unavailable_error(table_name: &str) -> DynamoDbError { /// The continuous-backups description every table reports. /// /// `ContinuousBackupsStatus` is always `ENABLED`, matching the service, which -/// reports `ENABLED` unconditionally. `PointInTimeRecoveryStatus` is always -/// `DISABLED` because no storage backend implements point-in-time recovery, -/// and the restorable-time fields are omitted, as the service omits them for -/// a table whose recovery is disabled. Owned by the engine rather than the -/// backends: the answer does not depend on stored state, so no backend is -/// consulted and all of them report identically. +/// reports `ENABLED` unconditionally. `PointInTimeRecoveryStatus` is +/// `DISABLED` and the restorable-time fields are omitted, as the service omits +/// them for a table whose recovery is disabled. The answer of every table +/// where no item-history store is available. fn default_continuous_backups_description() -> ContinuousBackupsDescription { ContinuousBackupsDescription { continuous_backups_status: "ENABLED".to_owned(), @@ -162,6 +170,8 @@ fn default_continuous_backups_description() -> ContinuousBackupsDescription { point_in_time_recovery_status: "DISABLED".to_owned(), earliest_restorable_date_time: None, latest_restorable_date_time: None, + recovery_period_in_days: None, + point_in_time_recovery_window_days: None, }), } } @@ -175,7 +185,7 @@ fn default_continuous_backups_description() -> ContinuousBackupsDescription { async fn require_table_exists( ctx: &OperationContext, table_name: &str, -) -> Result<(), DynamoDbError> { +) -> Result { match ctx .storage .describe_table( @@ -186,7 +196,7 @@ async fn require_table_exists( ) .await { - Ok(_) => Ok(()), + Ok(table) => Ok(table), Err(StorageError::TableNotFound(_)) => Err(DynamoDbError::TableNotFoundException(format!( "Table not found: {table_name}" ))), @@ -194,25 +204,38 @@ async fn require_table_exists( } } -/// `PointInTimeRecoverySpecification.PointInTimeRecoveryEnabled`, which the -/// model requires: a request without the specification, or with one that -/// does not hold the flag, is refused with the service's text for the -/// missing member. The JSON strings `"true"` and `"false"` are taken as the -/// booleans (the service answers 200 for `"false"`, measured); any other -/// value that is not a boolean is refused as a missing flag. -fn point_in_time_recovery_enabled(body: &Value) -> Result { +/// The model path of `PointInTimeRecoveryEnabled`. +const FLAG_PATH: &str = "pointInTimeRecoverySpecification.pointInTimeRecoveryEnabled"; + +/// The `Value null ... Member must not be null` constraint text of a missing +/// `PointInTimeRecoveryEnabled`, without the count prefix. +#[cfg(feature = "pitr")] +fn flag_null_constraint() -> String { + format!("Value null at '{FLAG_PATH}' failed to satisfy constraint: Member must not be null") +} + +/// `PointInTimeRecoverySpecification.PointInTimeRecoveryEnabled`: `None` +/// when the specification does not hold the flag, and the service's +/// missing-member refusal when there is no specification. The JSON strings +/// `"true"` and `"false"` are taken as the booleans (the service answers 200 +/// for `"false"`, measured); any other value that is not a boolean is taken +/// as a missing flag. +fn recovery_flag(body: &Value) -> Result, DynamoDbError> { let spec = body .get("PointInTimeRecoverySpecification") .filter(|v| !v.is_null()) .ok_or_else(|| member_null("pointInTimeRecoverySpecification"))?; - match spec.get("PointInTimeRecoveryEnabled") { - Some(Value::Bool(flag)) => Ok(*flag), - Some(Value::String(s)) if s == "true" => Ok(true), - Some(Value::String(s)) if s == "false" => Ok(false), - _ => Err(member_null( - "pointInTimeRecoverySpecification.pointInTimeRecoveryEnabled", - )), - } + Ok(match spec.get("PointInTimeRecoveryEnabled") { + Some(Value::Bool(flag)) => Some(*flag), + Some(Value::String(s)) if s == "true" => Some(true), + Some(Value::String(s)) if s == "false" => Some(false), + _ => None, + }) +} + +/// [`recovery_flag`], with a missing flag refused with the service's text. +fn point_in_time_recovery_enabled(body: &Value) -> Result { + recovery_flag(body)?.ok_or_else(|| member_null(FLAG_PATH)) } /// The restore's source table name, from `SourceTableName`, checked against @@ -261,15 +284,22 @@ fn check_restore_time_members(body: &Value) -> Result<(), DynamoDbError> { /// Handle `DescribeContinuousBackups`. /// /// Continuous backups report `ENABLED` (the service reports this -/// unconditionally) with `PointInTimeRecoveryStatus: DISABLED` and no -/// restorable-time window. +/// unconditionally). Without an item-history store, +/// `PointInTimeRecoveryStatus` is `DISABLED` with no restorable-time window; +/// with one, the table's recovery state and window (`capture::describe`). pub(crate) async fn handle_describe_continuous_backups( body: Value, ctx: &OperationContext, ) -> Result { let table_name = table_name_member(&body, "TableName")?; check_table_name(table_name, super::table_name_bounds(&ctx.limits))?; - require_table_exists(ctx, table_name).await?; + let table = require_table_exists(ctx, table_name).await?; + #[cfg(feature = "pitr")] + if let Some(store) = ctx.storage.as_pitr() { + let description = capture::describe(store, &table).await?; + return serialize_output(&json!({ "ContinuousBackupsDescription": description })); + } + let _ = table; serialize_output( &json!({ "ContinuousBackupsDescription": default_continuous_backups_description() }), ) @@ -277,16 +307,20 @@ pub(crate) async fn handle_describe_continuous_backups( /// Handle `UpdateContinuousBackups`. /// -/// Enabling point-in-time recovery is refused with -/// `ContinuousBackupsUnavailableException`, because no storage backend -/// implements it and reporting it as enabled would promise a restore that can -/// never happen. Disabling it returns the description -/// `DescribeContinuousBackups` reports. The specification is checked before -/// the table, as the service checks it. +/// Where no item-history store is available, enabling point-in-time recovery +/// is refused with `ContinuousBackupsUnavailableException`: reporting it as +/// enabled would promise a restore that cannot happen. Disabling it returns +/// the description `DescribeContinuousBackups` reports. With a store, the +/// request enables or disables recovery (`capture::update`). The +/// specification is checked before the table, as the service checks it. pub(crate) async fn handle_update_continuous_backups( body: Value, ctx: &OperationContext, ) -> Result { + #[cfg(feature = "pitr")] + if let Some(store) = ctx.storage.as_pitr() { + return update_with_history(store, &body, ctx).await; + } let table_name = table_name_member(&body, "TableName")?; check_table_name(table_name, super::table_name_bounds(&ctx.limits))?; let pitr_enabled = point_in_time_recovery_enabled(&body)?; @@ -299,15 +333,89 @@ pub(crate) async fn handle_update_continuous_backups( ) } +/// The JSON-type checks the service makes before every member constraint of +/// `operation`, `TableName`'s included: on `UpdateContinuousBackups` with an +/// item-history store, the type of `RecoveryPeriodInDays` (measured: a +/// TableName the pattern refuses with a string period is the period's +/// `SerializationException`). The server runs this before its table name +/// check (`authorize_request`); the handler makes the same check. +/// +/// # Errors +/// +/// The service's `SerializationException` for the member. +pub fn check_member_types( + operation: &str, + body: &Value, + storage: &dyn extenddb_storage::StorageEngine, +) -> Result<(), DynamoDbError> { + #[cfg(feature = "pitr")] + if operation == "UpdateContinuousBackups" && storage.as_pitr().is_some() { + capture::period_member(body)?; + } + let _ = (operation, body, storage); + Ok(()) +} + +/// The service's answer to an enable of a CREATING table (measured: two +/// tables right after CreateTable, two requests each, and two +/// RestoreTableFromBackup targets polled every 4.5 s). The first seconds +/// of any new table are `TableNotFoundException` (to 9 s and 22 s on the +/// two targets); a restore target then answers +/// `ContinuousBackupsUnavailableException` "Backups are being enabled for +/// the table: . Please retry later" for the rest of its load (to +/// 796 s on one target). ExtendDB answers each kind of table with the +/// answer of most of its CREATING time, and does not time the first +/// seconds. +#[cfg(feature = "pitr")] +fn enable_while_creating(table: &TableDescription) -> DynamoDbError { + if table.restore_summary.is_some() { + DynamoDbError::ContinuousBackupsUnavailableException(format!( + "Backups are being enabled for the table: {}. Please retry later", + table.table_name + )) + } else { + DynamoDbError::TableNotFoundException(format!("Table not found: {}", table.table_name)) + } +} + +/// `UpdateContinuousBackups` with an item-history store. Measured on the +/// live service, two samples each: the JSON type of `RecoveryPeriodInDays` +/// is checked before every other member, `TableName` included; the +/// specification's constraints are reported together; an enable of a +/// CREATING table is refused ([`enable_while_creating`]). +#[cfg(feature = "pitr")] +async fn update_with_history( + store: &dyn extenddb_storage::pitr::PitrCaptureStore, + body: &Value, + ctx: &OperationContext, +) -> Result { + let days = capture::period_member(body)?; + let table_name = table_name_member(body, "TableName")?; + check_table_name(table_name, super::table_name_bounds(&ctx.limits))?; + let flag = recovery_flag(body)?; + capture::check_specification(days, flag.is_none().then(flag_null_constraint))?; + let pitr_enabled = flag.ok_or_else(|| member_null(FLAG_PATH))?; + let period = capture::recovery_period(days, pitr_enabled)?; + let table = require_table_exists(ctx, table_name).await?; + // An enable never runs on a table still being created: a restore + // target's load writes no history. + if pitr_enabled && table.table_status == extenddb_core::types::TableStatus::Creating { + return Err(enable_while_creating(&table)); + } + let description = capture::update(store, &table, pitr_enabled, period).await?; + serialize_output(&json!({ "ContinuousBackupsDescription": description })) +} + /// Handle `RestoreTableToPointInTime`. /// /// An existing source table is refused with /// `PointInTimeRecoveryUnavailableException`, the exception the service models /// on this operation (`ContinuousBackupsUnavailableException` belongs to -/// `UpdateContinuousBackups`). A previous implementation faked a restore by -/// snapshotting the current table state and ignoring `RestoreDateTime`, which -/// violates tenet 1 (fidelity over features): a restore that silently returns -/// the wrong data is worse than a refusal the caller can act on. +/// `UpdateContinuousBackups`), whether or not its recovery is enabled. A +/// previous implementation faked a restore by snapshotting the current table +/// state and ignoring `RestoreDateTime`, which violates tenet 1 (fidelity +/// over features): a restore that silently returns the wrong data is worse +/// than a refusal the caller can act on. pub(crate) async fn handle_restore_table_to_point_in_time( body: Value, ctx: &OperationContext, @@ -322,5 +430,8 @@ pub(crate) async fn handle_restore_table_to_point_in_time( Err(pitr_restore_unavailable_error(&source_table_name)) } +#[cfg(feature = "pitr")] +mod capture; + #[cfg(test)] mod tests; diff --git a/crates/engine/src/backup/pitr/capture.rs b/crates/engine/src/backup/pitr/capture.rs new file mode 100644 index 000000000..e260cde30 --- /dev/null +++ b/crates/engine/src/backup/pitr/capture.rs @@ -0,0 +1,526 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Point-in-time recovery on a backend with an item-history store +//! (`extenddb_storage::pitr::PitrCaptureStore`): the enable, the disable and +//! the description. Measured on the live service, two samples each: +//! +//! - `RecoveryPeriodInDays` is read as the service's Integer member: a JSON +//! number with a fraction is truncated toward zero (7.9 is 7), a number +//! outside the 32-bit range is clamped to it, and any other JSON type is a +//! `SerializationException` checked before every other member. The value +//! is then 1 to 35, checked before the table, on an enable and on a +//! disable, together with a missing `PointInTimeRecoveryEnabled`; a disable +//! that gives one is refused. A fresh enable without one takes 35; an +//! enable on an enabled table changes the period only when one is given, +//! and keeps the window. +//! - A disable drops the history; a re-enable starts a new window. +//! - The description carries `RecoveryPeriodInDays` and an undocumented +//! `PointInTimeRecoveryWindowDays` equal to it. Right after the enable, +//! `EarliestRestorableDateTime` and `LatestRestorableDateTime` are the enable +//! instant in whole seconds. `LatestRestorableDateTime` stays there for the +//! first minutes and then runs about five minutes behind the request. + +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::types::{ + ContinuousBackupsDescription, PointInTimeRecoveryDescription, TableDescription, TableKeyInfo, +}; +use extenddb_storage::pitr::{PitrCaptureStore, PitrState, retention_floor}; +use serde_json::Value; + +use super::super::storage_err_to_dynamo; + +/// How far `LatestRestorableDateTime` runs behind the request, never before +/// the enable instant. Measured: still the enable instant 228 s after the +/// enable, and 299 s behind the request about 7 minutes after it. +pub(super) const LATEST_RESTORABLE_LAG: Duration = Duration::from_secs(300); + +/// The model's bounds on `RecoveryPeriodInDays`. +const PERIOD_MIN: i32 = 1; +const PERIOD_MAX: i32 = 35; + +const PERIOD_ON_DISABLE: &str = + "Invalid Request: Cannot specify RecoveryPeriodInDays when disabling point-in-time recovery."; + +fn period_constraint(value: i32, constraint: &str) -> String { + format!( + "Value '{value}' at 'pointInTimeRecoverySpecification.recoveryPeriodInDays' failed to \ + satisfy constraint: Member must have value {constraint}" + ) +} + +/// `PointInTimeRecoverySpecification.RecoveryPeriodInDays` as the service's +/// Integer member reads it, before any other member is checked (measured on +/// the live service, two samples each): a fraction is +/// truncated toward zero, a value outside the 32-bit range is clamped to it, +/// and a string, a boolean, a list or an object is a `SerializationException` +/// with the deserializer's text (none for a list). `None` when the member is +/// absent or null. +pub(super) fn period_member(body: &Value) -> Result, DynamoDbError> { + let member = body + .get("PointInTimeRecoverySpecification") + .and_then(|spec| spec.get("RecoveryPeriodInDays")); + let serialization = |m: &str| Err(DynamoDbError::SerializationException(m.to_owned())); + match member { + None | Some(Value::Null) => Ok(None), + Some(Value::Number(n)) => Ok(Some(match (n.as_i64(), n.as_u64()) { + (Some(i), _) => i32::try_from(i).unwrap_or(if i < 0 { i32::MIN } else { i32::MAX }), + (None, Some(_)) => i32::MAX, + // A float: `as` truncates toward zero and saturates at the bounds. + #[allow(clippy::cast_possible_truncation)] + (None, None) => n.as_f64().map_or(0, |f| f as i32), + })), + Some(Value::String(_)) => serialization("STRING_VALUE cannot be converted to Integer"), + Some(Value::Bool(true)) => serialization("TRUE_VALUE cannot be converted to Integer"), + Some(Value::Bool(false)) => serialization("FALSE_VALUE cannot be converted to Integer"), + Some(Value::Array(_)) => serialization(""), + Some(Value::Object(_)) => { + serialization("Start of structure or map found where not expected") + } + } +} + +/// The specification's model constraints, reported together in the +/// service's order: the period's bounds, then `flag_null`, the text of a +/// missing `PointInTimeRecoveryEnabled` (measured: "2 +/// validation errors detected: Value '0' at '...recoveryPeriodInDays' ...; +/// Value null at '...pointInTimeRecoveryEnabled' ..."). +pub(super) fn check_specification( + days: Option, + flag_null: Option, +) -> Result<(), DynamoDbError> { + let bound = days.and_then(|d| { + if d < PERIOD_MIN { + Some(period_constraint(d, "greater than or equal to 1")) + } else if d > PERIOD_MAX { + Some(period_constraint(d, "less than or equal to 35")) + } else { + None + } + }); + let errors: Vec = bound.into_iter().chain(flag_null).collect(); + match errors.len() { + 0 => Ok(()), + 1 => Err(DynamoDbError::ValidationException(format!( + "1 validation error detected: {}", + errors[0] + ))), + n => Err(DynamoDbError::ValidationException(format!( + "{n} validation errors detected: {}", + errors.join("; ") + ))), + } +} + +/// The period an update applies, its bounds already checked: refused on a +/// disable, `None` when the request gives none. +pub(super) fn recovery_period( + days: Option, + enabling: bool, +) -> Result, DynamoDbError> { + let Some(days) = days else { + return Ok(None); + }; + if !enabling { + return Err(DynamoDbError::ValidationException( + PERIOD_ON_DISABLE.to_owned(), + )); + } + u8::try_from(days).map(Some).map_err(|_| { + DynamoDbError::ValidationException(format!( + "1 validation error detected: {}", + period_constraint(days, "less than or equal to 35") + )) + }) +} + +fn key_info(table: &TableDescription) -> TableKeyInfo { + TableKeyInfo { + table_name: table.table_name.clone(), + table_id: table.table_id.clone(), + key_schema: table.key_schema.clone(), + base_key_schema: table.key_schema.clone(), + attribute_definitions: table.attribute_definitions.clone(), + ..Default::default() + } +} + +fn epoch_seconds(t: SystemTime) -> Duration { + t.duration_since(UNIX_EPOCH).unwrap_or_default() +} + +/// The floor of the restorable window of `state` at `now`, from +/// [`retention_floor`]: the enable instant in whole seconds, as the service +/// reports it, or the later floor the retention sets. +pub(crate) fn window_floor(state: &PitrState, now: SystemTime) -> Duration { + let floor = retention_floor(state, now); + let since = epoch_seconds(floor); + if floor == state.enabled_at { + Duration::from_secs(since.as_secs()) + } else { + since + } +} + +/// The restorable window of `state` at `now`: the floor ([`window_floor`]), +/// and a ceiling [`LATEST_RESTORABLE_LAG`] behind `now`, in milliseconds, +/// never before the floor. +pub(crate) fn restorable_window(state: &PitrState, now: SystemTime) -> (f64, f64) { + let earliest = window_floor(state, now); + let lagging = epoch_seconds(now).saturating_sub(LATEST_RESTORABLE_LAG); + let latest = Duration::from_millis(u64::try_from(lagging.as_millis()).unwrap_or(u64::MAX)); + let latest = latest.max(earliest); + (earliest.as_secs_f64(), latest.as_secs_f64()) +} + +fn description(state: Option<&PitrState>, now: SystemTime) -> ContinuousBackupsDescription { + let pitr = match state { + Some(state) => { + let (earliest, latest) = restorable_window(state, now); + PointInTimeRecoveryDescription { + point_in_time_recovery_status: "ENABLED".to_owned(), + earliest_restorable_date_time: Some(earliest), + latest_restorable_date_time: Some(latest), + recovery_period_in_days: Some(state.recovery_period_days), + point_in_time_recovery_window_days: Some(state.recovery_period_days), + } + } + None => PointInTimeRecoveryDescription { + point_in_time_recovery_status: "DISABLED".to_owned(), + earliest_restorable_date_time: None, + latest_restorable_date_time: None, + recovery_period_in_days: None, + point_in_time_recovery_window_days: None, + }, + }; + ContinuousBackupsDescription { + continuous_backups_status: "ENABLED".to_owned(), + point_in_time_recovery_description: Some(pitr), + } +} + +/// `DescribeContinuousBackups` for `table`. +pub(super) async fn describe( + store: &dyn PitrCaptureStore, + table: &TableDescription, +) -> Result { + let state = store + .pitr_state(&table.table_id) + .await + .map_err(storage_err_to_dynamo)?; + Ok(description(state.as_ref(), SystemTime::now())) +} + +/// A store error as the operation answers it: a table a DeleteTable removed +/// while the request ran is the service's missing-table text. +fn store_err(e: extenddb_storage::error::StorageError, table_name: &str) -> DynamoDbError { + match e { + extenddb_storage::error::StorageError::TableNotFound(_) => { + DynamoDbError::TableNotFoundException(format!("Table not found: {table_name}")) + } + other => storage_err_to_dynamo(other), + } +} + +/// `UpdateContinuousBackups` for `table`, its members already checked. +pub(super) async fn update( + store: &dyn PitrCaptureStore, + table: &TableDescription, + enabling: bool, + period: Option, +) -> Result { + let info = key_info(table); + let now = SystemTime::now(); + if enabling { + let state = store + .enable_pitr(&info, period, now) + .await + .map_err(|e| store_err(e, &table.table_name))?; + Ok(description(Some(&state), now)) + } else { + store + .disable_pitr(&info) + .await + .map_err(|e| store_err(e, &table.table_name))?; + Ok(description(None, now)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn state(enabled: u64, days: u8) -> PitrState { + PitrState::new(days, UNIX_EPOCH + Duration::from_micros(enabled)) + } + + #[test] + fn the_window_is_the_enable_second_until_the_lag_passes() { + let s = state(1_000_000_400_250_000, 35); + let now = |secs: u64| UNIX_EPOCH + Duration::from_secs(1_000_000_400 + secs); + assert_eq!( + restorable_window(&s, now(0)), + (1_000_000_400.0, 1_000_000_400.0) + ); + assert_eq!( + restorable_window(&s, now(228)), + (1_000_000_400.0, 1_000_000_400.0) + ); + assert_eq!( + restorable_window(&s, now(420)), + (1_000_000_400.0, 1_000_000_520.0) + ); + } + + #[test] + fn the_floor_moves_with_the_retention() { + let s = state(1_000_000_000_000_000, 7); + let now = UNIX_EPOCH + Duration::from_secs(1_000_000_000 + 8 * 86_400); + assert_eq!( + restorable_window(&s, now).0, + (1_000_000_000 + 86_400) as f64 + ); + } + + /// After a prune, the window starts where the history does, even when + /// a raised period would reach further back. + #[test] + fn the_floor_is_never_before_the_pruned_history() { + let mut s = state(1_000_000_000_000_000, 35); + s.pruned_to = UNIX_EPOCH + Duration::from_secs(1_000_000_000 + 9 * 86_400); + let now = UNIX_EPOCH + Duration::from_secs(1_000_000_000 + 10 * 86_400); + assert_eq!( + restorable_window(&s, now).0, + (1_000_000_000 + 9 * 86_400) as f64 + ); + } + + /// A table a DeleteTable removed while the request ran answers as a + /// missing table does. + #[test] + fn a_table_gone_from_the_store_is_the_missing_table_text() { + let e = store_err( + extenddb_storage::error::StorageError::TableNotFound("x".to_owned()), + "t1", + ); + match e { + DynamoDbError::TableNotFoundException(m) => assert_eq!(m, "Table not found: t1"), + other => panic!("{other:?}"), + } + } + + /// The answer to a specification, through the three checks in the + /// handler's order: the member's type, the constraints with the flag, + /// the disable. + fn period(spec: Value) -> Result, DynamoDbError> { + let body = json!({"PointInTimeRecoverySpecification": spec}); + let days = period_member(&body)?; + let flag = spec + .get("PointInTimeRecoveryEnabled") + .and_then(Value::as_bool); + let flag_null = flag.is_none().then(|| { + "Value null at 'pointInTimeRecoverySpecification.pointInTimeRecoveryEnabled' \ + failed to satisfy constraint: Member must not be null" + .to_owned() + }); + check_specification(days, flag_null)?; + recovery_period(days, flag.unwrap_or(false)) + } + + fn validation(r: Result, DynamoDbError>) -> String { + match r { + Err(DynamoDbError::ValidationException(m)) => m, + other => panic!("{other:?}"), + } + } + + fn serialization(r: Result, DynamoDbError>) -> String { + match r { + Err(DynamoDbError::SerializationException(m)) => m, + other => panic!("{other:?}"), + } + } + + const AT: &str = "at 'pointInTimeRecoverySpecification.recoveryPeriodInDays' failed to \ + satisfy constraint: Member must have value"; + const FLAG_NULL: &str = "Value null at \ + 'pointInTimeRecoverySpecification.pointInTimeRecoveryEnabled' failed to satisfy \ + constraint: Member must not be null"; + + fn too_low(v: &str) -> String { + format!("1 validation error detected: Value '{v}' {AT} greater than or equal to 1") + } + + fn too_high(v: &str) -> String { + format!("1 validation error detected: Value '{v}' {AT} less than or equal to 35") + } + + fn on(v: Value) -> Value { + json!({"PointInTimeRecoveryEnabled": true, "RecoveryPeriodInDays": v}) + } + + fn off(v: Value) -> Value { + json!({"PointInTimeRecoveryEnabled": false, "RecoveryPeriodInDays": v}) + } + + /// Measured on the live service, two samples each. + #[test] + fn the_period_bounds_are_the_services() { + assert_eq!(validation(period(on(json!(0)))), too_low("0")); + assert_eq!(validation(period(on(json!(-1)))), too_low("-1")); + assert_eq!(validation(period(on(json!(36)))), too_high("36")); + assert_eq!(validation(period(off(json!(0)))), too_low("0")); + assert_eq!(validation(period(off(json!(36)))), too_high("36")); + assert_eq!(validation(period(off(json!(7)))), PERIOD_ON_DISABLE); + assert_eq!(period(on(json!(7))).ok(), Some(Some(7))); + assert_eq!(period(on(json!(1))).ok(), Some(Some(1))); + assert_eq!(period(on(json!(35))).ok(), Some(Some(35))); + assert_eq!(period(on(Value::Null)).ok(), Some(None)); + assert_eq!( + period(json!({"PointInTimeRecoveryEnabled": true})).ok(), + Some(None) + ); + } + + /// A fraction is truncated toward zero, then checked (measured): 7.9 is + /// stored as 7; 7.5, 7.0 and 10.0 pass the bounds; 7.5 on a disable is + /// the disable refusal; 0.5, 0.9, 1e-5 and -0.5 are '0'; 1.9 is 1; 35.9 + /// passes; 36.5 is '36'; -1.5 is '-1'. + #[test] + fn a_fraction_is_truncated_toward_zero() { + assert_eq!(period(on(json!(7.9))).ok(), Some(Some(7))); + assert_eq!(period(on(json!(7.5))).ok(), Some(Some(7))); + assert_eq!(period(on(json!(7.0))).ok(), Some(Some(7))); + assert_eq!(period(on(json!(10.0))).ok(), Some(Some(10))); + assert_eq!(period(on(json!(1.9))).ok(), Some(Some(1))); + assert_eq!(period(on(json!(35.9))).ok(), Some(Some(35))); + assert_eq!(period(on(json!(3.5e1))).ok(), Some(Some(35))); + assert_eq!(validation(period(off(json!(7.5)))), PERIOD_ON_DISABLE); + for v in [json!(0.5), json!(0.9), json!(1e-5), json!(-0.5)] { + assert_eq!(validation(period(on(v))), too_low("0")); + } + assert_eq!(validation(period(on(json!(-1.5)))), too_low("-1")); + assert_eq!(validation(period(on(json!(36.5)))), too_high("36")); + assert_eq!(validation(period(off(json!(36.5)))), too_high("36")); + } + + /// Outside the 32-bit range the value is clamped to it (measured): 1e20, + /// 2147483648 and 2^63 are '2147483647'; -2147483649 is '-2147483648'; + /// and 2147483647, 2147483647.5, -2147483648 and -1e20. + #[test] + fn a_value_outside_the_integer_range_is_clamped() { + for v in [ + json!(2_147_483_647_i64), + json!(2_147_483_647.5), + json!(2_147_483_648_i64), + json!(9_223_372_036_854_775_808_u64), + json!(1e20), + ] { + assert_eq!(validation(period(on(v))), too_high("2147483647")); + } + for v in [ + json!(-2_147_483_648_i64), + json!(-2_147_483_649_i64), + json!(-1e20), + ] { + assert_eq!(validation(period(on(v))), too_low("-2147483648")); + } + } + + /// Every other JSON type is a SerializationException with the + /// deserializer's text (measured): "7", "abc", "", "7.5", true, [] and {}, + /// "7" on a disable, false and "35". + #[test] + fn another_json_type_is_a_serialization_error() { + for v in ["7", "abc", "", "7.5", "35"] { + assert_eq!( + serialization(period(on(json!(v)))), + "STRING_VALUE cannot be converted to Integer" + ); + } + assert_eq!( + serialization(period(off(json!("7")))), + "STRING_VALUE cannot be converted to Integer" + ); + assert_eq!( + serialization(period(on(json!(true)))), + "TRUE_VALUE cannot be converted to Integer" + ); + assert_eq!( + serialization(period(on(json!(false)))), + "FALSE_VALUE cannot be converted to Integer" + ); + assert_eq!(serialization(period(on(json!([])))), ""); + assert_eq!( + serialization(period(on(json!({})))), + "Start of structure or map found where not expected" + ); + } + + /// A bad period and a missing flag are one message, the period first: + /// 0, 36, 0.5, 1e20, -2147483649, and 0 with the flag null (measured). + /// A valid period with a missing flag names only the flag. + #[test] + fn a_bad_period_and_a_missing_flag_are_reported_together() { + let two = |v: &str, c: &str| { + format!("2 validation errors detected: Value '{v}' {AT} {c}; {FLAG_NULL}") + }; + let spec = |v: Value| json!({"RecoveryPeriodInDays": v}); + assert_eq!( + validation(period(spec(json!(0)))), + two("0", "greater than or equal to 1") + ); + assert_eq!( + validation(period( + json!({"PointInTimeRecoveryEnabled": null, "RecoveryPeriodInDays": 0}) + )), + two("0", "greater than or equal to 1") + ); + assert_eq!( + validation(period(spec(json!(36)))), + two("36", "less than or equal to 35") + ); + assert_eq!( + validation(period(spec(json!(0.5)))), + two("0", "greater than or equal to 1") + ); + assert_eq!( + validation(period(spec(json!(1e20)))), + two("2147483647", "less than or equal to 35") + ); + assert_eq!( + validation(period(spec(json!(-2_147_483_649_i64)))), + two("-2147483648", "greater than or equal to 1") + ); + assert_eq!( + validation(period(spec(json!(7)))), + format!("1 validation error detected: {FLAG_NULL}") + ); + } + + #[test] + fn the_description_carries_the_window_days_member() { + let d = serde_json::to_value(description( + Some(&state(1_000_000_000_000_000, 20)), + UNIX_EPOCH + Duration::from_secs(1_000_000_001), + )) + .expect("json"); + assert_eq!( + d, + json!({"ContinuousBackupsStatus": "ENABLED", "PointInTimeRecoveryDescription": { + "PointInTimeRecoveryStatus": "ENABLED", + "EarliestRestorableDateTime": 1_000_000_000.0, + "LatestRestorableDateTime": 1_000_000_000.0, + "RecoveryPeriodInDays": 20, + "PointInTimeRecoveryWindowDays": 20}}) + ); + assert_eq!( + serde_json::to_value(description(None, UNIX_EPOCH)).expect("json"), + json!({"ContinuousBackupsStatus": "ENABLED", + "PointInTimeRecoveryDescription": {"PointInTimeRecoveryStatus": "DISABLED"}}) + ); + } +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 89cb73648..023a70d74 100755 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -12,7 +12,8 @@ pub use backup::{ BackupRuntime, ConfiguredStore, HeldClaim, JobSettings, KillPoint, LOCAL_CHECK_TIMEOUT, OTHER_STORE, RESTORE_REGISTRATION_TIMEOUT, S3_CHECK_TIMEOUT, STORE_IDENTITY_KEY, Spool, StoreFence, StoreHealth, StoreHealthReport, StoreIdentityFile, StoreProblem, TARGET_NOT_EMPTY, - check_configured_store, maintenance, read_identity_bounded, reconciler, store_identity, sync, + check_configured_store, check_member_types, maintenance, read_identity_bounded, reconciler, + store_identity, sync, }; mod batch_get_item; mod batch_write_item; diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index 02e5c992b..ae5072f49 100755 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -18,6 +18,8 @@ mongodb-test-hooks = [] # `extenddb-storage/test-hooks` arms them in whichever backend the build # selects without naming a backend crate here (the server depends on none). backup-test-hooks = ["extenddb-engine/backup-test-hooks", "extenddb-storage/test-hooks"] +# Point-in-time recovery, off by default (see the engine's feature). +pitr = ["extenddb-engine/pitr"] [dependencies] extenddb-cache = { workspace = true } diff --git a/crates/server/src/request_helpers.rs b/crates/server/src/request_helpers.rs index 2faaabfdf..0a8a2a489 100755 --- a/crates/server/src/request_helpers.rs +++ b/crates/server/src/request_helpers.rs @@ -114,6 +114,9 @@ pub(crate) async fn authorize_request( // here, before the key info and resource tag lookups below and before the // developer-mode return, the cross-account refusal and the ARN shape // checks (see `table_name_gate`). + // The JSON types of members the service reads before every constraint + // check, TableName's included (`check_member_types`). + extenddb_engine::check_member_types(operation, input, &*state.storage)?; crate::table_name_gate::check_table_name_chars(operation, input, account_id)?; let table_name = extract_table_name(input); diff --git a/tests/test_continuous_backups.py b/tests/test_continuous_backups.py index a2a29df92..2b92dd3a6 100644 --- a/tests/test_continuous_backups.py +++ b/tests/test_continuous_backups.py @@ -9,7 +9,9 @@ recovery with ContinuousBackupsUnavailableException, and RestoreTableToPointInTime refuses with PointInTimeRecoveryUnavailableException after resolving the source table. These tests pin that divergence, so they run -only against ExtendDB. +only against ExtendDB. On a server built with the pitr feature +(EXTENDDB_TEST_PITR=1) an enable succeeds, so the test of the enable refusal +is skipped there; the other answers are the same on both builds. The request validation before those answers, and its order, are the service's: a missing table name member and the table name constraints, the @@ -32,6 +34,12 @@ reason="pins ExtendDB's documented divergence: point-in-time recovery is unsupported", ) +needs_no_pitr = pytest.mark.skipif( + os.environ.get("EXTENDDB_TEST_PITR") == "1", + reason="a server built with the pitr feature enables recovery " + "(tests/test_pitr_enable.py pins that answer)", +) + TABLE_NAME_REQUIRED = ( "The parameter 'TableName' is required but was not present in the request" ) @@ -106,6 +114,7 @@ def test_describe_continuous_backups_reports_recovery_disabled( assert pitr == {"PointInTimeRecoveryStatus": "DISABLED"} +@needs_no_pitr def test_enable_point_in_time_recovery_is_refused( dynamodb_client, create_and_cleanup_table, unique_table_name ): diff --git a/tests/test_pitr_enable.py b/tests/test_pitr_enable.py new file mode 100644 index 000000000..d55a19602 --- /dev/null +++ b/tests/test_pitr_enable.py @@ -0,0 +1,537 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Point-in-time recovery: the enable, the disable and the description. + +Every expectation was measured on DynamoDB, two samples each. Runs against +real DynamoDB, and against ExtendDB when the server is built with the pitr +feature (EXTENDDB_TEST_PITR=1). + +- An enable answers ENABLED at once, with RecoveryPeriodInDays (35 when the + request gives none) and an undocumented PointInTimeRecoveryWindowDays equal + to it, and with EarliestRestorableDateTime and LatestRestorableDateTime both + the enable instant in whole seconds. +- An enable on an enabled table keeps the window; it changes the period only + when the request gives one. +- RecoveryPeriodInDays is 1 to 35, checked before the table lookup, on an + enable and on a disable; a disable that gives one is refused. A number with + a fraction is truncated toward zero, one outside the 32-bit range is + clamped to it, and another JSON type is a SerializationException checked + before every other member. A bad period and a missing + PointInTimeRecoveryEnabled are reported in one message, the period first. +- An enable of a RestoreTableFromBackup target being created is refused: + TableNotFoundException for its first seconds, then + ContinuousBackupsUnavailableException "Backups are being enabled". +- A disable answers DISABLED with no window; a re-enable starts a new window + and takes 35 when it gives no period. +""" + +from __future__ import annotations + +import json +import os +import time +import uuid + +import pytest +from botocore.exceptions import ClientError + +pytestmark = pytest.mark.skipif( + bool(os.environ.get("EXTENDDB_TEST_ENDPOINT", "").strip()) + and os.environ.get("EXTENDDB_TEST_PITR") != "1", + reason="needs a server built with the pitr feature (EXTENDDB_TEST_PITR=1)", +) + +PERIOD_PATH = "pointInTimeRecoverySpecification.recoveryPeriodInDays" + + +def _too_low(value: int) -> str: + return ( + f"1 validation error detected: Value '{value}' at '{PERIOD_PATH}' failed to " + "satisfy constraint: Member must have value greater than or equal to 1" + ) + + +def _too_high(value: int) -> str: + return ( + f"1 validation error detected: Value '{value}' at '{PERIOD_PATH}' failed to " + "satisfy constraint: Member must have value less than or equal to 35" + ) + + +PERIOD_ON_DISABLE = ( + "Invalid Request: Cannot specify RecoveryPeriodInDays when disabling " + "point-in-time recovery." +) + + +@pytest.fixture() +def raw_bodies(dynamodb_client_no_validation): + """Every response body the no-validation client receives during the test, + as parsed JSON, including members the SDK model does not know.""" + bodies: list[dict] = [] + + def keep(http_response, **_): + bodies.append(json.loads(http_response.text or "{}")) + + events = dynamodb_client_no_validation.meta.events + events.register("after-call.dynamodb", keep, unique_id="pitr-raw-bodies") + yield bodies + events.unregister("after-call.dynamodb", unique_id="pitr-raw-bodies") + + +def _update(client, name, **spec): + return client.update_continuous_backups( + TableName=name, PointInTimeRecoverySpecification=spec + ) + + +def _while_backups_start(call): + """`call` on a new table. The service refuses an enable and a backup for + a few seconds after the table turns ACTIVE (measured: "Backups are being + enabled for the table: . Please retry later"), so retry that one + refusal for up to 30 s.""" + deadline = time.time() + 30 + while True: + try: + return call() + except ClientError as e: + err = e.response["Error"] + starting = err["Code"] == "ContinuousBackupsUnavailableException" and err[ + "Message" + ].startswith("Backups are being enabled for the table") + if not starting or time.time() > deadline: + raise + time.sleep(2) + + +def _first_enable(client, name, **spec): + """The first enable of a new table.""" + return _while_backups_start( + lambda: _update(client, name, PointInTimeRecoveryEnabled=True, **spec) + ) + + +def _error(call, **params) -> tuple[str, str]: + with pytest.raises(ClientError) as excinfo: + call(**params) + err = excinfo.value.response["Error"] + return err["Code"], err["Message"] + + +def test_an_enable_reports_the_window_and_the_period( + dynamodb_client_no_validation, create_and_cleanup_table, unique_table_name, raw_bodies +): + client = dynamodb_client_no_validation + create_and_cleanup_table() + bodies = raw_bodies + _first_enable(client, unique_table_name) + after = time.time() + before = int(after) - 2 + client.describe_continuous_backups(TableName=unique_table_name) + for body in bodies[-2:]: + desc = body["ContinuousBackupsDescription"] + assert desc["ContinuousBackupsStatus"] == "ENABLED" + pitr = desc["PointInTimeRecoveryDescription"] + assert pitr["PointInTimeRecoveryStatus"] == "ENABLED" + assert pitr["RecoveryPeriodInDays"] == 35 + assert pitr["PointInTimeRecoveryWindowDays"] == 35 + earliest = pitr["EarliestRestorableDateTime"] + assert earliest == int(earliest), pitr + assert before - 1 <= earliest <= after, pitr + assert pitr["LatestRestorableDateTime"] == earliest, pitr + + +def test_an_enable_on_an_enabled_table_changes_only_a_given_period( + dynamodb_client_no_validation, create_and_cleanup_table, unique_table_name +): + client = dynamodb_client_no_validation + create_and_cleanup_table() + + def pitr(resp): + return resp["ContinuousBackupsDescription"]["PointInTimeRecoveryDescription"] + + first = pitr(_first_enable(client, unique_table_name, RecoveryPeriodInDays=7)) + assert first["RecoveryPeriodInDays"] == 7 + for _ in range(2): + again = pitr( + _update( + client, unique_table_name, PointInTimeRecoveryEnabled=True, RecoveryPeriodInDays=20 + ) + ) + assert again["RecoveryPeriodInDays"] == 20 + assert again["EarliestRestorableDateTime"] == first["EarliestRestorableDateTime"] + for _ in range(2): + kept = pitr(_update(client, unique_table_name, PointInTimeRecoveryEnabled=True)) + assert kept["RecoveryPeriodInDays"] == 20 + assert kept["EarliestRestorableDateTime"] == first["EarliestRestorableDateTime"] + + +@pytest.mark.parametrize( + ("enabled", "value", "message"), + [ + (True, 0, _too_low(0)), + (True, -1, _too_low(-1)), + (True, 36, _too_high(36)), + (False, 0, _too_low(0)), + (False, 7, PERIOD_ON_DISABLE), + ], +) +def test_the_period_is_checked_before_the_table( + dynamodb_client_no_validation, create_and_cleanup_table, unique_table_name, enabled, value, message +): + client = dynamodb_client_no_validation + create_and_cleanup_table() + missing = f"extenddb-missing-{uuid.uuid4().hex[:12]}" + for name in (unique_table_name, missing): + for _ in range(2): + assert _error( + client.update_continuous_backups, + TableName=name, + PointInTimeRecoverySpecification={ + "PointInTimeRecoveryEnabled": enabled, + "RecoveryPeriodInDays": value, + }, + ) == ("ValidationException", message), name + + +def test_a_valid_period_on_a_missing_table_is_table_not_found(dynamodb_client_no_validation): + missing = f"extenddb-missing-{uuid.uuid4().hex[:12]}" + assert _error( + dynamodb_client_no_validation.update_continuous_backups, + TableName=missing, + PointInTimeRecoverySpecification={ + "PointInTimeRecoveryEnabled": True, + "RecoveryPeriodInDays": 7, + }, + ) == ("TableNotFoundException", f"Table not found: {missing}") + + +def test_a_disable_drops_the_window_and_a_re_enable_starts_a_new_one( + dynamodb_client_no_validation, create_and_cleanup_table, unique_table_name, raw_bodies +): + client = dynamodb_client_no_validation + create_and_cleanup_table() + bodies = raw_bodies + first = _first_enable(client, unique_table_name, RecoveryPeriodInDays=7)[ + "ContinuousBackupsDescription" + ]["PointInTimeRecoveryDescription"] + time.sleep(1.5) + for _ in range(2): + _update(client, unique_table_name, PointInTimeRecoveryEnabled=False) + client.describe_continuous_backups(TableName=unique_table_name) + for body in bodies[-4:]: + assert body == { + "ContinuousBackupsDescription": { + "ContinuousBackupsStatus": "ENABLED", + "PointInTimeRecoveryDescription": {"PointInTimeRecoveryStatus": "DISABLED"}, + } + } + again = _update(client, unique_table_name, PointInTimeRecoveryEnabled=True)[ + "ContinuousBackupsDescription" + ]["PointInTimeRecoveryDescription"] + assert again["RecoveryPeriodInDays"] == 35 + assert again["EarliestRestorableDateTime"] > first["EarliestRestorableDateTime"] + + +FLAG_NULL = ( + "Value null at 'pointInTimeRecoverySpecification.pointInTimeRecoveryEnabled' " + "failed to satisfy constraint: Member must not be null" +) +NAME_PATTERN = ( + "1 validation error detected: Value 'bad name!' at 'tableName' failed to " + "satisfy constraint: Member must satisfy regular expression pattern: " + "[a-zA-Z0-9_.-]+" +) + + +def _missing() -> str: + return f"extenddb-missing-{uuid.uuid4().hex[:12]}" + + +def _twice(call, **params) -> tuple[str, str]: + """The answer to two identical requests, which must be equal.""" + first = _error(call, **params) + assert _error(call, **params) == first, params + return first + + +def _with_period(value, enabled=True) -> dict: + return {"PointInTimeRecoveryEnabled": enabled, "RecoveryPeriodInDays": value} + + +@pytest.mark.parametrize( + ("case", "value", "message"), + [ + ("'7'", "7", "STRING_VALUE cannot be converted to Integer"), + ("'abc'", "abc", "STRING_VALUE cannot be converted to Integer"), + ("''", "", "STRING_VALUE cannot be converted to Integer"), + ("'7.5'", "7.5", "STRING_VALUE cannot be converted to Integer"), + ("true", True, "TRUE_VALUE cannot be converted to Integer"), + ("false", False, "FALSE_VALUE cannot be converted to Integer"), + ("[]", [], ""), + ("{}", {}, "Start of structure or map found where not expected"), + ], +) +def test_a_period_of_another_json_type_is_a_serialization_error( + dynamodb_client_no_validation, case, value, message +): + """Measured on the live service, two samples each.""" + assert _twice( + dynamodb_client_no_validation.update_continuous_backups, + TableName=_missing(), + PointInTimeRecoverySpecification=_with_period(value), + ) == ("SerializationException", message), case + + +@pytest.mark.parametrize( + ("case", "params", "message"), + [ + ("a TableName the pattern refuses", {"TableName": "bad name!"}, "STRING_VALUE cannot be converted to Integer"), + ("no TableName", {}, "STRING_VALUE cannot be converted to Integer"), + ("empty TableName", {"TableName": ""}, "STRING_VALUE cannot be converted to Integer"), + ], +) +def test_the_period_type_is_checked_before_the_table_name( + dynamodb_client_no_validation, case, params, message +): + """A type error in RecoveryPeriodInDays comes before a missing TableName + and before its constraints (measured).""" + assert _twice( + dynamodb_client_no_validation.update_continuous_backups, + PointInTimeRecoverySpecification=_with_period("x"), + **params, + ) == ("SerializationException", message), case + + +def test_the_table_name_constraints_come_before_the_period_bounds( + dynamodb_client_no_validation, +): + """A value the member clamps is no type error: the TableName pattern is + reported, and the period bounds are not (measured); without TableName + the required-member text comes first.""" + update = dynamodb_client_no_validation.update_continuous_backups + for spec in (_with_period(1e20), {"RecoveryPeriodInDays": 0}): + assert _twice( + update, TableName="bad name!", PointInTimeRecoverySpecification=spec + ) == ("ValidationException", NAME_PATTERN), spec + assert _twice( + update, TableName="ab", PointInTimeRecoverySpecification={"RecoveryPeriodInDays": 0} + ) == ( + "ValidationException", + "1 validation error detected: Value 'ab' at 'tableName' failed to satisfy " + "constraint: Member must have length greater than or equal to 3", + ) + assert _twice(update, PointInTimeRecoverySpecification=_with_period(0)) == ( + "ValidationException", + "The parameter 'TableName' is required but was not present in the request", + ) + + +def _two(value: str, constraint: str) -> str: + return ( + f"2 validation errors detected: Value '{value}' at '{PERIOD_PATH}' failed to " + f"satisfy constraint: Member must have value {constraint}; {FLAG_NULL}" + ) + + +@pytest.mark.parametrize( + ("case", "spec", "message"), + [ + ("0", {"RecoveryPeriodInDays": 0}, _two("0", "greater than or equal to 1")), + ("36", {"RecoveryPeriodInDays": 36}, _two("36", "less than or equal to 35")), + ("0.5", {"RecoveryPeriodInDays": 0.5}, _two("0", "greater than or equal to 1")), + ( + "1e20", + {"RecoveryPeriodInDays": 1e20}, + _two("2147483647", "less than or equal to 35"), + ), + ( + "-2147483649", + {"RecoveryPeriodInDays": -2147483649}, + _two("-2147483648", "greater than or equal to 1"), + ), + ( + "null flag", + {"PointInTimeRecoveryEnabled": None, "RecoveryPeriodInDays": 0}, + _two("0", "greater than or equal to 1"), + ), + ("7", {"RecoveryPeriodInDays": 7}, f"1 validation error detected: {FLAG_NULL}"), + ], +) +def test_a_bad_period_and_a_missing_flag_are_reported_together( + dynamodb_client_no_validation, case, spec, message +): + assert _twice( + dynamodb_client_no_validation.update_continuous_backups, + TableName=_missing(), + PointInTimeRecoverySpecification=spec, + ) == ("ValidationException", message), case + + +@pytest.mark.parametrize( + ("case", "value", "message"), + [ + ("0.5", 0.5, _too_low(0)), + ("0.9", 0.9, _too_low(0)), + ("1e-5", 1e-5, _too_low(0)), + ("-0.5", -0.5, _too_low(0)), + ("-1.5", -1.5, _too_low(-1)), + ("36.5", 36.5, _too_high(36)), + ("1e20", 1e20, _too_high(2147483647)), + ("2^31", 2**31, _too_high(2147483647)), + ("2^63", 2**63, _too_high(2147483647)), + ("2147483647.5", 2147483647.5, _too_high(2147483647)), + ("-2^31-1", -(2**31) - 1, _too_low(-2147483648)), + ("-1e20", -1e20, _too_low(-2147483648)), + ], +) +def test_a_period_is_truncated_and_clamped_before_its_bounds( + dynamodb_client_no_validation, case, value, message +): + """Measured on the live service, two samples each.""" + assert _twice( + dynamodb_client_no_validation.update_continuous_backups, + TableName=_missing(), + PointInTimeRecoverySpecification=_with_period(value), + ) == ("ValidationException", message), case + + +def test_a_period_in_bounds_after_truncation_reaches_the_table( + dynamodb_client_no_validation, create_and_cleanup_table, unique_table_name +): + """7.5, 7.0, 10.0, 1.9 and 35.9 on a missing table are + TableNotFoundException; on a disable 7.5 is the disable refusal; on an + enabled table 7.9 is stored as 7 (measured, two samples each).""" + client = dynamodb_client_no_validation + missing = _missing() + for value in (7.5, 7.0, 10.0, 1.9, 35.9): + assert _twice( + client.update_continuous_backups, + TableName=missing, + PointInTimeRecoverySpecification=_with_period(value), + ) == ("TableNotFoundException", f"Table not found: {missing}"), value + assert _twice( + client.update_continuous_backups, + TableName=missing, + PointInTimeRecoverySpecification=_with_period(7.5, enabled=False), + ) == ("ValidationException", PERIOD_ON_DISABLE) + create_and_cleanup_table() + _first_enable(client, unique_table_name) + for _ in range(2): + pitr = _update( + client, unique_table_name, PointInTimeRecoveryEnabled=True, RecoveryPeriodInDays=7.9 + )["ContinuousBackupsDescription"]["PointInTimeRecoveryDescription"] + assert pitr["RecoveryPeriodInDays"] == 7 + for _ in range(2): + described = client.describe_continuous_backups(TableName=unique_table_name) + pitr = described["ContinuousBackupsDescription"]["PointInTimeRecoveryDescription"] + assert pitr["RecoveryPeriodInDays"] == 7 + + +def _restore_source_items() -> int: + """Enough items that loading the restore target outlasts several enable + attempts on ExtendDB; on DynamoDB every restore takes minutes.""" + return 25 if not os.environ.get("EXTENDDB_TEST_ENDPOINT", "").strip() else 20_000 + + +def test_an_enable_of_a_restore_target_being_created_is_refused( + dynamodb_client, create_and_cleanup_table, unique_table_name +): + """While a RestoreTableFromBackup target is CREATING, an enable answers + TableNotFoundException "Table not found: " for the target's first + seconds, then ContinuousBackupsUnavailableException "Backups are being + enabled for the table: . Please retry later" until it is ACTIVE + (measured on DynamoDB, two targets polled every 4.5 s). ExtendDB gives + the second answer for the whole load. So recovery is never reported + enabled for a table whose load wrote no history: once the target is + ACTIVE its recovery is DISABLED, and an enable then takes the loaded + items as its baseline. An enable after a refusal that says the backups + are being enabled may succeed as the load ends.""" + client = dynamodb_client + create_and_cleanup_table() + target = f"{unique_table_name}-r" + not_found = ("TableNotFoundException", f"Table not found: {target}") + starting = ( + "ContinuousBackupsUnavailableException", + f"Backups are being enabled for the table: {target}. Please retry later", + ) + total = _restore_source_items() + for start in range(0, total, 25): + client.batch_write_item( + RequestItems={ + unique_table_name: [ + {"PutRequest": {"Item": {"pk": {"S": f"k{i}"}, "v": {"S": "x" * 300}}}} + for i in range(start, min(start + 25, total)) + ] + } + ) + arn = _while_backups_start( + lambda: client.create_backup(TableName=unique_table_name, BackupName="pitr-creating") + )["BackupDetails"]["BackupArn"] + try: + deadline = time.time() + 600 + while ( + client.describe_backup(BackupArn=arn)["BackupDescription"]["BackupDetails"][ + "BackupStatus" + ] + != "AVAILABLE" + ): + assert time.time() < deadline, "the backup did not become AVAILABLE" + time.sleep(1) + client.restore_table_from_backup(TargetTableName=target, BackupArn=arn) + try: + answers = [] + enabled_at_the_flip = False + deadline = time.time() + 1800 + while True: + before = client.describe_table(TableName=target)["Table"]["TableStatus"] + if before != "CREATING": + break + try: + _update(client, target, PointInTimeRecoveryEnabled=True) + answer = None + except ClientError as e: + answer = (e.response["Error"]["Code"], e.response["Error"]["Message"]) + if answer is None: + # The service accepts an enable at the end of the load + # while DescribeTable still reports CREATING (measured). + enabled_at_the_flip = True + break + assert answer in (not_found, starting), answer + assert not (answers and answers[-1] == starting and answer == not_found) + answers.append(answer) + assert time.time() < deadline, "the restore did not finish" + time.sleep(0.05 if os.environ.get("EXTENDDB_TEST_ENDPOINT") else 4) + assert starting in answers, answers + client.get_waiter("table_exists").wait(TableName=target) + for _ in range(2): + pitr = client.describe_continuous_backups(TableName=target)[ + "ContinuousBackupsDescription" + ]["PointInTimeRecoveryDescription"] + expected = "ENABLED" if enabled_at_the_flip else "DISABLED" + assert pitr["PointInTimeRecoveryStatus"] == expected + enabled = _first_enable(client, target)["ContinuousBackupsDescription"] + assert ( + enabled["PointInTimeRecoveryDescription"]["PointInTimeRecoveryStatus"] + == "ENABLED" + ) + finally: + _delete_when_possible(client, target) + finally: + client.delete_backup(BackupArn=arn) + + +def _delete_when_possible(client, name: str) -> None: + deadline = time.time() + 1800 + while True: + try: + client.delete_table(TableName=name) + break + except client.exceptions.ResourceNotFoundException: + return + except client.exceptions.ResourceInUseException: + assert time.time() < deadline, f"{name} could not be deleted" + time.sleep(5) + waiter = client.get_waiter("table_not_exists") + waiter.wait(TableName=name, WaiterConfig={"Delay": 2, "MaxAttempts": 300}) From f5cc587512101f9bb6ecaaedcd97492416f8f53b Mon Sep 17 00:00:00 2001 From: Elise Ong Date: Tue, 10 Nov 2026 12:25:22 -0500 Subject: [PATCH 4/6] feat(engine): point-in-time restore checks and reconstruct stream On the service, RestoreTableToPointInTime checks the time members' JSON types first, then the model's member constraints (the overrides' shapes and enums, a target name under 3 characters), then the source table and its recovery, then the time, then the overrides against the source, then the target name. The time's floor is the reported EarliestRestorableDateTime and its ceiling the request's own time: a restore to two seconds ago was accepted while the reported LatestRestorableDateTime was minutes behind, two seconds ahead was refused with InvalidRestoreTimeException "RestoreDateTime '' must be within EarliestRestorableDateTime '' and LatestRestorableDateTime '' for table ''": the requested time to the millisecond, the floor as reported, and Latest as the request's time less five minutes. RestoreDateTime is rounded to the millisecond and saturated to the 64-bit millisecond range, RestoreSummary.RestoreDateTime keeps the milliseconds, and UseLatestRestorableTime takes string spellings of true and false. BillingModeOverride PROVISIONED without a throughput override is "Invalid Request: Must override ProvisionedThroughput if BillingMode is overridden to PROVISIONED", after the time check, and on a table with a GSI the table rule comes before the index rule. A taken target name is TableAlreadyExists, a target still being restored TableInUse "Table is in the process of being created: " (two samples of each). Behind the pitr feature, and not yet called: the member checks, the time check and the RestoreSummary time, the target's CreateTableInput from the source's description with the overrides (the restore input a backup of the table would give), the target name check, and the source's items at the restore time as a stream over reconstruct pages. Adds InvalidRestoreTimeException. RestoreTableToPointInTime still answers PointInTimeRecoveryUnavailableException. Verified: engine unit tests with the measured texts, order and window, including the target name check; a negative control for the ceiling, the millisecond rounding, the table rule's order, the target name refusals and the stream's paging. --- crates/core/src/error/mod.rs | 11 + crates/engine/src/backup/pitr.rs | 2 + crates/engine/src/backup/pitr/restore.rs | 385 ++++++++++++ .../src/backup/pitr/restore_target_tests.rs | 130 ++++ .../engine/src/backup/pitr/restore_tests.rs | 583 ++++++++++++++++++ 5 files changed, 1111 insertions(+) create mode 100644 crates/engine/src/backup/pitr/restore.rs create mode 100644 crates/engine/src/backup/pitr/restore_target_tests.rs create mode 100644 crates/engine/src/backup/pitr/restore_tests.rs diff --git a/crates/core/src/error/mod.rs b/crates/core/src/error/mod.rs index 2e27ccb30..a04b2c9a2 100755 --- a/crates/core/src/error/mod.rs +++ b/crates/core/src/error/mod.rs @@ -113,6 +113,10 @@ pub enum DynamoDbError { /// on `UpdateContinuousBackups`. #[error("{0}")] PointInTimeRecoveryUnavailableException(String), + /// A `RestoreDateTime` outside the source table's restorable window, the + /// exception the service models on `RestoreTableToPointInTime`. + #[error("{0}")] + InvalidRestoreTimeException(String), #[error("{0}")] ServiceUnavailable(String), /// SP-ERR-002: Concurrent modification of the same item in a transaction. @@ -173,6 +177,7 @@ impl DynamoDbError { | Self::ExpiredIteratorException(_) | Self::ContinuousBackupsUnavailableException(_) | Self::PointInTimeRecoveryUnavailableException(_) + | Self::InvalidRestoreTimeException(_) | Self::TransactionConflictException(_) | Self::ProvisionedThroughputExceededException(_) | Self::ThrottlingException(_) @@ -230,6 +235,7 @@ impl DynamoDbError { Self::PointInTimeRecoveryUnavailableException(_) => { "PointInTimeRecoveryUnavailableException" } + Self::InvalidRestoreTimeException(_) => "InvalidRestoreTimeException", Self::ServiceUnavailable(_) => "ServiceUnavailable", Self::TransactionConflictException(_) => "TransactionConflictException", Self::ProvisionedThroughputExceededException(_) => { @@ -302,6 +308,7 @@ impl DynamoDbError { | Self::ExpiredIteratorException(m) | Self::ContinuousBackupsUnavailableException(m) | Self::PointInTimeRecoveryUnavailableException(m) + | Self::InvalidRestoreTimeException(m) | Self::ServiceUnavailable(m) | Self::TransactionConflictException(m) | Self::ProvisionedThroughputExceededException(m) @@ -424,6 +431,10 @@ mod tests { DynamoDbError::PointInTimeRecoveryUnavailableException(String::new()), 400, ), + ( + DynamoDbError::InvalidRestoreTimeException(String::new()), + 400, + ), (DynamoDbError::TableNotFoundException(String::new()), 400), (DynamoDbError::InvalidSignatureException(String::new()), 400), ]; diff --git a/crates/engine/src/backup/pitr.rs b/crates/engine/src/backup/pitr.rs index 5859e686c..508776c8e 100644 --- a/crates/engine/src/backup/pitr.rs +++ b/crates/engine/src/backup/pitr.rs @@ -432,6 +432,8 @@ pub(crate) async fn handle_restore_table_to_point_in_time( #[cfg(feature = "pitr")] mod capture; +#[cfg(feature = "pitr")] +mod restore; #[cfg(test)] mod tests; diff --git a/crates/engine/src/backup/pitr/restore.rs b/crates/engine/src/backup/pitr/restore.rs new file mode 100644 index 000000000..f309c0867 --- /dev/null +++ b/crates/engine/src/backup/pitr/restore.rs @@ -0,0 +1,385 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `RestoreTableToPointInTime` on a backend with an item-history store: the +//! request's checks in the service's order, the target's definition, and the +//! stream of the source's items at the restore time. Nothing calls them yet: +//! the handler answers `PointInTimeRecoveryUnavailableException`. +//! +//! Measured on the live service, two samples each: +//! - Order: the JSON types of the time members first, before the source +//! member; then the model's member constraints (the overrides' shapes and +//! enums, the target name's length); then the source table and the +//! recovery check; then the time window; then the overrides against the +//! source; then the target name. +//! - `RestoreDateTime` is epoch seconds rounded to the millisecond (.0005 is +//! .001, .0004 is .000), saturated to the 64-bit millisecond range. +//! `UseLatestRestorableTime` takes the strings "true", "TRUE", "yes" and +//! others as true and "false", "no" and others as false. +//! - The window's floor is the reported EarliestRestorableDateTime and its +//! ceiling the request's own time: restores to now - 2 s were accepted +//! while the reported Latest was minutes behind, now + 1.5 s refused, and +//! so was the floor less 1 ms. The refusal prints the requested time to +//! the millisecond, the floor as reported, and as Latest the request's +//! time less five minutes (never before the floor) to the nanosecond. +//! - `RestoreSummary.RestoreDateTime` is the requested time to the +//! millisecond; with `UseLatestRestorableTime` it is the reported Latest. +//! A restore at the floor holds the items written before the enable. +//! - The target name taken: `TableAlreadyExistsException` "Table already +//! exists: "; a restore target still being created: +//! `TableInUseException` "Table is in the process of being created: ". + +#![allow(dead_code)] + +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::limits::LimitsConfig; +use extenddb_core::types::{ + CreateTableInput, DescribeTableInput, Item, TableDescription, TableStatus, +}; +use extenddb_storage::backup_definition::restore_input_from_manifest; +use extenddb_storage::error::StorageError; +use extenddb_storage::pitr::{PitrCaptureStore, PitrState}; +use extenddb_storage::snapshot::source_table_schema; +use extenddb_storage::{RestoreTableOverrides, TableEngine}; +use futures::stream::{self, BoxStream, StreamExt}; +use serde_json::Value; + +use super::super::overrides::IndexOverrides; +use extenddb_core::types::BillingMode; + +use super::super::{ + parse_restore_table_overrides, storage_err_to_dynamo, validate_override_for_source, +}; +use super::capture::{LATEST_RESTORABLE_LAG, restorable_window, window_floor}; + +/// Items read from the history per page while a restore loads its target. +pub(crate) const RECONSTRUCT_PAGE: u32 = 1_000; + +/// The members checked before any table is read: the time members, the +/// overrides, and the target name's length and pattern. +pub(crate) struct RestoreMembers { + pub time: TimeMembers, + pub overrides: RestoreTableOverrides, + pub index_overrides: IndexOverrides, + pub target_table_name: String, +} + +/// `RestoreDateTime` in epoch milliseconds and `UseLatestRestorableTime`, as +/// the service's deserializer reads them. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct TimeMembers { + pub restore_date_time_ms: Option, + pub use_latest: bool, +} + +/// The time a restore asks for, its time members' exclusivity checked. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Requested { + Latest, + AtMillis(i64), +} + +/// The time a restore reconstructs at, and the one its `RestoreSummary` +/// reports, epoch seconds. +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) struct RestoreTime { + pub at: SystemTime, + pub restore_date_time: f64, +} + +fn serialization(message: &str) -> DynamoDbError { + DynamoDbError::SerializationException(message.to_owned()) +} + +/// `RestoreDateTime`, as the service reads it (measured). +fn restore_date_time_ms(member: Option<&Value>) -> Result, DynamoDbError> { + match member { + None | Some(Value::Null) => Ok(None), + Some(Value::Number(n)) => Ok(Some(match n.as_i64() { + Some(secs) => secs.saturating_mul(1_000), + // `as` saturates at the bounds of i64. + #[allow(clippy::cast_possible_truncation)] + None => n + .as_f64() + .map_or(i64::MAX, |f| (f * 1_000.0).round() as i64), + })), + Some(Value::String(_)) => Err(serialization("STRING_VALUE cannot be converted to Date")), + Some(Value::Bool(true)) => Err(serialization("TRUE_VALUE cannot be converted to Date")), + Some(Value::Bool(false)) => Err(serialization("FALSE_VALUE cannot be converted to Date")), + Some(Value::Array(_)) => Err(serialization("")), + Some(Value::Object(_)) => Err(serialization( + "Start of structure or map found where not expected", + )), + } +} + +/// `UseLatestRestorableTime`, as the service reads it (measured). +fn use_latest(member: Option<&Value>) -> Result { + match member { + None | Some(Value::Null) => Ok(false), + Some(Value::Bool(b)) => Ok(*b), + Some(Value::String(s)) => match s.as_str() { + "true" | "True" | "TRUE" | "yes" | "y" | "t" | "1" => Ok(true), + "false" | "False" | "FALSE" | "no" | "n" | "f" | "0" => Ok(false), + _ => Err(serialization("Unexpected token received from parser")), + }, + Some(Value::Number(n)) if n.is_f64() => Err(serialization( + "DECIMAL_VALUE cannot be converted to Boolean", + )), + Some(Value::Number(_)) => Err(serialization("NUMBER_VALUE cannot be converted to Boolean")), + Some(Value::Array(_)) => Err(serialization("")), + Some(Value::Object(_)) => Err(serialization( + "Start of structure or map found where not expected", + )), + } +} + +/// The time members' JSON types, before every other member. +pub(crate) fn time_members(body: &Value) -> Result { + Ok(TimeMembers { + restore_date_time_ms: restore_date_time_ms(body.get("RestoreDateTime"))?, + use_latest: use_latest(body.get("UseLatestRestorableTime"))?, + }) +} + +impl TimeMembers { + /// Exactly one of the two, with the service's texts (measured). + pub(crate) fn requested(self) -> Result { + match (self.restore_date_time_ms, self.use_latest) { + (Some(_), true) => Err(DynamoDbError::ValidationException( + super::TWO_RESTORE_TIMES.to_owned(), + )), + (None, false) => Err(DynamoDbError::ValidationException( + super::NO_RESTORE_TIME.to_owned(), + )), + (Some(ms), false) => Ok(Requested::AtMillis(ms)), + (None, true) => Ok(Requested::Latest), + } + } +} + +/// Step (a): the time members' types, then the model's member constraints, +/// before the source is read. +pub(crate) fn restore_members( + body: &Value, + bounds: (usize, usize), +) -> Result { + let time = time_members(body)?; + let overrides = parse_restore_table_overrides(body)?; + if let Some(throughput) = &overrides.provisioned_throughput { + extenddb_core::validation::validate_provisioned_throughput(&CreateTableInput { + billing_mode: Some(BillingMode::Provisioned), + provisioned_throughput: Some(throughput.clone()), + ..Default::default() + })?; + } + let index_overrides = IndexOverrides::parse(body)?; + let target = body + .get("TargetTableName") + .and_then(Value::as_str) + .unwrap_or_default(); + let (min, max) = bounds; + let length = target.chars().count(); + if length < min || length > max { + return Err(DynamoDbError::ValidationException(format!( + "TableName must be at least {min} characters long and at most {max} characters long" + ))); + } + super::super::validate_name_member(target, "targetTableName", bounds)?; + Ok(RestoreMembers { + time, + overrides, + index_overrides, + target_table_name: target.to_owned(), + }) +} + +/// A UTC instant as the service prints it in the window refusal: +/// `YYYY-MM-DDTHH:MM:SS` then the fraction in groups of three digits as +/// needed, then `Z`. A year past 9999 takes a `+` and a year before 0 a `-` +/// (measured: "+292278994-08-17T07:12:55.807Z", "+10000-01-01T00:00:00Z", +/// "0000-12-31T23:59:59Z", "-292275055-05-16T16:47:04.192Z"). +fn utc_instant(epoch_secs: i64, nanos: u32) -> String { + let days = epoch_secs.div_euclid(86_400); + let secs = epoch_secs.rem_euclid(86_400); + // Days since 1970-01-01 to a civil date (H. Hinnant's algorithm). + let z = days + 719_468; + let era = z.div_euclid(146_097); + let doe = z.rem_euclid(146_097); + let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let day = doy - (153 * mp + 2) / 5 + 1; + let month = if mp < 10 { mp + 3 } else { mp - 9 }; + let year = yoe + era * 400 + i64::from(month <= 2); + let year = if year > 9_999 { + format!("+{year}") + } else if year < 0 { + format!("-{:04}", -year) + } else { + format!("{year:04}") + }; + let fraction = if nanos == 0 { + String::new() + } else if nanos.is_multiple_of(1_000_000) { + format!(".{:03}", nanos / 1_000_000) + } else if nanos.is_multiple_of(1_000) { + format!(".{:06}", nanos / 1_000) + } else { + format!(".{nanos:09}") + }; + format!( + "{year}-{month:02}-{day:02}T{:02}:{:02}:{:02}{fraction}Z", + secs / 3_600, + secs % 3_600 / 60, + secs % 60 + ) +} + +fn utc_millis(ms: i64) -> String { + let nanos = u32::try_from(ms.rem_euclid(1_000)).expect("below 1000") * 1_000_000; + utc_instant(ms.div_euclid(1_000), nanos) +} + +fn utc_duration(since_epoch: Duration) -> String { + let secs = i64::try_from(since_epoch.as_secs()).unwrap_or(i64::MAX); + utc_instant(secs, since_epoch.subsec_nanos()) +} + +/// Step (c): the restore time against the window of `state` at `now`. The +/// reconstruct runs at the requested time, or at the enable instant for a +/// time inside the enable's second, which the window reports as its floor. +pub(crate) fn restore_time( + requested: Requested, + state: &PitrState, + now: SystemTime, + source_table_name: &str, +) -> Result { + let floor = window_floor(state, now); + let (_, latest) = restorable_window(state, now); + let ms = match requested { + // The reported Latest is whole milliseconds. + #[allow(clippy::cast_possible_truncation)] + Requested::Latest => (latest * 1_000.0).round() as i64, + Requested::AtMillis(ms) => ms, + }; + let now_since = now.duration_since(UNIX_EPOCH).unwrap_or_default(); + let requested_ns = i128::from(ms) * 1_000_000; + let as_ns = |d: Duration| i128::try_from(d.as_nanos()).unwrap_or(i128::MAX); + if requested_ns < as_ns(floor) || requested_ns > as_ns(now_since) { + let latest_text = now_since.saturating_sub(LATEST_RESTORABLE_LAG).max(floor); + return Err(DynamoDbError::InvalidRestoreTimeException(format!( + "RestoreDateTime '{}' must be within EarliestRestorableDateTime '{}' and \ + LatestRestorableDateTime '{}' for table '{source_table_name}'", + utc_millis(ms), + utc_duration(floor), + utc_duration(latest_text), + ))); + } + let since = Duration::from_millis(u64::try_from(ms).expect("at or after the floor")); + Ok(RestoreTime { + at: (UNIX_EPOCH + since).max(state.enabled_at), + #[allow(clippy::cast_precision_loss)] + restore_date_time: ms as f64 / 1_000.0, + }) +} + +/// The service's text for `BillingModeOverride` PROVISIONED without a +/// `ProvisionedThroughputOverride`, checked after the time window (measured; +/// RestoreTableFromBackup answers this case earlier and with other text). +const PROVISIONED_NEEDS_THROUGHPUT: &str = "Invalid Request: Must override ProvisionedThroughput if BillingMode is overridden to \ + PROVISIONED"; + +/// Step (d): the target's `CreateTableInput` from the source's description, +/// as a backup of it would restore (the source's keys, billing mode and +/// indexes; not TTL, streams, tags or recovery), with the overrides applied +/// and checked as `CreateTable` checks its input. +pub(crate) fn target_input( + source: &TableDescription, + members: &RestoreMembers, + limits: &LimitsConfig, +) -> Result { + if members.overrides.billing_mode == Some(BillingMode::Provisioned) + && members.overrides.provisioned_throughput.is_none() + { + return Err(DynamoDbError::ValidationException( + PROVISIONED_NEEDS_THROUGHPUT.to_owned(), + )); + } + let schema = source_table_schema(source, None).map_err(storage_err_to_dynamo)?; + validate_override_for_source( + &members.overrides, + extenddb_storage::backup_catalog::billing_mode_name(schema.billing_mode), + )?; + let mut input = + restore_input_from_manifest(&schema, &members.target_table_name, &members.overrides); + members + .index_overrides + .apply(&mut input, members.overrides.billing_mode)?; + extenddb_core::validation::validate_create_table(&input, limits)?; + Ok(input) +} + +/// Step (e): the target name under the account. +pub(crate) async fn check_target_name( + storage: &S, + account_id: &str, + target: &str, +) -> Result<(), DynamoDbError> { + let probe = storage + .describe_table( + account_id, + DescribeTableInput { + table_name: target.to_owned(), + }, + ) + .await; + match probe { + Ok(desc) + if desc.table_status == TableStatus::Creating && desc.restore_summary.is_some() => + { + Err(DynamoDbError::TableInUseException(format!( + "Table is in the process of being created: {target}" + ))) + } + Ok(_) => Err(DynamoDbError::TableAlreadyExistsException(format!( + "Table already exists: {target}" + ))), + Err(StorageError::TableNotFound(_)) => Ok(()), + Err(e) => Err(storage_err_to_dynamo(e)), + } +} + +/// The source's items at `at`, read from the history a page at a time. +pub(crate) fn reconstruct_stream<'a>( + store: &'a dyn PitrCaptureStore, + table_id: &'a str, + at: SystemTime, + page: u32, +) -> BoxStream<'a, Result> { + // State: the cursor of the next page, `None` once the last was read. + stream::unfold(Some(None), move |cursor: Option>| async move { + let after = cursor?; + match store.reconstruct_page(table_id, at, after, page).await { + Ok(p) => { + let next = p.next.map(Some); + Some(( + stream::iter(p.items.into_iter().map(Ok)).left_stream(), + next, + )) + } + Err(e) => Some((stream::once(async { Err(e) }).right_stream(), None)), + } + }) + .flatten() + .boxed() +} + +#[cfg(test)] +#[path = "restore_target_tests.rs"] +mod target_tests; +#[cfg(test)] +#[path = "restore_tests.rs"] +mod tests; diff --git a/crates/engine/src/backup/pitr/restore_target_tests.rs b/crates/engine/src/backup/pitr/restore_target_tests.rs new file mode 100644 index 000000000..8a9e889a2 --- /dev/null +++ b/crates/engine/src/backup/pitr/restore_target_tests.rs @@ -0,0 +1,130 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +use futures::future::BoxFuture; +use serde_json::json; + +use super::*; + +/// A catalog of named tables, for the target-name check. +struct Catalog(std::collections::HashMap>); + +fn unused() -> BoxFuture<'static, Result> { + unimplemented!("the target-name check reads describe_table only") +} + +impl TableEngine for Catalog { + fn create_table( + &self, + _: &str, + _: CreateTableInput, + ) -> BoxFuture<'_, Result> { + unused() + } + fn delete_table( + &self, + _: &str, + _: extenddb_core::types::DeleteTableInput, + ) -> BoxFuture<'_, Result> { + unused() + } + fn describe_table( + &self, + _: &str, + input: DescribeTableInput, + ) -> BoxFuture<'_, Result> { + let found = self.0.get(&input.table_name).cloned(); + Box::pin(async move { found.unwrap_or(Err(StorageError::TableNotFound(input.table_name))) }) + } + fn list_tables( + &self, + _: &str, + _: extenddb_core::types::ListTablesInput, + ) -> BoxFuture<'_, Result> { + unused() + } + fn update_table( + &self, + _: &str, + _: extenddb_core::types::UpdateTableInput, + ) -> BoxFuture<'_, Result> { + unused() + } + fn table_key_info( + &self, + _: &str, + _: &str, + ) -> BoxFuture<'_, Result> { + unused() + } + fn index_info( + &self, + _: &str, + _: &str, + _: &str, + ) -> BoxFuture<'_, Result> { + unused() + } + fn index_info_by_table_id( + &self, + _: &str, + _: &str, + ) -> BoxFuture<'_, Result> { + unused() + } +} + +fn table(name: &str, status: TableStatus, restoring: bool) -> TableDescription { + TableDescription { + table_name: name.to_owned(), + table_status: status, + restore_summary: restoring.then(|| { + serde_json::from_value(json!({ + "SourceBackupArn": "arn:aws:dynamodb:us-east-1:000000000000:table/src/backup/b", + "RestoreDateTime": 1_794_284_272.0, + "RestoreInProgress": true + })) + .expect("summary") + }), + ..Default::default() + } +} + +/// Measured on the live service: an ACTIVE target (two samples) and a +/// restore target still being created. +#[tokio::test] +async fn the_target_name_must_be_free() { + let catalog = Catalog( + [ + ("taken", Ok(table("taken", TableStatus::Active, false))), + ("loading", Ok(table("loading", TableStatus::Creating, true))), + ( + "broken", + Err(StorageError::Internal( + "the catalog is unreadable".to_owned(), + )), + ), + ] + .into_iter() + .map(|(n, t)| (n.to_owned(), t)) + .collect(), + ); + let check = |name: &'static str| check_target_name(&catalog, "000000000000", name); + match check("taken").await { + Err(DynamoDbError::TableAlreadyExistsException(m)) => { + assert_eq!(m, "Table already exists: taken"); + } + other => panic!("{other:?}"), + } + match check("loading").await { + Err(DynamoDbError::TableInUseException(m)) => { + assert_eq!(m, "Table is in the process of being created: loading"); + } + other => panic!("{other:?}"), + } + assert!(check("free").await.is_ok()); + assert!(matches!( + check("broken").await, + Err(DynamoDbError::InternalServerError(_)) + )); +} diff --git a/crates/engine/src/backup/pitr/restore_tests.rs b/crates/engine/src/backup/pitr/restore_tests.rs new file mode 100644 index 000000000..bdb8ccfef --- /dev/null +++ b/crates/engine/src/backup/pitr/restore_tests.rs @@ -0,0 +1,583 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +use std::sync::Mutex; + +use extenddb_core::types::AttributeValue; +use extenddb_storage::pitr::ReconstructPage; +use futures::future::BoxFuture; +use serde_json::json; + +use super::*; + +/// A state enabled at `enabled` (epoch seconds and nanoseconds). +fn state_at(secs: u64, nanos: u32) -> PitrState { + PitrState::new(35, UNIX_EPOCH + Duration::new(secs, nanos)) +} + +fn at(secs: u64, nanos: u32) -> SystemTime { + UNIX_EPOCH + Duration::new(secs, nanos) +} + +fn invalid(r: Result) -> String { + match r { + Err(DynamoDbError::InvalidRestoreTimeException(m)) => m, + other => panic!("{other:?}"), + } +} + +/// The restore time of a request body, through the two checks. +fn time_of( + body: Value, + s: &PitrState, + now: SystemTime, + table: &str, +) -> Result { + restore_time(time_members(&body)?.requested()?, s, now, table) +} + +fn window_text(requested: &str, earliest: &str, latest: &str, table: &str) -> String { + format!( + "RestoreDateTime '{requested}' must be within EarliestRestorableDateTime '{earliest}' \ + and LatestRestorableDateTime '{latest}' for table '{table}'" + ) +} + +#[test] +fn an_instant_prints_as_the_service_prints_it() { + assert_eq!(utc_instant(1_794_275_217, 0), "2026-11-10T01:46:57Z"); + assert_eq!(utc_instant(0, 0), "1970-01-01T00:00:00Z"); + assert_eq!(utc_instant(951_782_400, 0), "2000-02-29T00:00:00Z"); + assert_eq!(utc_instant(0, 1_000_000), "1970-01-01T00:00:00.001Z"); + assert_eq!(utc_instant(0, 1_000), "1970-01-01T00:00:00.000001Z"); + assert_eq!( + utc_instant(1_794_284_405, 708_986_387), + "2026-11-10T04:20:05.708986387Z" + ); + // A time past the 64-bit millisecond range prints at its end. + assert_eq!(utc_millis(i64::MAX), "+292278994-08-17T07:12:55.807Z"); + assert_eq!(utc_millis(i64::MIN), "-292275055-05-16T16:47:04.192Z"); + assert_eq!(utc_millis(253_402_300_800_000), "+10000-01-01T00:00:00Z"); + assert_eq!(utc_millis(253_402_300_799_999), "9999-12-31T23:59:59.999Z"); + assert_eq!(utc_millis(-62_135_596_801_000), "0000-12-31T23:59:59Z"); + assert_eq!(utc_millis(-1_000), "1969-12-31T23:59:59Z"); +} + +/// Measured on the live service, two samples each, on a table enabled in +/// second 1794284219. +#[test] +fn the_refusal_prints_the_requested_time_to_the_millisecond() { + let table = "src-e"; + let s = state_at(1_794_284_219, 950_000_000); + let early = at(1_794_284_268, 157_390_000); + let floor = "2026-11-10T04:16:59Z"; + let date = + |v: Value, now: SystemTime| invalid(time_of(json!({"RestoreDateTime": v}), &s, now, table)); + assert_eq!( + date(json!(1_794_284_159), early), + window_text("2026-11-10T04:15:59Z", floor, floor, table) + ); + assert_eq!( + date(json!(1_794_284_298.157_383), early), + window_text("2026-11-10T04:18:18.157Z", floor, floor, table) + ); + assert_eq!( + date(json!(-1), early), + window_text("1969-12-31T23:59:59Z", floor, floor, table) + ); + assert_eq!( + date(json!(1e20), early), + window_text("+292278994-08-17T07:12:55.807Z", floor, floor, table) + ); + // Once the lag passes, Latest is the request's time less five + // minutes, to the nanosecond. + assert_eq!( + date(json!(1_794_284_218.0), at(1_794_284_705, 708_986_387)), + window_text( + "2026-11-10T04:16:58Z", + floor, + "2026-11-10T04:20:05.708986387Z", + table + ) + ); +} + +/// Measured on the live service (three times refused, two samples each; two +/// accepted) on a table enabled in second 1794284900. +#[test] +fn the_window_is_checked_to_the_millisecond() { + let table = "src-f"; + let s = state_at(1_794_284_900, 900_000_000); + let now = at(1_794_284_904, 900_000_000); + let floor = "2026-11-10T04:28:20Z"; + let date = |v: Value| time_of(json!({"RestoreDateTime": v}), &s, now, table); + for (v, text) in [ + (json!(1_794_284_899.5), "2026-11-10T04:28:19.500Z"), + (json!(1_794_284_899.999), "2026-11-10T04:28:19.999Z"), + (json!(1_794_284_906.400_066_1), "2026-11-10T04:28:26.400Z"), + (json!(0.001), "1970-01-01T00:00:00.001Z"), + (json!(0.0004), "1970-01-01T00:00:00Z"), + ] { + assert_eq!(invalid(date(v)), window_text(text, floor, floor, table)); + } + let a1 = date(json!(1_794_284_900.0)).expect("the floor"); + assert_eq!(a1.restore_date_time, 1_794_284_900.0); + let a2 = date(json!(1_794_284_900.5)).expect("inside the window"); + assert_eq!(a2.restore_date_time, 1_794_284_900.5); + assert_eq!(a2.at, at(1_794_284_900, 900_000_000)); +} + +/// Measured on the live service, two samples each, on a table enabled in +/// second 1794289159; the requested time rounds to the millisecond and +/// saturates at the 64-bit millisecond range. +#[test] +fn the_requested_time_rounds_and_saturates() { + let table = "src-v"; + let s = state_at(1_794_289_159, 400_000_000); + let now = at(1_794_289_170, 0); + let floor = "2026-11-10T05:39:19Z"; + for (v, text) in [ + (json!(1_794_284_900.000_5), "2026-11-10T04:28:20.001Z"), + (json!(1_794_284_900.001_5), "2026-11-10T04:28:20.002Z"), + (json!(1e19), "+292278994-08-17T07:12:55.807Z"), + (json!(-1e20), "-292275055-05-16T16:47:04.192Z"), + (json!(253_402_300_800_i64), "+10000-01-01T00:00:00Z"), + (json!(253_402_300_799.999), "9999-12-31T23:59:59.999Z"), + (json!(-62_135_596_801_i64), "0000-12-31T23:59:59Z"), + ] { + assert_eq!( + invalid(time_of(json!({"RestoreDateTime": v}), &s, now, table)), + window_text(text, floor, floor, table) + ); + } +} + +/// Measured on the live service: RestoreSummary.RestoreDateTime +/// keeps the requested milliseconds, rounded. +#[test] +fn the_restore_runs_at_the_requested_millisecond() { + let s = state_at(1_794_284_219, 950_000_000); + let rs2 = time_of( + json!({"RestoreDateTime": 1_794_284_271.294_530_2}), + &s, + at(1_794_284_273, 294_535_900), + "src", + ) + .expect("a time with milliseconds"); + assert_eq!(rs2.restore_date_time, 1_794_284_271.295); + assert_eq!(rs2.at, at(1_794_284_271, 295_000_000)); + let re10 = time_of( + json!({"RestoreDateTime": 1_794_284_703.024_852}), + &s, + at(1_794_284_705, 24_859_000), + "src", + ) + .expect("a time with microseconds"); + assert_eq!(re10.restore_date_time, 1_794_284_703.025); +} + +/// A restore at the reported floor reconstructs at the enable instant, so +/// it holds the baseline the enable took (measured: live restores at +/// EarliestRestorableDateTime hold the items written before the enable). +#[test] +fn a_restore_at_the_floor_reconstructs_at_the_enable_instant() { + let s = state_at(1_794_284_900, 900_000_000); + let r = time_of( + json!({"RestoreDateTime": 1_794_284_900}), + &s, + at(1_794_284_905, 0), + "src", + ) + .expect("the floor"); + assert_eq!(r.at, s.enabled_at); + assert_eq!(r.restore_date_time, 1_794_284_900.0); +} + +/// Measured on the live service: with UseLatestRestorableTime the summary +/// is the reported Latest, the enable second for the first five minutes. +#[test] +fn use_latest_restores_to_the_reported_latest() { + let s = state_at(1_794_284_219, 250_000_000); + let latest = |now: SystemTime| { + time_of(json!({"UseLatestRestorableTime": true}), &s, now, "src") + .expect("latest") + .restore_date_time + }; + assert_eq!(latest(at(1_794_284_272, 648_000_000)), 1_794_284_219.0); + assert_eq!(latest(at(1_794_284_639, 500_000_000)), 1_794_284_339.5); +} + +fn serialization_text(body: Value) -> String { + match time_members(&body) { + Err(DynamoDbError::SerializationException(m)) => m, + other => panic!("{other:?}"), + } +} + +/// Measured on the live service: the deserializer's text +/// for a RestoreDateTime that is not a number. +#[test] +fn a_restore_date_time_that_is_not_a_number_is_a_serialization_error() { + for v in [ + json!("abc"), + json!("2026-01-01T00:00:00Z"), + json!("1794284900"), + ] { + assert_eq!( + serialization_text(json!({"RestoreDateTime": v})), + "STRING_VALUE cannot be converted to Date" + ); + } + assert_eq!( + serialization_text(json!({"RestoreDateTime": true})), + "TRUE_VALUE cannot be converted to Date" + ); + assert_eq!( + serialization_text(json!({"RestoreDateTime": false})), + "FALSE_VALUE cannot be converted to Date" + ); + assert_eq!(serialization_text(json!({"RestoreDateTime": []})), ""); + assert_eq!( + serialization_text(json!({"RestoreDateTime": {}})), + "Start of structure or map found where not expected" + ); +} + +/// Measured on the live service: the spellings +/// of UseLatestRestorableTime, and the deserializer's text for the rest. +#[test] +fn use_latest_takes_the_services_spellings() { + let flag = |v: Value| { + time_members(&json!({"UseLatestRestorableTime": v})) + .expect("a flag") + .use_latest + }; + for v in ["true", "True", "TRUE", "yes", "y", "t", "1"] { + assert!(flag(json!(v)), "{v}"); + } + for v in ["false", "False", "FALSE", "no", "n", "f", "0"] { + assert!(!flag(json!(v)), "{v}"); + } + assert!(flag(json!(true))); + assert!(!flag(Value::Null)); + for v in ["off", " true", "", "null"] { + assert_eq!( + serialization_text(json!({"UseLatestRestorableTime": v})), + "Unexpected token received from parser" + ); + } + for v in [json!(0), json!(1)] { + assert_eq!( + serialization_text(json!({"UseLatestRestorableTime": v})), + "NUMBER_VALUE cannot be converted to Boolean" + ); + } + assert_eq!( + serialization_text(json!({"UseLatestRestorableTime": 1.5})), + "DECIMAL_VALUE cannot be converted to Boolean" + ); + assert_eq!( + serialization_text(json!({"UseLatestRestorableTime": []})), + "" + ); + assert_eq!( + serialization_text(json!({"UseLatestRestorableTime": {}})), + "Start of structure or map found where not expected" + ); +} + +/// Measured on the live service: a string true with a RestoreDateTime is the +/// two-times refusal; a string false with one restores to that time. +#[test] +fn the_spellings_count_in_the_exclusivity_check() { + let requested = |v: &str| { + time_members(&json!({"UseLatestRestorableTime": v, "RestoreDateTime": 1000})) + .expect("types") + .requested() + }; + for v in ["TRUE", "yes"] { + match requested(v) { + Err(DynamoDbError::ValidationException(m)) => { + assert_eq!(m, super::super::TWO_RESTORE_TIMES); + } + other => panic!("{other:?}"), + } + } + assert_eq!( + requested("false").ok(), + Some(Requested::AtMillis(1_000_000)) + ); +} + +#[test] +fn the_members_are_checked_before_any_table() { + let bounds = (3, 255); + let msg = |body: Value| match restore_members(&body, bounds) { + Err(DynamoDbError::ValidationException(m)) => m, + Ok(_) => "ok".to_owned(), + Err(other) => format!("{other:?}"), + }; + assert_eq!( + msg(json!({"TargetTableName": "x"})), + "TableName must be at least 3 characters long and at most 255 characters long" + ); + assert!( + msg(json!({"TargetTableName": "abc", "BillingModeOverride": "NOPE"})).contains( + "Value 'NOPE' at 'billingModeOverride' failed to satisfy constraint: Member must \ + satisfy enum value set: [PROVISIONED, PAY_PER_REQUEST]" + ) + ); + assert_eq!(msg(json!({"TargetTableName": "abc"})), "ok"); + // The time members' types come before the other members (measured with + // a source name the pattern refuses, and with a bad billing mode and a + // target name too short). + for body in [ + json!({"TargetTableName": "x", "RestoreDateTime": "abc"}), + json!({"TargetTableName": "abc", "BillingModeOverride": "BAD", "RestoreDateTime": "abc"}), + ] { + assert_eq!( + msg(body), + "SerializationException(\"STRING_VALUE cannot be converted to Date\")" + ); + } + assert_eq!( + msg( + json!({"TargetTableName": "abc", "BillingModeOverride": "BAD", + "UseLatestRestorableTime": "off"}) + ), + "SerializationException(\"Unexpected token received from parser\")" + ); +} + +/// A floor the retention sets is not a whole second, and the window is +/// checked against it to the millisecond. +#[test] +fn a_time_before_a_retention_floor_is_refused() { + let s = PitrState::new(1, UNIX_EPOCH + Duration::from_secs(1_794_000_000)); + let now = at(1_794_284_900, 700_000_000); + let floor = 1_794_284_900.7 - 86_400.0; + let date = |t: f64| time_of(json!({"RestoreDateTime": t}), &s, now, "src"); + assert!(matches!( + date(floor - 0.3), + Err(DynamoDbError::InvalidRestoreTimeException(_)) + )); + assert_eq!(date(floor).expect("the floor").restore_date_time, floor); +} + +/// A store that serves fixed pages and records each call's cursor. +struct Pages { + pages: Vec, + calls: Mutex>>, +} + +impl PitrCaptureStore for Pages { + fn pitr_state<'a>( + &'a self, + _: &'a str, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async { Ok(None) }) + } + fn enable_pitr<'a>( + &'a self, + _: &'a extenddb_core::types::TableKeyInfo, + _: Option, + _: SystemTime, + ) -> BoxFuture<'a, Result> { + unimplemented!() + } + fn disable_pitr<'a>( + &'a self, + _: &'a extenddb_core::types::TableKeyInfo, + ) -> BoxFuture<'a, Result<(), StorageError>> { + unimplemented!() + } + fn reconstruct_page<'a>( + &'a self, + _: &'a str, + _: SystemTime, + after: Option, + _: u32, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mut calls = self.calls.lock().expect("calls"); + let i = calls.len(); + calls.push(after); + match self.pages.get(i) { + Some(p) => Ok(ReconstructPage { + items: p.items.clone(), + next: p.next, + }), + None => Err(StorageError::Internal("read past the last page".to_owned())), + } + }) + } + fn prune_before<'a>( + &'a self, + _: &'a str, + _: SystemTime, + _: u32, + ) -> BoxFuture<'a, Result> { + unimplemented!() + } + fn pitr_tables(&self) -> BoxFuture<'_, Result, StorageError>> { + unimplemented!() + } + fn remove_orphaned_history(&self, _: u32) -> BoxFuture<'_, Result> { + unimplemented!() + } +} + +fn item(pk: &str) -> Item { + Item::from([("pk".to_owned(), AttributeValue::S(pk.to_owned()))]) +} + +#[tokio::test] +async fn the_stream_follows_the_cursors_and_stops_after_the_last_page() { + let store = Pages { + pages: vec![ + ReconstructPage { + items: vec![item("a"), item("b")], + next: Some(7), + }, + ReconstructPage { + items: vec![], + next: Some(9), + }, + ReconstructPage { + items: vec![item("c")], + next: None, + }, + ], + calls: Mutex::new(Vec::new()), + }; + let items: Vec = reconstruct_stream(&store, "t", at(1, 0), 2) + .map(|r| r.expect("item")) + .collect() + .await; + assert_eq!(items, vec![item("a"), item("b"), item("c")]); + assert_eq!( + *store.calls.lock().expect("calls"), + vec![None, Some(7), Some(9)] + ); +} + +#[tokio::test] +async fn a_failed_page_ends_the_stream_with_its_error() { + let store = Pages { + pages: vec![ReconstructPage { + items: vec![item("a")], + next: Some(1), + }], + calls: Mutex::new(Vec::new()), + }; + let got: Vec> = + reconstruct_stream(&store, "t", at(1, 0), 1).collect().await; + assert_eq!(got.len(), 2); + assert!(got[0].is_ok()); + assert!(got[1].is_err()); +} + +fn source() -> TableDescription { + TableDescription { + table_name: "src".to_owned(), + table_id: "src-id".to_owned(), + table_arn: "arn:aws:dynamodb:us-east-1:000000000000:table/src".to_owned(), + key_schema: serde_json::from_value(json!([{"AttributeName": "pk", "KeyType": "HASH"}])) + .expect("keys"), + attribute_definitions: serde_json::from_value(json!([ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "g", "AttributeType": "S"} + ])) + .expect("attributes"), + billing_mode_summary: Some( + serde_json::from_value(json!({"BillingMode": "PAY_PER_REQUEST"})).expect("billing"), + ), + global_secondary_indexes: Some(vec![ + serde_json::from_value(json!({ + "IndexName": "gsi", + "KeySchema": [{"AttributeName": "g", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "ALL"}, + "IndexStatus": "ACTIVE", + "IndexSizeBytes": 0, + "ItemCount": 0, + "IndexArn": "arn:aws:dynamodb:us-east-1:000000000000:table/src/index/gsi" + })) + .expect("gsi"), + ]), + stream_specification: serde_json::from_value( + json!({"StreamEnabled": true, "StreamViewType": "NEW_IMAGE"}), + ) + .ok(), + ..Default::default() + } +} + +fn members(body: Value) -> RestoreMembers { + restore_members(&body, (3, 255)).expect("members") +} + +#[tokio::test] +async fn the_target_takes_the_source_definition_with_the_overrides() { + let source = source(); + let limits = LimitsConfig::default(); + let input = target_input( + &source, + &members(json!({"TargetTableName": "dst"})), + &limits, + ) + .expect("input"); + assert_eq!(input.table_name, "dst"); + assert_eq!(input.key_schema, source.key_schema); + assert_eq!( + input.billing_mode, + Some(extenddb_core::types::BillingMode::PayPerRequest) + ); + assert_eq!( + input + .global_secondary_indexes + .as_ref() + .map(|g| g.iter().map(|i| i.index_name.clone()).collect::>()), + Some(vec!["gsi".to_owned()]) + ); + assert!(input.stream_specification.is_none()); + + let dropped = target_input( + &source, + &members(json!({"TargetTableName": "dst", "GlobalSecondaryIndexOverride": []})), + &limits, + ) + .expect("input"); + assert!(dropped.global_secondary_indexes.is_none()); + + let refused = target_input( + &source, + &members(json!({"TargetTableName": "dst", "BillingModeOverride": "PROVISIONED"})), + &limits, + ); + match refused { + Err(DynamoDbError::ValidationException(m)) => assert_eq!( + m, + "Invalid Request: Must override ProvisionedThroughput if BillingMode is overridden \ + to PROVISIONED" + ), + other => panic!("{other:?}"), + } + + let gsi = target_input( + &source, + &members(json!({ + "TargetTableName": "dst", + "BillingModeOverride": "PROVISIONED", + "ProvisionedThroughputOverride": {"ReadCapacityUnits": 5, "WriteCapacityUnits": 5} + })), + &limits, + ); + match gsi { + Err(DynamoDbError::ValidationException(m)) => assert_eq!( + m, + "Invalid Request: Must override ProvisionedThroughput if BillingMode is overridden \ + to PROVISIONED for index gsi" + ), + other => panic!("{other:?}"), + } +} From 62889b9835da8cc7c6dd02a758f4627826ce5fea Mon Sep 17 00:00:00 2001 From: Elise Ong Date: Tue, 10 Nov 2026 12:25:22 -0500 Subject: [PATCH 5/6] ci: build, lint and test the point-in-time recovery features No job built the pitr features, so the item-history stores, the write hooks and the continuous-backups path of the engine were neither linted nor tested. clippy gains a pass with them on; the unit test job runs the storage, SQLite and engine crates with them; the PostgreSQL storage job runs tests/pitr_item_history.rs, which fails rather than skips under EXTENDDB_TEST_PG_REQUIRED; a new build job builds a SQLite and a PostgreSQL server with pitr once, and a new test job runs the continuous-backups wire tests against each, with PostgreSQL started only on the PostgreSQL leg and the server's log printed on failure. actionlint reports nothing new. --- .github/workflows/clippy.yml | 4 + .github/workflows/integration.yml | 134 +++++++++++++++++++++++++++++- .github/workflows/test.yml | 3 + 3 files changed, 140 insertions(+), 1 deletion(-) diff --git a/.github/workflows/clippy.yml b/.github/workflows/clippy.yml index fafd81908..aba9106f7 100644 --- a/.github/workflows/clippy.yml +++ b/.github/workflows/clippy.yml @@ -40,3 +40,7 @@ jobs: # `devtools/backup-kill-points.sh` runs by hand on a build with the # feature; no other job builds it. - run: cargo clippy --all-targets --features extenddb-engine/backup-test-hooks -- -D warnings + # Point-in-time recovery (the `pitr` features), off in every other + # build: the engine's operations, the item-history stores and their + # write hooks on SQLite and PostgreSQL. + - run: cargo clippy --all-targets --features extenddb/pitr,extenddb-storage-sqlite/pitr,extenddb-storage-postgres/pitr -- -D warnings diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 071500c00..760f10f2c 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -781,6 +781,14 @@ jobs: EXTENDDB_TEST_PG_REQUIRED: "1" run: cargo test --release -p extenddb-storage-postgres --tests + # The item-history store's checks, compiled only with the crate's + # `pitr` feature. + - name: Run the PostgreSQL point-in-time recovery store tests + env: + EXTENDDB_TEST_PG_CONNECTION_STRING: postgresql://postgres:devpass@127.0.0.1:5432 + EXTENDDB_TEST_PG_REQUIRED: "1" + run: cargo test --release -p extenddb-storage-postgres --features pitr --test pitr_item_history + # The runtime-detection proof: the same binary, against a PostgreSQL with no # pgvector, must refuse vector indexes over the wire. Today the job above runs # on a plain image too, so this looks like a duplicate of it; it is not, because @@ -966,6 +974,128 @@ jobs: if: failure() run: sudo journalctl -t extenddb --no-pager -n 500 || true + # A build with point-in-time recovery (the `pitr` features), which no other + # build job makes: one binary per backend whose item-history store it + # selects. + build-pitr: + runs-on: extenddb_ubuntu-2404_4-core + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + backend: [sqlite, postgres] + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: stable + + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + # The two legs compile different features; a key per leg keeps each + # leg's cache its own. + key: ${{ matrix.backend }}-pitr + cache-on-failure: true + # Only pushes to main save, so other runs restore main's cache. + save-if: ${{ github.ref == 'refs/heads/main' }} + + - name: Build release (${{ matrix.backend }},pitr) + run: cargo build --release -p extenddb --no-default-features --features ${{ matrix.backend }},pitr + + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: extenddb-${{ matrix.backend }}-pitr + path: target/release/extenddb + if-no-files-found: error + retention-days: 1 + # A re-run of a failed job uploads the same name again. + overwrite: true + + # The continuous-backups wire tests on the pitr builds, SQLite and + # PostgreSQL. test_continuous_backups.py skips here only the cells whose + # answer the feature changes; test_pitr_enable.py skips on every other job. + run-pitr-wire: + runs-on: extenddb_ubuntu-2404_4-core + timeout-minutes: 40 + needs: build-pitr + strategy: + fail-fast: false + matrix: + include: + - backend: sqlite + init: --backend sqlite + - backend: postgres + init: --pg-host 127.0.0.1 --pg-port 5432 --pg-user postgres --pg-pass devpass + services: + postgres: + # Services take no `if`. The runner skips a service whose image is + # empty, so only the postgres leg starts the container. + image: ${{ matrix.backend == 'postgres' && 'pgvector/pgvector:pg16' || '' }} + env: + POSTGRES_PASSWORD: devpass + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 5s + --health-timeout 5s + --health-retries 5 + ports: + - 127.0.0.1:5432:5432 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: extenddb-${{ matrix.backend }}-pitr + path: target/release + + # Artifacts do not keep file modes. + - run: chmod +x target/release/extenddb + + - name: Initialize ExtendDB + id: init + run: | + output=$(./target/release/extenddb init ${{ matrix.init }} --config extenddb.toml 2>&1) + echo "$output" + password=$(echo "$output" | grep -oP 'Password: \K\S+') + echo "admin_password=$password" >> "$GITHUB_OUTPUT" + + # A foreground server logs to stderr, so its log is this file. + - name: Start ExtendDB + run: | + ./target/release/extenddb serve --config extenddb.toml --foreground > extenddb-server.log 2>&1 & + for i in $(seq 1 30); do + if curl -sk https://127.0.0.1:18443/health | python3 -c 'import json, sys; sys.exit(0 if json.load(sys.stdin).get("status") == "healthy" else 1)' 2>/dev/null; then + echo "Server ready" + exit 0 + fi + sleep 1 + done + echo "Server failed to start" + curl -sk https://127.0.0.1:18443/health || true + exit 1 + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Python dependencies + run: pip install -r requirements.txt + + - name: Run the continuous-backups wire tests + env: + EXTENDDB_TEST_ENDPOINT: https://127.0.0.1:18443 + EXTENDDB_ADMIN_USER: admin + EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }} + EXTENDDB_TEST_PITR: "1" + run: | + eval "$(python3 devtools/provision-test-credentials)" + python3 -m pytest tests/test_pitr_enable.py tests/test_continuous_backups.py -v -rs -p no:cacheprovider + + - name: Print the server log on failure + if: failure() + run: cat extenddb-server.log || true + integration: runs-on: extenddb_ubuntu-2404_4-core timeout-minutes: 40 @@ -979,6 +1109,7 @@ jobs: run-postgres-storage-tests, run-rust-integration-postgres-novector, run-rust-integration-sqlite, + run-pitr-wire, ] if: always() steps: @@ -990,7 +1121,8 @@ jobs: [ "${{ needs.run-rust-integration.result }}" != "success" ] || \ [ "${{ needs.run-postgres-storage-tests.result }}" != "success" ] || \ [ "${{ needs.run-rust-integration-postgres-novector.result }}" != "success" ] || \ - [ "${{ needs.run-rust-integration-sqlite.result }}" != "success" ]; then + [ "${{ needs.run-rust-integration-sqlite.result }}" != "success" ] || \ + [ "${{ needs.run-pitr-wire.result }}" != "success" ]; then exit 1 fi diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ef1b68230..d95c4654f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,6 +64,9 @@ jobs: # AWS_ACCESS_KEY_ID in dev mode, and a test of that warning would see # these). - run: cargo test --locked --workspace --exclude extenddb-backup-store + # The same crates with point-in-time recovery on: the shared item-history + # checks on SQLite, the engine's operations and the write hooks. + - run: cargo test --locked -p extenddb-storage -p extenddb-storage-sqlite -p extenddb-engine --features extenddb-storage-sqlite/pitr,extenddb-engine/pitr - run: cargo test --locked -p extenddb-backup-store env: EXTENDDB_TEST_S3_REQUIRED: "1" From 74a4e29bb2bbb092145698cfcea16531f404cb6c Mon Sep 17 00:00:00 2001 From: Elise Ong Date: Tue, 10 Nov 2026 14:59:51 -0500 Subject: [PATCH 6/6] fix(storage-postgres): take history commit times under the row lock A reconstruct keeps, for each key, the row with the highest seq among the rows committed at or before the restore time. A put or delete with no condition, index or stream read the ExtendDB server's clock before its statement took the item's row lock, so a racing write could take the lower time and the higher seq. With 16 writers on one key for 20 s, 2,062 and 2,117 of about 11,800 rows had a time earlier than the row before them, by up to 19 ms; a restore inside such a gap returns a value committed after the restore time. Two ExtendDB servers whose clocks were 5 s apart gave gaps of 5 s on every path. On the service, two writes of one key 400 ms apart within one second restore to the first at the midpoint and to the second after it (two samples). Every PostgreSQL history row now reads the database server's clock, clock_timestamp(), after its data write: the autocommit put and delete in their own statement, the other writes in the history insert, and a transaction once, at its first recorded member, for all of its members. Every ExtendDB server of one database reads the same clock. SQLite reads the server's clock under its write lock and is unchanged (0 of about 11,800 rows, two samples). Verified: racing_writes_of_one_key_take_their_times_in_seq_order passes three runs in a row and fails with the earlier clock read (546 and 533 of about 2,300 rows); the same 16-writer load gives 0 inversions on both paths, and 0 with one of two servers 5 s ahead (two samples each); the PostgreSQL storage tests pass with and without the feature (400 and 439); release-build put throughput on a table without recovery is within 1.3 percent of a build without the feature (sequential and 8 writers, 6 runs each). --- .../storage-postgres/src/data/delete_item.rs | 18 +------ crates/storage-postgres/src/data/put_item.rs | 20 ++----- .../storage-postgres/src/data/transactions.rs | 9 ++-- .../storage-postgres/src/data/update_item.rs | 9 +--- crates/storage-postgres/src/pitr.rs | 47 ++++++++++------ .../storage-postgres/src/pitr/autocommit.rs | 26 ++++----- .../tests/pitr_item_history.rs | 53 +++++++++++++++++++ 7 files changed, 106 insertions(+), 76 deletions(-) diff --git a/crates/storage-postgres/src/data/delete_item.rs b/crates/storage-postgres/src/data/delete_item.rs index 6ff44b17c..59031cbd5 100755 --- a/crates/storage-postgres/src/data/delete_item.rs +++ b/crates/storage-postgres/src/data/delete_item.rs @@ -186,14 +186,7 @@ impl PostgresEngine { } // Reached only with a row deleted: a missing item returned above. #[cfg(feature = "pitr")] - crate::pitr::record_item_history_in_tx( - &mut tx, - key_info, - key, - None, - std::time::SystemTime::now(), - ) - .await?; + crate::pitr::record_item_history_in_tx(&mut tx, key_info, key, None, None).await?; // Persist async GSI work inside the same transaction — one row // per async index, each honoring its own propagation delay. let mut async_enqueued = enqueue_async_indexes( @@ -372,14 +365,7 @@ impl PostgresEngine { } // Reached only with a row deleted: a missing item returned above. #[cfg(feature = "pitr")] - crate::pitr::record_item_history_in_tx( - &mut tx, - key_info, - key, - None, - std::time::SystemTime::now(), - ) - .await?; + crate::pitr::record_item_history_in_tx(&mut tx, key_info, key, None, None).await?; // Persist async GSI work inside the same transaction — one row // per async index, each honoring its own propagation delay. let mut async_enqueued = enqueue_async_indexes( diff --git a/crates/storage-postgres/src/data/put_item.rs b/crates/storage-postgres/src/data/put_item.rs index c2d1edf22..92034a9b4 100755 --- a/crates/storage-postgres/src/data/put_item.rs +++ b/crates/storage-postgres/src/data/put_item.rs @@ -204,14 +204,8 @@ impl PostgresEngine { .await?; } #[cfg(feature = "pitr")] - crate::pitr::record_item_history_in_tx( - &mut tx, - key_info, - &item, - Some(&item), - std::time::SystemTime::now(), - ) - .await?; + crate::pitr::record_item_history_in_tx(&mut tx, key_info, &item, Some(&item), None) + .await?; // Persist async GSI work inside the same transaction — one row // per async index, each honoring its own propagation delay. let mut async_enqueued = enqueue_async_indexes( @@ -405,14 +399,8 @@ impl PostgresEngine { .await?; } #[cfg(feature = "pitr")] - crate::pitr::record_item_history_in_tx( - &mut tx, - key_info, - &item, - Some(&item), - std::time::SystemTime::now(), - ) - .await?; + crate::pitr::record_item_history_in_tx(&mut tx, key_info, &item, Some(&item), None) + .await?; // Persist async GSI work inside the same transaction — one row // per async index, each honoring its own propagation delay. let mut async_enqueued = enqueue_async_indexes( diff --git a/crates/storage-postgres/src/data/transactions.rs b/crates/storage-postgres/src/data/transactions.rs index c983d9ed4..cccd9128d 100644 --- a/crates/storage-postgres/src/data/transactions.rs +++ b/crates/storage-postgres/src/data/transactions.rs @@ -230,11 +230,12 @@ impl PostgresEngine { }); } - // Item history for every member at one instant, so a reconstruct - // holds all of the transaction or none of it. + // Item history for every member at one instant, read after every + // member's write, so a reconstruct holds all of the transaction or + // none of it. #[cfg(feature = "pitr")] { - let commit_time = std::time::SystemTime::now(); + let mut commit_time = None; for (op, (old_item, new_item)) in ops.iter().zip(op_items.iter()) { let (TransactWriteOp::Put { key_info, .. } | TransactWriteOp::Delete { key_info, .. } @@ -245,7 +246,7 @@ impl PostgresEngine { key_info, old_item.as_ref(), new_item.as_ref(), - commit_time, + &mut commit_time, ) .await .map_err(cancel_all)?; diff --git a/crates/storage-postgres/src/data/update_item.rs b/crates/storage-postgres/src/data/update_item.rs index b32080a91..82e890f33 100755 --- a/crates/storage-postgres/src/data/update_item.rs +++ b/crates/storage-postgres/src/data/update_item.rs @@ -330,14 +330,7 @@ impl PostgresEngine { .await?; } #[cfg(feature = "pitr")] - crate::pitr::record_item_history_in_tx( - &mut tx, - key_info, - &item, - Some(&item), - std::time::SystemTime::now(), - ) - .await?; + crate::pitr::record_item_history_in_tx(&mut tx, key_info, &item, Some(&item), None).await?; // Persist async GSI work inside the same transaction — one row per // async index, each honoring its own propagation delay. let mut async_enqueued = enqueue_async_indexes( diff --git a/crates/storage-postgres/src/pitr.rs b/crates/storage-postgres/src/pitr.rs index 7b9f4f69d..878323a5b 100644 --- a/crates/storage-postgres/src/pitr.rs +++ b/crates/storage-postgres/src/pitr.rs @@ -16,6 +16,11 @@ //! statement takes the data table's lock before it takes its snapshot, so a //! write held by an enable reads the `pitr_state` row the enable committed. //! +//! A row's commit time is read from the PostgreSQL server's clock +//! ([`DB_NOW_US`]) after the data write, so under the item's row lock: two +//! writes of one key take their times in the order they take their `seq`, +//! and every ExtendDB server of the database reads one clock. +//! //! A page first pins its time in `pitr_state` and a prune lowers its cutoff //! to a pin that holds; both lock the table's `pitr_state` row, so a prune //! either sees the pin or has already moved `pruned_to_us` past the time and @@ -39,6 +44,10 @@ use crate::data::{data_table_name, json_to_item}; mod autocommit; pub(crate) use autocommit::{autocommit_delete, autocommit_put}; +/// The PostgreSQL server's clock in microseconds since the Unix epoch, read +/// when the expression is evaluated. +pub(crate) const DB_NOW_US: &str = "(extract(epoch FROM clock_timestamp()) * 1000000)::bigint"; + /// Superseded: a later row of the same key committed at or before `$2`. const SUPERSEDED: &str = "EXISTS (SELECT 1 FROM item_history n \ WHERE n.table_id = h.table_id AND n.pk = h.pk AND n.sk = h.sk \ @@ -89,15 +98,16 @@ fn sk_sql(sk: Option) -> &'static str { /// Record one committed mutation of the table `key_info` names, in the /// mutation's data transaction, after its data write. `key` is any image that /// carries the primary key; `image` is the item after the mutation, `None` -/// for a delete. Returns whether a row was added, which is when the table's +/// for a delete. `commit_time` `None` reads [`DB_NOW_US`]. Returns the +/// recorded commit time when a row was added, which is when the table's /// recovery is enabled. pub(crate) async fn record_item_history_in_tx( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, key_info: &TableKeyInfo, key: &Item, image: Option<&Item>, - commit_time: SystemTime, -) -> Result { + commit_time: Option, +) -> Result, StorageError> { let pk_name = &key_info.key_schema[0].attribute_name; let pk_value = key .get(pk_name) @@ -116,19 +126,20 @@ pub(crate) async fn record_item_history_in_tx( .map(serde_json::to_value) .transpose() .map_err(|e| StorageError::Internal(e.to_string()))?; - let done = sqlx::query( + let commit_time = commit_time.map(unix_micros).transpose()?; + sqlx::query_scalar(&format!( "INSERT INTO item_history (table_id, pk, sk, commit_time_us, item_data) \ - SELECT $1, $2, $3, $4, $5 WHERE EXISTS (SELECT 1 FROM pitr_state WHERE table_id = $1)", - ) + SELECT $1, $2, $3, coalesce($4, {DB_NOW_US}), $5 \ + WHERE EXISTS (SELECT 1 FROM pitr_state WHERE table_id = $1) RETURNING commit_time_us" + )) .bind(&key_info.table_id) .bind(pk_text.as_ref()) .bind(sk) - .bind(unix_micros(commit_time)?) + .bind(commit_time) .bind(item_json) - .execute(&mut **tx) + .fetch_optional(&mut **tx) .await - .map_err(|e| store_err("record", e))?; - Ok(done.rows_affected() == 1) + .map_err(|e| store_err("record", e)) } /// The write paths' hook: record a mutation from its before and after images, @@ -136,19 +147,23 @@ pub(crate) async fn record_item_history_in_tx( /// neither image changed nothing (a delete of a missing item, a condition /// check) and records nothing: a delete of a missing key holds no row lock, so /// its row could take a higher `seq` than a concurrent first put of the key -/// that commits after it. Every mutation of one transaction passes the same -/// `commit_time`, so a reconstruct never splits a transaction. +/// that commits after it. The mutations of one transaction share +/// `commit_time`: the first one recorded reads [`DB_NOW_US`] and sets it, so +/// a reconstruct never splits a transaction. pub(crate) async fn capture_in_tx( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, key_info: &TableKeyInfo, old: Option<&Item>, new: Option<&Item>, - commit_time: SystemTime, + commit_time: &mut Option, ) -> Result<(), StorageError> { let Some(key) = new.or(old) else { return Ok(()); }; - record_item_history_in_tx(tx, key_info, key, new, commit_time).await?; + let at = commit_time.map(from_unix_micros); + if let Some(recorded) = record_item_history_in_tx(tx, key_info, key, new, at).await? { + *commit_time = Some(recorded); + } Ok(()) } @@ -558,7 +573,9 @@ impl PostgresEngine { .begin() .await .map_err(|e| store_err("begin", e))?; - let added = record_item_history_in_tx(&mut tx, key_info, key, image, commit_time).await?; + let added = record_item_history_in_tx(&mut tx, key_info, key, image, Some(commit_time)) + .await? + .is_some(); if commit { tx.commit().await.map_err(|e| store_err("commit", e))?; } else { diff --git a/crates/storage-postgres/src/pitr/autocommit.rs b/crates/storage-postgres/src/pitr/autocommit.rs index 0c10979b8..ec8d50cb5 100644 --- a/crates/storage-postgres/src/pitr/autocommit.rs +++ b/crates/storage-postgres/src/pitr/autocommit.rs @@ -5,24 +5,22 @@ //! statement that also records the write's history row when the table's //! recovery is enabled. -use std::time::SystemTime; - use extenddb_core::types::{ScalarAttributeType, TableKeyInfo}; use extenddb_storage::error::StorageError; -use extenddb_storage::pitr::unix_micros; use extenddb_storage::util::{SortKeyValue, sk_column}; -use super::sk_text; +use super::{DB_NOW_US, sk_text}; use crate::data::data_table_name; use crate::data::index::db_error; /// The history row of a write that is its statement's only data row, added /// when the table has a `pitr_state` row. `$1` is the data pk, then -/// `$table`, `$sk`, `$time` and `$image` name the statement's parameters. -fn history_from(w: &str, table: &str, sk: &str, time: &str, image: &str) -> String { +/// `$table`, `$sk` and `$image` name the statement's parameters. The commit +/// time is read per row of `w`, so after the data write took the row lock. +fn history_from(w: &str, table: &str, sk: &str, image: &str) -> String { format!( "INSERT INTO item_history (table_id, pk, sk, commit_time_us, item_data) \ - SELECT {table}, $1, {sk}, {time}, {image} FROM {w} \ + SELECT {table}, $1, {sk}, {DB_NOW_US}, {image} FROM {w} \ WHERE EXISTS (SELECT 1 FROM pitr_state WHERE table_id = {table})" ) } @@ -37,18 +35,16 @@ pub(crate) async fn autocommit_put( item_json: &serde_json::Value, ) -> Result<(), StorageError> { let ddb_table = data_table_name(&key_info.table_id); - let commit_time = unix_micros(SystemTime::now())?; let Some(sk) = sk else { let sql = format!( "WITH w AS (INSERT INTO {ddb_table} (pk, item_data) VALUES ($1, $2) \ ON CONFLICT (pk) DO UPDATE SET item_data = EXCLUDED.item_data RETURNING 1) {}", - history_from("w", "$3", "''", "$4", "$2") + history_from("w", "$3", "''", "$2") ); sqlx::query(&sql) .bind(pk_text) .bind(item_json) .bind(&key_info.table_id) - .bind(commit_time) .execute(pool) .await .map_err(db_error)?; @@ -58,14 +54,13 @@ pub(crate) async fn autocommit_put( let sql = format!( "WITH w AS (INSERT INTO {ddb_table} (pk, {sk_col}, item_data) VALUES ($1, $2, $3) \ ON CONFLICT (pk, {sk_col}) DO UPDATE SET item_data = EXCLUDED.item_data RETURNING 1) {}", - history_from("w", "$4", "$5", "$6", "$3") + history_from("w", "$4", "$5", "$3") ); let query = bind_typed_sk(sqlx::query(&sql).bind(pk_text), sk); query .bind(item_json) .bind(&key_info.table_id) .bind(sk_text(sk)) - .bind(commit_time) .execute(pool) .await .map_err(db_error)?; @@ -82,16 +77,14 @@ pub(crate) async fn autocommit_delete( sk: Option<&SortKeyValue>, ) -> Result<(), StorageError> { let ddb_table = data_table_name(&key_info.table_id); - let commit_time = unix_micros(SystemTime::now())?; let Some(sk) = sk else { let sql = format!( "WITH w AS (DELETE FROM {ddb_table} WHERE pk = $1 RETURNING 1) {}", - history_from("w", "$2", "''", "$3", "NULL") + history_from("w", "$2", "''", "NULL") ); sqlx::query(&sql) .bind(pk_text) .bind(&key_info.table_id) - .bind(commit_time) .execute(pool) .await .map_err(|e| StorageError::Internal(e.to_string()))?; @@ -100,12 +93,11 @@ pub(crate) async fn autocommit_delete( let sk_col = sk_column(sk_type(sk)); let sql = format!( "WITH w AS (DELETE FROM {ddb_table} WHERE pk = $1 AND {sk_col} = $2 RETURNING 1) {}", - history_from("w", "$3", "$4", "$5", "NULL") + history_from("w", "$3", "$4", "NULL") ); bind_typed_sk(sqlx::query(&sql).bind(pk_text), sk) .bind(&key_info.table_id) .bind(sk_text(sk)) - .bind(commit_time) .execute(pool) .await .map_err(|e| StorageError::Internal(e.to_string()))?; diff --git a/crates/storage-postgres/tests/pitr_item_history.rs b/crates/storage-postgres/tests/pitr_item_history.rs index 25cb52ed1..43509ed03 100644 --- a/crates/storage-postgres/tests/pitr_item_history.rs +++ b/crates/storage-postgres/tests/pitr_item_history.rs @@ -120,6 +120,59 @@ async fn concurrent_writers_leave_the_state_equal_to_the_live_table() { close(s).await; } +/// Writers racing on one key: a key's history rows take their commit times +/// in the order of their `seq`. A row with a higher `seq` and an earlier time +/// would make a restore between the two times return the later write. +#[tokio::test] +async fn racing_writes_of_one_key_take_their_times_in_seq_order() { + use extenddb_core::expression::ExpressionMaps; + let Some(s) = open().await else { return }; + let info = s.table("t", None, true).await; + let engine = std::sync::Arc::new(s); + let mut tasks = Vec::new(); + for task in 0..16_u64 { + let engine = engine.clone(); + let info = info.clone(); + tasks.push(tokio::spawn(async move { + let maps = ExpressionMaps::default(); + let key = Item::from([("pk".to_owned(), AttributeValue::S("k".to_owned()))]); + for i in 0..150_u64 { + if i % 10 == 9 { + engine + .engine + .delete_item(&info, &key, false, None, &maps, None) + .await + .expect("delete"); + } else { + let mut item = key.clone(); + item.insert("v".to_owned(), AttributeValue::S(format!("{task}-{i}"))); + engine + .engine + .put_item(&info, item, false, None, &maps, None) + .await + .expect("put"); + } + } + })); + } + for t in tasks { + t.await.expect("writer"); + } + let s = std::sync::Arc::into_inner(engine).expect("sole owner"); + let (rows, inversions): (i64, i64) = sqlx::query_as( + "SELECT count(*), count(*) FILTER (WHERE commit_time_us < prev) FROM \ + (SELECT commit_time_us, lag(commit_time_us) OVER (PARTITION BY pk, sk ORDER BY seq) \ + AS prev FROM item_history WHERE table_id = $1) x", + ) + .bind(&info.table_id) + .fetch_one(&s.db) + .await + .expect("count"); + assert!(rows > 2_000, "{rows} rows"); + assert_eq!(inversions, 0, "{inversions} of {rows} rows"); + close(s).await; +} + /// A delete of a key whose first put is still uncommitted deletes nothing and /// records nothing. The delete takes no row lock, so a recorded delete would /// draw a higher `seq` than the put that commits after it, and the reconstruct