feat: point-in-time recovery capture, enable, disable and describe on SQLite and PostgreSQL

.github/workflows/clippy.yml

@@ -40,3 +40,7 @@ jobs:
       # `devtools/backup-kill-points.sh` runs by hand on a build with the
       # feature; no other job builds it.
       - run: cargo clippy --all-targets --features extenddb-engine/backup-test-hooks -- -D warnings
+      # Point-in-time recovery (the `pitr` features), off in every other
+      # build: the engine's operations, the item-history stores and their
+      # write hooks on SQLite and PostgreSQL.
+      - run: cargo clippy --all-targets --features extenddb/pitr,extenddb-storage-sqlite/pitr,extenddb-storage-postgres/pitr -- -D warnings

.github/workflows/integration.yml

@@ -781,6 +781,14 @@ jobs:
           EXTENDDB_TEST_PG_REQUIRED: "1"
         run: cargo test --release -p extenddb-storage-postgres --tests
 
+      # The item-history store's checks, compiled only with the crate's
+      # `pitr` feature.
+      - name: Run the PostgreSQL point-in-time recovery store tests
+        env:
+          EXTENDDB_TEST_PG_CONNECTION_STRING: postgresql://postgres:devpass@127.0.0.1:5432
+          EXTENDDB_TEST_PG_REQUIRED: "1"
+        run: cargo test --release -p extenddb-storage-postgres --features pitr --test pitr_item_history
+
   # The runtime-detection proof: the same binary, against a PostgreSQL with no
   # pgvector, must refuse vector indexes over the wire. Today the job above runs
   # on a plain image too, so this looks like a duplicate of it; it is not, because
@@ -966,6 +974,128 @@ jobs:
         if: failure()
         run: sudo journalctl -t extenddb --no-pager -n 500 || true
 
+  # A build with point-in-time recovery (the `pitr` features), which no other
+  # build job makes: one binary per backend whose item-history store it
+  # selects.
+  build-pitr:
+    runs-on: extenddb_ubuntu-2404_4-core
+    timeout-minutes: 40
+    strategy:
+      fail-fast: false
+      matrix:
+        backend: [sqlite, postgres]
+    steps:
+      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
+
+      - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
+        with:
+          toolchain: stable
+
+      - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
+        with:
+          # The two legs compile different features; a key per leg keeps each
+          # leg's cache its own.
+          key: ${{ matrix.backend }}-pitr
+          cache-on-failure: true
+          # Only pushes to main save, so other runs restore main's cache.
+          save-if: ${{ github.ref == 'refs/heads/main' }}
+
+      - name: Build release (${{ matrix.backend }},pitr)
+        run: cargo build --release -p extenddb --no-default-features --features ${{ matrix.backend }},pitr
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+        with:
+          name: extenddb-${{ matrix.backend }}-pitr
+          path: target/release/extenddb
+          if-no-files-found: error
+          retention-days: 1
+          # A re-run of a failed job uploads the same name again.
+          overwrite: true
+
+  # The continuous-backups wire tests on the pitr builds, SQLite and
+  # PostgreSQL. test_continuous_backups.py skips here only the cells whose
+  # answer the feature changes; test_pitr_enable.py skips on every other job.
+  run-pitr-wire:
+    runs-on: extenddb_ubuntu-2404_4-core
+    timeout-minutes: 40
+    needs: build-pitr
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - backend: sqlite
+            init: --backend sqlite
+          - backend: postgres
+            init: --pg-host 127.0.0.1 --pg-port 5432 --pg-user postgres --pg-pass devpass
+    services:
+      postgres:
+        # Services take no `if`. The runner skips a service whose image is
+        # empty, so only the postgres leg starts the container.
+        image: ${{ matrix.backend == 'postgres' && 'pgvector/pgvector:pg16' || '' }}
+        env:
+          POSTGRES_PASSWORD: devpass
+        options: >-
+          --health-cmd "pg_isready -U postgres"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 127.0.0.1:5432:5432
+    steps:
+      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
+
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
+        with:
+          name: extenddb-${{ matrix.backend }}-pitr
+          path: target/release
+
+      # Artifacts do not keep file modes.
+      - run: chmod +x target/release/extenddb
+
+      - name: Initialize ExtendDB
+        id: init
+        run: |
+          output=$(./target/release/extenddb init ${{ matrix.init }} --config extenddb.toml 2>&1)
+          echo "$output"
+          password=$(echo "$output" | grep -oP 'Password: \K\S+')
+          echo "admin_password=$password" >> "$GITHUB_OUTPUT"
+
+      # A foreground server logs to stderr, so its log is this file.
+      - name: Start ExtendDB
+        run: |
+          ./target/release/extenddb serve --config extenddb.toml --foreground > extenddb-server.log 2>&1 &
+          for i in $(seq 1 30); do
+            if curl -sk https://127.0.0.1:18443/health | python3 -c 'import json, sys; sys.exit(0 if json.load(sys.stdin).get("status") == "healthy" else 1)' 2>/dev/null; then
+              echo "Server ready"
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "Server failed to start"
+          curl -sk https://127.0.0.1:18443/health || true
+          exit 1
+
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Install Python dependencies
+        run: pip install -r requirements.txt
+
+      - name: Run the continuous-backups wire tests
+        env:
+          EXTENDDB_TEST_ENDPOINT: https://127.0.0.1:18443
+          EXTENDDB_ADMIN_USER: admin
+          EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }}
+          EXTENDDB_TEST_PITR: "1"
+        run: |
+          eval "$(python3 devtools/provision-test-credentials)"
+          python3 -m pytest tests/test_pitr_enable.py tests/test_continuous_backups.py -v -rs -p no:cacheprovider
+
+      - name: Print the server log on failure
+        if: failure()
+        run: cat extenddb-server.log || true
+
   integration:
     runs-on: extenddb_ubuntu-2404_4-core
     timeout-minutes: 40
@@ -979,6 +1109,7 @@ jobs:
         run-postgres-storage-tests,
         run-rust-integration-postgres-novector,
         run-rust-integration-sqlite,
+        run-pitr-wire,
       ]
     if: always()
     steps:
@@ -990,7 +1121,8 @@ jobs:
              [ "${{ needs.run-rust-integration.result }}" != "success" ] || \
              [ "${{ needs.run-postgres-storage-tests.result }}" != "success" ] || \
              [ "${{ needs.run-rust-integration-postgres-novector.result }}" != "success" ] || \
-             [ "${{ needs.run-rust-integration-sqlite.result }}" != "success" ]; then
+             [ "${{ needs.run-rust-integration-sqlite.result }}" != "success" ] || \
+             [ "${{ needs.run-pitr-wire.result }}" != "success" ]; then
             exit 1
           fi
 

.github/workflows/test.yml

@@ -64,6 +64,9 @@ jobs:
       # AWS_ACCESS_KEY_ID in dev mode, and a test of that warning would see
       # these).
       - run: cargo test --locked --workspace --exclude extenddb-backup-store
+      # The same crates with point-in-time recovery on: the shared item-history
+      # checks on SQLite, the engine's operations and the write hooks.
+      - run: cargo test --locked -p extenddb-storage -p extenddb-storage-sqlite -p extenddb-engine --features extenddb-storage-sqlite/pitr,extenddb-engine/pitr
       - run: cargo test --locked -p extenddb-backup-store
         env:
           EXTENDDB_TEST_S3_REQUIRED: "1"

CONTRIBUTING.md

@@ -359,28 +359,29 @@ SQLite, PostgreSQL, Cassandra and MongoDB migrate through this framework.
 
 ### Example: a nullable column on `tables`
 
-Migration 10, `table_note` (10 being the next free id at the time of writing;
-migrations 6, 7, 8 and 9 are `backup_catalog_index`, `stream_pending`,
-`restore_progress` and `warm_throughput`), adds a nullable `note TEXT` column
-to the catalog's `tables` table on SQLite and PostgreSQL.
+Migration 11, `table_note` (11 being the next free id at the time of writing;
+migrations 6 to 10 are `backup_catalog_index`, `stream_pending`,
+`restore_progress`, `warm_throughput` and `point_in_time_recovery`), adds a
+nullable `note TEXT` column to the catalog's `tables` table on SQLite and
+PostgreSQL.
 
 In `crates/storage/src/migrations/registry.rs`, at the end of the
 `migration_ids!` list:
 
 ```rust
     /// Adds a nullable `note` column to the `tables` catalog table.
-    TableNote = 10, "table_note",
+    TableNote = 11, "table_note",
         "add a nullable note column to the tables catalog";
 ```
 
 PostgreSQL has `ADD COLUMN IF NOT EXISTS`, so its body is a file,
-`crates/storage-postgres/migrations/0010_table_note.sql`:
+`crates/storage-postgres/migrations/0011_table_note.sql`:
 
 ```sql
 -- Copyright 2026 ExtendDB contributors
 -- SPDX-License-Identifier: Apache-2.0
 --
--- Migration 10, table_note, PostgreSQL catalog database: a nullable note
+-- Migration 11, table_note, PostgreSQL catalog database: a nullable note
 -- column on tables. IF NOT EXISTS makes a second run change nothing.
 
 ALTER TABLE tables ADD COLUMN IF NOT EXISTS note TEXT;
@@ -395,7 +396,7 @@ In `crates/storage-postgres/src/migrations/mod.rs`, the arm in `migration()`:
             bodies: vec![file_body!(
                 StoreKind::Catalog,
                 "migrations",
-                "0010_table_note.sql"
+                "0011_table_note.sql"
             )],
         },
 ```
@@ -417,13 +418,13 @@ and the id in the transactional arm of `runs_in()`:
 SQLite has no `ADD COLUMN IF NOT EXISTS`, and a file with a plain
 `ADD COLUMN` fails when it runs a second time, so the SQLite body is code
 that checks for the column first.
-`crates/storage-sqlite/src/migrations/m0010_table_note.rs`:
+`crates/storage-sqlite/src/migrations/m0011_table_note.rs`:
 
 ```rust
 // Copyright 2026 ExtendDB contributors
 // SPDX-License-Identifier: Apache-2.0
 
-//! Migration 10, `table_note`, SQLite: a nullable `note` column on `tables`.
+//! Migration 11, `table_note`, SQLite: a nullable `note` column on `tables`.
 //!
 //! SQLite has no `ADD COLUMN IF NOT EXISTS`, so the body reads the table's
 //! columns first and adds the column only when it is missing. A second run
@@ -435,10 +436,10 @@ use extenddb_storage::migrations::{CodeBody, Done, MigrationError};
 use super::SqliteStores;
 
 /// This file, whose SHA-256 the checksum test pins.
-pub(crate) const SOURCE: &str = include_str!("m0010_table_note.rs");
+pub(crate) const SOURCE: &str = include_str!("m0011_table_note.rs");
 
 /// Recorded as this migration's checksum. Changes only with a new migration.
-const FINGERPRINT: &str = "0010_table_note/sqlite/1";
+const FINGERPRINT: &str = "0011_table_note/sqlite/1";
 
 pub(crate) struct TableNote;
 
@@ -477,7 +478,7 @@ beside the others:
 ```rust
 #[rustfmt::skip]
 #[allow(clippy::all, deprecated)]
-mod m0010_table_note;
+mod m0011_table_note;
 ```
 
 and the arm in `migration()`:
@@ -488,8 +489,8 @@ and the arm in `migration()`:
             convergent: true,
             bodies: vec![StoreBody::code(
                 StoreKind::Catalog,
-                &m0010_table_note::TableNote,
-                m0010_table_note::SOURCE,
+                &m0011_table_note::TableNote,
+                m0011_table_note::SOURCE,
             )],
         },
 ```
@@ -501,8 +502,8 @@ rows of `tables`:
 
 ```rust
 #[tokio::test]
-async fn migration_10_adds_the_note_column_to_tables() {
-    let db = temp_db("m10");
+async fn migration_11_adds_the_note_column_to_tables() {
+    let db = temp_db("m11");
     load_dump(&db, DEV_0_1_14).await;
     run(&db).await.expect("migrate");
     let mut text = dump(
@@ -512,12 +513,12 @@ async fn migration_10_adds_the_note_column_to_tables() {
     )
     .await;
     text.push_str(&dump(&db, "SELECT table_name, note FROM tables ORDER BY table_name").await);
-    golden("0010_table_note", &text);
+    golden("0011_table_note", &text);
 }
 ```
 
-`EXTENDDB_BLESS_EXPECTED=1 cargo test -p extenddb-storage-sqlite --lib migration_10`
-writes `crates/storage-sqlite/tests/fixtures/golden/0010_table_note.txt`, one
+`EXTENDDB_BLESS_EXPECTED=1 cargo test -p extenddb-storage-sqlite --lib migration_11`
+writes `crates/storage-sqlite/tests/fixtures/golden/0011_table_note.txt`, one
 tab-separated line per row under each query:
 
 ```
@@ -532,15 +533,15 @@ The PostgreSQL body is a file, so it has no golden test.
 The example declares `Kind::Breaking`. A nullable column can be additive by
 the rule in step 4, but declaring it additive needs the old-binary job, which
 this example does not add. As a breaking migration it makes a 1.x binary
-without migration 10 refuse a catalog that has it, and a rollback restores the
+without migration 11 refuse a catalog that has it, and a rollback restores the
 backup.
 
 The same PR changes the code that reads and writes `tables.note` in each
 backend's catalog store, and commits the `docs/schema/sqlite.txt` and
 `docs/schema/postgres.txt` that the integration jobs regenerate. It changes
 no per-table DDL, because `tables` is a catalog table, and adds no `RELEASED`
-entry. `extenddb migrate --list` then shows migration 10, and on a catalog at
-migration 9 `extenddb migrate` lists migration 10 as pending and exits 3.
+entry. `extenddb migrate --list` then shows migration 11, and on a catalog at
+migration 10 `extenddb migrate` lists migration 11 as pending and exits 3.
 
 ### Running the migration tests
 

crates/app/Cargo.toml

@@ -14,6 +14,7 @@ license.workspace = true
 dev-mode = ["extenddb-config/dev-mode"]
 mongodb-test-hooks = ["extenddb-server/mongodb-test-hooks"]
 backup-test-hooks = ["extenddb-server/backup-test-hooks"]
+pitr = ["extenddb-server/pitr"]
 
 [dependencies]
 extenddb-auth = { workspace = true }

crates/bin/Cargo.toml

@@ -30,6 +30,14 @@ mongodb-test-hooks = [
 # requires a dev backend (`sqlite`/`sqlite-memory`) and fails to build with a
 # production backend like `postgres` (enforced at compile time in `main.rs`).
 dev-mode = ["extenddb-app/dev-mode"]
+# Point-in-time recovery, off by default: the engine's operations and the
+# item-history store of the backend this build selects. A backend without a
+# store (Cassandra, MongoDB) keeps the unavailable answer.
+pitr = [
+    "extenddb-app/pitr",
+    "extenddb-storage-sqlite?/pitr",
+    "extenddb-storage-postgres?/pitr",
+]
 
 [dependencies]
 extenddb-app = { workspace = true }

crates/core/src/error/mod.rs

@@ -113,6 +113,10 @@ pub enum DynamoDbError {
     /// on `UpdateContinuousBackups`.
     #[error("{0}")]
     PointInTimeRecoveryUnavailableException(String),
+    /// A `RestoreDateTime` outside the source table's restorable window, the
+    /// exception the service models on `RestoreTableToPointInTime`.
+    #[error("{0}")]
+    InvalidRestoreTimeException(String),
     #[error("{0}")]
     ServiceUnavailable(String),
     /// SP-ERR-002: Concurrent modification of the same item in a transaction.
@@ -173,6 +177,7 @@ impl DynamoDbError {
             | Self::ExpiredIteratorException(_)
             | Self::ContinuousBackupsUnavailableException(_)
             | Self::PointInTimeRecoveryUnavailableException(_)
+            | Self::InvalidRestoreTimeException(_)
             | Self::TransactionConflictException(_)
             | Self::ProvisionedThroughputExceededException(_)
             | Self::ThrottlingException(_)
@@ -230,6 +235,7 @@ impl DynamoDbError {
             Self::PointInTimeRecoveryUnavailableException(_) => {
                 "PointInTimeRecoveryUnavailableException"
             }
+            Self::InvalidRestoreTimeException(_) => "InvalidRestoreTimeException",
             Self::ServiceUnavailable(_) => "ServiceUnavailable",
             Self::TransactionConflictException(_) => "TransactionConflictException",
             Self::ProvisionedThroughputExceededException(_) => {
@@ -302,6 +308,7 @@ impl DynamoDbError {
             | Self::ExpiredIteratorException(m)
             | Self::ContinuousBackupsUnavailableException(m)
             | Self::PointInTimeRecoveryUnavailableException(m)
+            | Self::InvalidRestoreTimeException(m)
             | Self::ServiceUnavailable(m)
             | Self::TransactionConflictException(m)
             | Self::ProvisionedThroughputExceededException(m)
@@ -424,6 +431,10 @@ mod tests {
                 DynamoDbError::PointInTimeRecoveryUnavailableException(String::new()),
                 400,
             ),
+            (
+                DynamoDbError::InvalidRestoreTimeException(String::new()),
+                400,
+            ),
             (DynamoDbError::TableNotFoundException(String::new()), 400),
             (DynamoDbError::InvalidSignatureException(String::new()), 400),
         ];