From 05da4cfad21954ef19ab79ca875d52305c3469fe Mon Sep 17 00:00:00 2001 From: aaltshuler Date: Wed, 5 Aug 2026 22:17:20 +0300 Subject: [PATCH] feat(cdc): establish graph change-feed foundation --- crates/omnigraph/src/changes/mod.rs | 75 +- crates/omnigraph/src/db/commit_graph.rs | 12 + crates/omnigraph/src/db/graph_coordinator.rs | 32 +- crates/omnigraph/src/db/omnigraph.rs | 17 +- crates/omnigraph/tests/forbidden_apis.rs | 5 + .../omnigraph/tests/lance_surface_guards.rs | 232 +++++- docs/dev/testing.md | 4 +- docs/rfcs/0030-cdc-time-travel.md | 676 ++++++++++++++++++ 8 files changed, 1020 insertions(+), 33 deletions(-) create mode 100644 docs/rfcs/0030-cdc-time-travel.md diff --git a/crates/omnigraph/src/changes/mod.rs b/crates/omnigraph/src/changes/mod.rs index 52178979..2b811fba 100644 --- a/crates/omnigraph/src/changes/mod.rs +++ b/crates/omnigraph/src/changes/mod.rs @@ -1,11 +1,11 @@ -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashSet}; use arrow_array::{Array, RecordBatch, StringArray, UInt64Array}; use arrow_cast::display::array_value_to_string; use lance::dataset::scanner::ColumnOrdering; use crate::db::SubTableEntry; -use crate::db::manifest::Snapshot; +use crate::db::manifest::{Snapshot, TableIdentity}; use crate::error::Result; use crate::storage_layer::{SnapshotHandle, TableStorage}; use crate::table_store::TableStore; @@ -104,6 +104,45 @@ impl ChangeFilter { // ─── Core diff ────────────────────────────────────────────────────────────── +/// One immutable table lifetime whose physical state differs between two +/// graph snapshots. +/// +/// Identity, not alias, pairs the endpoints. A rename therefore stays one +/// interval (and is elided when its physical state did not move), while a +/// drop/re-add under the same public name remains two distinct lifetimes. +#[derive(Debug, Clone, Copy)] +pub(crate) struct TableChangeInterval<'a> { + pub(crate) identity: TableIdentity, + pub(crate) from: Option<&'a SubTableEntry>, + pub(crate) to: Option<&'a SubTableEntry>, +} + +/// Derive changed table lifetimes in stable immutable-identity order. +/// +/// This is the graph-commit CDC pruning layer: later row enumeration only +/// needs to inspect these exact endpoint pairs. It persists no parallel change +/// log and does not infer identity from an alias, path, or Lance version. +pub(crate) fn changed_table_intervals<'a>( + from: &'a Snapshot, + to: &'a Snapshot, +) -> Vec> { + let mut by_identity = + BTreeMap::, Option<&'a SubTableEntry>)>::new(); + for entry in from.entries() { + by_identity.entry(entry.identity).or_default().0 = Some(entry); + } + for entry in to.entries() { + by_identity.entry(entry.identity).or_default().1 = Some(entry); + } + + by_identity + .into_iter() + .filter_map(|(identity, (from, to))| { + (!same_state(from, to)).then_some(TableChangeInterval { identity, from, to }) + }) + .collect() +} + /// Net-current diff between two snapshots. /// /// Uses a three-level algorithm: @@ -117,40 +156,28 @@ pub(crate) async fn diff_snapshots( filter: &ChangeFilter, branch: Option, ) -> Result { - let from_by_identity = from - .entries() - .map(|entry| (entry.identity, entry)) - .collect::>(); - let to_by_identity = to - .entries() - .map(|entry| (entry.identity, entry)) - .collect::>(); - let all_identities = from_by_identity - .keys() - .chain(to_by_identity.keys()) - .copied() - .collect::>(); - let mut changes = Vec::new(); - for identity in all_identities { - let from_entry = from_by_identity.get(&identity).copied(); - let to_entry = to_by_identity.get(&identity).copied(); + for interval in changed_table_intervals(from, to) { + let from_entry = interval.from; + let to_entry = interval.to; // Prefer the destination alias for a rename; a removed table has only // its source alias. Logical pairing never depends on either name. let table_key = &to_entry .or(from_entry) .expect("identity came from one snapshot") .table_key; + debug_assert!( + from_entry + .into_iter() + .chain(to_entry) + .all(|entry| entry.identity == interval.identity), + "table interval endpoints must retain their immutable identity" + ); if !filter.matches_table(table_key) { continue; } - // Skip if both snapshots have identical state for this table - if same_state(from_entry, to_entry) { - continue; - } - let (kind, type_name) = parse_table_key(table_key); let is_edge = kind == EntityKind::Edge; diff --git a/crates/omnigraph/src/db/commit_graph.rs b/crates/omnigraph/src/db/commit_graph.rs index f971c46d..13c2fd87 100644 --- a/crates/omnigraph/src/db/commit_graph.rs +++ b/crates/omnigraph/src/db/commit_graph.rs @@ -31,6 +31,18 @@ impl GraphCommit { } } +/// One observable graph transition on a branch's first-parent lineage. +/// +/// CDC compares exactly these two immutable graph commits. A merge commit is +/// compared with the branch state it landed on (`parent`); its +/// `merged_parent_commit_id` remains provenance and is never traversed as a +/// second feed path. +#[derive(Debug, Clone)] +pub(crate) struct FirstParentEdge { + pub(crate) parent: GraphCommit, + pub(crate) child: GraphCommit, +} + /// A pure projection of the graph lineage that lives in `__manifest` /// (`graph_commit` + `graph_head` rows, RFC-013 Phase 7). It opens NO Lance /// dataset (Phase B retired `_graph_commits.lance` / `_graph_commit_actors.lance`): diff --git a/crates/omnigraph/src/db/graph_coordinator.rs b/crates/omnigraph/src/db/graph_coordinator.rs index 672755a0..d849b671 100644 --- a/crates/omnigraph/src/db/graph_coordinator.rs +++ b/crates/omnigraph/src/db/graph_coordinator.rs @@ -7,7 +7,7 @@ use crate::error::{OmniError, Result}; use crate::failpoints; use crate::storage::{StorageAdapter, normalize_root_uri}; -use super::commit_graph::{CommitGraph, GraphCommit}; +use super::commit_graph::{CommitGraph, FirstParentEdge, GraphCommit}; use super::is_internal_system_branch; use super::manifest::{ CapturedManifestProbe, ExpectedTableVersions, LineageIntent, ManifestChange, @@ -83,6 +83,14 @@ pub struct ResolvedTarget { pub snapshot: Snapshot, } +/// Internal lineage classification for an existing two-commit diff request. +/// Arbitrary ranges retain net-current semantics; direct adjacency is derived +/// only from the child's persisted first-parent pointer. +pub(crate) enum ResolvedCommitRange { + FirstParent(FirstParentEdge), + Arbitrary { from: GraphCommit, to: GraphCommit }, +} + #[derive(Debug, Clone)] pub(crate) struct PublishedSnapshot { pub manifest_version: u64, @@ -426,6 +434,28 @@ impl GraphCoordinator { ))) } + /// Resolve both endpoints and classify direct first-parent adjacency from + /// the child's persisted parent pointer. + /// + /// This is deliberately O(1) after the two commits are resolved: it adds + /// no ancestry index or history walk. Arbitrary ranges retain the existing + /// net-current diff semantics. + pub(crate) async fn resolve_commit_range( + &self, + from_id: &SnapshotId, + to_id: &SnapshotId, + ) -> Result { + let from = self.resolve_commit(from_id).await?; + let to = self.resolve_commit(to_id).await?; + if to.parent_commit_id.as_deref() != Some(from.graph_commit_id.as_str()) { + return Ok(ResolvedCommitRange::Arbitrary { from, to }); + } + Ok(ResolvedCommitRange::FirstParent(FirstParentEdge { + parent: from, + child: to, + })) + } + pub(crate) async fn head_commit_id(&self) -> Result> { self.commit_graph .head_commit_id() diff --git a/crates/omnigraph/src/db/omnigraph.rs b/crates/omnigraph/src/db/omnigraph.rs index b955fbd0..de5559ae 100644 --- a/crates/omnigraph/src/db/omnigraph.rs +++ b/crates/omnigraph/src/db/omnigraph.rs @@ -25,7 +25,7 @@ use omnigraph_compiler::{ plan_schema_migration, }; -use crate::db::graph_coordinator::{GraphCoordinator, PublishedSnapshot}; +use crate::db::graph_coordinator::{GraphCoordinator, PublishedSnapshot, ResolvedCommitRange}; use crate::error::{OmniError, Result}; use crate::runtime_cache::RuntimeCache; use crate::storage::{ @@ -2559,10 +2559,19 @@ impl Omnigraph { filter: &crate::changes::ChangeFilter, ) -> Result { let coord = self.coordinator.read().await; - let from_commit = coord - .resolve_commit(&SnapshotId::new(from_commit_id)) + let range = coord + .resolve_commit_range( + &SnapshotId::new(from_commit_id), + &SnapshotId::new(to_commit_id), + ) .await?; - let to_commit = coord.resolve_commit(&SnapshotId::new(to_commit_id)).await?; + // Classify direct adjacency from the child's persisted first-parent + // pointer without changing this API's net-current result shape. The + // future feed can reuse that relationship without an ancestry index. + let (from_commit, to_commit) = match range { + ResolvedCommitRange::FirstParent(edge) => (edge.parent, edge.child), + ResolvedCommitRange::Arbitrary { from, to } => (from, to), + }; let from_snap = coord .resolve_target(&ReadTarget::Snapshot(SnapshotId::new( from_commit.graph_commit_id.clone(), diff --git a/crates/omnigraph/tests/forbidden_apis.rs b/crates/omnigraph/tests/forbidden_apis.rs index 8586bf43..dbeba781 100644 --- a/crates/omnigraph/tests/forbidden_apis.rs +++ b/crates/omnigraph/tests/forbidden_apis.rs @@ -517,6 +517,11 @@ const LOW_LEVEL_READ_ONLY_SURFACES: &[(&str, &str, &str)] = &[ "GraphCoordinator", "resolve_commit", ), + ( + "db/graph_coordinator.rs", + "GraphCoordinator", + "resolve_commit_range", + ), ( "db/graph_coordinator.rs", "GraphCoordinator", diff --git a/crates/omnigraph/tests/lance_surface_guards.rs b/crates/omnigraph/tests/lance_surface_guards.rs index 7adb29d4..25a11ea9 100644 --- a/crates/omnigraph/tests/lance_surface_guards.rs +++ b/crates/omnigraph/tests/lance_surface_guards.rs @@ -23,12 +23,14 @@ //! Functions decorated `#[tokio::test]` actually run; they construct real //! values and assert field shapes / types. -use std::collections::HashMap; +mod helpers; + +use std::collections::{HashMap, HashSet}; use std::fmt; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; -use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator, StringArray}; +use arrow_array::{Array, Int32Array, RecordBatch, RecordBatchIterator, StringArray, UInt64Array}; use arrow_schema::{DataType, Field, Schema}; use async_trait::async_trait; use futures::TryStreamExt; @@ -79,6 +81,8 @@ use object_store::{ }; use omnigraph_compiler::schema::parser::parse_schema; +use helpers::{init_and_load, open_dataset_head, snapshot_main}; + #[test] fn compiler_rejects_five_surveyed_lance_virtual_system_columns() { let names = [ @@ -2140,6 +2144,230 @@ async fn _compile_scalar_index_coverage_surface() -> lance::Result<()> { Ok(()) } +// --- CDC C0 guards: exact-end deltas and production row-version shape ------- +// +// Lance's explicit delta range controls the version-column predicate, but the +// row images are scanned from the `Dataset` handle used to build the delta. A +// historical interval therefore needs a handle checked out at its exact end: +// asking a later HEAD for the same interval can lose a row that changed again. +// Keep this regression beside the other Lance surface probes so a dependency +// bump cannot silently invalidate RFC-0030's candidate-pruning contract. + +#[tokio::test] +async fn dataset_delta_historical_images_require_the_exact_end_handle() { + async fn commit_alice_value(dataset: Dataset, value: i32) -> Dataset { + let batch = pk_full_row(&dataset, "alice", value); + let staged = stage_pk_merge( + Arc::new(dataset.clone()), + batch, + "id", + WhenMatched::UpdateAll, + WhenNotMatched::InsertAll, + Some(false), + ) + .await; + CommitBuilder::new(Arc::new(dataset)) + .with_skip_auto_cleanup(true) + .execute(staged.transaction) + .await + .expect("the guard update must commit") + } + + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().join("cdc-exact-end-delta.lance"); + let initial = fresh_pk_dataset(uri.to_str().unwrap()).await; + let begin_version = initial.version().version; + + let exact_end = commit_alice_value(initial, 20).await; + let end_version = exact_end.version().version; + assert_eq!( + end_version, + begin_version + 1, + "the exact-end fixture needs one adjacent update" + ); + + let current_head = commit_alice_value(exact_end.clone(), 30).await; + assert_eq!( + current_head.version().version, + end_version + 1, + "the negative control needs the same row updated after the selected end" + ); + + let exact_delta = exact_end + .delta() + .with_begin_version(begin_version) + .with_end_version(end_version) + .build() + .unwrap(); + let exact_batches: Vec = exact_delta + .get_updated_rows() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + let exact_rows = exact_batches + .iter() + .map(RecordBatch::num_rows) + .sum::(); + assert_eq!( + exact_rows, 1, + "the exact-end delta must retain the one row changed in the interval" + ); + let exact_batch = exact_batches + .iter() + .find(|batch| batch.num_rows() == 1) + .expect("the one changed row must be materialized"); + let exact_ids = exact_batch["id"] + .as_any() + .downcast_ref::() + .unwrap(); + let exact_values = exact_batch["value"] + .as_any() + .downcast_ref::() + .unwrap(); + let exact_updated_versions = exact_batch[ROW_LAST_UPDATED_AT_VERSION] + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(exact_ids.value(0), "alice"); + assert_eq!( + exact_values.value(0), + 20, + "the delta must return the image at the selected end, not a later image" + ); + assert_eq!(exact_updated_versions.value(0), end_version); + + let stale_interval_on_head = current_head + .delta() + .with_begin_version(begin_version) + .with_end_version(end_version) + .build() + .unwrap(); + let head_batches: Vec = stale_interval_on_head + .get_updated_rows() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!( + head_batches + .iter() + .map(RecordBatch::num_rows) + .sum::(), + 0, + "pinned Lance scans row images from the builder's handle: a later HEAD is \ + not a valid source for an older interval whose row changed again; RFC-0030 \ + must check out the exact end version before constructing DatasetDelta" + ); +} + +#[tokio::test] +async fn omnigraph_graph_tables_enable_stable_row_ids_and_version_columns() { + let dir = tempfile::tempdir().unwrap(); + let db = init_and_load(&dir).await; + let snapshot = snapshot_main(&db).await.unwrap(); + let entries = snapshot + .entries() + .map(|entry| { + ( + entry.table_key.clone(), + entry.table_path.clone(), + entry.table_version, + entry.table_branch.clone(), + ) + }) + .collect::>(); + assert_eq!( + entries.len(), + 4, + "the shared fixture must exercise every declared node and edge table" + ); + + for (table_key, table_path, table_version, table_branch) in entries { + let table_uri = dir.path().join(table_path); + let head = open_dataset_head(table_uri.to_str().unwrap(), table_branch.as_deref()).await; + let table = if head.version().version == table_version { + head + } else { + head.checkout_version(table_version).await.unwrap() + }; + + assert!( + table.manifest().uses_stable_row_ids(), + "OmniGraph-created graph table {table_key} must keep Lance stable row IDs enabled" + ); + + let selected_version = table.version().version; + let mut scanner = table.scan(); + scanner + .project(&[ + "id", + ROW_ID, + ROW_CREATED_AT_VERSION, + ROW_LAST_UPDATED_AT_VERSION, + ]) + .expect("stable row-id and row-version columns must be projectable"); + scanner + .filter(&format!( + "{ROW_CREATED_AT_VERSION} <= {ROW_LAST_UPDATED_AT_VERSION} AND \ + {ROW_LAST_UPDATED_AT_VERSION} <= {selected_version}" + )) + .expect("row-version columns must be usable in a scan predicate"); + let batches: Vec = scanner + .try_into_stream() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert!( + batches.iter().any(|batch| batch.num_rows() > 0), + "the shared loaded fixture must contain rows in {table_key}" + ); + + let mut row_ids = HashSet::new(); + for batch in batches { + let ids = batch[ROW_ID] + .as_any() + .downcast_ref::() + .expect("_rowid must retain Lance's UInt64 surface"); + let created = batch[ROW_CREATED_AT_VERSION] + .as_any() + .downcast_ref::() + .expect("_row_created_at_version must retain Lance's UInt64 surface"); + let updated = batch[ROW_LAST_UPDATED_AT_VERSION] + .as_any() + .downcast_ref::() + .expect("_row_last_updated_at_version must retain Lance's UInt64 surface"); + assert_eq!( + ids.null_count(), + 0, + "live {table_key} rows need concrete stable row IDs" + ); + assert_eq!( + created.null_count(), + 0, + "live {table_key} rows need a creation version" + ); + assert_eq!( + updated.null_count(), + 0, + "live {table_key} rows need an update version" + ); + for row in 0..batch.num_rows() { + assert!( + row_ids.insert(ids.value(row)), + "stable row IDs must be unique within {table_key}" + ); + assert!(created.value(row) <= updated.value(row)); + assert!(updated.value(row) <= selected_version); + } + } + } +} + // --- Guard 12: can a scalar BTREE be built on a system version column? -------- // // The deferred persisted-adjacency artifact plan assumed a cheap delta read of diff --git a/docs/dev/testing.md b/docs/dev/testing.md index 5dc3d3c5..c27f1a22 100644 --- a/docs/dev/testing.md +++ b/docs/dev/testing.md @@ -75,7 +75,7 @@ it is not inferred from a local syntax check. See [ci.md](ci.md). | `writes.rs` | Direct-publish writes: cancellation, RFC-022 non-strict full-attempt reprepare from fresh branch authority, strict stale-write conflicts, multi-statement atomicity, MR-794 staged-write rewire (D₂ rejection, insert+update coalesce, multi-append coalesce, partial-failure recovery, load RI/cardinality recovery); RFC-023 pins the inclusive 8,192-row keyed input ceiling, the same exact/+1 boundary on streamed mutation-update matches, no-effect state for both refusals, and oversized stored-Blob rejection before payload read. Crate-internal pending-scan cells pin inclusive/+1 32 MiB accounting plus pending-key shadow-before-charge. The lance#7444 row-id-overlap regression (`filtered_read_after_merge_update_and_delete_keeps_row_ids_consistent` — merge-load → same-key merge-load → delete → keyed point lookup, green only under the vendored lance-table patch — plus its append-only control) | | `src/table_store/staged_tests.rs` | Crate-internal staged primitives. RFC-023 pins one exact target preflight for general StrictInsert, durable v1 mint/commit/reopen/history persistence, exact-`id` filter emission, typed `KeyConflict`, and missing/wrong PK refusal. `all_new_upsert_certifies_insert_absence_and_persists_it_in_history` proves an all-new completed Upsert receives the optional certificate, a mixed/update Upsert does not, unrelated transaction properties survive, and UUID rebinding does not erase it. Proven-insert cells show the opaque path performs zero strict preflights; stages with `InsertBuilder` but commits the full pure-insert `Update` shape (exact parent and `id` filter, `RewriteRows`, no updates/removals, full nested schema preorder, physical rows); persists/re-admits its own output for proof composition; leaves new fragments outside old index coverage; and fails same-key races loudly in proven/proven and proven/general orders. The in-source `exec/merge.rs` certificate unit table rejects missing/unknown properties, wrong parent/filter/full-preorder/mode/offsets, rewrite/removal shapes, missing `physical_rows`, and Append. Source-interval cells pin exact selection, lazy retained-parent splitting, coalescing, and pinned Lance's approximate raw-emission boundary while every normalized/writer chunk remains hard-capped. Generic `stage_append`/`stage_merge_insert` remain primitive tests only. The file also owns index staging and `commit_staged{,_exact}` | | `forbidden_apis.rs` | Defense-in-depth syntax-tree/source guard over the whole engine. The primary boundary is Rust visibility: raw storage/coordinator/handle-cache modules are crate-private; public `Snapshot::open` returns `SnapshotTable`; and `SnapshotScanner` executes reads without exposing Lance's raw scanner or physical plan. The guard pins those visibility/return-type boundaries, classifies public async inherent `Omnigraph` methods plus loader conveniences, classifies every crate-visible async method on `GraphCoordinator` / `ManifestCoordinator`, and exact-counts registered method/UFCS durable-call shapes including recovery. RFC-023 rejects production graph call sites of generic `stage_append{,_stream}` and `proven_insert_capability_has_one_production_mint_site` pins `ProvenInsertChunk::from_verified_history` to the complete-history classifier in `exec/merge.rs`, preventing the no-preflight capability from becoming a reusable bypass. At the RFC-026 Phase-A checkpoint the guard registered only the exact v10 enrollment gateway and feature-gated test seam, counted its sidecar/index/shard durability primitives, and kept every row-put/ack/fold surface absent; the Phase-B1 owner below exact-counts only the approved crate-private put/fold durable-call sites. V11 adds only the bounded checked offline profile/runtime authority factories and recovery-v13 profile gateway; the guard must keep ambient constructors and public row, enrollment, drain, claim, lifecycle, SDK, HTTP, CLI, and OpenAPI side doors absent. F5a classifies the doc-hidden start/shutdown bridge as composed recovery-v14 fold orchestration while adding no durable primitive allowance for the supervisor module itself. F6b1 structurally pins the doc-hidden export cut as private-field, move-only, and non-forgeable while adding no public export seam. F7a registers exactly one doc-hidden public graph-ingest bridge as composed recovery-v14/v21 ownership while retaining the ban on raw table/lane writers and public management side doors. The embedded SDK's read-only `stream_status` is classified separately and must remain free of durable calls. B2a inventories the only production `_mem_wal` literal owners, forbids MemWAL reclamation/adoption symbols and destructive primitives in the adapter, keeps generic maintenance unaware of MemWAL, and keeps raw inventory/classifier helpers private. This remains defense in depth rather than macro expansion, alias, or data-flow analysis; the visibility boundary and behavior tests are still primary. It also counts selected raw `SnapshotHandle` / Dataset shapes, rejects renamed-owner/macro/include/path-lookalike forms, skips structurally test-only code, and pins retired escape hatches absent. `// forbidden-api-allow: ` exempts reviewed inline-Lance lines only | -| `lance_surface_guards.rs` | Pins the Lance API surfaces omnigraph depends on (named runtime + compile-only guards; see [lance.md](lance.md)) — the first smoke check on any Lance version bump. `cached_and_zero_cache_sessions_share_store_registry_not_metadata_cache` proves a cached data Session and zero-cache control Session reuse one live `ObjectStoreRegistry` client while their metadata caches remain isolated. `_compile_uncommitted_full_table_vector_index_shape` pins the public `IndexMetadata` shape suitable for `Operation::CreateIndex`; `compact_files_succeeds_on_blob_columns` pins blob-v2 compaction; Guard 9 pins clone-only branch reclaim semantics. RFC-023's `unenforced_pk_filter_shape_is_route_dependent` explicitly forces v2 versus indexed routes and pins the `Some(populated)` / `Some(empty)` / `None` key-filter shapes; `unenforced_pk_conflict_matrix_is_directional` pins the directional filtered/unfiltered and filtered/Append matrix. RFC-024's compile guard pins the public `BranchIdentifier` + current table version + current `Transaction.uuid` + `ManifestLocation.e_tag` current-HEAD witness; the local/shared-`Session` guard proves unchanged-reopen stability, ordinary-commit movement, and same-version ABA, while RustFS covers object-store ABA. RFC-025 adds exact main/named-branch tag-target, sparse cleanup pin/unpin, and branch-tree-deletion guards. RFC-026 pins doc-hidden `has_successor_version`, initializer/readback/shard-writer/durability/fencing, flush/drain, replay watermark, scanner, and merged-generation shapes; runtime Gate E0 classification belongs to `memwal_enrollment_gate.rs`, while v7 Phase-A and v8 B1 publication/recovery belong to the manifest/failpoint suites. B2b adds `cleanup_old_versions_does_not_reclaim_mem_wal_objects` and `mem_wal_deleted_fence_slot_allows_stale_writer_success_on_pinned_lance`: the first proves generic cleanup leaves the present MemWAL fixture unchanged and the second proves deleting the successor's empty fence sentinel is unsafe. The pinned source audit, not those two tests alone, establishes that stock RC.1 exposes no owned MemWAL reclamation API. The RC.1 compiler guard pins the five surveyed public Lance virtual system-column constants to early `.pg` rejection. These guards prove substrate shapes/tokens and negative ownership boundaries; they do not by themselves prove heads/checkpoint activation, the current publisher, or a safe reclamation implementation | +| `lance_surface_guards.rs` | Pins the Lance API surfaces omnigraph depends on (named runtime + compile-only guards; see [lance.md](lance.md)) — the first smoke check on any Lance version bump. `cached_and_zero_cache_sessions_share_store_registry_not_metadata_cache` proves a cached data Session and zero-cache control Session reuse one live `ObjectStoreRegistry` client while their metadata caches remain isolated. `_compile_uncommitted_full_table_vector_index_shape` pins the public `IndexMetadata` shape suitable for `Operation::CreateIndex`; `compact_files_succeeds_on_blob_columns` pins blob-v2 compaction; Guard 9 pins clone-only branch reclaim semantics. RFC-023's `unenforced_pk_filter_shape_is_route_dependent` explicitly forces v2 versus indexed routes and pins the `Some(populated)` / `Some(empty)` / `None` key-filter shapes; `unenforced_pk_conflict_matrix_is_directional` pins the directional filtered/unfiltered and filtered/Append matrix. RFC-024's compile guard pins the public `BranchIdentifier` + current table version + current `Transaction.uuid` + `ManifestLocation.e_tag` current-HEAD witness; the local/shared-`Session` guard proves unchanged-reopen stability, ordinary-commit movement, and same-version ABA, while RustFS covers object-store ABA. RFC-025 adds exact main/named-branch tag-target, sparse cleanup pin/unpin, and branch-tree-deletion guards. RFC-026 pins doc-hidden `has_successor_version`, initializer/readback/shard-writer/durability/fencing, flush/drain, replay watermark, scanner, and merged-generation shapes; runtime Gate E0 classification belongs to `memwal_enrollment_gate.rs`, while v7 Phase-A and v8 B1 publication/recovery belong to the manifest/failpoint suites. B2b adds `cleanup_old_versions_does_not_reclaim_mem_wal_objects` and `mem_wal_deleted_fence_slot_allows_stale_writer_success_on_pinned_lance`: the first proves generic cleanup leaves the present MemWAL fixture unchanged and the second proves deleting the successor's empty fence sentinel is unsafe. RFC-0030 C0 adds `dataset_delta_historical_images_require_the_exact_end_handle`, which proves an explicit end bound does not replace checking out the exact-end handle, and `omnigraph_graph_tables_enable_stable_row_ids_and_version_columns`, which exercises every node/edge table in the shared graph fixture. The pinned source audit, not those runtime tests alone, establishes the corresponding substrate contract. The RC.1 compiler guard pins the five surveyed public Lance virtual system-column constants to early `.pg` rejection. These guards prove substrate shapes/tokens and negative ownership boundaries; they do not by themselves prove heads/checkpoint activation, the current publisher, or a safe reclamation implementation | | `memwal_enrollment_gate.rs` | RFC-026's green production-neutral Gate E0 harness, isolated from the production manifest and graph writer. Fourteen substantive local cells plus one explicit unconfigured-S3 skip cover exact no-effect / `N + 1` index / pre-minted empty-shard classification, buried-effect refusal, marker survival, strict inventory/error handling, and the broad fail-closed matrix. The rejected first instrument used `checkout_latest` plus `IOTracker`, which missed local `read_dir`. The accepted exact-version classifier pins doc-hidden `has_successor_version`; its `AttemptTracker` records failed/`NotFound` attempts before forwarding and proves the identical complete six-attempt shape at baseline versions 8/80: four successful manifest HEADs, one `NotFound` manifest HEAD, one successful manifest GET, zero lists. A Unix execute-only `_versions` tripwire proves exact probing works when latest enumeration fails and an unreadable exact HEAD errors. The configured RustFS exact cell passes non-vacuously with the same zero-list shape and owns the positive lost-result/index/empty-shard/reopen sequence plus foreign shard, malformed/loose root, durable WAL, persisted cursor, and corrupt-manifest negatives. S3 ABA remains in `lance_surface_guards.rs`; CI rejects skipped E0/ABA cells. This file never mutates production manifest/schema state or deletes ambiguous artifacts; Phase A consumes its classifier through the private adapter | | `memwal_stream.rs` | Feature-gated RFC-026 private B1 mechanics, B2 compare-and-chain behavior, B2a provider-failure evidence, and the hidden lifecycle-v3 integration. B1 owns bounded put/ack/replay, authority, cancellation, and manifest-only visibility, including row-local value rejection before any WAL or manifest effect. B2 owns idempotency conflicts, same-generation overlays, stale-authority recapture, durable attribution, and one watcher/fence result over a distinct-key contiguous multi-row physical prefix. The hidden F4 request proof additionally pins graph-scoped `stream_ingest` policy and exact checked-runtime authority before body work; separate root-wide and per-actor transport admission before polling; incremental NDJSON framing across accepted chunks, CRLF, EOF, and over-limit-line boundaries without whole-request retention; strict `$stream` parsing; duplicate, unknown, and reserved-field refusal; explicit canonical IDs; dense schema-ordered node/edge conversion; scalar/list/enum/vector and value-constraint validation before recovery entry; and effect-free Blob-table refusal through a deliberately stale pre-schema-apply handle before either the request or lower B1/B2 seam can invoke MemWAL. Run-splitting cells cover invalid lines, repeated keys, token dispositions, and row/byte ceilings; bounded result/reorder ownership preserves caller order and stop-tail `blocking_ordinal` precedence, while disconnect stops new body polling/admission and transfers the invoked tail to root-owned settlement. The graph-native hidden slice pins strict mixed node/edge rows with no caller-visible table key, one graph-scoped policy decision, catalog-resolved lazy enrollment, move-only bounded normalization, declaration handoff through the existing finite node-before-edge driver round, graph-wide ambiguous-result blocking, and a scalar logical result whitelist with no physical evidence. The bodyless prepare cells pin effect-free witness challenge, checked-runtime/policy ordering, Blob refusal before enrollment, actor-bound durable-receipt replay, and two concurrent request IDs converging on one OPEN lane before ingest/fold composition. F5a extends this same owner with checked-runtime start refusal, coalesced timer/cap folding, trigger-during-fold preservation, cold-reopen discovery without an in-memory pending bit, deterministic finite node-before-edge rounds, root singleflight, retry/backoff visibility, and bounded supervisor shutdown; every effect is still proved by the existing recovery-v14 fold/crash cells rather than a duplicate replay suite. F6a adds one hidden in-process candidate-runtime composition: prepare, ordered NDJSON, an automatic mixed visible/dead-letter fold, stopped/offline selected-token list/export, an ordinary corrected successor, driver restart, clean shutdown ownership, and checked offline disable. F6b1 adds ambient-enrolled pre-byte refusal; managed and unmanaged-terminal checked success; checked `WITHDRAWN | DEAD_LETTERED` pre-byte refusal; one immutable exact-version cut across a later writer; sole nonwaiting root-slot exclusion and release; named-branch delete/recreate exclusion; and preservation of a post-start storage error with slot release. It remains inaccessible to production callers and exposes no SDK/HTTP/CLI/API/OpenAPI ingress, public driver status, or public rebind surface. Lifecycle-v3 owns recovery-covered cold/fold claims, exact full-generation projection after flush/reopen, recovery-v14 ordinary/drain folds, empty and non-empty `OPEN → DRAINING → SEALED`, an empty successor after an ordinary published fold, durable/reopen-stable typed `DataBlock` publication with no base or graph commit (including a fresh-source minimum-cardinality violation whose streamed edge supplies correction identity), idempotent same-request restart, conflicting request/stale-revision refusal, and the claim-before-seal plus seal-before-fold crash boundaries. Recovery-v15 adds receipt-first idempotent `SEALED → OPEN` resume, guarded `DRAINING → OPEN` abort, higher-epoch claim, terminal receipt publication, named-branch refusal, and current-binding-chain ancestry checks. F6b8 adds compile-enforced root-producer ownership transfer through detached resume installation, urgent trigger-before-release, exact empty-owner housekeeping before the unchanged node-before-edge round, prompt retirement, driver-first and resume-first/caller-cancelled races, cross-lane slot reuse, and shutdown waiting for the detached owner. The strict-block path streams `DRAINING` validation directly into the bounded evidence collector; unit owners below pin the detailed cap, empty-evidence refusal, and non-materialized overflow digest. F3d's checked-offline cell releases the served runtime, reaches terminal `DISABLED`, crashes after arming v18 but before a physical effect, and proves retry selects one fresh `SEALED` scope while retaining the old MemWAL inventory; repeating the same occurrence is physically effect-free. B2a injects a recording/failing store at the real Lance table-store boundary and covers post-invocation ambiguity and inert orphan residue. F7a extends this owner with the production checked-runtime graph token bridge, redacted newline result contract, and a non-resetting 50-ms same-declaration coalescing boundary that acknowledges a complete row while its body stream remains pending; server and CLI owners cover transport. No test here implies a supported public lifecycle API. | | `memwal_stream_cost.rs` | Feature-gated RFC-026 B1, Gate-R0, and B2a decision instrument. It separately measures warm already-claimed durability acknowledgement, cold replay, selected-generation fold scanning, visibility, retained merged metadata, the uncompacted graph-manifest term, legal no-roll estimates, and paired peak RSS. Gate R0 adds a revision-pinned source-audit tripwire, strict current-object classification/reference census, listed path/class/size retain-all comparisons at one/four/eight folds, referenced-cut retry reuse, and deterministic high-entropy near-cap local/configured-RustFS cells. The near-cap cell proves the exact B2-attributed boundary through the real adapter: 3,742 payload bytes per row admits 8,192 rows at 33,550,336 logical bytes, while 3,743 is rejected effect-free at 33,558,528 bytes. The legal generation acknowledges without graph visibility, then folds and publishes exactly once after logical-slice charging plus dense per-scanner-batch take. The reference-environment paired fold peak-RSS lift measured 286,441,472 bytes (about 273 MiB), below a one-sided 384-MiB remeasurement tripwire; common initialization may censor that lifetime high-water lift to zero or a negative value on another runner, and the tripwire is not a runtime allocator limit. B2a adds 1/8/32/128 local and configured-RustFS retained-history sweeps whose terms remain separate: warm ack, cold reopen/replay, fold, visibility, MemWAL/base-table/token-authority/other table-store work, graph-manifest/adapter work, advisory current-object bytes, and whole-process peak RSS. Older retained roots must receive zero reads, writes, or deletes. The only allowed delete shape is Lance's losing manifest-CAS `.binpb.tmp.` staging; canonical durable MemWAL delete requests remain zero. LIST totals, wall times, and RSS are advisory—not a quota, SLO, isolated WAL slope, or provider billing. A green test proves private closure/retention behavior; it does not activate a public API. | @@ -91,7 +91,7 @@ it is not inferred from a local syntax check. See [ci.md](ci.md). | `benchmark_scenario_contract.rs` | Source/protocol contract for the non-CI scenario harness. RFC-023 pins the production route's explicit `strict_insert_preflight_calls == 0` assertion and emitted `probe_strict_insert_preflight_calls` field, alongside route labels, clean-tree/binary identity, child-protocol refusal, and exact-content verification fields. A benchmark record therefore cannot silently claim the proven path after paying a target preflight | | `lifecycle.rs` | Graph lifecycle and schema state, including the v6-origin creation invariant—preserved through current v19—that every fresh node/edge table declares exactly physical `id` as Lance's unenforced PK | | `point_in_time.rs` | Snapshots, time travel (`snapshot_at_version`, `entity_at`) | -| `changes.rs` | `diff_between` / `diff_commits` | +| `changes.rs` | `diff_between` / `diff_commits`, including immutable-identity table pairing: pure renames stay empty while drop/re-add under one alias remains two table lifetimes. The production net diff consumes the same identity-keyed table-interval derivation reserved for later per-commit CDC row walking; it does not claim CDC block or cursor ordering | | `consistency.rs` | Cross-table snapshot isolation and atomic publish; RFC-023 cells prove `LoadMode::Append` is strict (existing `id` rejected without update/version movement), pin the inclusive 8,192-row load ceiling with a one-over pre-effect refusal, reject an input above 32 MiB through the shared Mutation/Load staging seam with raw table HEAD/manifest/sidecar unchanged, reject an oversized external blob on a lazy branch from object metadata before payload access/ref creation/sidecar arm, and use a barrier-synchronized stress cell over 16 pre-opened handles to prove one same-key winner, 15 typed `KeyConflict` losers, exactly one stored row carrying the winner's value, and survival of disjoint IDs | | `lineage_projection.rs` | RFC-013 Phase 7 acceptance gate: graph lineage lives ONLY in `__manifest` — over a realistic history (main commits, a branch, a merge, actors), the production coordinator reconstructs manifest snapshot state and the full DAG projection from one coherent manifest scan (commit set, parents, merge parents + merge actor, per-branch heads, inline actors), and the `_graph_commits.lance` / `_graph_commit_actors.lance` dataset directories are never created at all | | `schema_apply.rs` | Migration plan + apply, schema-apply lock; schema-contract publication is pinned by `read_only_open_holds_schema_gate_through_catalog_capture` and `refresh_holds_schema_gate_through_catalog_publication` (source, accepted IR/state, and compiled catalog are captured under one root schema gate). `long_lived_handle_uses_the_schema_catalog_bound_to_its_write_token` covers mutation/load plus a post-apply new node type merged through the pre-apply handle; `stale_handle_branch_delete_gates_tables_added_by_schema_apply` parks delete over that new type while a legacy index reconciler waits, proving merge planning and native-control table envelopes use an operation-local accepted catalog rather than stale ArcSwap state. Index materialization is deferred to the reconciler (iss-848): `apply_schema_defers_vector_index_on_empty_table` (an empty-table Vector `@index` never aborts the apply) and `index_only_constraint_apply_touches_no_table_data` (adding an `@index` is metadata-only — no table-version bump); enum widening (iss-enum-widening-migration): `enum_widening_apply_is_metadata_only_and_accepts_new_variant` (no table-version bump; new variant accepted, out-of-set still rejected) + `enum_narrowing_apply_is_refused` (OG-MF-106 with the graph left writable). The planner's widening/narrowing matrix lives in `schema_plan.rs`'s in-source tests. RFC-023 assertions prove exact-`id` PK metadata survives rewrites, applies to added types, remains on retained types across drop/re-add, and is present after reopen | diff --git a/docs/rfcs/0030-cdc-time-travel.md b/docs/rfcs/0030-cdc-time-travel.md new file mode 100644 index 00000000..f9f0d14c --- /dev/null +++ b/docs/rfcs/0030-cdc-time-travel.md @@ -0,0 +1,676 @@ +--- +type: spec +title: "RFC-030 — Graph change feed and retained-history contract" +description: Defines a graph-commit change feed, caller-owned cursors, and honest retention failures by composing OmniGraph lineage with Lance's native row-version tracking; it does not create a second WAL or expose physical datasets. +status: draft +tags: [eng, rfc, cdc, change-feed, time-travel, provenance, lineage, audit, omnigraph] +timestamp: 2026-08-05 +owner: OmniGraph maintainers +--- + +# RFC-030: Graph change feed and retained-history contract + +**Status:** Draft + +**Date:** 2026-08-05 + +**Author track:** Maintainer design series + +**Depends on:** RFC-013 Phase 7 graph lineage, RFC-022 snapshot capture and +publication, RFC-023 exact-`id` table fencing, and RFC-028 immutable table +identity. + +**Surveyed:** OmniGraph `main` at commit +`12a8596626c80a7dceed0fd72182d421052ff8d1` (internal schema v19) and Lance +9.0.0. + +**Audience:** engine, server, CLI, and documentation maintainers. + +--- + +## 0. Decision + +OmniGraph will expose changes as an ordered sequence of **graph commit +blocks**. A block is the logical difference between one graph commit and its +first parent, plus the cause already recorded on that commit. The user never +sees or resumes a feed for an individual Lance dataset. + +The implementation reuses the coordinator we already have: + +1. `__manifest` graph lineage selects the branch path and supplies commit, + parent, merge-parent, actor, and graph-snapshot authority. +2. The two exact manifest snapshots select each table lifetime's exact Lance + begin/end versions. +3. Lance's native row-version tracking supplies inserts and updates from the + dataset checked out at the **exact end version**. +4. Deletes come from an exact, bounded comparison of live logical IDs at the + begin and end snapshots. Lance 9 has no complete deleted-row feed. +5. One opaque caller-owned cursor resumes the graph feed. OmniGraph stores no + per-consumer state. + +This is a derived read model. It adds no WAL, transaction manager, change-log +table, server-side cursor registry, delete tombstone, or second coordinator. +It does not expose Lance paths, branches, fragment IDs, row addresses, or +per-table versions as public CDC concepts. + +The first contract is a **cause-carrying entity change feed**. Inserts and +updates carry the exact logical after-image from the child snapshot; deletes +carry the exact logical before-image from the parent snapshot. This is enough +to apply retained entity changes downstream and to derive the existing `diff` +result. + +It is not a complete empty-store graph replay log: graph-schema evolution is +separated in §10 because the current lineage does not preserve an accepted +SchemaIR for every historical graph commit. The limitation is schema replay, +not row-image availability. + +## 1. What Lance gives us—and what it does not + +The design is based on the pinned Lance 9.0.0 implementation as well as the +published format documentation. + +| Lance surface | Safe use in this RFC | Boundary | +|---|---|---| +| Immutable dataset versions and exact checkout | Read the exact table state selected by each graph manifest snapshot | Cleanup may remove an old version permanently | +| Stable row IDs and `_row_created_at_version` / `_row_last_updated_at_version` | Classify live rows inserted or updated in a table-version interval | Available only when stable row IDs were enabled at dataset creation | +| Public `Dataset::delta()` with streaming inserted/updated/upserted rows | Preferred insert/update substrate after the bounded-batch guard in §9 | It scans the `Dataset` handle's snapshot; asking a current-HEAD handle about an old interval can omit a row changed again later | +| Transaction files | Optional, fail-closed proof that an interval cannot contain logical deletes | They describe physical operations, are reclaimable, and do not persist deleted logical keys | +| `include_deleted_rows()` | Debugging and physical alignment only | It omits compacted tombstones and whole-fragment deletes and returns null `_rowid` for deleted rows | +| Manifest version timestamp | Optional derived publication-time evidence | Writer-clock based, not guaranteed monotonic; enumerating all timestamps reads all retained manifests | +| Tags, branches, and cleanup | Retain exact versions and reclaim old history | Retention can contain holes; it is not one scalar floor shared by all graph tables | + +Primary references: + +- [Lance row ID and lineage specification](https://lance.org/format/table/row_id_lineage/) +- [Lance transaction specification](https://lance.org/format/table/transaction/) +- [Lance versioning guide](https://lance.org/quickstart/versioning/) +- [Lance read/write and cleanup guide](https://lance.org/guide/read_and_write/) +- [Lance 9.0.0 `DatasetDelta` source](https://github.com/lance-format/lance/blob/v9.0.0/rust/lance/src/dataset/delta.rs) + +The `DatasetDelta` source on current Lance `main` is still byte-identical to +the v9.0.0 file surveyed here. There is no merged deleted-row API. Draft +[Lance PR #5002](https://github.com/lance-format/lance/pull/5002) explores +`_row_deleted_at_version`; its continuation +[PR #6671](https://github.com/lance-format/lance/pull/6671) closed without +merging. RFC-030 keeps an adoption seam for a future complete upstream delete +surface but does not depend on either proposal. + +Two details are load-bearing: + +- A delta range is exact only when its base `Dataset` is checked out at the + requested end version. The numeric end version is a row predicate, not a + snapshot pin. A later update or delete on the handle can otherwise change or + remove the result for an older interval. +- There is no native complete delete stream. Persisted `Delete` transactions + carry updated fragments, deleted fragment IDs, and a predicate—not the + logical keys. Merge-delete is represented as `Update` and likewise does not + retain those keys. + +These facts rule out both a custom tombstone interpretation and the claim that +Lance makes the entire graph feed free. Lance owns the table history; OmniGraph +still owns the small amount of graph-level coordination needed to compose it. + +## 2. Existing truth + +The required durable authority already exists: + +- Each successful graph publish atomically writes its `graph_commit` and + `graph_head` rows with the table-version changes they describe. +- A `GraphCommit` records `graph_commit_id`, first parent, optional merge + parent, actor, branch, manifest version, and `created_at`. +- A historical manifest snapshot maps immutable table identity + `(stable_table_id, table_incarnation_id)` to the exact table branch and Lance + version visible at that graph commit. +- Every current v19 graph table was created with stable row IDs and exact + non-null logical `id` as its unenforced primary key. +- `diff_between` and `diff_commits` already know how to compare two graph + snapshots, including table creation, removal, rename, and cross-lineage ID + comparison. + +Two current names must not be allowed to overstate their meaning: + +- `GraphCommit.created_at` is minted before table effects and remains fixed + across retries and recovery. Public CDC calls it **`authored_at`**, not + `committed_at` or `published_at`. Renaming the persisted column is unnecessary + and would create a format change. +- `EntityChange.manifest_version` currently mixes table-local row-version + stamps with a graph-manifest fallback. A graph feed does not carry this field + forward. Commit cause belongs on the block; physical table versions remain + implementation details. + +## 3. Public semantic model + +### 3.1 One logical block per graph commit + +Conceptually, the engine returns: + +```text +GraphChangeBlock { + cause: { + graph_commit_id, + parent_commit_id, + merged_parent_commit_id?, + authored_branch, + graph_snapshot_version, + actor_id?, + authored_at + }, + part, + commit_complete, + changes: [ + { kind, type_name, id, op, endpoints?, before?, after? } + ] +} +``` + +The exact Rust and wire DTOs land with the implementation. The semantics above +are fixed: + +- `kind` is node or edge. +- `type_name` is a graph-schema name, never a dataset name or path. +- `id` is the graph's exact logical `id`. +- `op` is `INSERT`, `UPDATE`, or `DELETE`. +- `endpoints` is present for edge changes, including deletes, and is read from + the appropriate exact snapshot. +- `after` contains the complete user-visible logical row for inserts and + updates and is absent for deletes. +- `before` contains the complete user-visible logical row for deletes and is + absent for inserts and updates. Update before-images are not required to + apply the feed and are deferred. +- Images use the same canonical logical value conversion as graph export. Lance + `_row*` columns and OmniGraph's trusted hidden stream metadata are never + exposed. +- Cause is stated once on the block, not copied onto every entity. +- `authored_branch` is the branch on which the commit originally landed. The + selected feed branch is page/request context; inherited commits on a named + branch do not have their cause rewritten. +- `part` is zero-based and `commit_complete` is true only on the final part of + a commit split across pages. No later commit appears before that terminal + part. Each transmitted part repeats the same cause; parts never mix commits. + +A physical-only graph commit such as compaction produces an empty block. The +cursor still advances over it. A pure type rename also produces no row changes; +future changes use the destination name because immutable identity, not alias, +pairs the table lifetime. + +### 3.2 Branch and merge order + +The feed order is the **first-parent chain of one captured graph branch +incarnation**. It is not a global sort of every commit in the DAG. + +For a merge commit, changes are computed against the first parent: this is the +state transition observed by a consumer tailing the target branch. The merged +parent ID remains on the cause so a DAG-aware caller can inspect it separately. +No `both sides` mode is part of v1. + +The existing `GraphCommit::lineage_key()` remains useful for deterministic +listing and head selection inside one lineage projection. It is not encoded as +the CDC order and is not a cross-branch cursor. + +### 3.3 Graph-level filtering + +Filters may select graph concepts—node/edge kind, graph type name, and +operation. They may not select a Lance dataset, table path, native branch, +fragment, or table version. + +The cursor binds the canonical filter and image contract. Reusing it with a +different filter or image contract fails as `CursorScopeMismatch`; it never +silently skips data. + +## 4. Exact per-commit derivation + +For each first-parent edge `P -> C`: + +1. Load the exact graph snapshots named by `P` and `C`. +2. Pair their table entries by immutable table identity, not alias. +3. Skip identical table branch/version pairs. +4. Derive changes for each remaining table lifetime. + +Logical operation is defined only by the two graph-visible states: + +- absent in `P`, present in `C` → insert; +- present in both with different canonical logical images → update; +- present in `P`, absent in `C` → delete; +- present in both with equal images, or absent in both → no entity change. + +Lance row-version columns are candidate pruning. They do not override this +definition. In particular, overwrite or restore can make a row look physically +new while reusing a logical graph `id` that was already present. + +### 4.1 Table addition and removal + +- A lifetime present only in `C` emits all live end rows as inserts with + after-images. +- A lifetime present only in `P` emits all live begin rows as deletes with + before-images. +- Drop/re-add is two lifetimes even when the public alias and logical IDs are + reused. The internal continuation key includes immutable lifetime identity so + pagination cannot conflate them. + +### 4.2 Inserts and updates on one lifetime + +Open the table at the exact end version selected by `C`. For the table-version +interval `(begin, end]`, stream rows matching Lance's documented row-version +predicates and partition the physical candidates as: + +- insert when `_row_created_at_version > begin`; +- update otherwise when `_row_last_updated_at_version > begin`. + +The adapter treats these rows as candidates. For **every** candidate it performs +a bounded parent membership/image probe: parent absence means insert; parent +presence plus a different logical image means update; equal user-visible images +mean no logical change. This suppresses physical no-ops and +hidden-metadata-only movement as well as closing overwrite and delete/reinsert +cases without turning physical row lineage into graph identity. + +Membership/image checks are coalesced into bounded structured exact-ID batches; +the design does not authorize one object-store round trip per candidate or an +ad-hoc string `IN (...)` filter. + +The adapter prefers `DatasetDelta::get_upserted_rows()` if its runtime surface +passes the projection, blob, and batch-memory gates in §9. Lance 9's convenience +builder hardcodes wildcard projection and does not expose row/byte batch limits. +If that cannot satisfy OmniGraph's bounds, the adapter uses one thin +`DatasetScanner` over the same public version columns and predicates, with the +required projection and batch ceilings. It must not create a second lineage +algorithm. + +Every invocation asserts: + +- the handle is pinned to `end`, not current HEAD; +- stable row IDs and both version columns are genuinely active; +- hidden OmniGraph stream metadata is excluded from the public projection; +- every emitted insert/update image is taken from this exact end handle. + +If table branch lineage changes, the end version does not advance from the +begin version, or the exact transaction interval contains an operation whose +row-version behavior is not proven, the optimization is unavailable. The +correct fallback is a bounded ordered comparison of complete logical rows at +the exact parent and child snapshots. `Restore`, unknown operations, and a lazy +branch fork therefore cannot disappear from CDC merely because their row +version stamps predate the graph commit. + +### 4.3 Deletes + +Correctness is an ordered, bounded merge of live logical IDs from the exact +begin and end snapshots. IDs present only at the begin snapshot are deletes; +their edge endpoints and logical before-images are read from that same begin +snapshot. + +An optimization may skip this comparison only after inspecting every exact +table transaction in the interval and proving that every operation is a mature, +row-set-preserving shape used by OmniGraph. Missing transaction files, cleaned +version holes, `Overwrite`, `Restore`, delete-capable `Update`, unknown/new +operation variants, and experimental Lance operations all mean **unknown** and +fall back to the exact ID comparison. The optimization is never authority. + +Forbidden delete shortcuts: + +- treating Lance's deletion-vector scan as complete; +- parsing a transaction predicate to rediscover keys; +- inferring keys from fragment IDs or row addresses; +- persisting OmniGraph tombstones only to make this reader cheaper. + +### 4.4 Bounds and deterministic continuation + +The engine implementation is streaming. It does not build a delta-wide +`Vec` or a delta-wide set of row images before applying page +limits. + +Each request has three independent ceilings: + +- graph commits examined; +- entity changes returned; +- retained/serialized bytes. + +This prevents a sparse filter from scanning unbounded history merely to fill a +row limit. A commit block is kept whole when it fits. A larger commit is split +at a deterministic internal key composed from immutable table lifetime, +logical ID, and operation rank; the key remains opaque on the wire. Replaying +the same page input against the same retained cut returns the same ordered +events. + +The total event order inside a block is +`(table_key, stable_table_id, table_incarnation_id, id, operation_rank)`. +Stable/incarnation IDs are hidden tie-breakers in the opaque cursor, not public +dataset handles. Operation rank is frozen as `INSERT = 0`, `UPDATE = 1`, +`DELETE = 2` for cursor v1. + +A consumer that needs graph-commit atomicity durably buffers non-terminal +parts and commits that buffer together with the cursor from the +`commit_complete = true` part. It does not apply a partial commit. Retrying from +the cursor before a part replays that part exactly; advancing a durable cursor +without durably retaining the corresponding part is caller data loss, just as +with any caller-owned offset. + +The byte ceiling is chosen at least as large as OmniGraph's maximum legal +logical row image. If historical or blob-backed data still produces one image +larger than that ceiling, the request fails with a typed resource-limit error; +it does not truncate the image or silently switch to keys-only output. + +The implementation must prove the ordering path is bounded. It may use Lance's +ordered scan or a bounded merge, but it may not sort an unbounded graph commit +in memory or depend on unspecified concurrent scan order. + +## 5. Cursor contract + +The wire cursor is opaque and versioned. Its encoding is deliberately not +documented as colon-separated fields. Semantically it binds: + +- graph identity (derived from the persisted schema identity domain); +- lineage/storage-strand incarnation (the first-parent root/genesis commit); +- cursor purpose and traversal direction (`changes/forward` for cursor v1); +- normalized graph branch name; +- Lance native branch identifier, closing delete/recreate ABA; +- canonical filter and logical-image contract digest; +- last completed graph commit; +- a captured upper-cut commit for an in-progress page sequence; +- within-block continuation when the current commit is split; +- cursor format version and corruption checksum. + +The cursor is not an authorization token. Every request is authorized normally, +then the cursor scope is validated. Cursors from different graphs, branch +incarnations, filters, or cursor versions fail loudly and are not comparable. + +On the first poll, the engine captures the branch head as the upper cut. Page +continuations keep that cut even if new commits arrive. Once the cut is reached, +the returned cursor is caught up; the next poll captures a new head and begins +after the last completed commit. This gives one coherent finite replay window +without hiding later work. + +The server persists no cursor or consumer offset. Durability belongs to the +caller. A cursor is not a retention lease: cleanup may reclaim versions after +the cursor is issued. + +### 5.1 Starting a feed + +The first request chooses one explicit start mode: + +- `Now` (default): capture the current head and return a cursor positioned + after it; no accidental replay of the graph's entire history. +- `AfterCommit(id)`: begin after an exact commit that must lie on the captured + branch incarnation's first-parent chain. +- `Beginning`: begin before that chain's root, including inherited history for + a named branch, and fail with a typed gap if the required data is no longer + retained. + +After validating the start, the same coherent branch snapshot supplies the +upper cut. A missing cursor is not an ambiguous alias for `Beginning`. + +### 5.2 Exact bootstrap and reset + +C2 ships one baseline handshake with the public feed: + +```text +capture_change_baseline(branch, feed_scope) -> { + snapshot_commit_id, + exact_graph_snapshot, + resume_cursor +} +``` + +The coordinator validates the filter/image scope, captures one branch +incarnation and head `H`, exports the graph snapshot pinned to `H`, and creates +a cursor equivalent to `AfterCommit(H)` in that same feed scope. Concurrent +commits after `H` are picked up on the next poll. The caller durably installs +the snapshot before it durably installs the resume cursor. If exact snapshot +construction fails or cleanup removes a participant, the handshake returns no +usable cursor. + +This is not today's current-HEAD-only export with a commit ID added afterward; +the export itself is opened at the captured graph commit. The same primitive is +the only supported reset after a retention gap, closing the head-capture/export +race. + +## 6. Retention and failure semantics + +There is no scalar `oldest_readable_manifest_version()` in this RFC. + +Lance cleanup acts independently on physical datasets, may retain tagged or +branched versions, may leave version holes, and OmniGraph cleanup records +per-table failures while allowing other tables to converge. A graph commit is +readable only when every exact table endpoint needed for its transition can be +opened. That is a property of a concrete first-parent edge, not a comparison +against one numeric floor. + +Before deriving an edge, the feed verifies the exact required begin/end table +versions. Failures are translated into graph-level typed outcomes: + +- `HistoricalDataReclaimed { graph_commit_id, type_name }` for direct time + travel to a graph snapshot whose participant is gone; +- `ChangeFeedGap { cursor, first_unreadable_commit_id }` when + a feed cannot continue contiguously. + +The public error names graph concepts. Exact physical paths and table versions +may appear in operator diagnostics/logs, not in the public CDC contract. + +The recovery action is §5.2's exact baseline handshake, not a suggested commit +ID that can race before export. Computing the oldest contiguous resumable suffix +requires walking and validating real participant pins; if a later +implementation exposes that answer, it must cache and cost-test the derivation +rather than guessing from HEAD arithmetic or table minima. + +A page is atomic. If an endpoint becomes unreadable while constructing it, the +request returns the typed gap and no cursor advancement; the caller retries +from its previously durable cursor or deliberately resets from a snapshot. The +engine streams scans into one bounded page buffer, but a transport does not +publish that page or its cursor until construction succeeds. + +`commit list` continues to list durable lineage even when old table data is no +longer readable. This RFC does not add `--mark-readable`: determining that for +every historical commit is a history walk, not a cheap annotation. + +## 7. Time semantics + +Exact time travel by graph commit ID or graph snapshot version remains the +authoritative contract. + +This RFC does **not** add `--at-time` in its core phases. The former proposal +used `GraphCommit.created_at`, called the result committed time, and promised a +binary search. All three parts were wrong: + +- `created_at` is intent/authorship time, minted before effects and recovery; +- Lance's `__manifest` version timestamp is a better publication-time witness + and can be read with `read_version_transaction(version)`; +- neither timestamp is guaranteed monotonic, and `Dataset::versions()` reads + every retained manifest. Lance's own date-range delta resolver performs a + full scan. + +A later publication-time slice may expose `published_at` as derived metadata +and define a deterministic wall-clock selection rule. It must first measure the +cold history cost and state explicitly that writer-clock timestamps are not a +linearizable real-time oracle. It may not introduce a binary search without a +proven monotonic index. + +## 8. Relationship to existing diff + +`diff_between` remains the direct net-current comparison API while the feed is +built. It does not receive optional cause fields: a range collapsed across +multiple commits has no single honest actor or commit. + +Once the graph feed is proven, its exact adjacent enumerator supplies most of +the same machinery. That does **not** make arbitrary-range net diff a free +operation algebra. Update-then-revert must disappear; delete-then-reinsert may +be update relative to the range baseline; table lifetimes can change; and +intermediate history may be reclaimed while both endpoint snapshots remain +readable. + +Any future feed-backed net diff must therefore reduce against baseline +membership/images and final membership/images, not merely fold operation +labels. The direct endpoint snapshot algorithm remains a valid and often +lower-cost reconciliation path. Both paths share table-identity interval +construction, canonical image comparison, and a conformance suite; they do not +grow separate definitions of insert, update, or delete. + +## 9. Evidence gates + +Implementation cannot move a phase to accepted without the matching evidence. +Extend existing owners before adding a new test silo: `changes.rs`, +`point_in_time.rs`, `lineage_projection.rs`, `maintenance.rs`, and +`lance_surface_guards.rs`. + +### L0 — pinned Lance surface + +- Exact-end regression: a row updated in v2 and again in v4 is present with its + v2 value when the delta is run on a v2 handle, and is not trusted when run on + a v4 handle for `(v1, v2]`. +- Stable-row-ID/version-column guard on every OmniGraph-created table. +- `DatasetDelta` batch-row, batch-byte, blob, and hidden-column observations. + If bounds cannot be enforced, select the bounded scanner adapter before C1. +- Exact insert/update after-images and delete before-images, including nested + values, blobs, edge endpoints, and exclusion of every reserved system field. +- Overwrite with an existing logical `id`, delete/reinsert, restore, and a + later update prove graph membership/image comparison—not physical + `_row_created_at_version` alone—selects the logical operation. +- Delete guards covering deletion vectors, whole-fragment delete, update, + merge-delete, compaction, and missing transaction files. +- A source-walk or exhaustive match makes new Lance `Operation` variants fall + back to exact ID comparison until reviewed. + +### G0 — graph semantics + +- Interleaved main/named-branch history follows one branch incarnation's + first-parent chain. +- Merge output is exactly first-parent-relative and carries the merged parent. +- Table rename is row-neutral; drop/re-add with the same alias and IDs emits + distinct deletes/inserts without cursor collision. +- Physical-only commits emit empty blocks and still advance the cursor. +- Existing streaming folds, ordinary mutation/load, schema table add/drop, + branch merge, and recovery-completed publication share the same block model. + +### P0 — cursor and pagination + +- Cursor graph/branch-incarnation/filter/version mismatches are typed refusals. +- Cursor lineage/genesis mismatch is refused even when a rebuilt graph reuses a + schema identity domain or main branch name. +- `Now`, `AfterCommit`, and `Beginning` have exact named-branch inheritance and + reclaimed-history behavior. +- Baseline capture concurrent with a new commit exports exactly captured `H` + and the returned cursor later yields `H + 1`; a failed export yields no usable + cursor. +- New commits arriving between pages do not enter the captured cut. +- Oversized single commits split and replay exactly under row, byte, and + commits-scanned ceilings. +- Sparse filters stop at the commits-scanned bound. +- Reopen and another process can resume from the caller's cursor with no server + state. + +### R0 — retention + +- Cleanup one participant past a required version while retaining others: + exact `ChangeFeedGap`, no partial page, and successful reset only through the + exact snapshot+cursor baseline handshake. +- Tagged/branched holes and per-table cleanup failure do not produce a false + global watermark. +- Direct snapshot access maps reclaimed participant versions to + `HistoricalDataReclaimed` rather than leaking a raw Lance error. + +### C0 — cost + +Using `helpers::cost` and realistic history depth, record separate curves for: + +- O(1) adjacent first-parent classification from an already-loaded child; +- an unchanged warm caught-up poll through the existing freshness probe; +- refresh after one new commit, including the known current + `__manifest` full-fold/history term rather than mislabeling it flat; +- page navigation at increasing backlog depth; +- exact-end insert/update enumeration against increasing table size and + changed-row count; +- parent membership/image probes; +- the no-delete proof for explicitly allowed complete transaction intervals; +- the delete/full-row fallback against increasing table size. + +These are measurements, not pre-declared flatness results. Any flat claim lands +only for the dimensions the instrument proves. The acceptance constraint is +that this RFC adds no O(history log history) binary-lifting state to normal +open, refresh, or existing adjacent `diff_commits`, and that every non-flat +term is documented before its public surface ships. + +## 10. Explicit exclusions and future work + +### 10.1 Graph-schema replay + +The entity feed carries canonical logical row images, but it still cannot +recreate a graph from an empty store by itself. The graph lineage does not +retain the accepted SchemaIR for every commit. Property additions, removals, +renames, constraints, and annotations therefore cannot be reconstructed as an +exact historical schema stream with today's authority. + +A schema-feed extension must decide: + +- historical SchemaIR identity and schema-change event encoding; +- bootstrap semantics for a consumer starting from an empty store; +- whether a full schema snapshot rides every change or only schema commits; +- retention and rebuild behavior for schema history; +- whether the required authority earns an internal-format strand. + +It must not infer graph property identity from Lance field IDs or physical +column names. Until that extension lands, the entity feed is replayable only +against a compatible graph schema established out of band. + +### 10.2 Other exclusions + +- **Delete tombstones:** not needed for retained-history entity CDC; adding + them changes every write path and the storage format. +- **Push delivery:** poll + cursor is the contract. SSE or another transport + may wrap it later without changing semantics. +- **Branch lifecycle events:** branch create/delete are control-plane events, + not graph content commits. +- **Retention pins/checkpoints:** RFC-025's domain. +- **Cross-rebuild history:** export/import rebuild preserves logical graph data, + not the old storage strand's commit feed. +- **History-flat arbitrary catch-up:** requires measured substrate support; no + speculative index or shadow log is authorized here. + +## 11. Format and compatibility audit + +The C0–C3 core below persists nothing and therefore requires no internal-schema +or recovery-schema bump: + +- lineage and table pins already exist; +- runtime path/cursor indexes are derived and rebuildable; +- cursors are caller-owned wire values; +- typed errors and new read APIs are additive. + +The opaque cursor has its own wire version. An unsupported cursor version is a +typed error, not best-effort decoding. + +Any implementation that proposes a stored watermark, feed offset, operation +summary, delete tombstone, or historical SchemaIR changes this conclusion and +must return to this RFC's format audit before landing. + +## 12. Phasing + +| Phase | Ships | Safe stop | +|---|---|---| +| C0 — foundation correction | Identity-keyed table intervals; O(1) adjacent first-parent validation; Lance surface guards; remove speculative binary lifting | No public behavior and no persistent state | +| C1 — engine feed | Internal graph commit blocks with exact logical images, exact-end insert/update adapter, complete delete fallback, bounded page/cursor engine, typed gaps | Engine-only contract can be exercised before wire commitment | +| C2 — graph surfaces | SDK, exact snapshot+cursor baseline, `omnigraph changes`, HTTP/OpenAPI, docs, authorization and parity tests | Useful caller-owned entity feed; compatible schema is established out of band | +| C3 — entity history | Newest-first history derived from the same per-commit enumerator, with a separately versioned `history/backward` cursor | Investigation surface; no new storage authority | +| C4 — publication time, optional | Derived `published_at` and possibly a measured as-of-time selector | Lands only after its semantics and cold-history cost pass §7 | +| C5 — schema replay, separate decision | Historical SchemaIR authority and schema-change events | Requires its own format conclusion before implementation | + +C0 deliberately does **not** add a second coordinator or an O(history log +history) ancestry index. `CommitGraph` already holds the warm lineage +projection; the feed may add at most the minimal first-parent navigation view +whose cost is justified by C1. + +## 13. Resolved decisions + +1. Public unit: graph commit block, not table/dataset delta. +2. Merge default: first parent only. +3. Cause placement: once per block. +4. Commit time field in v1: `authored_at`; no false `committed_at` label. +5. Cursor: opaque, caller-owned, graph/branch/filter/purpose/direction bound, + fixed-cut paging. +6. Delete authority: exact begin/end logical-ID comparison; transaction history + may only prove that comparison unnecessary. +7. Retention: validate concrete participant versions; no scalar watermark. +8. Existing `diff`: distinct net-current API with shared primitives, no + optional multi-commit attribution. +9. Row images: exact after-image for insert/update and exact before-image for + delete; graph-schema replay remains separate. +10. Reset: one exact graph snapshot and its `AfterCommit` cursor are captured + together; a bare head ID is not a safe bootstrap. +11. Format: no bump for the entity feed; revisit before persisting any new + history authority.