diff --git a/README.md b/README.md index 2b2be4f..03601b0 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ All coordination state lives under the `_pg2iceberg` schema in the source (or a | `log_index` | Sparse index of staged Parquet files with offset ranges + LSN | | `mat_cursor` | Materializer progress (last committed offset per table) | | `consumer` | Heartbeat registry for distributed materializer workers | -| `pipeline_meta` | Singleton: source-cluster `system_identifier` (DSN-swap detection) | +| `pipeline_meta` | Singleton: source-cluster `system_identifier` (DSN-swap detection), and the consumer group the schema serves | | `flushed_lsn` | Singleton: highest LSN we've acked the slot to (slot-tamper detection) | | `tables` | Per-table snapshot status + `pg_class.oid` (drop-recreate detection) | | `snapshot_progress` | Per-table mid-snapshot resume cursor | diff --git a/crates/pg2iceberg-coord/src/lib.rs b/crates/pg2iceberg-coord/src/lib.rs index 10c9e8d..523f751 100644 --- a/crates/pg2iceberg-coord/src/lib.rs +++ b/crates/pg2iceberg-coord/src/lib.rs @@ -46,6 +46,16 @@ pub enum CoordError { `pg2iceberg cleanup` first; otherwise verify the source DSN." )] SystemIdMismatch { stored: u64, connected: u64 }, + /// The coordinator schema serves another consumer group. One schema + /// serves one group: the staged log, its snapshot status and its + /// truncation all assume a single reader. + #[error( + "this coordinator schema serves consumer group {holder:?}, not {configured:?}: \ + one schema serves one group. To materialize the same tables into another \ + sink, run another pipeline, with a replication slot and \ + state.coordinator_schema of its own. A group can't be renamed." + )] + GroupMismatch { holder: String, configured: String }, #[error("other: {0}")] Other(String), } @@ -294,6 +304,13 @@ pub trait Coordinator: Send + Sync { /// startup right after `IDENTIFY_SYSTEM`. async fn set_pipeline_system_identifier(&self, sysid: u64) -> Result<()>; + /// Bind the schema to consumer group `group`, as every process that + /// reads or moves cursors does at startup: the first records it; any + /// other group is refused with [`CoordError::GroupMismatch`] — as is + /// one starting on a schema that predates the record and holds cursors + /// of another (a group's own `#…` cursors are its). + async fn claim_group(&self, group: &str) -> Result<()>; + // ── Last-acked LSN (tamper detection) ─────────────────── /// Read the highest LSN we've ever told the replication slot to /// flush past. Returns `Ok(Lsn::ZERO)` if no row exists yet diff --git a/crates/pg2iceberg-coord/src/prod/coord.rs b/crates/pg2iceberg-coord/src/prod/coord.rs index 9155101..f400860 100644 --- a/crates/pg2iceberg-coord/src/prod/coord.rs +++ b/crates/pg2iceberg-coord/src/prod/coord.rs @@ -492,6 +492,44 @@ impl Coordinator for PostgresCoordinator { Ok(()) } + async fn claim_group(&self, group: &str) -> Result<()> { + // One transaction, the pipeline-meta row locked: concurrent + // claims of different groups can't both win. + let mut client = self.client().await?; + let tx = client.transaction().await.map_err(pg)?; + tx.execute(&sql::ensure_pipeline_meta(&self.schema), &[]) + .await + .map_err(pg)?; + let holder: Option = tx + .query_one(&sql::select_pipeline_group(&self.schema), &[]) + .await + .map_err(pg)? + .get(0); + let holder = match holder { + Some(holder) => Some(holder), + // Claimed by none yet: another group's cursors hold it anyway. + None => tx + .query(&sql::other_cursor_groups(&self.schema), &[&group]) + .await + .map_err(pg)? + .first() + .map(|r| r.get::<_, String>(0)), + }; + match holder { + Some(holder) if holder != group => Err(CoordError::GroupMismatch { + holder, + configured: group.to_string(), + }), + _ => { + tx.execute(&sql::set_pipeline_group(&self.schema), &[&group]) + .await + .map_err(pg)?; + tx.commit().await.map_err(pg)?; + Ok(()) + } + } + } + async fn flushed_lsn(&self) -> Result { let client = self.client().await?; let rows = client diff --git a/crates/pg2iceberg-coord/src/sql.rs b/crates/pg2iceberg-coord/src/sql.rs index 739384c..0d5671f 100644 --- a/crates/pg2iceberg-coord/src/sql.rs +++ b/crates/pg2iceberg-coord/src/sql.rs @@ -101,6 +101,12 @@ pub fn migrate(schema: &CoordSchema) -> Vec { )", schema.qualify("pipeline_meta") ), + // The consumer group the schema serves (`claim_group`); NULL + // until a process claims it. + format!( + "ALTER TABLE {} ADD COLUMN IF NOT EXISTS consumer_group TEXT", + schema.qualify("pipeline_meta") + ), // Single-row record of the highest LSN we've ever told the // replication slot to flush past. Updated *before* every // `send_standby`, so on a crash between the two writes the @@ -409,6 +415,39 @@ pub fn upsert_pipeline_meta(schema: &CoordSchema) -> String { ) } +/// Ensure the pipeline-meta row, so `claim_group` can lock it. +pub fn ensure_pipeline_meta(schema: &CoordSchema) -> String { + format!( + "INSERT INTO {} (id) VALUES (1) ON CONFLICT (id) DO NOTHING", + schema.qualify("pipeline_meta") + ) +} + +/// The claimed consumer group, the row locked until the transaction ends. +pub fn select_pipeline_group(schema: &CoordSchema) -> String { + format!( + "SELECT consumer_group FROM {} WHERE id = 1 FOR UPDATE", + schema.qualify("pipeline_meta") + ) +} + +/// Groups other than `$1` with cursors: a group's own cursors are named +/// `group` and `group#…`. +pub fn other_cursor_groups(schema: &CoordSchema) -> String { + format!( + "SELECT DISTINCT split_part(group_name, '#', 1) AS g FROM {} \ + WHERE split_part(group_name, '#', 1) <> $1 ORDER BY g", + schema.qualify("mat_cursor") + ) +} + +pub fn set_pipeline_group(schema: &CoordSchema) -> String { + format!( + "UPDATE {} SET consumer_group = $1, updated_at = now() WHERE id = 1", + schema.qualify("pipeline_meta") + ) +} + // ── Last-acked LSN (single-row tamper detection) ────────────────────── pub fn select_flushed_lsn(schema: &CoordSchema) -> String { @@ -562,14 +601,15 @@ mod tests { let stmts = migrate(&s); // 6 base CREATE TABLEs (log_seq, log_index, mat_cursor, // consumer, pending_markers, marker_emissions) - // + 1 ALTER TABLE (log_index.flushable_lsn) + // + 2 ALTER TABLEs (log_index.flushable_lsn, + // pipeline_meta.consumer_group) // + 1 DROP TABLE (the retired `lock` table) // + 5 new per-concern tables (pipeline_meta, flushed_lsn, // replicated_lsn, tables, snapshot_progress) // + table_epoch + writes_in_flight - // = 15. Each statement is idempotent (CREATE TABLE IF NOT + // = 16. Each statement is idempotent (CREATE TABLE IF NOT // EXISTS, ADD COLUMN IF NOT EXISTS or DROP TABLE IF EXISTS). - assert_eq!(stmts.len(), 15); + assert_eq!(stmts.len(), 16); for stmt in &stmts { assert!( stmt.contains("IF NOT EXISTS") || stmt.contains("IF EXISTS"), diff --git a/crates/pg2iceberg-logical/src/materializer.rs b/crates/pg2iceberg-logical/src/materializer.rs index e6f880d..b20fec9 100644 --- a/crates/pg2iceberg-logical/src/materializer.rs +++ b/crates/pg2iceberg-logical/src/materializer.rs @@ -119,6 +119,17 @@ pub enum MaterializerError { /// update never happened. Handled within a cycle, never returned. #[error("log already applied up to offset {end}")] AlreadyApplied { end: u64 }, + /// The table's log starts past its cursor: entries it hasn't applied + /// are gone ([`Materializer::log_continues`]). + #[error( + "{table}'s staged log starts at offset {start}, past its cursor at {cursor}: \ + changes it hasn't applied are gone, and the table needs a new snapshot" + )] + LogGap { + table: TableIdent, + cursor: i64, + start: u64, + }, } pub type Result = std::result::Result; @@ -1575,8 +1586,13 @@ impl Materializer { snapshot: true, ..Unit::default() }; + let group = self.snapshot_group(); let mut log = self.staged_log(ident, from.max(0) as u64); + let mut first = true; while let Some((e, bytes)) = log.next().await? { + if std::mem::take(&mut first) && !self.log_continues(ident, &group, from, &e).await? { + return Ok(0); + } let rows: Vec = decode_chunk(&bytes)? .into_iter() .filter(|evt| is_snapshot_xid(evt.xid)) @@ -2762,6 +2778,40 @@ impl Materializer { Ok(folded) } + /// Whether `first`, the first entry read past `cursor` of `group`, + /// starts where the cursor sits, as entries follow one another. If + /// not, those in between are gone: fine if the cursor has moved past + /// them since it was read — another process applied and truncated + /// them, a worker that owned the table too for a moment — and this + /// cycle has nothing to do (`false`); otherwise they're lost, and + /// it's a [`MaterializerError::LogGap`]. + async fn log_continues( + &self, + ident: &TableIdent, + group: &str, + cursor: i64, + first: &LogEntry, + ) -> Result { + if first.start_offset == cursor.max(0) as u64 { + return Ok(true); + } + let now = self.coord.get_cursor(group, ident).await?.unwrap_or(-1); + if now > cursor { + tracing::info!( + table = %ident, + cursor, + now, + "another process applied the log meanwhile" + ); + return Ok(false); + } + Err(MaterializerError::LogGap { + table: ident.clone(), + cursor, + start: first.start_offset, + }) + } + /// Truncating the table's log ([`truncate_log`]): once this process /// starts, then whenever it has applied the log past where it's /// truncated to. Not before the table's backfill is looked up: that @@ -2777,7 +2827,10 @@ impl Materializer { let coord = self.coord.clone(); let blob_store = self.blob_store.clone(); let ident = ident.clone(); - async move { truncate_log(coord.as_ref(), blob_store.as_ref(), &ident, due?).await } + // The group's cursors over the log: its changes', and its + // backfill's while one applies. + let groups = [self.group.clone(), self.snapshot_group()]; + async move { truncate_log(coord.as_ref(), blob_store.as_ref(), &ident, &groups, due?).await } } /// Fold and commit the table's changes past its cursor, in units. @@ -2796,6 +2849,9 @@ impl Materializer { let mut read_any = false; let mut log = self.staged_log(ident, cursor.max(0) as u64); while let Some((e, bytes)) = log.next().await? { + if !read_any && !self.log_continues(ident, &self.group, cursor, &e).await? { + return Ok(0); + } read_any = true; let events = decode_chunk(&bytes)?; // A schema change starts its entry (the pipeline stages it @@ -3459,11 +3515,11 @@ fn now_micros() -> i64 { .unwrap_or(0) } -/// Drop `ident`'s log before the lowest cursor reading it, and delete the -/// files those entries staged: every reader has applied them, and none -/// reads them again. Every group that registered the table holds its log — -/// its changes' cursor, and its backfill's while one applies. The entry at -/// the cursor stays, so the log still shows where it's applied to. At most +/// Drop `ident`'s log before the lowest of `groups`' cursors — the consumer +/// group's, which the schema serves alone ([`Coordinator::claim_group`]) — +/// and delete the files those entries staged: they're applied, and nothing +/// reads them again. The entry at the cursor stays, so the log still shows +/// where it's applied to. At most /// [`TRUNCATE_BATCH`] entries at once, past `log_start`, where it was /// truncated to before. Returns where it's truncated to now, unless it /// left some for next time. Best effort: what this leaves the next one @@ -3472,6 +3528,7 @@ async fn truncate_log( coord: &dyn Coordinator, blob_store: &dyn BlobStore, ident: &TableIdent, + groups: &[String], log_start: i64, ) -> Option { let cursors = match coord.table_cursors(ident).await { @@ -3483,7 +3540,7 @@ async fn truncate_log( }; let keep_from = cursors .into_iter() - .filter(|(group, _)| !group.ends_with(SNAPSHOT_LSN_SUFFIX)) + .filter(|(group, _)| groups.contains(group)) .map(|(_, cursor)| cursor) .min()?; if keep_from <= log_start.max(0) { diff --git a/crates/pg2iceberg-sim/src/coord.rs b/crates/pg2iceberg-sim/src/coord.rs index 677d5ce..2525cb5 100644 --- a/crates/pg2iceberg-sim/src/coord.rs +++ b/crates/pg2iceberg-sim/src/coord.rs @@ -50,6 +50,8 @@ struct State { consumers: BTreeMap<(String, WorkerId), Timestamp>, /// `pipeline_meta.system_identifier` (zero = unset). system_identifier: u64, + /// `pipeline_meta.consumer_group` (`None` = unclaimed). + consumer_group: Option, /// `flushed_lsn.lsn` — highest LSN we've ever acked. Compared to /// `slot.confirmed_flush_lsn` at startup to catch external slot /// advancement. @@ -73,11 +75,15 @@ struct State { marker_emissions: BTreeSet<(String, TableIdent)>, } +/// Work [`MemoryCoordinator::before_next_read_log`] runs. +type Interlude = Box; + #[derive(Clone)] pub struct MemoryCoordinator { schema: CoordSchema, clock: Arc, state: Arc>, + before_read_log: Arc>>, } impl MemoryCoordinator { @@ -86,6 +92,7 @@ impl MemoryCoordinator { schema, clock, state: Arc::new(Mutex::new(State::default())), + before_read_log: Arc::default(), } } @@ -100,6 +107,13 @@ impl MemoryCoordinator { &self.schema } + /// Test hook: run `f` just before the next `read_log` reads — as + /// another process's work landing between a read of the cursor and of + /// the log. `f` may call this coordinator. + pub fn before_next_read_log(&self, f: impl FnOnce() + Send + 'static) { + *self.before_read_log.lock().unwrap() = Some(Box::new(f)); + } + /// Test hook: from now on, `set_cursor` for `groups` fails, writing /// nothing — as for a process that dies before its cursor write /// reaches the coordinator. An empty `groups` ends it. @@ -238,6 +252,10 @@ impl Coordinator for MemoryCoordinator { if i64::try_from(limit).is_err() { return Err(CoordError::Other("limit > i64".into())); } + let interlude = self.before_read_log.lock().unwrap().take(); + if let Some(f) = interlude { + f(); + } let state = self.state.lock().unwrap(); let mut out: Vec = state .log @@ -387,6 +405,29 @@ impl Coordinator for MemoryCoordinator { Ok(()) } + async fn claim_group(&self, group: &str) -> Result<()> { + let mut state = self.state.lock().unwrap(); + // Claimed by none yet: another group's cursors hold it anyway. + let holder = state.consumer_group.clone().or_else(|| { + state + .cursors + .keys() + .map(|(g, _)| g.split('#').next().unwrap_or_default().to_string()) + .filter(|g| g != group) + .min() + }); + match holder { + Some(holder) if holder != group => Err(CoordError::GroupMismatch { + holder, + configured: group.to_string(), + }), + _ => { + state.consumer_group = Some(group.to_string()); + Ok(()) + } + } + } + async fn flushed_lsn(&self) -> Result { Ok(self.state.lock().unwrap().flushed_lsn) } diff --git a/crates/pg2iceberg-sim/src/fault.rs b/crates/pg2iceberg-sim/src/fault.rs index 203a86a..57256b1 100644 --- a/crates/pg2iceberg-sim/src/fault.rs +++ b/crates/pg2iceberg-sim/src/fault.rs @@ -335,6 +335,10 @@ impl Coordinator for FaultyCoordinator { self.inner.set_pipeline_system_identifier(sysid).await } + async fn claim_group(&self, group: &str) -> CoordResult<()> { + self.inner.claim_group(group).await + } + async fn flushed_lsn(&self) -> CoordResult { self.inner.flushed_lsn().await } diff --git a/crates/pg2iceberg-sim/tests/coord_invariants.rs b/crates/pg2iceberg-sim/tests/coord_invariants.rs index f4ceeb0..a4b7a40 100644 --- a/crates/pg2iceberg-sim/tests/coord_invariants.rs +++ b/crates/pg2iceberg-sim/tests/coord_invariants.rs @@ -4,7 +4,9 @@ //! — the future PG impl will be tested against the same suite via a shared //! `coord_conformance` test crate. -use pg2iceberg_coord::{schema::CoordSchema, CommitBatch, Coordinator, OffsetClaim, StagedLogSize}; +use pg2iceberg_coord::{ + schema::CoordSchema, CommitBatch, CoordError, Coordinator, OffsetClaim, StagedLogSize, +}; use pg2iceberg_core::{Lsn, Namespace, TableIdent, WorkerId}; use pg2iceberg_sim::{clock::TestClock, coord::MemoryCoordinator}; use pollster::block_on; @@ -223,6 +225,33 @@ fn staged_log_size_counts_what_the_log_holds() { // ---------- cursor semantics ---------- +#[test] +fn claim_group_binds_the_schema_to_one_group() { + let (c, _) = coord_at_t0(); + block_on(c.claim_group("a")).unwrap(); + block_on(c.claim_group("a")).unwrap(); + let refused = block_on(c.claim_group("b")).unwrap_err(); + assert!( + matches!(&refused, CoordError::GroupMismatch { holder, configured } if holder == "a" && configured == "b"), + "{refused}" + ); +} + +#[test] +fn claim_group_refuses_a_schema_holding_another_groups_cursors() { + let (c, _) = coord_at_t0(); + // Cursors from before the schema recorded its group. + block_on(c.ensure_cursor("old#snapshot", &ident("t"))).unwrap(); + block_on(c.ensure_cursor("old", &ident("t"))).unwrap(); + let refused = block_on(c.claim_group("new")).unwrap_err(); + assert!( + matches!(&refused, CoordError::GroupMismatch { holder, .. } if holder == "old"), + "{refused}" + ); + // Its own, `#…` ones too, are no obstacle. + block_on(c.claim_group("old")).unwrap(); +} + #[test] fn table_cursors_lists_every_groups_cursor_of_the_table() { let (c, _) = coord_at_t0(); diff --git a/crates/pg2iceberg-tests/tests/dst.rs b/crates/pg2iceberg-tests/tests/dst.rs index d9a8621..a965841 100644 --- a/crates/pg2iceberg-tests/tests/dst.rs +++ b/crates/pg2iceberg-tests/tests/dst.rs @@ -6455,62 +6455,70 @@ fn materializer_idempotent_when_run_extra_times() { check_invariants(&mut h).unwrap(); } -/// Two consumer groups materialize one staged log, each into a catalog of -/// its own. The one ahead truncates the log only as far as the one behind -/// has read it: a group that registered the table holds its log. +/// Entries a table hasn't applied, gone from its log — however that came +/// about — fail its cycle, loudly: what they held is lost, and the table +/// needs a new snapshot. Reading on from what's left would lose them +/// silently. #[test] -fn a_group_behind_holds_the_log() { +fn a_gap_in_the_log_fails_the_cycle() { let mut h = DstHarness::boot(); - let catalog = Arc::new(MemoryCatalog::new()); - let mut behind = Materializer::new( - h.coord.clone() as Arc, - h.blob_store.clone(), - catalog.clone(), - mat_namer(&h.id_gen), - "behind", - MAT_BATCH, - ); - block_on(behind.register_table(schema())).unwrap(); - for step in [ Step::Insert { id: 1, qty: 10 }, - Step::Insert { id: 2, qty: 20 }, Step::DriveFlush, Step::MaterializerCycle, - Step::Insert { id: 3, qty: 30 }, + Step::Insert { id: 2, qty: 20 }, Step::DriveFlush, - Step::MaterializerCycle, - Step::Update { id: 1, qty: 11 }, - Step::Insert { id: 4, qty: 40 }, + Step::Insert { id: 3, qty: 30 }, Step::DriveFlush, - Step::MaterializerCycle, ] { h.run_step(&step); } - // Ahead and caught up, the group still finds the log the other hasn't - // read: all of it. - assert_eq!( - block_on(h.coord.read_log(&ident(), 0, 1_000_000)).unwrap(), - h.coord.claimed_log(&ident()), - "the group ahead truncated what the group behind hasn't read" - ); - while block_on(behind.cycle()).unwrap() > 0 {} - - let files = live_files_from_history(&catalog.history(&ident())); - let schema = block_on(catalog.load_table(&ident())) + let cursor = block_on(h.coord.get_cursor("default", &ident())) .unwrap() - .expect("table exists") - .schema; - let mut behinds = block_on(engine_read(h.blob_store.as_ref(), &schema, &files)).unwrap(); - sort_by_pk(&mut behinds); - let mut pg = h.db.read_table(&ident()).map(stored_rows).unwrap(); - sort_by_pk(&mut pg); - assert_eq!( - on_source_columns(&h.db, behinds), - on_source_columns(&h.db, pg), - "the group behind lost changes the group ahead truncated" + .unwrap(); + let unapplied = block_on(h.coord.read_log(&ident(), cursor as u64, 1_000_000)).unwrap(); + assert!(unapplied.len() > 1, "{unapplied:?}"); + // The first entry past the cursor goes (with the applied one before). + block_on( + h.coord + .truncate_log(&ident(), unapplied[0].end_offset, 1_000), + ) + .unwrap(); + + let failed = block_on(h.materializer.cycle()).expect_err("the cycle read past the gap"); + assert!( + failed + .to_string() + .contains("the table needs a new snapshot"), + "{failed}" ); - check_invariants(&mut h).unwrap(); +} + +/// Another process — a worker that owned the table too, for a moment — +/// applies the table's log and truncates it between this one reading the +/// cursor and reading the log. The gap that leaves is no loss: the cycle +/// finds nothing to do, rather than failing for a new snapshot. +#[test] +fn a_gap_another_process_left_is_no_loss() { + let mut h = DstHarness::boot(); + for step in [ + Step::Insert { id: 1, qty: 10 }, + Step::DriveFlush, + Step::MaterializerCycle, + Step::Insert { id: 2, qty: 20 }, + Step::DriveFlush, + Step::Insert { id: 3, qty: 30 }, + Step::DriveFlush, + ] { + h.run_step(&step); + } + let end = h.coord.claimed_log(&ident()).last().unwrap().end_offset; + let coord = h.coord.clone(); + h.coord.before_next_read_log(move || { + block_on(coord.set_cursor("default", &ident(), end as i64)).unwrap(); + block_on(coord.truncate_log(&ident(), end - 1, 1_000)).unwrap(); + }); + assert_eq!(block_on(h.materializer.cycle()).unwrap(), 0); } /// A cycle that stops at its row budget leaves a backlog: truncating the diff --git a/crates/pg2iceberg-tests/tests/dst_concurrency.rs b/crates/pg2iceberg-tests/tests/dst_concurrency.rs index f8fda53..c935ddc 100644 --- a/crates/pg2iceberg-tests/tests/dst_concurrency.rs +++ b/crates/pg2iceberg-tests/tests/dst_concurrency.rs @@ -357,6 +357,9 @@ impl Coordinator for OneConnection { self.request(self.inner.set_pipeline_system_identifier(sysid)) .await } + async fn claim_group(&self, group: &str) -> CoordResult<()> { + self.request(self.inner.claim_group(group)).await + } async fn flushed_lsn(&self) -> CoordResult { self.request(self.inner.flushed_lsn()).await } diff --git a/crates/pg2iceberg-tests/tests/dst_fault.rs b/crates/pg2iceberg-tests/tests/dst_fault.rs index b1c1ff4..aec451f 100644 --- a/crates/pg2iceberg-tests/tests/dst_fault.rs +++ b/crates/pg2iceberg-tests/tests/dst_fault.rs @@ -1964,6 +1964,40 @@ fn seeded_db() -> SimPostgres { db } +/// A process of another consumer group than the one the coordinator +/// schema serves refuses to start, before it reads or moves a cursor: the +/// schema's log, snapshot status and truncation assume one group. +#[tokio::test(start_paused = true)] +async fn another_consumer_group_is_refused() { + let db = seeded_db(); + let stores = SimStores::new(); + pg2iceberg_validate::run_logical_lifecycle( + sim_lifecycle_on(&db, &stores, every_second()), + Box::pin(std::future::ready(())), + ) + .await + .expect("the first start"); + + let mut other = sim_lifecycle_on(&db, &stores, every_second()); + other.group = "other".into(); + let refused = + pg2iceberg_validate::run_logical_lifecycle(other, Box::pin(std::future::ready(()))) + .await + .expect_err("another group started"); + assert!( + refused + .to_string() + .contains("serves consumer group \"default\", not \"other\""), + "{refused}" + ); + pg2iceberg_validate::run_logical_lifecycle( + sim_lifecycle_on(&db, &stores, every_second()), + Box::pin(std::future::ready(())), + ) + .await + .expect("the group's own next start"); +} + /// The coordinator can't take the LSN the standby tick would ack. The /// slot must not be acked past what's recorded: the next start would find /// it ahead, and refuse to start as if it had been tampered with. diff --git a/crates/pg2iceberg-tests/tests/integration_coord.rs b/crates/pg2iceberg-tests/tests/integration_coord.rs index fac793c..0ec69a0 100644 --- a/crates/pg2iceberg-tests/tests/integration_coord.rs +++ b/crates/pg2iceberg-tests/tests/integration_coord.rs @@ -28,7 +28,7 @@ use std::time::Duration; use pg2iceberg_coord::{ prod::{connect_with, PostgresCoordinator, TlsMode}, schema::CoordSchema, - CommitBatch, Coordinator, MarkerInfo, OffsetClaim, StagedLogSize, + CommitBatch, CoordError, Coordinator, MarkerInfo, OffsetClaim, StagedLogSize, }; use pg2iceberg_core::{Lsn, Namespace, TableIdent, WorkerId}; use testcontainers_modules::postgres::Postgres; @@ -469,6 +469,36 @@ async fn cursor_round_trip() { assert!(matches!(err, pg2iceberg_coord::CoordError::NotFound(_))); } +#[tokio::test] +async fn claim_group_binds_the_schema_to_one_group() { + let coord = fresh_coord().await; + coord.claim_group("a").await.unwrap(); + coord.claim_group("a").await.unwrap(); + let refused = coord.claim_group("b").await.unwrap_err(); + assert!( + matches!(&refused, CoordError::GroupMismatch { holder, configured } if holder == "a" && configured == "b"), + "{refused}" + ); + // The refusal changed nothing. + coord.claim_group("a").await.unwrap(); +} + +#[tokio::test] +async fn claim_group_refuses_a_schema_holding_another_groups_cursors() { + let coord = fresh_coord().await; + let t = ident("public", "orders"); + // Cursors from before the schema recorded its group. + coord.ensure_cursor("old#snapshot", &t).await.unwrap(); + coord.ensure_cursor("old", &t).await.unwrap(); + let refused = coord.claim_group("new").await.unwrap_err(); + assert!( + matches!(&refused, CoordError::GroupMismatch { holder, .. } if holder == "old"), + "{refused}" + ); + // Its own, `#…` ones too, are no obstacle. + coord.claim_group("old").await.unwrap(); +} + #[tokio::test] async fn table_cursors_lists_every_groups_cursor_of_the_table() { let coord = fresh_coord().await; diff --git a/crates/pg2iceberg-validate/src/instrument.rs b/crates/pg2iceberg-validate/src/instrument.rs index b8c6540..b1519c1 100644 --- a/crates/pg2iceberg-validate/src/instrument.rs +++ b/crates/pg2iceberg-validate/src/instrument.rs @@ -422,6 +422,11 @@ impl Coordinator for Instrumented { .await } + async fn claim_group(&self, group: &str) -> pg2iceberg_coord::Result<()> { + self.coord("claim_group", self.inner.claim_group(group)) + .await + } + async fn flushed_lsn(&self) -> pg2iceberg_coord::Result { self.coord("flushed_lsn", self.inner.flushed_lsn()).await } diff --git a/crates/pg2iceberg-validate/src/runtime.rs b/crates/pg2iceberg-validate/src/runtime.rs index 82b3f77..cedcd73 100644 --- a/crates/pg2iceberg-validate/src/runtime.rs +++ b/crates/pg2iceberg-validate/src/runtime.rs @@ -485,6 +485,12 @@ async fn start( .await .map_err(|e| LifecycleError::Coord(e.to_string()))?; } + // One schema serves one consumer group: refuse another before it + // reads or moves a cursor. + lc.coord + .claim_group(&lc.group) + .await + .map_err(|e| LifecycleError::Coord(e.to_string()))?; // 2. Startup validation. Refuses to start on any of the // invariants `pipeline/validate.go` enforces. Cluster diff --git a/crates/pg2iceberg/src/config.rs b/crates/pg2iceberg/src/config.rs index a595944..309874c 100644 --- a/crates/pg2iceberg/src/config.rs +++ b/crates/pg2iceberg/src/config.rs @@ -681,7 +681,9 @@ pub struct StateConfig { pub postgres_url: String, #[serde(default = "default_coord_schema")] pub coordinator_schema: String, - /// Materialization group name (consumer / mat_cursor key). + /// Materialization group name (consumer / mat_cursor key): the + /// distributed workers of one pipeline. A coordinator schema serves + /// one group ([`pg2iceberg_coord::Coordinator::claim_group`]). /// Defaults to `"default"`. #[serde(default = "default_group")] pub group: String, @@ -698,6 +700,20 @@ impl Default for StateConfig { } } +impl StateConfig { + /// A group names its cursors `group` and `group#…`: `#` would make one + /// group's cursors look like another's. + pub fn validate_group(&self) -> Result<()> { + if self.group.is_empty() || self.group.contains('#') { + anyhow::bail!( + "state.group must be a non-empty name without '#', not {:?}", + self.group + ); + } + Ok(()) + } +} + fn default_coord_schema() -> String { "_pg2iceberg".into() } @@ -753,6 +769,7 @@ impl Config { self.sink.validate_compaction()?; self.sink.validate_materializer()?; self.source.logical.validate_snapshot()?; + self.state.validate_group()?; self.metrics_addr()?; self.liveness_timeout()?; self.sink.resolved_maintenance()?; @@ -1375,6 +1392,17 @@ state: assert!(err.contains("snapshot_concurrency"), "{err}"); } + #[test] + fn group_is_a_name_without_hashes() { + let mut cfg: Config = serde_yaml::from_str(SAMPLE).unwrap(); + cfg.state.validate_group().unwrap(); + for bad in ["", "a#b"] { + cfg.state.group = bad.into(); + let err = cfg.state.validate_group().unwrap_err().to_string(); + assert!(err.contains("state.group"), "{err}"); + } + } + #[test] fn materializer_concurrency_is_at_least_one() { let mut cfg: Config = serde_yaml::from_str(SAMPLE).unwrap(); diff --git a/crates/pg2iceberg/src/run.rs b/crates/pg2iceberg/src/run.rs index 96edc4c..5e9eadf 100644 --- a/crates/pg2iceberg/src/run.rs +++ b/crates/pg2iceberg/src/run.rs @@ -421,7 +421,7 @@ pub async fn run_materializer_only( } let mut materializer = traced( pg2iceberg_logical::work_span!("startup"), - build_one_shot_materializer(cfg.clone(), metrics.clone()), + build_one_shot_materializer(cfg.clone(), metrics.clone(), Cursors::Claim), ) .await?; materializer.set_catalog_cache_ttl( @@ -567,7 +567,8 @@ async fn compact(cfg: Config) -> Result<()> { Set a non-zero value (e.g. 134217728 for 128 MiB) to enable." ); } - let mut materializer = build_one_shot_materializer(cfg.clone(), Arc::new(NoopMetrics)).await?; + let mut materializer = + build_one_shot_materializer(cfg.clone(), Arc::new(NoopMetrics), Cursors::Untouched).await?; // Its passes' writes in flight aren't those of `run`'s passes. materializer.set_compaction_writer(pg2iceberg_logical::materializer::writers::COMPACT_COMMAND); let cfg_compact = cfg.sink.compaction_config(); @@ -631,7 +632,8 @@ async fn maintain(cfg: Config, retention_override: Option) -> Result<()> ); } - let mut materializer = build_one_shot_materializer(cfg.clone(), Arc::new(NoopMetrics)).await?; + let mut materializer = + build_one_shot_materializer(cfg.clone(), Arc::new(NoopMetrics), Cursors::Untouched).await?; // Step 1: snapshot expiry. if !retention_str.is_empty() { @@ -717,9 +719,20 @@ fn instrument( /// opening the replication slot or installing signal handlers — for /// one-shot subcommands (`compact`, `maintain`) and `materializer-only`, /// recording into `metrics`. +/// Whether a one-shot materializer reads and moves the tables' cursors. +#[derive(Clone, Copy, PartialEq)] +enum Cursors { + /// It does (`materializer-only`): the schema must serve its consumer + /// group, which it claims ([`Coordinator::claim_group`]). + Claim, + /// It doesn't (`compact`, `maintain`). + Untouched, +} + async fn build_one_shot_materializer( cfg: Config, metrics: Arc, + cursors: Cursors, ) -> Result> { cfg.require_catalog()?; let catalog = IcebergRustCatalog::new(Arc::new(build_rest_catalog(&cfg).await?)) @@ -737,7 +750,17 @@ async fn build_one_shot_materializer( .await .context("coord connect")?; let coord_schema = CoordSchema::sanitize(&cfg.state.coordinator_schema); - let coord: Arc = Arc::new(PostgresCoordinator::new(coord_conn, coord_schema)); + let concrete = PostgresCoordinator::new(coord_conn, coord_schema); + if cursors == Cursors::Claim { + // Started before the process that replicates, it migrates the + // schema itself: the claim needs its newest columns. + concrete.migrate().await.context("coord migrate")?; + concrete + .claim_group(&cfg.state.group) + .await + .context("claim the consumer group")?; + } + let coord: Arc = Arc::new(concrete); let (catalog, blob, coord) = instrument(catalog, storage.blob, coord, &metrics); // Resolve schemas. Discovery requires PG; tables with explicit @@ -1054,6 +1077,10 @@ async fn snapshot_only(cfg: Config, metrics: Arc) -> Result<()> { let coord_schema = CoordSchema::sanitize(&cfg.state.coordinator_schema); let coord_concrete = Arc::new(PostgresCoordinator::new(coord_conn, coord_schema)); coord_concrete.migrate().await.context("coord migrate")?; + coord_concrete + .claim_group(&cfg.state.group) + .await + .context("claim the consumer group")?; let (catalog, blob, coord) = instrument(catalog, blob, coord_concrete, &metrics); // ── pg + slot ────────────────────────────────────────────────── diff --git a/docs/architecture/distributed.md b/docs/architecture/distributed.md index 40dbe25..18639db 100644 --- a/docs/architecture/distributed.md +++ b/docs/architecture/distributed.md @@ -59,7 +59,7 @@ A worker leaving (crash or graceful shutdown) is detected within one heartbeat T `mat_cursor` is keyed by `(group_name, table_name)` — workers within the same group share cursors per table because the deterministic round-robin assignment guarantees only one worker materializes any given table per cycle. There's no `worker_id` in the cursor key; ownership is recomputed at each cycle's start, not persisted. -The log is truncated behind the lowest cursor of any group that registered the table, so a group holds the log it hasn't read yet — from when it registers the table: a group added later starts where the log starts then. +A coordinator schema serves one consumer group. The first process to start records its `state.group`; a process configured with another refuses to start. The staged log, the tables' snapshot status and the log's truncation all assume one reader, so to materialize the same tables into another sink, run another pipeline — with a replication slot and `state.coordinator_schema` of its own. A group can't be renamed: its name is in its cursors and in the tables' Iceberg metadata. ## Commit isolation diff --git a/docs/architecture/flusher.md b/docs/architecture/flusher.md index 70b9f6f..c0bd383 100644 --- a/docs/architecture/flusher.md +++ b/docs/architecture/flusher.md @@ -43,7 +43,7 @@ The coordinator role is filled by the **source PostgreSQL database** itself. Thi | `log_index` | Sparse index: maps offset ranges to staged-Parquet S3 paths + LSNs | | `mat_cursor` | Per-`(group, table)` cursor tracking the last materialized offset | | `consumer` | Heartbeat registry for distributed materializer workers | -| `pipeline_meta` | Singleton: source-cluster `system_identifier` | +| `pipeline_meta` | Singleton: source-cluster `system_identifier`, and the consumer group the schema serves | | `flushed_lsn` | Singleton: highest LSN we've acked the slot to | | `tables` | Per-table snapshot status + `pg_class.oid` | | `snapshot_progress` | Per-table mid-snapshot resume cursor | @@ -66,7 +66,7 @@ The materializer reads `log_index` entries with offsets greater than its cursor, **Truncate** (materializer → log): -Once a table's entries are applied, the materializer drops them from `log_index` and deletes their files — while its next cycle applies more, since the two touch different entries. It truncates before the lowest cursor reading the log: every consumer group that registered the table holds it, and so does a backfill until its rows are applied. The entry that cursor sits at the end of stays, so the log still shows how far it's applied. A log grown long under a release that didn't truncate it is worked off at most 1,000 entries a table a cycle; a file whose delete fails stays behind. `pg2iceberg_staged_log_files` and `pg2iceberg_staged_log_bytes` gauge what each table's log still holds. +Once a table's entries are applied, the materializer drops them from `log_index` and deletes their files — while its next cycle applies more, since the two touch different entries. It truncates before the table's cursor — or its backfill's, until a backfill's rows are applied. The entry that cursor sits at the end of stays, so the log still shows how far it's applied, and a materializer that finds its log starting past its cursor stops with an error rather than read on: what's missing can't be applied, and the table needs a new snapshot. A log grown long under a release that didn't truncate it is worked off at most 1,000 entries a table a cycle; a file whose delete fails stays behind. `pg2iceberg_staged_log_files` and `pg2iceberg_staged_log_bytes` gauge what each table's log still holds. ## Restart record diff --git a/docs/usage/configuration.md b/docs/usage/configuration.md index 5c9fb66..a20e523 100644 --- a/docs/usage/configuration.md +++ b/docs/usage/configuration.md @@ -197,7 +197,7 @@ state: # When empty, the source PG hosts it. # postgres_url: postgres://coord_user:secret@coord-db.example.com/coord?sslmode=require coordinator_schema: _pg2iceberg - group: default # consumer group name (distributed mode) + group: default # consumer group name (distributed mode); one per coordinator_schema metrics_addr: ":9090" # /metrics, /healthz, /readyz; `off` serves nothing liveness_timeout: 5m # /healthz fails once nothing completes for this long diff --git a/docs/usage/modes.md b/docs/usage/modes.md index bfe01dd..ee19c2c 100644 --- a/docs/usage/modes.md +++ b/docs/usage/modes.md @@ -67,7 +67,7 @@ pg2iceberg materializer-only --config config.yaml --worker-id worker-b Each worker registers a heartbeat in `_pg2iceberg.consumer` keyed by `state.group` and `--worker-id`. On every cycle, every worker reads the active-worker list and computes the same deterministic assignment (sorted tables → sorted workers → `[i % N]`). Adding or removing a worker rebalances on the next cycle automatically; no leader election. !!! note - All workers must share the same `state.group` (default `default`) and the same configured table list. Different `--worker-id` per process is mandatory — duplicates trample each other's heartbeat row and produce undefined assignment. + All workers must share the same `state.group` (default `default`) and the same configured table list. A coordinator schema serves one group: a process configured with another refuses to start. Different `--worker-id` per process is mandatory — duplicates trample each other's heartbeat row and produce undefined assignment. ## `snapshot` — one-shot initial snapshot