Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ All coordination state lives under the `_pg2iceberg` schema in the source (or a
| `log_index` | Sparse index of staged Parquet files with offset ranges + LSN |
| `mat_cursor` | Materializer progress (last committed offset per table) |
| `consumer` | Heartbeat registry for distributed materializer workers |
| `pipeline_meta` | Singleton: source-cluster `system_identifier` (DSN-swap detection) |
| `pipeline_meta` | Singleton: source-cluster `system_identifier` (DSN-swap detection), and the consumer group the schema serves |
| `flushed_lsn` | Singleton: highest LSN we've acked the slot to (slot-tamper detection) |
| `tables` | Per-table snapshot status + `pg_class.oid` (drop-recreate detection) |
| `snapshot_progress` | Per-table mid-snapshot resume cursor |
Expand Down
17 changes: 17 additions & 0 deletions crates/pg2iceberg-coord/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,16 @@ pub enum CoordError {
`pg2iceberg cleanup` first; otherwise verify the source DSN."
)]
SystemIdMismatch { stored: u64, connected: u64 },
/// The coordinator schema serves another consumer group. One schema
/// serves one group: the staged log, its snapshot status and its
/// truncation all assume a single reader.
#[error(
"this coordinator schema serves consumer group {holder:?}, not {configured:?}: \
one schema serves one group. To materialize the same tables into another \
sink, run another pipeline, with a replication slot and \
state.coordinator_schema of its own. A group can't be renamed."
)]
GroupMismatch { holder: String, configured: String },
#[error("other: {0}")]
Other(String),
}
Expand Down Expand Up @@ -294,6 +304,13 @@ pub trait Coordinator: Send + Sync {
/// startup right after `IDENTIFY_SYSTEM`.
async fn set_pipeline_system_identifier(&self, sysid: u64) -> Result<()>;

/// Bind the schema to consumer group `group`, as every process that
/// reads or moves cursors does at startup: the first records it; any
/// other group is refused with [`CoordError::GroupMismatch`] — as is
/// one starting on a schema that predates the record and holds cursors
/// of another (a group's own `#…` cursors are its).
async fn claim_group(&self, group: &str) -> Result<()>;

// ── Last-acked LSN (tamper detection) ───────────────────
/// Read the highest LSN we've ever told the replication slot to
/// flush past. Returns `Ok(Lsn::ZERO)` if no row exists yet
Expand Down
38 changes: 38 additions & 0 deletions crates/pg2iceberg-coord/src/prod/coord.rs
Original file line number Diff line number Diff line change
Expand Up @@ -492,6 +492,44 @@ impl Coordinator for PostgresCoordinator {
Ok(())
}

async fn claim_group(&self, group: &str) -> Result<()> {
// One transaction, the pipeline-meta row locked: concurrent
// claims of different groups can't both win.
let mut client = self.client().await?;
let tx = client.transaction().await.map_err(pg)?;
tx.execute(&sql::ensure_pipeline_meta(&self.schema), &[])
.await
.map_err(pg)?;
let holder: Option<String> = tx
.query_one(&sql::select_pipeline_group(&self.schema), &[])
.await
.map_err(pg)?
.get(0);
let holder = match holder {
Some(holder) => Some(holder),
// Claimed by none yet: another group's cursors hold it anyway.
None => tx
.query(&sql::other_cursor_groups(&self.schema), &[&group])
.await
.map_err(pg)?
.first()
.map(|r| r.get::<_, String>(0)),
};
match holder {
Some(holder) if holder != group => Err(CoordError::GroupMismatch {
holder,
configured: group.to_string(),
}),
_ => {
tx.execute(&sql::set_pipeline_group(&self.schema), &[&group])
.await
.map_err(pg)?;
tx.commit().await.map_err(pg)?;
Ok(())
}
}
}

async fn flushed_lsn(&self) -> Result<Lsn> {
let client = self.client().await?;
let rows = client
Expand Down
46 changes: 43 additions & 3 deletions crates/pg2iceberg-coord/src/sql.rs
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,12 @@ pub fn migrate(schema: &CoordSchema) -> Vec<String> {
)",
schema.qualify("pipeline_meta")
),
// The consumer group the schema serves (`claim_group`); NULL
// until a process claims it.
format!(
"ALTER TABLE {} ADD COLUMN IF NOT EXISTS consumer_group TEXT",
schema.qualify("pipeline_meta")
),
// Single-row record of the highest LSN we've ever told the
// replication slot to flush past. Updated *before* every
// `send_standby`, so on a crash between the two writes the
Expand Down Expand Up @@ -409,6 +415,39 @@ pub fn upsert_pipeline_meta(schema: &CoordSchema) -> String {
)
}

/// Ensure the pipeline-meta row, so `claim_group` can lock it.
pub fn ensure_pipeline_meta(schema: &CoordSchema) -> String {
format!(
"INSERT INTO {} (id) VALUES (1) ON CONFLICT (id) DO NOTHING",
schema.qualify("pipeline_meta")
)
}

/// The claimed consumer group, the row locked until the transaction ends.
pub fn select_pipeline_group(schema: &CoordSchema) -> String {
format!(
"SELECT consumer_group FROM {} WHERE id = 1 FOR UPDATE",
schema.qualify("pipeline_meta")
)
}

/// Groups other than `$1` with cursors: a group's own cursors are named
/// `group` and `group#…`.
pub fn other_cursor_groups(schema: &CoordSchema) -> String {
format!(
"SELECT DISTINCT split_part(group_name, '#', 1) AS g FROM {} \
WHERE split_part(group_name, '#', 1) <> $1 ORDER BY g",
schema.qualify("mat_cursor")
)
}

pub fn set_pipeline_group(schema: &CoordSchema) -> String {
format!(
"UPDATE {} SET consumer_group = $1, updated_at = now() WHERE id = 1",
schema.qualify("pipeline_meta")
)
}

// ── Last-acked LSN (single-row tamper detection) ──────────────────────

pub fn select_flushed_lsn(schema: &CoordSchema) -> String {
Expand Down Expand Up @@ -562,14 +601,15 @@ mod tests {
let stmts = migrate(&s);
// 6 base CREATE TABLEs (log_seq, log_index, mat_cursor,
// consumer, pending_markers, marker_emissions)
// + 1 ALTER TABLE (log_index.flushable_lsn)
// + 2 ALTER TABLEs (log_index.flushable_lsn,
// pipeline_meta.consumer_group)
// + 1 DROP TABLE (the retired `lock` table)
// + 5 new per-concern tables (pipeline_meta, flushed_lsn,
// replicated_lsn, tables, snapshot_progress)
// + table_epoch + writes_in_flight
// = 15. Each statement is idempotent (CREATE TABLE IF NOT
// = 16. Each statement is idempotent (CREATE TABLE IF NOT
// EXISTS, ADD COLUMN IF NOT EXISTS or DROP TABLE IF EXISTS).
assert_eq!(stmts.len(), 15);
assert_eq!(stmts.len(), 16);
for stmt in &stmts {
assert!(
stmt.contains("IF NOT EXISTS") || stmt.contains("IF EXISTS"),
Expand Down
71 changes: 64 additions & 7 deletions crates/pg2iceberg-logical/src/materializer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,17 @@ pub enum MaterializerError {
/// update never happened. Handled within a cycle, never returned.
#[error("log already applied up to offset {end}")]
AlreadyApplied { end: u64 },
/// The table's log starts past its cursor: entries it hasn't applied
/// are gone ([`Materializer::log_continues`]).
#[error(
"{table}'s staged log starts at offset {start}, past its cursor at {cursor}: \
changes it hasn't applied are gone, and the table needs a new snapshot"
)]
LogGap {
table: TableIdent,
cursor: i64,
start: u64,
},
}

pub type Result<T> = std::result::Result<T, MaterializerError>;
Expand Down Expand Up @@ -1575,8 +1586,13 @@ impl<C: Catalog> Materializer<C> {
snapshot: true,
..Unit::default()
};
let group = self.snapshot_group();
let mut log = self.staged_log(ident, from.max(0) as u64);
let mut first = true;
while let Some((e, bytes)) = log.next().await? {
if std::mem::take(&mut first) && !self.log_continues(ident, &group, from, &e).await? {
return Ok(0);
}
let rows: Vec<MatEvent> = decode_chunk(&bytes)?
.into_iter()
.filter(|evt| is_snapshot_xid(evt.xid))
Expand Down Expand Up @@ -2762,6 +2778,40 @@ impl<C: Catalog> Materializer<C> {
Ok(folded)
}

/// Whether `first`, the first entry read past `cursor` of `group`,
/// starts where the cursor sits, as entries follow one another. If
/// not, those in between are gone: fine if the cursor has moved past
/// them since it was read — another process applied and truncated
/// them, a worker that owned the table too for a moment — and this
/// cycle has nothing to do (`false`); otherwise they're lost, and
/// it's a [`MaterializerError::LogGap`].
async fn log_continues(
&self,
ident: &TableIdent,
group: &str,
cursor: i64,
first: &LogEntry,
) -> Result<bool> {
if first.start_offset == cursor.max(0) as u64 {
return Ok(true);
}
let now = self.coord.get_cursor(group, ident).await?.unwrap_or(-1);
if now > cursor {
tracing::info!(
table = %ident,
cursor,
now,
"another process applied the log meanwhile"
);
return Ok(false);
}
Err(MaterializerError::LogGap {
table: ident.clone(),
cursor,
start: first.start_offset,
})
}

/// Truncating the table's log ([`truncate_log`]): once this process
/// starts, then whenever it has applied the log past where it's
/// truncated to. Not before the table's backfill is looked up: that
Expand All @@ -2777,7 +2827,10 @@ impl<C: Catalog> Materializer<C> {
let coord = self.coord.clone();
let blob_store = self.blob_store.clone();
let ident = ident.clone();
async move { truncate_log(coord.as_ref(), blob_store.as_ref(), &ident, due?).await }
// The group's cursors over the log: its changes', and its
// backfill's while one applies.
let groups = [self.group.clone(), self.snapshot_group()];
async move { truncate_log(coord.as_ref(), blob_store.as_ref(), &ident, &groups, due?).await }
}

/// Fold and commit the table's changes past its cursor, in units.
Expand All @@ -2796,6 +2849,9 @@ impl<C: Catalog> Materializer<C> {
let mut read_any = false;
let mut log = self.staged_log(ident, cursor.max(0) as u64);
while let Some((e, bytes)) = log.next().await? {
if !read_any && !self.log_continues(ident, &self.group, cursor, &e).await? {
return Ok(0);
}
read_any = true;
let events = decode_chunk(&bytes)?;
// A schema change starts its entry (the pipeline stages it
Expand Down Expand Up @@ -3459,11 +3515,11 @@ fn now_micros() -> i64 {
.unwrap_or(0)
}

/// Drop `ident`'s log before the lowest cursor reading it, and delete the
/// files those entries staged: every reader has applied them, and none
/// reads them again. Every group that registered the table holds its log —
/// its changes' cursor, and its backfill's while one applies. The entry at
/// the cursor stays, so the log still shows where it's applied to. At most
/// Drop `ident`'s log before the lowest of `groups`' cursors — the consumer
/// group's, which the schema serves alone ([`Coordinator::claim_group`]) —
/// and delete the files those entries staged: they're applied, and nothing
/// reads them again. The entry at the cursor stays, so the log still shows
/// where it's applied to. At most
/// [`TRUNCATE_BATCH`] entries at once, past `log_start`, where it was
/// truncated to before. Returns where it's truncated to now, unless it
/// left some for next time. Best effort: what this leaves the next one
Expand All @@ -3472,6 +3528,7 @@ async fn truncate_log(
coord: &dyn Coordinator,
blob_store: &dyn BlobStore,
ident: &TableIdent,
groups: &[String],
log_start: i64,
) -> Option<i64> {
let cursors = match coord.table_cursors(ident).await {
Expand All @@ -3483,7 +3540,7 @@ async fn truncate_log(
};
let keep_from = cursors
.into_iter()
.filter(|(group, _)| !group.ends_with(SNAPSHOT_LSN_SUFFIX))
.filter(|(group, _)| groups.contains(group))
.map(|(_, cursor)| cursor)
.min()?;
if keep_from <= log_start.max(0) {
Expand Down
41 changes: 41 additions & 0 deletions crates/pg2iceberg-sim/src/coord.rs
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ struct State {
consumers: BTreeMap<(String, WorkerId), Timestamp>,
/// `pipeline_meta.system_identifier` (zero = unset).
system_identifier: u64,
/// `pipeline_meta.consumer_group` (`None` = unclaimed).
consumer_group: Option<String>,
/// `flushed_lsn.lsn` — highest LSN we've ever acked. Compared to
/// `slot.confirmed_flush_lsn` at startup to catch external slot
/// advancement.
Expand All @@ -73,11 +75,15 @@ struct State {
marker_emissions: BTreeSet<(String, TableIdent)>,
}

/// Work [`MemoryCoordinator::before_next_read_log`] runs.
type Interlude = Box<dyn FnOnce() + Send>;

#[derive(Clone)]
pub struct MemoryCoordinator {
schema: CoordSchema,
clock: Arc<dyn Clock>,
state: Arc<Mutex<State>>,
before_read_log: Arc<Mutex<Option<Interlude>>>,
}

impl MemoryCoordinator {
Expand All @@ -86,6 +92,7 @@ impl MemoryCoordinator {
schema,
clock,
state: Arc::new(Mutex::new(State::default())),
before_read_log: Arc::default(),
}
}

Expand All @@ -100,6 +107,13 @@ impl MemoryCoordinator {
&self.schema
}

/// Test hook: run `f` just before the next `read_log` reads — as
/// another process's work landing between a read of the cursor and of
/// the log. `f` may call this coordinator.
pub fn before_next_read_log(&self, f: impl FnOnce() + Send + 'static) {
*self.before_read_log.lock().unwrap() = Some(Box::new(f));
}

/// Test hook: from now on, `set_cursor` for `groups` fails, writing
/// nothing — as for a process that dies before its cursor write
/// reaches the coordinator. An empty `groups` ends it.
Expand Down Expand Up @@ -238,6 +252,10 @@ impl Coordinator for MemoryCoordinator {
if i64::try_from(limit).is_err() {
return Err(CoordError::Other("limit > i64".into()));
}
let interlude = self.before_read_log.lock().unwrap().take();
if let Some(f) = interlude {
f();
}
let state = self.state.lock().unwrap();
let mut out: Vec<LogEntry> = state
.log
Expand Down Expand Up @@ -387,6 +405,29 @@ impl Coordinator for MemoryCoordinator {
Ok(())
}

async fn claim_group(&self, group: &str) -> Result<()> {
let mut state = self.state.lock().unwrap();
// Claimed by none yet: another group's cursors hold it anyway.
let holder = state.consumer_group.clone().or_else(|| {
state
.cursors
.keys()
.map(|(g, _)| g.split('#').next().unwrap_or_default().to_string())
.filter(|g| g != group)
.min()
});
match holder {
Some(holder) if holder != group => Err(CoordError::GroupMismatch {
holder,
configured: group.to_string(),
}),
_ => {
state.consumer_group = Some(group.to_string());
Ok(())
}
}
}

async fn flushed_lsn(&self) -> Result<Lsn> {
Ok(self.state.lock().unwrap().flushed_lsn)
}
Expand Down
4 changes: 4 additions & 0 deletions crates/pg2iceberg-sim/src/fault.rs
Original file line number Diff line number Diff line change
Expand Up @@ -335,6 +335,10 @@ impl Coordinator for FaultyCoordinator {
self.inner.set_pipeline_system_identifier(sysid).await
}

async fn claim_group(&self, group: &str) -> CoordResult<()> {
self.inner.claim_group(group).await
}

async fn flushed_lsn(&self) -> CoordResult<Lsn> {
self.inner.flushed_lsn().await
}
Expand Down
Loading
Loading