From bfec6e078135dc18a7989ee49ebc14083424ec12 Mon Sep 17 00:00:00 2001 From: Mayur Kale Date: Thu, 12 Nov 2026 20:17:05 +0000 Subject: [PATCH] fix(storage): name truncated history and read rewritten manifests An incremental read walks back from `to` to its checkpoint `from`. It looked `from` up too, so a checkpoint that expiry removed failed the read even when its child was kept, and a gap in the history failed with the same NotFound as a snapshot that was never an ancestor. The walk now stops at the child of `from`, and a missing snapshot other than `to` is the new Error::TruncatedHistory that ADR 0009 calls for. The walk is its own iterator so the plan-time range check can share it. files_added_by read only the manifests written inside the range and kept only Added entries. A replace that rewrites a manifest carries a file forward as Existing under the id of the snapshot that added it, so those rows were dropped. It now reads every manifest `to` lists and keeps the live entries added in the range. Part of #89 Signed-off-by: Mayur Kale --- crates/model/src/lib.rs | 5 + crates/storage/src/read.rs | 199 ++++++++++++++++++++++++++++++++----- 2 files changed, 179 insertions(+), 25 deletions(-) diff --git a/crates/model/src/lib.rs b/crates/model/src/lib.rs index b33f8f0..d153008 100644 --- a/crates/model/src/lib.rs +++ b/crates/model/src/lib.rs @@ -165,6 +165,10 @@ pub enum Error { /// A commit lost a race: the table changed after the writer read it. /// Retrying from a fresh read is safe. Conflict(String), + /// A read needs a snapshot that expiry removed, so the rows committed + /// since its checkpoint can't be told apart. Retrying never helps; the + /// reader has to read the table again in full (ADR 0009). + TruncatedHistory(String), } impl fmt::Display for Error { @@ -176,6 +180,7 @@ impl fmt::Display for Error { Error::Execution(m) => write!(f, "execution error: {m}"), Error::NotFound(m) => write!(f, "not found: {m}"), Error::Conflict(m) => write!(f, "conflict: {m}"), + Error::TruncatedHistory(m) => write!(f, "truncated history: {m}"), } } } diff --git a/crates/storage/src/read.rs b/crates/storage/src/read.rs index caf5ffd..b804073 100644 --- a/crates/storage/src/read.rs +++ b/crates/storage/src/read.rs @@ -7,7 +7,7 @@ use std::collections::HashSet; use arrow_array::RecordBatch; use futures::{StreamExt, TryStreamExt}; use iceberg::io::FileIO; -use iceberg::spec::{ManifestStatus, Operation, TableMetadata}; +use iceberg::spec::{Operation, SnapshotRef, TableMetadata}; use iceberg::table::{StaticTable, Table}; use infostash_model::{Error, Result, SnapshotId, SnapshotRange, TableRef}; @@ -88,8 +88,8 @@ pub async fn read_metadata_file(table: &TableRef, location: &str) -> Result Result> { let mut snapshots = HashSet::new(); - let mut cursor = Some(to); - while let Some(id) = cursor { - // Look the snapshot up first, so a made-up `from == to` is not found. - let snapshot = metadata - .snapshot_by_id(id) - .ok_or_else(|| Error::NotFound(format!("{table}: snapshot {id}")))?; - if id == from { - return Ok(snapshots); - } + for snapshot in ancestors_after(metadata, table, from, to) { + let snapshot = snapshot?; let operation = &snapshot.summary().operation; if *operation != Operation::Append { return Err(Error::Storage(format!( - "{table}: snapshot {id} is a `{}`, not an append, so it can't be read \ + "{table}: snapshot {} is a `{}`, not an append, so it can't be read \ incrementally", + snapshot.snapshot_id(), operation.as_str() ))); } - snapshots.insert(id); - cursor = snapshot.parent_snapshot_id(); + snapshots.insert(snapshot.snapshot_id()); } - Err(Error::NotFound(format!( - "{table}: snapshot {from} is not an ancestor of snapshot {to}" - ))) + Ok(snapshots) } -/// Paths of the data files that `snapshots` added, as listed by snapshot `to`. +/// The snapshots after `from` up to and including `to`, newest first. `to` +/// is looked up first, so a made-up `from == to` is not found. The walk +/// stops at the child of `from` without looking `from` up, so a checkpoint +/// that expiry removed still bounds the range while its child is kept. +fn ancestors_after<'a>( + metadata: &'a TableMetadata, + table: &'a TableRef, + from: SnapshotId, + to: SnapshotId, +) -> impl Iterator> + 'a { + let first = metadata + .snapshot_by_id(to) + .ok_or_else(|| Error::NotFound(format!("{table}: snapshot {to}"))); + let mut next = match first { + Ok(_) if to == from => None, + found => Some(found), + }; + std::iter::from_fn(move || { + let current = next.take()?; + if let Ok(snapshot) = ¤t { + next = match snapshot.parent_snapshot_id() { + Some(parent) if parent == from => None, + Some(parent) => Some(metadata.snapshot_by_id(parent).ok_or_else(|| { + Error::TruncatedHistory(format!( + "{table}: snapshot {parent}, between {from} and {to}, has expired" + )) + })), + None => Some(Err(Error::NotFound(format!( + "{table}: snapshot {from} is not an ancestor of snapshot {to}" + )))), + }; + } + Some(current) + }) +} + +/// Paths of the data files that `snapshots` added and that snapshot `to` +/// still lists as live. pub(crate) async fn files_added_by( table: &Table, to: SnapshotId, @@ -139,17 +167,16 @@ pub(crate) async fn files_added_by( .map_err(storage)?; let mut files = HashSet::new(); for manifest in list.entries() { - if !snapshots.contains(&manifest.added_snapshot_id) { - continue; - } let manifest = table .manifest_reader() .read(manifest) .await .map_err(storage)?; + // A later manifest rewrite carries a file forward as `Existing` with + // the id of the snapshot that added it, so every manifest is read. for entry in manifest.entries() { let added_here = entry.snapshot_id().is_some_and(|s| snapshots.contains(&s)); - if added_here && entry.status() == ManifestStatus::Added { + if added_here && entry.is_alive() { files.insert(entry.file_path().to_string()); } } @@ -192,11 +219,15 @@ pub(crate) async fn scan_at( #[cfg(test)] mod tests { - use infostash_model::{Error, Result, SnapshotRange}; + use std::collections::HashSet; - use super::appends_between; + use iceberg::spec::TableMetadata; + use infostash_model::{Error, Result, SnapshotId, SnapshotRange}; + + use super::{appends_between, files_added_by}; use crate::Replacement; use crate::test_support::{all_ids, files_of, ids, job, orders, position, two_appends}; + use crate::write::write_files; #[tokio::test] async fn an_overwrite_in_the_range_is_refused() { @@ -250,4 +281,122 @@ mod tests { assert_eq!(all_ids(&since), [6]); assert_eq!(all_ids(&wh.read_all(&orders).await.unwrap()), [1, 2, 6, 30]); } + + /// A file that a later replace carries forward as `Existing` still + /// counts as added by the snapshot that wrote it, and a file the range + /// deleted does not. + #[tokio::test] + async fn files_added_in_the_range_are_found_in_a_rewritten_manifest() { + let dir = tempfile::tempdir().unwrap(); + let (wh, orders) = orders(dir.path()).await; + let (_, second) = two_appends(&wh, &orders).await; + + // Two prebuilt files in one replace share one manifest. + let loaded = wh.load(&orders).await.unwrap(); + let (mut files, _) = write_files(&loaded, &orders, &[ids(&[10])]).await.unwrap(); + files.extend( + write_files(&loaded, &orders, &[ids(&[20])]) + .await + .unwrap() + .0, + ); + let (ten, twenty) = ( + files[0].file_path().to_string(), + files[1].file_path().to_string(), + ); + let built = Replacement { + remove: files_of(&wh, &orders, &second).await, + files, + ..Replacement::default() + }; + let built = wh + .replace_if_head(&orders, second.snapshot, built, &job("a")) + .await + .unwrap(); + // Removing one of them rewrites that manifest. + let partial = Replacement { + remove: vec![ten], + ..Replacement::default() + }; + let partial = wh + .replace_if_head(&orders, built.snapshot, partial, &job("b")) + .await + .unwrap(); + + let loaded = wh.load(&orders).await.unwrap(); + let found = files_added_by(&loaded, partial.snapshot, &HashSet::from([built.snapshot])) + .await + .unwrap(); + assert_eq!(found, HashSet::from([twenty])); + } + + /// Four appends, with expiry simulated by dropping snapshots from the + /// metadata, as the issue allows for a hand-built overwrite. + async fn four_appends() -> (tempfile::TempDir, TableMetadata, [SnapshotId; 4]) { + let dir = tempfile::tempdir().unwrap(); + let (wh, orders) = orders(dir.path()).await; + let mut head = None; + let mut ids_ = [0; 4]; + for (i, id) in ids_.iter_mut().enumerate() { + let value = i64::try_from(i).unwrap(); + let commit = wh + .append_if_head(&orders, &[ids(&[value])], head, &position("p")) + .await + .unwrap(); + *id = commit.snapshot; + head = Some(commit.snapshot); + } + let metadata = wh.load(&orders).await.unwrap().metadata().clone(); + (dir, metadata, ids_) + } + + fn expire(metadata: &TableMetadata, gone: &[SnapshotId]) -> TableMetadata { + metadata + .clone() + .into_builder(None) + .remove_snapshots(gone) + .build() + .unwrap() + .metadata + } + + #[tokio::test] + async fn a_range_whose_history_expired_is_truncated_not_missing() { + let (_dir, metadata, [s1, s2, s3, s4]) = four_appends().await; + let orders = infostash_model::TableRef::new(infostash_model::Layer::Raw, "orders"); + let after_s2 = HashSet::from([s3, s4]); + + // Snapshots older than the checkpoint, or the checkpoint alone with + // its child kept, leave the range readable. + let older = expire(&metadata, &[s1]); + assert_eq!(appends_between(&older, &orders, s2, s4).unwrap(), after_s2); + let checkpoint = expire(&metadata, &[s2]); + assert_eq!( + appends_between(&checkpoint, &orders, s2, s4).unwrap(), + after_s2 + ); + + // Losing the checkpoint's child too truncates the range. + let truncated = expire(&metadata, &[s2, s3]); + let err = appends_between(&truncated, &orders, s2, s4).unwrap_err(); + assert!(matches!(err, Error::TruncatedHistory(_)), "{err:?}"); + assert_eq!( + err.to_string(), + format!( + "truncated history: bronze.orders: snapshot {s3}, between {s2} and {s4}, has expired" + ) + ); + + // A snapshot that was never an ancestor is still not found, and so + // is a missing `to`. + let never = appends_between(&metadata, &orders, 7, s4).unwrap_err(); + assert!(matches!(never, Error::NotFound(m) if m.contains("not an ancestor"))); + let no_to = appends_between(&truncated, &orders, s3, s3).unwrap_err(); + assert!(matches!(no_to, Error::NotFound(_)), "{no_to:?}"); + assert!( + appends_between(&metadata, &orders, s4, s4) + .unwrap() + .is_empty() + ); + } }