Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions crates/model/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,10 @@ pub enum Error {
/// A commit lost a race: the table changed after the writer read it.
/// Retrying from a fresh read is safe.
Conflict(String),
/// A read needs a snapshot that expiry removed, so the rows committed
/// since its checkpoint can't be told apart. Retrying never helps; the
/// reader has to read the table again in full (ADR 0009).
TruncatedHistory(String),
}

impl fmt::Display for Error {
Expand All @@ -176,6 +180,7 @@ impl fmt::Display for Error {
Error::Execution(m) => write!(f, "execution error: {m}"),
Error::NotFound(m) => write!(f, "not found: {m}"),
Error::Conflict(m) => write!(f, "conflict: {m}"),
Error::TruncatedHistory(m) => write!(f, "truncated history: {m}"),
}
}
}
Expand Down
199 changes: 174 additions & 25 deletions crates/storage/src/read.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ use std::collections::HashSet;
use arrow_array::RecordBatch;
use futures::{StreamExt, TryStreamExt};
use iceberg::io::FileIO;
use iceberg::spec::{ManifestStatus, Operation, TableMetadata};
use iceberg::spec::{Operation, SnapshotRef, TableMetadata};
use iceberg::table::{StaticTable, Table};
use infostash_model::{Error, Result, SnapshotId, SnapshotRange, TableRef};

Expand Down Expand Up @@ -88,41 +88,69 @@ pub async fn read_metadata_file(table: &TableRef, location: &str) -> Result<Vec<
}

/// The snapshots after `from` up to and including `to`, walking back from
/// `to`. Fails if one of them is not an append, or if the walk never reaches
/// `from`.
/// `to`. Fails if one of them is not an append, if the walk never reaches
/// `from`, or if expiry removed a snapshot it needs.
fn appends_between(
metadata: &TableMetadata,
table: &TableRef,
from: SnapshotId,
to: SnapshotId,
) -> Result<HashSet<SnapshotId>> {
let mut snapshots = HashSet::new();
let mut cursor = Some(to);
while let Some(id) = cursor {
// Look the snapshot up first, so a made-up `from == to` is not found.
let snapshot = metadata
.snapshot_by_id(id)
.ok_or_else(|| Error::NotFound(format!("{table}: snapshot {id}")))?;
if id == from {
return Ok(snapshots);
}
for snapshot in ancestors_after(metadata, table, from, to) {
let snapshot = snapshot?;
let operation = &snapshot.summary().operation;
if *operation != Operation::Append {
return Err(Error::Storage(format!(
"{table}: snapshot {id} is a `{}`, not an append, so it can't be read \
"{table}: snapshot {} is a `{}`, not an append, so it can't be read \
incrementally",
snapshot.snapshot_id(),
operation.as_str()
)));
}
snapshots.insert(id);
cursor = snapshot.parent_snapshot_id();
snapshots.insert(snapshot.snapshot_id());
}
Err(Error::NotFound(format!(
"{table}: snapshot {from} is not an ancestor of snapshot {to}"
)))
Ok(snapshots)
}

/// Paths of the data files that `snapshots` added, as listed by snapshot `to`.
/// The snapshots after `from` up to and including `to`, newest first. `to`
/// is looked up first, so a made-up `from == to` is not found. The walk
/// stops at the child of `from` without looking `from` up, so a checkpoint
/// that expiry removed still bounds the range while its child is kept.
fn ancestors_after<'a>(
metadata: &'a TableMetadata,
table: &'a TableRef,
from: SnapshotId,
to: SnapshotId,
) -> impl Iterator<Item = Result<&'a SnapshotRef>> + 'a {
let first = metadata
.snapshot_by_id(to)
.ok_or_else(|| Error::NotFound(format!("{table}: snapshot {to}")));
let mut next = match first {
Ok(_) if to == from => None,
found => Some(found),
};
std::iter::from_fn(move || {
let current = next.take()?;
if let Ok(snapshot) = &current {
next = match snapshot.parent_snapshot_id() {
Some(parent) if parent == from => None,
Some(parent) => Some(metadata.snapshot_by_id(parent).ok_or_else(|| {
Error::TruncatedHistory(format!(
"{table}: snapshot {parent}, between {from} and {to}, has expired"
))
})),
None => Some(Err(Error::NotFound(format!(
"{table}: snapshot {from} is not an ancestor of snapshot {to}"
)))),
};
}
Some(current)
})
}

/// Paths of the data files that `snapshots` added and that snapshot `to`
/// still lists as live.
pub(crate) async fn files_added_by(
table: &Table,
to: SnapshotId,
Expand All @@ -139,17 +167,16 @@ pub(crate) async fn files_added_by(
.map_err(storage)?;
let mut files = HashSet::new();
for manifest in list.entries() {
if !snapshots.contains(&manifest.added_snapshot_id) {
continue;
}
let manifest = table
.manifest_reader()
.read(manifest)
.await
.map_err(storage)?;
// A later manifest rewrite carries a file forward as `Existing` with
// the id of the snapshot that added it, so every manifest is read.
for entry in manifest.entries() {
let added_here = entry.snapshot_id().is_some_and(|s| snapshots.contains(&s));
if added_here && entry.status() == ManifestStatus::Added {
if added_here && entry.is_alive() {
files.insert(entry.file_path().to_string());
}
}
Expand Down Expand Up @@ -192,11 +219,15 @@ pub(crate) async fn scan_at(

#[cfg(test)]
mod tests {
use infostash_model::{Error, Result, SnapshotRange};
use std::collections::HashSet;

use super::appends_between;
use iceberg::spec::TableMetadata;
use infostash_model::{Error, Result, SnapshotId, SnapshotRange};

use super::{appends_between, files_added_by};
use crate::Replacement;
use crate::test_support::{all_ids, files_of, ids, job, orders, position, two_appends};
use crate::write::write_files;

#[tokio::test]
async fn an_overwrite_in_the_range_is_refused() {
Expand Down Expand Up @@ -250,4 +281,122 @@ mod tests {
assert_eq!(all_ids(&since), [6]);
assert_eq!(all_ids(&wh.read_all(&orders).await.unwrap()), [1, 2, 6, 30]);
}

/// A file that a later replace carries forward as `Existing` still
/// counts as added by the snapshot that wrote it, and a file the range
/// deleted does not.
#[tokio::test]
async fn files_added_in_the_range_are_found_in_a_rewritten_manifest() {
let dir = tempfile::tempdir().unwrap();
let (wh, orders) = orders(dir.path()).await;
let (_, second) = two_appends(&wh, &orders).await;

// Two prebuilt files in one replace share one manifest.
let loaded = wh.load(&orders).await.unwrap();
let (mut files, _) = write_files(&loaded, &orders, &[ids(&[10])]).await.unwrap();
files.extend(
write_files(&loaded, &orders, &[ids(&[20])])
.await
.unwrap()
.0,
);
let (ten, twenty) = (
files[0].file_path().to_string(),
files[1].file_path().to_string(),
);
let built = Replacement {
remove: files_of(&wh, &orders, &second).await,
files,
..Replacement::default()
};
let built = wh
.replace_if_head(&orders, second.snapshot, built, &job("a"))
.await
.unwrap();
// Removing one of them rewrites that manifest.
let partial = Replacement {
remove: vec![ten],
..Replacement::default()
};
let partial = wh
.replace_if_head(&orders, built.snapshot, partial, &job("b"))
.await
.unwrap();

let loaded = wh.load(&orders).await.unwrap();
let found = files_added_by(&loaded, partial.snapshot, &HashSet::from([built.snapshot]))
.await
.unwrap();
assert_eq!(found, HashSet::from([twenty]));
}

/// Four appends, with expiry simulated by dropping snapshots from the
/// metadata, as the issue allows for a hand-built overwrite.
async fn four_appends() -> (tempfile::TempDir, TableMetadata, [SnapshotId; 4]) {
let dir = tempfile::tempdir().unwrap();
let (wh, orders) = orders(dir.path()).await;
let mut head = None;
let mut ids_ = [0; 4];
for (i, id) in ids_.iter_mut().enumerate() {
let value = i64::try_from(i).unwrap();
let commit = wh
.append_if_head(&orders, &[ids(&[value])], head, &position("p"))
.await
.unwrap();
*id = commit.snapshot;
head = Some(commit.snapshot);
}
let metadata = wh.load(&orders).await.unwrap().metadata().clone();
(dir, metadata, ids_)
}

fn expire(metadata: &TableMetadata, gone: &[SnapshotId]) -> TableMetadata {
metadata
.clone()
.into_builder(None)
.remove_snapshots(gone)
.build()
.unwrap()
.metadata
}

#[tokio::test]
async fn a_range_whose_history_expired_is_truncated_not_missing() {
let (_dir, metadata, [s1, s2, s3, s4]) = four_appends().await;
let orders = infostash_model::TableRef::new(infostash_model::Layer::Raw, "orders");
let after_s2 = HashSet::from([s3, s4]);

// Snapshots older than the checkpoint, or the checkpoint alone with
// its child kept, leave the range readable.
let older = expire(&metadata, &[s1]);
assert_eq!(appends_between(&older, &orders, s2, s4).unwrap(), after_s2);
let checkpoint = expire(&metadata, &[s2]);
assert_eq!(
appends_between(&checkpoint, &orders, s2, s4).unwrap(),
after_s2
);

// Losing the checkpoint's child too truncates the range.
let truncated = expire(&metadata, &[s2, s3]);
let err = appends_between(&truncated, &orders, s2, s4).unwrap_err();
assert!(matches!(err, Error::TruncatedHistory(_)), "{err:?}");
assert_eq!(
err.to_string(),
format!(
"truncated history: bronze.orders: snapshot {s3}, between {s2} and {s4}, has expired"
)
);

// A snapshot that was never an ancestor is still not found, and so
// is a missing `to`.
let never = appends_between(&metadata, &orders, 7, s4).unwrap_err();
assert!(matches!(never, Error::NotFound(m) if m.contains("not an ancestor")));
let no_to = appends_between(&truncated, &orders, s3, s3).unwrap_err();
assert!(matches!(no_to, Error::NotFound(_)), "{no_to:?}");
assert!(
appends_between(&metadata, &orders, s4, s4)
.unwrap()
.is_empty()
);
}
}
Loading