Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
6e4fc17
feat(transaction): add MergingSnapshotProducer and RewriteFilesAction
hotcache Aug 21, 2026
64a5b1f
fix(transaction): support non-default partition specs in manifest fil…
hotcache Aug 21, 2026
b02fb8e
refactor(transaction): encapsulate MergingSnapshotProducer fields
hotcache Aug 21, 2026
1f412f8
refactor(transaction): address review feedback for PR1
hotcache Aug 21, 2026
0fb348c
test(transaction): add coverage for empty table rewrite and manifest …
hotcache Aug 21, 2026
1d48610
Merge branch 'main' into feat/rewrite-files-action
hotcache Aug 25, 2026
4208c44
fix(transaction): address review feedback from #3046
hotcache Aug 25, 2026
1dd97f0
Preserve survivor entries as Existing when filtering manifests
jopdorp Sep 18, 2026
243f8f3
Allow a rewrite to set the data sequence number of added files
jopdorp Sep 18, 2026
291b9b6
Run the producer validations before committing a rewrite
jopdorp Sep 18, 2026
d969ab3
Keep manifest names unique across commit attempts
jopdorp Sep 18, 2026
82ad3ae
Drop manifests an earlier snapshot left all-deleted
jopdorp Sep 18, 2026
ddafede
Refuse a rewrite when deletes landed after its starting snapshot
jopdorp Sep 18, 2026
5f81639
Run the compaction and partial rewrites against a V2 table
jopdorp Sep 18, 2026
021c3b3
Format the new code and record the rewrite action in the public API
jopdorp Sep 18, 2026
4886b5a
Merge apache/main into feat/rewrite-files-action
hotcache Sep 28, 2026
7d63061
test(transaction): run the rewrite tests against a V1 table
hotcache Sep 28, 2026
318758c
fix(transaction): name the file in the added-file content type error
hotcache Sep 28, 2026
1af9f1d
fix(transaction): fail a rewrite whose starting snapshot is gone
hotcache Sep 28, 2026
9c9f1ed
fix(transaction): fail a rewrite planned before the table had a snapshot
hotcache Sep 28, 2026
544a91f
docs(transaction): describe when a rewrite refuses to commit
hotcache Sep 28, 2026
ebf1946
refactor(transaction): build the rewrite errors with invalid_data!
hotcache Sep 28, 2026
43a09a5
test(transaction): cover rewrites on a spec-evolved table
rexminnis Sep 28, 2026
89786a8
fix(transaction): keep partition metrics in the rewrite summary
rexminnis Sep 28, 2026
8a4f780
fix(transaction): record removed files as DELETED manifest entries
hotcache Sep 29, 2026
04e4668
Merge apache/main into feat/rewrite-files-action
hotcache Sep 29, 2026
e864184
Allow rewrites without output files
jopdorp Oct 13, 2026
6956f97
add msp
CTTY Oct 27, 2026
ebb9357
robust caching
CTTY Oct 28, 2026
ff97a62
use manifest's schema
CTTY Oct 28, 2026
1f14509
Merge remote-tracking branch 'origin/main' into feat/rewrite-files-ac…
jopdorp Oct 30, 2026
682a615
Keep the rewrite's snapshot id across commit attempts
jopdorp Oct 30, 2026
443a1f7
Stack RewriteFilesAction on the MergingSnapshotProducer from #3564
jopdorp Nov 3, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions crates/iceberg/public-api.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3465,13 +3465,20 @@ pub fn iceberg::transaction::ReplaceSortOrderAction::desc(self, name: &str, null
pub fn iceberg::transaction::ReplaceSortOrderAction::desc_with_transform(self, name: &str, transform: iceberg::spec::Transform, null_order: iceberg::spec::NullOrder) -> Self
impl core::clone::Clone for iceberg::transaction::ReplaceSortOrderAction
pub fn iceberg::transaction::ReplaceSortOrderAction::clone(&self) -> iceberg::transaction::ReplaceSortOrderAction
pub struct iceberg::transaction::RewriteFilesAction
impl iceberg::transaction::RewriteFilesAction
pub fn iceberg::transaction::RewriteFilesAction::add_file(self, file: iceberg::spec::DataFile) -> Self
pub fn iceberg::transaction::RewriteFilesAction::delete_file(self, file: iceberg::spec::DataFile) -> Self
impl core::clone::Clone for iceberg::transaction::RewriteFilesAction
pub fn iceberg::transaction::RewriteFilesAction::clone(&self) -> iceberg::transaction::RewriteFilesAction
pub struct iceberg::transaction::Transaction
impl iceberg::transaction::Transaction
pub async fn iceberg::transaction::Transaction::commit(self, catalog: &dyn iceberg::Catalog) -> iceberg::Result<iceberg::table::Table>
pub fn iceberg::transaction::Transaction::expire_snapshots(&self) -> iceberg::transaction::ExpireSnapshotsAction
pub fn iceberg::transaction::Transaction::fast_append(&self) -> iceberg::transaction::FastAppendAction
pub fn iceberg::transaction::Transaction::new(table: &iceberg::table::Table) -> Self
pub fn iceberg::transaction::Transaction::replace_sort_order(&self) -> iceberg::transaction::ReplaceSortOrderAction
pub fn iceberg::transaction::Transaction::rewrite_files(&self) -> iceberg::transaction::RewriteFilesAction
pub fn iceberg::transaction::Transaction::update_location(&self) -> iceberg::transaction::UpdateLocationAction
pub fn iceberg::transaction::Transaction::update_schema(&self) -> iceberg::transaction::UpdateSchemaAction
pub fn iceberg::transaction::Transaction::update_statistics(&self) -> iceberg::transaction::UpdateStatisticsAction
Expand Down
6 changes: 5 additions & 1 deletion crates/iceberg/src/io/object_cache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,10 @@ pub(crate) enum CachedObjectKey {
Manifest(ManifestKey),
}

/// Identifies a parsed manifest: its path plus every manifest-list value the
/// reader inherits onto its entries. Two references to one physical manifest
/// with different inherited values parse to different entries, so anything
/// derived from parsed entries must be keyed by this, not by path alone.
#[derive(Clone, Debug, Hash, Eq, PartialEq)]
pub(crate) struct ManifestKey {
path: String,
Expand All @@ -61,7 +65,7 @@ pub(crate) struct ManifestKey {
}

impl ManifestKey {
fn new(manifest_file: &ManifestFile) -> Self {
pub(crate) fn new(manifest_file: &ManifestFile) -> Self {
Self {
path: manifest_file.manifest_path.clone(),
added_snapshot_id: manifest_file.added_snapshot_id,
Expand Down
8 changes: 0 additions & 8 deletions crates/iceberg/src/spec/manifest/writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -315,10 +315,6 @@ impl ManifestWriter {
/// Add a delete manifest entry. This method will update following status of the entry:
/// - Update the entry status to `Deleted`
/// - Set the snapshot id to the current snapshot id
///
/// # TODO
/// Remove this allow later
#[allow(dead_code)]
pub(crate) fn add_delete_entry(&mut self, mut entry: ManifestEntry) -> Result<()> {
self.check_data_file(&entry.data_file)?;
entry.status = ManifestStatus::Deleted;
Expand Down Expand Up @@ -350,10 +346,6 @@ impl ManifestWriter {

/// Add an existing manifest entry. This method will update following status of the entry:
/// - Update the entry status to `Existing`
///
/// # TODO
/// Remove this allow later
#[allow(dead_code)]
pub(crate) fn add_existing_entry(&mut self, mut entry: ManifestEntry) -> Result<()> {
self.check_data_file(&entry.data_file)?;
entry.status = ManifestStatus::Existing;
Expand Down
1 change: 1 addition & 0 deletions crates/iceberg/src/spec/snapshot_summary.rs
Original file line number Diff line number Diff line change
Expand Up @@ -348,6 +348,7 @@ pub(crate) fn update_snapshot_summaries(
if summary.operation != Operation::Append
&& summary.operation != Operation::Overwrite
&& summary.operation != Operation::Delete
&& summary.operation != Operation::Replace
{
return Err(invalid_data!("Operation is not supported."));
}
Expand Down
18 changes: 18 additions & 0 deletions crates/iceberg/src/spec/transform.rs
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,9 @@ impl Transform {
(Self::Hour, PrimitiveLiteral::Int(ordinal)) => Self::human_hour(ordinal),
(_, value) => {
let field_type = field_type.as_primitive_type().unwrap();
// A value written before its source column was promoted (`int`
// to `long`, `float` to `double`) still carries the old type.
let value = value.promote_to(field_type).unwrap_or(value);
Datum::new(field_type.clone(), value).to_human_string()
}
}
Expand Down Expand Up @@ -1322,6 +1325,21 @@ mod tests {
);
}

/// A partition value written before its source column was promoted keeps
/// the old literal type; rendering it under the promoted type must widen it
/// rather than panic.
#[test]
fn test_to_human_string_promotes_value_to_declared_type() {
assert_eq!(human(Transform::Identity, PrimitiveType::Long, 7), "7");
assert_eq!(
Transform::Identity.to_human_string(
&Type::Primitive(PrimitiveType::Double),
Some(&Literal::float(1.5_f32))
),
"1.5"
);
}

#[test]
fn test_to_human_string_null_cases() {
assert_eq!(human(Transform::Void, PrimitiveType::Int, 47), "null");
Expand Down
73 changes: 70 additions & 3 deletions crates/iceberg/src/transaction/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ mod action;
pub use action::*;
mod append;
mod expire_snapshots;
mod rewrite;
mod snapshot;
mod sort_order;
mod update_location;
Expand All @@ -75,6 +76,7 @@ use crate::table::Table;
use crate::transaction::action::TransactionActionEntry;
pub use crate::transaction::append::FastAppendAction;
pub use crate::transaction::expire_snapshots::ExpireSnapshotsAction;
pub use crate::transaction::rewrite::RewriteFilesAction;
pub use crate::transaction::sort_order::ReplaceSortOrderAction;
pub use crate::transaction::update_location::UpdateLocationAction;
pub use crate::transaction::update_properties::UpdatePropertiesAction;
Expand Down Expand Up @@ -150,6 +152,15 @@ impl Transaction {
FastAppendAction::new()
}

/// Creates a rewrite files action for compaction.
///
/// This action replaces a set of data files with a new set while keeping
/// the logical table contents unchanged. The resulting snapshot uses
/// [`Operation::Replace`](crate::spec::Operation::Replace).
pub fn rewrite_files(&self) -> RewriteFilesAction {
RewriteFilesAction::new(self.table.metadata().current_snapshot_id())
}

/// Creates replace sort order action.
pub fn replace_sort_order(&self) -> ReplaceSortOrderAction {
ReplaceSortOrderAction::new()
Expand Down Expand Up @@ -264,14 +275,39 @@ mod tests {
use crate::io::FileIO;
use crate::memory::tests::new_memory_catalog;
use crate::spec::{
DataContentType, DataFileBuilder, DataFileFormat, Literal, Struct, TableMetadata,
DataContentType, DataFile, DataFileBuilder, DataFileFormat, Literal, Struct, TableMetadata,
TableProperties,
};
use crate::table::{StaticTable, Table};
use crate::test_utils::{make_encrypted_table, test_runtime};
use crate::transaction::{ApplyTransactionAction, Transaction};
use crate::{Catalog, Error, ErrorKind, TableCreation, TableIdent};

/// Create a test data file with the given path, record count, and file size.
pub(crate) fn make_data_file(table: &Table, path: &str, records: u64, size: u64) -> DataFile {
DataFileBuilder::default()
.content(DataContentType::Data)
.file_path(path.to_string())
.file_format(DataFileFormat::Parquet)
.file_size_in_bytes(size)
.record_count(records)
.partition(Struct::from_iter([Some(Literal::long(1))]))
.partition_spec_id(table.metadata().default_partition_spec_id())
.build()
.unwrap()
}

/// Append data files to a table via fast_append and return the updated table.
pub(crate) async fn append_files(
catalog: &impl Catalog,
table: &Table,
files: Vec<DataFile>,
) -> Table {
let tx = Transaction::new(table);
let tx = tx.fast_append().add_data_files(files).apply(tx).unwrap();
tx.commit(catalog).await.unwrap()
}

pub fn make_v1_table() -> Table {
let file = File::open(format!(
"{}/testdata/table_metadata/{}",
Expand Down Expand Up @@ -332,6 +368,24 @@ mod tests {
.unwrap()
}

pub(crate) async fn make_v1_minimal_table_in_catalog(catalog: &impl Catalog) -> Table {
make_minimal_table_in_catalog(
catalog,
"TableMetadataV1Valid.json",
crate::spec::FormatVersion::V1,
)
.await
}

pub(crate) async fn make_v2_minimal_table_in_catalog(catalog: &impl Catalog) -> Table {
make_minimal_table_in_catalog(
catalog,
"TableMetadataV2ValidMinimal.json",
crate::spec::FormatVersion::V2,
)
.await
}

pub(crate) fn make_v3_minimal_table() -> Table {
let file = File::open(format!(
"{}/testdata/table_metadata/{}",
Expand Down Expand Up @@ -365,6 +419,19 @@ mod tests {
}

pub(crate) async fn make_v3_minimal_table_in_catalog(catalog: &impl Catalog) -> Table {
make_minimal_table_in_catalog(
catalog,
"TableMetadataV3ValidMinimal.json",
crate::spec::FormatVersion::V3,
)
.await
}

async fn make_minimal_table_in_catalog(
catalog: &impl Catalog,
metadata_file: &str,
format_version: crate::spec::FormatVersion,
) -> Table {
let table_ident =
TableIdent::from_strs([format!("ns1-{}", uuid::Uuid::new_v4()), "test1".to_string()])
.unwrap();
Expand All @@ -377,7 +444,7 @@ mod tests {
let file = File::open(format!(
"{}/testdata/table_metadata/{}",
env!("CARGO_MANIFEST_DIR"),
"TableMetadataV3ValidMinimal.json"
metadata_file
))
.unwrap();
let reader = BufReader::new(file);
Expand All @@ -388,7 +455,7 @@ mod tests {
.partition_spec((**base_metadata.default_partition_spec()).clone())
.sort_order((**base_metadata.default_sort_order()).clone())
.name(table_ident.name().to_string())
.format_version(crate::spec::FormatVersion::V3)
.format_version(format_version)
.build();

catalog
Expand Down
Loading