diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 6dd6bd5f37..9aaacd0fb3 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -3691,6 +3691,15 @@ impl core::fmt::Debug for iceberg::writer::file_writer::location_generator::Obje pub fn iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl iceberg::writer::file_writer::location_generator::LocationGenerator for iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator pub fn iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String +pub struct iceberg::writer::file_writer::location_generator::TableLocationGenerator +impl iceberg::writer::file_writer::location_generator::TableLocationGenerator +pub fn iceberg::writer::file_writer::location_generator::TableLocationGenerator::new(table_metadata: &iceberg::spec::TableMetadata) -> iceberg::Result +impl core::clone::Clone for iceberg::writer::file_writer::location_generator::TableLocationGenerator +pub fn iceberg::writer::file_writer::location_generator::TableLocationGenerator::clone(&self) -> iceberg::writer::file_writer::location_generator::TableLocationGenerator +impl core::fmt::Debug for iceberg::writer::file_writer::location_generator::TableLocationGenerator +pub fn iceberg::writer::file_writer::location_generator::TableLocationGenerator::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::writer::file_writer::location_generator::LocationGenerator for iceberg::writer::file_writer::location_generator::TableLocationGenerator +pub fn iceberg::writer::file_writer::location_generator::TableLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String pub trait iceberg::writer::file_writer::location_generator::FileNameGenerator: core::clone::Clone + core::marker::Send + core::marker::Sync + 'static pub fn iceberg::writer::file_writer::location_generator::FileNameGenerator::generate_file_name(&self) -> alloc::string::String impl iceberg::writer::file_writer::location_generator::FileNameGenerator for iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator @@ -3701,6 +3710,8 @@ impl iceberg::writer::file_writer::location_generator::LocationGenerator for ice pub fn iceberg::writer::file_writer::location_generator::DefaultLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String impl iceberg::writer::file_writer::location_generator::LocationGenerator for iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator pub fn iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String +impl iceberg::writer::file_writer::location_generator::LocationGenerator for iceberg::writer::file_writer::location_generator::TableLocationGenerator +pub fn iceberg::writer::file_writer::location_generator::TableLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String pub mod iceberg::writer::file_writer::rolling_writer pub struct iceberg::writer::file_writer::rolling_writer::RollingFileWriter impl iceberg::writer::file_writer::rolling_writer::RollingFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator diff --git a/crates/iceberg/src/cow_rewrite/writer.rs b/crates/iceberg/src/cow_rewrite/writer.rs index ce235e889b..4641e45300 100644 --- a/crates/iceberg/src/cow_rewrite/writer.rs +++ b/crates/iceberg/src/cow_rewrite/writer.rs @@ -28,8 +28,7 @@ use crate::writer::IcebergWriterBuilder; use crate::writer::base_writer::data_file_writer::DataFileWriterBuilder; use crate::writer::file_writer::ParquetWriterBuilder; use crate::writer::file_writer::location_generator::{ - DefaultFileNameGenerator, DefaultLocationGenerator, LocationGenerator, - ObjectStorageLocationGenerator, + DefaultFileNameGenerator, TableLocationGenerator, }; use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder; @@ -44,9 +43,8 @@ use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder; /// batch is written, so it is safe to construct one optimistically and only /// write to it once a source file is known to have changed. /// -/// The location layout follows `write.object-storage.enabled`: tables that opt -/// into the object-storage layout get hash-entropy paths from -/// [`ObjectStorageLocationGenerator`], everything else gets the default layout. +/// File locations follow the layout selected by the table properties; see +/// [`TableLocationGenerator`]. pub(crate) async fn build_replacement_writer( table: &Table, write_schema: SchemaRef, @@ -72,27 +70,12 @@ pub(crate) async fn build_replacement_writer( parquet_builder = parquet_builder.with_encryption_manager(encryption_manager.clone()); } - if table_props.write_object_storage_enabled()? { - let location_generator = ObjectStorageLocationGenerator::new(table.metadata())?; - build_data_file_writer(table, parquet_builder, location_generator, partition_key).await - } else { - let location_generator = DefaultLocationGenerator::new(table.metadata())?; - build_data_file_writer(table, parquet_builder, location_generator, partition_key).await - } -} - -async fn build_data_file_writer( - table: &Table, - parquet_builder: ParquetWriterBuilder, - location_generator: L, - partition_key: Option, -) -> Result> { + let location_generator = TableLocationGenerator::new(table.metadata())?; let file_name_generator = DefaultFileNameGenerator::new( format!("cow-rewrite-{}", Uuid::now_v7()), None, DataFileFormat::Parquet, ); - let table_props = table.metadata().table_properties(); let rolling_builder = RollingFileWriterBuilder::new( parquet_builder, table_props.write_target_file_size_bytes()?, diff --git a/crates/iceberg/src/writer/file_writer/location_generator.rs b/crates/iceberg/src/writer/file_writer/location_generator.rs index 26207d3e1d..bcdc2d2a2f 100644 --- a/crates/iceberg/src/writer/file_writer/location_generator.rs +++ b/crates/iceberg/src/writer/file_writer/location_generator.rs @@ -54,6 +54,10 @@ const ENTROPY_DIR_DEPTH: usize = 3; #[derive(Clone, Debug)] /// `DefaultLocationGenerator` used to generate the data dir location of data file. /// The location is generated based on the table location and the data location in table properties. +/// +/// Paths use `{data location}/{partition path}/{file name}`, omitting the +/// partition path for unpartitioned files. This generator always uses this layout. +/// Use [`TableLocationGenerator`] to select the layout from table properties. pub struct DefaultLocationGenerator { data_location: String, } @@ -203,6 +207,52 @@ impl LocationGenerator for ObjectStorageLocationGenerator { } } +/// Generates data file paths using the layout configured in table properties. +/// +/// It uses [`ObjectStorageLocationGenerator`] when +/// [`write.object-storage.enabled`](crate::spec::TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED) +/// is `true`, and [`DefaultLocationGenerator`] otherwise. +#[derive(Clone, Debug)] +pub struct TableLocationGenerator { + inner: Layout, +} + +#[derive(Clone, Debug)] +enum Layout { + Default(DefaultLocationGenerator), + ObjectStorage(ObjectStorageLocationGenerator), +} + +impl TableLocationGenerator { + /// Creates a generator from table metadata. + /// + /// # Errors + /// + /// Returns an error if a layout-related table property has an invalid value. + pub fn new(table_metadata: &TableMetadata) -> Result { + let inner = if table_metadata + .table_properties() + .write_object_storage_enabled()? + { + Layout::ObjectStorage(ObjectStorageLocationGenerator::new(table_metadata)?) + } else { + Layout::Default(DefaultLocationGenerator::new(table_metadata)?) + }; + Ok(Self { inner }) + } +} + +impl LocationGenerator for TableLocationGenerator { + fn generate_location(&self, partition_key: Option<&PartitionKey>, file_name: &str) -> String { + match &self.inner { + Layout::Default(generator) => generator.generate_location(partition_key, file_name), + Layout::ObjectStorage(generator) => { + generator.generate_location(partition_key, file_name) + } + } + } +} + /// Derive the `{parent}/{name}` context from a table location, mirroring Hadoop's /// `Path.getParent().getName()` / `Path.getName()`. fn path_context(table_location: &str) -> String { @@ -291,15 +341,18 @@ pub(crate) mod test { use std::collections::HashMap; use std::sync::Arc; + use rstest::rstest; use uuid::Uuid; use super::LocationGenerator; + use crate::ErrorKind; use crate::spec::{ FormatVersion, Literal, NestedField, PartitionKey, PartitionSpec, PrimitiveType, Schema, Struct, StructType, TableMetadata, TableProperties, Transform, Type, }; use crate::writer::file_writer::location_generator::{ DefaultLocationGenerator, FileNameGenerator, ObjectStorageLocationGenerator, + TableLocationGenerator, }; #[test] @@ -609,6 +662,85 @@ pub(crate) mod test { ); } + #[rstest] + #[case::unset(&[], [ + "s3://data.db/table/data/test.parquet", + "s3://data.db/table/data/data%231=val%231/test.parquet", + ])] + #[case::disabled(&[("write.object-storage.enabled", "false")], [ + "s3://data.db/table/data/test.parquet", + "s3://data.db/table/data/data%231=val%231/test.parquet", + ])] + #[case::enabled(&[("write.object-storage.enabled", "true")], [ + "s3://data.db/table/data/0110/1010/0011/11101000/test.parquet", + "s3://data.db/table/data/0000/1011/0110/00001000/data%231=val%231/test.parquet", + ])] + #[case::uppercase(&[("write.object-storage.enabled", "TRUE")], [ + "s3://data.db/table/data/0110/1010/0011/11101000/test.parquet", + "s3://data.db/table/data/0000/1011/0110/00001000/data%231=val%231/test.parquet", + ])] + #[case::without_partition_paths(&[ + ("write.object-storage.enabled", "true"), + ("write.object-storage.partitioned-paths", "false"), + ], [ + "s3://data.db/table/data/0110/1010/0011/11101000-test.parquet", + "s3://data.db/table/data/0110/1010/0011/11101000-test.parquet", + ])] + fn table_location_follows_properties( + #[case] properties: &[(&str, &str)], + #[case] expected: [&str; 2], + ) { + let metadata = table_metadata_with( + "s3://data.db/table", + properties + .iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(), + ); + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "data#1", Type::Primitive(PrimitiveType::String)) + .into(), + ]) + .build() + .unwrap(), + ); + let partition_spec = PartitionSpec::builder(schema.clone()) + .add_partition_field("data#1", "data#1", Transform::Identity) + .unwrap() + .build() + .unwrap(); + let partition_key = PartitionKey::new( + partition_spec, + schema, + Struct::from_iter([Some(Literal::string("val#1"))]), + ); + let generator = TableLocationGenerator::new(&metadata).unwrap(); + + assert_eq!( + [ + generator.generate_location(None, "test.parquet"), + generator.generate_location(Some(&partition_key), "test.parquet"), + ], + expected + ); + } + + #[test] + fn table_location_rejects_invalid_property() { + let metadata = table_metadata_with( + "s3://data.db/table", + HashMap::from([( + "write.object-storage.enabled".to_string(), + "yes".to_string(), + )]), + ); + let err = TableLocationGenerator::new(&metadata).unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + /// Build a minimal `TableMetadata` for location generator tests. fn table_metadata_with(location: &str, properties: HashMap) -> TableMetadata { TableMetadata { diff --git a/crates/iceberg/src/writer/mod.rs b/crates/iceberg/src/writer/mod.rs index ca401097ab..db4653af5b 100644 --- a/crates/iceberg/src/writer/mod.rs +++ b/crates/iceberg/src/writer/mod.rs @@ -51,7 +51,7 @@ //! use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; //! use iceberg::writer::file_writer::ParquetWriterBuilder; //! use iceberg::writer::file_writer::location_generator::{ -//! DefaultFileNameGenerator, DefaultLocationGenerator, +//! DefaultFileNameGenerator, TableLocationGenerator, //! }; //! use iceberg::writer::{IcebergWriter, IcebergWriterBuilder}; //! use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, Result, TableIdent}; @@ -78,7 +78,7 @@ //! let table = catalog //! .load_table(&TableIdent::from_strs(["hello", "world"])?) //! .await?; -//! let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap(); +//! let location_generator = TableLocationGenerator::new(table.metadata()).unwrap(); //! let file_name_generator = DefaultFileNameGenerator::new( //! "test".to_string(), //! None, @@ -125,7 +125,7 @@ //! use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; //! use iceberg::writer::file_writer::ParquetWriterBuilder; //! use iceberg::writer::file_writer::location_generator::{ -//! DefaultFileNameGenerator, DefaultLocationGenerator, +//! DefaultFileNameGenerator, TableLocationGenerator, //! }; //! use iceberg::writer::{IcebergWriter, IcebergWriterBuilder}; //! use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, Result, TableIdent}; @@ -207,7 +207,7 @@ //! table.metadata().current_schema().clone(), //! Struct::from_iter(vec![Some(Literal::string("Seattle"))]), //! ); -//! let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap(); +//! let location_generator = TableLocationGenerator::new(table.metadata()).unwrap(); //! let file_name_generator = DefaultFileNameGenerator::new( //! "test".to_string(), //! None, @@ -262,7 +262,7 @@ //! # use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; //! # use iceberg::writer::file_writer::ParquetWriterBuilder; //! # use iceberg::writer::file_writer::location_generator::{ -//! # DefaultFileNameGenerator, DefaultLocationGenerator, +//! # DefaultFileNameGenerator, TableLocationGenerator, //! # }; //! # use parquet::file::properties::WriterProperties; //! # use std::collections::HashMap; @@ -272,7 +272,7 @@ //! # .load("memory", HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), "file:///path/to/warehouse".to_string())])) //! # .await?; //! # let table = catalog.load_table(&TableIdent::from_strs(["hello", "world"])?).await?; -//! # let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap(); +//! # let location_generator = TableLocationGenerator::new(table.metadata()).unwrap(); //! # let file_name_generator = DefaultFileNameGenerator::new("test".to_string(), None, iceberg::spec::DataFileFormat::Parquet); //! # let parquet_writer_builder = ParquetWriterBuilder::new(WriterProperties::default(), table.metadata().current_schema().clone()); //! # let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( @@ -326,7 +326,7 @@ //! # use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; //! # use iceberg::writer::file_writer::ParquetWriterBuilder; //! # use iceberg::writer::file_writer::location_generator::{ -//! # DefaultFileNameGenerator, DefaultLocationGenerator, +//! # DefaultFileNameGenerator, TableLocationGenerator, //! # }; //! # use parquet::file::properties::WriterProperties; //! # use std::collections::HashMap; @@ -336,7 +336,7 @@ //! # .load("memory", HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), "file:///path/to/warehouse".to_string())])) //! # .await?; //! # let table = catalog.load_table(&TableIdent::from_strs(["hello", "world"])?).await?; -//! # let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap(); +//! # let location_generator = TableLocationGenerator::new(table.metadata()).unwrap(); //! # let file_name_generator = DefaultFileNameGenerator::new("test".to_string(), None, iceberg::spec::DataFileFormat::Parquet); //! # let parquet_writer_builder = ParquetWriterBuilder::new(WriterProperties::default(), table.metadata().current_schema().clone()); //! # let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size(