From cb224a5f0cd0b4d226aa63efadcbda0a08ae861a Mon Sep 17 00:00:00 2001 From: Sreesh Maheshwar Date: Fri, 6 Nov 2026 00:32:33 +0000 Subject: [PATCH 1/4] feat(writer): add a location generator that follows table properties Writers that build a location generator from table metadata had to choose between `DefaultLocationGenerator` and `ObjectStorageLocationGenerator` themselves, or ignore `write.object-storage.enabled`. Add `TableLocationGenerator`, which makes that choice from the table properties, and use it in the copy-on-write rewrite writer. The layout is chosen internally, so more layouts can be added later without breaking callers. --- crates/iceberg/public-api.txt | 11 ++ crates/iceberg/src/cow_rewrite/writer.rs | 25 +--- .../writer/file_writer/location_generator.rs | 133 ++++++++++++++++++ crates/iceberg/src/writer/mod.rs | 16 +-- 4 files changed, 156 insertions(+), 29 deletions(-) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 29acb011c4..e92d7377b1 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -3611,6 +3611,15 @@ impl core::fmt::Debug for iceberg::writer::file_writer::location_generator::Obje pub fn iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl iceberg::writer::file_writer::location_generator::LocationGenerator for iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator pub fn iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String +pub struct iceberg::writer::file_writer::location_generator::TableLocationGenerator +impl iceberg::writer::file_writer::location_generator::TableLocationGenerator +pub fn iceberg::writer::file_writer::location_generator::TableLocationGenerator::new(table_metadata: &iceberg::spec::TableMetadata) -> iceberg::Result +impl core::clone::Clone for iceberg::writer::file_writer::location_generator::TableLocationGenerator +pub fn iceberg::writer::file_writer::location_generator::TableLocationGenerator::clone(&self) -> iceberg::writer::file_writer::location_generator::TableLocationGenerator +impl core::fmt::Debug for iceberg::writer::file_writer::location_generator::TableLocationGenerator +pub fn iceberg::writer::file_writer::location_generator::TableLocationGenerator::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::writer::file_writer::location_generator::LocationGenerator for iceberg::writer::file_writer::location_generator::TableLocationGenerator +pub fn iceberg::writer::file_writer::location_generator::TableLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String pub trait iceberg::writer::file_writer::location_generator::FileNameGenerator: core::clone::Clone + core::marker::Send + core::marker::Sync + 'static pub fn iceberg::writer::file_writer::location_generator::FileNameGenerator::generate_file_name(&self) -> alloc::string::String impl iceberg::writer::file_writer::location_generator::FileNameGenerator for iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator @@ -3621,6 +3630,8 @@ impl iceberg::writer::file_writer::location_generator::LocationGenerator for ice pub fn iceberg::writer::file_writer::location_generator::DefaultLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String impl iceberg::writer::file_writer::location_generator::LocationGenerator for iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator pub fn iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String +impl iceberg::writer::file_writer::location_generator::LocationGenerator for iceberg::writer::file_writer::location_generator::TableLocationGenerator +pub fn iceberg::writer::file_writer::location_generator::TableLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String pub mod iceberg::writer::file_writer::rolling_writer pub struct iceberg::writer::file_writer::rolling_writer::RollingFileWriter impl iceberg::writer::file_writer::rolling_writer::RollingFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator diff --git a/crates/iceberg/src/cow_rewrite/writer.rs b/crates/iceberg/src/cow_rewrite/writer.rs index ce235e889b..4641e45300 100644 --- a/crates/iceberg/src/cow_rewrite/writer.rs +++ b/crates/iceberg/src/cow_rewrite/writer.rs @@ -28,8 +28,7 @@ use crate::writer::IcebergWriterBuilder; use crate::writer::base_writer::data_file_writer::DataFileWriterBuilder; use crate::writer::file_writer::ParquetWriterBuilder; use crate::writer::file_writer::location_generator::{ - DefaultFileNameGenerator, DefaultLocationGenerator, LocationGenerator, - ObjectStorageLocationGenerator, + DefaultFileNameGenerator, TableLocationGenerator, }; use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder; @@ -44,9 +43,8 @@ use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder; /// batch is written, so it is safe to construct one optimistically and only /// write to it once a source file is known to have changed. /// -/// The location layout follows `write.object-storage.enabled`: tables that opt -/// into the object-storage layout get hash-entropy paths from -/// [`ObjectStorageLocationGenerator`], everything else gets the default layout. +/// File locations follow the layout selected by the table properties; see +/// [`TableLocationGenerator`]. pub(crate) async fn build_replacement_writer( table: &Table, write_schema: SchemaRef, @@ -72,27 +70,12 @@ pub(crate) async fn build_replacement_writer( parquet_builder = parquet_builder.with_encryption_manager(encryption_manager.clone()); } - if table_props.write_object_storage_enabled()? { - let location_generator = ObjectStorageLocationGenerator::new(table.metadata())?; - build_data_file_writer(table, parquet_builder, location_generator, partition_key).await - } else { - let location_generator = DefaultLocationGenerator::new(table.metadata())?; - build_data_file_writer(table, parquet_builder, location_generator, partition_key).await - } -} - -async fn build_data_file_writer( - table: &Table, - parquet_builder: ParquetWriterBuilder, - location_generator: L, - partition_key: Option, -) -> Result> { + let location_generator = TableLocationGenerator::new(table.metadata())?; let file_name_generator = DefaultFileNameGenerator::new( format!("cow-rewrite-{}", Uuid::now_v7()), None, DataFileFormat::Parquet, ); - let table_props = table.metadata().table_properties(); let rolling_builder = RollingFileWriterBuilder::new( parquet_builder, table_props.write_target_file_size_bytes()?, diff --git a/crates/iceberg/src/writer/file_writer/location_generator.rs b/crates/iceberg/src/writer/file_writer/location_generator.rs index 26207d3e1d..02538a173f 100644 --- a/crates/iceberg/src/writer/file_writer/location_generator.rs +++ b/crates/iceberg/src/writer/file_writer/location_generator.rs @@ -54,6 +54,9 @@ const ENTROPY_DIR_DEPTH: usize = 3; #[derive(Clone, Debug)] /// `DefaultLocationGenerator` used to generate the data dir location of data file. /// The location is generated based on the table location and the data location in table properties. +/// +/// It always uses the default layout. Use [`TableLocationGenerator`] to follow the layout selected +/// by the table properties. pub struct DefaultLocationGenerator { data_location: String, } @@ -203,6 +206,56 @@ impl LocationGenerator for ObjectStorageLocationGenerator { } } +/// `TableLocationGenerator` generates data file locations in the layout that the table +/// properties select. +/// +/// It uses [`ObjectStorageLocationGenerator`] when +/// [`write.object-storage.enabled`](crate::spec::TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED) +/// is `true`, and [`DefaultLocationGenerator`] otherwise. A custom location provider configured +/// with `write.location-provider.impl` is ignored, and the default or object-storage layout is +/// used instead. +/// +/// Writers that produce data files for a table should use this generator so that the files +/// follow the table's configured layout. +#[derive(Clone, Debug)] +pub struct TableLocationGenerator { + inner: Layout, +} + +#[derive(Clone, Debug)] +enum Layout { + Default(DefaultLocationGenerator), + ObjectStorage(ObjectStorageLocationGenerator), +} + +impl TableLocationGenerator { + /// Create a new `TableLocationGenerator` from table metadata. + /// + /// Returns an error if a layout-related table property has an invalid value. + pub fn new(table_metadata: &TableMetadata) -> Result { + let inner = if table_metadata + .table_properties() + .write_object_storage_enabled()? + { + Layout::ObjectStorage(ObjectStorageLocationGenerator::new(table_metadata)?) + } else { + Layout::Default(DefaultLocationGenerator::new(table_metadata)?) + }; + Ok(Self { inner }) + } +} + +impl LocationGenerator for TableLocationGenerator { + fn generate_location(&self, partition_key: Option<&PartitionKey>, file_name: &str) -> String { + match &self.inner { + Layout::Default(generator) => generator.generate_location(partition_key, file_name), + Layout::ObjectStorage(generator) => { + generator.generate_location(partition_key, file_name) + } + } + } +} + /// Derive the `{parent}/{name}` context from a table location, mirroring Hadoop's /// `Path.getParent().getName()` / `Path.getName()`. fn path_context(table_location: &str) -> String { @@ -294,12 +347,14 @@ pub(crate) mod test { use uuid::Uuid; use super::LocationGenerator; + use crate::ErrorKind; use crate::spec::{ FormatVersion, Literal, NestedField, PartitionKey, PartitionSpec, PrimitiveType, Schema, Struct, StructType, TableMetadata, TableProperties, Transform, Type, }; use crate::writer::file_writer::location_generator::{ DefaultLocationGenerator, FileNameGenerator, ObjectStorageLocationGenerator, + TableLocationGenerator, }; #[test] @@ -609,6 +664,84 @@ pub(crate) mod test { ); } + #[test] + fn test_table_location_generator_default_layout() { + let enabled = TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED; + for properties in [vec![], vec![(enabled, "false")]] { + let table_metadata = table_metadata_with_properties(&properties); + let expected = DefaultLocationGenerator::new(&table_metadata).unwrap(); + assert_same_locations(&table_metadata, &expected); + } + } + + #[test] + fn test_table_location_generator_object_storage_layout() { + let enabled = TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED; + let partitioned_paths = TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS; + let cases = [ + vec![(enabled, "true"), (partitioned_paths, "false")], + vec![(enabled, "true")], + vec![(enabled, "TRUE")], + ]; + for properties in cases { + let table_metadata = table_metadata_with_properties(&properties); + let expected = ObjectStorageLocationGenerator::new(&table_metadata).unwrap(); + assert_same_locations(&table_metadata, &expected); + } + } + + #[test] + fn test_table_location_generator_invalid_property() { + let table_metadata = table_metadata_with_properties(&[( + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED, + "yes", + )]); + let err = TableLocationGenerator::new(&table_metadata).unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + /// Assert that a `TableLocationGenerator` built from `table_metadata` generates the same + /// locations as `expected`, with and without a partition key. + fn assert_same_locations(table_metadata: &TableMetadata, expected: &impl LocationGenerator) { + let location_gen = TableLocationGenerator::new(table_metadata).unwrap(); + let partition_key = partition_key(); + for partition_key in [None, Some(&partition_key)] { + assert_eq!( + location_gen.generate_location(partition_key, "test.parquet"), + expected.generate_location(partition_key, "test.parquet") + ); + } + } + + /// Build a partition key with an identity partition on `id = 42`. + fn partition_key() -> PartitionKey { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + let partition_spec = PartitionSpec::builder(schema.clone()) + .add_partition_field("id", "id", Transform::Identity) + .unwrap() + .build() + .unwrap(); + let partition_data = Struct::from_iter([Some(Literal::int(42))]); + PartitionKey::new(partition_spec, schema, partition_data) + } + + /// Build a minimal `TableMetadata` at `s3://data.db/table` with the given properties. + fn table_metadata_with_properties(properties: &[(&str, &str)]) -> TableMetadata { + let properties = properties + .iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(); + table_metadata_with("s3://data.db/table", properties) + } + /// Build a minimal `TableMetadata` for location generator tests. fn table_metadata_with(location: &str, properties: HashMap) -> TableMetadata { TableMetadata { diff --git a/crates/iceberg/src/writer/mod.rs b/crates/iceberg/src/writer/mod.rs index ca401097ab..db4653af5b 100644 --- a/crates/iceberg/src/writer/mod.rs +++ b/crates/iceberg/src/writer/mod.rs @@ -51,7 +51,7 @@ //! use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; //! use iceberg::writer::file_writer::ParquetWriterBuilder; //! use iceberg::writer::file_writer::location_generator::{ -//! DefaultFileNameGenerator, DefaultLocationGenerator, +//! DefaultFileNameGenerator, TableLocationGenerator, //! }; //! use iceberg::writer::{IcebergWriter, IcebergWriterBuilder}; //! use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, Result, TableIdent}; @@ -78,7 +78,7 @@ //! let table = catalog //! .load_table(&TableIdent::from_strs(["hello", "world"])?) //! .await?; -//! let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap(); +//! let location_generator = TableLocationGenerator::new(table.metadata()).unwrap(); //! let file_name_generator = DefaultFileNameGenerator::new( //! "test".to_string(), //! None, @@ -125,7 +125,7 @@ //! use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; //! use iceberg::writer::file_writer::ParquetWriterBuilder; //! use iceberg::writer::file_writer::location_generator::{ -//! DefaultFileNameGenerator, DefaultLocationGenerator, +//! DefaultFileNameGenerator, TableLocationGenerator, //! }; //! use iceberg::writer::{IcebergWriter, IcebergWriterBuilder}; //! use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, Result, TableIdent}; @@ -207,7 +207,7 @@ //! table.metadata().current_schema().clone(), //! Struct::from_iter(vec![Some(Literal::string("Seattle"))]), //! ); -//! let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap(); +//! let location_generator = TableLocationGenerator::new(table.metadata()).unwrap(); //! let file_name_generator = DefaultFileNameGenerator::new( //! "test".to_string(), //! None, @@ -262,7 +262,7 @@ //! # use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; //! # use iceberg::writer::file_writer::ParquetWriterBuilder; //! # use iceberg::writer::file_writer::location_generator::{ -//! # DefaultFileNameGenerator, DefaultLocationGenerator, +//! # DefaultFileNameGenerator, TableLocationGenerator, //! # }; //! # use parquet::file::properties::WriterProperties; //! # use std::collections::HashMap; @@ -272,7 +272,7 @@ //! # .load("memory", HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), "file:///path/to/warehouse".to_string())])) //! # .await?; //! # let table = catalog.load_table(&TableIdent::from_strs(["hello", "world"])?).await?; -//! # let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap(); +//! # let location_generator = TableLocationGenerator::new(table.metadata()).unwrap(); //! # let file_name_generator = DefaultFileNameGenerator::new("test".to_string(), None, iceberg::spec::DataFileFormat::Parquet); //! # let parquet_writer_builder = ParquetWriterBuilder::new(WriterProperties::default(), table.metadata().current_schema().clone()); //! # let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( @@ -326,7 +326,7 @@ //! # use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; //! # use iceberg::writer::file_writer::ParquetWriterBuilder; //! # use iceberg::writer::file_writer::location_generator::{ -//! # DefaultFileNameGenerator, DefaultLocationGenerator, +//! # DefaultFileNameGenerator, TableLocationGenerator, //! # }; //! # use parquet::file::properties::WriterProperties; //! # use std::collections::HashMap; @@ -336,7 +336,7 @@ //! # .load("memory", HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), "file:///path/to/warehouse".to_string())])) //! # .await?; //! # let table = catalog.load_table(&TableIdent::from_strs(["hello", "world"])?).await?; -//! # let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap(); +//! # let location_generator = TableLocationGenerator::new(table.metadata()).unwrap(); //! # let file_name_generator = DefaultFileNameGenerator::new("test".to_string(), None, iceberg::spec::DataFileFormat::Parquet); //! # let parquet_writer_builder = ParquetWriterBuilder::new(WriterProperties::default(), table.metadata().current_schema().clone()); //! # let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( From df3f93add5efa192d5c4365b388fbc2bf3565f8d Mon Sep 17 00:00:00 2001 From: Sreesh Maheshwar Date: Fri, 6 Nov 2026 02:41:08 +0000 Subject: [PATCH 2/4] docs: clarify that DefaultLocationGenerator ignores the table layout --- crates/iceberg/src/writer/file_writer/location_generator.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/iceberg/src/writer/file_writer/location_generator.rs b/crates/iceberg/src/writer/file_writer/location_generator.rs index 02538a173f..399242d757 100644 --- a/crates/iceberg/src/writer/file_writer/location_generator.rs +++ b/crates/iceberg/src/writer/file_writer/location_generator.rs @@ -55,8 +55,9 @@ const ENTROPY_DIR_DEPTH: usize = 3; /// `DefaultLocationGenerator` used to generate the data dir location of data file. /// The location is generated based on the table location and the data location in table properties. /// -/// It always uses the default layout. Use [`TableLocationGenerator`] to follow the layout selected -/// by the table properties. +/// "Default" names the layout, not the table's choice: this generator always writes +/// `{data location}/{partition path}/{file name}` and ignores `write.object-storage.enabled`. +/// Use [`TableLocationGenerator`] to follow the layout selected by the table properties. pub struct DefaultLocationGenerator { data_location: String, } From a9b322cef3615e123e8aafac24761a4bd8856be0 Mon Sep 17 00:00:00 2001 From: Sreesh Maheshwar Date: Thu, 19 Nov 2026 11:50:08 +0000 Subject: [PATCH 3/4] test(writer): simplify location generator tests and docs Generated-by: Codex --- .../writer/file_writer/location_generator.rs | 151 +++++++++--------- 1 file changed, 73 insertions(+), 78 deletions(-) diff --git a/crates/iceberg/src/writer/file_writer/location_generator.rs b/crates/iceberg/src/writer/file_writer/location_generator.rs index 399242d757..6a1dac464b 100644 --- a/crates/iceberg/src/writer/file_writer/location_generator.rs +++ b/crates/iceberg/src/writer/file_writer/location_generator.rs @@ -51,13 +51,10 @@ const ENTROPY_DIR_LENGTH: usize = 4; /// Number of entropy directories generated from the hash. const ENTROPY_DIR_DEPTH: usize = 3; -#[derive(Clone, Debug)] -/// `DefaultLocationGenerator` used to generate the data dir location of data file. -/// The location is generated based on the table location and the data location in table properties. +/// Generates paths under the data directory, with partition subdirectories. /// -/// "Default" names the layout, not the table's choice: this generator always writes -/// `{data location}/{partition path}/{file name}` and ignores `write.object-storage.enabled`. -/// Use [`TableLocationGenerator`] to follow the layout selected by the table properties. +/// Use [`TableLocationGenerator`] to honor `write.object-storage.enabled`. +#[derive(Clone, Debug)] pub struct DefaultLocationGenerator { data_location: String, } @@ -207,17 +204,11 @@ impl LocationGenerator for ObjectStorageLocationGenerator { } } -/// `TableLocationGenerator` generates data file locations in the layout that the table -/// properties select. +/// Generates data file paths using the layout configured in table properties. /// /// It uses [`ObjectStorageLocationGenerator`] when /// [`write.object-storage.enabled`](crate::spec::TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED) -/// is `true`, and [`DefaultLocationGenerator`] otherwise. A custom location provider configured -/// with `write.location-provider.impl` is ignored, and the default or object-storage layout is -/// used instead. -/// -/// Writers that produce data files for a table should use this generator so that the files -/// follow the table's configured layout. +/// is `true`, and [`DefaultLocationGenerator`] otherwise. #[derive(Clone, Debug)] pub struct TableLocationGenerator { inner: Layout, @@ -230,7 +221,9 @@ enum Layout { } impl TableLocationGenerator { - /// Create a new `TableLocationGenerator` from table metadata. + /// Creates a generator from table metadata. + /// + /// # Errors /// /// Returns an error if a layout-related table property has an invalid value. pub fn new(table_metadata: &TableMetadata) -> Result { @@ -345,6 +338,7 @@ pub(crate) mod test { use std::collections::HashMap; use std::sync::Arc; + use rstest::rstest; use uuid::Uuid; use super::LocationGenerator; @@ -665,82 +659,83 @@ pub(crate) mod test { ); } - #[test] - fn test_table_location_generator_default_layout() { - let enabled = TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED; - for properties in [vec![], vec![(enabled, "false")]] { - let table_metadata = table_metadata_with_properties(&properties); - let expected = DefaultLocationGenerator::new(&table_metadata).unwrap(); - assert_same_locations(&table_metadata, &expected); - } - } - - #[test] - fn test_table_location_generator_object_storage_layout() { - let enabled = TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED; - let partitioned_paths = TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS; - let cases = [ - vec![(enabled, "true"), (partitioned_paths, "false")], - vec![(enabled, "true")], - vec![(enabled, "TRUE")], - ]; - for properties in cases { - let table_metadata = table_metadata_with_properties(&properties); - let expected = ObjectStorageLocationGenerator::new(&table_metadata).unwrap(); - assert_same_locations(&table_metadata, &expected); - } - } - - #[test] - fn test_table_location_generator_invalid_property() { - let table_metadata = table_metadata_with_properties(&[( - TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED, - "yes", - )]); - let err = TableLocationGenerator::new(&table_metadata).unwrap_err(); - - assert_eq!(err.kind(), ErrorKind::DataInvalid); - } - - /// Assert that a `TableLocationGenerator` built from `table_metadata` generates the same - /// locations as `expected`, with and without a partition key. - fn assert_same_locations(table_metadata: &TableMetadata, expected: &impl LocationGenerator) { - let location_gen = TableLocationGenerator::new(table_metadata).unwrap(); - let partition_key = partition_key(); - for partition_key in [None, Some(&partition_key)] { - assert_eq!( - location_gen.generate_location(partition_key, "test.parquet"), - expected.generate_location(partition_key, "test.parquet") - ); - } - } - - /// Build a partition key with an identity partition on `id = 42`. - fn partition_key() -> PartitionKey { + #[rstest] + #[case::unset(&[], [ + "s3://data.db/table/data/test.parquet", + "s3://data.db/table/data/data%231=val%231/test.parquet", + ])] + #[case::disabled(&[("write.object-storage.enabled", "false")], [ + "s3://data.db/table/data/test.parquet", + "s3://data.db/table/data/data%231=val%231/test.parquet", + ])] + #[case::enabled(&[("write.object-storage.enabled", "true")], [ + "s3://data.db/table/data/0110/1010/0011/11101000/test.parquet", + "s3://data.db/table/data/0000/1011/0110/00001000/data%231=val%231/test.parquet", + ])] + #[case::uppercase(&[("write.object-storage.enabled", "TRUE")], [ + "s3://data.db/table/data/0110/1010/0011/11101000/test.parquet", + "s3://data.db/table/data/0000/1011/0110/00001000/data%231=val%231/test.parquet", + ])] + #[case::without_partition_paths(&[ + ("write.object-storage.enabled", "true"), + ("write.object-storage.partitioned-paths", "false"), + ], [ + "s3://data.db/table/data/0110/1010/0011/11101000-test.parquet", + "s3://data.db/table/data/0110/1010/0011/11101000-test.parquet", + ])] + fn table_location_follows_properties( + #[case] properties: &[(&str, &str)], + #[case] expected: [&str; 2], + ) { + let metadata = table_metadata_with( + "s3://data.db/table", + properties + .iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(), + ); let schema = Arc::new( Schema::builder() .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(1, "data#1", Type::Primitive(PrimitiveType::String)) + .into(), ]) .build() .unwrap(), ); let partition_spec = PartitionSpec::builder(schema.clone()) - .add_partition_field("id", "id", Transform::Identity) + .add_partition_field("data#1", "data#1", Transform::Identity) .unwrap() .build() .unwrap(); - let partition_data = Struct::from_iter([Some(Literal::int(42))]); - PartitionKey::new(partition_spec, schema, partition_data) + let partition_key = PartitionKey::new( + partition_spec, + schema, + Struct::from_iter([Some(Literal::string("val#1"))]), + ); + let generator = TableLocationGenerator::new(&metadata).unwrap(); + + assert_eq!( + [ + generator.generate_location(None, "test.parquet"), + generator.generate_location(Some(&partition_key), "test.parquet"), + ], + expected + ); } - /// Build a minimal `TableMetadata` at `s3://data.db/table` with the given properties. - fn table_metadata_with_properties(properties: &[(&str, &str)]) -> TableMetadata { - let properties = properties - .iter() - .map(|(key, value)| (key.to_string(), value.to_string())) - .collect(); - table_metadata_with("s3://data.db/table", properties) + #[test] + fn table_location_rejects_invalid_property() { + let metadata = table_metadata_with( + "s3://data.db/table", + HashMap::from([( + "write.object-storage.enabled".to_string(), + "yes".to_string(), + )]), + ); + let err = TableLocationGenerator::new(&metadata).unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); } /// Build a minimal `TableMetadata` for location generator tests. From 1597f9461d260dd42f69c3a88fe911f5dde2d35f Mon Sep 17 00:00:00 2001 From: Sreesh Maheshwar Date: Thu, 19 Nov 2026 11:56:07 +0000 Subject: [PATCH 4/4] docs(writer): clarify the fixed default file layout Generated-by: Codex --- .../iceberg/src/writer/file_writer/location_generator.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/iceberg/src/writer/file_writer/location_generator.rs b/crates/iceberg/src/writer/file_writer/location_generator.rs index 6a1dac464b..bcdc2d2a2f 100644 --- a/crates/iceberg/src/writer/file_writer/location_generator.rs +++ b/crates/iceberg/src/writer/file_writer/location_generator.rs @@ -51,10 +51,13 @@ const ENTROPY_DIR_LENGTH: usize = 4; /// Number of entropy directories generated from the hash. const ENTROPY_DIR_DEPTH: usize = 3; -/// Generates paths under the data directory, with partition subdirectories. -/// -/// Use [`TableLocationGenerator`] to honor `write.object-storage.enabled`. #[derive(Clone, Debug)] +/// `DefaultLocationGenerator` used to generate the data dir location of data file. +/// The location is generated based on the table location and the data location in table properties. +/// +/// Paths use `{data location}/{partition path}/{file name}`, omitting the +/// partition path for unpartitioned files. This generator always uses this layout. +/// Use [`TableLocationGenerator`] to select the layout from table properties. pub struct DefaultLocationGenerator { data_location: String, }