From ca73173e58ddbbccc38813ff657141efe67d12e7 Mon Sep 17 00:00:00 2001 From: Sreesh Maheshwar Date: Fri, 6 Nov 2026 00:57:01 +0000 Subject: [PATCH 1/2] fix: honor `write.object-storage.enabled` in `IcebergWriteExec` `IcebergWriteExec` always used `DefaultLocationGenerator`, so tables with `write.object-storage.enabled=true` got the default `{data}/{partition path}/{file}` layout instead of hashed paths, and `write.object-storage.partitioned-paths=false` had no effect. Use `TableLocationGenerator`, which picks the layout from the table properties. INSERT now fails when a `write.object-storage.*` property has an invalid value, instead of ignoring it. --- crates/datafusion/src/write/exec.rs | 5 +- .../tests/integration_datafusion_test.rs | 132 +++++++++++++++++- 2 files changed, 132 insertions(+), 5 deletions(-) diff --git a/crates/datafusion/src/write/exec.rs b/crates/datafusion/src/write/exec.rs index 0beaaa9..7c537fa 100644 --- a/crates/datafusion/src/write/exec.rs +++ b/crates/datafusion/src/write/exec.rs @@ -44,7 +44,7 @@ use iceberg::table::Table; use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; use iceberg::writer::file_writer::ParquetWriterBuilder; use iceberg::writer::file_writer::location_generator::{ - DefaultFileNameGenerator, DefaultLocationGenerator, + DefaultFileNameGenerator, TableLocationGenerator, }; use iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder; use uuid::Uuid; @@ -276,8 +276,7 @@ impl ExecutionPlan for IcebergWriteExec { .map_err(to_datafusion_error)?; let file_io = self.table.file_io().clone(); - // todo location_gen and file_name_gen should be configurable - let location_generator = DefaultLocationGenerator::new(self.table.metadata()) + let location_generator = TableLocationGenerator::new(self.table.metadata()) .map_err(to_datafusion_error)?; // todo filename prefix/suffix should be configurable let file_name_generator = diff --git a/crates/datafusion/tests/integration_datafusion_test.rs b/crates/datafusion/tests/integration_datafusion_test.rs index 42d350a..7308c8a 100644 --- a/crates/datafusion/tests/integration_datafusion_test.rs +++ b/crates/datafusion/tests/integration_datafusion_test.rs @@ -38,10 +38,15 @@ use datafusion::physical_plan::common::collect; use iceberg::io::LocalFsStorageFactory; use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; use iceberg::spec::{ - Literal, NestedField, PrimitiveType, Schema, Struct, StructType, Transform, Type, - UnboundPartitionField, UnboundPartitionSpec, + DataFile, Literal, NestedField, PartitionKey, PrimitiveType, Schema, Struct, + StructType, TableProperties, Transform, Type, UnboundPartitionField, + UnboundPartitionSpec, }; +use iceberg::table::Table; use iceberg::transaction::{AddColumn, ApplyTransactionAction, Transaction}; +use iceberg::writer::file_writer::location_generator::{ + LocationGenerator, ObjectStorageLocationGenerator, +}; use iceberg::{ Catalog, CatalogBuilder, MemoryCatalog, NamespaceIdent, Result as IcebergResult, TableCreation, TableIdent, @@ -895,6 +900,129 @@ async fn test_insert_into_partitioned() -> Result<(), Box> { Ok(()) } +#[tokio::test] +async fn test_insert_into_object_storage_layout_without_partition_paths() +-> Result<(), Box> { + let (table, data_files) = insert_into_partitioned_table(&[ + ( + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED, + "true", + ), + ( + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS, + "false", + ), + ]) + .await?; + + let location_generator = ObjectStorageLocationGenerator::new(table.metadata())?; + assert_eq!(data_files.len(), 2); + for data_file in data_files { + let path = data_file.file_path(); + // Without partition paths, the last entropy directory and the file name are + // joined with `-`, and the location doesn't depend on the partition. + let (_, file_name) = path.rsplit_once('/').unwrap().1.split_once('-').unwrap(); + assert_eq!(path, location_generator.generate_location(None, file_name)); + assert!(!path.contains("category="), "{path}"); + } + + Ok(()) +} + +#[tokio::test] +async fn test_insert_into_object_storage_layout_with_partition_paths() +-> Result<(), Box> { + let (table, data_files) = insert_into_partitioned_table(&[( + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED, + "true", + )]) + .await?; + + let metadata = table.metadata(); + let location_generator = ObjectStorageLocationGenerator::new(metadata)?; + assert_eq!(data_files.len(), 2); + for data_file in data_files { + let path = data_file.file_path(); + let (_, file_name) = path.rsplit_once('/').unwrap(); + let partition_key = PartitionKey::new( + metadata.default_partition_spec().as_ref().clone(), + metadata.current_schema().clone(), + data_file.partition().clone(), + ); + assert_eq!( + path, + location_generator.generate_location(Some(&partition_key), file_name) + ); + } + + Ok(()) +} + +/// Creates a table with `properties`, identity-partitioned on `category`, inserts a row +/// into each of two partitions, and returns the table and the data files it committed. +async fn insert_into_partitioned_table( + properties: &[(&str, &str)], +) -> Result<(Table, Vec), Box> { + let iceberg_catalog = get_iceberg_catalog().await; + let namespace = NamespaceIdent::new("ns".to_string()); + set_test_namespace(&iceberg_catalog, &namespace).await?; + + let schema = Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "category", Type::Primitive(PrimitiveType::String)) + .into(), + ]) + .build()?; + let partition_spec = UnboundPartitionSpec::builder() + .with_spec_id(0) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .name("category") + .transform(Transform::Identity) + .build()?, + )? + .build(); + let creation = TableCreation::builder() + .name("t".to_string()) + .location(temp_path()) + .schema(schema) + .partition_spec(partition_spec) + .properties( + properties + .iter() + .map(|(key, value)| (key.to_string(), value.to_string())), + ) + .build(); + iceberg_catalog.create_table(&namespace, creation).await?; + + let client = Arc::new(iceberg_catalog); + let catalog = Arc::new(IcebergCatalogProvider::try_new(client.clone()).await?); + let ctx = SessionContext::new(); + ctx.register_catalog("catalog", catalog); + ctx.sql("INSERT INTO catalog.ns.t VALUES (1, 'a'), (2, 'b')") + .await? + .collect() + .await?; + + let table = client + .load_table(&TableIdent::new(namespace, "t".to_string())) + .await?; + let snapshot = table.metadata().current_snapshot().unwrap(); + let manifest_list = table.manifest_list_reader(snapshot).load().await?; + let mut data_files = Vec::new(); + for entry in manifest_list.entries() { + let manifest = table.manifest_reader().read(entry).await?; + for entry in manifest.entries() { + data_files.push(entry.data_file().clone()); + } + } + + Ok((table, data_files)) +} + /// Executes `plan`, which must have a single partition, and returns its rows. async fn run_batches( plan: &dyn ExecutionPlan, From 0338059da8af103f813ba361aa61046b12c5b0de Mon Sep 17 00:00:00 2001 From: Sreesh Maheshwar Date: Thu, 19 Nov 2026 12:20:52 +0000 Subject: [PATCH 2/2] test: assert object-storage INSERT paths directly Compare each data file path with its expected layout, with the hash directories and generated file name redacted, instead of rebuilding the expected path with `ObjectStorageLocationGenerator`. Also check that INSERT rejects an invalid `write.object-storage.enabled`. Generated-by: Claude Code Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + crates/datafusion/Cargo.toml | 1 + crates/datafusion/tests/insert_into.rs | 147 +++++++++++++++---------- 3 files changed, 90 insertions(+), 59 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a2a9af5..fe872b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3248,6 +3248,7 @@ dependencies = [ "iceberg-datafusion-test-tools", "insta", "parquet", + "regex", "tempfile", "tokio", "uuid", diff --git a/crates/datafusion/Cargo.toml b/crates/datafusion/Cargo.toml index e7fb281..f934128 100644 --- a/crates/datafusion/Cargo.toml +++ b/crates/datafusion/Cargo.toml @@ -44,6 +44,7 @@ uuid = { version = "1.18", features = ["v7"] } iceberg-datafusion-test-tools = { workspace = true } insta = "1" parquet = "59.2" +regex = "1" tempfile = "3.18" # Every integration test is a module of `tests/main.rs`, so they all link into one binary diff --git a/crates/datafusion/tests/insert_into.rs b/crates/datafusion/tests/insert_into.rs index ef048fb..7f4cbe4 100644 --- a/crates/datafusion/tests/insert_into.rs +++ b/crates/datafusion/tests/insert_into.rs @@ -28,15 +28,12 @@ use datafusion::arrow::util::pretty::pretty_format_batches; use datafusion::execution::context::{SessionConfig, SessionContext}; use iceberg::TableIdent; use iceberg::spec::{ - DataFile, Literal, NestedField, PartitionKey, PrimitiveType, Schema, Struct, - StructType, TableProperties, Transform, Type, -}; -use iceberg::table::Table; -use iceberg::writer::file_writer::location_generator::{ - LocationGenerator, ObjectStorageLocationGenerator, + Literal, NestedField, PrimitiveType, Schema, Struct, StructType, TableProperties, + Transform, Type, }; use iceberg_datafusion::IcebergCatalogProvider; -use iceberg_datafusion_test_tools::{FixtureCatalog, display_schema}; +use iceberg_datafusion_test_tools::{FixtureCatalog, data_file_name, display_schema}; +use regex::Regex; #[tokio::test] async fn test_insert_into() -> Result<(), Box> { @@ -495,11 +492,37 @@ async fn test_insert_into_partitioned() -> Result<(), Box> { Ok(()) } +#[tokio::test] +async fn test_insert_into_object_storage_layout_with_partition_paths() +-> Result<(), Box> { + let fixtures = FixtureCatalog::try_new().await?; + let ctx = create_category_table( + &fixtures, + &[( + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED, + "true", + )], + ) + .await?; + + ctx.sql("INSERT INTO catalog.ns.t VALUES (1, 'a')") + .await? + .collect() + .await?; + + assert_eq!( + redacted_data_file_paths(&fixtures).await?, + ["/category=a/"] + ); + + Ok(()) +} + #[tokio::test] async fn test_insert_into_object_storage_layout_without_partition_paths() -> Result<(), Box> { let fixtures = FixtureCatalog::try_new().await?; - let (table, data_files) = insert_into_partitioned_table( + let ctx = create_category_table( &fixtures, &[ ( @@ -514,95 +537,101 @@ async fn test_insert_into_object_storage_layout_without_partition_paths() ) .await?; - let location_generator = ObjectStorageLocationGenerator::new(table.metadata())?; - assert_eq!(data_files.len(), 2); - for data_file in data_files { - let path = data_file.file_path(); - // Without partition paths, the last entropy directory and the file name are - // joined with `-`, and the location doesn't depend on the partition. - let (_, file_name) = path.rsplit_once('/').unwrap().1.split_once('-').unwrap(); - assert_eq!(path, location_generator.generate_location(None, file_name)); - assert!(!path.contains("category="), "{path}"); - } + ctx.sql("INSERT INTO catalog.ns.t VALUES (1, 'a')") + .await? + .collect() + .await?; + + assert_eq!( + redacted_data_file_paths(&fixtures).await?, + ["-"] + ); Ok(()) } #[tokio::test] -async fn test_insert_into_object_storage_layout_with_partition_paths() --> Result<(), Box> { +async fn test_insert_into_invalid_object_storage_enabled() -> Result<(), Box> { let fixtures = FixtureCatalog::try_new().await?; - let (table, data_files) = insert_into_partitioned_table( + let ctx = create_category_table( &fixtures, &[( TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED, - "true", + "yes", )], ) .await?; - let metadata = table.metadata(); - let location_generator = ObjectStorageLocationGenerator::new(metadata)?; - assert_eq!(data_files.len(), 2); - for data_file in data_files { - let path = data_file.file_path(); - let (_, file_name) = path.rsplit_once('/').unwrap(); - let partition_key = PartitionKey::new( - metadata.default_partition_spec().as_ref().clone(), - metadata.current_schema().clone(), - data_file.partition().clone(), - ); - assert_eq!( - path, - location_generator.generate_location(Some(&partition_key), file_name) - ); - } + let error = ctx + .sql("INSERT INTO catalog.ns.t VALUES (1, 'a')") + .await? + .collect() + .await + .unwrap_err(); + + insta::assert_snapshot!(error, @r" + IcebergError(DataInvalid) + caused by + Execution error: DataInvalid => Invalid value for write.object-storage.enabled: provided string was not `true` or `false` + "); Ok(()) } -/// Creates a table with `properties`, identity-partitioned on `category`, inserts a row -/// into each of two partitions, and returns the table and the data files it committed. -async fn insert_into_partitioned_table( +/// Creates `ns.t (id int, category string)`, identity-partitioned on `category`, with +/// table `properties`, and returns a session with its catalog registered as `catalog`. +async fn create_category_table( fixtures: &FixtureCatalog, properties: &[(&str, &str)], -) -> Result<(Table, Vec), Box> { - let ident = TableIdent::from_strs(["ns", "t"])?; +) -> Result> { let schema = Schema::builder() - .with_schema_id(0) .with_fields(vec![ NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), NestedField::required(2, "category", Type::Primitive(PrimitiveType::String)) .into(), ]) .build()?; - let mut builder = fixtures - .table(ident.clone(), schema) + let mut table = fixtures + .table(TableIdent::from_strs(["ns", "t"])?, schema) .partition_by("category", Transform::Identity); for (key, value) in properties { - builder = builder.property(*key, *value); + table = table.property(*key, *value); } - builder.create().await?; + table.create().await?; - let client = fixtures.catalog(); - let catalog = Arc::new(IcebergCatalogProvider::try_new(client.clone()).await?); let ctx = SessionContext::new(); - ctx.register_catalog("catalog", catalog); - ctx.sql("INSERT INTO catalog.ns.t VALUES (1, 'a'), (2, 'b')") - .await? - .collect() - .await?; + ctx.register_catalog( + "catalog", + Arc::new(IcebergCatalogProvider::try_new(fixtures.catalog()).await?), + ); + Ok(ctx) +} - let table = client.load_table(&ident).await?; +/// Returns the paths of `ns.t`'s data files relative to its data directory, sorted, with +/// leading object-storage hash directories such as `0101/1100/0011/01010101` shown as +/// `` and generated file names as ``. +async fn redacted_data_file_paths( + fixtures: &FixtureCatalog, +) -> Result, Box> { + let hash = Regex::new("^[01]{4}/[01]{4}/[01]{4}/[01]{8}")?; + let file_name = Regex::new(r"[0-9a-f-]{36}-[0-9]{5}\.parquet$")?; + + let table = fixtures + .catalog() + .load_table(&TableIdent::from_strs(["ns", "t"])?) + .await?; let snapshot = table.metadata().current_snapshot().unwrap(); let manifest_list = table.manifest_list_reader(snapshot).load().await?; - let mut data_files = Vec::new(); + let mut paths = Vec::new(); for entry in manifest_list.entries() { let manifest = table.manifest_reader().read(entry).await?; for entry in manifest.entries() { - data_files.push(entry.data_file().clone()); + let path = data_file_name(entry.data_file().file_path()); + let path = hash.replace(path, ""); + paths.push(file_name.replace(&path, "").into_owned()); } } + paths.sort(); - Ok((table, data_files)) + Ok(paths) }