diff --git a/Cargo.lock b/Cargo.lock index a2a9af5..fe872b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3248,6 +3248,7 @@ dependencies = [ "iceberg-datafusion-test-tools", "insta", "parquet", + "regex", "tempfile", "tokio", "uuid", diff --git a/crates/datafusion/Cargo.toml b/crates/datafusion/Cargo.toml index e7fb281..f934128 100644 --- a/crates/datafusion/Cargo.toml +++ b/crates/datafusion/Cargo.toml @@ -44,6 +44,7 @@ uuid = { version = "1.18", features = ["v7"] } iceberg-datafusion-test-tools = { workspace = true } insta = "1" parquet = "59.2" +regex = "1" tempfile = "3.18" # Every integration test is a module of `tests/main.rs`, so they all link into one binary diff --git a/crates/datafusion/src/write/exec.rs b/crates/datafusion/src/write/exec.rs index 0beaaa9..7c537fa 100644 --- a/crates/datafusion/src/write/exec.rs +++ b/crates/datafusion/src/write/exec.rs @@ -44,7 +44,7 @@ use iceberg::table::Table; use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; use iceberg::writer::file_writer::ParquetWriterBuilder; use iceberg::writer::file_writer::location_generator::{ - DefaultFileNameGenerator, DefaultLocationGenerator, + DefaultFileNameGenerator, TableLocationGenerator, }; use iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder; use uuid::Uuid; @@ -276,8 +276,7 @@ impl ExecutionPlan for IcebergWriteExec { .map_err(to_datafusion_error)?; let file_io = self.table.file_io().clone(); - // todo location_gen and file_name_gen should be configurable - let location_generator = DefaultLocationGenerator::new(self.table.metadata()) + let location_generator = TableLocationGenerator::new(self.table.metadata()) .map_err(to_datafusion_error)?; // todo filename prefix/suffix should be configurable let file_name_generator = diff --git a/crates/datafusion/tests/insert_into.rs b/crates/datafusion/tests/insert_into.rs index 58ca01d..7f4cbe4 100644 --- a/crates/datafusion/tests/insert_into.rs +++ b/crates/datafusion/tests/insert_into.rs @@ -28,10 +28,12 @@ use datafusion::arrow::util::pretty::pretty_format_batches; use datafusion::execution::context::{SessionConfig, SessionContext}; use iceberg::TableIdent; use iceberg::spec::{ - Literal, NestedField, PrimitiveType, Schema, Struct, StructType, Transform, Type, + Literal, NestedField, PrimitiveType, Schema, Struct, StructType, TableProperties, + Transform, Type, }; use iceberg_datafusion::IcebergCatalogProvider; -use iceberg_datafusion_test_tools::{FixtureCatalog, display_schema}; +use iceberg_datafusion_test_tools::{FixtureCatalog, data_file_name, display_schema}; +use regex::Regex; #[tokio::test] async fn test_insert_into() -> Result<(), Box> { @@ -489,3 +491,147 @@ async fn test_insert_into_partitioned() -> Result<(), Box> { Ok(()) } + +#[tokio::test] +async fn test_insert_into_object_storage_layout_with_partition_paths() +-> Result<(), Box> { + let fixtures = FixtureCatalog::try_new().await?; + let ctx = create_category_table( + &fixtures, + &[( + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED, + "true", + )], + ) + .await?; + + ctx.sql("INSERT INTO catalog.ns.t VALUES (1, 'a')") + .await? + .collect() + .await?; + + assert_eq!( + redacted_data_file_paths(&fixtures).await?, + ["/category=a/"] + ); + + Ok(()) +} + +#[tokio::test] +async fn test_insert_into_object_storage_layout_without_partition_paths() +-> Result<(), Box> { + let fixtures = FixtureCatalog::try_new().await?; + let ctx = create_category_table( + &fixtures, + &[ + ( + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED, + "true", + ), + ( + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS, + "false", + ), + ], + ) + .await?; + + ctx.sql("INSERT INTO catalog.ns.t VALUES (1, 'a')") + .await? + .collect() + .await?; + + assert_eq!( + redacted_data_file_paths(&fixtures).await?, + ["-"] + ); + + Ok(()) +} + +#[tokio::test] +async fn test_insert_into_invalid_object_storage_enabled() -> Result<(), Box> { + let fixtures = FixtureCatalog::try_new().await?; + let ctx = create_category_table( + &fixtures, + &[( + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED, + "yes", + )], + ) + .await?; + + let error = ctx + .sql("INSERT INTO catalog.ns.t VALUES (1, 'a')") + .await? + .collect() + .await + .unwrap_err(); + + insta::assert_snapshot!(error, @r" + IcebergError(DataInvalid) + caused by + Execution error: DataInvalid => Invalid value for write.object-storage.enabled: provided string was not `true` or `false` + "); + + Ok(()) +} + +/// Creates `ns.t (id int, category string)`, identity-partitioned on `category`, with +/// table `properties`, and returns a session with its catalog registered as `catalog`. +async fn create_category_table( + fixtures: &FixtureCatalog, + properties: &[(&str, &str)], +) -> Result> { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "category", Type::Primitive(PrimitiveType::String)) + .into(), + ]) + .build()?; + let mut table = fixtures + .table(TableIdent::from_strs(["ns", "t"])?, schema) + .partition_by("category", Transform::Identity); + for (key, value) in properties { + table = table.property(*key, *value); + } + table.create().await?; + + let ctx = SessionContext::new(); + ctx.register_catalog( + "catalog", + Arc::new(IcebergCatalogProvider::try_new(fixtures.catalog()).await?), + ); + Ok(ctx) +} + +/// Returns the paths of `ns.t`'s data files relative to its data directory, sorted, with +/// leading object-storage hash directories such as `0101/1100/0011/01010101` shown as +/// `` and generated file names as ``. +async fn redacted_data_file_paths( + fixtures: &FixtureCatalog, +) -> Result, Box> { + let hash = Regex::new("^[01]{4}/[01]{4}/[01]{4}/[01]{8}")?; + let file_name = Regex::new(r"[0-9a-f-]{36}-[0-9]{5}\.parquet$")?; + + let table = fixtures + .catalog() + .load_table(&TableIdent::from_strs(["ns", "t"])?) + .await?; + let snapshot = table.metadata().current_snapshot().unwrap(); + let manifest_list = table.manifest_list_reader(snapshot).load().await?; + let mut paths = Vec::new(); + for entry in manifest_list.entries() { + let manifest = table.manifest_reader().read(entry).await?; + for entry in manifest.entries() { + let path = data_file_name(entry.data_file().file_path()); + let path = hash.replace(path, ""); + paths.push(file_name.replace(&path, "").into_owned()); + } + } + paths.sort(); + + Ok(paths) +}