Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions crates/datafusion/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ uuid = { version = "1.18", features = ["v7"] }
iceberg-datafusion-test-tools = { workspace = true }
insta = "1"
parquet = "59.2"
regex = "1"
tempfile = "3.18"

# Every integration test is a module of `tests/main.rs`, so they all link into one binary
Expand Down
5 changes: 2 additions & 3 deletions crates/datafusion/src/write/exec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ use iceberg::table::Table;
use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder;
use iceberg::writer::file_writer::ParquetWriterBuilder;
use iceberg::writer::file_writer::location_generator::{
DefaultFileNameGenerator, DefaultLocationGenerator,
DefaultFileNameGenerator, TableLocationGenerator,
};
use iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder;
use uuid::Uuid;
Expand Down Expand Up @@ -276,8 +276,7 @@ impl ExecutionPlan for IcebergWriteExec {
.map_err(to_datafusion_error)?;

let file_io = self.table.file_io().clone();
// todo location_gen and file_name_gen should be configurable
let location_generator = DefaultLocationGenerator::new(self.table.metadata())
let location_generator = TableLocationGenerator::new(self.table.metadata())
.map_err(to_datafusion_error)?;
// todo filename prefix/suffix should be configurable
let file_name_generator =
Expand Down
150 changes: 148 additions & 2 deletions crates/datafusion/tests/insert_into.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,12 @@ use datafusion::arrow::util::pretty::pretty_format_batches;
use datafusion::execution::context::{SessionConfig, SessionContext};
use iceberg::TableIdent;
use iceberg::spec::{
Literal, NestedField, PrimitiveType, Schema, Struct, StructType, Transform, Type,
Literal, NestedField, PrimitiveType, Schema, Struct, StructType, TableProperties,
Transform, Type,
};
use iceberg_datafusion::IcebergCatalogProvider;
use iceberg_datafusion_test_tools::{FixtureCatalog, display_schema};
use iceberg_datafusion_test_tools::{FixtureCatalog, data_file_name, display_schema};
use regex::Regex;

#[tokio::test]
async fn test_insert_into() -> Result<(), Box<dyn Error>> {
Expand Down Expand Up @@ -489,3 +491,147 @@ async fn test_insert_into_partitioned() -> Result<(), Box<dyn Error>> {

Ok(())
}

#[tokio::test]
async fn test_insert_into_object_storage_layout_with_partition_paths()
-> Result<(), Box<dyn Error>> {
let fixtures = FixtureCatalog::try_new().await?;
let ctx = create_category_table(
&fixtures,
&[(
TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED,
"true",
)],
)
.await?;

ctx.sql("INSERT INTO catalog.ns.t VALUES (1, 'a')")
.await?
.collect()
.await?;

assert_eq!(
redacted_data_file_paths(&fixtures).await?,

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

With apache/iceberg-rust#3730, we could rely on Iceberg Rust behaviour and instead here just check that the file path matches what ObjectStorageLocationGenerator would yield by using it directly. I thought that being more explicit in the actual paths we expect might be nicer hence gone with this

["<hash>/category=a/<file>"]
);

Ok(())
}

#[tokio::test]
async fn test_insert_into_object_storage_layout_without_partition_paths()
-> Result<(), Box<dyn Error>> {
let fixtures = FixtureCatalog::try_new().await?;
let ctx = create_category_table(
&fixtures,
&[
(
TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED,
"true",
),
(
TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS,
"false",
),
],
)
.await?;

ctx.sql("INSERT INTO catalog.ns.t VALUES (1, 'a')")
.await?
.collect()
.await?;

assert_eq!(
redacted_data_file_paths(&fixtures).await?,
["<hash>-<file>"]
);

Ok(())
}

#[tokio::test]
async fn test_insert_into_invalid_object_storage_enabled() -> Result<(), Box<dyn Error>> {
let fixtures = FixtureCatalog::try_new().await?;
let ctx = create_category_table(
&fixtures,
&[(
TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED,
"yes",
)],
)
.await?;

let error = ctx
.sql("INSERT INTO catalog.ns.t VALUES (1, 'a')")
.await?
.collect()
.await
.unwrap_err();

insta::assert_snapshot!(error, @r"
IcebergError(DataInvalid)
caused by
Execution error: DataInvalid => Invalid value for write.object-storage.enabled: provided string was not `true` or `false`
");

Ok(())
}

/// Creates `ns.t (id int, category string)`, identity-partitioned on `category`, with
/// table `properties`, and returns a session with its catalog registered as `catalog`.
async fn create_category_table(
fixtures: &FixtureCatalog,
properties: &[(&str, &str)],
) -> Result<SessionContext, Box<dyn Error>> {
let schema = Schema::builder()
.with_fields(vec![
NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
NestedField::required(2, "category", Type::Primitive(PrimitiveType::String))
.into(),
])
.build()?;
let mut table = fixtures
.table(TableIdent::from_strs(["ns", "t"])?, schema)
.partition_by("category", Transform::Identity);
for (key, value) in properties {
table = table.property(*key, *value);
}
table.create().await?;

let ctx = SessionContext::new();
ctx.register_catalog(
"catalog",
Arc::new(IcebergCatalogProvider::try_new(fixtures.catalog()).await?),
);
Ok(ctx)
}

/// Returns the paths of `ns.t`'s data files relative to its data directory, sorted, with
/// leading object-storage hash directories such as `0101/1100/0011/01010101` shown as
/// `<hash>` and generated file names as `<file>`.
async fn redacted_data_file_paths(
fixtures: &FixtureCatalog,
) -> Result<Vec<String>, Box<dyn Error>> {
let hash = Regex::new("^[01]{4}/[01]{4}/[01]{4}/[01]{8}")?;
let file_name = Regex::new(r"[0-9a-f-]{36}-[0-9]{5}\.parquet$")?;

let table = fixtures
.catalog()
.load_table(&TableIdent::from_strs(["ns", "t"])?)
.await?;
let snapshot = table.metadata().current_snapshot().unwrap();
let manifest_list = table.manifest_list_reader(snapshot).load().await?;
let mut paths = Vec::new();
for entry in manifest_list.entries() {
let manifest = table.manifest_reader().read(entry).await?;
for entry in manifest.entries() {
let path = data_file_name(entry.data_file().file_path());
let path = hash.replace(path, "<hash>");
paths.push(file_name.replace(&path, "<file>").into_owned());
}
}
paths.sort();

Ok(paths)
}