Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions crates/iceberg/public-api.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3691,6 +3691,15 @@ impl core::fmt::Debug for iceberg::writer::file_writer::location_generator::Obje
pub fn iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
impl iceberg::writer::file_writer::location_generator::LocationGenerator for iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator
pub fn iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String
pub struct iceberg::writer::file_writer::location_generator::TableLocationGenerator
impl iceberg::writer::file_writer::location_generator::TableLocationGenerator
pub fn iceberg::writer::file_writer::location_generator::TableLocationGenerator::new(table_metadata: &iceberg::spec::TableMetadata) -> iceberg::Result<Self>
impl core::clone::Clone for iceberg::writer::file_writer::location_generator::TableLocationGenerator
pub fn iceberg::writer::file_writer::location_generator::TableLocationGenerator::clone(&self) -> iceberg::writer::file_writer::location_generator::TableLocationGenerator
impl core::fmt::Debug for iceberg::writer::file_writer::location_generator::TableLocationGenerator
pub fn iceberg::writer::file_writer::location_generator::TableLocationGenerator::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
impl iceberg::writer::file_writer::location_generator::LocationGenerator for iceberg::writer::file_writer::location_generator::TableLocationGenerator
pub fn iceberg::writer::file_writer::location_generator::TableLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String
pub trait iceberg::writer::file_writer::location_generator::FileNameGenerator: core::clone::Clone + core::marker::Send + core::marker::Sync + 'static
pub fn iceberg::writer::file_writer::location_generator::FileNameGenerator::generate_file_name(&self) -> alloc::string::String
impl iceberg::writer::file_writer::location_generator::FileNameGenerator for iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator
Expand All @@ -3701,6 +3710,8 @@ impl iceberg::writer::file_writer::location_generator::LocationGenerator for ice
pub fn iceberg::writer::file_writer::location_generator::DefaultLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String
impl iceberg::writer::file_writer::location_generator::LocationGenerator for iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator
pub fn iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String
impl iceberg::writer::file_writer::location_generator::LocationGenerator for iceberg::writer::file_writer::location_generator::TableLocationGenerator
pub fn iceberg::writer::file_writer::location_generator::TableLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String
pub mod iceberg::writer::file_writer::rolling_writer
pub struct iceberg::writer::file_writer::rolling_writer::RollingFileWriter<B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator>
impl<B, L, F> iceberg::writer::file_writer::rolling_writer::RollingFileWriter<B, L, F> where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator
Expand Down
25 changes: 4 additions & 21 deletions crates/iceberg/src/cow_rewrite/writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,7 @@ use crate::writer::IcebergWriterBuilder;
use crate::writer::base_writer::data_file_writer::DataFileWriterBuilder;
use crate::writer::file_writer::ParquetWriterBuilder;
use crate::writer::file_writer::location_generator::{
DefaultFileNameGenerator, DefaultLocationGenerator, LocationGenerator,
ObjectStorageLocationGenerator,
DefaultFileNameGenerator, TableLocationGenerator,
};
use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder;

Expand All @@ -44,9 +43,8 @@ use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder;
/// batch is written, so it is safe to construct one optimistically and only
/// write to it once a source file is known to have changed.
///
/// The location layout follows `write.object-storage.enabled`: tables that opt
/// into the object-storage layout get hash-entropy paths from
/// [`ObjectStorageLocationGenerator`], everything else gets the default layout.
/// File locations follow the layout selected by the table properties; see
/// [`TableLocationGenerator`].
pub(crate) async fn build_replacement_writer(
table: &Table,
write_schema: SchemaRef,
Expand All @@ -72,27 +70,12 @@ pub(crate) async fn build_replacement_writer(
parquet_builder = parquet_builder.with_encryption_manager(encryption_manager.clone());
}

if table_props.write_object_storage_enabled()? {
let location_generator = ObjectStorageLocationGenerator::new(table.metadata())?;
build_data_file_writer(table, parquet_builder, location_generator, partition_key).await
} else {
let location_generator = DefaultLocationGenerator::new(table.metadata())?;
build_data_file_writer(table, parquet_builder, location_generator, partition_key).await
}
}

async fn build_data_file_writer<L: LocationGenerator>(
table: &Table,
parquet_builder: ParquetWriterBuilder,
location_generator: L,
partition_key: Option<PartitionKey>,
) -> Result<Box<dyn crate::writer::IcebergWriter>> {
let location_generator = TableLocationGenerator::new(table.metadata())?;
let file_name_generator = DefaultFileNameGenerator::new(
format!("cow-rewrite-{}", Uuid::now_v7()),
None,
DataFileFormat::Parquet,
);
let table_props = table.metadata().table_properties();
let rolling_builder = RollingFileWriterBuilder::new(
parquet_builder,
table_props.write_target_file_size_bytes()?,
Expand Down
132 changes: 132 additions & 0 deletions crates/iceberg/src/writer/file_writer/location_generator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,10 @@ const ENTROPY_DIR_DEPTH: usize = 3;
#[derive(Clone, Debug)]
/// `DefaultLocationGenerator` used to generate the data dir location of data file.
/// The location is generated based on the table location and the data location in table properties.
///
/// Paths use `{data location}/{partition path}/{file name}`, omitting the
/// partition path for unpartitioned files. This generator always uses this layout.
/// Use [`TableLocationGenerator`] to select the layout from table properties.
pub struct DefaultLocationGenerator {
data_location: String,
}
Expand Down Expand Up @@ -203,6 +207,52 @@ impl LocationGenerator for ObjectStorageLocationGenerator {
}
}

/// Generates data file paths using the layout configured in table properties.
///
/// It uses [`ObjectStorageLocationGenerator`] when
/// [`write.object-storage.enabled`](crate::spec::TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED)
/// is `true`, and [`DefaultLocationGenerator`] otherwise.
#[derive(Clone, Debug)]
pub struct TableLocationGenerator {

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wrote up some context in #3729 on why I've gone with this approach but open to change / discussion if folks have thoughts. A new generator here lets us keep compatibility with current client behaviour, but the downside is that DefaultLocationGenerator reads naturally yet may not do what's desired.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

IAlso, Java and Python support custom location providers so that is worth considering too w.r.t. the API design we introduce here. I think the proposed design should be able to support that by e.g. adding Custom(Arc<dyn <ErasedGenerator>) to the Layout enum)

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I think we should consider custom location provider here. enum seems like an acceptable workaround to me, but I don't want to block the custom location provider support for datafusion. Would appreciate more clarity here!

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(You're right, I'm having a think here on the best shape)

inner: Layout,
}

#[derive(Clone, Debug)]
enum Layout {
Default(DefaultLocationGenerator),
ObjectStorage(ObjectStorageLocationGenerator),
}

impl TableLocationGenerator {
/// Creates a generator from table metadata.
///
/// # Errors
///
/// Returns an error if a layout-related table property has an invalid value.
pub fn new(table_metadata: &TableMetadata) -> Result<Self> {
let inner = if table_metadata
.table_properties()
.write_object_storage_enabled()?
{
Layout::ObjectStorage(ObjectStorageLocationGenerator::new(table_metadata)?)
} else {
Layout::Default(DefaultLocationGenerator::new(table_metadata)?)
};
Ok(Self { inner })
}
}

impl LocationGenerator for TableLocationGenerator {
fn generate_location(&self, partition_key: Option<&PartitionKey>, file_name: &str) -> String {
match &self.inner {
Layout::Default(generator) => generator.generate_location(partition_key, file_name),
Layout::ObjectStorage(generator) => {
generator.generate_location(partition_key, file_name)
}
}
}
}

/// Derive the `{parent}/{name}` context from a table location, mirroring Hadoop's
/// `Path.getParent().getName()` / `Path.getName()`.
fn path_context(table_location: &str) -> String {
Expand Down Expand Up @@ -291,15 +341,18 @@ pub(crate) mod test {
use std::collections::HashMap;
use std::sync::Arc;

use rstest::rstest;
use uuid::Uuid;

use super::LocationGenerator;
use crate::ErrorKind;
use crate::spec::{
FormatVersion, Literal, NestedField, PartitionKey, PartitionSpec, PrimitiveType, Schema,
Struct, StructType, TableMetadata, TableProperties, Transform, Type,
};
use crate::writer::file_writer::location_generator::{
DefaultLocationGenerator, FileNameGenerator, ObjectStorageLocationGenerator,
TableLocationGenerator,
};

#[test]
Expand Down Expand Up @@ -609,6 +662,85 @@ pub(crate) mod test {
);
}

#[rstest]

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm wary that I could be over-testing here (I did somehwat like how these tests read though)

#[case::unset(&[], [
"s3://data.db/table/data/test.parquet",
"s3://data.db/table/data/data%231=val%231/test.parquet",
])]
#[case::disabled(&[("write.object-storage.enabled", "false")], [
"s3://data.db/table/data/test.parquet",
"s3://data.db/table/data/data%231=val%231/test.parquet",
])]
#[case::enabled(&[("write.object-storage.enabled", "true")], [
"s3://data.db/table/data/0110/1010/0011/11101000/test.parquet",
"s3://data.db/table/data/0000/1011/0110/00001000/data%231=val%231/test.parquet",
])]
#[case::uppercase(&[("write.object-storage.enabled", "TRUE")], [
"s3://data.db/table/data/0110/1010/0011/11101000/test.parquet",
"s3://data.db/table/data/0000/1011/0110/00001000/data%231=val%231/test.parquet",
])]
#[case::without_partition_paths(&[
("write.object-storage.enabled", "true"),
("write.object-storage.partitioned-paths", "false"),
], [
"s3://data.db/table/data/0110/1010/0011/11101000-test.parquet",
"s3://data.db/table/data/0110/1010/0011/11101000-test.parquet",
])]
fn table_location_follows_properties(
#[case] properties: &[(&str, &str)],
#[case] expected: [&str; 2],
) {
let metadata = table_metadata_with(
"s3://data.db/table",
properties
.iter()
.map(|(key, value)| (key.to_string(), value.to_string()))
.collect(),
);
let schema = Arc::new(
Schema::builder()
.with_fields(vec![
NestedField::required(1, "data#1", Type::Primitive(PrimitiveType::String))
.into(),
])
.build()
.unwrap(),
);
let partition_spec = PartitionSpec::builder(schema.clone())
.add_partition_field("data#1", "data#1", Transform::Identity)
.unwrap()
.build()
.unwrap();
let partition_key = PartitionKey::new(
partition_spec,
schema,
Struct::from_iter([Some(Literal::string("val#1"))]),
);
let generator = TableLocationGenerator::new(&metadata).unwrap();

assert_eq!(
[
generator.generate_location(None, "test.parquet"),
generator.generate_location(Some(&partition_key), "test.parquet"),
],
expected
);
}

#[test]
fn table_location_rejects_invalid_property() {
let metadata = table_metadata_with(
"s3://data.db/table",
HashMap::from([(
"write.object-storage.enabled".to_string(),
"yes".to_string(),
)]),
);
let err = TableLocationGenerator::new(&metadata).unwrap_err();

assert_eq!(err.kind(), ErrorKind::DataInvalid);
}

/// Build a minimal `TableMetadata` for location generator tests.
fn table_metadata_with(location: &str, properties: HashMap<String, String>) -> TableMetadata {
TableMetadata {
Expand Down
16 changes: 8 additions & 8 deletions crates/iceberg/src/writer/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
//! use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder;
//! use iceberg::writer::file_writer::ParquetWriterBuilder;
//! use iceberg::writer::file_writer::location_generator::{
//! DefaultFileNameGenerator, DefaultLocationGenerator,
//! DefaultFileNameGenerator, TableLocationGenerator,
//! };
//! use iceberg::writer::{IcebergWriter, IcebergWriterBuilder};
//! use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, Result, TableIdent};
Expand All @@ -78,7 +78,7 @@
//! let table = catalog
//! .load_table(&TableIdent::from_strs(["hello", "world"])?)
//! .await?;
//! let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap();
//! let location_generator = TableLocationGenerator::new(table.metadata()).unwrap();
//! let file_name_generator = DefaultFileNameGenerator::new(
//! "test".to_string(),
//! None,
Expand Down Expand Up @@ -125,7 +125,7 @@
//! use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder;
//! use iceberg::writer::file_writer::ParquetWriterBuilder;
//! use iceberg::writer::file_writer::location_generator::{
//! DefaultFileNameGenerator, DefaultLocationGenerator,
//! DefaultFileNameGenerator, TableLocationGenerator,
//! };
//! use iceberg::writer::{IcebergWriter, IcebergWriterBuilder};
//! use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, Result, TableIdent};
Expand Down Expand Up @@ -207,7 +207,7 @@
//! table.metadata().current_schema().clone(),
//! Struct::from_iter(vec![Some(Literal::string("Seattle"))]),
//! );
//! let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap();
//! let location_generator = TableLocationGenerator::new(table.metadata()).unwrap();
//! let file_name_generator = DefaultFileNameGenerator::new(
//! "test".to_string(),
//! None,
Expand Down Expand Up @@ -262,7 +262,7 @@
//! # use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder;
//! # use iceberg::writer::file_writer::ParquetWriterBuilder;
//! # use iceberg::writer::file_writer::location_generator::{
//! # DefaultFileNameGenerator, DefaultLocationGenerator,
//! # DefaultFileNameGenerator, TableLocationGenerator,
//! # };
//! # use parquet::file::properties::WriterProperties;
//! # use std::collections::HashMap;
Expand All @@ -272,7 +272,7 @@
//! # .load("memory", HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), "file:///path/to/warehouse".to_string())]))
//! # .await?;
//! # let table = catalog.load_table(&TableIdent::from_strs(["hello", "world"])?).await?;
//! # let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap();
//! # let location_generator = TableLocationGenerator::new(table.metadata()).unwrap();
//! # let file_name_generator = DefaultFileNameGenerator::new("test".to_string(), None, iceberg::spec::DataFileFormat::Parquet);
//! # let parquet_writer_builder = ParquetWriterBuilder::new(WriterProperties::default(), table.metadata().current_schema().clone());
//! # let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size(
Expand Down Expand Up @@ -326,7 +326,7 @@
//! # use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder;
//! # use iceberg::writer::file_writer::ParquetWriterBuilder;
//! # use iceberg::writer::file_writer::location_generator::{
//! # DefaultFileNameGenerator, DefaultLocationGenerator,
//! # DefaultFileNameGenerator, TableLocationGenerator,
//! # };
//! # use parquet::file::properties::WriterProperties;
//! # use std::collections::HashMap;
Expand All @@ -336,7 +336,7 @@
//! # .load("memory", HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), "file:///path/to/warehouse".to_string())]))
//! # .await?;
//! # let table = catalog.load_table(&TableIdent::from_strs(["hello", "world"])?).await?;
//! # let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap();
//! # let location_generator = TableLocationGenerator::new(table.metadata()).unwrap();
//! # let file_name_generator = DefaultFileNameGenerator::new("test".to_string(), None, iceberg::spec::DataFileFormat::Parquet);
//! # let parquet_writer_builder = ParquetWriterBuilder::new(WriterProperties::default(), table.metadata().current_schema().clone());
//! # let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size(
Expand Down
Loading