Repository navigation
feat(parquet): write Arrow timestamps to INT96 columns #12263
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -33,7 +33,7 @@ use crate::column::writer::encoder::ColumnValueEncoder; | |
| use crate::column::writer::{ | ||
| ColumnCloseResult, ColumnWriter, ColumnWriterImpl, GenericColumnWriter, get_column_writer, | ||
| }; | ||
| use crate::data_type::{ByteArray, FixedLenByteArray, FixedLenByteArrayType}; | ||
| use crate::data_type::{ByteArray, FixedLenByteArray, FixedLenByteArrayType, Int96}; | ||
| use std::collections::HashSet; | ||
| type DistinctValuesSet = HashSet<u64>; | ||
| #[cfg(feature = "encryption")] | ||
|
|
@@ -1768,8 +1768,46 @@ fn write_leaf( | |
| levels, | ||
| }, | ||
| ), | ||
| ColumnWriter::Int96ColumnWriter(_typed) => { | ||
| unreachable!("Currently unreachable because data type not supported") | ||
| ColumnWriter::Int96ColumnWriter(typed) => { | ||
| let (values, nanos_per_unit) = match column.data_type() { | ||
| ArrowDataType::Timestamp(TimeUnit::Second, _) => ( | ||
| column.as_primitive::<TimestampSecondType>().values(), | ||
| 1_000_000_000, | ||
| ), | ||
| ArrowDataType::Timestamp(TimeUnit::Millisecond, _) => ( | ||
| column.as_primitive::<TimestampMillisecondType>().values(), | ||
| 1_000_000, | ||
| ), | ||
| ArrowDataType::Timestamp(TimeUnit::Microsecond, _) => ( | ||
| column.as_primitive::<TimestampMicrosecondType>().values(), | ||
| 1_000, | ||
| ), | ||
| ArrowDataType::Timestamp(TimeUnit::Nanosecond, _) => { | ||
| (column.as_primitive::<TimestampNanosecondType>().values(), 1) | ||
| } | ||
| d => return Err(ParquetError::General(format!("Cannot coerce {d} to INT96"))), | ||
|
Comment on lines
+1772
to
+1788
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This writes The arrow-rs reader makes the opposite assumption. Without the embedded Arrow schema, it reads INT96 as Should INT96 accept only timestamps with a time zone, and return the "Cannot coerce" error for |
||
| }; | ||
| let values = indices | ||
| .iter() | ||
| .map(|&i| { | ||
| Int96::from_epoch_nanos(values[i] as i128 * nanos_per_unit).ok_or_else(|| { | ||
| ParquetError::General(format!( | ||
| "{} value {} is out of range for INT96", | ||
| column.data_type(), | ||
| values[i] | ||
| )) | ||
| }) | ||
| }) | ||
| .collect::<Result<Vec<_>>>()?; | ||
|
Comment on lines
+1790
to
+1801
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
| typed.write_batch_internal( | ||
| &values, | ||
| None, | ||
| levels.def_level_data().as_ref(), | ||
| levels.rep_level_data().as_ref(), | ||
| None, | ||
| None, | ||
| None, | ||
| ) | ||
| } | ||
| ColumnWriter::FloatColumnWriter(typed) => { | ||
| let array = column.as_primitive::<Float32Type>(); | ||
|
|
@@ -6342,4 +6380,97 @@ mod tests { | |
| assert_eq!(fixed, (-1, 1), "incorrect Arrow FixedSizeBinaryArray stats"); | ||
| assert_eq!(binary, (-1, 1), "incorrect Arrow BinaryArray stats"); | ||
| } | ||
|
|
||
| #[test] | ||
| fn arrow_writer_timestamps_to_int96() { | ||
| let seconds = | ||
| TimestampSecondArray::from(vec![Some(-1), None, Some(1_700_000_000), None, None]); | ||
| let micros = TimestampMicrosecondArray::from(vec![ | ||
| Some(0), | ||
| Some(-1), | ||
| None, | ||
| Some(-86_400_000_001), | ||
| Some(1_700_000_000_123_456), | ||
| ]) | ||
| .with_timezone("UTC"); | ||
| let nanos = TimestampNanosecondArray::from(vec![ | ||
| Some(1), | ||
| Some(-1), | ||
| Some(1_700_000_000_123_456_789), | ||
| None, | ||
| Some(-86_400_000_000_001), | ||
| ]); | ||
| let millis = TimestampMillisecondArray::from(vec![Some(-1), None, Some(1_700_000_000_123)]); | ||
| let list = ListArray::new( | ||
| Arc::new(Field::new("element", millis.data_type().clone(), true)), | ||
| OffsetBuffer::from_lengths([2, 0, 1, 0, 0]), | ||
| Arc::new(millis), | ||
| None, | ||
| ); | ||
| let schema = Arc::new(Schema::new(vec![ | ||
| Field::new("s", seconds.data_type().clone(), true), | ||
| Field::new("us", micros.data_type().clone(), true), | ||
| Field::new("ns", nanos.data_type().clone(), true), | ||
| Field::new("list", list.data_type().clone(), false), | ||
| ])); | ||
| let batch = RecordBatch::try_new( | ||
| schema.clone(), | ||
| vec![ | ||
| Arc::new(seconds), | ||
| Arc::new(micros), | ||
| Arc::new(nanos), | ||
| Arc::new(list), | ||
| ], | ||
| ) | ||
| .unwrap(); | ||
| let parquet_schema = crate::schema::parser::parse_message_type( | ||
| "message schema { | ||
| OPTIONAL INT96 s; | ||
| OPTIONAL INT96 us; | ||
| OPTIONAL INT96 ns; | ||
| REQUIRED group list (LIST) { | ||
| REPEATED group list { | ||
| OPTIONAL INT96 element; | ||
| } | ||
| } | ||
| }", | ||
| ) | ||
| .unwrap(); | ||
| let options = ArrowWriterOptions::new() | ||
| .with_parquet_schema(SchemaDescriptor::new(Arc::new(parquet_schema))); | ||
| let mut buf = Vec::new(); | ||
| let mut writer = ArrowWriter::try_new_with_options(&mut buf, schema, options).unwrap(); | ||
| writer.write(&batch).unwrap(); | ||
| writer.close().unwrap(); | ||
|
|
||
| let builder = ParquetRecordBatchReaderBuilder::try_new(Bytes::from(buf)).unwrap(); | ||
| let descr = builder.metadata().file_metadata().schema_descr_ptr(); | ||
| assert!( | ||
| descr | ||
| .columns() | ||
| .iter() | ||
| .all(|c| c.physical_type() == crate::basic::Type::INT96) | ||
| ); | ||
| let read = builder.build().unwrap().next().unwrap().unwrap(); | ||
| assert_eq!(read, batch); | ||
| } | ||
|
|
||
| #[test] | ||
| fn arrow_writer_timestamp_out_of_int96_range() { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This test checks one value far past the range. Could we test at the boundaries instead? For The conversion skips null slots on purpose, so an out-of-range value under a null doesn't fail the write. Could a test cover that too, with a null slot that holds |
||
| let seconds = TimestampSecondArray::from(vec![i64::MAX]); | ||
| let schema = Arc::new(Schema::new(vec![Field::new( | ||
| "s", | ||
| seconds.data_type().clone(), | ||
| false, | ||
| )])); | ||
| let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(seconds)]).unwrap(); | ||
| let parquet_schema = | ||
| crate::schema::parser::parse_message_type("message schema { REQUIRED INT96 s; }") | ||
| .unwrap(); | ||
| let options = ArrowWriterOptions::new() | ||
| .with_parquet_schema(SchemaDescriptor::new(Arc::new(parquet_schema))); | ||
| let mut writer = ArrowWriter::try_new_with_options(Vec::new(), schema, options).unwrap(); | ||
| let err = writer.write(&batch).unwrap_err(); | ||
| assert!(err.to_string().contains("out of range for INT96"), "{err}"); | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
With bloom filters enabled,
ArrowWriternow writes a bloom filter for an INT96 column, butArrowSbbfcan't check it. I wroteTimestamp(Microsecond, "UTC")values to an INT96 column withset_bloom_filter_enabled(true).ArrowSbbf::try_newwith that Arrow type then returned aCannot check a bloom filter of type INT96 with ...error.validateonly accepts the types thatarrow_to_parquet_typeproduces, and that function never produces INT96 (bloom_filter.rs). The INT96 arm ofcheckalso still saysArrowWriter does not write INT96 columns(bloom_filter.rs).ArrowSbbfexists so that callers can check Arrow values converted the same wayArrowWriterconverts them. Could the conversion move into avisit_int96that bothwrite_leafandArrowSbbf::checkcall, the wayvisit_i64works, withvalidateacceptingTimestampfor INT96 columns?