From 580478a27b0b0f23201ad635a40186ac8afafc70 Mon Sep 17 00:00:00 2001 From: Ran Reichman Date: Sat, 14 Nov 2026 23:38:02 -0500 Subject: [PATCH] feat(parquet): write Arrow timestamps to INT96 columns --- parquet/src/arrow/arrow_writer/mod.rs | 137 +++++++++++++++++++++++++- parquet/src/data_type.rs | 36 +++++++ 2 files changed, 170 insertions(+), 3 deletions(-) diff --git a/parquet/src/arrow/arrow_writer/mod.rs b/parquet/src/arrow/arrow_writer/mod.rs index 928a6fad414b..104206fcd15c 100644 --- a/parquet/src/arrow/arrow_writer/mod.rs +++ b/parquet/src/arrow/arrow_writer/mod.rs @@ -33,7 +33,7 @@ use crate::column::writer::encoder::ColumnValueEncoder; use crate::column::writer::{ ColumnCloseResult, ColumnWriter, ColumnWriterImpl, GenericColumnWriter, get_column_writer, }; -use crate::data_type::{ByteArray, FixedLenByteArray, FixedLenByteArrayType}; +use crate::data_type::{ByteArray, FixedLenByteArray, FixedLenByteArrayType, Int96}; use std::collections::HashSet; type DistinctValuesSet = HashSet; #[cfg(feature = "encryption")] @@ -1768,8 +1768,46 @@ fn write_leaf( levels, }, ), - ColumnWriter::Int96ColumnWriter(_typed) => { - unreachable!("Currently unreachable because data type not supported") + ColumnWriter::Int96ColumnWriter(typed) => { + let (values, nanos_per_unit) = match column.data_type() { + ArrowDataType::Timestamp(TimeUnit::Second, _) => ( + column.as_primitive::().values(), + 1_000_000_000, + ), + ArrowDataType::Timestamp(TimeUnit::Millisecond, _) => ( + column.as_primitive::().values(), + 1_000_000, + ), + ArrowDataType::Timestamp(TimeUnit::Microsecond, _) => ( + column.as_primitive::().values(), + 1_000, + ), + ArrowDataType::Timestamp(TimeUnit::Nanosecond, _) => { + (column.as_primitive::().values(), 1) + } + d => return Err(ParquetError::General(format!("Cannot coerce {d} to INT96"))), + }; + let values = indices + .iter() + .map(|&i| { + Int96::from_epoch_nanos(values[i] as i128 * nanos_per_unit).ok_or_else(|| { + ParquetError::General(format!( + "{} value {} is out of range for INT96", + column.data_type(), + values[i] + )) + }) + }) + .collect::>>()?; + typed.write_batch_internal( + &values, + None, + levels.def_level_data().as_ref(), + levels.rep_level_data().as_ref(), + None, + None, + None, + ) } ColumnWriter::FloatColumnWriter(typed) => { let array = column.as_primitive::(); @@ -6342,4 +6380,97 @@ mod tests { assert_eq!(fixed, (-1, 1), "incorrect Arrow FixedSizeBinaryArray stats"); assert_eq!(binary, (-1, 1), "incorrect Arrow BinaryArray stats"); } + + #[test] + fn arrow_writer_timestamps_to_int96() { + let seconds = + TimestampSecondArray::from(vec![Some(-1), None, Some(1_700_000_000), None, None]); + let micros = TimestampMicrosecondArray::from(vec![ + Some(0), + Some(-1), + None, + Some(-86_400_000_001), + Some(1_700_000_000_123_456), + ]) + .with_timezone("UTC"); + let nanos = TimestampNanosecondArray::from(vec![ + Some(1), + Some(-1), + Some(1_700_000_000_123_456_789), + None, + Some(-86_400_000_000_001), + ]); + let millis = TimestampMillisecondArray::from(vec![Some(-1), None, Some(1_700_000_000_123)]); + let list = ListArray::new( + Arc::new(Field::new("element", millis.data_type().clone(), true)), + OffsetBuffer::from_lengths([2, 0, 1, 0, 0]), + Arc::new(millis), + None, + ); + let schema = Arc::new(Schema::new(vec![ + Field::new("s", seconds.data_type().clone(), true), + Field::new("us", micros.data_type().clone(), true), + Field::new("ns", nanos.data_type().clone(), true), + Field::new("list", list.data_type().clone(), false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(seconds), + Arc::new(micros), + Arc::new(nanos), + Arc::new(list), + ], + ) + .unwrap(); + let parquet_schema = crate::schema::parser::parse_message_type( + "message schema { + OPTIONAL INT96 s; + OPTIONAL INT96 us; + OPTIONAL INT96 ns; + REQUIRED group list (LIST) { + REPEATED group list { + OPTIONAL INT96 element; + } + } + }", + ) + .unwrap(); + let options = ArrowWriterOptions::new() + .with_parquet_schema(SchemaDescriptor::new(Arc::new(parquet_schema))); + let mut buf = Vec::new(); + let mut writer = ArrowWriter::try_new_with_options(&mut buf, schema, options).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + let builder = ParquetRecordBatchReaderBuilder::try_new(Bytes::from(buf)).unwrap(); + let descr = builder.metadata().file_metadata().schema_descr_ptr(); + assert!( + descr + .columns() + .iter() + .all(|c| c.physical_type() == crate::basic::Type::INT96) + ); + let read = builder.build().unwrap().next().unwrap().unwrap(); + assert_eq!(read, batch); + } + + #[test] + fn arrow_writer_timestamp_out_of_int96_range() { + let seconds = TimestampSecondArray::from(vec![i64::MAX]); + let schema = Arc::new(Schema::new(vec![Field::new( + "s", + seconds.data_type().clone(), + false, + )])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(seconds)]).unwrap(); + let parquet_schema = + crate::schema::parser::parse_message_type("message schema { REQUIRED INT96 s; }") + .unwrap(); + let options = ArrowWriterOptions::new() + .with_parquet_schema(SchemaDescriptor::new(Arc::new(parquet_schema))); + let mut writer = ArrowWriter::try_new_with_options(Vec::new(), schema, options).unwrap(); + let err = writer.write(&batch).unwrap_err(); + assert!(err.to_string().contains("out of range for INT96"), "{err}"); + } } diff --git a/parquet/src/data_type.rs b/parquet/src/data_type.rs index 0b29ee73b44b..42d2c427416a 100644 --- a/parquet/src/data_type.rs +++ b/parquet/src/data_type.rs @@ -105,6 +105,18 @@ impl Int96 { .wrapping_add(nanos) } + /// Creates an INT96 from nanoseconds since EPOCH, or `None` if the Julian day exceeds `i32` + #[cfg(feature = "arrow")] + pub(crate) fn from_epoch_nanos(nanos: i128) -> Option { + let nanos_in_day = NANOSECONDS_IN_DAY as i128; + let day = + i32::try_from(nanos.div_euclid(nanos_in_day) + JULIAN_DAY_OF_EPOCH as i128).ok()?; + let nanos_of_day = nanos.rem_euclid(nanos_in_day) as u64; + let mut result = Self::new(); + result.set_data(nanos_of_day as u32, (nanos_of_day >> 32) as u32, day as u32); + Some(result) + } + #[inline] fn get_days(&self) -> i32 { self.data()[2] as i32 @@ -1400,6 +1412,30 @@ mod tests { ); } + #[test] + #[cfg(feature = "arrow")] + fn test_int96_from_epoch_nanos() { + for nanos in [ + 0, + 1, + -1, + 86_400_000_000_000, + -86_400_000_000_001, + 1_700_000_000_123_456_789, + ] { + assert_eq!( + Int96::from_epoch_nanos(nanos).unwrap().to_nanos() as i128, + nanos + ); + } + let nanos_of_day: u64 = 86_399_999_999_000; + assert_eq!( + Int96::from_epoch_nanos(-1_000).unwrap().data(), + &[nanos_of_day as u32, (nanos_of_day >> 32) as u32, 2_440_587] + ); + assert!(Int96::from_epoch_nanos(i64::MAX as i128 * 1_000_000_000).is_none()); + } + #[test] fn test_byte_array_from() { assert_eq!(ByteArray::from(b"ABC".to_vec()).data(), b"ABC");