From de9a613d3982dd78bcddfc3cae1645241c90483d Mon Sep 17 00:00:00 2001 From: Manu Zhang Date: Sun, 8 Nov 2026 01:28:56 +0900 Subject: [PATCH] feat(parquet): allow updating cached file key-value metadata --- parquet/src/file/metadata/mod.rs | 60 ++++++++++++++++++++++++++ parquet/tests/encryption/encryption.rs | 55 ++++++++++++++++++++++- 2 files changed, 114 insertions(+), 1 deletion(-) diff --git a/parquet/src/file/metadata/mod.rs b/parquet/src/file/metadata/mod.rs index 8b6d4edc0d1e..44dc8cac37f4 100644 --- a/parquet/src/file/metadata/mod.rs +++ b/parquet/src/file/metadata/mod.rs @@ -393,6 +393,18 @@ impl ParquetMetaDataBuilder { } } + /// Replaces the file-level key-value metadata. + /// + /// Pass `None` to remove all entries. This only updates the in-memory metadata; + /// it does not modify the Parquet file. + /// + /// Other metadata, including row groups, page indexes, and decryption state, + /// is preserved. Shared file metadata is cloned before modification. + pub fn set_key_value_metadata(mut self, key_value_metadata: Option>) -> Self { + Arc::make_mut(&mut self.file_metadata).key_value_metadata = key_value_metadata; + self + } + /// Adds a row group to the metadata /// /// If the row groups are shared with another `ParquetMetaData`, this clones @@ -2329,6 +2341,54 @@ mod tests { assert!(!page_index.has_offset_indexes()); } + #[test] + fn test_set_key_value_metadata_preserves_shared_metadata() { + let original = vec![KeyValue::new("original".to_string(), "value".to_string())]; + let replacement = vec![KeyValue::new("replacement".to_string(), "new".to_string())]; + for initial in [None, Some(original)] { + let file_metadata = FileMetaData::new( + 2, + 0, + Some("test writer".to_string()), + initial.clone(), + get_test_schema_descr(), + None, + ); + #[cfg(feature = "encryption")] + let file_metadata = file_metadata + .with_encryption_algorithm(Some(EncryptionAlgorithm::AES_GCM_V1( + crate::file::metadata::thrift::encryption::AesGcmV1 { + aad_prefix: Some(vec![1; 8]), + aad_file_unique: Some(vec![2; 8]), + supply_aad_prefix: Some(true), + }, + ))) + .with_footer_signing_key_metadata(Some(vec![3; 8])); + let cached = ParquetMetaDataBuilder::new(file_metadata) + .set_page_index(Some(Arc::new(PageIndexBuilder::new(0, 2).build()))) + .build(); + for values in [None, Some(vec![]), Some(replacement.clone())] { + let updated = cached + .clone() + .into_builder() + .set_key_value_metadata(values.clone()) + .build(); + let mut expected = cached.file_metadata().clone(); + expected.key_value_metadata = values; + assert_eq!(updated.file_metadata(), &expected); + assert_eq!( + cached.file_metadata().key_value_metadata(), + initial.as_ref() + ); + assert!(Arc::ptr_eq(&cached.row_groups, &updated.row_groups)); + assert!(Arc::ptr_eq( + cached.page_index().unwrap(), + updated.page_index().unwrap() + )); + } + } + } + #[test] fn test_replace_page_index_preserves_shared_metadata() { let schema_descr = get_test_schema_descr(); diff --git a/parquet/tests/encryption/encryption.rs b/parquet/tests/encryption/encryption.rs index 603f982dd6c0..ffddae6e44d5 100644 --- a/parquet/tests/encryption/encryption.rs +++ b/parquet/tests/encryption/encryption.rs @@ -26,7 +26,7 @@ use parquet::data_type::{ByteArray, ByteArrayType}; use parquet::encryption::decrypt::FileDecryptionProperties; use parquet::encryption::encrypt::FileEncryptionProperties; use parquet::errors::ParquetError; -use parquet::file::metadata::{ColumnChunkMetaData, PageIndexPolicy, ParquetMetaData}; +use parquet::file::metadata::{ColumnChunkMetaData, KeyValue, PageIndexPolicy, ParquetMetaData}; use parquet::file::properties::WriterProperties; use parquet::file::writer::SerializedFileWriter; use parquet::schema::parser::parse_message_type; @@ -119,6 +119,59 @@ fn test_non_uniform_encryption_plaintext_footer() { non_uniform_encryption_plaintext_footer(AES_256_FOOTER_KEY, AES_256_COLUMN_NAME_KEYS); } +#[test] +fn test_set_key_value_metadata_preserves_encrypted_reads() { + for filename in [ + "encrypt_columns_and_footer.parquet.encrypted", + "encrypt_columns_plaintext_footer.parquet.encrypted", + ] { + let path = encryption_util::encrypted_data_path(AES_128_FOOTER_KEY, filename); + let file = File::open(path).unwrap(); + let mut decryption = FileDecryptionProperties::builder(AES_128_FOOTER_KEY.to_vec()); + for (name, key) in AES_128_COLUMN_NAME_KEYS { + decryption = decryption.with_column_key(name, key.to_vec()); + } + let options = ArrowReaderOptions::new() + .with_file_decryption_properties(decryption.build().unwrap()) + .with_page_index_policy(PageIndexPolicy::Required); + let loaded = ArrowReaderMetadata::load(&file, options).unwrap(); + let cached = loaded.metadata(); + let original_values = cached.file_metadata().key_value_metadata().cloned(); + let values = vec![KeyValue::new( + "application.note".to_string(), + "updated".to_string(), + )]; + let updated = Arc::new( + cached + .as_ref() + .clone() + .into_builder() + .set_key_value_metadata(Some(values.clone())) + .build(), + ); + assert_eq!(updated.file_metadata().key_value_metadata(), Some(&values)); + assert_eq!( + cached.file_metadata().key_value_metadata(), + original_values.as_ref() + ); + assert!(Arc::ptr_eq( + cached.page_index().unwrap(), + updated.page_index().unwrap() + )); + verify_column_indexes(&updated); + + // The rebuilt metadata must retain the decryptor without supplying the keys again. + let metadata = + ArrowReaderMetadata::try_new(updated.clone(), ArrowReaderOptions::new()).unwrap(); + let batches = ParquetRecordBatchReaderBuilder::new_with_metadata(file, metadata) + .build() + .unwrap() + .collect::, _>>() + .unwrap(); + encryption_util::verify_encryption_test_data(batches, &updated); + } +} + #[test] fn test_plaintext_footer_signature_verification() { fn plaintext_footer_signature_verification(footer_key: &[u8], column_keys: &[(&str, &[u8])]) {