From 74563874f0126651f35b290981e503d14410c401 Mon Sep 17 00:00:00 2001 From: Gang Wu Date: Tue, 17 Nov 2026 11:22:17 +0800 Subject: [PATCH 1/2] fix: return null row lineage when first_row_id is unassigned --- src/iceberg/arrow/metadata_column_util.cc | 3 ++- src/iceberg/avro/avro_data_util.cc | 9 +++++---- src/iceberg/avro/avro_direct_decoder.cc | 9 +++++---- src/iceberg/parquet/parquet_data_util.cc | 7 ++++++- src/iceberg/test/avro_test.cc | 8 +++++++- src/iceberg/test/parquet_data_test.cc | 15 ++++++++------- src/iceberg/test/parquet_test.cc | 6 +++--- 7 files changed, 36 insertions(+), 21 deletions(-) diff --git a/src/iceberg/arrow/metadata_column_util.cc b/src/iceberg/arrow/metadata_column_util.cc index 0009bf37bf..266ddde923 100644 --- a/src/iceberg/arrow/metadata_column_util.cc +++ b/src/iceberg/arrow/metadata_column_util.cc @@ -35,7 +35,8 @@ bool CanInheritRowLineageValue(int32_t field_id, return metadata_context.first_row_id.has_value(); } if (field_id == MetadataColumns::kLastUpdatedSequenceNumberColumnId) { - return metadata_context.data_sequence_number.has_value(); + return metadata_context.first_row_id.has_value() && + metadata_context.data_sequence_number.has_value(); } return false; } diff --git a/src/iceberg/avro/avro_data_util.cc b/src/iceberg/avro/avro_data_util.cc index 39f2948b9f..27db9f814b 100644 --- a/src/iceberg/avro/avro_data_util.cc +++ b/src/iceberg/avro/avro_data_util.cc @@ -471,7 +471,11 @@ Status AppendFieldToBuilder(const ::avro::NodePtr& avro_node, const SchemaField& projected_field, const arrow::MetadataColumnContext& metadata_context, ::arrow::ArrayBuilder* array_builder) { - if (projection.kind == FieldProjection::Kind::kNull) { + const bool is_row_lineage = + MetadataColumns::IsRowLineageColumn(projected_field.field_id()); + + if (projection.kind == FieldProjection::Kind::kNull || + (is_row_lineage && !metadata_context.first_row_id.has_value())) { ICEBERG_ARROW_RETURN_NOT_OK(array_builder->AppendNull()); return {}; } @@ -480,9 +484,6 @@ Status AppendFieldToBuilder(const ::avro::NodePtr& avro_node, return AppendDefaultToBuilder(projection, array_builder); } - const bool is_row_lineage = - MetadataColumns::IsRowLineageColumn(projected_field.field_id()); - if (avro_node->type() == ::avro::AVRO_UNION) { size_t branch = avro_datum.unionBranch(); if (avro_node->leafAt(branch)->type() == ::avro::AVRO_NULL) { diff --git a/src/iceberg/avro/avro_direct_decoder.cc b/src/iceberg/avro/avro_direct_decoder.cc index 465bc4b240..cc4c301317 100644 --- a/src/iceberg/avro/avro_direct_decoder.cc +++ b/src/iceberg/avro/avro_direct_decoder.cc @@ -602,15 +602,16 @@ Status DecodeFieldToBuilder(const ::avro::NodePtr& avro_node, ::avro::Decoder& d const SchemaField& projected_field, const arrow::MetadataColumnContext& metadata_context, ::arrow::ArrayBuilder* array_builder, DecodeContext& ctx) { - if (projection.kind == FieldProjection::Kind::kNull) { + const bool is_row_lineage = + MetadataColumns::IsRowLineageColumn(projected_field.field_id()); + + if (projection.kind == FieldProjection::Kind::kNull || + (is_row_lineage && !metadata_context.first_row_id.has_value())) { ICEBERG_RETURN_UNEXPECTED(SkipAvroValue(avro_node, decoder)); ICEBERG_ARROW_RETURN_NOT_OK(array_builder->AppendNull()); return {}; } - const bool is_row_lineage = - MetadataColumns::IsRowLineageColumn(projected_field.field_id()); - if (avro_node->type() == ::avro::AVRO_UNION) { const size_t branch_index = decoder.decodeUnionIndex(); diff --git a/src/iceberg/parquet/parquet_data_util.cc b/src/iceberg/parquet/parquet_data_util.cc index 73597cbfb8..c149252526 100644 --- a/src/iceberg/parquet/parquet_data_util.cc +++ b/src/iceberg/parquet/parquet_data_util.cc @@ -222,7 +222,12 @@ Result> ProjectStructArray( std::shared_ptr<::arrow::Array> projected_array; - if (field_projection.kind == FieldProjection::Kind::kProjected) { + if (MetadataColumns::IsRowLineageColumn(projected_field.field_id()) && + !metadata_context.first_row_id.has_value()) { + ICEBERG_ASSIGN_OR_RAISE( + projected_array, + MakeNullArray(output_arrow_type, struct_array->length(), pool)); + } else if (field_projection.kind == FieldProjection::Kind::kProjected) { auto parquet_field_index = static_cast(std::get(field_projection.from)); if (parquet_field_index >= struct_array->num_fields()) { diff --git a/src/iceberg/test/avro_test.cc b/src/iceberg/test/avro_test.cc index a997592dd5..8e2070fa68 100644 --- a/src/iceberg/test/avro_test.cc +++ b/src/iceberg/test/avro_test.cc @@ -953,7 +953,8 @@ TEST_P(AvroReaderParameterizedTest, ReadPartialLineage) { ICEBERG_UNWRAP_OR_FAIL(auto reader_with_sequence, OpenRowLineageReader(schema, std::nullopt, 7L)); - VerifyNextBatch(*reader_with_sequence, R"([[1, null, 7], [2, null, 7], [3, null, 7]])"); + VerifyNextBatch(*reader_with_sequence, + R"([[1, null, null], [2, null, null], [3, null, null]])"); } TEST_P(AvroReaderParameterizedTest, ReadPhysicalLineage) { @@ -964,6 +965,11 @@ TEST_P(AvroReaderParameterizedTest, ReadPhysicalLineage) { ICEBERG_UNWRAP_OR_FAIL(auto reader, OpenRowLineageReader(schema, 100L)); VerifyNextBatch(*reader, R"([[1, 100, null], [2, 777, 8], [3, 102, null]])"); + + ICEBERG_UNWRAP_OR_FAIL(auto reader_without_row_id, + OpenRowLineageReader(schema, std::nullopt, 7L)); + VerifyNextBatch(*reader_without_row_id, + R"([[1, null, null], [2, null, null], [3, null, null]])"); } TEST_P(AvroReaderParameterizedTest, SplitWithRowPositionNotSupported) { diff --git a/src/iceberg/test/parquet_data_test.cc b/src/iceberg/test/parquet_data_test.cc index 1ec4d80655..11bc36ced0 100644 --- a/src/iceberg/test/parquet_data_test.cc +++ b/src/iceberg/test/parquet_data_test.cc @@ -193,15 +193,16 @@ TEST(ProjectRecordBatchTest, LineageWithoutRowPositions) { Schema lineage_schema({SchemaField::MakeRequired(1, "id", int32()), MetadataColumns::kRowId, MetadataColumns::kLastUpdatedSequenceNumber}); - ASSERT_NO_FATAL_FAILURE(VerifyProjectRecordBatch( - lineage_schema, data_schema, "[[10], [20], [30]]", - "[[10, null, 7], [20, null, 7], [30, null, 7]]", {.data_sequence_number = 7})); + const std::string expected = "[[10, null, null], [20, null, null], [30, null, null]]"; + ASSERT_NO_FATAL_FAILURE(VerifyProjectRecordBatch(lineage_schema, data_schema, + "[[10], [20], [30]]", expected, + {.data_sequence_number = 7})); const std::string physical_values = "[[10, null, null], [20, 777, 8], [30, null, 9]]"; - ASSERT_NO_FATAL_FAILURE(VerifyProjectRecordBatch( - lineage_schema, lineage_schema, physical_values, - "[[10, null, 7], [20, 777, 8], [30, null, 9]]", {.data_sequence_number = 7})); ASSERT_NO_FATAL_FAILURE(VerifyProjectRecordBatch(lineage_schema, lineage_schema, - physical_values, physical_values)); + physical_values, expected, + {.data_sequence_number = 7})); + ASSERT_NO_FATAL_FAILURE(VerifyProjectRecordBatch(lineage_schema, lineage_schema, + physical_values, expected)); } TEST(ProjectRecordBatchTest, RowPositionsAreIndependentOfPhysicalColumns) { diff --git a/src/iceberg/test/parquet_test.cc b/src/iceberg/test/parquet_test.cc index 6b0604a541..5ef42c8eea 100644 --- a/src/iceberg/test/parquet_test.cc +++ b/src/iceberg/test/parquet_test.cc @@ -1597,9 +1597,9 @@ TEST_F(ParquetReaderTest, ReadPartialLineage) { OpenRowLineageReader(schema, std::nullopt, 7L)); ASSERT_NO_FATAL_FAILURE(VerifyNextBatch(*reader_with_sequence, - R"([[1, null, 7], - [2, null, 7], - [3, null, 7]])")); + R"([[1, null, null], + [2, null, null], + [3, null, null]])")); } TEST_F(ParquetReaderTest, ReadPhysicalLineage) { From ec59a6bc110499ada61a6f5ccc2f82c899d2ed39 Mon Sep 17 00:00:00 2001 From: Gang Wu Date: Tue, 17 Nov 2026 12:55:56 +0800 Subject: [PATCH 2/2] address feedback --- src/iceberg/arrow/metadata_column_util.cc | 6 ++++++ src/iceberg/arrow/metadata_column_util_internal.h | 4 ++++ src/iceberg/avro/avro_data_util.cc | 8 ++++---- src/iceberg/avro/avro_direct_decoder.cc | 8 ++++---- src/iceberg/parquet/parquet_data_util.cc | 8 ++------ src/iceberg/test/avro_test.cc | 8 ++++++-- src/iceberg/test/parquet_test.cc | 10 ++++++++-- 7 files changed, 34 insertions(+), 18 deletions(-) diff --git a/src/iceberg/arrow/metadata_column_util.cc b/src/iceberg/arrow/metadata_column_util.cc index 266ddde923..3deedfcecb 100644 --- a/src/iceberg/arrow/metadata_column_util.cc +++ b/src/iceberg/arrow/metadata_column_util.cc @@ -29,6 +29,12 @@ namespace iceberg::arrow { +bool ShouldReadRowLineageAsNull(int32_t field_id, + const MetadataColumnContext& metadata_context) { + return MetadataColumns::IsRowLineageColumn(field_id) && + !metadata_context.first_row_id.has_value(); +} + bool CanInheritRowLineageValue(int32_t field_id, const MetadataColumnContext& metadata_context) { if (field_id == MetadataColumns::kRowIdColumnId) { diff --git a/src/iceberg/arrow/metadata_column_util_internal.h b/src/iceberg/arrow/metadata_column_util_internal.h index a08ea9bfb1..a2640db60a 100644 --- a/src/iceberg/arrow/metadata_column_util_internal.h +++ b/src/iceberg/arrow/metadata_column_util_internal.h @@ -46,6 +46,10 @@ struct MetadataColumnContext { std::optional data_sequence_number; // Data sequence number for lineage. }; +/// \brief Return whether a row lineage column must be read as null. +bool ShouldReadRowLineageAsNull(int32_t field_id, + const MetadataColumnContext& metadata_context); + /// \brief Check if row lineage metadata can be inherited for the column. bool CanInheritRowLineageValue(int32_t field_id, const MetadataColumnContext& metadata_context); diff --git a/src/iceberg/avro/avro_data_util.cc b/src/iceberg/avro/avro_data_util.cc index 27db9f814b..5c6c19decf 100644 --- a/src/iceberg/avro/avro_data_util.cc +++ b/src/iceberg/avro/avro_data_util.cc @@ -471,11 +471,8 @@ Status AppendFieldToBuilder(const ::avro::NodePtr& avro_node, const SchemaField& projected_field, const arrow::MetadataColumnContext& metadata_context, ::arrow::ArrayBuilder* array_builder) { - const bool is_row_lineage = - MetadataColumns::IsRowLineageColumn(projected_field.field_id()); - if (projection.kind == FieldProjection::Kind::kNull || - (is_row_lineage && !metadata_context.first_row_id.has_value())) { + arrow::ShouldReadRowLineageAsNull(projected_field.field_id(), metadata_context)) { ICEBERG_ARROW_RETURN_NOT_OK(array_builder->AppendNull()); return {}; } @@ -484,6 +481,9 @@ Status AppendFieldToBuilder(const ::avro::NodePtr& avro_node, return AppendDefaultToBuilder(projection, array_builder); } + const bool is_row_lineage = + MetadataColumns::IsRowLineageColumn(projected_field.field_id()); + if (avro_node->type() == ::avro::AVRO_UNION) { size_t branch = avro_datum.unionBranch(); if (avro_node->leafAt(branch)->type() == ::avro::AVRO_NULL) { diff --git a/src/iceberg/avro/avro_direct_decoder.cc b/src/iceberg/avro/avro_direct_decoder.cc index cc4c301317..c2d0a3c9c3 100644 --- a/src/iceberg/avro/avro_direct_decoder.cc +++ b/src/iceberg/avro/avro_direct_decoder.cc @@ -602,16 +602,16 @@ Status DecodeFieldToBuilder(const ::avro::NodePtr& avro_node, ::avro::Decoder& d const SchemaField& projected_field, const arrow::MetadataColumnContext& metadata_context, ::arrow::ArrayBuilder* array_builder, DecodeContext& ctx) { - const bool is_row_lineage = - MetadataColumns::IsRowLineageColumn(projected_field.field_id()); - if (projection.kind == FieldProjection::Kind::kNull || - (is_row_lineage && !metadata_context.first_row_id.has_value())) { + arrow::ShouldReadRowLineageAsNull(projected_field.field_id(), metadata_context)) { ICEBERG_RETURN_UNEXPECTED(SkipAvroValue(avro_node, decoder)); ICEBERG_ARROW_RETURN_NOT_OK(array_builder->AppendNull()); return {}; } + const bool is_row_lineage = + MetadataColumns::IsRowLineageColumn(projected_field.field_id()); + if (avro_node->type() == ::avro::AVRO_UNION) { const size_t branch_index = decoder.decodeUnionIndex(); diff --git a/src/iceberg/parquet/parquet_data_util.cc b/src/iceberg/parquet/parquet_data_util.cc index c149252526..b5024c4867 100644 --- a/src/iceberg/parquet/parquet_data_util.cc +++ b/src/iceberg/parquet/parquet_data_util.cc @@ -222,8 +222,8 @@ Result> ProjectStructArray( std::shared_ptr<::arrow::Array> projected_array; - if (MetadataColumns::IsRowLineageColumn(projected_field.field_id()) && - !metadata_context.first_row_id.has_value()) { + if (field_projection.kind == FieldProjection::Kind::kNull || + arrow::ShouldReadRowLineageAsNull(projected_field.field_id(), metadata_context)) { ICEBERG_ASSIGN_OR_RAISE( projected_array, MakeNullArray(output_arrow_type, struct_array->length(), pool)); @@ -252,10 +252,6 @@ Result> ProjectStructArray( projected_array, ProjectPrimitiveArray(parquet_array, output_arrow_type, pool)); } - } else if (field_projection.kind == FieldProjection::Kind::kNull) { - ICEBERG_ASSIGN_OR_RAISE( - projected_array, - MakeNullArray(output_arrow_type, struct_array->length(), pool)); } else if (field_projection.kind == FieldProjection::Kind::kDefault) { ICEBERG_ASSIGN_OR_RAISE( projected_array, diff --git a/src/iceberg/test/avro_test.cc b/src/iceberg/test/avro_test.cc index 8e2070fa68..71062375f7 100644 --- a/src/iceberg/test/avro_test.cc +++ b/src/iceberg/test/avro_test.cc @@ -179,7 +179,7 @@ class AvroReaderTest : public TempFileTestBase { ::arrow::json::ArrayFromJSONString(::arrow::struct_(arrow_schema->fields()), R"([[1, null, null], [2, 777, 8], - [3, null, null]])") + [3, 778, null]])") .ValueOrDie(); struct ArrowArray arrow_array; @@ -964,7 +964,11 @@ TEST_P(AvroReaderParameterizedTest, ReadPhysicalLineage) { auto schema = RowLineageSchema(); ICEBERG_UNWRAP_OR_FAIL(auto reader, OpenRowLineageReader(schema, 100L)); - VerifyNextBatch(*reader, R"([[1, 100, null], [2, 777, 8], [3, 102, null]])"); + VerifyNextBatch(*reader, R"([[1, 100, null], [2, 777, 8], [3, 778, null]])"); + + ICEBERG_UNWRAP_OR_FAIL(auto reader_with_sequence, + OpenRowLineageReader(schema, 100L, 7L)); + VerifyNextBatch(*reader_with_sequence, R"([[1, 100, 7], [2, 777, 8], [3, 778, 7]])"); ICEBERG_UNWRAP_OR_FAIL(auto reader_without_row_id, OpenRowLineageReader(schema, std::nullopt, 7L)); diff --git a/src/iceberg/test/parquet_test.cc b/src/iceberg/test/parquet_test.cc index 5ef42c8eea..e7642dbe0f 100644 --- a/src/iceberg/test/parquet_test.cc +++ b/src/iceberg/test/parquet_test.cc @@ -319,7 +319,7 @@ class ParquetReaderTest : public TempFileTestBase { ::arrow::json::ArrayFromJSONString(::arrow::struct_(arrow_schema->fields()), R"([[1, null, null], [2, 777, 8], - [3, null, null]])") + [3, 778, null]])") .ValueOrDie(); WriterProperties writer_properties; @@ -1611,7 +1611,13 @@ TEST_F(ParquetReaderTest, ReadPhysicalLineage) { ASSERT_NO_FATAL_FAILURE(VerifyNextBatch(*reader, R"([[1, 100, null], [2, 777, 8], - [3, 102, null]])")); + [3, 778, null]])")); + + ICEBERG_UNWRAP_OR_FAIL(auto reader_with_sequence, + OpenRowLineageReader(schema, 100L, 7L)); + ASSERT_NO_FATAL_FAILURE(VerifyNextBatch(*reader_with_sequence, R"([[1, 100, 7], + [2, 777, 8], + [3, 778, 7]])")); } TEST_F(ParquetReaderTest, ReadPositionsAndLineageAcrossRowGroups) {