Skip to content

Commit 75d594e

Browse files
manuzhangcodex
andcommitted
feat(inspect): unify and project metadata partition values
Extract a focused piece from the metadata table implementation with its tests. Co-authored-by: Codex <codex@openai.com>
1 parent 1957376 commit 75d594e

5 files changed

Lines changed: 386 additions & 0 deletions

File tree

‎src/iceberg/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ set(ICEBERG_SOURCES
5858
geospatial.cc
5959
incremental_append_scan.cc
6060
incremental_changelog_scan.cc
61+
inspect/metadata_partition_util_internal.cc
6162
inspect/history_table.cc
6263
inspect/metadata_scan.cc
6364
inspect/metadata_table.cc
Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,228 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#include "iceberg/inspect/metadata_partition_util_internal.h"
21+
22+
#include <algorithm>
23+
#include <cstdint>
24+
#include <functional>
25+
#include <map>
26+
#include <ranges>
27+
#include <string>
28+
#include <unordered_map>
29+
#include <unordered_set>
30+
#include <utility>
31+
#include <variant>
32+
#include <vector>
33+
34+
#include <nanoarrow/nanoarrow.h>
35+
36+
#include "iceberg/arrow_row_builder_internal.h"
37+
#include "iceberg/nanoarrow_status_internal.h"
38+
#include "iceberg/partition_spec.h"
39+
#include "iceberg/schema.h"
40+
#include "iceberg/schema_field.h"
41+
#include "iceberg/table_metadata.h"
42+
#include "iceberg/transform.h"
43+
#include "iceberg/type.h"
44+
#include "iceberg/util/checked_cast.h"
45+
#include "iceberg/util/macros.h"
46+
47+
namespace iceberg::internal {
48+
49+
Status AppendLiteral(ArrowArray* array, const Literal& literal) {
50+
if (literal.IsNull()) {
51+
return AppendNull(array);
52+
}
53+
if (literal.IsAboveMax() || literal.IsBelowMin()) {
54+
return InvalidArgument("Cannot append non-value partition literal {}",
55+
literal.ToString());
56+
}
57+
58+
switch (literal.type()->type_id()) {
59+
case TypeId::kBoolean:
60+
return AppendBoolean(array, std::get<bool>(literal.value()));
61+
case TypeId::kInt:
62+
case TypeId::kDate:
63+
return AppendInt(array, std::get<int32_t>(literal.value()));
64+
case TypeId::kLong:
65+
case TypeId::kTime:
66+
case TypeId::kTimestamp:
67+
case TypeId::kTimestampTz:
68+
case TypeId::kTimestampNs:
69+
case TypeId::kTimestampTzNs:
70+
return AppendInt(array, std::get<int64_t>(literal.value()));
71+
case TypeId::kFloat:
72+
return AppendDouble(array, std::get<float>(literal.value()));
73+
case TypeId::kDouble:
74+
return AppendDouble(array, std::get<double>(literal.value()));
75+
case TypeId::kString:
76+
return AppendString(array, std::get<std::string>(literal.value()));
77+
case TypeId::kBinary:
78+
case TypeId::kFixed:
79+
return AppendBytes(array, std::get<std::vector<uint8_t>>(literal.value()));
80+
case TypeId::kDecimal:
81+
return AppendDecimal(array, std::get<Decimal>(literal.value()));
82+
case TypeId::kUuid:
83+
return AppendBytes(array, std::get<Uuid>(literal.value()).bytes());
84+
case TypeId::kUnknown:
85+
case TypeId::kStruct:
86+
case TypeId::kList:
87+
case TypeId::kMap:
88+
case TypeId::kVariant:
89+
case TypeId::kGeometry:
90+
case TypeId::kGeography:
91+
return NotSupported("Cannot append partition literal of type {}",
92+
literal.type()->ToString());
93+
}
94+
std::unreachable();
95+
}
96+
97+
Result<std::shared_ptr<StructType>> UnifiedPartitionType(const TableMetadata& metadata) {
98+
ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema());
99+
auto specs = metadata.partition_specs;
100+
for (const auto& spec : specs) {
101+
ICEBERG_PRECHECK(spec != nullptr, "Partition spec cannot be null");
102+
}
103+
std::ranges::sort(specs, std::greater{}, &PartitionSpec::spec_id);
104+
105+
std::unordered_set<int32_t> active_field_ids;
106+
for (const auto& spec : specs) {
107+
for (const auto& field : spec->fields()) {
108+
ICEBERG_PRECHECK(field.transform() != nullptr,
109+
"Partition field {} has a null transform", field.field_id());
110+
ICEBERG_CHECK(field.transform()->transform_type() != TransformType::kUnknown,
111+
"Cannot build table partition type with unknown transform '{}'",
112+
field.transform()->ToString());
113+
ICEBERG_ASSIGN_OR_RAISE(auto source_field,
114+
schema->FindFieldById(field.source_id()));
115+
if (source_field.has_value()) {
116+
active_field_ids.insert(field.field_id());
117+
}
118+
}
119+
}
120+
121+
struct ProjectedField {
122+
const PartitionField* definition;
123+
std::string name;
124+
std::shared_ptr<Type> type;
125+
};
126+
std::map<int32_t, ProjectedField> fields_by_id;
127+
for (const auto& spec : specs) {
128+
ICEBERG_ASSIGN_OR_RAISE(auto spec_type, spec->PartitionType(*schema));
129+
ICEBERG_PRECHECK(spec_type->fields().size() == spec->fields().size(),
130+
"Partition spec {} has mismatched field and type counts",
131+
spec->spec_id());
132+
for (size_t index = 0; index < spec->fields().size(); ++index) {
133+
const auto& partition_field = spec->fields()[index];
134+
if (!active_field_ids.contains(partition_field.field_id())) {
135+
continue;
136+
}
137+
138+
const auto& spec_field = spec_type->fields()[index];
139+
auto [iter, inserted] =
140+
fields_by_id.try_emplace(partition_field.field_id(),
141+
ProjectedField{.definition = &partition_field,
142+
.name = std::string(spec_field.name()),
143+
.type = spec_field.type()});
144+
if (!inserted) {
145+
const auto& existing = *iter->second.definition;
146+
const auto current_transform = partition_field.transform()->transform_type();
147+
const auto existing_transform = existing.transform()->transform_type();
148+
const bool compatible_transform =
149+
*partition_field.transform() == *existing.transform() ||
150+
current_transform == TransformType::kVoid ||
151+
existing_transform == TransformType::kVoid;
152+
ICEBERG_CHECK(
153+
partition_field.source_id() == existing.source_id() && compatible_transform,
154+
"Conflicting partition fields with ID {}: '{}' and '{}'",
155+
partition_field.field_id(), partition_field.ToString(), existing.ToString());
156+
157+
if (existing_transform == TransformType::kVoid &&
158+
current_transform != TransformType::kVoid) {
159+
iter->second.definition = &partition_field;
160+
iter->second.type = spec_field.type();
161+
}
162+
}
163+
}
164+
}
165+
166+
std::vector<SchemaField> fields;
167+
fields.reserve(fields_by_id.size());
168+
for (auto& [_, field] : fields_by_id) {
169+
fields.emplace_back(field.definition->field_id(), std::move(field.name),
170+
std::move(field.type), /*optional=*/true);
171+
}
172+
return std::make_shared<StructType>(std::move(fields));
173+
}
174+
175+
Result<PartitionValues> ProjectPartitionValues(const StructType& partition_type,
176+
const PartitionSpec& spec,
177+
const PartitionValues& values) {
178+
ICEBERG_PRECHECK(values.num_fields() == spec.fields().size(),
179+
"Partition has {} values but spec {} has {} fields",
180+
values.num_fields(), spec.spec_id(), spec.fields().size());
181+
182+
std::unordered_map<int32_t, size_t> positions;
183+
positions.reserve(spec.fields().size());
184+
for (size_t index = 0; index < spec.fields().size(); ++index) {
185+
positions.emplace(spec.fields()[index].field_id(), index);
186+
}
187+
188+
std::vector<Literal> projected;
189+
projected.reserve(partition_type.fields().size());
190+
for (const auto& field : partition_type.fields()) {
191+
auto target_type = checked_pointer_cast<PrimitiveType>(field.type());
192+
auto position = positions.find(field.field_id());
193+
if (position == positions.end()) {
194+
projected.push_back(Literal::Null(std::move(target_type)));
195+
continue;
196+
}
197+
ICEBERG_ASSIGN_OR_RAISE(auto value, values.ValueAt(position->second));
198+
if (value.get().IsNull()) {
199+
projected.push_back(Literal::Null(std::move(target_type)));
200+
} else if (value.get().type()->type_id() == target_type->type_id()) {
201+
projected.push_back(value.get());
202+
} else {
203+
ICEBERG_ASSIGN_OR_RAISE(auto coerced, value.get().CastTo(target_type));
204+
projected.push_back(std::move(coerced));
205+
}
206+
}
207+
return PartitionValues(std::move(projected));
208+
}
209+
210+
Status AppendPartitionValues(ArrowArray* array, const StructType& partition_type,
211+
const PartitionValues& values) {
212+
ICEBERG_PRECHECK(array != nullptr, "Partition Arrow array cannot be null");
213+
ICEBERG_PRECHECK(std::cmp_equal(array->n_children, partition_type.fields().size()),
214+
"Partition Arrow array has {} fields but schema has {}",
215+
array->n_children, partition_type.fields().size());
216+
ICEBERG_PRECHECK(values.num_fields() == partition_type.fields().size(),
217+
"Partition has {} values but schema has {} fields",
218+
values.num_fields(), partition_type.fields().size());
219+
220+
for (size_t index = 0; index < values.num_fields(); ++index) {
221+
ICEBERG_ASSIGN_OR_RAISE(auto value, values.ValueAt(index));
222+
ICEBERG_RETURN_UNEXPECTED(AppendLiteral(array->children[index], value.get()));
223+
}
224+
ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array));
225+
return {};
226+
}
227+
228+
} // namespace iceberg::internal
Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#pragma once
21+
22+
#include <memory>
23+
24+
#include "iceberg/arrow_c_data.h"
25+
#include "iceberg/iceberg_export.h"
26+
#include "iceberg/result.h"
27+
#include "iceberg/row/partition_values.h"
28+
#include "iceberg/type_fwd.h"
29+
30+
namespace iceberg::internal {
31+
32+
/// \brief Build the Java-compatible union of active partition fields across all specs.
33+
ICEBERG_EXPORT Result<std::shared_ptr<StructType>> UnifiedPartitionType(
34+
const TableMetadata& metadata);
35+
36+
/// \brief Project values written with one spec into the table-wide partition type.
37+
ICEBERG_EXPORT Result<PartitionValues> ProjectPartitionValues(
38+
const StructType& partition_type, const PartitionSpec& spec,
39+
const PartitionValues& values);
40+
41+
/// \brief Append a primitive partition value or decoded metric bound.
42+
ICEBERG_EXPORT Status AppendLiteral(ArrowArray* array, const Literal& literal);
43+
44+
/// \brief Append partition values to an Arrow struct builder.
45+
ICEBERG_EXPORT Status AppendPartitionValues(ArrowArray* array,
46+
const StructType& partition_type,
47+
const PartitionValues& values);
48+
49+
} // namespace iceberg::internal

‎src/iceberg/test/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,7 @@ if(ICEBERG_BUILD_BUNDLE)
206206
add_iceberg_test(metadata_table_test
207207
USE_BUNDLE
208208
SOURCES
209+
inspect/metadata_partition_util_test.cc
209210
inspect/history_table_test.cc
210211
inspect/metadata_table_test.cc
211212
inspect/refs_table_test.cc

0 commit comments

Comments
 (0)