|
| 1 | +/* |
| 2 | + * Licensed to the Apache Software Foundation (ASF) under one |
| 3 | + * or more contributor license agreements. See the NOTICE file |
| 4 | + * distributed with this work for additional information |
| 5 | + * regarding copyright ownership. The ASF licenses this file |
| 6 | + * to you under the Apache License, Version 2.0 (the |
| 7 | + * "License"); you may not use this file except in compliance |
| 8 | + * with the License. You may obtain a copy of the License at |
| 9 | + * |
| 10 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | + * |
| 12 | + * Unless required by applicable law or agreed to in writing, |
| 13 | + * software distributed under the License is distributed on an |
| 14 | + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 15 | + * KIND, either express or implied. See the License for the |
| 16 | + * specific language governing permissions and limitations |
| 17 | + * under the License. |
| 18 | + */ |
| 19 | + |
| 20 | +#include "iceberg/inspect/metadata_partition_util_internal.h" |
| 21 | + |
| 22 | +#include <algorithm> |
| 23 | +#include <cstdint> |
| 24 | +#include <functional> |
| 25 | +#include <map> |
| 26 | +#include <ranges> |
| 27 | +#include <string> |
| 28 | +#include <unordered_map> |
| 29 | +#include <unordered_set> |
| 30 | +#include <utility> |
| 31 | +#include <variant> |
| 32 | +#include <vector> |
| 33 | + |
| 34 | +#include <nanoarrow/nanoarrow.h> |
| 35 | + |
| 36 | +#include "iceberg/arrow_row_builder_internal.h" |
| 37 | +#include "iceberg/nanoarrow_status_internal.h" |
| 38 | +#include "iceberg/partition_spec.h" |
| 39 | +#include "iceberg/schema.h" |
| 40 | +#include "iceberg/schema_field.h" |
| 41 | +#include "iceberg/table_metadata.h" |
| 42 | +#include "iceberg/transform.h" |
| 43 | +#include "iceberg/type.h" |
| 44 | +#include "iceberg/util/checked_cast.h" |
| 45 | +#include "iceberg/util/macros.h" |
| 46 | + |
| 47 | +namespace iceberg::internal { |
| 48 | + |
| 49 | +Status AppendLiteral(ArrowArray* array, const Literal& literal) { |
| 50 | + if (literal.IsNull()) { |
| 51 | + return AppendNull(array); |
| 52 | + } |
| 53 | + if (literal.IsAboveMax() || literal.IsBelowMin()) { |
| 54 | + return InvalidArgument("Cannot append non-value partition literal {}", |
| 55 | + literal.ToString()); |
| 56 | + } |
| 57 | + |
| 58 | + switch (literal.type()->type_id()) { |
| 59 | + case TypeId::kBoolean: |
| 60 | + return AppendBoolean(array, std::get<bool>(literal.value())); |
| 61 | + case TypeId::kInt: |
| 62 | + case TypeId::kDate: |
| 63 | + return AppendInt(array, std::get<int32_t>(literal.value())); |
| 64 | + case TypeId::kLong: |
| 65 | + case TypeId::kTime: |
| 66 | + case TypeId::kTimestamp: |
| 67 | + case TypeId::kTimestampTz: |
| 68 | + case TypeId::kTimestampNs: |
| 69 | + case TypeId::kTimestampTzNs: |
| 70 | + return AppendInt(array, std::get<int64_t>(literal.value())); |
| 71 | + case TypeId::kFloat: |
| 72 | + return AppendDouble(array, std::get<float>(literal.value())); |
| 73 | + case TypeId::kDouble: |
| 74 | + return AppendDouble(array, std::get<double>(literal.value())); |
| 75 | + case TypeId::kString: |
| 76 | + return AppendString(array, std::get<std::string>(literal.value())); |
| 77 | + case TypeId::kBinary: |
| 78 | + case TypeId::kFixed: |
| 79 | + return AppendBytes(array, std::get<std::vector<uint8_t>>(literal.value())); |
| 80 | + case TypeId::kDecimal: |
| 81 | + return AppendDecimal(array, std::get<Decimal>(literal.value())); |
| 82 | + case TypeId::kUuid: |
| 83 | + return AppendBytes(array, std::get<Uuid>(literal.value()).bytes()); |
| 84 | + case TypeId::kUnknown: |
| 85 | + case TypeId::kStruct: |
| 86 | + case TypeId::kList: |
| 87 | + case TypeId::kMap: |
| 88 | + case TypeId::kVariant: |
| 89 | + case TypeId::kGeometry: |
| 90 | + case TypeId::kGeography: |
| 91 | + return NotSupported("Cannot append partition literal of type {}", |
| 92 | + literal.type()->ToString()); |
| 93 | + } |
| 94 | + std::unreachable(); |
| 95 | +} |
| 96 | + |
| 97 | +Result<std::shared_ptr<StructType>> UnifiedPartitionType(const TableMetadata& metadata) { |
| 98 | + ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); |
| 99 | + auto specs = metadata.partition_specs; |
| 100 | + for (const auto& spec : specs) { |
| 101 | + ICEBERG_PRECHECK(spec != nullptr, "Partition spec cannot be null"); |
| 102 | + } |
| 103 | + std::ranges::sort(specs, std::greater{}, &PartitionSpec::spec_id); |
| 104 | + |
| 105 | + std::unordered_set<int32_t> active_field_ids; |
| 106 | + for (const auto& spec : specs) { |
| 107 | + for (const auto& field : spec->fields()) { |
| 108 | + ICEBERG_PRECHECK(field.transform() != nullptr, |
| 109 | + "Partition field {} has a null transform", field.field_id()); |
| 110 | + ICEBERG_CHECK(field.transform()->transform_type() != TransformType::kUnknown, |
| 111 | + "Cannot build table partition type with unknown transform '{}'", |
| 112 | + field.transform()->ToString()); |
| 113 | + ICEBERG_ASSIGN_OR_RAISE(auto source_field, |
| 114 | + schema->FindFieldById(field.source_id())); |
| 115 | + if (source_field.has_value()) { |
| 116 | + active_field_ids.insert(field.field_id()); |
| 117 | + } |
| 118 | + } |
| 119 | + } |
| 120 | + |
| 121 | + struct ProjectedField { |
| 122 | + const PartitionField* definition; |
| 123 | + std::string name; |
| 124 | + std::shared_ptr<Type> type; |
| 125 | + }; |
| 126 | + std::map<int32_t, ProjectedField> fields_by_id; |
| 127 | + for (const auto& spec : specs) { |
| 128 | + ICEBERG_ASSIGN_OR_RAISE(auto spec_type, spec->PartitionType(*schema)); |
| 129 | + ICEBERG_PRECHECK(spec_type->fields().size() == spec->fields().size(), |
| 130 | + "Partition spec {} has mismatched field and type counts", |
| 131 | + spec->spec_id()); |
| 132 | + for (size_t index = 0; index < spec->fields().size(); ++index) { |
| 133 | + const auto& partition_field = spec->fields()[index]; |
| 134 | + if (!active_field_ids.contains(partition_field.field_id())) { |
| 135 | + continue; |
| 136 | + } |
| 137 | + |
| 138 | + const auto& spec_field = spec_type->fields()[index]; |
| 139 | + auto [iter, inserted] = |
| 140 | + fields_by_id.try_emplace(partition_field.field_id(), |
| 141 | + ProjectedField{.definition = &partition_field, |
| 142 | + .name = std::string(spec_field.name()), |
| 143 | + .type = spec_field.type()}); |
| 144 | + if (!inserted) { |
| 145 | + const auto& existing = *iter->second.definition; |
| 146 | + const auto current_transform = partition_field.transform()->transform_type(); |
| 147 | + const auto existing_transform = existing.transform()->transform_type(); |
| 148 | + const bool compatible_transform = |
| 149 | + *partition_field.transform() == *existing.transform() || |
| 150 | + current_transform == TransformType::kVoid || |
| 151 | + existing_transform == TransformType::kVoid; |
| 152 | + ICEBERG_CHECK( |
| 153 | + partition_field.source_id() == existing.source_id() && compatible_transform, |
| 154 | + "Conflicting partition fields with ID {}: '{}' and '{}'", |
| 155 | + partition_field.field_id(), partition_field.ToString(), existing.ToString()); |
| 156 | + |
| 157 | + if (existing_transform == TransformType::kVoid && |
| 158 | + current_transform != TransformType::kVoid) { |
| 159 | + iter->second.definition = &partition_field; |
| 160 | + iter->second.type = spec_field.type(); |
| 161 | + } |
| 162 | + } |
| 163 | + } |
| 164 | + } |
| 165 | + |
| 166 | + std::vector<SchemaField> fields; |
| 167 | + fields.reserve(fields_by_id.size()); |
| 168 | + for (auto& [_, field] : fields_by_id) { |
| 169 | + fields.emplace_back(field.definition->field_id(), std::move(field.name), |
| 170 | + std::move(field.type), /*optional=*/true); |
| 171 | + } |
| 172 | + return std::make_shared<StructType>(std::move(fields)); |
| 173 | +} |
| 174 | + |
| 175 | +Result<PartitionValues> ProjectPartitionValues(const StructType& partition_type, |
| 176 | + const PartitionSpec& spec, |
| 177 | + const PartitionValues& values) { |
| 178 | + ICEBERG_PRECHECK(values.num_fields() == spec.fields().size(), |
| 179 | + "Partition has {} values but spec {} has {} fields", |
| 180 | + values.num_fields(), spec.spec_id(), spec.fields().size()); |
| 181 | + |
| 182 | + std::unordered_map<int32_t, size_t> positions; |
| 183 | + positions.reserve(spec.fields().size()); |
| 184 | + for (size_t index = 0; index < spec.fields().size(); ++index) { |
| 185 | + positions.emplace(spec.fields()[index].field_id(), index); |
| 186 | + } |
| 187 | + |
| 188 | + std::vector<Literal> projected; |
| 189 | + projected.reserve(partition_type.fields().size()); |
| 190 | + for (const auto& field : partition_type.fields()) { |
| 191 | + auto target_type = checked_pointer_cast<PrimitiveType>(field.type()); |
| 192 | + auto position = positions.find(field.field_id()); |
| 193 | + if (position == positions.end()) { |
| 194 | + projected.push_back(Literal::Null(std::move(target_type))); |
| 195 | + continue; |
| 196 | + } |
| 197 | + ICEBERG_ASSIGN_OR_RAISE(auto value, values.ValueAt(position->second)); |
| 198 | + if (value.get().IsNull()) { |
| 199 | + projected.push_back(Literal::Null(std::move(target_type))); |
| 200 | + } else if (value.get().type()->type_id() == target_type->type_id()) { |
| 201 | + projected.push_back(value.get()); |
| 202 | + } else { |
| 203 | + ICEBERG_ASSIGN_OR_RAISE(auto coerced, value.get().CastTo(target_type)); |
| 204 | + projected.push_back(std::move(coerced)); |
| 205 | + } |
| 206 | + } |
| 207 | + return PartitionValues(std::move(projected)); |
| 208 | +} |
| 209 | + |
| 210 | +Status AppendPartitionValues(ArrowArray* array, const StructType& partition_type, |
| 211 | + const PartitionValues& values) { |
| 212 | + ICEBERG_PRECHECK(array != nullptr, "Partition Arrow array cannot be null"); |
| 213 | + ICEBERG_PRECHECK(std::cmp_equal(array->n_children, partition_type.fields().size()), |
| 214 | + "Partition Arrow array has {} fields but schema has {}", |
| 215 | + array->n_children, partition_type.fields().size()); |
| 216 | + ICEBERG_PRECHECK(values.num_fields() == partition_type.fields().size(), |
| 217 | + "Partition has {} values but schema has {} fields", |
| 218 | + values.num_fields(), partition_type.fields().size()); |
| 219 | + |
| 220 | + for (size_t index = 0; index < values.num_fields(); ++index) { |
| 221 | + ICEBERG_ASSIGN_OR_RAISE(auto value, values.ValueAt(index)); |
| 222 | + ICEBERG_RETURN_UNEXPECTED(AppendLiteral(array->children[index], value.get())); |
| 223 | + } |
| 224 | + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array)); |
| 225 | + return {}; |
| 226 | +} |
| 227 | + |
| 228 | +} // namespace iceberg::internal |
0 commit comments