|
17 | 17 | * under the License. |
18 | 18 | */ |
19 | 19 |
|
| 20 | +#include <cerrno> |
20 | 21 | #include <cstdint> |
21 | 22 | #include <limits> |
22 | 23 | #include <memory> |
23 | 24 | #include <mutex> |
24 | 25 | #include <span> |
| 26 | +#include <string> |
| 27 | +#include <string_view> |
| 28 | +#include <type_traits> |
25 | 29 | #include <utility> |
| 30 | +#include <variant> |
26 | 31 | #include <vector> |
27 | 32 |
|
28 | 33 | #include <nanoarrow/nanoarrow.h> |
29 | 34 |
|
30 | 35 | #include "iceberg/arrow_c_data_guard_internal.h" |
31 | 36 | #include "iceberg/arrow_c_data_util_internal.h" |
| 37 | +#include "iceberg/expression/literal.h" |
32 | 38 | #include "iceberg/file_reader.h" |
33 | 39 | #include "iceberg/nanoarrow_status_internal.h" |
34 | 40 | #include "iceberg/result.h" |
35 | 41 | #include "iceberg/schema.h" |
36 | 42 | #include "iceberg/schema_field.h" |
37 | 43 | #include "iceberg/schema_internal.h" |
| 44 | +#include "iceberg/schema_util.h" |
| 45 | +#include "iceberg/type.h" |
| 46 | +#include "iceberg/util/checked_cast.h" |
| 47 | +#include "iceberg/util/decimal.h" |
38 | 48 | #include "iceberg/util/macros.h" |
| 49 | +#include "iceberg/util/uuid.h" |
39 | 50 |
|
40 | 51 | namespace iceberg { |
41 | 52 |
|
@@ -394,4 +405,172 @@ Result<ArrowArray> ProjectBatch(ArrowArray* input_batch, |
394 | 405 | projection.selected_field_indices()); |
395 | 406 | } |
396 | 407 |
|
| 408 | +namespace { |
| 409 | + |
| 410 | +int AppendWriteDefault(ArrowArray* array, const Literal& literal) { |
| 411 | + return std::visit( |
| 412 | + [&](const auto& value) -> int { |
| 413 | + using Value = std::decay_t<decltype(value)>; |
| 414 | + if constexpr (std::is_same_v<Value, std::monostate>) { |
| 415 | + return ArrowArrayAppendNull(array, 1); |
| 416 | + } else if constexpr (std::is_integral_v<Value>) { |
| 417 | + return ArrowArrayAppendInt(array, value); |
| 418 | + } else if constexpr (std::is_floating_point_v<Value>) { |
| 419 | + return ArrowArrayAppendDouble(array, value); |
| 420 | + } else if constexpr (std::is_same_v<Value, std::string>) { |
| 421 | + return ArrowArrayAppendString(array, |
| 422 | + ArrowStringView(value.data(), value.size())); |
| 423 | + } else if constexpr (std::is_same_v<Value, std::vector<uint8_t>>) { |
| 424 | + ArrowBufferView bytes; |
| 425 | + bytes.data.as_uint8 = value.data(); |
| 426 | + bytes.size_bytes = static_cast<int64_t>(value.size()); |
| 427 | + return ArrowArrayAppendBytes(array, bytes); |
| 428 | + } else if constexpr (std::is_same_v<Value, Uuid>) { |
| 429 | + ArrowBufferView bytes; |
| 430 | + bytes.data.as_uint8 = value.bytes().data(); |
| 431 | + bytes.size_bytes = Uuid::kLength; |
| 432 | + return ArrowArrayAppendBytes(array, bytes); |
| 433 | + } else if constexpr (std::is_same_v<Value, Decimal>) { |
| 434 | + const auto& type = internal::checked_cast<const DecimalType&>(*literal.type()); |
| 435 | + ArrowDecimal decimal; |
| 436 | + ArrowDecimalInit(&decimal, Decimal::kBitWidth, type.precision(), type.scale()); |
| 437 | + ArrowDecimalSetBytes(&decimal, value.native_endian_bytes()); |
| 438 | + return ArrowArrayAppendDecimal(array, &decimal); |
| 439 | + } else { |
| 440 | + return EINVAL; |
| 441 | + } |
| 442 | + }, |
| 443 | + literal.value()); |
| 444 | +} |
| 445 | + |
| 446 | +Result<ArrowArray> MaterializeWriteField(const FieldProjection& projection, |
| 447 | + const ArrowSchema& schema, int64_t length) { |
| 448 | + ArrowArray result{}; |
| 449 | + ArrowError error; |
| 450 | + ICEBERG_NANOARROW_RETURN_UNEXPECTED_WITH_ERROR( |
| 451 | + ArrowArrayInitFromSchema(&result, &schema, &error), error); |
| 452 | + internal::ArrowArrayGuard guard(&result); |
| 453 | + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayStartAppending(&result)); |
| 454 | + if (projection.kind == FieldProjection::Kind::kNull) { |
| 455 | + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(&result, length)); |
| 456 | + } else { |
| 457 | + const auto& literal = std::get<Literal>(projection.from); |
| 458 | + for (int64_t row = 0; row < length; ++row) { |
| 459 | + ICEBERG_NANOARROW_RETURN_UNEXPECTED(AppendWriteDefault(&result, literal)); |
| 460 | + } |
| 461 | + } |
| 462 | + ICEBERG_NANOARROW_RETURN_UNEXPECTED_WITH_ERROR( |
| 463 | + ArrowArrayFinishBuildingDefault(&result, &error), error); |
| 464 | + return std::exchange(result, ArrowArray{}); |
| 465 | +} |
| 466 | + |
| 467 | +Result<ArrowArray> MaterializeWriteProjection( |
| 468 | + ArrowArray& input, const ArrowSchema& schema, |
| 469 | + std::span<const FieldProjection> projections) { |
| 470 | + ArrowArray result{}; |
| 471 | + ICEBERG_NANOARROW_RETURN_UNEXPECTED( |
| 472 | + ArrowArrayInitFromType(&result, NANOARROW_TYPE_UNINITIALIZED)); |
| 473 | + internal::ArrowArrayGuard guard(&result); |
| 474 | + ICEBERG_NANOARROW_RETURN_UNEXPECTED( |
| 475 | + ArrowArrayAllocateChildren(&result, schema.n_children)); |
| 476 | + result.n_buffers = input.n_buffers; |
| 477 | + result.length = input.length; |
| 478 | + result.offset = input.offset; |
| 479 | + result.null_count = input.null_count; |
| 480 | + // MoveShared makes every input buffer independently reference-counted. Clone only |
| 481 | + // the parent buffers; existing leaves can then move directly into the new tree. |
| 482 | + for (int64_t index = 0; index < input.n_buffers; ++index) { |
| 483 | + ArrowBuffer buffer; |
| 484 | + ArrowBufferInit(&buffer); |
| 485 | + internal::ArrowArrayBufferGuard buffer_guard(&buffer); |
| 486 | + ICEBERG_NANOARROW_RETURN_UNEXPECTED( |
| 487 | + ArrowSharedBufferClone(ArrowArrayBuffer(&input, index), &buffer)); |
| 488 | + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArraySetBuffer(&result, index, &buffer)); |
| 489 | + } |
| 490 | + if (std::string_view(schema.format) == "+m") { |
| 491 | + // Maps add an entries struct between the map and its key/value projection. |
| 492 | + ICEBERG_ASSIGN_OR_RAISE( |
| 493 | + *result.children[0], |
| 494 | + MaterializeWriteProjection(*input.children[0], *schema.children[0], projections)); |
| 495 | + } else { |
| 496 | + const int64_t length = std::string_view(schema.format) == "+s" |
| 497 | + ? input.offset + input.length |
| 498 | + : input.children[0]->length; |
| 499 | + for (size_t index = 0; index < projections.size(); ++index) { |
| 500 | + const auto& projection = projections[index]; |
| 501 | + const auto& child_schema = *schema.children[index]; |
| 502 | + auto* output_child = result.children[index]; |
| 503 | + if (projection.kind == FieldProjection::Kind::kProjected) { |
| 504 | + auto* child = input.children[std::get<size_t>(projection.from)]; |
| 505 | + if (projection.children.empty()) { |
| 506 | + ArrowArrayMove(child, output_child); |
| 507 | + } else { |
| 508 | + ICEBERG_ASSIGN_OR_RAISE( |
| 509 | + *output_child, |
| 510 | + MaterializeWriteProjection(*child, child_schema, projection.children)); |
| 511 | + } |
| 512 | + } else { |
| 513 | + ICEBERG_ASSIGN_OR_RAISE(*output_child, |
| 514 | + MaterializeWriteField(projection, child_schema, length)); |
| 515 | + } |
| 516 | + } |
| 517 | + } |
| 518 | + return std::exchange(result, ArrowArray{}); |
| 519 | +} |
| 520 | + |
| 521 | +} // namespace |
| 522 | + |
| 523 | +class WriteProjection::Impl { |
| 524 | + public: |
| 525 | + bool identical = false; |
| 526 | + SchemaProjection projection; |
| 527 | + ArrowSchema write_schema{}; |
| 528 | + ArrowArrayView input_view{}; |
| 529 | + |
| 530 | + ~Impl() { |
| 531 | + ArrowArrayViewReset(&input_view); |
| 532 | + if (write_schema.release != nullptr) { |
| 533 | + ArrowSchemaRelease(&write_schema); |
| 534 | + } |
| 535 | + } |
| 536 | +}; |
| 537 | + |
| 538 | +WriteProjection::WriteProjection(std::unique_ptr<Impl> impl) : impl_(std::move(impl)) {} |
| 539 | +WriteProjection::~WriteProjection() = default; |
| 540 | + |
| 541 | +Result<std::unique_ptr<WriteProjection>> WriteProjection::Make( |
| 542 | + const Schema& input_schema, const Schema& write_schema) { |
| 543 | + auto impl = std::make_unique<Impl>(); |
| 544 | + impl->identical = input_schema == write_schema; |
| 545 | + if (!impl->identical) { |
| 546 | + ICEBERG_ASSIGN_OR_RAISE(impl->projection, |
| 547 | + Project(write_schema, input_schema, /*prune_source=*/false, |
| 548 | + ProjectionMode::kWrite)); |
| 549 | + ICEBERG_RETURN_UNEXPECTED(ToArrowSchema(write_schema, &impl->write_schema)); |
| 550 | + ArrowSchema input_arrow_schema{}; |
| 551 | + ICEBERG_RETURN_UNEXPECTED(ToArrowSchema(input_schema, &input_arrow_schema)); |
| 552 | + internal::ArrowSchemaGuard guard(&input_arrow_schema); |
| 553 | + ArrowError error; |
| 554 | + ICEBERG_NANOARROW_RETURN_UNEXPECTED_WITH_ERROR( |
| 555 | + ArrowArrayViewInitFromSchema(&impl->input_view, &input_arrow_schema, &error), |
| 556 | + error); |
| 557 | + } |
| 558 | + return std::unique_ptr<WriteProjection>(new WriteProjection(std::move(impl))); |
| 559 | +} |
| 560 | + |
| 561 | +Result<ArrowArray> WriteProjection::AlignBatch(ArrowArray* input_batch) const { |
| 562 | + if (impl_->identical) { |
| 563 | + return std::exchange(*input_batch, ArrowArray{}); |
| 564 | + } |
| 565 | + internal::ArrowArrayGuard input_guard(input_batch); |
| 566 | + ArrowError error; |
| 567 | + ICEBERG_NANOARROW_RETURN_UNEXPECTED_WITH_ERROR( |
| 568 | + ArrowArrayViewSetArray(&impl_->input_view, input_batch, &error), error); |
| 569 | + ArrowArray shared{}; |
| 570 | + internal::ArrowArrayGuard shared_guard(&shared); |
| 571 | + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayMoveShared(input_batch, &shared)); |
| 572 | + return MaterializeWriteProjection(shared, impl_->write_schema, |
| 573 | + impl_->projection.fields); |
| 574 | +} |
| 575 | + |
397 | 576 | } // namespace iceberg |
0 commit comments