Skip to content

Commit 942e9af

Browse files
authored
feat: apply write defaults in DataWriter (#942)
Writing a batch with an older schema should fill omitted columns with Iceberg write-default values. This PR adds optional `DataWriterOptions::input_schema`; for a required column with initial-default 42 and write-default 7, input `[[1], [2]]` writes `[[1, 7], [2, 7]]`. DataWriter prepares a reusable `WriteProjection` once in `Make` and applies it to each batch. The projection matches fields by ID recursively, preserves supplied values and explicit nulls, fills missing optional fields with typed nulls, and materializes missing fields from their write-default. When no `input_schema` is provided, or the schemas are identical, the original Arrow batch is passed through without projection. Incompatible schemas fail before opening the file, including extra input IDs, missing required fields without a write-default, optional-to-required fields, and concrete-to-unknown writes. Existing primitive types must match; read projection behavior is unchanged. Parquet and Avro writers remain unchanged and continue to expect batches matching their complete target schema. The shared projection is implemented in core using nanoarrow shared buffers without adding an Arrow C++ dependency. Addresses the write-path portion of #637.
1 parent 61fa6ed commit 942e9af

10 files changed

Lines changed: 856 additions & 20 deletions

‎src/iceberg/arrow_c_data_util.cc‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,25 +17,36 @@
1717
* under the License.
1818
*/
1919

20+
#include <cerrno>
2021
#include <cstdint>
2122
#include <limits>
2223
#include <memory>
2324
#include <mutex>
2425
#include <span>
26+
#include <string>
27+
#include <string_view>
28+
#include <type_traits>
2529
#include <utility>
30+
#include <variant>
2631
#include <vector>
2732

2833
#include <nanoarrow/nanoarrow.h>
2934

3035
#include "iceberg/arrow_c_data_guard_internal.h"
3136
#include "iceberg/arrow_c_data_util_internal.h"
37+
#include "iceberg/expression/literal.h"
3238
#include "iceberg/file_reader.h"
3339
#include "iceberg/nanoarrow_status_internal.h"
3440
#include "iceberg/result.h"
3541
#include "iceberg/schema.h"
3642
#include "iceberg/schema_field.h"
3743
#include "iceberg/schema_internal.h"
44+
#include "iceberg/schema_util.h"
45+
#include "iceberg/type.h"
46+
#include "iceberg/util/checked_cast.h"
47+
#include "iceberg/util/decimal.h"
3848
#include "iceberg/util/macros.h"
49+
#include "iceberg/util/uuid.h"
3950

4051
namespace iceberg {
4152

@@ -394,4 +405,172 @@ Result<ArrowArray> ProjectBatch(ArrowArray* input_batch,
394405
projection.selected_field_indices());
395406
}
396407

408+
namespace {
409+
410+
int AppendWriteDefault(ArrowArray* array, const Literal& literal) {
411+
return std::visit(
412+
[&](const auto& value) -> int {
413+
using Value = std::decay_t<decltype(value)>;
414+
if constexpr (std::is_same_v<Value, std::monostate>) {
415+
return ArrowArrayAppendNull(array, 1);
416+
} else if constexpr (std::is_integral_v<Value>) {
417+
return ArrowArrayAppendInt(array, value);
418+
} else if constexpr (std::is_floating_point_v<Value>) {
419+
return ArrowArrayAppendDouble(array, value);
420+
} else if constexpr (std::is_same_v<Value, std::string>) {
421+
return ArrowArrayAppendString(array,
422+
ArrowStringView(value.data(), value.size()));
423+
} else if constexpr (std::is_same_v<Value, std::vector<uint8_t>>) {
424+
ArrowBufferView bytes;
425+
bytes.data.as_uint8 = value.data();
426+
bytes.size_bytes = static_cast<int64_t>(value.size());
427+
return ArrowArrayAppendBytes(array, bytes);
428+
} else if constexpr (std::is_same_v<Value, Uuid>) {
429+
ArrowBufferView bytes;
430+
bytes.data.as_uint8 = value.bytes().data();
431+
bytes.size_bytes = Uuid::kLength;
432+
return ArrowArrayAppendBytes(array, bytes);
433+
} else if constexpr (std::is_same_v<Value, Decimal>) {
434+
const auto& type = internal::checked_cast<const DecimalType&>(*literal.type());
435+
ArrowDecimal decimal;
436+
ArrowDecimalInit(&decimal, Decimal::kBitWidth, type.precision(), type.scale());
437+
ArrowDecimalSetBytes(&decimal, value.native_endian_bytes());
438+
return ArrowArrayAppendDecimal(array, &decimal);
439+
} else {
440+
return EINVAL;
441+
}
442+
},
443+
literal.value());
444+
}
445+
446+
Result<ArrowArray> MaterializeWriteField(const FieldProjection& projection,
447+
const ArrowSchema& schema, int64_t length) {
448+
ArrowArray result{};
449+
ArrowError error;
450+
ICEBERG_NANOARROW_RETURN_UNEXPECTED_WITH_ERROR(
451+
ArrowArrayInitFromSchema(&result, &schema, &error), error);
452+
internal::ArrowArrayGuard guard(&result);
453+
ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayStartAppending(&result));
454+
if (projection.kind == FieldProjection::Kind::kNull) {
455+
ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(&result, length));
456+
} else {
457+
const auto& literal = std::get<Literal>(projection.from);
458+
for (int64_t row = 0; row < length; ++row) {
459+
ICEBERG_NANOARROW_RETURN_UNEXPECTED(AppendWriteDefault(&result, literal));
460+
}
461+
}
462+
ICEBERG_NANOARROW_RETURN_UNEXPECTED_WITH_ERROR(
463+
ArrowArrayFinishBuildingDefault(&result, &error), error);
464+
return std::exchange(result, ArrowArray{});
465+
}
466+
467+
Result<ArrowArray> MaterializeWriteProjection(
468+
ArrowArray& input, const ArrowSchema& schema,
469+
std::span<const FieldProjection> projections) {
470+
ArrowArray result{};
471+
ICEBERG_NANOARROW_RETURN_UNEXPECTED(
472+
ArrowArrayInitFromType(&result, NANOARROW_TYPE_UNINITIALIZED));
473+
internal::ArrowArrayGuard guard(&result);
474+
ICEBERG_NANOARROW_RETURN_UNEXPECTED(
475+
ArrowArrayAllocateChildren(&result, schema.n_children));
476+
result.n_buffers = input.n_buffers;
477+
result.length = input.length;
478+
result.offset = input.offset;
479+
result.null_count = input.null_count;
480+
// MoveShared makes every input buffer independently reference-counted. Clone only
481+
// the parent buffers; existing leaves can then move directly into the new tree.
482+
for (int64_t index = 0; index < input.n_buffers; ++index) {
483+
ArrowBuffer buffer;
484+
ArrowBufferInit(&buffer);
485+
internal::ArrowArrayBufferGuard buffer_guard(&buffer);
486+
ICEBERG_NANOARROW_RETURN_UNEXPECTED(
487+
ArrowSharedBufferClone(ArrowArrayBuffer(&input, index), &buffer));
488+
ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArraySetBuffer(&result, index, &buffer));
489+
}
490+
if (std::string_view(schema.format) == "+m") {
491+
// Maps add an entries struct between the map and its key/value projection.
492+
ICEBERG_ASSIGN_OR_RAISE(
493+
*result.children[0],
494+
MaterializeWriteProjection(*input.children[0], *schema.children[0], projections));
495+
} else {
496+
const int64_t length = std::string_view(schema.format) == "+s"
497+
? input.offset + input.length
498+
: input.children[0]->length;
499+
for (size_t index = 0; index < projections.size(); ++index) {
500+
const auto& projection = projections[index];
501+
const auto& child_schema = *schema.children[index];
502+
auto* output_child = result.children[index];
503+
if (projection.kind == FieldProjection::Kind::kProjected) {
504+
auto* child = input.children[std::get<size_t>(projection.from)];
505+
if (projection.children.empty()) {
506+
ArrowArrayMove(child, output_child);
507+
} else {
508+
ICEBERG_ASSIGN_OR_RAISE(
509+
*output_child,
510+
MaterializeWriteProjection(*child, child_schema, projection.children));
511+
}
512+
} else {
513+
ICEBERG_ASSIGN_OR_RAISE(*output_child,
514+
MaterializeWriteField(projection, child_schema, length));
515+
}
516+
}
517+
}
518+
return std::exchange(result, ArrowArray{});
519+
}
520+
521+
} // namespace
522+
523+
class WriteProjection::Impl {
524+
public:
525+
bool identical = false;
526+
SchemaProjection projection;
527+
ArrowSchema write_schema{};
528+
ArrowArrayView input_view{};
529+
530+
~Impl() {
531+
ArrowArrayViewReset(&input_view);
532+
if (write_schema.release != nullptr) {
533+
ArrowSchemaRelease(&write_schema);
534+
}
535+
}
536+
};
537+
538+
WriteProjection::WriteProjection(std::unique_ptr<Impl> impl) : impl_(std::move(impl)) {}
539+
WriteProjection::~WriteProjection() = default;
540+
541+
Result<std::unique_ptr<WriteProjection>> WriteProjection::Make(
542+
const Schema& input_schema, const Schema& write_schema) {
543+
auto impl = std::make_unique<Impl>();
544+
impl->identical = input_schema == write_schema;
545+
if (!impl->identical) {
546+
ICEBERG_ASSIGN_OR_RAISE(impl->projection,
547+
Project(write_schema, input_schema, /*prune_source=*/false,
548+
ProjectionMode::kWrite));
549+
ICEBERG_RETURN_UNEXPECTED(ToArrowSchema(write_schema, &impl->write_schema));
550+
ArrowSchema input_arrow_schema{};
551+
ICEBERG_RETURN_UNEXPECTED(ToArrowSchema(input_schema, &input_arrow_schema));
552+
internal::ArrowSchemaGuard guard(&input_arrow_schema);
553+
ArrowError error;
554+
ICEBERG_NANOARROW_RETURN_UNEXPECTED_WITH_ERROR(
555+
ArrowArrayViewInitFromSchema(&impl->input_view, &input_arrow_schema, &error),
556+
error);
557+
}
558+
return std::unique_ptr<WriteProjection>(new WriteProjection(std::move(impl)));
559+
}
560+
561+
Result<ArrowArray> WriteProjection::AlignBatch(ArrowArray* input_batch) const {
562+
if (impl_->identical) {
563+
return std::exchange(*input_batch, ArrowArray{});
564+
}
565+
internal::ArrowArrayGuard input_guard(input_batch);
566+
ArrowError error;
567+
ICEBERG_NANOARROW_RETURN_UNEXPECTED_WITH_ERROR(
568+
ArrowArrayViewSetArray(&impl_->input_view, input_batch, &error), error);
569+
ArrowArray shared{};
570+
internal::ArrowArrayGuard shared_guard(&shared);
571+
ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayMoveShared(input_batch, &shared));
572+
return MaterializeWriteProjection(shared, impl_->write_schema,
573+
impl_->projection.fields);
574+
}
575+
397576
} // namespace iceberg

‎src/iceberg/arrow_c_data_util_internal.h‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -242,4 +242,25 @@ ICEBERG_EXPORT Result<ArrowArray> ProjectBatch(ArrowArray* input_batch,
242242
std::span<const int32_t> row_indices,
243243
ProjectionContext& projection);
244244

245+
/// \brief A write projection prepared once for a stable input/write schema pair.
246+
///
247+
/// Matches fields by id, fills missing fields with write-defaults or typed nulls,
248+
/// and rejects incompatible schemas at Make. Existing values, including explicit
249+
/// nulls, are preserved. Do not share a projection across concurrent calls.
250+
class ICEBERG_EXPORT WriteProjection {
251+
public:
252+
static Result<std::unique_ptr<WriteProjection>> Make(const Schema& input_schema,
253+
const Schema& write_schema);
254+
~WriteProjection();
255+
256+
/// \brief Align a batch, consuming it even on failure.
257+
/// Identical schemas transfer the original batch without conversion.
258+
Result<ArrowArray> AlignBatch(ArrowArray* input_batch) const;
259+
260+
private:
261+
class Impl;
262+
explicit WriteProjection(std::unique_ptr<Impl> impl);
263+
std::unique_ptr<Impl> impl_;
264+
};
265+
245266
} // namespace iceberg

‎src/iceberg/data/data_writer.cc‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
#include <map>
2323
#include <utility>
2424

25+
#include "iceberg/arrow_c_data_guard_internal.h"
26+
#include "iceberg/arrow_c_data_util_internal.h"
2527
#include "iceberg/data/writer.h" // IWYU pragma: keep
2628
#include "iceberg/file_writer.h"
2729
#include "iceberg/manifest/manifest_entry.h"
@@ -33,6 +35,11 @@ namespace iceberg {
3335
class DataWriter::Impl {
3436
public:
3537
static Result<std::unique_ptr<Impl>> Make(DataWriterOptions options) {
38+
ICEBERG_ASSIGN_OR_RAISE(
39+
auto projection,
40+
WriteProjection::Make(
41+
options.input_schema ? *options.input_schema : *options.schema,
42+
*options.schema));
3643
WriterOptions writer_options{
3744
.path = options.path,
3845
.schema = options.schema,
@@ -43,10 +50,15 @@ class DataWriter::Impl {
4350
ICEBERG_ASSIGN_OR_RAISE(auto writer,
4451
WriterFactoryRegistry::Open(options.format, writer_options));
4552

46-
return std::unique_ptr<Impl>(new Impl(std::move(options), std::move(writer)));
53+
return std::unique_ptr<Impl>(
54+
new Impl(std::move(options), std::move(writer), std::move(projection)));
4755
}
4856

49-
Status Write(ArrowArray* data) { return writer_->Write(data); }
57+
Status Write(ArrowArray* data) {
58+
ICEBERG_ASSIGN_OR_RAISE(auto aligned, projection_->AlignBatch(data));
59+
internal::ArrowArrayGuard guard(&aligned);
60+
return writer_->Write(&aligned);
61+
}
5062

5163
Result<int64_t> Length() const { return writer_->length(); }
5264

@@ -106,11 +118,15 @@ class DataWriter::Impl {
106118
}
107119

108120
private:
109-
Impl(DataWriterOptions options, std::unique_ptr<Writer> writer)
110-
: options_(std::move(options)), writer_(std::move(writer)) {}
121+
Impl(DataWriterOptions options, std::unique_ptr<Writer> writer,
122+
std::unique_ptr<WriteProjection> projection)
123+
: options_(std::move(options)),
124+
writer_(std::move(writer)),
125+
projection_(std::move(projection)) {}
111126

112127
DataWriterOptions options_;
113128
std::unique_ptr<Writer> writer_;
129+
std::unique_ptr<WriteProjection> projection_;
114130
bool closed_ = false;
115131
};
116132

‎src/iceberg/data/data_writer.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@ struct ICEBERG_DATA_EXPORT DataWriterOptions {
4848
std::shared_ptr<FileIO> io;
4949
std::optional<int32_t> sort_order_id;
5050
std::unordered_map<std::string, std::string> properties;
51+
/// \brief Schema of input batches; when omitted, batches use the target schema.
52+
std::shared_ptr<Schema> input_schema;
5153
};
5254

5355
/// \brief Writer for Iceberg data files.

0 commit comments

Comments
 (0)