Skip to content

Commit bb8cd9f

Browse files
committed
feat(avro,parquet): read the full file schema when no projection is set
When ReaderOptions::projection is not set, the Avro and Parquet readers now derive the read schema from the file schema instead of failing. Also export FromArrowSchema for the bundle library so shared builds can link it. Closes #514 Signed-off-by: Vincent <1729065730@qq.com>
1 parent a899511 commit bb8cd9f

12 files changed

Lines changed: 573 additions & 15 deletions

‎src/iceberg/avro/avro_reader.cc‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -284,13 +284,7 @@ std::shared_ptr<::arrow::DataType> StorageTypeForBuilder(
284284
class AvroReader::Impl {
285285
public:
286286
Status Open(const ReaderOptions& options) {
287-
// TODO(gangwu): perhaps adding a ReaderOptions::Validate() method
288-
if (options.projection == nullptr) {
289-
return InvalidArgument("Projected schema is required by Avro reader");
290-
}
291-
292287
batch_size_ = options.properties.Get(ReaderProperties::kBatchSize);
293-
read_schema_ = options.projection;
294288

295289
// Open the input stream and adapt to the avro interface.
296290
ICEBERG_ASSIGN_OR_RAISE(
@@ -328,6 +322,13 @@ class AvroReader::Impl {
328322
return InvalidSchema("Not all fields in the Avro file schema have field IDs");
329323
}
330324

325+
if (options.projection != nullptr) {
326+
read_schema_ = options.projection;
327+
} else {
328+
// Read all fields of the file schema when no projection is given.
329+
ICEBERG_ASSIGN_OR_RAISE(read_schema_, FromAvroSchema(file_schema.root()));
330+
}
331+
331332
// Project the read schema on top of the file schema.
332333
ICEBERG_ASSIGN_OR_RAISE(projection_, Project(*read_schema_, file_schema.root(),
333334
/*prune_source=*/false));

‎src/iceberg/avro/avro_schema_util.cc‎

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -903,6 +903,166 @@ Result<SchemaProjection> Project(const Schema& expected_schema,
903903

904904
namespace {
905905

906+
Result<std::shared_ptr<Type>> FromAvroNode(const ::avro::NodePtr& node);
907+
908+
std::optional<std::string> GetIcebergFieldName(const ::avro::NodePtr& node,
909+
size_t field_idx) {
910+
if (field_idx >= node->customAttributes()) {
911+
return std::nullopt;
912+
}
913+
auto name = node->customAttributesAt(field_idx).getAttribute(
914+
std::string(kIcebergFieldNameProp));
915+
if (name.has_value() && name->size() >= 2 && name->front() == '"' &&
916+
name->back() == '"') {
917+
return name->substr(1, name->size() - 2);
918+
}
919+
return name;
920+
}
921+
922+
// Converts an Avro field node to an Iceberg field. A union with null is read as an
923+
// optional field and a bare null node as an optional field of unknown type.
924+
Result<SchemaField> FromAvroField(int32_t field_id, std::string_view name,
925+
const ::avro::NodePtr& node) {
926+
if (node->type() == ::avro::AVRO_NULL) {
927+
return SchemaField::MakeOptional(field_id, name, iceberg::unknown());
928+
}
929+
::avro::NodePtr value_node;
930+
ICEBERG_RETURN_UNEXPECTED(UnwrapUnion(node, &value_node));
931+
ICEBERG_ASSIGN_OR_RAISE(auto type, FromAvroNode(value_node));
932+
bool optional = node->type() == ::avro::AVRO_UNION;
933+
return SchemaField(field_id, name, std::move(type), optional);
934+
}
935+
936+
Result<std::vector<SchemaField>> FromAvroRecordFields(const ::avro::NodePtr& node) {
937+
if (node->type() != ::avro::AVRO_RECORD) {
938+
return InvalidSchema("Expected AVRO_RECORD type, but got {}", ToString(node));
939+
}
940+
std::vector<SchemaField> fields;
941+
fields.reserve(node->leaves());
942+
for (size_t i = 0; i < node->leaves(); ++i) {
943+
ICEBERG_ASSIGN_OR_RAISE(int32_t field_id, GetFieldId(node, i));
944+
std::string name = GetIcebergFieldName(node, i).value_or(node->nameAt(i));
945+
ICEBERG_ASSIGN_OR_RAISE(auto field, FromAvroField(field_id, name, node->leafAt(i)));
946+
fields.emplace_back(std::move(field));
947+
}
948+
return fields;
949+
}
950+
951+
Result<std::shared_ptr<Type>> FromAvroTimestamp(const ::avro::NodePtr& node, bool nanos) {
952+
bool adjust_to_utc = GetAdjustToUtc(node).value_or("false") == "true";
953+
if (nanos) {
954+
return adjust_to_utc ? std::shared_ptr<Type>(iceberg::timestamptz_ns())
955+
: std::shared_ptr<Type>(iceberg::timestamp_ns());
956+
}
957+
return adjust_to_utc ? std::shared_ptr<Type>(iceberg::timestamp_tz())
958+
: std::shared_ptr<Type>(iceberg::timestamp());
959+
}
960+
961+
Result<std::shared_ptr<Type>> FromAvroNode(const ::avro::NodePtr& node) {
962+
switch (node->type()) {
963+
case ::avro::AVRO_RECORD: {
964+
ICEBERG_ASSIGN_OR_RAISE(auto fields, FromAvroRecordFields(node));
965+
return iceberg::struct_(std::move(fields));
966+
}
967+
case ::avro::AVRO_ARRAY: {
968+
if (node->leaves() != 1) {
969+
return InvalidSchema("Array type must have exactly one node, got {}",
970+
node->leaves());
971+
}
972+
if (HasMapLogicalType(node)) {
973+
const auto& entry_node = node->leafAt(0);
974+
ICEBERG_ASSIGN_OR_RAISE(auto entry_fields, FromAvroRecordFields(entry_node));
975+
if (entry_fields.size() != 2) {
976+
return InvalidSchema(
977+
"Array-backed map type must have a record node with two fields");
978+
}
979+
SchemaField key(entry_fields[0].field_id(), MapType::kKeyName,
980+
entry_fields[0].type(), entry_fields[0].optional());
981+
SchemaField value(entry_fields[1].field_id(), MapType::kValueName,
982+
entry_fields[1].type(), entry_fields[1].optional());
983+
return iceberg::map(std::move(key), std::move(value));
984+
}
985+
ICEBERG_ASSIGN_OR_RAISE(int32_t element_id, GetElementId(node));
986+
ICEBERG_ASSIGN_OR_RAISE(
987+
auto element,
988+
FromAvroField(element_id, ListType::kElementName, node->leafAt(0)));
989+
return iceberg::list(std::move(element));
990+
}
991+
case ::avro::AVRO_MAP: {
992+
if (node->leaves() != 2) {
993+
return InvalidSchema("Map type must have exactly two nodes, got {}",
994+
node->leaves());
995+
}
996+
ICEBERG_ASSIGN_OR_RAISE(int32_t key_id, GetKeyId(node));
997+
ICEBERG_ASSIGN_OR_RAISE(int32_t value_id, GetValueId(node));
998+
auto key = SchemaField::MakeRequired(key_id, MapType::kKeyName, iceberg::string());
999+
ICEBERG_ASSIGN_OR_RAISE(
1000+
auto value, FromAvroField(value_id, MapType::kValueName, node->leafAt(1)));
1001+
return iceberg::map(std::move(key), std::move(value));
1002+
}
1003+
case ::avro::AVRO_BOOL:
1004+
return iceberg::boolean();
1005+
case ::avro::AVRO_INT:
1006+
if (HasLogicalType(node, ::avro::LogicalType::DATE)) {
1007+
return iceberg::date();
1008+
}
1009+
if (HasLogicalType(node, ::avro::LogicalType::NONE)) {
1010+
return iceberg::int32();
1011+
}
1012+
break;
1013+
case ::avro::AVRO_LONG:
1014+
switch (node->logicalType().type()) {
1015+
case ::avro::LogicalType::NONE:
1016+
return iceberg::int64();
1017+
case ::avro::LogicalType::TIME_MICROS:
1018+
return iceberg::time();
1019+
case ::avro::LogicalType::TIMESTAMP_MICROS:
1020+
return FromAvroTimestamp(node, /*nanos=*/false);
1021+
case ::avro::LogicalType::TIMESTAMP_NANOS:
1022+
return FromAvroTimestamp(node, /*nanos=*/true);
1023+
default:
1024+
break;
1025+
}
1026+
break;
1027+
case ::avro::AVRO_FLOAT:
1028+
return iceberg::float32();
1029+
case ::avro::AVRO_DOUBLE:
1030+
return iceberg::float64();
1031+
case ::avro::AVRO_STRING:
1032+
return iceberg::string();
1033+
case ::avro::AVRO_BYTES:
1034+
if (HasLogicalType(node, ::avro::LogicalType::NONE)) {
1035+
return iceberg::binary();
1036+
}
1037+
break;
1038+
case ::avro::AVRO_FIXED:
1039+
if (HasLogicalType(node, ::avro::LogicalType::DECIMAL)) {
1040+
const auto logical_type = node->logicalType();
1041+
return iceberg::decimal(static_cast<int32_t>(logical_type.precision()),
1042+
static_cast<int32_t>(logical_type.scale()));
1043+
}
1044+
if (HasLogicalType(node, ::avro::LogicalType::UUID) && node->fixedSize() == 16) {
1045+
return iceberg::uuid();
1046+
}
1047+
if (HasLogicalType(node, ::avro::LogicalType::NONE)) {
1048+
return iceberg::fixed(static_cast<int32_t>(node->fixedSize()));
1049+
}
1050+
break;
1051+
default:
1052+
break;
1053+
}
1054+
return NotSupported("Cannot convert Avro type {} to an Iceberg type", ToString(node));
1055+
}
1056+
1057+
} // namespace
1058+
1059+
Result<std::unique_ptr<Schema>> FromAvroSchema(const ::avro::NodePtr& avro_node) {
1060+
ICEBERG_ASSIGN_OR_RAISE(auto fields, FromAvroRecordFields(avro_node));
1061+
return std::make_unique<Schema>(std::move(fields));
1062+
}
1063+
1064+
namespace {
1065+
9061066
class NamesGuard {
9071067
public:
9081068
NamesGuard(std::vector<std::string>& names, const std::string& name) : names_(names) {

‎src/iceberg/avro/avro_schema_util_internal.h‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,17 @@ class HasIdVisitor {
146146
Result<SchemaProjection> Project(const Schema& expected_schema,
147147
const ::avro::NodePtr& avro_node, bool prune_source);
148148

149+
/// \brief Convert an Avro record schema annotated with Iceberg field IDs to an
150+
/// Iceberg Schema.
151+
///
152+
/// Every field must carry a field ID (and every array or map an element, key or
153+
/// value ID). A union with null becomes an optional field.
154+
///
155+
/// \param avro_node The Avro record node to convert.
156+
/// \return The converted Iceberg Schema, or an error if the Avro schema cannot be
157+
/// represented in Iceberg.
158+
Result<std::unique_ptr<Schema>> FromAvroSchema(const ::avro::NodePtr& avro_node);
159+
149160
std::string ToString(const ::avro::NodePtr& node);
150161
std::string ToString(const ::avro::LogicalType& logical_type);
151162
std::string ToString(const ::avro::LogicalType::Type& logical_type);

‎src/iceberg/file_reader.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,8 @@ struct ICEBERG_EXPORT ReaderOptions {
108108
std::optional<Split> split;
109109
/// \brief FileIO instance to open the file.
110110
std::shared_ptr<class FileIO> io;
111-
/// \brief The projection schema to read from the file. This field is required.
111+
/// \brief The projection schema to read from the file. If not set, all fields of
112+
/// the file schema are read.
112113
std::shared_ptr<class Schema> projection;
113114
/// \brief The filter to apply to the data. Reader implementations may ignore this if
114115
/// the file format does not support filtering.

‎src/iceberg/parquet/parquet_reader.cc‎

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,27 @@ Result<std::shared_ptr<::arrow::io::RandomAccessFile>> OpenInputStream(
6262
return arrow::OpenArrowInputStream(options.io, options.path, options.length);
6363
}
6464

65+
// SchemaManifest holds pointers into its own fields, so it is filled in place.
66+
Status MakeSchemaManifest(::parquet::arrow::FileReader* reader,
67+
const NameMapping* name_mapping,
68+
::parquet::arrow::SchemaManifest* schema_manifest) {
69+
auto metadata = reader->parquet_reader()->metadata();
70+
71+
ICEBERG_ARROW_RETURN_NOT_OK(::parquet::arrow::SchemaManifest::Make(
72+
metadata->schema(), metadata->key_value_metadata(), reader->properties(),
73+
schema_manifest));
74+
75+
if (!HasFieldIds(metadata->schema()->schema_root())) {
76+
if (name_mapping == nullptr) {
77+
return InvalidSchema(
78+
"Parquet file schema has no field IDs and no name mapping provided");
79+
}
80+
ApplyNameMapping(*name_mapping, schema_manifest);
81+
}
82+
83+
return {};
84+
}
85+
6586
Result<SchemaProjection> BuildProjection(::parquet::arrow::FileReader* reader,
6687
const Schema& read_schema,
6788
const NameMapping* name_mapping) {
@@ -278,11 +299,6 @@ class ParquetReader::Impl {
278299
public:
279300
// Open the Parquet reader with the given options
280301
Status Open(const ReaderOptions& options) {
281-
if (options.projection == nullptr) {
282-
return InvalidArgument("Projected schema is required by Parquet reader");
283-
}
284-
285-
read_schema_ = options.projection;
286302
stats_filter_.reset();
287303

288304
// Prepare reader properties
@@ -304,6 +320,16 @@ class ParquetReader::Impl {
304320
reader_, ::parquet::arrow::FileReader::Make(pool_, std::move(file_reader),
305321
arrow_reader_properties));
306322

323+
::parquet::arrow::SchemaManifest schema_manifest;
324+
ICEBERG_RETURN_UNEXPECTED(
325+
MakeSchemaManifest(reader_.get(), options.name_mapping.get(), &schema_manifest));
326+
if (options.projection != nullptr) {
327+
read_schema_ = options.projection;
328+
} else {
329+
// Read all fields of the file schema when no projection is given.
330+
ICEBERG_ASSIGN_OR_RAISE(read_schema_, FromParquetSchema(schema_manifest));
331+
}
332+
307333
// Project read schema onto the Parquet file schema
308334
ICEBERG_ASSIGN_OR_RAISE(projection_, BuildProjection(reader_.get(), *read_schema_,
309335
options.name_mapping.get()));
@@ -313,7 +339,7 @@ class ParquetReader::Impl {
313339
ICEBERG_ASSIGN_OR_RAISE(auto is_bound, IsBoundVisitor::IsBound(filter));
314340
if (!is_bound) {
315341
ICEBERG_ASSIGN_OR_RAISE(
316-
filter, Binder::Bind(*options.projection, filter, options.case_sensitive));
342+
filter, Binder::Bind(*read_schema_, filter, options.case_sensitive));
317343
}
318344
ICEBERG_ASSIGN_OR_RAISE(stats_filter_, ParquetMetricsRowGroupFilter::Make(
319345
filter, *reader_->manifest().descr));

‎src/iceberg/parquet/parquet_schema_util.cc‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,15 @@
1717
* under the License.
1818
*/
1919

20+
#include <arrow/c/bridge.h>
2021
#include <arrow/extension_type.h>
2122
#include <arrow/type.h>
2223
#include <arrow/type_fwd.h>
2324
#include <arrow/util/key_value_metadata.h>
2425
#include <parquet/arrow/schema.h>
2526
#include <parquet/schema.h>
2627

28+
#include "iceberg/arrow_c_data_guard_internal.h"
2729
#include "iceberg/constants.h"
2830
#include "iceberg/metadata_columns.h"
2931
#include "iceberg/name_mapping.h"
@@ -781,6 +783,60 @@ void ApplyNameMappingToField(const NameMapping& mapping, std::vector<std::string
781783
}
782784
}
783785

786+
// Rebuilds the Arrow field of a Parquet schema field so that nested fields carry the
787+
// field IDs stored in the SchemaField tree (e.g. those assigned by a name mapping).
788+
// List elements and map keys/values use the Iceberg names. Returns nullptr if the field,
789+
// or any list element or map key/value inside it, has no field ID, since such a field
790+
// cannot be read by ID.
791+
std::shared_ptr<::arrow::Field> ToArrowFieldWithIds(
792+
const ::parquet::arrow::SchemaField& parquet_field) {
793+
if (!GetFieldId(parquet_field).has_value()) {
794+
return nullptr;
795+
}
796+
797+
const auto& field = parquet_field.field;
798+
const auto& children = parquet_field.children;
799+
switch (field->type()->id()) {
800+
case ::arrow::Type::STRUCT: {
801+
::arrow::FieldVector child_fields;
802+
child_fields.reserve(children.size());
803+
for (const auto& child : children) {
804+
if (auto child_field = ToArrowFieldWithIds(child); child_field != nullptr) {
805+
child_fields.push_back(std::move(child_field));
806+
}
807+
}
808+
return field->WithType(::arrow::struct_(std::move(child_fields)));
809+
}
810+
case ::arrow::Type::LIST:
811+
case ::arrow::Type::LARGE_LIST: {
812+
if (children.size() != 1) {
813+
return nullptr;
814+
}
815+
auto element_field = ToArrowFieldWithIds(children[0]);
816+
if (element_field == nullptr) {
817+
return nullptr;
818+
}
819+
return field->WithType(
820+
::arrow::list(element_field->WithName(std::string(ListType::kElementName))));
821+
}
822+
case ::arrow::Type::MAP: {
823+
if (children.size() != 1 || children[0].children.size() != 2) {
824+
return nullptr;
825+
}
826+
auto key_field = ToArrowFieldWithIds(children[0].children[0]);
827+
auto value_field = ToArrowFieldWithIds(children[0].children[1]);
828+
if (key_field == nullptr || value_field == nullptr) {
829+
return nullptr;
830+
}
831+
return field->WithType(std::make_shared<::arrow::MapType>(
832+
key_field->WithName(std::string(MapType::kKeyName)),
833+
value_field->WithName(std::string(MapType::kValueName))));
834+
}
835+
default:
836+
return field;
837+
}
838+
}
839+
784840
} // namespace
785841

786842
Result<SchemaProjection> Project(const Schema& expected_schema,
@@ -826,6 +882,27 @@ void ApplyNameMapping(const NameMapping& mapping,
826882
}
827883
}
828884

885+
Result<std::unique_ptr<Schema>> FromParquetSchema(
886+
const ::parquet::arrow::SchemaManifest& parquet_schema) {
887+
::arrow::FieldVector fields;
888+
fields.reserve(parquet_schema.schema_fields.size());
889+
for (const auto& parquet_field : parquet_schema.schema_fields) {
890+
if (auto field = ToArrowFieldWithIds(parquet_field); field != nullptr) {
891+
fields.push_back(std::move(field));
892+
}
893+
}
894+
895+
ArrowSchema arrow_schema;
896+
if (auto status =
897+
::arrow::ExportSchema(::arrow::Schema(std::move(fields)), &arrow_schema);
898+
!status.ok()) {
899+
return InvalidSchema("Failed to export Parquet file schema to Arrow: {}",
900+
status.message());
901+
}
902+
internal::ArrowSchemaGuard guard(&arrow_schema);
903+
return FromArrowSchema(arrow_schema, /*schema_id=*/std::nullopt);
904+
}
905+
829906
Result<std::shared_ptr<::parquet::SchemaDescriptor>> ToParquetSchema(
830907
const Schema& iceberg_schema) {
831908
::parquet::schema::NodeVector fields;

0 commit comments

Comments
 (0)