diff --git a/reference-tables/GenerateTables.java b/reference-tables/GenerateTables.java index 9474b5c..9c8c4fd 100644 --- a/reference-tables/GenerateTables.java +++ b/reference-tables/GenerateTables.java @@ -29,13 +29,17 @@ import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Stream; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.AppendFiles; import org.apache.iceberg.DataFile; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.IcebergBuild; @@ -65,6 +69,16 @@ import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.JsonUtil; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.example.data.simple.SimpleGroupFactory; +import org.apache.parquet.hadoop.ParquetFileWriter; +import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.example.ExampleParquetWriter; +import org.apache.parquet.schema.GroupType; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; +import org.apache.parquet.schema.Type.Repetition; /** * Writes reference tables from the Input block in each table's README.md. @@ -131,6 +145,13 @@ static void generate(HadoopTables tables, String name, JsonNode input) throws IO Map props = new HashMap<>(); props.put(TableProperties.FORMAT_VERSION, String.valueOf(formatVersion)); + if (input.has("properties")) { + // a value that is not a string, such as a name mapping, is written as its compact JSON + for (Map.Entry e : input.get("properties").properties()) { + JsonNode value = e.getValue(); + props.put(e.getKey(), value.isTextual() ? value.textValue() : value.toString()); + } + } Table table = tables.create(schema, spec, props, SENTINEL + "/" + name); for (JsonNode commit : input.get("commits")) { @@ -146,6 +167,11 @@ static void generate(HadoopTables tables, String name, JsonNode input) throws IO updateSchema(table, commit.get("schema-updates")); continue; } + if (commit.has("file-schema")) { + DataFile file = writeWithFileSchema(table, commit); + table.newAppend().appendFile(file).commit(); + continue; + } List records = new ArrayList<>(); for (JsonNode row : commit.get("rows")) { records.add(record(table.schema().asStruct(), row)); @@ -280,6 +306,190 @@ static DataFile write(Table table, String tag, List records) throws IOEx return writer.toDataFile(); } + /** + * Writes a data file whose own schema differs from the table's, with parquet-mr rather than + * Iceberg's writer, so that the file can carry column names, column order and field IDs the table + * schema does not. The file schema is given in the spec's JSON form. Its field IDs key the rows + * and are written to the file, except those listed in {@code without-field-ids}, which are + * written with no field ID at all. The data file entry carries the record count and no column + * metrics, since metrics are keyed by field ID. + */ + static DataFile writeWithFileSchema(Table table, JsonNode commit) throws IOException { + if (table.spec().isPartitioned()) { + throw new IllegalArgumentException("file-schema is supported for unpartitioned tables only"); + } + Schema fileSchema = SchemaParser.fromJson(commit.get("file-schema")); + Set withoutIds = new HashSet<>(); + if (commit.has("without-field-ids")) { + for (JsonNode id : commit.get("without-field-ids")) { + withoutIds.add(id.intValue()); + } + } + + List columns = new ArrayList<>(); + for (Types.NestedField field : fileSchema.columns()) { + columns.add(parquetType(field, withoutIds)); + } + MessageType message = new MessageType("table", columns); + + String location = table.location() + "/data/" + commit.get("data-file").asText() + ".parquet"; + SimpleGroupFactory groups = new SimpleGroupFactory(message); + long count = 0; + try (ParquetWriter writer = + ExampleParquetWriter.builder(new org.apache.hadoop.fs.Path(location)) + .withConf(new Configuration()) + .withType(message) + .withWriteMode(ParquetFileWriter.Mode.OVERWRITE) + .build()) { + for (JsonNode row : commit.get("rows")) { + Group group = groups.newGroup(); + fill(group, fileSchema.asStruct(), record(fileSchema.asStruct(), row)); + writer.write(group); + count += 1; + } + } + + return DataFiles.builder(table.spec()) + .withInputFile(table.io().newInputFile(location)) + .withFormat(FileFormat.PARQUET) + .withRecordCount(count) + .build(); + } + + /** A field as a Parquet type, with three-level lists and maps as the spec's Parquet mapping. */ + static org.apache.parquet.schema.Type parquetType( + Types.NestedField field, Set withoutIds) { + Repetition repetition = field.isOptional() ? Repetition.OPTIONAL : Repetition.REQUIRED; + Type type = field.type(); + org.apache.parquet.schema.Type result; + switch (type.typeId()) { + case LONG: + result = + org.apache.parquet.schema.Types.primitive(PrimitiveTypeName.INT64, repetition) + .named(field.name()); + break; + case INTEGER: + result = + org.apache.parquet.schema.Types.primitive(PrimitiveTypeName.INT32, repetition) + .named(field.name()); + break; + case DOUBLE: + result = + org.apache.parquet.schema.Types.primitive(PrimitiveTypeName.DOUBLE, repetition) + .named(field.name()); + break; + case STRING: + result = + org.apache.parquet.schema.Types.primitive(PrimitiveTypeName.BINARY, repetition) + .as(LogicalTypeAnnotation.stringType()) + .named(field.name()); + break; + case STRUCT: + { + List fields = new ArrayList<>(); + for (Types.NestedField child : type.asStructType().fields()) { + fields.add(parquetType(child, withoutIds)); + } + result = new GroupType(repetition, field.name(), fields); + break; + } + case LIST: + { + Types.NestedField element = type.asListType().fields().get(0); + result = + org.apache.parquet.schema.Types.buildGroup(repetition) + .as(LogicalTypeAnnotation.listType()) + .addField( + new GroupType( + Repetition.REPEATED, + "list", + parquetType(rename(element, "element"), withoutIds))) + .named(field.name()); + break; + } + case MAP: + { + Types.NestedField key = type.asMapType().fields().get(0); + Types.NestedField value = type.asMapType().fields().get(1); + result = + org.apache.parquet.schema.Types.buildGroup(repetition) + .as(LogicalTypeAnnotation.mapType()) + .addField( + new GroupType( + Repetition.REPEATED, + "key_value", + parquetType(rename(key, "key"), withoutIds), + parquetType(rename(value, "value"), withoutIds))) + .named(field.name()); + break; + } + default: + throw new UnsupportedOperationException("No Parquet type for " + type); + } + return withoutIds.contains(field.fieldId()) ? result : result.withId(field.fieldId()); + } + + static Types.NestedField rename(Types.NestedField field, String name) { + return Types.NestedField.from(field).withName(name).build(); + } + + /** Copies a record into a Parquet group of the same shape. A null value is left unset. */ + static void fill(Group group, Types.StructType struct, Record record) { + List fields = struct.fields(); + for (int i = 0; i < fields.size(); i += 1) { + Types.NestedField field = fields.get(i); + Object value = record.get(i); + if (value != null) { + add(group, field.name(), field.type(), value); + } + } + } + + static void add(Group group, String name, Type type, Object value) { + switch (type.typeId()) { + case LONG: + group.add(name, (Long) value); + return; + case INTEGER: + group.add(name, (Integer) value); + return; + case DOUBLE: + group.add(name, (Double) value); + return; + case STRING: + group.add(name, (String) value); + return; + case STRUCT: + fill(group.addGroup(name), type.asStructType(), (Record) value); + return; + case LIST: + { + Group list = group.addGroup(name); + for (Object element : (List) value) { + Group entry = list.addGroup("list"); + if (element != null) { + add(entry, "element", type.asListType().elementType(), element); + } + } + return; + } + case MAP: + { + Group map = group.addGroup(name); + for (Map.Entry e : ((Map) value).entrySet()) { + Group entry = map.addGroup("key_value"); + add(entry, "key", type.asMapType().keyType(), e.getKey()); + if (e.getValue() != null) { + add(entry, "value", type.asMapType().valueType(), e.getValue()); + } + } + return; + } + default: + throw new UnsupportedOperationException("No Parquet value for " + type); + } + } + // input values, in the spec's single-value JSON keyed by field id, with long values as strings static Record record(Types.StructType struct, JsonNode node) { diff --git a/reference-tables/README.md b/reference-tables/README.md index 585b117..56272a9 100644 --- a/reference-tables/README.md +++ b/reference-tables/README.md @@ -174,6 +174,18 @@ symlink, and the consumer symlink is removed first. The generated `data/` and `metadata/` directories then replace the table's own, which are removed first so no earlier UUID-named files remain. +The Input may also give `properties`, table properties set when the table is +created. A value that is not a JSON string, such as a name mapping, is set as its +compact JSON. A commit may give a `file-schema`, in the spec's schema JSON, for a +data file whose own schema differs from the table's: other column names, another +column order, fields the table has and the file lacks. Its rows are keyed by the +file schema's field IDs, and the file carries those IDs except the ones listed in +`without-field-ids`, which it writes with no field ID. Lists and maps take the +three-level form the spec's Parquet mapping gives. Such a file is written with +parquet-mr's example writer rather than Iceberg's, uncompressed, and its manifest +entry carries the record count and no column metrics, because those are keyed by +field ID. This is supported for unpartitioned tables only. + Data files are GZIP-compressed although each table sets `write.parquet.compression-codec` to `zstd`, because the generator builds the Parquet writer without the table's properties. diff --git a/reference-tables/v2/partial-field-ids-parquet-data/README.md b/reference-tables/v2/partial-field-ids-parquet-data/README.md new file mode 100644 index 0000000..ef5d6d1 --- /dev/null +++ b/reference-tables/v2/partial-field-ids-parquet-data/README.md @@ -0,0 +1,231 @@ + + +# v2/partial-field-ids-parquet-data + +A table with `schema.name-mapping.default` set and one Parquet data file that +carries field IDs on some fields and not others. The file has field IDs on every +top-level column and on the struct's fields. It has none on the list element, the +map key or the map value. + +Each ID-carrying column has a physical name the name mapping does not list (`c1`, +`c2`, and so on), so it resolves only by field ID. The physical column order +differs from the table schema's, inside the struct as well. The list and map use +the three-level form of the spec's Parquet mapping, so the ID-less fields are +named `element`, `key` and `value`, which are the names the mapping gives them. +Field 6, `added`, is in the table schema and the name mapping and is not in the +file. + +| Field ID | Table name | Physical name | Field ID in the file | +| -- | -- | -- | -- | +| 1 | `id` | `c1` | yes | +| 2 | `name` | `c2` | yes | +| 3 | `location` | `c3` | yes | +| 7 | `location.lat` | `c3.c7` | yes | +| 8 | `location.lon` | `c3.c8` | yes | +| 4 | `tags` | `c4` | yes | +| 9 | `tags.element` | `c4.list.element` | no | +| 5 | `props` | `c5` | yes | +| 10 | `props.key` | `c5.key_value.key` | no | +| 11 | `props.value` | `c5.key_value.value` | no | +| 6 | `added` | not in the file | | + +## Scope + +Partial field IDs in a Parquet data file: every top-level column and the struct's +fields carry IDs, while the list element, map key and map value do not, with a +name mapping that names the physical columns. The open case is how a reader +resolves the ID-less fields, so the table has no `expected/scan.json` until the +question on dev@iceberg.apache.org is settled. The values the spec already fixes +are stated in "What the spec already fixes". + +## Waiting on + +This table has no `expected/scan.json` and no `index.json` entry yet. The spec +does not settle how a reader resolves the fields of this file that carry no field +ID, and the question is under discussion on dev@iceberg.apache.org in +[Name mapping for data files that carry only some field IDs](https://lists.apache.org/thread/vgxtsjvcfhjq06lcg3mnstp2fyglgzbd). + +Two sentences of the spec's +[Column Projection](https://iceberg.apache.org/spec/#column-projection) section +bear on it. One says to use `schema.name-mapping.default` "to map field id to +columns without field id". The other says the mappings provide fallback field IDs +"when a data file does not contain field id information". This file contains +field ID information, and three of its fields have none. + +The open question is what a read of fields 4 and 5 returns. Read field by field, +under the first sentence, the ID-less element, key and value take their IDs from +the mapping. Read file by file, under the second, the mapping does not apply to +this file, and the question becomes what a list or map whose element, key or +value has no field ID reads as: a null list or map, a failure, or a behavior the +spec leaves unspecified. + +The expected values are written once the thread settles that. + +## What the spec already fixes + +Unless the answer is that a file like this one cannot be read at all, the spec +fixes these values for a scan of this table that does not project fields 4 and 5. +They are stated here rather than as a partial `expected/scan.json`, because the +reference tables have no format for a scan that covers only some fields. + +- Fields 1, 2, 3, 7 and 8 are read from columns `c1`, `c2`, `c3`, `c3.c7` and + `c3.c8` by their field IDs, as the rows in the Input give them. Columns are + selected by field ID. Under the first sentence above the mapping applies only + to columns without a field ID, and under the second it does not apply to a file + that carries field IDs, so neither reading lets it resolve these columns. +- Field 6 reads as `null` in every row. The table is unpartitioned, the file has + no column for it, and it has no `initial-default`. The mapping names it `added`, + and the file has no column of that name. + +A projection of one of these columns, or of these columns in another order, +selects the same values by field ID. + +## Input + +Input to `reference-tables/GenerateTables.java`. The schema, partition spec and +name mapping are the spec's JSON forms. The commit gives the data file's own +schema, whose field IDs key the rows and are written to the file except those in +`without-field-ids`. Rows are in the spec's single-value JSON keyed by field ID, +with `long` values as strings. + +```json +{ + "format-version": 2, + "description": "Name mapping set, and one Parquet data file with field IDs on its top-level columns and struct fields but not on its list element, map key or map value. ID-carrying columns have names the mapping does not list. One snapshot, three rows.", + "properties": { + "schema.name-mapping.default": [ + {"field-id": 1, "names": ["id"]}, + {"field-id": 2, "names": ["name"]}, + { + "field-id": 3, + "names": ["location"], + "fields": [ + {"field-id": 7, "names": ["lat"]}, + {"field-id": 8, "names": ["lon"]} + ] + }, + {"field-id": 4, "names": ["tags"], "fields": [{"field-id": 9, "names": ["element"]}]}, + { + "field-id": 5, + "names": ["props"], + "fields": [ + {"field-id": 10, "names": ["key"]}, + {"field-id": 11, "names": ["value"]} + ] + }, + {"field-id": 6, "names": ["added"]} + ] + }, + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [ + {"id": 1, "name": "id", "required": true, "type": "long"}, + {"id": 2, "name": "name", "required": false, "type": "string"}, + { + "id": 3, + "name": "location", + "required": false, + "type": { + "type": "struct", + "fields": [ + {"id": 7, "name": "lat", "required": true, "type": "double"}, + {"id": 8, "name": "lon", "required": true, "type": "double"} + ] + } + }, + { + "id": 4, + "name": "tags", + "required": false, + "type": {"type": "list", "element-id": 9, "element": "string", "element-required": false} + }, + { + "id": 5, + "name": "props", + "required": false, + "type": { + "type": "map", + "key-id": 10, + "key": "string", + "value-id": 11, + "value": "int", + "value-required": false + } + }, + {"id": 6, "name": "added", "required": false, "type": "string"} + ] + }, + "partition-spec": { + "spec-id": 0, + "fields": [] + }, + "commits": [ + { + "data-file": "a", + "file-schema": { + "type": "struct", + "schema-id": 0, + "fields": [ + { + "id": 4, + "name": "c4", + "required": false, + "type": {"type": "list", "element-id": 9, "element": "string", "element-required": false} + }, + {"id": 1, "name": "c1", "required": true, "type": "long"}, + { + "id": 5, + "name": "c5", + "required": false, + "type": { + "type": "map", + "key-id": 10, + "key": "string", + "value-id": 11, + "value": "int", + "value-required": false + } + }, + { + "id": 3, + "name": "c3", + "required": false, + "type": { + "type": "struct", + "fields": [ + {"id": 8, "name": "c8", "required": true, "type": "double"}, + {"id": 7, "name": "c7", "required": true, "type": "double"} + ] + } + }, + {"id": 2, "name": "c2", "required": false, "type": "string"} + ] + }, + "without-field-ids": [9, 10, 11], + "rows": [ + {"1": "1", "2": "alpha", "3": {"7": 40.7128, "8": -74.006}, "4": ["east", "coastal"], "5": {"keys": ["pop"], "values": [8336817]}}, + {"1": "2", "2": null, "3": null, "4": null, "5": null}, + {"1": "3", "2": "charlie", "3": {"7": 51.5074, "8": -0.1278}, "4": ["west", null], "5": {"keys": ["pop", "area"], "values": [null, 1572]}} + ] + } + ] +} +``` diff --git a/reference-tables/v2/partial-field-ids-parquet-data/data/a.parquet b/reference-tables/v2/partial-field-ids-parquet-data/data/a.parquet new file mode 100644 index 0000000..5ddcf52 Binary files /dev/null and b/reference-tables/v2/partial-field-ids-parquet-data/data/a.parquet differ diff --git a/reference-tables/v2/partial-field-ids-parquet-data/metadata/10b4fea3-5958-4012-a791-53d0c8661288-m0.avro b/reference-tables/v2/partial-field-ids-parquet-data/metadata/10b4fea3-5958-4012-a791-53d0c8661288-m0.avro new file mode 100644 index 0000000..bea3565 Binary files /dev/null and b/reference-tables/v2/partial-field-ids-parquet-data/metadata/10b4fea3-5958-4012-a791-53d0c8661288-m0.avro differ diff --git a/reference-tables/v2/partial-field-ids-parquet-data/metadata/snap-8822764460534905462-1-10b4fea3-5958-4012-a791-53d0c8661288.avro b/reference-tables/v2/partial-field-ids-parquet-data/metadata/snap-8822764460534905462-1-10b4fea3-5958-4012-a791-53d0c8661288.avro new file mode 100644 index 0000000..9adf2fa Binary files /dev/null and b/reference-tables/v2/partial-field-ids-parquet-data/metadata/snap-8822764460534905462-1-10b4fea3-5958-4012-a791-53d0c8661288.avro differ diff --git a/reference-tables/v2/partial-field-ids-parquet-data/metadata/v2.metadata.json b/reference-tables/v2/partial-field-ids-parquet-data/metadata/v2.metadata.json new file mode 100644 index 0000000..91888af --- /dev/null +++ b/reference-tables/v2/partial-field-ids-parquet-data/metadata/v2.metadata.json @@ -0,0 +1 @@ +{"format-version":2,"table-uuid":"c1a57077-2296-496c-91f6-b1002eb575cf","location":"file:///tmp/iceberg-verification/reference-tables/v2/partial-field-ids-parquet-data","last-sequence-number":1,"last-updated-ms":1794824816527,"last-column-id":11,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"name","required":false,"type":"string"},{"id":3,"name":"location","required":false,"type":{"type":"struct","fields":[{"id":7,"name":"lat","required":true,"type":"double"},{"id":8,"name":"lon","required":true,"type":"double"}]}},{"id":4,"name":"tags","required":false,"type":{"type":"list","element-id":9,"element":"string","element-required":false}},{"id":5,"name":"props","required":false,"type":{"type":"map","key-id":10,"key":"string","value-id":11,"value":"int","value-required":false}},{"id":6,"name":"added","required":false,"type":"string"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"schema.name-mapping.default":"[{\"field-id\":1,\"names\":[\"id\"]},{\"field-id\":2,\"names\":[\"name\"]},{\"field-id\":3,\"names\":[\"location\"],\"fields\":[{\"field-id\":7,\"names\":[\"lat\"]},{\"field-id\":8,\"names\":[\"lon\"]}]},{\"field-id\":4,\"names\":[\"tags\"],\"fields\":[{\"field-id\":9,\"names\":[\"element\"]}]},{\"field-id\":5,\"names\":[\"props\"],\"fields\":[{\"field-id\":10,\"names\":[\"key\"]},{\"field-id\":11,\"names\":[\"value\"]}]},{\"field-id\":6,\"names\":[\"added\"]}]","write.parquet.compression-codec":"zstd"},"current-snapshot-id":8822764460534905462,"refs":{"main":{"snapshot-id":8822764460534905462,"type":"branch"}},"snapshots":[{"sequence-number":1,"snapshot-id":8822764460534905462,"timestamp-ms":1794824816520,"summary":{"operation":"append","manifests-created":"1","manifests-kept":"0","manifests-replaced":"0","added-data-files":"1","added-records":"3","added-files-size":"1665","changed-partition-count":"1","total-records":"3","total-files-size":"1665","total-data-files":"1","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","iceberg-version":"Apache Iceberg 1.11.0 (commit 6976e020b894f6a6777704df2b8c4458cb291ae9)"},"manifest-list":"file:/tmp/iceberg-verification/reference-tables/v2/partial-field-ids-parquet-data/metadata/snap-8822764460534905462-1-10b4fea3-5958-4012-a791-53d0c8661288.avro","schema-id":0}],"statistics":[],"partition-statistics":[],"snapshot-log":[{"timestamp-ms":1794824816520,"snapshot-id":8822764460534905462}],"metadata-log":[{"timestamp-ms":1794824815755,"metadata-file":"file:/tmp/iceberg-verification/reference-tables/v2/partial-field-ids-parquet-data/metadata/v1.metadata.json"}]} \ No newline at end of file