From b6667ed74b479c3ff40b30bca1ea0351a8ee78df Mon Sep 17 00:00:00 2001 From: Sylwester Lachiewicz Date: Mon, 16 Nov 2026 11:26:13 +0100 Subject: [PATCH 1/2] Support table properties and data file schemas in the table generator A commit's file-schema is written with parquet-mr so a data file can carry column names, order and field IDs the table schema does not, including none on chosen fields; Iceberg's writer always writes every ID. --- reference-tables/GenerateTables.java | 210 +++++++++++++++++++++++++++ reference-tables/README.md | 12 ++ 2 files changed, 222 insertions(+) diff --git a/reference-tables/GenerateTables.java b/reference-tables/GenerateTables.java index 9474b5c..9c8c4fd 100644 --- a/reference-tables/GenerateTables.java +++ b/reference-tables/GenerateTables.java @@ -29,13 +29,17 @@ import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Stream; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.AppendFiles; import org.apache.iceberg.DataFile; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.IcebergBuild; @@ -65,6 +69,16 @@ import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.JsonUtil; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.example.data.simple.SimpleGroupFactory; +import org.apache.parquet.hadoop.ParquetFileWriter; +import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.example.ExampleParquetWriter; +import org.apache.parquet.schema.GroupType; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; +import org.apache.parquet.schema.Type.Repetition; /** * Writes reference tables from the Input block in each table's README.md. @@ -131,6 +145,13 @@ static void generate(HadoopTables tables, String name, JsonNode input) throws IO Map props = new HashMap<>(); props.put(TableProperties.FORMAT_VERSION, String.valueOf(formatVersion)); + if (input.has("properties")) { + // a value that is not a string, such as a name mapping, is written as its compact JSON + for (Map.Entry e : input.get("properties").properties()) { + JsonNode value = e.getValue(); + props.put(e.getKey(), value.isTextual() ? value.textValue() : value.toString()); + } + } Table table = tables.create(schema, spec, props, SENTINEL + "/" + name); for (JsonNode commit : input.get("commits")) { @@ -146,6 +167,11 @@ static void generate(HadoopTables tables, String name, JsonNode input) throws IO updateSchema(table, commit.get("schema-updates")); continue; } + if (commit.has("file-schema")) { + DataFile file = writeWithFileSchema(table, commit); + table.newAppend().appendFile(file).commit(); + continue; + } List records = new ArrayList<>(); for (JsonNode row : commit.get("rows")) { records.add(record(table.schema().asStruct(), row)); @@ -280,6 +306,190 @@ static DataFile write(Table table, String tag, List records) throws IOEx return writer.toDataFile(); } + /** + * Writes a data file whose own schema differs from the table's, with parquet-mr rather than + * Iceberg's writer, so that the file can carry column names, column order and field IDs the table + * schema does not. The file schema is given in the spec's JSON form. Its field IDs key the rows + * and are written to the file, except those listed in {@code without-field-ids}, which are + * written with no field ID at all. The data file entry carries the record count and no column + * metrics, since metrics are keyed by field ID. + */ + static DataFile writeWithFileSchema(Table table, JsonNode commit) throws IOException { + if (table.spec().isPartitioned()) { + throw new IllegalArgumentException("file-schema is supported for unpartitioned tables only"); + } + Schema fileSchema = SchemaParser.fromJson(commit.get("file-schema")); + Set withoutIds = new HashSet<>(); + if (commit.has("without-field-ids")) { + for (JsonNode id : commit.get("without-field-ids")) { + withoutIds.add(id.intValue()); + } + } + + List columns = new ArrayList<>(); + for (Types.NestedField field : fileSchema.columns()) { + columns.add(parquetType(field, withoutIds)); + } + MessageType message = new MessageType("table", columns); + + String location = table.location() + "/data/" + commit.get("data-file").asText() + ".parquet"; + SimpleGroupFactory groups = new SimpleGroupFactory(message); + long count = 0; + try (ParquetWriter writer = + ExampleParquetWriter.builder(new org.apache.hadoop.fs.Path(location)) + .withConf(new Configuration()) + .withType(message) + .withWriteMode(ParquetFileWriter.Mode.OVERWRITE) + .build()) { + for (JsonNode row : commit.get("rows")) { + Group group = groups.newGroup(); + fill(group, fileSchema.asStruct(), record(fileSchema.asStruct(), row)); + writer.write(group); + count += 1; + } + } + + return DataFiles.builder(table.spec()) + .withInputFile(table.io().newInputFile(location)) + .withFormat(FileFormat.PARQUET) + .withRecordCount(count) + .build(); + } + + /** A field as a Parquet type, with three-level lists and maps as the spec's Parquet mapping. */ + static org.apache.parquet.schema.Type parquetType( + Types.NestedField field, Set withoutIds) { + Repetition repetition = field.isOptional() ? Repetition.OPTIONAL : Repetition.REQUIRED; + Type type = field.type(); + org.apache.parquet.schema.Type result; + switch (type.typeId()) { + case LONG: + result = + org.apache.parquet.schema.Types.primitive(PrimitiveTypeName.INT64, repetition) + .named(field.name()); + break; + case INTEGER: + result = + org.apache.parquet.schema.Types.primitive(PrimitiveTypeName.INT32, repetition) + .named(field.name()); + break; + case DOUBLE: + result = + org.apache.parquet.schema.Types.primitive(PrimitiveTypeName.DOUBLE, repetition) + .named(field.name()); + break; + case STRING: + result = + org.apache.parquet.schema.Types.primitive(PrimitiveTypeName.BINARY, repetition) + .as(LogicalTypeAnnotation.stringType()) + .named(field.name()); + break; + case STRUCT: + { + List fields = new ArrayList<>(); + for (Types.NestedField child : type.asStructType().fields()) { + fields.add(parquetType(child, withoutIds)); + } + result = new GroupType(repetition, field.name(), fields); + break; + } + case LIST: + { + Types.NestedField element = type.asListType().fields().get(0); + result = + org.apache.parquet.schema.Types.buildGroup(repetition) + .as(LogicalTypeAnnotation.listType()) + .addField( + new GroupType( + Repetition.REPEATED, + "list", + parquetType(rename(element, "element"), withoutIds))) + .named(field.name()); + break; + } + case MAP: + { + Types.NestedField key = type.asMapType().fields().get(0); + Types.NestedField value = type.asMapType().fields().get(1); + result = + org.apache.parquet.schema.Types.buildGroup(repetition) + .as(LogicalTypeAnnotation.mapType()) + .addField( + new GroupType( + Repetition.REPEATED, + "key_value", + parquetType(rename(key, "key"), withoutIds), + parquetType(rename(value, "value"), withoutIds))) + .named(field.name()); + break; + } + default: + throw new UnsupportedOperationException("No Parquet type for " + type); + } + return withoutIds.contains(field.fieldId()) ? result : result.withId(field.fieldId()); + } + + static Types.NestedField rename(Types.NestedField field, String name) { + return Types.NestedField.from(field).withName(name).build(); + } + + /** Copies a record into a Parquet group of the same shape. A null value is left unset. */ + static void fill(Group group, Types.StructType struct, Record record) { + List fields = struct.fields(); + for (int i = 0; i < fields.size(); i += 1) { + Types.NestedField field = fields.get(i); + Object value = record.get(i); + if (value != null) { + add(group, field.name(), field.type(), value); + } + } + } + + static void add(Group group, String name, Type type, Object value) { + switch (type.typeId()) { + case LONG: + group.add(name, (Long) value); + return; + case INTEGER: + group.add(name, (Integer) value); + return; + case DOUBLE: + group.add(name, (Double) value); + return; + case STRING: + group.add(name, (String) value); + return; + case STRUCT: + fill(group.addGroup(name), type.asStructType(), (Record) value); + return; + case LIST: + { + Group list = group.addGroup(name); + for (Object element : (List) value) { + Group entry = list.addGroup("list"); + if (element != null) { + add(entry, "element", type.asListType().elementType(), element); + } + } + return; + } + case MAP: + { + Group map = group.addGroup(name); + for (Map.Entry e : ((Map) value).entrySet()) { + Group entry = map.addGroup("key_value"); + add(entry, "key", type.asMapType().keyType(), e.getKey()); + if (e.getValue() != null) { + add(entry, "value", type.asMapType().valueType(), e.getValue()); + } + } + return; + } + default: + throw new UnsupportedOperationException("No Parquet value for " + type); + } + } + // input values, in the spec's single-value JSON keyed by field id, with long values as strings static Record record(Types.StructType struct, JsonNode node) { diff --git a/reference-tables/README.md b/reference-tables/README.md index 585b117..56272a9 100644 --- a/reference-tables/README.md +++ b/reference-tables/README.md @@ -174,6 +174,18 @@ symlink, and the consumer symlink is removed first. The generated `data/` and `metadata/` directories then replace the table's own, which are removed first so no earlier UUID-named files remain. +The Input may also give `properties`, table properties set when the table is +created. A value that is not a JSON string, such as a name mapping, is set as its +compact JSON. A commit may give a `file-schema`, in the spec's schema JSON, for a +data file whose own schema differs from the table's: other column names, another +column order, fields the table has and the file lacks. Its rows are keyed by the +file schema's field IDs, and the file carries those IDs except the ones listed in +`without-field-ids`, which it writes with no field ID. Lists and maps take the +three-level form the spec's Parquet mapping gives. Such a file is written with +parquet-mr's example writer rather than Iceberg's, uncompressed, and its manifest +entry carries the record count and no column metrics, because those are keyed by +field ID. This is supported for unpartitioned tables only. + Data files are GZIP-compressed although each table sets `write.parquet.compression-codec` to `zstd`, because the generator builds the Parquet writer without the table's properties. From 9542d5291c100cb036648beb11920feb1b5f69c1 Mon Sep 17 00:00:00 2001 From: Sylwester Lachiewicz Date: Mon, 16 Nov 2026 11:28:34 +0100 Subject: [PATCH 2/2] Add reference table input for a data file with partial field IDs --- .../partial-field-ids-parquet-data/README.md | 231 ++++++++++++++++++ .../data/a.parquet | Bin 0 -> 1665 bytes ...b4fea3-5958-4012-a791-53d0c8661288-m0.avro | Bin 0 -> 7491 bytes ...-10b4fea3-5958-4012-a791-53d0c8661288.avro | Bin 0 -> 4476 bytes .../metadata/v2.metadata.json | 1 + 5 files changed, 232 insertions(+) create mode 100644 reference-tables/v2/partial-field-ids-parquet-data/README.md create mode 100644 reference-tables/v2/partial-field-ids-parquet-data/data/a.parquet create mode 100644 reference-tables/v2/partial-field-ids-parquet-data/metadata/10b4fea3-5958-4012-a791-53d0c8661288-m0.avro create mode 100644 reference-tables/v2/partial-field-ids-parquet-data/metadata/snap-8822764460534905462-1-10b4fea3-5958-4012-a791-53d0c8661288.avro create mode 100644 reference-tables/v2/partial-field-ids-parquet-data/metadata/v2.metadata.json diff --git a/reference-tables/v2/partial-field-ids-parquet-data/README.md b/reference-tables/v2/partial-field-ids-parquet-data/README.md new file mode 100644 index 0000000..ef5d6d1 --- /dev/null +++ b/reference-tables/v2/partial-field-ids-parquet-data/README.md @@ -0,0 +1,231 @@ + + +# v2/partial-field-ids-parquet-data + +A table with `schema.name-mapping.default` set and one Parquet data file that +carries field IDs on some fields and not others. The file has field IDs on every +top-level column and on the struct's fields. It has none on the list element, the +map key or the map value. + +Each ID-carrying column has a physical name the name mapping does not list (`c1`, +`c2`, and so on), so it resolves only by field ID. The physical column order +differs from the table schema's, inside the struct as well. The list and map use +the three-level form of the spec's Parquet mapping, so the ID-less fields are +named `element`, `key` and `value`, which are the names the mapping gives them. +Field 6, `added`, is in the table schema and the name mapping and is not in the +file. + +| Field ID | Table name | Physical name | Field ID in the file | +| -- | -- | -- | -- | +| 1 | `id` | `c1` | yes | +| 2 | `name` | `c2` | yes | +| 3 | `location` | `c3` | yes | +| 7 | `location.lat` | `c3.c7` | yes | +| 8 | `location.lon` | `c3.c8` | yes | +| 4 | `tags` | `c4` | yes | +| 9 | `tags.element` | `c4.list.element` | no | +| 5 | `props` | `c5` | yes | +| 10 | `props.key` | `c5.key_value.key` | no | +| 11 | `props.value` | `c5.key_value.value` | no | +| 6 | `added` | not in the file | | + +## Scope + +Partial field IDs in a Parquet data file: every top-level column and the struct's +fields carry IDs, while the list element, map key and map value do not, with a +name mapping that names the physical columns. The open case is how a reader +resolves the ID-less fields, so the table has no `expected/scan.json` until the +question on dev@iceberg.apache.org is settled. The values the spec already fixes +are stated in "What the spec already fixes". + +## Waiting on + +This table has no `expected/scan.json` and no `index.json` entry yet. The spec +does not settle how a reader resolves the fields of this file that carry no field +ID, and the question is under discussion on dev@iceberg.apache.org in +[Name mapping for data files that carry only some field IDs](https://lists.apache.org/thread/vgxtsjvcfhjq06lcg3mnstp2fyglgzbd). + +Two sentences of the spec's +[Column Projection](https://iceberg.apache.org/spec/#column-projection) section +bear on it. One says to use `schema.name-mapping.default` "to map field id to +columns without field id". The other says the mappings provide fallback field IDs +"when a data file does not contain field id information". This file contains +field ID information, and three of its fields have none. + +The open question is what a read of fields 4 and 5 returns. Read field by field, +under the first sentence, the ID-less element, key and value take their IDs from +the mapping. Read file by file, under the second, the mapping does not apply to +this file, and the question becomes what a list or map whose element, key or +value has no field ID reads as: a null list or map, a failure, or a behavior the +spec leaves unspecified. + +The expected values are written once the thread settles that. + +## What the spec already fixes + +Unless the answer is that a file like this one cannot be read at all, the spec +fixes these values for a scan of this table that does not project fields 4 and 5. +They are stated here rather than as a partial `expected/scan.json`, because the +reference tables have no format for a scan that covers only some fields. + +- Fields 1, 2, 3, 7 and 8 are read from columns `c1`, `c2`, `c3`, `c3.c7` and + `c3.c8` by their field IDs, as the rows in the Input give them. Columns are + selected by field ID. Under the first sentence above the mapping applies only + to columns without a field ID, and under the second it does not apply to a file + that carries field IDs, so neither reading lets it resolve these columns. +- Field 6 reads as `null` in every row. The table is unpartitioned, the file has + no column for it, and it has no `initial-default`. The mapping names it `added`, + and the file has no column of that name. + +A projection of one of these columns, or of these columns in another order, +selects the same values by field ID. + +## Input + +Input to `reference-tables/GenerateTables.java`. The schema, partition spec and +name mapping are the spec's JSON forms. The commit gives the data file's own +schema, whose field IDs key the rows and are written to the file except those in +`without-field-ids`. Rows are in the spec's single-value JSON keyed by field ID, +with `long` values as strings. + +```json +{ + "format-version": 2, + "description": "Name mapping set, and one Parquet data file with field IDs on its top-level columns and struct fields but not on its list element, map key or map value. ID-carrying columns have names the mapping does not list. One snapshot, three rows.", + "properties": { + "schema.name-mapping.default": [ + {"field-id": 1, "names": ["id"]}, + {"field-id": 2, "names": ["name"]}, + { + "field-id": 3, + "names": ["location"], + "fields": [ + {"field-id": 7, "names": ["lat"]}, + {"field-id": 8, "names": ["lon"]} + ] + }, + {"field-id": 4, "names": ["tags"], "fields": [{"field-id": 9, "names": ["element"]}]}, + { + "field-id": 5, + "names": ["props"], + "fields": [ + {"field-id": 10, "names": ["key"]}, + {"field-id": 11, "names": ["value"]} + ] + }, + {"field-id": 6, "names": ["added"]} + ] + }, + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [ + {"id": 1, "name": "id", "required": true, "type": "long"}, + {"id": 2, "name": "name", "required": false, "type": "string"}, + { + "id": 3, + "name": "location", + "required": false, + "type": { + "type": "struct", + "fields": [ + {"id": 7, "name": "lat", "required": true, "type": "double"}, + {"id": 8, "name": "lon", "required": true, "type": "double"} + ] + } + }, + { + "id": 4, + "name": "tags", + "required": false, + "type": {"type": "list", "element-id": 9, "element": "string", "element-required": false} + }, + { + "id": 5, + "name": "props", + "required": false, + "type": { + "type": "map", + "key-id": 10, + "key": "string", + "value-id": 11, + "value": "int", + "value-required": false + } + }, + {"id": 6, "name": "added", "required": false, "type": "string"} + ] + }, + "partition-spec": { + "spec-id": 0, + "fields": [] + }, + "commits": [ + { + "data-file": "a", + "file-schema": { + "type": "struct", + "schema-id": 0, + "fields": [ + { + "id": 4, + "name": "c4", + "required": false, + "type": {"type": "list", "element-id": 9, "element": "string", "element-required": false} + }, + {"id": 1, "name": "c1", "required": true, "type": "long"}, + { + "id": 5, + "name": "c5", + "required": false, + "type": { + "type": "map", + "key-id": 10, + "key": "string", + "value-id": 11, + "value": "int", + "value-required": false + } + }, + { + "id": 3, + "name": "c3", + "required": false, + "type": { + "type": "struct", + "fields": [ + {"id": 8, "name": "c8", "required": true, "type": "double"}, + {"id": 7, "name": "c7", "required": true, "type": "double"} + ] + } + }, + {"id": 2, "name": "c2", "required": false, "type": "string"} + ] + }, + "without-field-ids": [9, 10, 11], + "rows": [ + {"1": "1", "2": "alpha", "3": {"7": 40.7128, "8": -74.006}, "4": ["east", "coastal"], "5": {"keys": ["pop"], "values": [8336817]}}, + {"1": "2", "2": null, "3": null, "4": null, "5": null}, + {"1": "3", "2": "charlie", "3": {"7": 51.5074, "8": -0.1278}, "4": ["west", null], "5": {"keys": ["pop", "area"], "values": [null, 1572]}} + ] + } + ] +} +``` diff --git a/reference-tables/v2/partial-field-ids-parquet-data/data/a.parquet b/reference-tables/v2/partial-field-ids-parquet-data/data/a.parquet new file mode 100644 index 0000000000000000000000000000000000000000..5ddcf52e5a1650ce5f518da38e079dc643182b5d GIT binary patch literal 1665 zcmZuyU5Hyn6h4{Ey}28fVr6HTDVHS==90>~Y?94SAxagM`pZ&SShlcolT2ywW;bTD ztzubaANnF#5T!3Z`QlncP_VuT?n^;I>JOqIiw{~vTPP@4#E0rRb90lF^-jp0nRCuJ z-}&a8Wa;6PMFC<7$E~YBd~nDYSzNlp1z-S>zBf(ZUonm^6K)PN_-<|BB#db5oY@?p z{{oKJuk3&7_!tVp6COYc&GZlqJp@FL;9?fX?_YoX<8%WT2t%gcMvnl)z6lW_hvTiA z|9rxH1mhEYJj>@DNToTt-H+qaYnT2?liMWi*zFhiu<{Drhq<6}4#&k?J6|9qade5J zP0_jh$G6WkUp={Z^XKZ3y;FOi4V{1b?%!{`1I}wteV%#n)T76*-udX|Klh$Ej?RlX z{(kS`yVD1p9m*)`oe8aMXTpBeG5RpmDl^njD{duZT5=t@TtCi$xnO;~L_w`=I*0`V z9U(Szl1i&I3P-yP!b>S%n4{{P7+1)DA%#sv1w5v zaaNWBXkU6_n;6PjT3kqpHC)kjmx*VsVTM+>nOGSbuBF9l;=@PN9Y-{+yCU}cC!9gp zjEu-ag(=or6im1k&u<`hg)1I)unZZ9={!>z6Pd1AALQX_pA_SHz=+~9i$g0I$|_lZ zE)Jb~-n{hO*)ZBNScn)8DvdiyEJHlUTI1ld;E%Z=vKF$wd`~j1Egy;xXRT5m8ToXw zbhWR~2jGiooyS|LBX7$zVJJXm<+j`oyhfE5Y$1#a14Y8h!rY-DXD2EH<1o`EZ@a2u<432G#6)^o-AVr{;dD>Sz{Q9D72foK2@+JK9`47PWHh%yB literal 0 HcmV?d00001 diff --git a/reference-tables/v2/partial-field-ids-parquet-data/metadata/10b4fea3-5958-4012-a791-53d0c8661288-m0.avro b/reference-tables/v2/partial-field-ids-parquet-data/metadata/10b4fea3-5958-4012-a791-53d0c8661288-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..bea35657d5b187199fbe32151a3790b03114ae03 GIT binary patch literal 7491 zcmcIp-)kI29B+%LsMLa`Sm?vB4`Sg`ZgTcI?B&D`!>vhCjPy1SQ> zwgF##(=^0fT5PF`AgzKAK1$#E;FBQuX9yL2_B%5>KX&i-dUuHsLUwoN`{VQdG2hSR zoGB!Mw%sEH#2Gm|1#x!MzdZs+CB-q)Mzf=Hs~9(W`wvy z*TYlINXVyMKOyh|c9WWr6-42Ru~Fk&EV+g4)Hk(pkg8o^fv@ee<-Q|q1srJhvDhzd9K&l#bc!e8XAK$Hgt$>cCFJSX!J*$KY32Z2$vV8( z0-i5PGn{oP0(pjWi`hD!-85QaZHVJ^HOd^8)_lXF5q5(B_Oo<><8F`{P5M69 z$Z^pnK}nF%B~IAwv`8Y=k1oyDq_zEkIO(26PHQ4PoMDH`m8MmKoLI1?A6PdEGk{(0 z&vU_xT*!;F{9}ruHp*5BqMDkheuk>8A@fWer9LfYdD|nLAMh+bh(l$3QZ!5lLXTa z;b)`+Cz*U-ZYeiRbJkfi&4%!XAA5O+piQx|mB3C7P2M8q;v7cJ3A3?5!x=;?zKeq^ z`~aqXc!PEsmc{3BOtI!3BHu$<1V0oZ1`e$sKfM7KRX3%2qihoo@cuwG2T)n{NuZi5 zo278=hsvt&0VTxn@}t0mzVI!XXEFSE$Z3Q}3i94)+}_y8&R?a}#8n-B$#|vdx~G5G;Ds(|U!x zLJT$89(e^1O*UI^9zk@IBu)!3hp}7qkUjJ?Yli08>;%wRGu4GUpfg*u6Jf{vMa3=1EWJ^big5r)_GImh`8x_odbUFsaXKW!bV}&UB!e3#&zp0h zo2_e>}Y7LExt#_j|hB3~G+U=BNqi3&}19bLd+O|3jFH^A@25JU1CS+H) zBU>MB#B2$Z{TLQ;u23{+Mj3QY46JU{fg#IJ;XG#NJ6d?3Y8hgip2lgE09uI$xjcw> zP$Oi3F2hlhMOzW{ODda(6K@?cC&4Wb2?mbd?H{U7 z(!PiJEJ|lVm^^WYFdVe`gbxE_Zo_c1A>bSO=u8i;ql^u@kw@Ia0BV3Ua)M9bQ)?ti zfkzy&3_0db(J>cD!=d^K-zCtJS8^ix^(i^^jdsrgELbo@!6FsOEs%d2B-)vZw`}YgqqPPtaV$ote1HqYyoWKlio1;K0|=_{!GgltA0LKOHR%++p3I#Z#LC9> z%z~<|@(c=Wub%aBDCvf+?xP3Lr7^v*A(1M)upxAN_u_@*M*3x`>*o@W8Pg{Vu8u2w zvH&~bQ_BxAyZvP^#;F{gb|~{z*_Rl`xaL=vjIEW-r>LkO_bXa5TG4QT*x zF+)j!QlrkE^1NT6M|Q^n|K@-}U;bAE+GKtMBqnbqu+R$!5CUbM%E8j?uY?Wm-2<(I zM9htmP5cGM4JQZE&S}-k$~>UL(87dNDrYKPTkW!e;iD z7MBj^^Wm#>>^$(~_?_d&p8WJ>bMwRBc7Fcn?4f@Tx~*G(KmY65_6KjicL#gR%Z;Zy z+r7KjzjHPhA3o^a-`%>l_2c69wXNQE@BY=z-h=O*tzCHB{p$V?i<>)JcdtEm+TS`~ zT)nc{+xh0+c5nCYqkFwek6%A}vUYyz_wx(2g+E?->+@R|&0h~Zd-(L?pGSUq=j6+8 LFSnKt^jY&C&OEy; literal 0 HcmV?d00001 diff --git a/reference-tables/v2/partial-field-ids-parquet-data/metadata/snap-8822764460534905462-1-10b4fea3-5958-4012-a791-53d0c8661288.avro b/reference-tables/v2/partial-field-ids-parquet-data/metadata/snap-8822764460534905462-1-10b4fea3-5958-4012-a791-53d0c8661288.avro new file mode 100644 index 0000000000000000000000000000000000000000..9adf2fa521bcec8754275b3333a53b7217bc7123 GIT binary patch literal 4476 zcmbVPONbmr7|w2@=pN!?!iu;tWfLWbUFn{knVlV6P~5D9EV^VOh%t^e-8DOH`ek?Z zj^na|qDGD2AsX}$5f5rWJ$T6>7x7|(2Q??z9D*0Y97IG=Z~oPfs_L2Pd3chJsO z`(Ix_(RhAh?+o7b-3J4`MQr@*!7YV$Jfc(;pXjb{D07N~ZT8#7j@cvutv5}Jz~iP# zEF(~=C%1Y|c$l`}L;@60-*h(Mx8dr5v*zlUny!PMJbo0lP1-`MH59RP)OpiTswGw3 zp6kW7h_fN1TdstC!ll?kEI<^PFA-!qsL`P$NW)ElH3`_mJ~df9^}r)~9hiCGrbBtu zHTV-9T|r?$3`AYj_D$LyZ$X09U5AqB{X_*B?9uqo0J%*>TLh)jR*|}FV2bCEwrmiK z^hB;LN)!QkAta7Y>P~1kh%e&KNK#8qq4*SPw}_J=MQt1)Y#4;yJW6V;l2($qY!mbu ziEqtqLy@5La7qiAkZ;;WRhFotQ0jq$y`bgNOhU33OHQz?&=#ig9R{$d3q35XQi8hf zhIyHKgz17Hk$wlI$0Uu^q6A4^Gy@8i9yQ27)Fa8s0l>2OaVZ$XaaW?685D1?(7xO5 zi8Z5hECD;#PXSFXdal4wP_95OuAHn?fT6Id0J*4gzIf-e@ep_ENyP~*OM$VIbS?IM z+=0KL*=?A0{CW3q^h;_bt?A56WOD$UP5@2HfK%Lt>skam@E{(FC;SkmRTE*S6OXSJ zYf5My%R)R(w27^-d+01l6)4Ce=HH?~ z1K6FDg?rP1KOgUzNp05D39O}!#96#?A8#Ho1fkAWYwS59{nU)1NcMlP_eh+D539)Z zVPjPRitw%)!)dURjajr_7Va@}crx=b*K-Jp0;k};EV+#w64@~HcORTUTjT`7cZ>i# zjt>!JB~F&+TsV*%(u`oWF&v3tEpyiXUSt&J5xof~RTnsEKUXpeWo-nPBKV43d|7{z z!zUL?uU`@1B5!<8kB!gwwrq(6g-BAmvd3mfS9o6nO>|P6kc=b9(I-=UJ#K~vuZd)H zea438`bZ8$Fo^V~@{CQ8%F_>D6=Q(U8Rl(-E=_C&d|dK*Y=LiuV_oZvKagw>pWM>N zUpz&Zc;yNc5<4Ejo|)CD8yRLs`Z*w>Xr&Kr1Vkcs`2wK!vw<$Ilsn-RDyUtn|Mpa*yw5v!RWYs*N;EHe(COWe{I~f zKfStg>$P{yoqcZ}`{D5Kd+wTYZaI7TyUWdwb{{Zqu3kO=^xKa;b@=Ozvu~YwcJBudFK1MmGjF7Z~Xo5%Uj#!52vq(ul+O89pZno_V6kI literal 0 HcmV?d00001 diff --git a/reference-tables/v2/partial-field-ids-parquet-data/metadata/v2.metadata.json b/reference-tables/v2/partial-field-ids-parquet-data/metadata/v2.metadata.json new file mode 100644 index 0000000..91888af --- /dev/null +++ b/reference-tables/v2/partial-field-ids-parquet-data/metadata/v2.metadata.json @@ -0,0 +1 @@ +{"format-version":2,"table-uuid":"c1a57077-2296-496c-91f6-b1002eb575cf","location":"file:///tmp/iceberg-verification/reference-tables/v2/partial-field-ids-parquet-data","last-sequence-number":1,"last-updated-ms":1794824816527,"last-column-id":11,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"name","required":false,"type":"string"},{"id":3,"name":"location","required":false,"type":{"type":"struct","fields":[{"id":7,"name":"lat","required":true,"type":"double"},{"id":8,"name":"lon","required":true,"type":"double"}]}},{"id":4,"name":"tags","required":false,"type":{"type":"list","element-id":9,"element":"string","element-required":false}},{"id":5,"name":"props","required":false,"type":{"type":"map","key-id":10,"key":"string","value-id":11,"value":"int","value-required":false}},{"id":6,"name":"added","required":false,"type":"string"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"schema.name-mapping.default":"[{\"field-id\":1,\"names\":[\"id\"]},{\"field-id\":2,\"names\":[\"name\"]},{\"field-id\":3,\"names\":[\"location\"],\"fields\":[{\"field-id\":7,\"names\":[\"lat\"]},{\"field-id\":8,\"names\":[\"lon\"]}]},{\"field-id\":4,\"names\":[\"tags\"],\"fields\":[{\"field-id\":9,\"names\":[\"element\"]}]},{\"field-id\":5,\"names\":[\"props\"],\"fields\":[{\"field-id\":10,\"names\":[\"key\"]},{\"field-id\":11,\"names\":[\"value\"]}]},{\"field-id\":6,\"names\":[\"added\"]}]","write.parquet.compression-codec":"zstd"},"current-snapshot-id":8822764460534905462,"refs":{"main":{"snapshot-id":8822764460534905462,"type":"branch"}},"snapshots":[{"sequence-number":1,"snapshot-id":8822764460534905462,"timestamp-ms":1794824816520,"summary":{"operation":"append","manifests-created":"1","manifests-kept":"0","manifests-replaced":"0","added-data-files":"1","added-records":"3","added-files-size":"1665","changed-partition-count":"1","total-records":"3","total-files-size":"1665","total-data-files":"1","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","iceberg-version":"Apache Iceberg 1.11.0 (commit 6976e020b894f6a6777704df2b8c4458cb291ae9)"},"manifest-list":"file:/tmp/iceberg-verification/reference-tables/v2/partial-field-ids-parquet-data/metadata/snap-8822764460534905462-1-10b4fea3-5958-4012-a791-53d0c8661288.avro","schema-id":0}],"statistics":[],"partition-statistics":[],"snapshot-log":[{"timestamp-ms":1794824816520,"snapshot-id":8822764460534905462}],"metadata-log":[{"timestamp-ms":1794824815755,"metadata-file":"file:/tmp/iceberg-verification/reference-tables/v2/partial-field-ids-parquet-data/metadata/v1.metadata.json"}]} \ No newline at end of file