diff --git a/reference-tables/GenerateTables.java b/reference-tables/GenerateTables.java index 9474b5c..af4874e 100644 --- a/reference-tables/GenerateTables.java +++ b/reference-tables/GenerateTables.java @@ -39,6 +39,7 @@ import org.apache.iceberg.FileScanTask; import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.IcebergBuild; +import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.PartitionKey; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionSpecParser; @@ -63,6 +64,7 @@ import org.apache.iceberg.io.OutputFile; import org.apache.iceberg.parquet.Parquet; import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.JsonUtil; @@ -146,6 +148,10 @@ static void generate(HadoopTables tables, String name, JsonNode input) throws IO updateSchema(table, commit.get("schema-updates")); continue; } + if (commit.has("overwrite")) { + overwrite(table, commit.get("overwrite")); + continue; + } List records = new ArrayList<>(); for (JsonNode row : commit.get("rows")) { records.add(record(table.schema().asStruct(), row)); @@ -173,20 +179,7 @@ static void appendFiles(Table table, JsonNode files) throws IOException { /** One deletion vector for the given row positions of a live data file, in a row delta. */ static void addDeletionVector(Table table, JsonNode dv) throws IOException { String tag = dv.get("data-file").asText(); - DataFile target = null; - try (CloseableIterable tasks = table.newScan().planFiles()) { - for (FileScanTask task : tasks) { - if (task.file().location().endsWith("/data/" + tag + ".parquet")) { - if (!task.deletes().isEmpty()) { - throw new IllegalArgumentException("Data file " + tag + " already has deletes"); - } - target = task.file(); - } - } - } - if (target == null) { - throw new IllegalArgumentException("No live data file " + tag); - } + DataFile target = liveDataFile(table, tag); String location = table.location() + "/data/" + tag + "-deletes.puffin"; BaseDVFileWriter writer = @@ -244,8 +237,51 @@ static Literal literal(Type type, JsonNode node) { return value == null ? null : Expressions.lit(value); } - /** A data file belongs to one partition, so every record shares the first record's value. */ + /** The live data file with the given tag, which must have no deletes. */ + static DataFile liveDataFile(Table table, String tag) throws IOException { + DataFile target = null; + try (CloseableIterable tasks = table.newScan().planFiles()) { + for (FileScanTask task : tasks) { + if (task.file().location().endsWith("/data/" + tag + ".parquet")) { + if (!task.deletes().isEmpty()) { + throw new IllegalArgumentException("Data file " + tag + " already has deletes"); + } + target = task.file(); + } + } + } + if (target == null) { + throw new IllegalArgumentException("No live data file " + tag); + } + return target; + } + + /** + * Replaces a live data file with a new one in an overwrite. The new file's rows also carry the + * row lineage columns, {@code _row_id} and {@code _last_updated_sequence_number}, keyed by their + * reserved field IDs, so a row moved from the old file keeps its row ID. + */ + static void overwrite(Table table, JsonNode overwrite) throws IOException { + DataFile replaced = liveDataFile(table, overwrite.get("replaces").asText()); + Schema schema = + TypeUtil.join( + table.schema(), + new Schema(MetadataColumns.ROW_ID, MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER)); + List records = new ArrayList<>(); + for (JsonNode row : overwrite.get("rows")) { + records.add(record(schema.asStruct(), row)); + } + DataFile file = write(table, schema, overwrite.get("data-file").asText(), records); + table.newOverwrite().deleteFile(replaced).addFile(file).commit(); + } + static DataFile write(Table table, String tag, List records) throws IOException { + return write(table, table.schema(), tag, records); + } + + /** A data file belongs to one partition, so every record shares the first record's value. */ + static DataFile write(Table table, Schema schema, String tag, List records) + throws IOException { StructLike partition = null; if (table.spec().isPartitioned()) { InternalRecordWrapper wrapper = new InternalRecordWrapper(table.schema().asStruct()); @@ -265,7 +301,7 @@ static DataFile write(Table table, String tag, List records) throws IOEx OutputFile out = table.io().newOutputFile(table.location() + "/data/" + tag + ".parquet"); DataWriter writer = Parquet.writeData(out) - .schema(table.schema()) + .schema(schema) .createWriterFunc(GenericParquetWriter::create) .withSpec(table.spec()) .withPartition(partition) diff --git a/reference-tables/README.md b/reference-tables/README.md index 585b117..5580504 100644 --- a/reference-tables/README.md +++ b/reference-tables/README.md @@ -34,7 +34,7 @@ exactly the rows in the table's `expected/scan.json`. Bytes are not compared. ## Size -A table should stay under 100 KB. The six here are 16 KB to 31 KB. +A table should stay under 100 KB. The seven here are 16 KB to 52 KB. Size tracks commit count, and almost nothing else. A commit costs roughly 12 to 14 KB, most of it the manifest and manifest list it produces. Rows are close to @@ -156,14 +156,16 @@ table apart: it states the reader behavior the table exercises, and what it deliberately leaves out and why. The Input section gives the table's format version, schema, partition spec, and each commit: the rows of one data file, the data files of one append, the row positions of one data file that a deletion -vector deletes, or the updates of a schema change. The generator reads that -input, creates the table with `HadoopTables`, and commits each in turn, writing -a deletion vector with Iceberg's `BaseDVFileWriter` in a row delta. It writes -each table directly at the sentinel root, so the absolute paths are correct as -written and nothing is rewritten. After writing, it prunes earlier -`metadata.json` files and deletes Hadoop's `.crc` sidecars and -`version-hint.text`. Output is not byte-reproducible, because Iceberg assigns -random UUIDs, snapshot IDs and timestamps. +vector deletes, the updates of a schema change, or a data file that replaces +another in an overwrite. The generator reads that input, creates the table with +`HadoopTables`, and commits each in turn, writing a deletion vector with Iceberg's +`BaseDVFileWriter` in a row delta. A replacing data file's rows may also carry +the row lineage columns `_row_id` and `_last_updated_sequence_number`, keyed by +their reserved field IDs. It writes each table directly at the sentinel root, +so the absolute paths are correct as written and nothing is rewritten. After +writing, it prunes earlier `metadata.json` files and deletes Hadoop's `.crc` +sidecars and `version-hint.text`. Output is not byte-reproducible, because Iceberg +assigns random UUIDs, snapshot IDs and timestamps. A table is added by writing its `README.md` and running the generator with the `reference-tables` directory and the table's name as arguments, for example @@ -253,6 +255,12 @@ Each table directory holds `expected/scan.json`, with one entry per snapshot: [JSON single-value serialization](https://github.com/apache/iceberg/blob/main/format/spec.md#json-single-value-serialization): keyed by field ID, with every nested value in the same form. +A v3 table that tests row lineage also writes the +[row lineage](https://iceberg.apache.org/spec/#row-lineage) metadata columns in +each row: `_row_id` under `"2147483540"` and `_last_updated_sequence_number` +under `"2147483539"`, both `long`. Their values are the ones a reader assigns, +by inheritance where the data file stores null or omits the column. + `snapshot-id` and every `long` value are written as strings, following the 64-bit integer rule in `CONTRIBUTING.md`. This is the one departure from the single-value serialization, which writes a `long` as a JSON number. @@ -264,14 +272,19 @@ list, each manifest's entries not marked `DELETED`, and the rows of each data file those entries name, read by field ID with any field a file lacks filled from its `initial-default`, less the positions any deletion vector that applies to it deletes. The rows of `v3/deletion-vector-parquet-data` were instead derived -from its Input, and a PyIceberg 0.12.0 scan returns the same rows. +from its Input, and a PyIceberg 0.12.0 scan returns the same rows. The rows of +`v3/row-lineage-parquet-data` were derived from its Input and the spec's +inheritance rules, and an Iceberg Java 1.11.0 scan that projects the row lineage +columns returns the same rows. ## Procedure Each entry in `snapshots` is checked on its own. The table is scanned at that snapshot, as in the spec's [Point in Time Reads](https://iceberg.apache.org/spec/#point-in-time-reads-time-travel), -through the implementation's scan API, with no filter and every column projected. +through the implementation's scan API, with no filter and every column projected, +and with `_row_id` and `_last_updated_sequence_number` projected too where `rows` +carries their field IDs. Reading the data files directly bypasses scan planning, so it does not satisfy the assertion. diff --git a/reference-tables/index.json b/reference-tables/index.json index 3904f09..86df973 100644 --- a/reference-tables/index.json +++ b/reference-tables/index.json @@ -47,6 +47,14 @@ "data-file-format": "parquet", "metadata": "v3/initial-default-parquet-data/metadata/v5.metadata.json", "description": "A required int added with an initial-default (42) whose write-default is then changed (10), an optional string added with an initial-default (\"none\"), and an optional string added without a default, after the first append. Two snapshots, four rows; rows written before the change read the initial-defaults and null." + }, + { + "name": "v3/row-lineage-parquet-data", + "format-version": 3, + "written-by": "Iceberg Java 1.11.0", + "data-file-format": "parquet", + "metadata": "v3/row-lineage-parquet-data/metadata/v4.metadata.json", + "description": "Row lineage. Two appends assign row IDs by inheritance; an overwrite then writes some row IDs and sequence numbers explicitly and leaves others null." } ] } diff --git a/reference-tables/v3/row-lineage-parquet-data/README.md b/reference-tables/v3/row-lineage-parquet-data/README.md new file mode 100644 index 0000000..6815906 --- /dev/null +++ b/reference-tables/v3/row-lineage-parquet-data/README.md @@ -0,0 +1,118 @@ + + +# v3/row-lineage-parquet-data + +Input to `reference-tables/GenerateTables.java`. The schema and partition spec are +the spec's JSON forms. Rows are written in the spec's single-value JSON keyed by +field ID, with `long` values as strings. The first commit appends one data file. +The second appends two data files in one snapshot. The third is an overwrite that +replaces data file `a` with `a2`, whose rows also carry the row lineage columns +`_row_id` (field ID 2147483540) and `_last_updated_sequence_number` (field ID +2147483539). + +`a2` copies the three rows of `a` with their row IDs, updates the `data` of id 2, +and adds id 7. Following +[Row lineage assignment](https://iceberg.apache.org/spec/#row-lineage-assignment), +the copied rows keep their `_last_updated_sequence_number`, while the updated row +and the added row write it as null, and the added row writes a null `_row_id`. + +## Scope + +Row lineage: two appends that assign row IDs by inheritance, then an overwrite +that writes some row IDs and sequence numbers explicitly and leaves others null. + +The two appends write six rows across three data files, so IDs must be assigned +in commit order and across files. The overwrite copies the first file's three +rows with their IDs and original last-updated sequence number, updates one of +them and adds a new one, so the reader must preserve the copied values and write +null for the changed and added rows. The metadata columns are keyed by their +reserved field IDs, 2147483540 and 2147483539, so a reader that treats them as +ordinary columns or resolves them by name returns wrong values. Other v3 +features are left to other tables. + +## Input + +```json +{ + "format-version": 3, + "description": "Row lineage. Two appends assign row IDs by inheritance; an overwrite then writes some row IDs and sequence numbers explicitly and leaves others null.", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [ + { + "id": 1, + "name": "id", + "required": true, + "type": "long" + }, + { + "id": 2, + "name": "data", + "required": false, + "type": "string" + } + ] + }, + "partition-spec": { + "spec-id": 0, + "fields": [] + }, + "commits": [ + { + "data-file": "a", + "rows": [ + {"1": "1", "2": "alpha"}, + {"1": "2", "2": "bravo"}, + {"1": "3", "2": "charlie"} + ] + }, + { + "data-files": [ + { + "data-file": "b", + "rows": [ + {"1": "4", "2": "delta"}, + {"1": "5", "2": "echo"} + ] + }, + { + "data-file": "c", + "rows": [ + {"1": "6", "2": "foxtrot"} + ] + } + ] + }, + { + "overwrite": { + "replaces": "a", + "data-file": "a2", + "rows": [ + {"1": "1", "2": "alpha", "2147483540": "0", "2147483539": "1"}, + {"1": "2", "2": "bravo-updated", "2147483540": "1", "2147483539": null}, + {"1": "3", "2": "charlie", "2147483540": "2", "2147483539": "1"}, + {"1": "7", "2": "golf", "2147483540": null, "2147483539": null} + ] + } + } + ] +} +``` diff --git a/reference-tables/v3/row-lineage-parquet-data/data/a.parquet b/reference-tables/v3/row-lineage-parquet-data/data/a.parquet new file mode 100644 index 0000000..f201779 Binary files /dev/null and b/reference-tables/v3/row-lineage-parquet-data/data/a.parquet differ diff --git a/reference-tables/v3/row-lineage-parquet-data/data/a2.parquet b/reference-tables/v3/row-lineage-parquet-data/data/a2.parquet new file mode 100644 index 0000000..1dcee75 Binary files /dev/null and b/reference-tables/v3/row-lineage-parquet-data/data/a2.parquet differ diff --git a/reference-tables/v3/row-lineage-parquet-data/data/b.parquet b/reference-tables/v3/row-lineage-parquet-data/data/b.parquet new file mode 100644 index 0000000..5fffbc2 Binary files /dev/null and b/reference-tables/v3/row-lineage-parquet-data/data/b.parquet differ diff --git a/reference-tables/v3/row-lineage-parquet-data/data/c.parquet b/reference-tables/v3/row-lineage-parquet-data/data/c.parquet new file mode 100644 index 0000000..4d8a376 Binary files /dev/null and b/reference-tables/v3/row-lineage-parquet-data/data/c.parquet differ diff --git a/reference-tables/v3/row-lineage-parquet-data/expected/scan.json b/reference-tables/v3/row-lineage-parquet-data/expected/scan.json new file mode 100644 index 0000000..ed00a99 --- /dev/null +++ b/reference-tables/v3/row-lineage-parquet-data/expected/scan.json @@ -0,0 +1,35 @@ +{ + "snapshots": [ + { + "snapshot-id": "8916293407817999201", + "rows": [ + {"1": "1", "2": "alpha", "2147483540": "0", "2147483539": "1"}, + {"1": "2", "2": "bravo", "2147483540": "1", "2147483539": "1"}, + {"1": "3", "2": "charlie", "2147483540": "2", "2147483539": "1"} + ] + }, + { + "snapshot-id": "4621771802524176542", + "rows": [ + {"1": "1", "2": "alpha", "2147483540": "0", "2147483539": "1"}, + {"1": "2", "2": "bravo", "2147483540": "1", "2147483539": "1"}, + {"1": "3", "2": "charlie", "2147483540": "2", "2147483539": "1"}, + {"1": "4", "2": "delta", "2147483540": "3", "2147483539": "2"}, + {"1": "5", "2": "echo", "2147483540": "4", "2147483539": "2"}, + {"1": "6", "2": "foxtrot", "2147483540": "5", "2147483539": "2"} + ] + }, + { + "snapshot-id": "3108891484559734981", + "rows": [ + {"1": "1", "2": "alpha", "2147483540": "0", "2147483539": "1"}, + {"1": "2", "2": "bravo-updated", "2147483540": "1", "2147483539": "3"}, + {"1": "3", "2": "charlie", "2147483540": "2", "2147483539": "1"}, + {"1": "4", "2": "delta", "2147483540": "3", "2147483539": "2"}, + {"1": "5", "2": "echo", "2147483540": "4", "2147483539": "2"}, + {"1": "6", "2": "foxtrot", "2147483540": "5", "2147483539": "2"}, + {"1": "7", "2": "golf", "2147483540": "9", "2147483539": "3"} + ] + } + ] +} diff --git a/reference-tables/v3/row-lineage-parquet-data/metadata/47a4731f-1794-4c9a-bde9-7b26462653cc-m0.avro b/reference-tables/v3/row-lineage-parquet-data/metadata/47a4731f-1794-4c9a-bde9-7b26462653cc-m0.avro new file mode 100644 index 0000000..7409b62 Binary files /dev/null and b/reference-tables/v3/row-lineage-parquet-data/metadata/47a4731f-1794-4c9a-bde9-7b26462653cc-m0.avro differ diff --git a/reference-tables/v3/row-lineage-parquet-data/metadata/77acae54-d569-4214-a217-6208595365e8-m0.avro b/reference-tables/v3/row-lineage-parquet-data/metadata/77acae54-d569-4214-a217-6208595365e8-m0.avro new file mode 100644 index 0000000..8a378de Binary files /dev/null and b/reference-tables/v3/row-lineage-parquet-data/metadata/77acae54-d569-4214-a217-6208595365e8-m0.avro differ diff --git a/reference-tables/v3/row-lineage-parquet-data/metadata/77acae54-d569-4214-a217-6208595365e8-m1.avro b/reference-tables/v3/row-lineage-parquet-data/metadata/77acae54-d569-4214-a217-6208595365e8-m1.avro new file mode 100644 index 0000000..3f9d3a7 Binary files /dev/null and b/reference-tables/v3/row-lineage-parquet-data/metadata/77acae54-d569-4214-a217-6208595365e8-m1.avro differ diff --git a/reference-tables/v3/row-lineage-parquet-data/metadata/f4cde2d4-0670-43ca-81a1-0eeff5f59e29-m0.avro b/reference-tables/v3/row-lineage-parquet-data/metadata/f4cde2d4-0670-43ca-81a1-0eeff5f59e29-m0.avro new file mode 100644 index 0000000..395a08a Binary files /dev/null and b/reference-tables/v3/row-lineage-parquet-data/metadata/f4cde2d4-0670-43ca-81a1-0eeff5f59e29-m0.avro differ diff --git a/reference-tables/v3/row-lineage-parquet-data/metadata/snap-3108891484559734981-1-77acae54-d569-4214-a217-6208595365e8.avro b/reference-tables/v3/row-lineage-parquet-data/metadata/snap-3108891484559734981-1-77acae54-d569-4214-a217-6208595365e8.avro new file mode 100644 index 0000000..5de29e8 Binary files /dev/null and b/reference-tables/v3/row-lineage-parquet-data/metadata/snap-3108891484559734981-1-77acae54-d569-4214-a217-6208595365e8.avro differ diff --git a/reference-tables/v3/row-lineage-parquet-data/metadata/snap-4621771802524176542-1-f4cde2d4-0670-43ca-81a1-0eeff5f59e29.avro b/reference-tables/v3/row-lineage-parquet-data/metadata/snap-4621771802524176542-1-f4cde2d4-0670-43ca-81a1-0eeff5f59e29.avro new file mode 100644 index 0000000..fb5ac0d Binary files /dev/null and b/reference-tables/v3/row-lineage-parquet-data/metadata/snap-4621771802524176542-1-f4cde2d4-0670-43ca-81a1-0eeff5f59e29.avro differ diff --git a/reference-tables/v3/row-lineage-parquet-data/metadata/snap-8916293407817999201-1-47a4731f-1794-4c9a-bde9-7b26462653cc.avro b/reference-tables/v3/row-lineage-parquet-data/metadata/snap-8916293407817999201-1-47a4731f-1794-4c9a-bde9-7b26462653cc.avro new file mode 100644 index 0000000..04cf69d Binary files /dev/null and b/reference-tables/v3/row-lineage-parquet-data/metadata/snap-8916293407817999201-1-47a4731f-1794-4c9a-bde9-7b26462653cc.avro differ diff --git a/reference-tables/v3/row-lineage-parquet-data/metadata/v4.metadata.json b/reference-tables/v3/row-lineage-parquet-data/metadata/v4.metadata.json new file mode 100644 index 0000000..3e8e17d --- /dev/null +++ b/reference-tables/v3/row-lineage-parquet-data/metadata/v4.metadata.json @@ -0,0 +1 @@ +{"format-version":3,"table-uuid":"aceb2436-004a-4dc4-b452-a230193dbf6d","location":"file:///tmp/iceberg-verification/reference-tables/v3/row-lineage-parquet-data","last-sequence-number":3,"last-updated-ms":1794822094921,"last-column-id":2,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"data","required":false,"type":"string"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"write.parquet.compression-codec":"zstd"},"current-snapshot-id":3108891484559734981,"next-row-id":10,"refs":{"main":{"snapshot-id":3108891484559734981,"type":"branch"}},"snapshots":[{"sequence-number":1,"snapshot-id":8916293407817999201,"timestamp-ms":1794822094634,"summary":{"operation":"append","manifests-created":"1","manifests-kept":"0","manifests-replaced":"0","added-data-files":"1","added-records":"3","added-files-size":"701","changed-partition-count":"1","total-records":"3","total-files-size":"701","total-data-files":"1","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","iceberg-version":"Apache Iceberg 1.11.0 (commit 6976e020b894f6a6777704df2b8c4458cb291ae9)"},"manifest-list":"file:/tmp/iceberg-verification/reference-tables/v3/row-lineage-parquet-data/metadata/snap-8916293407817999201-1-47a4731f-1794-4c9a-bde9-7b26462653cc.avro","schema-id":0,"first-row-id":0,"added-rows":3},{"sequence-number":2,"snapshot-id":4621771802524176542,"parent-snapshot-id":8916293407817999201,"timestamp-ms":1794822094779,"summary":{"operation":"append","manifests-created":"1","manifests-kept":"1","manifests-replaced":"0","added-data-files":"2","added-records":"3","added-files-size":"1382","changed-partition-count":"1","total-records":"6","total-files-size":"2083","total-data-files":"3","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","iceberg-version":"Apache Iceberg 1.11.0 (commit 6976e020b894f6a6777704df2b8c4458cb291ae9)"},"manifest-list":"file:/tmp/iceberg-verification/reference-tables/v3/row-lineage-parquet-data/metadata/snap-4621771802524176542-1-f4cde2d4-0670-43ca-81a1-0eeff5f59e29.avro","schema-id":0,"first-row-id":3,"added-rows":3},{"sequence-number":3,"snapshot-id":3108891484559734981,"parent-snapshot-id":4621771802524176542,"timestamp-ms":1794822094920,"summary":{"operation":"overwrite","manifests-created":"2","manifests-kept":"1","manifests-replaced":"1","added-data-files":"1","deleted-data-files":"1","added-records":"4","deleted-records":"3","added-files-size":"1509","removed-files-size":"701","changed-partition-count":"1","total-records":"7","total-files-size":"2891","total-data-files":"3","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","iceberg-version":"Apache Iceberg 1.11.0 (commit 6976e020b894f6a6777704df2b8c4458cb291ae9)"},"manifest-list":"file:/tmp/iceberg-verification/reference-tables/v3/row-lineage-parquet-data/metadata/snap-3108891484559734981-1-77acae54-d569-4214-a217-6208595365e8.avro","schema-id":0,"first-row-id":6,"added-rows":4}],"statistics":[],"partition-statistics":[],"snapshot-log":[{"timestamp-ms":1794822094634,"snapshot-id":8916293407817999201},{"timestamp-ms":1794822094779,"snapshot-id":4621771802524176542},{"timestamp-ms":1794822094920,"snapshot-id":3108891484559734981}],"metadata-log":[{"timestamp-ms":1794822094133,"metadata-file":"file:/tmp/iceberg-verification/reference-tables/v3/row-lineage-parquet-data/metadata/v1.metadata.json"},{"timestamp-ms":1794822094639,"metadata-file":"file:/tmp/iceberg-verification/reference-tables/v3/row-lineage-parquet-data/metadata/v2.metadata.json"},{"timestamp-ms":1794822094779,"metadata-file":"file:/tmp/iceberg-verification/reference-tables/v3/row-lineage-parquet-data/metadata/v3.metadata.json"}]} \ No newline at end of file