Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 52 additions & 16 deletions reference-tables/GenerateTables.java
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import org.apache.iceberg.FileScanTask;
import org.apache.iceberg.HasTableOperations;
import org.apache.iceberg.IcebergBuild;
import org.apache.iceberg.MetadataColumns;
import org.apache.iceberg.PartitionKey;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.PartitionSpecParser;
Expand All @@ -63,6 +64,7 @@
import org.apache.iceberg.io.OutputFile;
import org.apache.iceberg.parquet.Parquet;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.TypeUtil;
import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.JsonUtil;

Expand Down Expand Up @@ -146,6 +148,10 @@ static void generate(HadoopTables tables, String name, JsonNode input) throws IO
updateSchema(table, commit.get("schema-updates"));
continue;
}
if (commit.has("overwrite")) {
overwrite(table, commit.get("overwrite"));
continue;
}
List<Record> records = new ArrayList<>();
for (JsonNode row : commit.get("rows")) {
records.add(record(table.schema().asStruct(), row));
Expand Down Expand Up @@ -173,20 +179,7 @@ static void appendFiles(Table table, JsonNode files) throws IOException {
/** One deletion vector for the given row positions of a live data file, in a row delta. */
static void addDeletionVector(Table table, JsonNode dv) throws IOException {
String tag = dv.get("data-file").asText();
DataFile target = null;
try (CloseableIterable<FileScanTask> tasks = table.newScan().planFiles()) {
for (FileScanTask task : tasks) {
if (task.file().location().endsWith("/data/" + tag + ".parquet")) {
if (!task.deletes().isEmpty()) {
throw new IllegalArgumentException("Data file " + tag + " already has deletes");
}
target = task.file();
}
}
}
if (target == null) {
throw new IllegalArgumentException("No live data file " + tag);
}
DataFile target = liveDataFile(table, tag);

String location = table.location() + "/data/" + tag + "-deletes.puffin";
BaseDVFileWriter writer =
Expand Down Expand Up @@ -244,8 +237,51 @@ static Literal<?> literal(Type type, JsonNode node) {
return value == null ? null : Expressions.lit(value);
}

/** A data file belongs to one partition, so every record shares the first record's value. */
/** The live data file with the given tag, which must have no deletes. */
static DataFile liveDataFile(Table table, String tag) throws IOException {
DataFile target = null;
try (CloseableIterable<FileScanTask> tasks = table.newScan().planFiles()) {
for (FileScanTask task : tasks) {
if (task.file().location().endsWith("/data/" + tag + ".parquet")) {
if (!task.deletes().isEmpty()) {
throw new IllegalArgumentException("Data file " + tag + " already has deletes");
}
target = task.file();
}
}
}
if (target == null) {
throw new IllegalArgumentException("No live data file " + tag);
}
return target;
}

/**
* Replaces a live data file with a new one in an overwrite. The new file's rows also carry the
* row lineage columns, {@code _row_id} and {@code _last_updated_sequence_number}, keyed by their
* reserved field IDs, so a row moved from the old file keeps its row ID.
*/
static void overwrite(Table table, JsonNode overwrite) throws IOException {
DataFile replaced = liveDataFile(table, overwrite.get("replaces").asText());
Schema schema =
TypeUtil.join(
table.schema(),
new Schema(MetadataColumns.ROW_ID, MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER));
List<Record> records = new ArrayList<>();
for (JsonNode row : overwrite.get("rows")) {
records.add(record(schema.asStruct(), row));
}
DataFile file = write(table, schema, overwrite.get("data-file").asText(), records);
table.newOverwrite().deleteFile(replaced).addFile(file).commit();
}

static DataFile write(Table table, String tag, List<Record> records) throws IOException {
return write(table, table.schema(), tag, records);
}

/** A data file belongs to one partition, so every record shares the first record's value. */
static DataFile write(Table table, Schema schema, String tag, List<Record> records)
throws IOException {
StructLike partition = null;
if (table.spec().isPartitioned()) {
InternalRecordWrapper wrapper = new InternalRecordWrapper(table.schema().asStruct());
Expand All @@ -265,7 +301,7 @@ static DataFile write(Table table, String tag, List<Record> records) throws IOEx
OutputFile out = table.io().newOutputFile(table.location() + "/data/" + tag + ".parquet");
DataWriter<Record> writer =
Parquet.writeData(out)
.schema(table.schema())
.schema(schema)
.createWriterFunc(GenericParquetWriter::create)
.withSpec(table.spec())
.withPartition(partition)
Expand Down
35 changes: 24 additions & 11 deletions reference-tables/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ exactly the rows in the table's `expected/scan.json`. Bytes are not compared.

## Size

A table should stay under 100 KB. The six here are 16 KB to 31 KB.
A table should stay under 100 KB. The seven here are 16 KB to 52 KB.

Size tracks commit count, and almost nothing else. A commit costs roughly 12 to
14 KB, most of it the manifest and manifest list it produces. Rows are close to
Expand Down Expand Up @@ -156,14 +156,16 @@ table apart: it states the reader behavior the table exercises, and what it
deliberately leaves out and why. The Input section gives the table's format
version, schema, partition spec, and each commit: the rows of one data file, the
data files of one append, the row positions of one data file that a deletion
vector deletes, or the updates of a schema change. The generator reads that
input, creates the table with `HadoopTables`, and commits each in turn, writing
a deletion vector with Iceberg's `BaseDVFileWriter` in a row delta. It writes
each table directly at the sentinel root, so the absolute paths are correct as
written and nothing is rewritten. After writing, it prunes earlier
`metadata.json` files and deletes Hadoop's `.crc` sidecars and
`version-hint.text`. Output is not byte-reproducible, because Iceberg assigns
random UUIDs, snapshot IDs and timestamps.
vector deletes, the updates of a schema change, or a data file that replaces
another in an overwrite. The generator reads that input, creates the table with
`HadoopTables`, and commits each in turn, writing a deletion vector with Iceberg's
`BaseDVFileWriter` in a row delta. A replacing data file's rows may also carry
the row lineage columns `_row_id` and `_last_updated_sequence_number`, keyed by
their reserved field IDs. It writes each table directly at the sentinel root,
so the absolute paths are correct as written and nothing is rewritten. After
writing, it prunes earlier `metadata.json` files and deletes Hadoop's `.crc`
sidecars and `version-hint.text`. Output is not byte-reproducible, because Iceberg
assigns random UUIDs, snapshot IDs and timestamps.

A table is added by writing its `README.md` and running the generator with the
`reference-tables` directory and the table's name as arguments, for example
Expand Down Expand Up @@ -253,6 +255,12 @@ Each table directory holds `expected/scan.json`, with one entry per snapshot:
[JSON single-value serialization](https://github.com/apache/iceberg/blob/main/format/spec.md#json-single-value-serialization):
keyed by field ID, with every nested value in the same form.

A v3 table that tests row lineage also writes the
[row lineage](https://iceberg.apache.org/spec/#row-lineage) metadata columns in
each row: `_row_id` under `"2147483540"` and `_last_updated_sequence_number`
under `"2147483539"`, both `long`. Their values are the ones a reader assigns,
by inheritance where the data file stores null or omits the column.

`snapshot-id` and every `long` value are written as strings, following the
64-bit integer rule in `CONTRIBUTING.md`. This is the one departure from the
single-value serialization, which writes a `long` as a JSON number.
Expand All @@ -264,14 +272,19 @@ list, each manifest's entries not marked `DELETED`, and the rows of each data
file those entries name, read by field ID with any field a file lacks filled from
its `initial-default`, less the positions any deletion vector that applies to
it deletes. The rows of `v3/deletion-vector-parquet-data` were instead derived
from its Input, and a PyIceberg 0.12.0 scan returns the same rows.
from its Input, and a PyIceberg 0.12.0 scan returns the same rows. The rows of
`v3/row-lineage-parquet-data` were derived from its Input and the spec's
inheritance rules, and an Iceberg Java 1.11.0 scan that projects the row lineage
columns returns the same rows.

## Procedure

Each entry in `snapshots` is checked on its own. The table is scanned at that
snapshot, as in the spec's
[Point in Time Reads](https://iceberg.apache.org/spec/#point-in-time-reads-time-travel),
through the implementation's scan API, with no filter and every column projected.
through the implementation's scan API, with no filter and every column projected,
and with `_row_id` and `_last_updated_sequence_number` projected too where `rows`
carries their field IDs.
Reading the data files directly bypasses scan planning, so it does not satisfy
the assertion.

Expand Down
8 changes: 8 additions & 0 deletions reference-tables/index.json
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,14 @@
"data-file-format": "parquet",
"metadata": "v3/initial-default-parquet-data/metadata/v5.metadata.json",
"description": "A required int added with an initial-default (42) whose write-default is then changed (10), an optional string added with an initial-default (\"none\"), and an optional string added without a default, after the first append. Two snapshots, four rows; rows written before the change read the initial-defaults and null."
},
{
"name": "v3/row-lineage-parquet-data",
"format-version": 3,
"written-by": "Iceberg Java 1.11.0",
"data-file-format": "parquet",
"metadata": "v3/row-lineage-parquet-data/metadata/v4.metadata.json",
"description": "Row lineage. Two appends assign row IDs by inheritance; an overwrite then writes some row IDs and sequence numbers explicitly and leaves others null."
}
]
}
118 changes: 118 additions & 0 deletions reference-tables/v3/row-lineage-parquet-data/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing,
~ software distributed under the License is distributed on an
~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
~ KIND, either express or implied. See the License for the
~ specific language governing permissions and limitations
~ under the License.
-->

# v3/row-lineage-parquet-data

Input to `reference-tables/GenerateTables.java`. The schema and partition spec are
the spec's JSON forms. Rows are written in the spec's single-value JSON keyed by
field ID, with `long` values as strings. The first commit appends one data file.
The second appends two data files in one snapshot. The third is an overwrite that
replaces data file `a` with `a2`, whose rows also carry the row lineage columns
`_row_id` (field ID 2147483540) and `_last_updated_sequence_number` (field ID
2147483539).

`a2` copies the three rows of `a` with their row IDs, updates the `data` of id 2,
and adds id 7. Following
[Row lineage assignment](https://iceberg.apache.org/spec/#row-lineage-assignment),
the copied rows keep their `_last_updated_sequence_number`, while the updated row
and the added row write it as null, and the added row writes a null `_row_id`.

## Scope

Row lineage: two appends that assign row IDs by inheritance, then an overwrite
that writes some row IDs and sequence numbers explicitly and leaves others null.

The two appends write six rows across three data files, so IDs must be assigned
in commit order and across files. The overwrite copies the first file's three
rows with their IDs and original last-updated sequence number, updates one of
them and adds a new one, so the reader must preserve the copied values and write
null for the changed and added rows. The metadata columns are keyed by their
reserved field IDs, 2147483540 and 2147483539, so a reader that treats them as
ordinary columns or resolves them by name returns wrong values. Other v3
features are left to other tables.

## Input

```json
{
"format-version": 3,
"description": "Row lineage. Two appends assign row IDs by inheritance; an overwrite then writes some row IDs and sequence numbers explicitly and leaves others null.",
"schema": {
"type": "struct",
"schema-id": 0,
"fields": [
{
"id": 1,
"name": "id",
"required": true,
"type": "long"
},
{
"id": 2,
"name": "data",
"required": false,
"type": "string"
}
]
},
"partition-spec": {
"spec-id": 0,
"fields": []
},
"commits": [
{
"data-file": "a",
"rows": [
{"1": "1", "2": "alpha"},
{"1": "2", "2": "bravo"},
{"1": "3", "2": "charlie"}
]
},
{
"data-files": [
{
"data-file": "b",
"rows": [
{"1": "4", "2": "delta"},
{"1": "5", "2": "echo"}
]
},
{
"data-file": "c",
"rows": [
{"1": "6", "2": "foxtrot"}
]
}
]
},
{
"overwrite": {
"replaces": "a",
"data-file": "a2",
"rows": [
{"1": "1", "2": "alpha", "2147483540": "0", "2147483539": "1"},
{"1": "2", "2": "bravo-updated", "2147483540": "1", "2147483539": null},
{"1": "3", "2": "charlie", "2147483540": "2", "2147483539": "1"},
{"1": "7", "2": "golf", "2147483540": null, "2147483539": null}
]
}
}
]
}
```
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
35 changes: 35 additions & 0 deletions reference-tables/v3/row-lineage-parquet-data/expected/scan.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
{
"snapshots": [
{
"snapshot-id": "8916293407817999201",
"rows": [
{"1": "1", "2": "alpha", "2147483540": "0", "2147483539": "1"},
{"1": "2", "2": "bravo", "2147483540": "1", "2147483539": "1"},
{"1": "3", "2": "charlie", "2147483540": "2", "2147483539": "1"}
]
},
{
"snapshot-id": "4621771802524176542",
"rows": [
{"1": "1", "2": "alpha", "2147483540": "0", "2147483539": "1"},
{"1": "2", "2": "bravo", "2147483540": "1", "2147483539": "1"},
{"1": "3", "2": "charlie", "2147483540": "2", "2147483539": "1"},
{"1": "4", "2": "delta", "2147483540": "3", "2147483539": "2"},
{"1": "5", "2": "echo", "2147483540": "4", "2147483539": "2"},
{"1": "6", "2": "foxtrot", "2147483540": "5", "2147483539": "2"}
]
},
{
"snapshot-id": "3108891484559734981",
"rows": [
{"1": "1", "2": "alpha", "2147483540": "0", "2147483539": "1"},
{"1": "2", "2": "bravo-updated", "2147483540": "1", "2147483539": "3"},
{"1": "3", "2": "charlie", "2147483540": "2", "2147483539": "1"},
{"1": "4", "2": "delta", "2147483540": "3", "2147483539": "2"},
{"1": "5", "2": "echo", "2147483540": "4", "2147483539": "2"},
{"1": "6", "2": "foxtrot", "2147483540": "5", "2147483539": "2"},
{"1": "7", "2": "golf", "2147483540": "9", "2147483539": "3"}
]
}
]
}
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"format-version":3,"table-uuid":"aceb2436-004a-4dc4-b452-a230193dbf6d","location":"file:///tmp/iceberg-verification/reference-tables/v3/row-lineage-parquet-data","last-sequence-number":3,"last-updated-ms":1794822094921,"last-column-id":2,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"data","required":false,"type":"string"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"write.parquet.compression-codec":"zstd"},"current-snapshot-id":3108891484559734981,"next-row-id":10,"refs":{"main":{"snapshot-id":3108891484559734981,"type":"branch"}},"snapshots":[{"sequence-number":1,"snapshot-id":8916293407817999201,"timestamp-ms":1794822094634,"summary":{"operation":"append","manifests-created":"1","manifests-kept":"0","manifests-replaced":"0","added-data-files":"1","added-records":"3","added-files-size":"701","changed-partition-count":"1","total-records":"3","total-files-size":"701","total-data-files":"1","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","iceberg-version":"Apache Iceberg 1.11.0 (commit 6976e020b894f6a6777704df2b8c4458cb291ae9)"},"manifest-list":"file:/tmp/iceberg-verification/reference-tables/v3/row-lineage-parquet-data/metadata/snap-8916293407817999201-1-47a4731f-1794-4c9a-bde9-7b26462653cc.avro","schema-id":0,"first-row-id":0,"added-rows":3},{"sequence-number":2,"snapshot-id":4621771802524176542,"parent-snapshot-id":8916293407817999201,"timestamp-ms":1794822094779,"summary":{"operation":"append","manifests-created":"1","manifests-kept":"1","manifests-replaced":"0","added-data-files":"2","added-records":"3","added-files-size":"1382","changed-partition-count":"1","total-records":"6","total-files-size":"2083","total-data-files":"3","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","iceberg-version":"Apache Iceberg 1.11.0 (commit 6976e020b894f6a6777704df2b8c4458cb291ae9)"},"manifest-list":"file:/tmp/iceberg-verification/reference-tables/v3/row-lineage-parquet-data/metadata/snap-4621771802524176542-1-f4cde2d4-0670-43ca-81a1-0eeff5f59e29.avro","schema-id":0,"first-row-id":3,"added-rows":3},{"sequence-number":3,"snapshot-id":3108891484559734981,"parent-snapshot-id":4621771802524176542,"timestamp-ms":1794822094920,"summary":{"operation":"overwrite","manifests-created":"2","manifests-kept":"1","manifests-replaced":"1","added-data-files":"1","deleted-data-files":"1","added-records":"4","deleted-records":"3","added-files-size":"1509","removed-files-size":"701","changed-partition-count":"1","total-records":"7","total-files-size":"2891","total-data-files":"3","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","iceberg-version":"Apache Iceberg 1.11.0 (commit 6976e020b894f6a6777704df2b8c4458cb291ae9)"},"manifest-list":"file:/tmp/iceberg-verification/reference-tables/v3/row-lineage-parquet-data/metadata/snap-3108891484559734981-1-77acae54-d569-4214-a217-6208595365e8.avro","schema-id":0,"first-row-id":6,"added-rows":4}],"statistics":[],"partition-statistics":[],"snapshot-log":[{"timestamp-ms":1794822094634,"snapshot-id":8916293407817999201},{"timestamp-ms":1794822094779,"snapshot-id":4621771802524176542},{"timestamp-ms":1794822094920,"snapshot-id":3108891484559734981}],"metadata-log":[{"timestamp-ms":1794822094133,"metadata-file":"file:/tmp/iceberg-verification/reference-tables/v3/row-lineage-parquet-data/metadata/v1.metadata.json"},{"timestamp-ms":1794822094639,"metadata-file":"file:/tmp/iceberg-verification/reference-tables/v3/row-lineage-parquet-data/metadata/v2.metadata.json"},{"timestamp-ms":1794822094779,"metadata-file":"file:/tmp/iceberg-verification/reference-tables/v3/row-lineage-parquet-data/metadata/v3.metadata.json"}]}
Loading