From 8e0a88cd60fd1ae89b91caecc90771e18259e1ff Mon Sep 17 00:00:00 2001 From: Sylwester Lachiewicz Date: Mon, 16 Nov 2026 10:44:20 +0100 Subject: [PATCH] Add reference table for V3 row lineage --- reference-tables/GenerateTables.java | 68 +++++++--- reference-tables/README.md | 35 ++++-- reference-tables/index.json | 8 ++ .../v3/row-lineage-parquet-data/README.md | 118 ++++++++++++++++++ .../row-lineage-parquet-data/data/a.parquet | Bin 0 -> 701 bytes .../row-lineage-parquet-data/data/a2.parquet | Bin 0 -> 1509 bytes .../row-lineage-parquet-data/data/b.parquet | Bin 0 -> 681 bytes .../row-lineage-parquet-data/data/c.parquet | Bin 0 -> 701 bytes .../expected/scan.json | 35 ++++++ ...a4731f-1794-4c9a-bde9-7b26462653cc-m0.avro | Bin 0 -> 7838 bytes ...acae54-d569-4214-a217-6208595365e8-m0.avro | Bin 0 -> 7838 bytes ...acae54-d569-4214-a217-6208595365e8-m1.avro | Bin 0 -> 7874 bytes ...cde2d4-0670-43ca-81a1-0eeff5f59e29-m0.avro | Bin 0 -> 7871 bytes ...-77acae54-d569-4214-a217-6208595365e8.avro | Bin 0 -> 4866 bytes ...-f4cde2d4-0670-43ca-81a1-0eeff5f59e29.avro | Bin 0 -> 4839 bytes ...-47a4731f-1794-4c9a-bde9-7b26462653cc.avro | Bin 0 -> 4765 bytes .../metadata/v4.metadata.json | 1 + 17 files changed, 238 insertions(+), 27 deletions(-) create mode 100644 reference-tables/v3/row-lineage-parquet-data/README.md create mode 100644 reference-tables/v3/row-lineage-parquet-data/data/a.parquet create mode 100644 reference-tables/v3/row-lineage-parquet-data/data/a2.parquet create mode 100644 reference-tables/v3/row-lineage-parquet-data/data/b.parquet create mode 100644 reference-tables/v3/row-lineage-parquet-data/data/c.parquet create mode 100644 reference-tables/v3/row-lineage-parquet-data/expected/scan.json create mode 100644 reference-tables/v3/row-lineage-parquet-data/metadata/47a4731f-1794-4c9a-bde9-7b26462653cc-m0.avro create mode 100644 reference-tables/v3/row-lineage-parquet-data/metadata/77acae54-d569-4214-a217-6208595365e8-m0.avro create mode 100644 reference-tables/v3/row-lineage-parquet-data/metadata/77acae54-d569-4214-a217-6208595365e8-m1.avro create mode 100644 reference-tables/v3/row-lineage-parquet-data/metadata/f4cde2d4-0670-43ca-81a1-0eeff5f59e29-m0.avro create mode 100644 reference-tables/v3/row-lineage-parquet-data/metadata/snap-3108891484559734981-1-77acae54-d569-4214-a217-6208595365e8.avro create mode 100644 reference-tables/v3/row-lineage-parquet-data/metadata/snap-4621771802524176542-1-f4cde2d4-0670-43ca-81a1-0eeff5f59e29.avro create mode 100644 reference-tables/v3/row-lineage-parquet-data/metadata/snap-8916293407817999201-1-47a4731f-1794-4c9a-bde9-7b26462653cc.avro create mode 100644 reference-tables/v3/row-lineage-parquet-data/metadata/v4.metadata.json diff --git a/reference-tables/GenerateTables.java b/reference-tables/GenerateTables.java index 9474b5c..af4874e 100644 --- a/reference-tables/GenerateTables.java +++ b/reference-tables/GenerateTables.java @@ -39,6 +39,7 @@ import org.apache.iceberg.FileScanTask; import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.IcebergBuild; +import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.PartitionKey; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionSpecParser; @@ -63,6 +64,7 @@ import org.apache.iceberg.io.OutputFile; import org.apache.iceberg.parquet.Parquet; import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.JsonUtil; @@ -146,6 +148,10 @@ static void generate(HadoopTables tables, String name, JsonNode input) throws IO updateSchema(table, commit.get("schema-updates")); continue; } + if (commit.has("overwrite")) { + overwrite(table, commit.get("overwrite")); + continue; + } List records = new ArrayList<>(); for (JsonNode row : commit.get("rows")) { records.add(record(table.schema().asStruct(), row)); @@ -173,20 +179,7 @@ static void appendFiles(Table table, JsonNode files) throws IOException { /** One deletion vector for the given row positions of a live data file, in a row delta. */ static void addDeletionVector(Table table, JsonNode dv) throws IOException { String tag = dv.get("data-file").asText(); - DataFile target = null; - try (CloseableIterable tasks = table.newScan().planFiles()) { - for (FileScanTask task : tasks) { - if (task.file().location().endsWith("/data/" + tag + ".parquet")) { - if (!task.deletes().isEmpty()) { - throw new IllegalArgumentException("Data file " + tag + " already has deletes"); - } - target = task.file(); - } - } - } - if (target == null) { - throw new IllegalArgumentException("No live data file " + tag); - } + DataFile target = liveDataFile(table, tag); String location = table.location() + "/data/" + tag + "-deletes.puffin"; BaseDVFileWriter writer = @@ -244,8 +237,51 @@ static Literal literal(Type type, JsonNode node) { return value == null ? null : Expressions.lit(value); } - /** A data file belongs to one partition, so every record shares the first record's value. */ + /** The live data file with the given tag, which must have no deletes. */ + static DataFile liveDataFile(Table table, String tag) throws IOException { + DataFile target = null; + try (CloseableIterable tasks = table.newScan().planFiles()) { + for (FileScanTask task : tasks) { + if (task.file().location().endsWith("/data/" + tag + ".parquet")) { + if (!task.deletes().isEmpty()) { + throw new IllegalArgumentException("Data file " + tag + " already has deletes"); + } + target = task.file(); + } + } + } + if (target == null) { + throw new IllegalArgumentException("No live data file " + tag); + } + return target; + } + + /** + * Replaces a live data file with a new one in an overwrite. The new file's rows also carry the + * row lineage columns, {@code _row_id} and {@code _last_updated_sequence_number}, keyed by their + * reserved field IDs, so a row moved from the old file keeps its row ID. + */ + static void overwrite(Table table, JsonNode overwrite) throws IOException { + DataFile replaced = liveDataFile(table, overwrite.get("replaces").asText()); + Schema schema = + TypeUtil.join( + table.schema(), + new Schema(MetadataColumns.ROW_ID, MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER)); + List records = new ArrayList<>(); + for (JsonNode row : overwrite.get("rows")) { + records.add(record(schema.asStruct(), row)); + } + DataFile file = write(table, schema, overwrite.get("data-file").asText(), records); + table.newOverwrite().deleteFile(replaced).addFile(file).commit(); + } + static DataFile write(Table table, String tag, List records) throws IOException { + return write(table, table.schema(), tag, records); + } + + /** A data file belongs to one partition, so every record shares the first record's value. */ + static DataFile write(Table table, Schema schema, String tag, List records) + throws IOException { StructLike partition = null; if (table.spec().isPartitioned()) { InternalRecordWrapper wrapper = new InternalRecordWrapper(table.schema().asStruct()); @@ -265,7 +301,7 @@ static DataFile write(Table table, String tag, List records) throws IOEx OutputFile out = table.io().newOutputFile(table.location() + "/data/" + tag + ".parquet"); DataWriter writer = Parquet.writeData(out) - .schema(table.schema()) + .schema(schema) .createWriterFunc(GenericParquetWriter::create) .withSpec(table.spec()) .withPartition(partition) diff --git a/reference-tables/README.md b/reference-tables/README.md index 585b117..5580504 100644 --- a/reference-tables/README.md +++ b/reference-tables/README.md @@ -34,7 +34,7 @@ exactly the rows in the table's `expected/scan.json`. Bytes are not compared. ## Size -A table should stay under 100 KB. The six here are 16 KB to 31 KB. +A table should stay under 100 KB. The seven here are 16 KB to 52 KB. Size tracks commit count, and almost nothing else. A commit costs roughly 12 to 14 KB, most of it the manifest and manifest list it produces. Rows are close to @@ -156,14 +156,16 @@ table apart: it states the reader behavior the table exercises, and what it deliberately leaves out and why. The Input section gives the table's format version, schema, partition spec, and each commit: the rows of one data file, the data files of one append, the row positions of one data file that a deletion -vector deletes, or the updates of a schema change. The generator reads that -input, creates the table with `HadoopTables`, and commits each in turn, writing -a deletion vector with Iceberg's `BaseDVFileWriter` in a row delta. It writes -each table directly at the sentinel root, so the absolute paths are correct as -written and nothing is rewritten. After writing, it prunes earlier -`metadata.json` files and deletes Hadoop's `.crc` sidecars and -`version-hint.text`. Output is not byte-reproducible, because Iceberg assigns -random UUIDs, snapshot IDs and timestamps. +vector deletes, the updates of a schema change, or a data file that replaces +another in an overwrite. The generator reads that input, creates the table with +`HadoopTables`, and commits each in turn, writing a deletion vector with Iceberg's +`BaseDVFileWriter` in a row delta. A replacing data file's rows may also carry +the row lineage columns `_row_id` and `_last_updated_sequence_number`, keyed by +their reserved field IDs. It writes each table directly at the sentinel root, +so the absolute paths are correct as written and nothing is rewritten. After +writing, it prunes earlier `metadata.json` files and deletes Hadoop's `.crc` +sidecars and `version-hint.text`. Output is not byte-reproducible, because Iceberg +assigns random UUIDs, snapshot IDs and timestamps. A table is added by writing its `README.md` and running the generator with the `reference-tables` directory and the table's name as arguments, for example @@ -253,6 +255,12 @@ Each table directory holds `expected/scan.json`, with one entry per snapshot: [JSON single-value serialization](https://github.com/apache/iceberg/blob/main/format/spec.md#json-single-value-serialization): keyed by field ID, with every nested value in the same form. +A v3 table that tests row lineage also writes the +[row lineage](https://iceberg.apache.org/spec/#row-lineage) metadata columns in +each row: `_row_id` under `"2147483540"` and `_last_updated_sequence_number` +under `"2147483539"`, both `long`. Their values are the ones a reader assigns, +by inheritance where the data file stores null or omits the column. + `snapshot-id` and every `long` value are written as strings, following the 64-bit integer rule in `CONTRIBUTING.md`. This is the one departure from the single-value serialization, which writes a `long` as a JSON number. @@ -264,14 +272,19 @@ list, each manifest's entries not marked `DELETED`, and the rows of each data file those entries name, read by field ID with any field a file lacks filled from its `initial-default`, less the positions any deletion vector that applies to it deletes. The rows of `v3/deletion-vector-parquet-data` were instead derived -from its Input, and a PyIceberg 0.12.0 scan returns the same rows. +from its Input, and a PyIceberg 0.12.0 scan returns the same rows. The rows of +`v3/row-lineage-parquet-data` were derived from its Input and the spec's +inheritance rules, and an Iceberg Java 1.11.0 scan that projects the row lineage +columns returns the same rows. ## Procedure Each entry in `snapshots` is checked on its own. The table is scanned at that snapshot, as in the spec's [Point in Time Reads](https://iceberg.apache.org/spec/#point-in-time-reads-time-travel), -through the implementation's scan API, with no filter and every column projected. +through the implementation's scan API, with no filter and every column projected, +and with `_row_id` and `_last_updated_sequence_number` projected too where `rows` +carries their field IDs. Reading the data files directly bypasses scan planning, so it does not satisfy the assertion. diff --git a/reference-tables/index.json b/reference-tables/index.json index 3904f09..86df973 100644 --- a/reference-tables/index.json +++ b/reference-tables/index.json @@ -47,6 +47,14 @@ "data-file-format": "parquet", "metadata": "v3/initial-default-parquet-data/metadata/v5.metadata.json", "description": "A required int added with an initial-default (42) whose write-default is then changed (10), an optional string added with an initial-default (\"none\"), and an optional string added without a default, after the first append. Two snapshots, four rows; rows written before the change read the initial-defaults and null." + }, + { + "name": "v3/row-lineage-parquet-data", + "format-version": 3, + "written-by": "Iceberg Java 1.11.0", + "data-file-format": "parquet", + "metadata": "v3/row-lineage-parquet-data/metadata/v4.metadata.json", + "description": "Row lineage. Two appends assign row IDs by inheritance; an overwrite then writes some row IDs and sequence numbers explicitly and leaves others null." } ] } diff --git a/reference-tables/v3/row-lineage-parquet-data/README.md b/reference-tables/v3/row-lineage-parquet-data/README.md new file mode 100644 index 0000000..6815906 --- /dev/null +++ b/reference-tables/v3/row-lineage-parquet-data/README.md @@ -0,0 +1,118 @@ + + +# v3/row-lineage-parquet-data + +Input to `reference-tables/GenerateTables.java`. The schema and partition spec are +the spec's JSON forms. Rows are written in the spec's single-value JSON keyed by +field ID, with `long` values as strings. The first commit appends one data file. +The second appends two data files in one snapshot. The third is an overwrite that +replaces data file `a` with `a2`, whose rows also carry the row lineage columns +`_row_id` (field ID 2147483540) and `_last_updated_sequence_number` (field ID +2147483539). + +`a2` copies the three rows of `a` with their row IDs, updates the `data` of id 2, +and adds id 7. Following +[Row lineage assignment](https://iceberg.apache.org/spec/#row-lineage-assignment), +the copied rows keep their `_last_updated_sequence_number`, while the updated row +and the added row write it as null, and the added row writes a null `_row_id`. + +## Scope + +Row lineage: two appends that assign row IDs by inheritance, then an overwrite +that writes some row IDs and sequence numbers explicitly and leaves others null. + +The two appends write six rows across three data files, so IDs must be assigned +in commit order and across files. The overwrite copies the first file's three +rows with their IDs and original last-updated sequence number, updates one of +them and adds a new one, so the reader must preserve the copied values and write +null for the changed and added rows. The metadata columns are keyed by their +reserved field IDs, 2147483540 and 2147483539, so a reader that treats them as +ordinary columns or resolves them by name returns wrong values. Other v3 +features are left to other tables. + +## Input + +```json +{ + "format-version": 3, + "description": "Row lineage. Two appends assign row IDs by inheritance; an overwrite then writes some row IDs and sequence numbers explicitly and leaves others null.", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [ + { + "id": 1, + "name": "id", + "required": true, + "type": "long" + }, + { + "id": 2, + "name": "data", + "required": false, + "type": "string" + } + ] + }, + "partition-spec": { + "spec-id": 0, + "fields": [] + }, + "commits": [ + { + "data-file": "a", + "rows": [ + {"1": "1", "2": "alpha"}, + {"1": "2", "2": "bravo"}, + {"1": "3", "2": "charlie"} + ] + }, + { + "data-files": [ + { + "data-file": "b", + "rows": [ + {"1": "4", "2": "delta"}, + {"1": "5", "2": "echo"} + ] + }, + { + "data-file": "c", + "rows": [ + {"1": "6", "2": "foxtrot"} + ] + } + ] + }, + { + "overwrite": { + "replaces": "a", + "data-file": "a2", + "rows": [ + {"1": "1", "2": "alpha", "2147483540": "0", "2147483539": "1"}, + {"1": "2", "2": "bravo-updated", "2147483540": "1", "2147483539": null}, + {"1": "3", "2": "charlie", "2147483540": "2", "2147483539": "1"}, + {"1": "7", "2": "golf", "2147483540": null, "2147483539": null} + ] + } + } + ] +} +``` diff --git a/reference-tables/v3/row-lineage-parquet-data/data/a.parquet b/reference-tables/v3/row-lineage-parquet-data/data/a.parquet new file mode 100644 index 0000000000000000000000000000000000000000..f20177910d3d1ce6cc1c5acade2382f110a55c92 GIT binary patch literal 701 zcmZ8fO=}ZD7@pnkZWjrH&>41M!7!2#aT~Hp^Jyq}DdMRX1f?jM-JMMqHfg%M5u_A) z(;|Y6dJw#c2Y-Typ1g?{|AHR;0fK@ad?#rebq>S(@jTBw@62po*>W+$72Lw_KK%Hq z+t`5A#3n-LAHp_UesF+_rBAt6%Kb+lFIi9n%mxnd?>DdC=H?XRKZ@7u^}@bBJ#6dr zFu$>N-Mh7Z`SH<B$w=0G#=m4>5=0&>VVk^HaRZUh3NeZ%`KhePfvvNVFFcX-%c(yQ zLvB4&_E>grELex7Su*i6=CFBdH40d#;;=9hagegk_8yZJ*I^?*lz7M-mWbU+ln7`5 zTR10L@y@8v_MJJUdPp(yml=R^{^UNh1;q%YJO$S=c46aqz=59e6}y}UpC?6 F{sA~emHPky literal 0 HcmV?d00001 diff --git a/reference-tables/v3/row-lineage-parquet-data/data/a2.parquet b/reference-tables/v3/row-lineage-parquet-data/data/a2.parquet new file mode 100644 index 0000000000000000000000000000000000000000..1dcee752646bcbb92df728f84530e573aab16c34 GIT binary patch literal 1509 zcma)6Ux*V&7@uq=xkZlp2WQxYsO!qb2=Tbw=I)Yf5G_SKxreL82T{pQc4u=dn@!Aa zJVcI)wxE>KQfMh{Uwlx`_NhfEic}v}5XCn|C{^*r^TikO<@B4)=8u1RbP3sSzM0?e z`^|5@*_mDYas-i`WCywS^SM*;6p>*l6B(gRU&CMcnAS;@&fFe6fB#$_J>L9uod#)G zW|J>tb-{Z_nAaw=#8P)yFehamyc> zt;P?D8`F30OvTai;Xj`lZRQgBgnV=Pmn(z+O>9?BO1m?+@%fng#kY$a1rC!nWSG2q z@zuk4iAV&$y9f00pEDQ5ra=1|QXtQ7K6?;K$K*rOZ9bbszdvdGCLrosNmb++MB;&P zpdBG1sWh5SFt;(sR0UV-uHBdK_$XjfA540mwNpT#F;Y`hCN9v6C?J<&q7t1HqeY1{ zqKTzAu|)`&LwgbdtGFBvg5V*fNRG8%LS%4AQn1AWHiX7-DkaupNDY*m_4zU^SzLbo zdUZRuq3kk0D7PB0oLgm|A8m2Z8V-#h&mi4l*6F3V{j!Fp8t4Xq+-os=E64Zn0SW<9@!bA2SWAsui9hAmVCbU>- z8{S6(-XMB6NmBH?1Q`iwa75GDY{+zkHqwg{2<;q)(mf+8Wj>JYs{s|3l7!OhiAboZ zfI;ZW)`fKgdL@=#il={yr+OyzB8oHJ7VBk^nuXTJU=lbbUc#$W10Z2pNT846R8GtE z7G5vyrmh z=pvKL>K?0!^)SlnO=tn9$zcM}Jlo-N>t0n~$hIM)U5Mziw?H;?{T>PwIQ_U>ej;Bm z#`2&dsu}g<3LIhOtZvm!kT_LqxQ^)r8nm)D^{E!ju|RWtjkSV$jRlU$T=%%feBY^h zu!pYTsIXD&6<(?F|F4|Sqx+(@?~67+$34)*LwWPe*F+I5x?TFh;RU4ZZLlU>vS53y zsU71@->G|AZa6nFoYOLumg8F5M1d77W7HVomHfnb(W(>;i;vl(_9)L4jB&#p$(xK9 Ua@*8txZB+yS_@ym8Ti1z0d)gTg#Z8m literal 0 HcmV?d00001 diff --git a/reference-tables/v3/row-lineage-parquet-data/data/b.parquet b/reference-tables/v3/row-lineage-parquet-data/data/b.parquet new file mode 100644 index 0000000000000000000000000000000000000000..5fffbc2fb79961602f4913e161bfbaac33bab6cd GIT binary patch literal 681 zcmZ9K&ubGw6vyALld?i9Dw*kyEVv9Lq_7Ry{7wi(FGV~^5cHx#W_M@X4Vxd`Yy>GS zUPZhJUcD3q{R2D-Ui2@}s~(Ggg>RGCsB;)*-g}?#eBaFUZ{K$*P@C52`PrAZmn=$f zYfuB==1ZIw!mJT&aM4n zvcCuUPp@Dbaehtbra_k4fXm!!Np;Rv>0F@VRC1Fj|8TO3Tg;#y767-{ke+6x>^fkw zcPE!hFIJQQB?U0`D0rX)-7M%P31lh@&>)tDAS2x3P7%;FFMz?bN=g`E_n22SdTS?Z zdkjh!{?<{nfR})0Ll0HI>om{KHM$3x^d@g?WZH|P7wV7MX>r(VLTS;Bx$NyP3G4#c zFP+|C?{r!L_CaH$Xt1AJKI_%I68Xwg$w7JMAF7cw-)P4oeKu91D`sgj_fz4BWokDH zM7QFIFj8?a6WvG0B8#|=7|T&sk2^;s>S!J%3KK{x=Z{uA86SufXNgoVkn95gfkGM2 z))YF3GVaLw39fy(gd8fdV4_lk<*T%rYN zdEk&tJS2dt9&s!YsmOz-<+@r(33HiW1s5&mfz1pPElTJf-LKl+Wjo7=!sepiWmUJJ zm!R*4u68{mxIw>{a1V_2Dr=UF_w?$idylHB=q|=t1loH(;-dDKvmOrU4}#a}8-g96 zFH4kEZF*MPFe`A@vP8zRIH--2Lpg|?*QFD#o*hZv;bT=y62*N!gSIlkJ9VF@nastQ zcOIQ^Qwe-Nj0T3UGoKgoc#;)T16nQpxmP|K9`I9tCTT1rX0QK<(kLG9C|<;5FFrw1ROx9X-g^e<4-_>6aN8k#-0y**Y0K$q^MH8Gw=Q0 zZ{B z7em*4+@b5?f4UZucYAI~;0M@EbZ*NF{8epJPqvt1OAE(1v#E`}NbUjzF5S1Xc?7SA z!7B-n2d@f6x+m)!#;dhz@|&07{|Q`eoCWseHtQLmqbWW^0hwh zMr%QAv(Sk4;v0qD=Z3PbG#_P$C5enKUUL%Nls-fyB z>Q+a_scsOtu^afPMLZIdNJr)=h!1;i+~?o8S9VN>rJ`qqbaA{Uy{EYY=feQh0|zq) zZ@syIHr#j(&7VgspJ>GL_>F2+VTIXk2Vn;*apf$}LntU&gD4d>L~~U~LFmwbP#1@A z_Sqyes%mn;ikL+&&=+%WR5R?aZ1MolaU+OwisD|^BYF}cMok=%76N)H_re{Dy2Qyv zZz|Cz)nz+DPd!$a_uOlPr87O%WO0Pt#0h0CXL>M$-7Ca)eY@F*6Bn^3g~c6Q2x9E9 zbU_g;r{?)`_0ZI0a00K_@olxMDQqsb)w zWkPKfv=Z2}yj~iVEM8!?1I-AFo9PsW?H3w9g)p4kr%fphqrWvPDWmb1U70YKGGRg6JEx2D0 zZQKa^E!4n`a_&@+beGKzbr%UV4N739#Kmn?o@=JsQJ$De~JgK1Rb-TMu676QzINi)t8%3@d zhmd5A*UnhHqmZO#J9H7dLt>nc-npSR0VnR8Grkk{yKHWPeTeTvO)qGco{Ob+qo@mG zoE@~=5s637UVcNuK-ht~GM)M?c^RIkW@Frh@k`n+Zbz0n+VI&DCi`t5N+#MYplJ!{ z3?EpXpaVme8^L|dE-kchntCb0rlxTeguqtVf?OV^J17WoK<8mC44JSNe3ZJFQoC{6 z4M8M@!NyPzLfNp%`2ri)VLgKN;056*a#wwrC+YB!`7xdS*Z`UifWu$P^9ne<{K@8Cc!=o>`i z5q}k=5F6k50l>}9tO1GuguDv1oS2iW)5|H1ju_7|-R=Y};v9frJRFnbiB_ttlQ06s zH(_Z)kW{E3fkL0C9(NrAm*;9K`EO2%`5ba!gjWf8r{Ef%>-~b430h9{4#~b{L>c8T zw`Sib7@>T#C0!0E^N|cp2?I1ghv@r}tYjZocp=HH>a)xssqFNW#~n+8O49;zV7Rf} z*ZZ7V(BxTaQl(#jBd5>Hl)v;KpRpMELi3O2V}8Up0O?O2lgR!;W=oD~7xcfxBr`at z{0m56>CdHU`AdLwA`>df^4uH1D6R_M1f-W)kubT}0BJu{%7S95w{;DMon+oHX4rq| z>n3{xVY6KN6Zl9*yBZ%=l3%3@T5Fq}pb@pZjA2g1!0bwuDIJU!0~n&(aYzg%6xQy< zP%f%PP=>7JVfuNSk(p4nMUg>)?KH9hwJXA~AT84SWr(odW)v1AQb80J9No^Nc%lpy zahWGYTsg(L&6v#T!kA3pj>pt=eaznJ*@udB^2Ewf1)_v6%RZ+CrR9-I#w;!wODUQ0 z;j7Y3z_;Wj1o@zcYd3OH#m1Fl@g37k{1mE{$iG@JpN-XY0ZWtE$> zDi0?o(u#~`ZsDDXHHkqT1h_27=&~X#xzX$Y(q|H>oF)dOSP{knzP2ODq-U9>+%tWG zSJl_Z0Jf{bKv%)vaWObD*|h)S%7GNg(^Nw5ne-3+(!F-L-}vC~58Y2@{+ZaCaGJFT z-~W66qrWOwf3(bM)x-7McNV_=WAawz_j~om@?vH2ab>BpRli+t=(q0N`*U(}d8x8^ zcd7BPvQXbzs63jq9!+jdKE8YF&eC%I?8C|%`pf!q<^J-5zVP6wL)WjIGk)0f>*qh6 zdwlRa`|Y}3clD2tKL6g-v1g85YhJ>aE=?HwU;Ay(-XmZA{Mq|APJeyWs^6HLd*PcC SFItDc`1Ja|oBQ?;;QkN0>o1`I literal 0 HcmV?d00001 diff --git a/reference-tables/v3/row-lineage-parquet-data/metadata/77acae54-d569-4214-a217-6208595365e8-m0.avro b/reference-tables/v3/row-lineage-parquet-data/metadata/77acae54-d569-4214-a217-6208595365e8-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..8a378dea40478b06b2b1f2d6a5c5ff2b3205ec28 GIT binary patch literal 7838 zcmcIoO^6&t6iznA2+<%UNQ4~9KB>&gboXZeV1uGrvQgJ;5OL`Xb%kX+0`j2AH{MGwJq4(967n`aSxRn_&=J=2}p*%g*RdaB<0-dFFv zdSAWew)Z~$Wa_Pw4!$#X)yOuw#F#bGEa`cf(J=Tg<3Y=qZ8i))AYm&tW-nef=ohnL zM7TrO!~YsaLO$pP34tGAH)#l4VH~X(n+?9jl3Q9h!-Y*g4pX%Y5CnAJ^5!#mEs0;^ zKpvcs5EJ6X33Z!8=-?>uNt!t%%90KEu7S;$q#4e7Dg2a28ALQhwvIIM5yKc$UEdebg@`w}lIxb11`{AwGnl(QNiIeWw zHmvMKo6%C{8^359;Cs&c2vr zW>ZfNSdp>lCHiXa&1Ql94H*N zPR)zu`l0E`;KgCD6FGFbvMLHqGK-T0(+&}2qyu>@zn5FeWz(E;)=YC+L{mum%Y@pf zXeF>yV{;9MJyl&IN-YR;8X6t#D=UGA!}GiecW@W_pJ3xn+@)Oe2vN{NSqy)aAw~|Z zpCG*qh^m`XbGnR)AMox_H3L*x%~7bDDZ^5{<{76^Wi@v|70c%aWRe&5T5!K0+PD#p zT4)h3)^n$Wq`Mq;sJlp@Wl{nQB`$BH`dqX0j`GM{v#gN_WK1#Rkb}G{RA-v0chU$) zyH1up4P@CfBV?Q1R|Iw0X3vaB7QO0ey+U3g#+q!8yn<_!&DNVo7_XDWae;Ffy+p&= zK~J+LX`W4uK%F&NU8n;(vo$pmcPw7S;;kjgINOeMsy>ZWz0@=VI&KDDA=+=fu9B zl5F_w6*nXdgdLbG^Qq5Pm*F{THp4EAU&?lMJF@lBM$DEl*>3|8pJ+ z4h&gA3imO)w9vw9)JqLEJ&n^i0k)DBjOPVWy2=t^K4v)^$6C37lhL^Sczbsq{Bzy$9(o<186<~R>wS{pGQJ`l2C!va_H`& z5)pllDpINkW%ToK1#ck6I`G^=p&0U?@1OEtpp6*w;h16r=r}_dZvI?yzyzCPsOM21 zV{^@u=Qivsl~X^HJX7^48Lh8E=#0tS90HA+uu7?6uQ>5C>BUCg{()N3*N-M7(F#Z* zHol7kfSa9J0~9d`c^zs!F~?b_S5q1tF^(|Z?!_(Q9fDv&oRMQ(D~+#_Bn8DcW*btF zj8j1Zg+A6i7I*|M&y`&A-=2{3Ipn|)uM+T1AvCD|U}L+l_XV?{ z$&1wFO1}a}NuSp#f8{|vV=?lj<{!?-{E%+|%AYzWnf;Z_)*RC==zoPtWpGLPSCGKc zUr5vXmjLBNB^1x{(i^}qt}5RIl$U9lFs0W3Wxr6$l47d28wLzJeBLmp*njBjCVK;6 zvt0fY_~4`6tQb}JugV3jwM{|LklJ0vFe77Nc9qIh4u*>X3{mYkAO;HxYj8d;y(m0{SB7Ww@$K-g|G3L6rsA_^OhZs$=vQijU7tm87S zlH%NEOx9#&Ocrn_Vse8Bvv+#-VO%+RWaa1rQNdSbU($lg@=zsXRhNvdmCWeyb?GMI z+v*a6e9%=|1$;vS;E&w3a!t8WcTZWFRrrzJF$_}rkX=+jaBbutpvB~Of>@yp7EW}4 zbW`wD4wh7!GB)@S2cm;SROpHVPT4DftCw*S=Y(oyWyPip7l7t>2tTW$a#L2-;gm#L zmC?d2yc4meFzADTkOc)@R)wWBdi`JeOd(a%M4uF^!q~^xb|jhdtg=*ircdy?`WhO* zc2yYYD)>7l2S+BG9=o`FC`a-HmC$>Buz&pgyY^>eUrgVhTm9wXfwAAmJp9*{5C3`i z)wfT*{_D+a*YH8_g1>m@%dLgzqb)yapKUMQi{5Q7`4{}eUGTTQ-dbv3XfL%dC3m6) z_v~W(?$)ilTT5F%MT`EWwtMAH^oqNLgZ7Q+w7c-liJPY$eEjL(`==k@fBoL|@BjVN zb@6HM)^GRs+7la6o!ff3=)D;AnD>d)C%#{yqPJ`Ho=WRflk$E$ZnRz7z zNic>4663-}A;u*w;Hn#qOA=!=B*u*{j9aNdjVo6EpL=KS&&+!>yuQ|?Az|LV=jXfM zIrn_`NbTL#n>IxDBx&H!`YvekbdzY~S`>#ZJJu>%^2@O6Xyet2R(FZ#MB4a?3mW~R zS2Q0t=z92HMGMK17{{#$eoC`+eVQU$mt;tz)qK zsM`wC_cepajorXU4)I7#q6#whL4MeC<7xhldu7L@n`-n7kS2~NrT3IOa6T-6*8>|f z2XDQ32%U4|Np$cSVtk?*%j4IpRRtAhw;qHItmKumd=OGW$r?nV7)O+=6%>Rv{RcI1 z2xp%qnO;@N0V`sby@xSWQb7>N5nw@FU4NCLs661 zx$F%k`=q)oJ7}rL%6QMc9%gi=r~wVa?LP$0+1oai~|Rgu29T0 zUG1bk_E#J%a}>Zb$9iC!*;WK)Y%|At1dCqvv|fQ%h@mFhV^YDTWHZ&~;RWYNXw?97 z7`=GISw>GY_E4Vf?EyMtPjR7k=*-66p0H#2B9?C{NXBj@npU&9vLMlJX7OIwbFAj)ov6u zVT`kadOad>_w402Bn*TNm@Ct%&y<(pF={r(H5k97?c#Q1s-q2`En%`>LPW_#n+<4> z4CnwKSnZ$zLzWxCeatQ`bZ9^IQWBd=<0uFLtI!cpW-_%Kh>;Un{7I{UE!G#vnoa~@I8BP4Ya zQi0@Hbaz&X@IFTsDb<4_`-5-=Pa}pp;Oroe5BWRWr~F;C5#wYyrm;cjI7S$5{#0_n z1e?ZCj-xuprkW?^HtfrlQ#+CgQ`IRMotuQz8Iq|v1R6DDl~RIUcH(8yi;cYP1GRv! zJsOYr6Cj1y_|6XiZgyr3q6k39tD%+yvzv8#Ii=AN<5{NL?Z6@SMo7lPF&Rj-((n`s zBT#&OrX~c*Fcl*YI5L7rac+a-w%g_AMjI zD1W&%`#!+{<(p0EvQ3$HWne@Yp#1Eh@4K>+eO%#%B)6*1GVP$U(^DRIOc_*43*bO! zV=J%sIkKS1v(ThUzaWmBJ}*=L(t~`)V&DtS-<^+xUA_TGfAW|F`wNjRIi?lR|1u`Y z;GFU=0D;k;OVjd~0O>>$Dq(r<4WJuWg>M4V%cvks?lnN#&y=!&nCk6{2E$G=Z|Hm3 zf9UHbdjny!T>2CENJhJHKB^?YN*A=&E^>gn)b28eF_8naD^;d+&|MB-h-$|UIT%n_ zE0aUHs1`}-vXVRL=Ot!lK-Csy1_id<%-YnhNW%nLr1wh)VY|dEOdwK07A73s^0RoX z3>A49yG33(#ks_sj6H=p8Gs$nspk5az09n7g_k4NWDM#cz-2*3mleg58@>K7eI_H7(?pvTD~hoVucb&b=~=Rr zd!|qDs`~0WfTgN1&{gm^AO=S!o7S8-wK0uk9hK0lZhyYHb#mb6M}K^`q?E8);@ceh`^ NK6+`*_4RGJ{RdrON<#nu literal 0 HcmV?d00001 diff --git a/reference-tables/v3/row-lineage-parquet-data/metadata/f4cde2d4-0670-43ca-81a1-0eeff5f59e29-m0.avro b/reference-tables/v3/row-lineage-parquet-data/metadata/f4cde2d4-0670-43ca-81a1-0eeff5f59e29-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..395a08afdeb563da9c95a96c618722638b3cdefb GIT binary patch literal 7871 zcmcIoPiP!f7`JT%v87fl;vvFw4q|1aJ2PpT1gb6CK#R80RtyPY@^3G^yjOTj#ZCu0gb6EsTPu9cd*k{-w{UYQyD{R&$734YlF% zd5wP2OPYu4bUpl4(gN~g(+LRt0J}+v+j4zxN?Rz!TMV(KiX)ubRKsp4cR>UW-8Zpt zKb{Hv$KpV~e?o*95Ze!^+a!cK_M93CBa3)ZFbCh2V3Q?bgrjB%KP6c@9!-(0i!-EA z^0gjrgwuXxvDD%XnkZ>iQo~I*(uV1G)sn7S0$7M=6U%GXD67`xTS)BrIxIgq90(U~s@<79?QVz83-PV!sw6TlH<=##8Z+A@XY|nkI;P z8b;;))NKjr`x}1fM2_#FDsf3f!V)s}Lw?wGqB;JJdu7L@n`-n7kOq#XrT2t8a6T-6 zXM7to2hSfrjAor^8jYMlj88OUdHj00tf0c|*8HH3mAo>RM<5jxuR#=wVMMuFLVjS= z|4;)5aQ11E>1CB1up(mFi|F&Q*UKsNCl+{u&v7G&Vv3?>!zHCSLyRgoBvl0PQtX90 z6gG&R$=*=1kE_eF{ib@XjQ8B@QATHas`26wIk6LpT*h=OgPoU%<#<+Q4o+O;9v2pO zaMF*k%jkkaSWeCJ}@(&<{*G&4t2pcv!w`%*k%rO2^PKTX}tok5JOG0$GCzE$!4m}!}VuL zU{wHf7`=GI*+5S-1}M+=bpf3*kYA{6IS)7fOPK7}5m7wR zrUM#~0UhE4tL@ie$Z|rskJ+V#4nIx36vU>|IP?QRE2sjO+vyG(ggBrRuoi|)SPMQ1 zUG%HnxaI^Pk^+CWtp}lO*yMbYjq9);!Fuq5a2PsM9?X+;_(=Vj%zkVDO$NaHocq=D z2uO_tR3KF?x;w2zc%P$+l#YW!N zfm*=V8jVZ5DUd>JeCG!MH#@TiQTQO_)liFp*~>b;oYLrsv7703+piLPCnV$Ii1fx< zsc(h^At=7BrX~bQ9~C4}=v&LZj!oe5oJu7Bd;MZQ2M%=bDgo~lT*Gs{pYt+7%Zc70 z>9>q9rTpdA^!o$@ly5er%NAwck%4|;fbz4AzVFCN`f-I9lH963%d~>ZPEUE$^ z|7A>)!5QVB0|KKzm!`!p0n&*iRLt_s8$c(na^D1`mq9_8%xi$OpDJYmG1c284Tha~ z-q824Kj`Zwdjny!T=En6h)26&KB~mON*A=&7C1m1YIm8#kjR1Al`2y@=qv{?M73j^ z91JL|&B>uyREwl^S;_76^Exv#plb6ng96)VW-V%0q+tRrlKZ8Nuw7>sCJ-qn3lolR z<5}ERhKjt5y&|uS;#_A=#z1aP24Kf?syH5I@AT|LpLBBD%25TP3}2RgMhi;I9hHoc zUoxgrGTn!-N;eU{DK8=5gR0VU!q-Fq@gsMkT$66(-IG;jIetWUbe)hsWTzDnTpP&; zXfes1z*k6vsS|A@-DG%@gBg`3jkQ0-w$T9*Wprr)C+($zE0%F0&VJd-vWiU_P6e9W zA@NzIm7BCG4<{qi@{Fc#;f;tj8G||qa9NPjWqGk=Mz8-%pUFt&G|?i(@?vbkYdw-o zdX_BZp6L_3s=hi7V7)2~bQS#T6@w#_O+DiiJCjK6q7wR!-p^n8^X`TI0q>*vx-z=5IQshoXLfJjzWuezB%Ykys`s4PQJ(ze$l%f|TU7hK+yWJ8Zgk24SV(9vSK$_d>?RH=~Q)Z^r zN;XC`K`P{4~lZO-4K(=FP#u)t0nkvNEY^vDNOanc;bt5WkP98m;LM}Rs z+NWiuWL%cVBlWR$VvLE=byJ8vYC2d)G(lu*FA$^|C{uBW?WbD}tO$XN*m5+QPueaL zH4Vrtam{d8)+zWCZ5&1=n`Dt=qOzqqwfRPcu&QY|#C^ZYf*5Rr_ot1_9CGpm`P3$n zyf%v+96`$3EYV3_=E|S|VUs;2VyGl-lnNPQ@w@|+q^hTocM6sB#0Y?*GPV)UW(mEy zTU3uFB`#nUG~lNtyv=4AngscWeORc3c+(oF0RhTQC2bqHXy;8QP>|roswZewoIG~C zJ2XL&`|D6>r`T!LEQM8SJ>>;KA*~AXj|m*fF(D**Ub7wObaz0SMjff1Y!X-`-)n{J z^l^vXo@thEJ<*m~uFEyRvqu3v)`$T`Yf4x`2OSTk%GQX1Hgn=wtDU;`d_ zqBv@mK&^5J8x>E!@^mO+cyt}HJkegPmuf|4Na#Qg=7tX5N%ApwP6vvO2AElr!zJC3 zk~F8FP$YyDbQpgtoq>tsLh7?Bd0NcypYf#^xo;u&EtE4YgyGumY*K>4fyD(;Di-em z*nO*gU}b#gctnC}D}Xdqmr7!sT`W~96fh_Ty$Zo z-b4l|)8{tbzw3vGM}`r-MGF;;%0TR5*N>=XmZ`G7r*C--0I2!yHj;1_0Fj3dn{O#ng#nb384s^`8IqI9Vib)sYhWBCw@98JB z*WbQ2`Cs?83+HFgzj|cN2K)Lw-QTbI>)tsR%D=tQyXE+U^AhGChvqN4`R{$PlPh*~ zMt+()xoX|S%*v*jlc~k|!>9H?_j}ig(bZp#E;&5;-p@zRSx@zzy52MQ^5sj>w^rUX zr~ckD|JYUX?Zs8v2dCFE@R6m5|`$R zhDqRY(I95QRYrF1Z8=fFv;-#_fn4es)+GEc*gC|Suysrg+d_{&wiVS3T0-|{k%!Ks z#*Bh8l2p~%!PeL&u_oo{rY*(ZVN+}(mLPJCX9zMZl&@3b2I)2eYf|6}cBsMf$+;Dx z=Kz@}ZdjCOorOQq)=^Y-NdZwC)f|I1=Np&8>b6CR_kM!~IoN&vpDwbCh?WQns2xG- zrUIsT5NVqV#3U`5YbgoBCDT=6=_F@W%X#96yhD^^8>f(e3e`%)3W1^;b`dTV2)nsg zRG%d+DPffj;HN9R3A+YOf`Y>VEL1|iX%p121m&fYb1htPOEwJ^Bz&>P304(a!qmS* z2NZRv1%-8ro73%TRHe2sUJw-0siWYS#F3hiLXtg(OQF-f0qGcZtZ}kKV3~Zs6$%T- z9rb#qQ@*W4J9e!l*AUM>1?*Tm1~j?pIRgs@_^woQU9c;JiT z4yOugRYcgT`|?#{LkYuUnuzC#X0Tap7M+aLfdb469lnziVD4H76kBaD3#5pvCRIjQ zPGO-)ODUKz{!T6r6D5Q+U{&$8nB#8`q!)Q_A@40zv>k-u+TLt3f}(-t1yQY3t^wG4 zt9M{+cGv8n0@GFoX=p5!^en$vwpuP@Pz-hzg7uQ1v*UeKS>2~dJy#|avwgt(CW^^1 zOLyuOZ>z$4D0mO$O*`KaQj^=KXgILmVRmHE<&d5zh$~W7JZlocaZiD5gk9H|1ev#y zMQZH1%l7Y<(b2I{#BR|=MdK7F-lxZRiA+=48Y7yu6l=HrSP_ z&KAG#h`j2mA-mIXC#}w?;d>ldl!gQQbOb^XKJz{JAU04wiq?w49Z&>LNWCK+k${ME z2(C1e7m5Ip?M!=b!~wJv2N15Pi?F@8BmtJ>U_s4=KZ*bi5ti-2FA3CPj@sTSb%Wfc zbK;=t5C?7NnYw|@cHyA}UZM#v?4Ba<s$LFp>Hf#by6e%ziJtG7L zqb-0YCdn@d(?ISnBlrn5UyIk`$!nt6oS&YQtY@%jCl>Nr zA}*^Jm@q%-4vtIYD!QP_T#2AfWa6rJ-Q4=^=f3|aO=Hi#kS%@q!EH<4Tzvf8!{dv!RR?}c+;Hmb z)RnbE+NrY}4nIHs^yb|Mvz0r~96P%G#*6b8{#a)%nS1f-Y31j>W#=zF`QOaf2iB~A zZT~CF`eT>gDxSXh(&gpvKRUK&^kDy?Pkx;~c4hpG@zCt5&!)cGfBBofo2zd>(f8oL z8=m^KpBx>qCmuQBOuVu5)BW~ZSLM*-rKnO)n~VnMvknow=KJ z8>1+#H46G5iVxz0NPVb+PX)Cgq?8sxC`CjN6!FbRU+NF2=g!Z)cXoI7r;j^(=X~dW z=bZ0+^F(EJ$F32)?t2FV-6Re^t!yZ?StCkO@rmyFhBBtO*kQjN>{?Y4&~nwX2|TV^ z#5Mw@xU|uBqK2snC*nW>^(}V=ejA<+aTYusQ_FMF(@z~o4U3xSSP6ybJnFb*D8)%t z-5hI=Z4-Azj&6HW>_v}a8?gjYV4Wh!a#5v8Nsy!)2iBy(HSAN1u7L@xs}Puy}T z&$f#?w&t6Y!s?z&N%(%uf*fo%`ZGXY6;YF*gxVrf4;z@`F{B+f zh)vou*QOaldK$DxEGte_AXCO1KT&#qEo?#OLGNa1%;*-zhA)3-9#jV@60wryzBp01HE>@1!J{%e_Fc+X2%cRb02JQe-)$g(5GdV8i(P z4JxX6m)jIk1DJC8flgtLNVJ1yl=9YoOgAm6oC7F`Y*ih{TzWyNz>5F8H`*h4r7tQC-X54ohlo(F9I9-W(; zpF`{xeN;530vBZh<7j@UnhP5n|)~F{$w4EM_X`6+iU3rmgY+Ad*0l~M;q))U1y6waztTu)s)?d z-ATJMihYj{i_-F8pB5n$;WIyg4`Krqp=j+W+yQ0aq|`gp5ebMKhu}&hd7%st+0JzK zMjSw!<^aMK^$~Uumn6U@IapG2;g2#vQ-qZU@Jj-9ilcURO8p@B>6|#II>A9Zd8U3K zOMQ4Kfp@3{FYTT(@Z?5mcT*y`s2g_?1IOpCLN;syLKG=66ayn92BRZ@CMGE=2-86B zE;IN^HD8<8;>l~G*qon%!*hN@agO|~KoI3k=ouIwp{G-PRmcF>8RonMFI8*@TrRmD z+u*9fDH_m?Zi@EOT=aM0uvS`-NkW< zTtyeOm@5&qr5tHyy1Fb5lN0>67^*ColmwJ++E@Wk`(d(}BbDsZPUhEdp^u#X<-yPW ze_lEA%@>zuF1z}D8D3tz6k{Mh?{Y=3}GD$ibce#a?u z^vvsbna#&P{gb|Z{p`rvdtaTr^|rTuymor@rP_zv*Us;J=Z&9Vz5Az;rO7=H4R5T@ z-sv9Mcjdd`t<@8s75D!;d+!hDuFjBi*FRalY^*IB_H)1I9)9Lb@csBbGnao|-}UIX YSN{6@;>8m;ZY