diff --git a/reference-tables/GenerateTables.java b/reference-tables/GenerateTables.java index 9474b5c..9ddd5ad 100644 --- a/reference-tables/GenerateTables.java +++ b/reference-tables/GenerateTables.java @@ -21,10 +21,15 @@ import com.fasterxml.jackson.databind.JsonNode; import java.io.File; import java.io.IOException; +import java.math.BigDecimal; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Comparator; @@ -32,6 +37,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.stream.Stream; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.AppendFiles; @@ -281,6 +287,7 @@ static DataFile write(Table table, String tag, List records) throws IOEx } // input values, in the spec's single-value JSON keyed by field id, with long values as strings + // and binary and fixed values as uppercase hex static Record record(Types.StructType struct, JsonNode node) { Record record = GenericRecord.create(struct); @@ -295,19 +302,43 @@ static Object value(Type type, JsonNode node) { return null; } switch (type.typeId()) { + case BOOLEAN: + return node.booleanValue(); case LONG: return Long.parseLong(node.asText()); case INTEGER: return node.intValue(); + case FLOAT: + return node.floatValue(); case DOUBLE: return node.doubleValue(); + case DECIMAL: + BigDecimal decimal = new BigDecimal(node.textValue()); + if (decimal.scale() != ((Types.DecimalType) type).scale()) { + throw new IllegalArgumentException("Value " + node + " is not of type " + type); + } + return decimal; + case DATE: + return LocalDate.parse(node.textValue()); + case TIME: + return LocalTime.parse(node.textValue()); + case TIMESTAMP: + if (((Types.TimestampType) type).shouldAdjustToUTC()) { + return OffsetDateTime.parse(node.textValue()); + } + return LocalDateTime.parse(node.textValue()); case STRING: return node.textValue(); - case TIMESTAMP: - if (!((Types.TimestampType) type).shouldAdjustToUTC()) { - break; + case UUID: + return UUID.fromString(node.textValue()); + case FIXED: + byte[] fixed = hex(node.textValue()); + if (fixed.length != ((Types.FixedType) type).length()) { + throw new IllegalArgumentException("Value " + node + " is not of type " + type); } - return OffsetDateTime.parse(node.textValue()); + return fixed; + case BINARY: + return ByteBuffer.wrap(hex(node.textValue())); case STRUCT: return record(type.asStructType(), node); case LIST: @@ -332,6 +363,18 @@ static Object value(Type type, JsonNode node) { throw new UnsupportedOperationException("No input conversion for type " + type); } + /** Uppercase hex, the form CONTRIBUTING.md gives for binary values. */ + static byte[] hex(String text) { + if (text.length() % 2 != 0 || !text.matches("[0-9A-F]*")) { + throw new IllegalArgumentException("Not uppercase hex: \"" + text + "\""); + } + byte[] bytes = new byte[text.length() / 2]; + for (int i = 0; i < bytes.length; i += 1) { + bytes[i] = (byte) Integer.parseInt(text.substring(2 * i, 2 * i + 2), 16); + } + return bytes; + } + // post-processing /** Per table, in generation order, for the index. */ diff --git a/reference-tables/README.md b/reference-tables/README.md index acfdbb1..951155f 100644 --- a/reference-tables/README.md +++ b/reference-tables/README.md @@ -232,7 +232,10 @@ list, each manifest's entries not marked `DELETED`, and the rows of each data file those entries name, read by field ID with any field a file lacks filled from its `initial-default`, less the positions any deletion vector that applies to it deletes. The rows of `v3/deletion-vector-parquet-data` were instead derived -from its Input, and a PyIceberg 0.12.0 scan returns the same rows. +from its Input, and a PyIceberg 0.12.0 scan returns the same rows. The rows of +`v2/primitive-types-parquet-data` were also derived from its Input, then checked +against the same read with no Iceberg library and against scans with PyIceberg, +Iceberg Java, iceberg-go and iceberg-rust. ## Procedure diff --git a/reference-tables/index.json b/reference-tables/index.json index 3904f09..b48585d 100644 --- a/reference-tables/index.json +++ b/reference-tables/index.json @@ -40,6 +40,14 @@ "metadata": "v3/deletion-vector-parquet-data/metadata/v3.metadata.json", "description": "Flat schema of long, string and int. One append of two data files (five and two rows), then a deletion vector that deletes positions 0 and 3 of the first file only." }, + { + "name": "v2/primitive-types-parquet-data", + "format-version": 2, + "written-by": "Iceberg Java 1.11.0", + "data-file-format": "parquet", + "metadata": "v2/primitive-types-parquet-data/metadata/v2.metadata.json", + "description": "Every v1 and v2 primitive type, read back from Parquet. One append of one data file, three rows: typical values with int and long maxima, boundary values, and every optional column null." + }, { "name": "v3/initial-default-parquet-data", "format-version": 3, diff --git a/reference-tables/v2/primitive-types-parquet-data/README.md b/reference-tables/v2/primitive-types-parquet-data/README.md new file mode 100644 index 0000000..19f3fdf --- /dev/null +++ b/reference-tables/v2/primitive-types-parquet-data/README.md @@ -0,0 +1,189 @@ + + +# v2/primitive-types-parquet-data + +Input to `reference-tables/GenerateTables.java`. The schema and partition spec are +the spec's JSON forms. The one commit appends one data file holding its rows, +written in the spec's single-value JSON keyed by field ID, with `long` values as +strings. + +The table holds one column of every v1 and v2 primitive type. The three decimal +columns use each of the Parquet physical types the spec maps a decimal to: +`int32` for precision up to 9, `int64` up to 18, and `fixed_len_byte_array` +otherwise. A reader that decodes only one of them fails on the others. The +`timestamp` and `timestamptz` columns differ only in the `TIMESTAMP_MICROS` +annotation's `isAdjustedToUTC` flag, so a reader that ignores the flag returns +one of them with the wrong type, or shifted by its local zone. + +Row 1 holds typical values, except that `int_col` and `long_col` hold their +maxima and `time_col` holds `00:00:00`. Row 2 holds boundary values: the int and +long minima, a negative decimal with every digit of its precision used, a date +and timestamps one microsecond before the epoch, `23:59:59.999999`, an empty +string and an empty binary, and an all-ones uuid and fixed. Row 1's string is +non-ASCII and includes a character outside the Basic Multilingual Plane. Row 3 +is null in every optional column. + +Values are written as follows: + +- A decimal is the spec's decimal string, with exactly `scale` digits after the + point. The unscaled value of the `decimal(38, 10)` column overflows 64 bits, + and a string carries it without loss. +- `fixed` and `binary` values are uppercase hex, following `CONTRIBUTING.md`. + The spec's example is lowercase. The generator rejects lowercase hex. +- A `uuid` is the lowercase uuid string, as the spec says. +- Every `float` and `double` value is exactly representable in binary32 or + binary64, so the decimal text decodes to one value in every language. + +NaN, positive and negative infinity, and -0.0 are left out. JSON cannot carry +NaN or the infinities, and the single-value serialization does not define a form +for them. A -0.0 compares equal to 0.0 under most decoded-double equality, so a +multiset comparison would not tell them apart. These are open questions for the +dev list. + +## Input + +```json +{ + "format-version": 2, + "description": "Every v1 and v2 primitive type, read back from Parquet. One append of one data file, three rows: typical values with int and long maxima, boundary values, and every optional column null.", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [ + { + "id": 1, + "name": "id", + "required": true, + "type": "long" + }, + { + "id": 2, + "name": "bool_col", + "required": false, + "type": "boolean" + }, + { + "id": 3, + "name": "int_col", + "required": false, + "type": "int" + }, + { + "id": 4, + "name": "long_col", + "required": false, + "type": "long" + }, + { + "id": 5, + "name": "float_col", + "required": false, + "type": "float" + }, + { + "id": 6, + "name": "double_col", + "required": false, + "type": "double" + }, + { + "id": 7, + "name": "decimal_9_2", + "required": false, + "type": "decimal(9, 2)" + }, + { + "id": 8, + "name": "decimal_18_4", + "required": false, + "type": "decimal(18, 4)" + }, + { + "id": 9, + "name": "decimal_38_10", + "required": false, + "type": "decimal(38, 10)" + }, + { + "id": 10, + "name": "date_col", + "required": false, + "type": "date" + }, + { + "id": 11, + "name": "time_col", + "required": false, + "type": "time" + }, + { + "id": 12, + "name": "timestamp_col", + "required": false, + "type": "timestamp" + }, + { + "id": 13, + "name": "timestamptz_col", + "required": false, + "type": "timestamptz" + }, + { + "id": 14, + "name": "string_col", + "required": false, + "type": "string" + }, + { + "id": 15, + "name": "uuid_col", + "required": false, + "type": "uuid" + }, + { + "id": 16, + "name": "fixed_col", + "required": false, + "type": "fixed[4]" + }, + { + "id": 17, + "name": "binary_col", + "required": false, + "type": "binary" + } + ] + }, + "partition-spec": { + "spec-id": 0, + "fields": [] + }, + "commits": [ + { + "data-file": "a", + "rows": [ + {"1": "1", "2": true, "3": 2147483647, "4": "9223372036854775807", "5": 1.5, "6": -0.25, "7": "1234.56", "8": "12345678.9012", "9": "1234567890.0123456789", "10": "2026-01-15", "11": "00:00:00.000000", "12": "2026-01-15T10:30:00.123456", "13": "2026-01-15T10:30:00.123456+00:00", "14": "Grüße, 世界 🧊", "15": "f79c3e09-677c-4bbd-a479-3f349cb785e7", "16": "000102FF", "17": "DEADBEEF"}, + {"1": "2", "2": false, "3": -2147483648, "4": "-9223372036854775808", "5": -0.375, "6": 1048576.125, "7": "-9999999.99", "8": "-99999999999999.9999", "9": "-9999999999999999999999999999.9999999999", "10": "1969-12-31", "11": "23:59:59.999999", "12": "1969-12-31T23:59:59.999999", "13": "1969-12-31T23:59:59.999999+00:00", "14": "", "15": "ffffffff-ffff-ffff-ffff-ffffffffffff", "16": "FFFFFFFF", "17": ""}, + {"1": "3", "2": null, "3": null, "4": null, "5": null, "6": null, "7": null, "8": null, "9": null, "10": null, "11": null, "12": null, "13": null, "14": null, "15": null, "16": null, "17": null} + ] + } + ] +} +``` diff --git a/reference-tables/v2/primitive-types-parquet-data/data/a.parquet b/reference-tables/v2/primitive-types-parquet-data/data/a.parquet new file mode 100644 index 0000000..a2ee95b Binary files /dev/null and b/reference-tables/v2/primitive-types-parquet-data/data/a.parquet differ diff --git a/reference-tables/v2/primitive-types-parquet-data/expected/scan.json b/reference-tables/v2/primitive-types-parquet-data/expected/scan.json new file mode 100644 index 0000000..f9c65dc --- /dev/null +++ b/reference-tables/v2/primitive-types-parquet-data/expected/scan.json @@ -0,0 +1,12 @@ +{ + "snapshots": [ + { + "snapshot-id": "3884808864619474138", + "rows": [ + {"1": "1", "2": true, "3": 2147483647, "4": "9223372036854775807", "5": 1.5, "6": -0.25, "7": "1234.56", "8": "12345678.9012", "9": "1234567890.0123456789", "10": "2026-01-15", "11": "00:00:00.000000", "12": "2026-01-15T10:30:00.123456", "13": "2026-01-15T10:30:00.123456+00:00", "14": "Grüße, 世界 🧊", "15": "f79c3e09-677c-4bbd-a479-3f349cb785e7", "16": "000102FF", "17": "DEADBEEF"}, + {"1": "2", "2": false, "3": -2147483648, "4": "-9223372036854775808", "5": -0.375, "6": 1048576.125, "7": "-9999999.99", "8": "-99999999999999.9999", "9": "-9999999999999999999999999999.9999999999", "10": "1969-12-31", "11": "23:59:59.999999", "12": "1969-12-31T23:59:59.999999", "13": "1969-12-31T23:59:59.999999+00:00", "14": "", "15": "ffffffff-ffff-ffff-ffff-ffffffffffff", "16": "FFFFFFFF", "17": ""}, + {"1": "3", "2": null, "3": null, "4": null, "5": null, "6": null, "7": null, "8": null, "9": null, "10": null, "11": null, "12": null, "13": null, "14": null, "15": null, "16": null, "17": null} + ] + } + ] +} diff --git a/reference-tables/v2/primitive-types-parquet-data/metadata/0d0374c5-3a2f-4285-8c18-282c5fee3383-m0.avro b/reference-tables/v2/primitive-types-parquet-data/metadata/0d0374c5-3a2f-4285-8c18-282c5fee3383-m0.avro new file mode 100644 index 0000000..1a26b72 Binary files /dev/null and b/reference-tables/v2/primitive-types-parquet-data/metadata/0d0374c5-3a2f-4285-8c18-282c5fee3383-m0.avro differ diff --git a/reference-tables/v2/primitive-types-parquet-data/metadata/snap-3884808864619474138-1-0d0374c5-3a2f-4285-8c18-282c5fee3383.avro b/reference-tables/v2/primitive-types-parquet-data/metadata/snap-3884808864619474138-1-0d0374c5-3a2f-4285-8c18-282c5fee3383.avro new file mode 100644 index 0000000..793da9b Binary files /dev/null and b/reference-tables/v2/primitive-types-parquet-data/metadata/snap-3884808864619474138-1-0d0374c5-3a2f-4285-8c18-282c5fee3383.avro differ diff --git a/reference-tables/v2/primitive-types-parquet-data/metadata/v2.metadata.json b/reference-tables/v2/primitive-types-parquet-data/metadata/v2.metadata.json new file mode 100644 index 0000000..975cbd5 --- /dev/null +++ b/reference-tables/v2/primitive-types-parquet-data/metadata/v2.metadata.json @@ -0,0 +1 @@ +{"format-version":2,"table-uuid":"c0589ff2-c22b-4995-b137-0a0570580058","location":"file:///tmp/iceberg-verification/reference-tables/v2/primitive-types-parquet-data","last-sequence-number":1,"last-updated-ms":1794842319830,"last-column-id":17,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"bool_col","required":false,"type":"boolean"},{"id":3,"name":"int_col","required":false,"type":"int"},{"id":4,"name":"long_col","required":false,"type":"long"},{"id":5,"name":"float_col","required":false,"type":"float"},{"id":6,"name":"double_col","required":false,"type":"double"},{"id":7,"name":"decimal_9_2","required":false,"type":"decimal(9, 2)"},{"id":8,"name":"decimal_18_4","required":false,"type":"decimal(18, 4)"},{"id":9,"name":"decimal_38_10","required":false,"type":"decimal(38, 10)"},{"id":10,"name":"date_col","required":false,"type":"date"},{"id":11,"name":"time_col","required":false,"type":"time"},{"id":12,"name":"timestamp_col","required":false,"type":"timestamp"},{"id":13,"name":"timestamptz_col","required":false,"type":"timestamptz"},{"id":14,"name":"string_col","required":false,"type":"string"},{"id":15,"name":"uuid_col","required":false,"type":"uuid"},{"id":16,"name":"fixed_col","required":false,"type":"fixed[4]"},{"id":17,"name":"binary_col","required":false,"type":"binary"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"write.parquet.compression-codec":"zstd"},"current-snapshot-id":3884808864619474138,"refs":{"main":{"snapshot-id":3884808864619474138,"type":"branch"}},"snapshots":[{"sequence-number":1,"snapshot-id":3884808864619474138,"timestamp-ms":1794842319823,"summary":{"operation":"append","manifests-created":"1","manifests-kept":"0","manifests-replaced":"0","added-data-files":"1","added-records":"3","added-files-size":"4936","changed-partition-count":"1","total-records":"3","total-files-size":"4936","total-data-files":"1","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","iceberg-version":"Apache Iceberg 1.11.0 (commit 6976e020b894f6a6777704df2b8c4458cb291ae9)"},"manifest-list":"file:/tmp/iceberg-verification/reference-tables/v2/primitive-types-parquet-data/metadata/snap-3884808864619474138-1-0d0374c5-3a2f-4285-8c18-282c5fee3383.avro","schema-id":0}],"statistics":[],"partition-statistics":[],"snapshot-log":[{"timestamp-ms":1794842319823,"snapshot-id":3884808864619474138}],"metadata-log":[{"timestamp-ms":1794842319245,"metadata-file":"file:/tmp/iceberg-verification/reference-tables/v2/primitive-types-parquet-data/metadata/v1.metadata.json"}]} \ No newline at end of file