Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 47 additions & 4 deletions reference-tables/GenerateTables.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,17 +21,23 @@
import com.fasterxml.jackson.databind.JsonNode;
import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Stream;
import org.apache.hadoop.conf.Configuration;
import org.apache.iceberg.AppendFiles;
Expand Down Expand Up @@ -281,6 +287,7 @@ static DataFile write(Table table, String tag, List<Record> records) throws IOEx
}

// input values, in the spec's single-value JSON keyed by field id, with long values as strings
// and binary and fixed values as uppercase hex

static Record record(Types.StructType struct, JsonNode node) {
Record record = GenericRecord.create(struct);
Expand All @@ -295,19 +302,43 @@ static Object value(Type type, JsonNode node) {
return null;
}
switch (type.typeId()) {
case BOOLEAN:
return node.booleanValue();
case LONG:
return Long.parseLong(node.asText());
case INTEGER:
return node.intValue();
case FLOAT:
return node.floatValue();
case DOUBLE:
return node.doubleValue();
case DECIMAL:
BigDecimal decimal = new BigDecimal(node.textValue());
if (decimal.scale() != ((Types.DecimalType) type).scale()) {
throw new IllegalArgumentException("Value " + node + " is not of type " + type);
}
return decimal;
case DATE:
return LocalDate.parse(node.textValue());
case TIME:
return LocalTime.parse(node.textValue());
case TIMESTAMP:
if (((Types.TimestampType) type).shouldAdjustToUTC()) {
return OffsetDateTime.parse(node.textValue());
}
return LocalDateTime.parse(node.textValue());
case STRING:
return node.textValue();
case TIMESTAMP:
if (!((Types.TimestampType) type).shouldAdjustToUTC()) {
break;
case UUID:
return UUID.fromString(node.textValue());
case FIXED:
byte[] fixed = hex(node.textValue());
if (fixed.length != ((Types.FixedType) type).length()) {
throw new IllegalArgumentException("Value " + node + " is not of type " + type);
}
return OffsetDateTime.parse(node.textValue());
return fixed;
case BINARY:
return ByteBuffer.wrap(hex(node.textValue()));
case STRUCT:
return record(type.asStructType(), node);
case LIST:
Expand All @@ -332,6 +363,18 @@ static Object value(Type type, JsonNode node) {
throw new UnsupportedOperationException("No input conversion for type " + type);
}

/** Uppercase hex, the form CONTRIBUTING.md gives for binary values. */
static byte[] hex(String text) {
if (text.length() % 2 != 0 || !text.matches("[0-9A-F]*")) {
throw new IllegalArgumentException("Not uppercase hex: \"" + text + "\"");
}
byte[] bytes = new byte[text.length() / 2];
for (int i = 0; i < bytes.length; i += 1) {
bytes[i] = (byte) Integer.parseInt(text.substring(2 * i, 2 * i + 2), 16);
}
return bytes;
}

// post-processing

/** Per table, in generation order, for the index. */
Expand Down
5 changes: 4 additions & 1 deletion reference-tables/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,10 @@ list, each manifest's entries not marked `DELETED`, and the rows of each data
file those entries name, read by field ID with any field a file lacks filled from
its `initial-default`, less the positions any deletion vector that applies to
it deletes. The rows of `v3/deletion-vector-parquet-data` were instead derived
from its Input, and a PyIceberg 0.12.0 scan returns the same rows.
from its Input, and a PyIceberg 0.12.0 scan returns the same rows. The rows of
`v2/primitive-types-parquet-data` were also derived from its Input, then checked
against the same read with no Iceberg library and against scans with PyIceberg,
Iceberg Java, iceberg-go and iceberg-rust.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: the sentence before pins PyIceberg 0.12.0, could this one carry the versions too (PyIceberg 0.12.0, Java 1.11.0 and 1.12.0, iceberg-go a9e2cc3, iceberg-rust 0.11.0)? Right now they're only in the PR description.


## Procedure

Expand Down
8 changes: 8 additions & 0 deletions reference-tables/index.json
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,14 @@
"metadata": "v3/deletion-vector-parquet-data/metadata/v3.metadata.json",
"description": "Flat schema of long, string and int. One append of two data files (five and two rows), then a deletion vector that deletes positions 0 and 3 of the first file only."
},
{
"name": "v2/primitive-types-parquet-data",
"format-version": 2,
"written-by": "Iceberg Java 1.11.0",
"data-file-format": "parquet",
"metadata": "v2/primitive-types-parquet-data/metadata/v2.metadata.json",
"description": "Every v1 and v2 primitive type, read back from Parquet. One append of one data file, three rows: typical values with int and long maxima, boundary values, and every optional column null."
},
{
"name": "v3/initial-default-parquet-data",
"format-version": 3,
Expand Down
189 changes: 189 additions & 0 deletions reference-tables/v2/primitive-types-parquet-data/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,189 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing,
~ software distributed under the License is distributed on an
~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
~ KIND, either express or implied. See the License for the
~ specific language governing permissions and limitations
~ under the License.
-->

# v2/primitive-types-parquet-data

Input to `reference-tables/GenerateTables.java`. The schema and partition spec are
the spec's JSON forms. The one commit appends one data file holding its rows,
written in the spec's single-value JSON keyed by field ID, with `long` values as
strings.

The table holds one column of every v1 and v2 primitive type. The three decimal
columns use each of the Parquet physical types the spec maps a decimal to:
`int32` for precision up to 9, `int64` up to 18, and `fixed_len_byte_array`
otherwise. A reader that decodes only one of them fails on the others. The
`timestamp` and `timestamptz` columns differ only in the `TIMESTAMP_MICROS`
annotation's `isAdjustedToUTC` flag, so a reader that ignores the flag returns
one of them with the wrong type, or shifted by its local zone.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: timestamp_col and timestamptz_col hold the same micros in the file, and the Procedure decodes each column by its type in the Iceberg schema. A reader that ignores isAdjustedToUTC but takes the type from the schema still returns the right rows, and a zone shift only shows when the check runs outside UTC. Could this sentence say the check should run in a non-UTC zone, or limit the claim to readers that take the type from the Parquet annotation?


Row 1 holds typical values, except that `int_col` and `long_col` hold their
maxima and `time_col` holds `00:00:00`. Row 2 holds boundary values: the int and
long minima, a negative decimal with every digit of its precision used, a date
and timestamps one microsecond before the epoch, `23:59:59.999999`, an empty
string and an empty binary, and an all-ones uuid and fixed. Row 1's string is
non-ASCII and includes a character outside the Basic Multilingual Plane. Row 3
is null in every optional column.

Values are written as follows:

- A decimal is the spec's decimal string, with exactly `scale` digits after the
point. The unscaled value of the `decimal(38, 10)` column overflows 64 bits,
and a string carries it without loss.
- `fixed` and `binary` values are uppercase hex, following `CONTRIBUTING.md`.
The spec's example is lowercase. The generator rejects lowercase hex.
- A `uuid` is the lowercase uuid string, as the spec says.
- Every `float` and `double` value is exactly representable in binary32 or
binary64, so the decimal text decodes to one value in every language.

NaN, positive and negative infinity, and -0.0 are left out. JSON cannot carry
NaN or the infinities, and the single-value serialization does not define a form
for them. A -0.0 compares equal to 0.0 under most decoded-double equality, so a
multiset comparison would not tell them apart. These are open questions for the
dev list.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we add a dedicated Scope section now that #30 is in?

## Input

```json
{
"format-version": 2,
"description": "Every v1 and v2 primitive type, read back from Parquet. One append of one data file, three rows: typical values with int and long maxima, boundary values, and every optional column null.",
"schema": {
"type": "struct",
"schema-id": 0,
"fields": [
{
"id": 1,
"name": "id",
"required": true,
"type": "long"
},
{
"id": 2,
"name": "bool_col",
"required": false,
"type": "boolean"
},
{
"id": 3,
"name": "int_col",
"required": false,
"type": "int"
},
{
"id": 4,
"name": "long_col",
"required": false,
"type": "long"
},
{
"id": 5,
"name": "float_col",
"required": false,
"type": "float"
},
{
"id": 6,
"name": "double_col",
"required": false,
"type": "double"
},
{
"id": 7,
"name": "decimal_9_2",
"required": false,
"type": "decimal(9, 2)"
},
{
"id": 8,
"name": "decimal_18_4",
"required": false,
"type": "decimal(18, 4)"
},
{
"id": 9,
"name": "decimal_38_10",
"required": false,
"type": "decimal(38, 10)"
},
{
"id": 10,
"name": "date_col",
"required": false,
"type": "date"
},
{
"id": 11,
"name": "time_col",
"required": false,
"type": "time"
},
{
"id": 12,
"name": "timestamp_col",
"required": false,
"type": "timestamp"
},
{
"id": 13,
"name": "timestamptz_col",
"required": false,
"type": "timestamptz"
},
{
"id": 14,
"name": "string_col",
"required": false,
"type": "string"
},
{
"id": 15,
"name": "uuid_col",
"required": false,
"type": "uuid"
},
{
"id": 16,
"name": "fixed_col",
"required": false,
"type": "fixed[4]"
},
{
"id": 17,
"name": "binary_col",
"required": false,
"type": "binary"
}
]
},
"partition-spec": {
"spec-id": 0,
"fields": []
},
"commits": [
{
"data-file": "a",
"rows": [
{"1": "1", "2": true, "3": 2147483647, "4": "9223372036854775807", "5": 1.5, "6": -0.25, "7": "1234.56", "8": "12345678.9012", "9": "1234567890.0123456789", "10": "2026-01-15", "11": "00:00:00.000000", "12": "2026-01-15T10:30:00.123456", "13": "2026-01-15T10:30:00.123456+00:00", "14": "Grüße, 世界 🧊", "15": "f79c3e09-677c-4bbd-a479-3f349cb785e7", "16": "000102FF", "17": "DEADBEEF"},
{"1": "2", "2": false, "3": -2147483648, "4": "-9223372036854775808", "5": -0.375, "6": 1048576.125, "7": "-9999999.99", "8": "-99999999999999.9999", "9": "-9999999999999999999999999999.9999999999", "10": "1969-12-31", "11": "23:59:59.999999", "12": "1969-12-31T23:59:59.999999", "13": "1969-12-31T23:59:59.999999+00:00", "14": "", "15": "ffffffff-ffff-ffff-ffff-ffffffffffff", "16": "FFFFFFFF", "17": ""},
{"1": "3", "2": null, "3": null, "4": null, "5": null, "6": null, "7": null, "8": null, "9": null, "10": null, "11": null, "12": null, "13": null, "14": null, "15": null, "16": null, "17": null}
]
}
]
}
```
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
{
"snapshots": [
{
"snapshot-id": "3884808864619474138",
"rows": [
{"1": "1", "2": true, "3": 2147483647, "4": "9223372036854775807", "5": 1.5, "6": -0.25, "7": "1234.56", "8": "12345678.9012", "9": "1234567890.0123456789", "10": "2026-01-15", "11": "00:00:00.000000", "12": "2026-01-15T10:30:00.123456", "13": "2026-01-15T10:30:00.123456+00:00", "14": "Grüße, 世界 🧊", "15": "f79c3e09-677c-4bbd-a479-3f349cb785e7", "16": "000102FF", "17": "DEADBEEF"},
{"1": "2", "2": false, "3": -2147483648, "4": "-9223372036854775808", "5": -0.375, "6": 1048576.125, "7": "-9999999.99", "8": "-99999999999999.9999", "9": "-9999999999999999999999999999.9999999999", "10": "1969-12-31", "11": "23:59:59.999999", "12": "1969-12-31T23:59:59.999999", "13": "1969-12-31T23:59:59.999999+00:00", "14": "", "15": "ffffffff-ffff-ffff-ffff-ffffffffffff", "16": "FFFFFFFF", "17": ""},
{"1": "3", "2": null, "3": null, "4": null, "5": null, "6": null, "7": null, "8": null, "9": null, "10": null, "11": null, "12": null, "13": null, "14": null, "15": null, "16": null, "17": null}
]
}
]
}
Binary file not shown.
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"format-version":2,"table-uuid":"c0589ff2-c22b-4995-b137-0a0570580058","location":"file:///tmp/iceberg-verification/reference-tables/v2/primitive-types-parquet-data","last-sequence-number":1,"last-updated-ms":1794842319830,"last-column-id":17,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"bool_col","required":false,"type":"boolean"},{"id":3,"name":"int_col","required":false,"type":"int"},{"id":4,"name":"long_col","required":false,"type":"long"},{"id":5,"name":"float_col","required":false,"type":"float"},{"id":6,"name":"double_col","required":false,"type":"double"},{"id":7,"name":"decimal_9_2","required":false,"type":"decimal(9, 2)"},{"id":8,"name":"decimal_18_4","required":false,"type":"decimal(18, 4)"},{"id":9,"name":"decimal_38_10","required":false,"type":"decimal(38, 10)"},{"id":10,"name":"date_col","required":false,"type":"date"},{"id":11,"name":"time_col","required":false,"type":"time"},{"id":12,"name":"timestamp_col","required":false,"type":"timestamp"},{"id":13,"name":"timestamptz_col","required":false,"type":"timestamptz"},{"id":14,"name":"string_col","required":false,"type":"string"},{"id":15,"name":"uuid_col","required":false,"type":"uuid"},{"id":16,"name":"fixed_col","required":false,"type":"fixed[4]"},{"id":17,"name":"binary_col","required":false,"type":"binary"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"write.parquet.compression-codec":"zstd"},"current-snapshot-id":3884808864619474138,"refs":{"main":{"snapshot-id":3884808864619474138,"type":"branch"}},"snapshots":[{"sequence-number":1,"snapshot-id":3884808864619474138,"timestamp-ms":1794842319823,"summary":{"operation":"append","manifests-created":"1","manifests-kept":"0","manifests-replaced":"0","added-data-files":"1","added-records":"3","added-files-size":"4936","changed-partition-count":"1","total-records":"3","total-files-size":"4936","total-data-files":"1","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","iceberg-version":"Apache Iceberg 1.11.0 (commit 6976e020b894f6a6777704df2b8c4458cb291ae9)"},"manifest-list":"file:/tmp/iceberg-verification/reference-tables/v2/primitive-types-parquet-data/metadata/snap-3884808864619474138-1-0d0374c5-3a2f-4285-8c18-282c5fee3383.avro","schema-id":0}],"statistics":[],"partition-statistics":[],"snapshot-log":[{"timestamp-ms":1794842319823,"snapshot-id":3884808864619474138}],"metadata-log":[{"timestamp-ms":1794842319245,"metadata-file":"file:/tmp/iceberg-verification/reference-tables/v2/primitive-types-parquet-data/metadata/v1.metadata.json"}]}