Skip to content

Commit 1985222

Browse files
rahulsmahadevIsaac
andcommitted
Use internal values for temporal initial-defaults
A file that lacks a date, time, timestamp or timestamptz column with a non-null initial-default compared the Python value with bound literals, which hold days or microseconds. Equality and IN dropped matching rows, != kept them, and range predicates raised TypeError. Add a private NestedField accessor for the internal form, used by the translator, the Avro default reader and Record defaults. initial_default is unchanged. Co-authored-by: Isaac <no-reply@databricks.com>
1 parent 7c425ed commit 1985222

8 files changed

Lines changed: 135 additions & 4 deletions

File tree

‎pyiceberg/avro/resolver.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -430,7 +430,7 @@ def struct(self, struct: StructType, expected_struct: IcebergType | None, field_
430430
if isinstance(read_field, NestedField) and read_field.initial_default is not None:
431431
# The field is not in the file, but there is a default value
432432
# and that one can be required
433-
results.append((pos, DefaultReader(read_field.initial_default)))
433+
results.append((pos, DefaultReader(read_field._initial_default_value)))
434434
elif read_field.required:
435435
raise ResolveError(f"{read_field} is non-optional, and not part of the file schema")
436436
else:

‎pyiceberg/expressions/visitors.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -924,7 +924,9 @@ def visit_bound_predicate(self, predicate: BoundPredicate) -> BooleanExpression:
924924
# https://iceberg.apache.org/spec/#column-projection
925925
# Evaluate column projection first if it exists, otherwise default to the initial-default-value
926926
field_value = (
927-
self.projected_field_values[field_id] if field.field_id in self.projected_field_values else field.initial_default
927+
self.projected_field_values[field_id]
928+
if field.field_id in self.projected_field_values
929+
else field._initial_default_value
928930
)
929931
return (
930932
AlwaysTrue()

‎pyiceberg/typedef.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,9 @@ def _bind(cls, struct: StructType, **arguments: Any) -> Self:
191191
field_names = {field.name for field in struct.fields}
192192
if unknown_fields := arguments.keys() - field_names:
193193
raise TypeError(f"Unexpected {cls.__name__} fields: {', '.join(sorted(unknown_fields))}")
194-
return cls(*[arguments[field.name] if field.name in arguments else field.initial_default for field in struct.fields])
194+
return cls(
195+
*[arguments[field.name] if field.name in arguments else field._initial_default_value for field in struct.fields]
196+
)
195197

196198
def __init__(self, *data: Any) -> None:
197199
self._data = list(data)

‎pyiceberg/types.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -432,6 +432,15 @@ class NestedField(IcebergType):
432432
initial_default: DefaultValue | None = Field(alias="initial-default", default=None, repr=True) # type: ignore
433433
write_default: DefaultValue | None = Field(alias="write-default", default=None, repr=True) # type: ignore
434434

435+
@property
436+
def _initial_default_value(self) -> object:
437+
"""Return the initial-default as bound literals hold it: days or microseconds for temporal types."""
438+
if self.initial_default is not None and isinstance(self.field_type, (DateType, TimeType, TimestampType, TimestamptzType)):
439+
from pyiceberg.expressions.literals import literal
440+
441+
return literal(self.initial_default).to(self.field_type).value
442+
return self.initial_default
443+
435444
@field_validator("field_type", mode="before")
436445
def convert_field_type(cls, v: Any) -> IcebergType:
437446
"""Convert string values into IcebergType instances."""

‎tests/avro/test_resolver.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
# under the License.
1717

1818
from collections.abc import Callable
19+
from datetime import date
1920
from tempfile import TemporaryDirectory
2021

2122
import pytest
@@ -55,6 +56,7 @@
5556
from pyiceberg.types import (
5657
BinaryType,
5758
BooleanType,
59+
DateType,
5860
DecimalType,
5961
DoubleType,
6062
FloatType,
@@ -333,6 +335,20 @@ def test_resolver_initial_value() -> None:
333335
)
334336

335337

338+
@pytest.mark.parametrize("decoder_class", AVAILABLE_DECODERS)
339+
def test_resolver_temporal_initial_value_reads_like_stored_value(decoder_class: Callable[[bytes], BinaryDecoder]) -> None:
340+
"""A defaulted date must read as the same internal value as a date stored in the file."""
341+
read_schema = Schema(
342+
NestedField(2, "d", DateType(), required=True, initial_default=date(2020, 1, 1)),
343+
schema_id=2,
344+
)
345+
stored = resolve_reader(Schema(NestedField(2, "d", DateType(), required=True), schema_id=1), read_schema)
346+
defaulted = resolve_reader(Schema(schema_id=1), read_schema)
347+
348+
# 2020-01-01 is 18262 days after the epoch, zigzag and varint encoded as an Avro int
349+
assert stored.read(decoder_class(b"\xac\x9d\x02"))[0] == defaulted.read(decoder_class(b""))[0] == 18262
350+
351+
336352
def test_resolve_writer() -> None:
337353
actual = resolve_writer(record_schema=MANIFEST_ENTRY_SCHEMAS[2], file_schema=MANIFEST_ENTRY_SCHEMAS[1])
338354
expected = StructWriter(

‎tests/expressions/test_visitors.py‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
# under the License.
1717
# pylint:disable=redefined-outer-name
1818

19+
import datetime as dt
1920
from concurrent.futures import ThreadPoolExecutor
2021
from threading import Event
2122
from typing import Any
@@ -86,13 +87,17 @@
8687
from pyiceberg.types import (
8788
BinaryType,
8889
BooleanType,
90+
DateType,
8991
DoubleType,
9092
FloatType,
9193
IcebergType,
9294
IntegerType,
9395
NestedField,
9496
PrimitiveType,
9597
StringType,
98+
TimestampType,
99+
TimestamptzType,
100+
TimeType,
96101
)
97102

98103

@@ -1850,6 +1855,45 @@ def test_translate_column_names_missing_column_match_explicit_null() -> None:
18501855
assert translated_expr == AlwaysTrue()
18511856

18521857

1858+
def test_translate_column_names_missing_column_with_temporal_initial_default() -> None:
1859+
"""Test translate_column_names compares temporal initial_default values in the form the bound literal uses."""
1860+
file_schema = Schema(NestedField(field_id=1, name="existing_col", field_type=StringType(), required=False), schema_id=1)
1861+
cases = [
1862+
(DateType(), dt.date(2020, 1, 1), dt.date(2019, 12, 31), dt.date(2020, 1, 2)),
1863+
(TimeType(), dt.time(7, 15, 30), dt.time(7, 15, 29), dt.time(7, 15, 31)),
1864+
(TimestampType(), dt.datetime(2020, 1, 1, 12, 30), dt.datetime(2020, 1, 1, 12, 29), dt.datetime(2020, 1, 1, 12, 31)),
1865+
(
1866+
TimestamptzType(),
1867+
dt.datetime(2020, 1, 1, 12, 30, tzinfo=dt.UTC),
1868+
dt.datetime(2020, 1, 1, 12, 29, tzinfo=dt.UTC),
1869+
dt.datetime(2020, 1, 1, 12, 31, tzinfo=dt.UTC),
1870+
),
1871+
]
1872+
for field_type, default, earlier, later in cases:
1873+
original_schema = Schema(
1874+
NestedField(field_id=2, name="missing_col", field_type=field_type, required=False, initial_default=default),
1875+
schema_id=1,
1876+
)
1877+
for unbound_expr, expected in [
1878+
(EqualTo("missing_col", default), AlwaysTrue()),
1879+
(EqualTo("missing_col", later), AlwaysFalse()),
1880+
(NotEqualTo("missing_col", default), AlwaysFalse()),
1881+
(In("missing_col", {earlier, default}), AlwaysTrue()),
1882+
(GreaterThan("missing_col", earlier), AlwaysTrue()),
1883+
]:
1884+
bound_expr = visit(unbound_expr, visitor=BindVisitor(schema=original_schema, case_sensitive=True))
1885+
assert translate_column_names(bound_expr, file_schema, case_sensitive=True) == expected, (field_type, unbound_expr)
1886+
1887+
# A missing optional column without a default is null, and the conversion must leave it alone
1888+
null_schema = Schema(NestedField(2, "missing_col", field_type, required=False))
1889+
for unbound_expr, expected in [
1890+
(IsNull("missing_col"), AlwaysTrue()),
1891+
(EqualTo("missing_col", default), AlwaysFalse()),
1892+
]:
1893+
bound_expr = visit(unbound_expr, visitor=BindVisitor(schema=null_schema, case_sensitive=True))
1894+
assert translate_column_names(bound_expr, file_schema) == expected, (field_type, unbound_expr)
1895+
1896+
18531897
def test_translate_column_names_missing_column_with_initial_default() -> None:
18541898
"""Test translate_column_names when missing column's initial_default matches expression."""
18551899
# Original schema

‎tests/io/test_pyarrow.py‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1637,6 +1637,55 @@ def test_identity_transform_column_projection_uuid(tmp_path: str, catalog: InMem
16371637
assert result["other_field"].to_pylist() == expected, row_filter
16381638

16391639

1640+
def test_initial_default_temporal_filter_matches_projected_value(tmpdir: str) -> None:
1641+
"""A filter on a temporal column with an initial-default keeps the rows that project that default."""
1642+
table_schema = Schema(
1643+
NestedField(1, "id", LongType(), required=False),
1644+
NestedField(2, "date_col", DateType(), required=False, initial_default=date(2020, 1, 1)),
1645+
NestedField(3, "time_col", TimeType(), required=False, initial_default=time(7, 15, 30)),
1646+
NestedField(4, "ts_col", TimestampType(), required=False, initial_default=datetime(2020, 1, 1, 12, 30)),
1647+
NestedField(5, "tstz_col", TimestamptzType(), required=False, initial_default=datetime(2020, 1, 1, 12, 30, tzinfo=UTC)),
1648+
)
1649+
partition_spec = PartitionSpec()
1650+
table_metadata = TableMetadataV3(
1651+
location=str(tmpdir), last_column_id=5, schemas=[table_schema], partition_specs=[partition_spec], next_row_id=0
1652+
)
1653+
1654+
# The file was written before the temporal columns were added, so it only has `id`
1655+
arrow_schema = pa.schema([pa.field("id", pa.int64(), metadata={PYARROW_PARQUET_FIELD_ID_KEY: "1"})])
1656+
file_path = _write_table_to_file(f"{tmpdir}/old.parquet", arrow_schema, pa.table({"id": [1, 2, 3]}, schema=arrow_schema))
1657+
data_file = DataFile.from_args(
1658+
content=DataFileContent.DATA,
1659+
file_path=file_path,
1660+
file_format=FileFormat.PARQUET,
1661+
partition=Record(),
1662+
record_count=3,
1663+
file_size_in_bytes=os.path.getsize(file_path),
1664+
spec_id=partition_spec.spec_id,
1665+
)
1666+
1667+
def scan(row_filter: BooleanExpression) -> pa.Table:
1668+
return ArrowScan(table_metadata, PyArrowFileIO(), table_schema, row_filter).to_table([FileScanTask(data_file)])
1669+
1670+
assert scan(AlwaysTrue()).to_pydict() == {
1671+
"id": [1, 2, 3],
1672+
"date_col": [date(2020, 1, 1)] * 3,
1673+
"time_col": [time(7, 15, 30)] * 3,
1674+
"ts_col": [datetime(2020, 1, 1, 12, 30)] * 3,
1675+
"tstz_col": [datetime(2020, 1, 1, 12, 30, tzinfo=UTC)] * 3,
1676+
}
1677+
1678+
for row_filter, expected in [
1679+
(EqualTo("date_col", date(2020, 1, 1)), [1, 2, 3]),
1680+
(Not(EqualTo("date_col", date(2020, 1, 1))), []),
1681+
(GreaterThan("date_col", date(2019, 12, 31)), [1, 2, 3]),
1682+
(EqualTo("time_col", time(7, 15, 30)), [1, 2, 3]),
1683+
(EqualTo("ts_col", datetime(2020, 1, 1, 12, 30)), [1, 2, 3]),
1684+
(EqualTo("tstz_col", datetime(2020, 1, 1, 12, 30, tzinfo=UTC)), [1, 2, 3]),
1685+
]:
1686+
assert scan(row_filter)["id"].to_pylist() == expected, row_filter
1687+
1688+
16401689
def test_identity_transform_null_partition_value_takes_precedence_over_initial_default(tmpdir: str) -> None:
16411690
"""A null identity partition value must project as null, not as the field's initial-default."""
16421691
table_schema = Schema(

‎tests/test_typedef.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,12 @@
1414
# KIND, either express or implied. See the License for the
1515
# specific language governing permissions and limitations
1616
# under the License.
17+
from datetime import date
18+
1719
import pytest
1820

1921
from pyiceberg.typedef import FrozenDict, KeyDefaultDict, Record
20-
from pyiceberg.types import IntegerType, NestedField, StructType
22+
from pyiceberg.types import DateType, IntegerType, NestedField, StructType
2123

2224

2325
def test_setitem_frozendict() -> None:
@@ -57,6 +59,13 @@ def test_record_bind_rejects_unknown_arguments() -> None:
5759
Record._bind(struct, known=1, unknown=2)
5860

5961

62+
def test_record_bind_fills_temporal_initial_default_in_internal_form() -> None:
63+
struct = StructType(NestedField(1, "d", DateType(), initial_default=date(2020, 1, 1)))
64+
65+
# Records hold the internal form, days since the epoch, as values read from a file do
66+
assert Record._bind(struct)[0] == 18262
67+
68+
6069
def test_record_bind_rejects_non_schema_property() -> None:
6170
class RecordWithProperty(Record):
6271
@property

0 commit comments

Comments
 (0)