Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyiceberg/avro/resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -430,7 +430,7 @@ def struct(self, struct: StructType, expected_struct: IcebergType | None, field_
if isinstance(read_field, NestedField) and read_field.initial_default is not None:
# The field is not in the file, but there is a default value
# and that one can be required
results.append((pos, DefaultReader(read_field.initial_default)))
results.append((pos, DefaultReader(read_field._initial_default_value)))
elif read_field.required:
raise ResolveError(f"{read_field} is non-optional, and not part of the file schema")
else:
Expand Down
4 changes: 3 additions & 1 deletion pyiceberg/expressions/visitors.py
Original file line number Diff line number Diff line change
Expand Up @@ -924,7 +924,9 @@ def visit_bound_predicate(self, predicate: BoundPredicate) -> BooleanExpression:
# https://iceberg.apache.org/spec/#column-projection
# Evaluate column projection first if it exists, otherwise default to the initial-default-value
field_value = (
self.projected_field_values[field_id] if field.field_id in self.projected_field_values else field.initial_default
self.projected_field_values[field_id]
if field.field_id in self.projected_field_values
else field._initial_default_value
)
return (
AlwaysTrue()
Expand Down
4 changes: 3 additions & 1 deletion pyiceberg/typedef.py
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,9 @@ def _bind(cls, struct: StructType, **arguments: Any) -> Self:
field_names = {field.name for field in struct.fields}
if unknown_fields := arguments.keys() - field_names:
raise TypeError(f"Unexpected {cls.__name__} fields: {', '.join(sorted(unknown_fields))}")
return cls(*[arguments[field.name] if field.name in arguments else field.initial_default for field in struct.fields])
return cls(
*[arguments[field.name] if field.name in arguments else field._initial_default_value for field in struct.fields]
)

def __init__(self, *data: Any) -> None:
self._data = list(data)
Expand Down
9 changes: 9 additions & 0 deletions pyiceberg/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -432,6 +432,15 @@ class NestedField(IcebergType):
initial_default: DefaultValue | None = Field(alias="initial-default", default=None, repr=True) # type: ignore
write_default: DefaultValue | None = Field(alias="write-default", default=None, repr=True) # type: ignore

@property
def _initial_default_value(self) -> object:
"""Return the initial-default as bound literals hold it: days or microseconds for temporal types."""
if self.initial_default is not None and isinstance(self.field_type, (DateType, TimeType, TimestampType, TimestamptzType)):
from pyiceberg.expressions.literals import literal

return literal(self.initial_default).to(self.field_type).value
return self.initial_default
Comment on lines +435 to +442

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wouldn't it make more sense to do this when we create the type? We know the type then right away.


@field_validator("field_type", mode="before")
def convert_field_type(cls, v: Any) -> IcebergType:
"""Convert string values into IcebergType instances."""
Expand Down
16 changes: 16 additions & 0 deletions tests/avro/test_resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
# under the License.

from collections.abc import Callable
from datetime import date
from tempfile import TemporaryDirectory

import pytest
Expand Down Expand Up @@ -55,6 +56,7 @@
from pyiceberg.types import (
BinaryType,
BooleanType,
DateType,
DecimalType,
DoubleType,
FloatType,
Expand Down Expand Up @@ -333,6 +335,20 @@ def test_resolver_initial_value() -> None:
)


@pytest.mark.parametrize("decoder_class", AVAILABLE_DECODERS)
def test_resolver_temporal_initial_value_reads_like_stored_value(decoder_class: Callable[[bytes], BinaryDecoder]) -> None:
"""A defaulted date must read as the same internal value as a date stored in the file."""
read_schema = Schema(
NestedField(2, "d", DateType(), required=True, initial_default=date(2020, 1, 1)),
schema_id=2,
)
stored = resolve_reader(Schema(NestedField(2, "d", DateType(), required=True), schema_id=1), read_schema)
defaulted = resolve_reader(Schema(schema_id=1), read_schema)

# 2020-01-01 is 18262 days after the epoch, zigzag and varint encoded as an Avro int
assert stored.read(decoder_class(b"\xac\x9d\x02"))[0] == defaulted.read(decoder_class(b""))[0] == 18262


def test_resolve_writer() -> None:
actual = resolve_writer(record_schema=MANIFEST_ENTRY_SCHEMAS[2], file_schema=MANIFEST_ENTRY_SCHEMAS[1])
expected = StructWriter(
Expand Down
44 changes: 44 additions & 0 deletions tests/expressions/test_visitors.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
# under the License.
# pylint:disable=redefined-outer-name

import datetime as dt
from concurrent.futures import ThreadPoolExecutor
from threading import Event
from typing import Any
Expand Down Expand Up @@ -86,13 +87,17 @@
from pyiceberg.types import (
BinaryType,
BooleanType,
DateType,
DoubleType,
FloatType,
IcebergType,
IntegerType,
NestedField,
PrimitiveType,
StringType,
TimestampType,
TimestamptzType,
TimeType,
)


Expand Down Expand Up @@ -1850,6 +1855,45 @@ def test_translate_column_names_missing_column_match_explicit_null() -> None:
assert translated_expr == AlwaysTrue()


def test_translate_column_names_missing_column_with_temporal_initial_default() -> None:
"""Test translate_column_names compares temporal initial_default values in the form the bound literal uses."""
file_schema = Schema(NestedField(field_id=1, name="existing_col", field_type=StringType(), required=False), schema_id=1)
cases = [
(DateType(), dt.date(2020, 1, 1), dt.date(2019, 12, 31), dt.date(2020, 1, 2)),
(TimeType(), dt.time(7, 15, 30), dt.time(7, 15, 29), dt.time(7, 15, 31)),
(TimestampType(), dt.datetime(2020, 1, 1, 12, 30), dt.datetime(2020, 1, 1, 12, 29), dt.datetime(2020, 1, 1, 12, 31)),
(
TimestamptzType(),
dt.datetime(2020, 1, 1, 12, 30, tzinfo=dt.UTC),
dt.datetime(2020, 1, 1, 12, 29, tzinfo=dt.UTC),
dt.datetime(2020, 1, 1, 12, 31, tzinfo=dt.UTC),
),
]
for field_type, default, earlier, later in cases:
original_schema = Schema(
NestedField(field_id=2, name="missing_col", field_type=field_type, required=False, initial_default=default),
schema_id=1,
)
for unbound_expr, expected in [
(EqualTo("missing_col", default), AlwaysTrue()),
(EqualTo("missing_col", later), AlwaysFalse()),
(NotEqualTo("missing_col", default), AlwaysFalse()),
(In("missing_col", {earlier, default}), AlwaysTrue()),
(GreaterThan("missing_col", earlier), AlwaysTrue()),
]:
bound_expr = visit(unbound_expr, visitor=BindVisitor(schema=original_schema, case_sensitive=True))
assert translate_column_names(bound_expr, file_schema, case_sensitive=True) == expected, (field_type, unbound_expr)

# A missing optional column without a default is null, and the conversion must leave it alone
null_schema = Schema(NestedField(2, "missing_col", field_type, required=False))
for unbound_expr, expected in [
(IsNull("missing_col"), AlwaysTrue()),
(EqualTo("missing_col", default), AlwaysFalse()),
]:
bound_expr = visit(unbound_expr, visitor=BindVisitor(schema=null_schema, case_sensitive=True))
assert translate_column_names(bound_expr, file_schema) == expected, (field_type, unbound_expr)


def test_translate_column_names_missing_column_with_initial_default() -> None:
"""Test translate_column_names when missing column's initial_default matches expression."""
# Original schema
Expand Down
49 changes: 49 additions & 0 deletions tests/io/test_pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -1637,6 +1637,55 @@ def test_identity_transform_column_projection_uuid(tmp_path: str, catalog: InMem
assert result["other_field"].to_pylist() == expected, row_filter


def test_initial_default_temporal_filter_matches_projected_value(tmpdir: str) -> None:
"""A filter on a temporal column with an initial-default keeps the rows that project that default."""
table_schema = Schema(
NestedField(1, "id", LongType(), required=False),
NestedField(2, "date_col", DateType(), required=False, initial_default=date(2020, 1, 1)),
NestedField(3, "time_col", TimeType(), required=False, initial_default=time(7, 15, 30)),
NestedField(4, "ts_col", TimestampType(), required=False, initial_default=datetime(2020, 1, 1, 12, 30)),
NestedField(5, "tstz_col", TimestamptzType(), required=False, initial_default=datetime(2020, 1, 1, 12, 30, tzinfo=UTC)),
)
partition_spec = PartitionSpec()
table_metadata = TableMetadataV3(
location=str(tmpdir), last_column_id=5, schemas=[table_schema], partition_specs=[partition_spec], next_row_id=0
)

# The file was written before the temporal columns were added, so it only has `id`
arrow_schema = pa.schema([pa.field("id", pa.int64(), metadata={PYARROW_PARQUET_FIELD_ID_KEY: "1"})])
file_path = _write_table_to_file(f"{tmpdir}/old.parquet", arrow_schema, pa.table({"id": [1, 2, 3]}, schema=arrow_schema))
data_file = DataFile.from_args(
content=DataFileContent.DATA,
file_path=file_path,
file_format=FileFormat.PARQUET,
partition=Record(),
record_count=3,
file_size_in_bytes=os.path.getsize(file_path),
spec_id=partition_spec.spec_id,
)

def scan(row_filter: BooleanExpression) -> pa.Table:
return ArrowScan(table_metadata, PyArrowFileIO(), table_schema, row_filter).to_table([FileScanTask(data_file)])

assert scan(AlwaysTrue()).to_pydict() == {
"id": [1, 2, 3],
"date_col": [date(2020, 1, 1)] * 3,
"time_col": [time(7, 15, 30)] * 3,
"ts_col": [datetime(2020, 1, 1, 12, 30)] * 3,
"tstz_col": [datetime(2020, 1, 1, 12, 30, tzinfo=UTC)] * 3,
}

for row_filter, expected in [
(EqualTo("date_col", date(2020, 1, 1)), [1, 2, 3]),
(Not(EqualTo("date_col", date(2020, 1, 1))), []),
(GreaterThan("date_col", date(2019, 12, 31)), [1, 2, 3]),
(EqualTo("time_col", time(7, 15, 30)), [1, 2, 3]),
(EqualTo("ts_col", datetime(2020, 1, 1, 12, 30)), [1, 2, 3]),
(EqualTo("tstz_col", datetime(2020, 1, 1, 12, 30, tzinfo=UTC)), [1, 2, 3]),
]:
assert scan(row_filter)["id"].to_pylist() == expected, row_filter


def test_identity_transform_null_partition_value_takes_precedence_over_initial_default(tmpdir: str) -> None:
"""A null identity partition value must project as null, not as the field's initial-default."""
table_schema = Schema(
Expand Down
11 changes: 10 additions & 1 deletion tests/test_typedef.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,12 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from datetime import date

import pytest

from pyiceberg.typedef import FrozenDict, KeyDefaultDict, Record
from pyiceberg.types import IntegerType, NestedField, StructType
from pyiceberg.types import DateType, IntegerType, NestedField, StructType


def test_setitem_frozendict() -> None:
Expand Down Expand Up @@ -57,6 +59,13 @@ def test_record_bind_rejects_unknown_arguments() -> None:
Record._bind(struct, known=1, unknown=2)


def test_record_bind_fills_temporal_initial_default_in_internal_form() -> None:
struct = StructType(NestedField(1, "d", DateType(), initial_default=date(2020, 1, 1)))

# Records hold the internal form, days since the epoch, as values read from a file do
assert Record._bind(struct)[0] == 18262


def test_record_bind_rejects_non_schema_property() -> None:
class RecordWithProperty(Record):
@property
Expand Down
Loading