diff --git a/pyiceberg/avro/resolver.py b/pyiceberg/avro/resolver.py index f99438bcef..acdbc3013c 100644 --- a/pyiceberg/avro/resolver.py +++ b/pyiceberg/avro/resolver.py @@ -430,7 +430,7 @@ def struct(self, struct: StructType, expected_struct: IcebergType | None, field_ if isinstance(read_field, NestedField) and read_field.initial_default is not None: # The field is not in the file, but there is a default value # and that one can be required - results.append((pos, DefaultReader(read_field.initial_default))) + results.append((pos, DefaultReader(read_field._initial_default_value))) elif read_field.required: raise ResolveError(f"{read_field} is non-optional, and not part of the file schema") else: diff --git a/pyiceberg/expressions/visitors.py b/pyiceberg/expressions/visitors.py index 7504548507..03594dc179 100644 --- a/pyiceberg/expressions/visitors.py +++ b/pyiceberg/expressions/visitors.py @@ -924,7 +924,9 @@ def visit_bound_predicate(self, predicate: BoundPredicate) -> BooleanExpression: # https://iceberg.apache.org/spec/#column-projection # Evaluate column projection first if it exists, otherwise default to the initial-default-value field_value = ( - self.projected_field_values[field_id] if field.field_id in self.projected_field_values else field.initial_default + self.projected_field_values[field_id] + if field.field_id in self.projected_field_values + else field._initial_default_value ) return ( AlwaysTrue() diff --git a/pyiceberg/typedef.py b/pyiceberg/typedef.py index de4d036fe7..4bf2aadebf 100644 --- a/pyiceberg/typedef.py +++ b/pyiceberg/typedef.py @@ -191,7 +191,9 @@ def _bind(cls, struct: StructType, **arguments: Any) -> Self: field_names = {field.name for field in struct.fields} if unknown_fields := arguments.keys() - field_names: raise TypeError(f"Unexpected {cls.__name__} fields: {', '.join(sorted(unknown_fields))}") - return cls(*[arguments[field.name] if field.name in arguments else field.initial_default for field in struct.fields]) + return cls( + *[arguments[field.name] if field.name in arguments else field._initial_default_value for field in struct.fields] + ) def __init__(self, *data: Any) -> None: self._data = list(data) diff --git a/pyiceberg/types.py b/pyiceberg/types.py index 3295beb666..b981aa3ce4 100644 --- a/pyiceberg/types.py +++ b/pyiceberg/types.py @@ -432,6 +432,15 @@ class NestedField(IcebergType): initial_default: DefaultValue | None = Field(alias="initial-default", default=None, repr=True) # type: ignore write_default: DefaultValue | None = Field(alias="write-default", default=None, repr=True) # type: ignore + @property + def _initial_default_value(self) -> object: + """Return the initial-default as bound literals hold it: days or microseconds for temporal types.""" + if self.initial_default is not None and isinstance(self.field_type, (DateType, TimeType, TimestampType, TimestamptzType)): + from pyiceberg.expressions.literals import literal + + return literal(self.initial_default).to(self.field_type).value + return self.initial_default + @field_validator("field_type", mode="before") def convert_field_type(cls, v: Any) -> IcebergType: """Convert string values into IcebergType instances.""" diff --git a/tests/avro/test_resolver.py b/tests/avro/test_resolver.py index 106e4ad8ff..113f1be096 100644 --- a/tests/avro/test_resolver.py +++ b/tests/avro/test_resolver.py @@ -16,6 +16,7 @@ # under the License. from collections.abc import Callable +from datetime import date from tempfile import TemporaryDirectory import pytest @@ -55,6 +56,7 @@ from pyiceberg.types import ( BinaryType, BooleanType, + DateType, DecimalType, DoubleType, FloatType, @@ -333,6 +335,20 @@ def test_resolver_initial_value() -> None: ) +@pytest.mark.parametrize("decoder_class", AVAILABLE_DECODERS) +def test_resolver_temporal_initial_value_reads_like_stored_value(decoder_class: Callable[[bytes], BinaryDecoder]) -> None: + """A defaulted date must read as the same internal value as a date stored in the file.""" + read_schema = Schema( + NestedField(2, "d", DateType(), required=True, initial_default=date(2020, 1, 1)), + schema_id=2, + ) + stored = resolve_reader(Schema(NestedField(2, "d", DateType(), required=True), schema_id=1), read_schema) + defaulted = resolve_reader(Schema(schema_id=1), read_schema) + + # 2020-01-01 is 18262 days after the epoch, zigzag and varint encoded as an Avro int + assert stored.read(decoder_class(b"\xac\x9d\x02"))[0] == defaulted.read(decoder_class(b""))[0] == 18262 + + def test_resolve_writer() -> None: actual = resolve_writer(record_schema=MANIFEST_ENTRY_SCHEMAS[2], file_schema=MANIFEST_ENTRY_SCHEMAS[1]) expected = StructWriter( diff --git a/tests/expressions/test_visitors.py b/tests/expressions/test_visitors.py index 2bc77d50e9..e5d8da55b6 100644 --- a/tests/expressions/test_visitors.py +++ b/tests/expressions/test_visitors.py @@ -16,6 +16,7 @@ # under the License. # pylint:disable=redefined-outer-name +import datetime as dt from concurrent.futures import ThreadPoolExecutor from threading import Event from typing import Any @@ -86,6 +87,7 @@ from pyiceberg.types import ( BinaryType, BooleanType, + DateType, DoubleType, FloatType, IcebergType, @@ -93,6 +95,9 @@ NestedField, PrimitiveType, StringType, + TimestampType, + TimestamptzType, + TimeType, ) @@ -1850,6 +1855,45 @@ def test_translate_column_names_missing_column_match_explicit_null() -> None: assert translated_expr == AlwaysTrue() +def test_translate_column_names_missing_column_with_temporal_initial_default() -> None: + """Test translate_column_names compares temporal initial_default values in the form the bound literal uses.""" + file_schema = Schema(NestedField(field_id=1, name="existing_col", field_type=StringType(), required=False), schema_id=1) + cases = [ + (DateType(), dt.date(2020, 1, 1), dt.date(2019, 12, 31), dt.date(2020, 1, 2)), + (TimeType(), dt.time(7, 15, 30), dt.time(7, 15, 29), dt.time(7, 15, 31)), + (TimestampType(), dt.datetime(2020, 1, 1, 12, 30), dt.datetime(2020, 1, 1, 12, 29), dt.datetime(2020, 1, 1, 12, 31)), + ( + TimestamptzType(), + dt.datetime(2020, 1, 1, 12, 30, tzinfo=dt.UTC), + dt.datetime(2020, 1, 1, 12, 29, tzinfo=dt.UTC), + dt.datetime(2020, 1, 1, 12, 31, tzinfo=dt.UTC), + ), + ] + for field_type, default, earlier, later in cases: + original_schema = Schema( + NestedField(field_id=2, name="missing_col", field_type=field_type, required=False, initial_default=default), + schema_id=1, + ) + for unbound_expr, expected in [ + (EqualTo("missing_col", default), AlwaysTrue()), + (EqualTo("missing_col", later), AlwaysFalse()), + (NotEqualTo("missing_col", default), AlwaysFalse()), + (In("missing_col", {earlier, default}), AlwaysTrue()), + (GreaterThan("missing_col", earlier), AlwaysTrue()), + ]: + bound_expr = visit(unbound_expr, visitor=BindVisitor(schema=original_schema, case_sensitive=True)) + assert translate_column_names(bound_expr, file_schema, case_sensitive=True) == expected, (field_type, unbound_expr) + + # A missing optional column without a default is null, and the conversion must leave it alone + null_schema = Schema(NestedField(2, "missing_col", field_type, required=False)) + for unbound_expr, expected in [ + (IsNull("missing_col"), AlwaysTrue()), + (EqualTo("missing_col", default), AlwaysFalse()), + ]: + bound_expr = visit(unbound_expr, visitor=BindVisitor(schema=null_schema, case_sensitive=True)) + assert translate_column_names(bound_expr, file_schema) == expected, (field_type, unbound_expr) + + def test_translate_column_names_missing_column_with_initial_default() -> None: """Test translate_column_names when missing column's initial_default matches expression.""" # Original schema diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index ca21fd05f0..3f3d42b357 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -1637,6 +1637,55 @@ def test_identity_transform_column_projection_uuid(tmp_path: str, catalog: InMem assert result["other_field"].to_pylist() == expected, row_filter +def test_initial_default_temporal_filter_matches_projected_value(tmpdir: str) -> None: + """A filter on a temporal column with an initial-default keeps the rows that project that default.""" + table_schema = Schema( + NestedField(1, "id", LongType(), required=False), + NestedField(2, "date_col", DateType(), required=False, initial_default=date(2020, 1, 1)), + NestedField(3, "time_col", TimeType(), required=False, initial_default=time(7, 15, 30)), + NestedField(4, "ts_col", TimestampType(), required=False, initial_default=datetime(2020, 1, 1, 12, 30)), + NestedField(5, "tstz_col", TimestamptzType(), required=False, initial_default=datetime(2020, 1, 1, 12, 30, tzinfo=UTC)), + ) + partition_spec = PartitionSpec() + table_metadata = TableMetadataV3( + location=str(tmpdir), last_column_id=5, schemas=[table_schema], partition_specs=[partition_spec], next_row_id=0 + ) + + # The file was written before the temporal columns were added, so it only has `id` + arrow_schema = pa.schema([pa.field("id", pa.int64(), metadata={PYARROW_PARQUET_FIELD_ID_KEY: "1"})]) + file_path = _write_table_to_file(f"{tmpdir}/old.parquet", arrow_schema, pa.table({"id": [1, 2, 3]}, schema=arrow_schema)) + data_file = DataFile.from_args( + content=DataFileContent.DATA, + file_path=file_path, + file_format=FileFormat.PARQUET, + partition=Record(), + record_count=3, + file_size_in_bytes=os.path.getsize(file_path), + spec_id=partition_spec.spec_id, + ) + + def scan(row_filter: BooleanExpression) -> pa.Table: + return ArrowScan(table_metadata, PyArrowFileIO(), table_schema, row_filter).to_table([FileScanTask(data_file)]) + + assert scan(AlwaysTrue()).to_pydict() == { + "id": [1, 2, 3], + "date_col": [date(2020, 1, 1)] * 3, + "time_col": [time(7, 15, 30)] * 3, + "ts_col": [datetime(2020, 1, 1, 12, 30)] * 3, + "tstz_col": [datetime(2020, 1, 1, 12, 30, tzinfo=UTC)] * 3, + } + + for row_filter, expected in [ + (EqualTo("date_col", date(2020, 1, 1)), [1, 2, 3]), + (Not(EqualTo("date_col", date(2020, 1, 1))), []), + (GreaterThan("date_col", date(2019, 12, 31)), [1, 2, 3]), + (EqualTo("time_col", time(7, 15, 30)), [1, 2, 3]), + (EqualTo("ts_col", datetime(2020, 1, 1, 12, 30)), [1, 2, 3]), + (EqualTo("tstz_col", datetime(2020, 1, 1, 12, 30, tzinfo=UTC)), [1, 2, 3]), + ]: + assert scan(row_filter)["id"].to_pylist() == expected, row_filter + + def test_identity_transform_null_partition_value_takes_precedence_over_initial_default(tmpdir: str) -> None: """A null identity partition value must project as null, not as the field's initial-default.""" table_schema = Schema( diff --git a/tests/test_typedef.py b/tests/test_typedef.py index 32e4e54c2d..6c35bc7d03 100644 --- a/tests/test_typedef.py +++ b/tests/test_typedef.py @@ -14,10 +14,12 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from datetime import date + import pytest from pyiceberg.typedef import FrozenDict, KeyDefaultDict, Record -from pyiceberg.types import IntegerType, NestedField, StructType +from pyiceberg.types import DateType, IntegerType, NestedField, StructType def test_setitem_frozendict() -> None: @@ -57,6 +59,13 @@ def test_record_bind_rejects_unknown_arguments() -> None: Record._bind(struct, known=1, unknown=2) +def test_record_bind_fills_temporal_initial_default_in_internal_form() -> None: + struct = StructType(NestedField(1, "d", DateType(), initial_default=date(2020, 1, 1))) + + # Records hold the internal form, days since the epoch, as values read from a file do + assert Record._bind(struct)[0] == 18262 + + def test_record_bind_rejects_non_schema_property() -> None: class RecordWithProperty(Record): @property