Skip to content

Commit 1e6e94a

Browse files
fix: accept up to 9 fractional digits for micros timestamps (#4142)
Widen ISO_TIMESTAMP and ISO_TIMESTAMPTZ to accept 1-9 fractional digits and zone offsets with optional seconds, and parse micros timestamps through the nanosecond path, truncating to microseconds, mirroring the Java implementation. Previously, valid ISO-8601 strings with nanosecond precision were rejected with a ValueError when binding string literals to timestamp/timestamptz columns or parsing JSON single values. Signed-off-by: Shahzeb Ali <126076773+Shahzeb-DataEngineer@users.noreply.github.com>
1 parent 0a2e7ca commit 1e6e94a

3 files changed

Lines changed: 75 additions & 5 deletions

File tree

‎pyiceberg/utils/datetime.py‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,11 @@
2828

2929
EPOCH_DATE = date.fromisoformat("1970-01-01")
3030
EPOCH_TIMESTAMP = datetime.fromisoformat("1970-01-01T00:00:00.000000")
31-
ISO_TIMESTAMP = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(.\d{1,6})?")
31+
ISO_TIMESTAMP = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(.\d{1,9})?")
3232
ISO_TIMESTAMP_NANO = re.compile(r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(.\d{1,6})?(\d{1,3})?")
3333
EPOCH_TIMESTAMPTZ = datetime.fromisoformat("1970-01-01T00:00:00.000000+00:00")
34-
ISO_TIMESTAMPTZ = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(.\d{1,6})?[-+]\d{2}:\d{2}")
35-
ISO_TIMESTAMPTZ_NANO = re.compile(r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(.\d{1,6})?(\d{1,3})?([-+]\d{2}:\d{2})")
34+
ISO_TIMESTAMPTZ = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(.\d{1,9})?[-+]\d{2}:\d{2}(:\d{2})?")
35+
ISO_TIMESTAMPTZ_NANO = re.compile(r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(.\d{1,6})?(\d{1,3})?([-+]\d{2}:\d{2}(:\d{2})?)")
3636

3737

3838
def micros_to_days(timestamp: int) -> int:
@@ -86,7 +86,8 @@ def datetime_to_micros(dt: datetime) -> int:
8686
def timestamp_to_micros(timestamp_str: str) -> int:
8787
"""Convert an ISO-9601 formatted timestamp without zone to microseconds from 1970-01-01T00:00:00.000000."""
8888
if ISO_TIMESTAMP.fullmatch(timestamp_str):
89-
return datetime_to_micros(datetime.fromisoformat(timestamp_str))
89+
# Parse through the nanosecond path and truncate to microseconds, mirroring Java
90+
return nanos_to_micros(timestamp_to_nanos(timestamp_str))
9091
if ISO_TIMESTAMPTZ.fullmatch(timestamp_str):
9192
# When we can match a timestamp without a zone, we can give a more specific error
9293
raise ValueError(f"Zone offset provided, but not expected: {timestamp_str}")
@@ -168,7 +169,8 @@ def millis_to_datetime(millis: int) -> datetime:
168169
def timestamptz_to_micros(timestamptz_str: str) -> int:
169170
"""Convert an ISO-8601 formatted timestamp with zone to microseconds from 1970-01-01T00:00:00.000000+00:00."""
170171
if ISO_TIMESTAMPTZ.fullmatch(timestamptz_str):
171-
return datetime_to_micros(datetime.fromisoformat(timestamptz_str))
172+
# Parse through the nanosecond path and truncate to microseconds, mirroring Java
173+
return nanos_to_micros(timestamptz_to_nanos(timestamptz_str))
172174
if ISO_TIMESTAMP.fullmatch(timestamptz_str):
173175
# When we can match a timestamp without a zone, we can give a more specific error
174176
raise ValueError(f"Missing zone offset: {timestamptz_str} (must be ISO-8601)")

‎tests/test_conversions.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -616,6 +616,23 @@ def test_json_serialize_roundtrip(primitive_type: PrimitiveType, value: Any) ->
616616
assert value == conversions.from_json(primitive_type, conversions.to_json(primitive_type, value))
617617

618618

619+
@pytest.mark.parametrize(
620+
"primitive_type, json_val, expected",
621+
[
622+
(TimestampType(), "2017-11-16T22:31:08.123456789", datetime(2017, 11, 16, 22, 31, 8, 123456)),
623+
(
624+
TimestamptzType(),
625+
"2017-11-16T22:31:08.123456789+00:00",
626+
datetime(2017, 11, 16, 22, 31, 8, 123456, tzinfo=timezone.utc),
627+
),
628+
],
629+
)
630+
def test_json_single_deserialization_with_nano_precision(
631+
primitive_type: PrimitiveType, json_val: str, expected: datetime
632+
) -> None:
633+
assert conversions.from_json(primitive_type, json_val) == expected
634+
635+
619636
def test_unknown_type_conversions() -> None:
620637
"""Unknown values are always null, so they have no binary or partition representation."""
621638
unknown = UnknownType()

‎tests/utils/test_datetime.py‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,9 @@
2828
nanos_to_micros,
2929
time_str_to_nanos,
3030
time_to_nanos,
31+
timestamp_to_micros,
3132
timestamp_to_nanos,
33+
timestamptz_to_micros,
3234
timestamptz_to_nanos,
3335
)
3436

@@ -155,6 +157,55 @@ def test_timestamptz_to_nanos_invalid_timestamp() -> None:
155157
timestamptz_to_nanos("invalid")
156158

157159

160+
@pytest.mark.parametrize(
161+
"timestamp, micros",
162+
[
163+
("1970-01-01T00:00:00", 0),
164+
("2025-02-23T20:21:44.375612", 1740342104375612),
165+
("2025-02-23T20:21:44.3756127", 1740342104375612),
166+
("2025-02-23T20:21:44.37561278", 1740342104375612),
167+
("2025-02-23T20:21:44.375612001", 1740342104375612),
168+
],
169+
)
170+
def test_timestamp_to_micros(timestamp: str, micros: int) -> None:
171+
assert micros == timestamp_to_micros(timestamp)
172+
173+
174+
def test_timestamp_to_micros_unexpected_zone_offset() -> None:
175+
with pytest.raises(ValueError, match="Zone offset provided, but not expected: 2025-02-23T16:21:44.375612001-04:00"):
176+
timestamp_to_micros("2025-02-23T16:21:44.375612001-04:00")
177+
178+
179+
def test_timestamp_to_micros_invalid_timestamp() -> None:
180+
with pytest.raises(ValueError, match=re.escape("Invalid timestamp without zone: invalid (must be ISO-8601)")):
181+
timestamp_to_micros("invalid")
182+
183+
184+
@pytest.mark.parametrize(
185+
"timestamp, micros",
186+
[
187+
("1970-01-01T00:00:00+00:00", 0),
188+
("2025-02-23T16:21:44.375612-04:00", 1740342104375612),
189+
("2025-02-23T16:21:44.3756127-04:00", 1740342104375612),
190+
("2025-02-23T16:21:44.37561278-04:00", 1740342104375612),
191+
("2025-02-23T16:21:44.375612001-04:00", 1740342104375612),
192+
("2025-02-23T16:21:44-04:00:00", 1740342104000000),
193+
],
194+
)
195+
def test_timestamptz_to_micros(timestamp: str, micros: int) -> None:
196+
assert micros == timestamptz_to_micros(timestamp)
197+
198+
199+
def test_timestamptz_to_micros_missing_zone_offset() -> None:
200+
with pytest.raises(ValueError, match="Missing zone offset: 2025-02-23T20:21:44.375612001"):
201+
timestamptz_to_micros("2025-02-23T20:21:44.375612001")
202+
203+
204+
def test_timestamptz_to_micros_invalid_timestamp() -> None:
205+
with pytest.raises(ValueError, match=re.escape("Invalid timestamp with zone: invalid (must be ISO-8601)")):
206+
timestamptz_to_micros("invalid")
207+
208+
158209
@pytest.mark.parametrize("nanos, micros", [(1510871468000001001, 1510871468000001), (-1510871468000001001, -1510871468000002)])
159210
def test_nanos_to_micros(nanos: int, micros: int) -> None:
160211
assert micros == nanos_to_micros(nanos)

0 commit comments

Comments
 (0)