Skip to content

Commit da20b98

Browse files
committed
Add support for reading Theta sketch Puffin file
1 parent ce5a195 commit da20b98

5 files changed

Lines changed: 278 additions & 3 deletions

File tree

‎pyiceberg/table/theta_sketch.py‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
from __future__ import annotations
18+
19+
from typing import TYPE_CHECKING
20+
21+
from pyiceberg.table.puffin import PuffinBlobMetadata, PuffinFile
22+
23+
if TYPE_CHECKING:
24+
from datasketches import compact_theta_sketch
25+
26+
BLOB_TYPE_APACHE_DATASKETCHES_THETA_V1 = "apache-datasketches-theta-v1"
27+
28+
29+
class ThetaSketch:
30+
field_id: int
31+
_sketch: compact_theta_sketch
32+
33+
def __init__(self, field_id: int, sketch: compact_theta_sketch) -> None:
34+
self.field_id = field_id
35+
self._sketch = sketch
36+
37+
def get_estimate(self) -> float:
38+
return self._sketch.get_estimate()
39+
40+
def get_lower_bound(self, num_std_devs: int = 1) -> float:
41+
return self._sketch.get_lower_bound(num_std_devs)
42+
43+
def get_upper_bound(self, num_std_devs: int = 1) -> float:
44+
return self._sketch.get_upper_bound(num_std_devs)
45+
46+
def is_empty(self) -> bool:
47+
return self._sketch.is_empty()
48+
49+
def is_estimation_mode(self) -> bool:
50+
return self._sketch.is_estimation_mode()
51+
52+
@property
53+
def sketch(self) -> compact_theta_sketch:
54+
return self._sketch
55+
56+
57+
def _theta_sketches_from_blob(blob: PuffinBlobMetadata, payload: bytes) -> list[ThetaSketch]:
58+
from datasketches import compact_theta_sketch
59+
60+
sketch = compact_theta_sketch.deserialize(payload)
61+
return [ThetaSketch(field_id=field_id, sketch=sketch) for field_id in blob.fields]
62+
63+
64+
def theta_sketches_from_puffin_file(puffin_file: PuffinFile) -> list[ThetaSketch]:
65+
sketches = []
66+
for blob in puffin_file.footer.blobs:
67+
if blob.type == BLOB_TYPE_APACHE_DATASKETCHES_THETA_V1:
68+
sketches.extend(_theta_sketches_from_blob(blob, puffin_file.get_blob_payload(blob)))
69+
return sketches

‎pyproject.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ gcp-auth = ["google-auth>=2.4.0"]
101101
entra-auth = ["azure-identity>=1.25.1"]
102102
geoarrow = ["geoarrow-pyarrow>=0.2.0"]
103103
encryption = ["cryptography>=42.0.0"]
104+
datasketches = ["datasketches>=3.4.0,<6.0.0"]
104105

105106
[dependency-groups]
106107
dev = [
@@ -129,6 +130,7 @@ dev = [
129130
"nbformat>=5.10.0",
130131
"ipykernel>=6.29.0",
131132
"ty==0.0.92",
133+
"datasketches>=3.4.0,<6.0.0",
132134
]
133135
# for mkdocs
134136
docs = [
843 Bytes
Binary file not shown.

‎tests/table/test_theta_sketch.py‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
import json
18+
from os import path
19+
20+
import pytest
21+
from datasketches import compact_theta_sketch, update_theta_sketch
22+
23+
from pyiceberg.table.puffin import MAGIC_BYTES, PuffinFile
24+
from pyiceberg.table.theta_sketch import ThetaSketch, theta_sketches_from_puffin_file
25+
26+
27+
def _open_fixture(file: str) -> bytes:
28+
cur_dir = path.dirname(path.realpath(__file__))
29+
with open(f"{cur_dir}/puffin/v1/{file}", "rb") as f:
30+
return f.read()
31+
32+
33+
def _make_sketch(values: list[int]) -> compact_theta_sketch:
34+
ts = update_theta_sketch()
35+
for v in values:
36+
ts.update(v)
37+
return ts.compact()
38+
39+
40+
@pytest.fixture
41+
def empty_sketch_bytes() -> bytes:
42+
return update_theta_sketch().compact().serialize()
43+
44+
45+
@pytest.fixture
46+
def three_value_sketch_bytes() -> bytes:
47+
return _make_sketch([1, 2, 3]).serialize()
48+
49+
50+
def test_empty_sketch(empty_sketch_bytes: bytes) -> None:
51+
sketch = compact_theta_sketch.deserialize(empty_sketch_bytes)
52+
ts = ThetaSketch(field_id=1, sketch=sketch)
53+
54+
assert ts.is_empty()
55+
assert ts.get_estimate() == 0.0
56+
57+
58+
def test_sketch_estimate(three_value_sketch_bytes: bytes) -> None:
59+
sketch = compact_theta_sketch.deserialize(three_value_sketch_bytes)
60+
ts = ThetaSketch(field_id=1, sketch=sketch)
61+
62+
assert not ts.is_empty()
63+
assert ts.get_estimate() == pytest.approx(3.0)
64+
assert not ts.is_estimation_mode()
65+
66+
67+
def test_sketch_bounds_exact_mode(three_value_sketch_bytes: bytes) -> None:
68+
sketch = compact_theta_sketch.deserialize(three_value_sketch_bytes)
69+
ts = ThetaSketch(field_id=1, sketch=sketch)
70+
71+
assert ts.get_lower_bound(1) == pytest.approx(3.0)
72+
assert ts.get_upper_bound(1) == pytest.approx(3.0)
73+
74+
75+
def test_sketch_field_id() -> None:
76+
sketch = _make_sketch([10, 20, 30])
77+
ts = ThetaSketch(field_id=42, sketch=sketch)
78+
79+
assert ts.field_id == 42
80+
81+
82+
def test_sketch_property() -> None:
83+
sketch = _make_sketch([1, 2])
84+
ts = ThetaSketch(field_id=1, sketch=sketch)
85+
86+
assert ts.sketch is sketch
87+
88+
89+
def test_estimation_mode() -> None:
90+
ts_builder = update_theta_sketch(lg_k=5)
91+
for i in range(100):
92+
ts_builder.update(i)
93+
sketch = ts_builder.compact()
94+
ts = ThetaSketch(field_id=1, sketch=sketch)
95+
96+
assert ts.is_estimation_mode()
97+
assert ts.get_estimate() > 0
98+
assert ts.get_lower_bound(1) <= ts.get_estimate()
99+
assert ts.get_upper_bound(1) >= ts.get_estimate()
100+
101+
102+
def _build_puffin_file(blob_bytes: bytes, field_ids: list[int], snapshot_id: int = 1) -> bytes:
103+
# Puffin layout: magic(4) + blobs + footer_json + footer_size(4) + flags(4) + magic(4)
104+
# Blob offsets are file-absolute; first blob starts immediately after the 4-byte magic.
105+
blob_offset = 4
106+
footer = {
107+
"blobs": [
108+
{
109+
"type": "apache-datasketches-theta-v1",
110+
"snapshot-id": snapshot_id,
111+
"sequence-number": 1,
112+
"fields": field_ids,
113+
"offset": blob_offset,
114+
"length": len(blob_bytes),
115+
}
116+
],
117+
"properties": {},
118+
}
119+
footer_json = json.dumps(footer, separators=(",", ":")).encode("utf-8")
120+
footer_size_bytes = len(footer_json).to_bytes(4, byteorder="little")
121+
flags = b"\x00\x00\x00\x00"
122+
return MAGIC_BYTES + blob_bytes + footer_json + footer_size_bytes + flags + MAGIC_BYTES
123+
124+
125+
def test_theta_sketches_from_puffin_file_single_field(three_value_sketch_bytes: bytes) -> None:
126+
puffin_bytes = _build_puffin_file(three_value_sketch_bytes, field_ids=[5])
127+
puffin_file = PuffinFile(puffin_bytes)
128+
129+
sketches = theta_sketches_from_puffin_file(puffin_file)
130+
131+
assert len(sketches) == 1
132+
assert sketches[0].field_id == 5
133+
assert sketches[0].get_estimate() == pytest.approx(3.0)
134+
135+
136+
def test_theta_sketches_from_puffin_file_multiple_fields(three_value_sketch_bytes: bytes) -> None:
137+
puffin_bytes = _build_puffin_file(three_value_sketch_bytes, field_ids=[1, 2, 3])
138+
puffin_file = PuffinFile(puffin_bytes)
139+
140+
sketches = theta_sketches_from_puffin_file(puffin_file)
141+
142+
assert len(sketches) == 3
143+
assert [s.field_id for s in sketches] == [1, 2, 3]
144+
for sketch in sketches:
145+
assert sketch.get_estimate() == pytest.approx(3.0)
146+
147+
148+
def test_theta_sketches_from_puffin_file_empty_sketch(empty_sketch_bytes: bytes) -> None:
149+
puffin_bytes = _build_puffin_file(empty_sketch_bytes, field_ids=[7])
150+
puffin_file = PuffinFile(puffin_bytes)
151+
152+
sketches = theta_sketches_from_puffin_file(puffin_file)
153+
154+
assert len(sketches) == 1
155+
assert sketches[0].is_empty()
156+
assert sketches[0].get_estimate() == 0.0
157+
158+
159+
def test_theta_sketches_from_trino_written_puffin_file() -> None:
160+
puffin_file = PuffinFile(_open_fixture("theta-sketches.puffin"))
161+
sketches = theta_sketches_from_puffin_file(puffin_file)
162+
163+
assert len(sketches) == 3
164+
assert [s.field_id for s in sketches] == [1, 2, 3]
165+
for sketch in sketches:
166+
assert sketch.get_estimate() == pytest.approx(5.0)

‎uv.lock‎

Lines changed: 41 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)