Skip to content

Commit 0ebc58b

Browse files
authored
REST: Apply the scan filter when the server omits the residual (#4379)
* REST: Apply the scan filter when the server omits the residual Use the original request filter for file scan tasks without a server residual so count() applies the same filter as Arrow scans. * REST: Simplify the default scan filter --------- Co-authored-by: linzch3 <20380442+linzch3@users.noreply.github.com>
1 parent ab0afe7 commit 0ebc58b

3 files changed

Lines changed: 60 additions & 7 deletions

File tree

‎pyiceberg/catalog/rest/__init__.py‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@
7171
UnauthorizedError,
7272
ViewAlreadyExistsError,
7373
)
74+
from pyiceberg.expressions import AlwaysTrue, BooleanExpression
7475
from pyiceberg.io import (
7576
AWS_ACCESS_KEY_ID,
7677
AWS_PROFILE_NAME,
@@ -860,21 +861,23 @@ def _poll_until_completed(self, identifier: str | Identifier, plan_id: str) -> P
860861
sleep_ms = min(sleep_ms * REST_SCAN_PLANNING_POLL_SCALE_FACTOR, REST_SCAN_PLANNING_POLL_MAX_SLEEP_MS)
861862
retries += 1
862863

863-
def _expand_plan_tasks(self, identifier: str | Identifier, response: PlanCompleted) -> list[FileScanTask]:
864+
def _expand_plan_tasks(
865+
self, identifier: str | Identifier, response: PlanCompleted, fallback_residual: BooleanExpression
866+
) -> list[FileScanTask]:
864867
"""Expand a completed plan response into FileScanTask objects, including pagination."""
865868
tasks: list[FileScanTask] = []
866869

867870
# Collect tasks from initial response
868871
for task in response.file_scan_tasks:
869-
tasks.append(FileScanTask.from_rest_response(task, response.delete_files))
872+
tasks.append(FileScanTask.from_rest_response(task, response.delete_files, fallback_residual=fallback_residual))
870873

871874
# Fetch and collect from additional batches
872875
pending_tasks = deque(response.plan_tasks)
873876
while pending_tasks:
874877
plan_task = pending_tasks.popleft()
875878
batch = self._fetch_scan_tasks(identifier, plan_task)
876879
for task in batch.file_scan_tasks:
877-
tasks.append(FileScanTask.from_rest_response(task, batch.delete_files))
880+
tasks.append(FileScanTask.from_rest_response(task, batch.delete_files, fallback_residual=fallback_residual))
878881
pending_tasks.extend(batch.plan_tasks)
879882

880883
return tasks
@@ -901,7 +904,7 @@ def _plan_scan_result(self, identifier: str | Identifier, request: PlanTableScan
901904
if not isinstance(response, PlanCompleted):
902905
raise RuntimeError(f"Invalid planStatus for response: {type(response).__name__}")
903906

904-
tasks = self._expand_plan_tasks(identifier, response)
907+
tasks = self._expand_plan_tasks(identifier, response, request.filter or AlwaysTrue())
905908
return PlannedScanResult(
906909
tasks=tasks,
907910
storage_credentials=list(response.storage_credentials or []),

‎pyiceberg/table/__init__.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2343,12 +2343,15 @@ def __init__(
23432343
def from_rest_response(
23442344
rest_task: RESTFileScanTask,
23452345
delete_files: list[RESTDeleteFile],
2346+
*,
2347+
fallback_residual: BooleanExpression = ALWAYS_TRUE,
23462348
) -> FileScanTask:
23472349
"""Convert a RESTFileScanTask to a FileScanTask.
23482350
23492351
Args:
23502352
rest_task: The REST file scan task.
23512353
delete_files: The list of delete files from the ScanTasks response.
2354+
fallback_residual: The scan filter to use when the server omits the residual.
23522355
23532356
Returns:
23542357
A FileScanTask with the converted data and delete files.
@@ -2371,7 +2374,7 @@ def from_rest_response(
23712374
return FileScanTask(
23722375
data_file=data_file,
23732376
delete_files=resolved_deletes,
2374-
residual=rest_task.residual_filter if rest_task.residual_filter else ALWAYS_TRUE,
2377+
residual=rest_task.residual_filter if rest_task.residual_filter is not None else fallback_residual,
23752378
)
23762379

23772380

‎tests/catalog/test_scan_planning_models.py‎

Lines changed: 49 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,11 @@
1414
# KIND, either express or implied. See the License for the
1515
# specific language governing permissions and limitations
1616
# under the License.
17+
from pathlib import Path
1718
from typing import Any
1819

20+
import pyarrow as pa
21+
import pyarrow.parquet as pq
1922
import pytest
2023
from pydantic import TypeAdapter, ValidationError
2124
from requests_mock import Mocker
@@ -39,6 +42,7 @@
3942
ValueMap,
4043
)
4144
from pyiceberg.expressions import AlwaysTrue, EqualTo, Reference
45+
from pyiceberg.io.pyarrow import schema_to_pyarrow
4246
from pyiceberg.manifest import FileFormat
4347
from pyiceberg.table import FileScanTask
4448

@@ -462,6 +466,7 @@ def test_plan_scan_completed_single_batch(rest_scan_catalog: RestCatalog, reques
462466

463467
assert len(tasks) == 1
464468
assert tasks[0].file.file_path == "s3://bucket/tbl/data/file1.parquet"
469+
assert tasks[0].residual == AlwaysTrue()
465470

466471

467472
def test_plan_scan_with_pagination(rest_scan_catalog: RestCatalog, requests_mock: Mocker) -> None:
@@ -474,7 +479,7 @@ def test_plan_scan_with_pagination(rest_scan_catalog: RestCatalog, requests_mock
474479
"status": "completed",
475480
"plan-id": "plan-123",
476481
"delete-files": [],
477-
"file-scan-tasks": [{"data-file": file_one}],
482+
"file-scan-tasks": [{"data-file": file_one, "residual-filter": True}],
478483
"plan-tasks": ["token-batch-2"],
479484
},
480485
status_code=200,
@@ -490,13 +495,16 @@ def test_plan_scan_with_pagination(rest_scan_catalog: RestCatalog, requests_mock
490495
status_code=200,
491496
)
492497

493-
request = PlanTableScanRequest()
498+
row_filter = EqualTo("x", 1)
499+
request = PlanTableScanRequest(filter=row_filter)
494500

495501
tasks = rest_scan_catalog.plan_scan(("db", "tbl"), request)
496502

497503
assert len(tasks) == 2
498504
assert tasks[0].file.file_path == "s3://bucket/tbl/data/file1.parquet"
499505
assert tasks[1].file.file_path == "s3://bucket/tbl/data/file2.parquet"
506+
assert tasks[0].residual == AlwaysTrue()
507+
assert tasks[1].residual == row_filter
500508

501509

502510
def test_plan_scan_with_delete_files(rest_scan_catalog: RestCatalog, requests_mock: Mocker) -> None:
@@ -840,6 +848,45 @@ def test_scan_uses_server_side_planning_from_load_table_config(
840848
assert plan_mock.call_count == 1
841849

842850

851+
@pytest.mark.parametrize("paginated", [False, True])
852+
def test_count_applies_scan_filter_when_rest_residual_is_missing(
853+
rest_scan_catalog: RestCatalog,
854+
requests_mock: Mocker,
855+
example_table_metadata_v2: dict[str, Any],
856+
tmp_path: Path,
857+
paginated: bool,
858+
) -> None:
859+
_mock_load_table(requests_mock, example_table_metadata_v2, {})
860+
table = rest_scan_catalog.load_table(("db", "tbl"))
861+
data_file = tmp_path / "file.parquet"
862+
pq.write_table(
863+
pa.table({"x": [1, 2, 1], "y": [4, 5, 6], "z": [7, 8, 9]}, schema=schema_to_pyarrow(table.schema())), data_file
864+
)
865+
file_task = {
866+
"data-file": _rest_data_file(file_path=str(data_file), file_size_in_bytes=data_file.stat().st_size, record_count=3)
867+
}
868+
869+
requests_mock.post(
870+
f"{TEST_URI}v1/namespaces/db/tables/tbl/plan",
871+
json={
872+
"status": "completed",
873+
"plan-id": "plan-123",
874+
"delete-files": [],
875+
"file-scan-tasks": [] if paginated else [file_task],
876+
"plan-tasks": ["next"] if paginated else [],
877+
},
878+
)
879+
if paginated:
880+
requests_mock.post(
881+
f"{TEST_URI}v1/namespaces/db/tables/tbl/tasks",
882+
json={"delete-files": [], "file-scan-tasks": [file_task], "plan-tasks": []},
883+
)
884+
885+
scan = table.scan(row_filter="x != 1")
886+
assert scan.to_arrow().num_rows == 1
887+
assert scan.count() == 1
888+
889+
843890
def test_scan_keeps_client_side_planning_without_load_table_config(
844891
rest_client_planning_catalog: RestCatalog, requests_mock: Mocker, example_table_metadata_v2: dict[str, Any]
845892
) -> None:

0 commit comments

Comments
 (0)