1414# KIND, either express or implied. See the License for the
1515# specific language governing permissions and limitations
1616# under the License.
17+ from pathlib import Path
1718from typing import Any
1819
20+ import pyarrow as pa
21+ import pyarrow .parquet as pq
1922import pytest
2023from pydantic import TypeAdapter , ValidationError
2124from requests_mock import Mocker
3942 ValueMap ,
4043)
4144from pyiceberg .expressions import AlwaysTrue , EqualTo , Reference
45+ from pyiceberg .io .pyarrow import schema_to_pyarrow
4246from pyiceberg .manifest import FileFormat
4347from pyiceberg .table import FileScanTask
4448
@@ -462,6 +466,7 @@ def test_plan_scan_completed_single_batch(rest_scan_catalog: RestCatalog, reques
462466
463467 assert len (tasks ) == 1
464468 assert tasks [0 ].file .file_path == "s3://bucket/tbl/data/file1.parquet"
469+ assert tasks [0 ].residual == AlwaysTrue ()
465470
466471
467472def test_plan_scan_with_pagination (rest_scan_catalog : RestCatalog , requests_mock : Mocker ) -> None :
@@ -474,7 +479,7 @@ def test_plan_scan_with_pagination(rest_scan_catalog: RestCatalog, requests_mock
474479 "status" : "completed" ,
475480 "plan-id" : "plan-123" ,
476481 "delete-files" : [],
477- "file-scan-tasks" : [{"data-file" : file_one }],
482+ "file-scan-tasks" : [{"data-file" : file_one , "residual-filter" : True }],
478483 "plan-tasks" : ["token-batch-2" ],
479484 },
480485 status_code = 200 ,
@@ -490,13 +495,16 @@ def test_plan_scan_with_pagination(rest_scan_catalog: RestCatalog, requests_mock
490495 status_code = 200 ,
491496 )
492497
493- request = PlanTableScanRequest ()
498+ row_filter = EqualTo ("x" , 1 )
499+ request = PlanTableScanRequest (filter = row_filter )
494500
495501 tasks = rest_scan_catalog .plan_scan (("db" , "tbl" ), request )
496502
497503 assert len (tasks ) == 2
498504 assert tasks [0 ].file .file_path == "s3://bucket/tbl/data/file1.parquet"
499505 assert tasks [1 ].file .file_path == "s3://bucket/tbl/data/file2.parquet"
506+ assert tasks [0 ].residual == AlwaysTrue ()
507+ assert tasks [1 ].residual == row_filter
500508
501509
502510def test_plan_scan_with_delete_files (rest_scan_catalog : RestCatalog , requests_mock : Mocker ) -> None :
@@ -840,6 +848,45 @@ def test_scan_uses_server_side_planning_from_load_table_config(
840848 assert plan_mock .call_count == 1
841849
842850
851+ @pytest .mark .parametrize ("paginated" , [False , True ])
852+ def test_count_applies_scan_filter_when_rest_residual_is_missing (
853+ rest_scan_catalog : RestCatalog ,
854+ requests_mock : Mocker ,
855+ example_table_metadata_v2 : dict [str , Any ],
856+ tmp_path : Path ,
857+ paginated : bool ,
858+ ) -> None :
859+ _mock_load_table (requests_mock , example_table_metadata_v2 , {})
860+ table = rest_scan_catalog .load_table (("db" , "tbl" ))
861+ data_file = tmp_path / "file.parquet"
862+ pq .write_table (
863+ pa .table ({"x" : [1 , 2 , 1 ], "y" : [4 , 5 , 6 ], "z" : [7 , 8 , 9 ]}, schema = schema_to_pyarrow (table .schema ())), data_file
864+ )
865+ file_task = {
866+ "data-file" : _rest_data_file (file_path = str (data_file ), file_size_in_bytes = data_file .stat ().st_size , record_count = 3 )
867+ }
868+
869+ requests_mock .post (
870+ f"{ TEST_URI } v1/namespaces/db/tables/tbl/plan" ,
871+ json = {
872+ "status" : "completed" ,
873+ "plan-id" : "plan-123" ,
874+ "delete-files" : [],
875+ "file-scan-tasks" : [] if paginated else [file_task ],
876+ "plan-tasks" : ["next" ] if paginated else [],
877+ },
878+ )
879+ if paginated :
880+ requests_mock .post (
881+ f"{ TEST_URI } v1/namespaces/db/tables/tbl/tasks" ,
882+ json = {"delete-files" : [], "file-scan-tasks" : [file_task ], "plan-tasks" : []},
883+ )
884+
885+ scan = table .scan (row_filter = "x != 1" )
886+ assert scan .to_arrow ().num_rows == 1
887+ assert scan .count () == 1
888+
889+
843890def test_scan_keeps_client_side_planning_without_load_table_config (
844891 rest_client_planning_catalog : RestCatalog , requests_mock : Mocker , example_table_metadata_v2 : dict [str , Any ]
845892) -> None :
0 commit comments