test: iceberg_data_files's twin is a port by name (#1341)

test/iceberg_data_files.sh

@@ -83,6 +83,18 @@ LOC="$(python3 -c "import json;print(json.load(open('$MD'))['location'].replace(
 # symlink planted there resolves under it -- writing them elsewhere would derive a
 # wrong actual root and mask the boundary under a "file not found".
 MDDIR="$(dirname "$MD")"
+# After a survival check, wait (up to 60s) until the server accepts connections
+# again. A refusal that crashes the backend fails its own arm and the survival check;
+# without the wait, whether the NEXT arm also failed on the recovery depended on how
+# long the python step before it took (#1341).
+recovered() {
+	local i
+	for i in $(seq 240); do
+		[ "$(q 'SELECT 1')" = "1" ] && return 0
+		sleep 0.25
+	done
+}
+
 esc_case() {  # $1 payload manifest-list, writes a metadata variant, echoes its path
 	local payload="$1" out="$MDDIR/esc_$2.metadata.json"
 	python3 - "$MD" "$out" "$payload" <<'PY'
@@ -113,6 +125,7 @@ ln -sf "$OUTSIDE" "$DEST/db/events/metadata/sneaky.avro"
 check "a symlink under the location targeting outside is refused (22023, not XX001)" \
 	"$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_data_files('$(esc_case "file://$LOC/metadata/sneaky.avro" d)')")" "22023"
 check "backend still up after the traversal attempts" "$(q 'SELECT 1')" "1"
+recovered
 
 # ---- deletes are refused, not ignored --------------------------------------
 # pyiceberg 0.11.1 cannot write merge-on-read deletes, so craft the deny input:
@@ -178,6 +191,7 @@ PY
 check "a snapshot with a delete manifest is refused (feature_not_supported 0A000)" \
 	"$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_data_files('$DT/metadata/v1.metadata.json')")" "0A000"
 check "backend still up after the delete refusal" "$(q 'SELECT 1')" "1"
+recovered
 
 # ---- a delete ENTRY inside a data manifest is refused too (#1341 item 49) --
 # The arm above stops at the manifest LIST. A data manifest (content 0 in the
@@ -259,6 +273,7 @@ check "an equality-delete entry in a data manifest is refused (0A000)" \
 check "a DELETED (status 2) data entry is refused, not listed (0A000)" \
 	"$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_data_files('$(entry_tbl gone 2 0)')")" "0A000"
 check "backend still up after the entry refusals" "$(q 'SELECT 1')" "1"
+recovered
 
 # ---- no current snapshot: zero rows ----------------------------------------
 NOSNAP="$PGC_WORKDIR/no_current.metadata.json"
@@ -273,13 +288,23 @@ check "a table with no current snapshot yields no data files" \
 	"$(q "SELECT count(*) FROM pgcolumnar.iceberg_data_files('$NOSNAP')")" "0"
 
 # ---- privilege: needs pg_read_server_files ---------------------------------
-psql_run "DROP ROLE IF EXISTS icedf_unpriv; CREATE ROLE icedf_unpriv LOGIN;"
+# The role gets USAGE on the schema and EXECUTE on the function, so the refusal is
+# the function's own gate. Without them the 42501 is "permission denied for schema
+# pgcolumnar", the function never runs, and the arm cannot fail on the gate it names.
+DROP_UNPRIV="DO \$\$BEGIN IF EXISTS (SELECT FROM pg_roles WHERE rolname = 'icedf_unpriv')
+	THEN EXECUTE 'DROP OWNED BY icedf_unpriv'; END IF; END\$\$;
+DROP ROLE IF EXISTS icedf_unpriv;"
+psql_run "$DROP_UNPRIV
+CREATE ROLE icedf_unpriv LOGIN;
+GRANT USAGE ON SCHEMA pgcolumnar TO icedf_unpriv;
+GRANT EXECUTE ON FUNCTION pgcolumnar.iceberg_data_files TO icedf_unpriv;"
 check "an unprivileged role is refused (42501)" \
 	"$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U icedf_unpriv -d "$PGC_DB" \
 		-qtA 2>&1 <<SQL | sed -n 's/^ERROR:  \([0-9A-Z]\{5\}\).*/\1/p' | head -1
 \\set VERBOSITY sqlstate
 SELECT count(*) FROM pgcolumnar.iceberg_data_files('$MD');
 SQL
 )" "42501"
+psql_run "$DROP_UNPRIV"
 
 pgc_summary

test/pytest/TESTS.md

@@ -177,7 +177,7 @@ behaviour, the source of that number is named.
 - [129. test_savepoint_flush.py: a savepoint rolled back after a flush](#129-test_savepoint_flushpy-a-savepoint-rolled-back-after-a-flush)
 - [130. test_rls_query_paths.py: row-level security holds on every read path](#130-test_rls_query_pathspy-row-level-security-holds-on-every-read-path)
 - [131. test_type_matrix.py: types the differential matrix never stored answer as heap](#131-test_type_matrixpy-types-the-differential-matrix-never-stored-answer-as-heap)
-- [132. test_iceberg_data_files.py: a delete entry in a data manifest is refused](#132-test_iceberg_data_filespy-a-delete-entry-in-a-data-manifest-is-refused)
+- [132. test_iceberg_data_files.py: an Iceberg table's live data files, listed inside its location](#132-test_iceberg_data_filespy-an-iceberg-tables-live-data-files-listed-inside-its-location)
 - [133. test_native_skip.py: a type binary-coercible to its opclass's type prunes](#133-test_native_skippy-a-type-binary-coercible-to-its-opclasss-type-prunes)
 - [134. test_unique_conc.py: concurrent inserters of one unique key serialize](#134-test_unique_concpy-concurrent-inserters-of-one-unique-key-serialize)
 - [135. test_update_conc.py: two writers of one row serialize](#135-test_update_concpy-two-writers-of-one-row-serialize)
@@ -7337,17 +7337,28 @@ The pytest twin of `test/type_matrix.sh` (#1341), every check by the shell's nam
 | --- | --- |
 | `test_types_the_matrix_never_stored_answer_as_heap` | the 21 columns against heap: rows, counts, sixteen ranges, the three non-btree matches, and the special values |
 
-## 132. test_iceberg_data_files.py: a delete entry in a data manifest is refused
+## 132. test_iceberg_data_files.py: an Iceberg table's live data files, listed inside its location
 
-A partial pytest twin of `test/iceberg_data_files.sh` (#1341 item 49). It ports only the arms that item added, by the shell's names; the rest of the shell suite is declared INCOMPLETE in `test_compare_to_bash.py`. The lister refuses a delete manifest in the manifest list without opening it, but a data manifest can still hold an entry it must not list: a position or equality delete file (entry content 1 or 2), or a removed file (status 2, DELETED). Each table is crafted Avro, the same one-entry data manifest with one (status, content) pair, because no available writer emits these shapes. The (1, 0) control must be listed, so a refusal of the others comes from the entry and not from a fault in the crafted Avro. An EXISTING (0, 0) entry must be listed too, so a lister that keeps only ADDED entries fails.
+The port of `test/iceberg_data_files.sh`, by name, one check per test (#1341). The `ice` module fixture copies the committed warehouse away from its recorded root (`/tmp/pgc_ice_wh`), so rebasing is exercised for real. It stages every tampered and crafted table once and records every reading.
+
+- **The oracle is pyiceberg's own view,** committed as `expected_files.json`, so pyiceberg is not needed at test time.
+- **A manifest list outside the table's location is refused with 22023, by four routes:** a foreign absolute path, a `..` traversal, a sibling sharing the byte prefix, and a symlink under the location that targets a file outside it. XX001 would mean the backend opened the file.
+- **Deletes are refused, not ignored:** a delete manifest in the list, and a delete or removed entry inside a data manifest (#1341 item 49). Each table is crafted Avro, because no available writer emits these shapes. The added and the EXISTING entry must be listed, so a refusal comes from the entry, and a lister that keeps only ADDED entries fails.
+- **The privilege** is read under `SET ROLE` to a role without `pg_read_server_files`.
+
+The shell's two skip names, "iceberg warehouse fixture is missing" and "python3 (json, zlib) is needed", are carried as the same declines.
 
 ### Every test
 
 | test | what it holds |
 | --- | --- |
-| `test_the_control_entry_is_listed` | a data manifest with one added data entry lists that one file |
-| `test_an_existing_entry_is_listed` | an EXISTING (status 0) entry is listed too, so a lister that keeps only ADDED entries fails |
-| `test_a_delete_entry_is_refused` | a position-delete entry, an equality-delete entry and a DELETED entry are each refused with 0A000, and the backend stays up |
+| `test_the_warehouse_fixture_is_present`, `test_python_json_and_zlib_are_available` | the shell's skip names; when present, the oracle lists data files and zlib round-trips |
+| `test_the_data_files_match_pyicebergs_oracle`, `test_every_data_file_is_parquet` | the live data files, by record count, region and file name, all PARQUET |
+| `test_returned_paths_are_rebased_under_the_actual_location`, `test_no_returned_path_leaks_the_recorded_root` | the rebasing |
+| `test_a_manifest_list_outside_the_location_is_refused`, `test_the_backend_survives_the_traversal_attempts` | the four escapes, refused with 22023 |
+| `test_a_snapshot_with_a_delete_manifest_is_refused`, `test_the_backend_survives_the_delete_refusal` | a delete manifest in the list, refused with 0A000 |
+| `test_the_control_entry_is_listed`, `test_an_existing_entry_is_listed`, `test_a_delete_entry_is_refused`, `test_the_backend_survives_the_entry_refusals` | an added and an existing entry are listed; a position-delete, an equality-delete and a DELETED entry are refused with 0A000 |
+| `test_a_table_with_no_current_snapshot_yields_no_data_files`, `test_an_unprivileged_role_is_refused` | no current snapshot lists nothing; a caller without `pg_read_server_files` gets 42501 |
 
 ## 133. test_native_skip.py: a type binary-coercible to its opclass's type prunes
 

test/pytest/expected_tests.txt

@@ -850,4 +850,6 @@ guard_tests 442
 # nineteen tests, one per check, by collection over the cluster files on this tree.
 # cluster_tests 966 -> 990 for #1341, test_native_format.py ported by name, one check per
 # test (1 -> 25 tests), by collection over the cluster files.
-cluster_tests 990
+# cluster_tests 990 -> 1006 for #1341, test_iceberg_data_files.py ported by name, one check
+# per test (5 -> 21 tests), by collection over the cluster files.
+cluster_tests 1006

test/pytest/test_compare_to_bash.py

@@ -88,6 +88,7 @@
 COMPLETE = ["advisory_lock_class",
             "range_pruning",
             "native_format",
+            "iceberg_data_files",
             "alter_table_scans",
             "fdw_plan_privilege",
             "am_contracts",
@@ -241,13 +242,6 @@
     "drop_cleanup": "the #1265 arms (a columnar matview's options row goes with its "
                     "DROP and survives its REFRESH, with an unrelated row as the "
                     "control) are ported; the storage and catalog cleanup arms are not",
-    # #1341 item 49 ported the arms it added: a delete or removed entry inside a
-    # data manifest is refused, with a control that is listed. The rest of the
-    # shell suite (the pyiceberg oracle, path rebasing and its escapes, the
-    # delete-manifest refusal, no snapshot, the privilege refusal) is not.
-    "iceberg_data_files": "the delete-entry refusals and their control are ported; "
-                          "the oracle, rebasing, path-escape, delete-manifest, "
-                          "no-snapshot and privilege arms are not",
     # #1341's megabyte values: every arm is ported except the opt-in chunk over 1 GB
     # (PGC_RUN_OVER_1GB=1), which needs several GB of memory and which this layer,
     # having no SKIP verdict for an arm a run chooses not to make, does not carry.