test: every registered suite without a pytest twin is declared, in both directions (#1341)

test/pytest/TESTS.md

@@ -4196,6 +4196,7 @@ the tool grades THIS tree.
 | `test_a_longer_helper_name_is_not_shadowed_by_a_shorter_one` | `check_ratio` must not eat `check_ratio_needs_quiet_machine` |
 | `test_the_suite_local_helpers_are_known_and_excluded` | the four suite-local helpers, and that none of their suites is graded |
 | `test_every_pair_in_the_tree_is_declared` | the declaration is asserted BOTH ways, so a new pair cannot be silently ungraded |
+| `test_every_registered_suite_is_ported_or_declared_unported` | every registered suite with no twin is listed in `unported_suites.txt`, a ported one leaves it, and no listed stem is phantom (#1341); the parity goal (#432) is that file empty |
 | `test_the_ported_suites_in_this_tree_are_graded_one_for_one` | the standing arm: every pair in the tree, graded |
 
 ## 37. test_iceberg_fdw.py: the Iceberg FDW's pruning surface

test/pytest/expected_tests.txt

@@ -288,7 +288,9 @@
 # guard_tests 435 -> 438 for three arms in test_mutation_ledger.py: a mutation text holding
 # ';' is refused at merge, a ledger piece with whitespace at an end is refused, and the
 # committed ledger holds none, by collection over NO_CLUSTER.
-guard_tests 438
+# guard_tests 438 -> 439 for #1341's test_every_registered_suite_is_ported_or_declared_unported
+# in test_compare_to_bash.py: one test, by collection over NO_CLUSTER.
+guard_tests 439
 
 # The complement: tests that need the driver and a throwaway cluster. Until #1016 these ran
 # in no CI job at all -- a quarter of the corpus, green when somebody ran them by hand and

test/pytest/test_compare_to_bash.py

@@ -1542,6 +1542,65 @@ def test_every_pair_in_the_tree_is_declared(expect):
                 "every incomplete pair says why, at more than a passing word")
 
 
+def _registered_suites():
+    """The SUITES array, read the way run_all_versions.sh itself reads it: the same
+    sed range and an eval, through bash, so this guard and the runner cannot disagree
+    about which suites exist."""
+    import subprocess
+    runner = HERE.parent / "run_all_versions.sh"
+    out = subprocess.run(
+        ["bash", "-c", 'eval "$(sed -n \'/^SUITES=(/,/)$/p\' "$1")"; printf "%s\\n" "${SUITES[@]}"',
+         "bash", str(runner)],
+        capture_output=True, text=True, check=True).stdout
+    return [s for s in out.split("\n") if s]
+
+
+UNPORTED_FILE = HERE / "unported_suites.txt"
+
+
+def _unported():
+    if not UNPORTED_FILE.exists():
+        return None
+    return [ln.strip() for ln in UNPORTED_FILE.read_text().splitlines()
+            if ln.strip() and not ln.lstrip().startswith("#")]
+
+
+def test_every_registered_suite_is_ported_or_declared_unported(expect):
+    """NO SUITE IS TWIN-LESS WITHOUT SAYING SO (#1341).
+
+    `test_every_pair_in_the_tree_is_declared` grades the pairs that EXIST. A registered
+    shell suite with no pytest twin was in no list at all, so the parity goal (#432)
+    could not be read off the tree: 168 suites were twin-less and nothing named them,
+    and a new shell-only suite could land with nobody noticing. `unported_suites.txt`
+    declares them, one stem per line so that a port's removal and another port's
+    removal never conflict, and this arm holds it in both directions:
+
+        a registered suite with no twin, not declared     reddens, named
+        a declared stem that now has its twin             reddens, so the port must
+                                                          also move it to COMPLETE or
+                                                          INCOMPLETE
+        a declared stem that is not a registered suite    reddens, so the list cannot rot
+    """
+    reg = _registered_suites()
+    expect.at_least(len(reg), 200, "premise: the runner's SUITES array was read, not an empty one")
+    unp = _unported()
+    expect.text("present" if unp is not None else f"missing: {UNPORTED_FILE.name}", "present",
+                "premise: the unported declaration exists")
+    if unp is None:
+        return
+    expect.num(len(unp), len(set(unp)), "premise: no stem is declared unported twice")
+    paired = {s for s in reg if (HERE / f"test_{s}.py").exists()}
+    undeclared = sorted(s for s in reg if s not in paired and s not in set(unp))
+    ported = sorted(s for s in unp if s in paired)
+    phantom = sorted(set(unp) - set(reg))
+    expect.text(", ".join(undeclared) or "none", "none",
+                "every registered suite without a pytest twin is declared unported")
+    expect.text(", ".join(ported) or "none", "none",
+                "and a suite that gained its twin is no longer declared unported")
+    expect.text(", ".join(phantom) or "none", "none",
+                "and every declared unported stem is a registered suite")
+
+
 def test_the_ported_suites_in_this_tree_are_graded_one_for_one(expect):
     """THE STANDING ARM, and the reason this file is not only about fixtures.
 

test/pytest/unported_suites.txt

@@ -0,0 +1,176 @@
+# Registered shell suites that have no pytest twin yet (#1341, #432).
+#
+# One stem per line, sorted. test_compare_to_bash.py holds this list in both
+# directions: a registered suite with no test_<stem>.py must be listed here, and a
+# listed stem that gains its twin must be removed and declared COMPLETE or INCOMPLETE
+# in that file. A port PR removes its own line, so two ports never conflict here.
+#
+# The parity goal (#432) is this file empty.
+alter_column_type
+analyze_stats
+arrow_export
+arrow_nested
+arrow_nested_import
+audit
+autovacuum
+autovacuum_yield
+avro_manifest
+batch_fold_explain
+bench_guards
+bloom_lazy
+bloom_setting
+bloom_sizing
+cancel_decode
+catalog_natts
+column_projection
+concurrency
+concurrent_diff
+cost_written_geometry
+decode_interrupts
+decode_skip_interrupts
+dependency_estimate
+doc_parallel_premise
+docs_style
+eager_ordering_record
+encode_effort
+encode_invariants
+estimate_deleted
+export_sink
+fsst_margin
+fsst_verdict_cache
+fuzz
+fuzz_arrow
+fuzz_avro
+fuzz_listing
+fuzz_parquet
+generated_columns
+groupagg_table_sizing
+hardening
+harness_selftest
+hilbert_curve
+iceberg_catalog
+iceberg_deletes
+iceberg_dv
+iceberg_fdw_estimate
+iceberg_fdw_projection
+iceberg_malformed
+iceberg_name_mapping
+iceberg_name_mapping_memory
+iceberg_objstore
+iceberg_rest
+iceberg_rest_scan
+iceberg_rest_server
+iceberg_rest_vended
+iceberg_scan
+import_deferred
+import_exclusion
+index_delete_liveness
+index_only
+int8_agg_int128
+isolation
+local_open_race_free
+logical_decoding_cdc_recipe
+logical_decoding_source
+logical_subscriber
+maintenance_due
+native_agg_addcolumn
+native_agg_deletes
+native_bloom
+native_cancel
+native_cluster
+native_compact
+native_ctas
+native_encoding
+native_exact_selection
+native_fastdecode
+native_fetch_group_memo
+native_fetch_interrupt
+native_fetch_sort_context
+native_fold_deferral
+native_fold_skipguard
+native_gap
+native_groupagg_wide_cost
+native_join_runtime_filter
+native_late_materialization
+native_lazy_slot
+native_metadata_flush
+native_page_offset_bound
+native_param_pushdown
+native_parquet_codecs
+native_parquet_fdw
+native_parquet_fieldid
+native_parquet_hardening
+native_parquet_partition
+native_parquet_projection
+native_parquet_stack
+native_parquet_streaming
+native_parquet_units
+native_read_parquet
+native_reclaim_reconcile
+native_rewrite
+native_rewrite_conc
+native_roundtrip
+native_sort_by
+native_toasted_write
+native_truncate
+native_varlena_bound
+native_vecdecode
+native_zonemap
+native_zonemap_narrow
+native_zonemap_session
+objstore_addressing
+objstore_http_read
+objstore_listing
+objstore_module
+objstore_s3_read
+objstore_sink_write
+objstore_stash_recovery
+objstore_tls_read
+parallel
+parallel_copy
+parallel_copy_dedup
+parallel_degree
+parallel_export_parquet
+parallel_flush_optin
+parallel_vector_agg
+parquet_count_bounds
+parquet_export
+parquet_export_stats
+parquet_level_width
+parquet_nested
+parquet_nested_import
+pg19_vacuum_options
+phase2
+phase3
+phase4
+phase6
+planner_choice_quality
+preimage_rewrite
+projection_rename_restore
+projection_rewrite
+pushdown_report
+qual_order_selectivity
+read_stream
+reader_buffer_reuse
+recluster_extent
+recluster_gate
+rewrite_group_scan
+row_triggers
+scan_direction
+server_file_privilege
+smoke
+sort_status
+sort_status_privilege
+sorted_mark_rename
+tablesample
+ungrouped_vector_agg
+vacuum_sorted_gate
+vacuum_stripe_count
+vector_agg_rescan_memory
+vector_agg_tlist_shape
+vm_clear_on_renumber
+wal_envelope
+write_fsst_compressed
+write_minmax_fastpath
+zonemap_cost
+zonemap_estimate_sample