diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index a58a7caec..a2088e9c3 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -4196,6 +4196,7 @@ the tool grades THIS tree. | `test_a_longer_helper_name_is_not_shadowed_by_a_shorter_one` | `check_ratio` must not eat `check_ratio_needs_quiet_machine` | | `test_the_suite_local_helpers_are_known_and_excluded` | the four suite-local helpers, and that none of their suites is graded | | `test_every_pair_in_the_tree_is_declared` | the declaration is asserted BOTH ways, so a new pair cannot be silently ungraded | +| `test_every_registered_suite_is_ported_or_declared_unported` | every registered suite with no twin is listed in `unported_suites.txt`, a ported one leaves it, and no listed stem is phantom (#1341); the parity goal (#432) is that file empty | | `test_the_ported_suites_in_this_tree_are_graded_one_for_one` | the standing arm: every pair in the tree, graded | ## 37. test_iceberg_fdw.py: the Iceberg FDW's pruning surface diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index d11536d0e..a0e246733 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -288,7 +288,9 @@ # guard_tests 435 -> 438 for three arms in test_mutation_ledger.py: a mutation text holding # ';' is refused at merge, a ledger piece with whitespace at an end is refused, and the # committed ledger holds none, by collection over NO_CLUSTER. -guard_tests 438 +# guard_tests 438 -> 439 for #1341's test_every_registered_suite_is_ported_or_declared_unported +# in test_compare_to_bash.py: one test, by collection over NO_CLUSTER. +guard_tests 439 # The complement: tests that need the driver and a throwaway cluster. Until #1016 these ran # in no CI job at all -- a quarter of the corpus, green when somebody ran them by hand and diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 0e6afd0df..589701d7c 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -1542,6 +1542,65 @@ def test_every_pair_in_the_tree_is_declared(expect): "every incomplete pair says why, at more than a passing word") +def _registered_suites(): + """The SUITES array, read the way run_all_versions.sh itself reads it: the same + sed range and an eval, through bash, so this guard and the runner cannot disagree + about which suites exist.""" + import subprocess + runner = HERE.parent / "run_all_versions.sh" + out = subprocess.run( + ["bash", "-c", 'eval "$(sed -n \'/^SUITES=(/,/)$/p\' "$1")"; printf "%s\\n" "${SUITES[@]}"', + "bash", str(runner)], + capture_output=True, text=True, check=True).stdout + return [s for s in out.split("\n") if s] + + +UNPORTED_FILE = HERE / "unported_suites.txt" + + +def _unported(): + if not UNPORTED_FILE.exists(): + return None + return [ln.strip() for ln in UNPORTED_FILE.read_text().splitlines() + if ln.strip() and not ln.lstrip().startswith("#")] + + +def test_every_registered_suite_is_ported_or_declared_unported(expect): + """NO SUITE IS TWIN-LESS WITHOUT SAYING SO (#1341). + + `test_every_pair_in_the_tree_is_declared` grades the pairs that EXIST. A registered + shell suite with no pytest twin was in no list at all, so the parity goal (#432) + could not be read off the tree: 168 suites were twin-less and nothing named them, + and a new shell-only suite could land with nobody noticing. `unported_suites.txt` + declares them, one stem per line so that a port's removal and another port's + removal never conflict, and this arm holds it in both directions: + + a registered suite with no twin, not declared reddens, named + a declared stem that now has its twin reddens, so the port must + also move it to COMPLETE or + INCOMPLETE + a declared stem that is not a registered suite reddens, so the list cannot rot + """ + reg = _registered_suites() + expect.at_least(len(reg), 200, "premise: the runner's SUITES array was read, not an empty one") + unp = _unported() + expect.text("present" if unp is not None else f"missing: {UNPORTED_FILE.name}", "present", + "premise: the unported declaration exists") + if unp is None: + return + expect.num(len(unp), len(set(unp)), "premise: no stem is declared unported twice") + paired = {s for s in reg if (HERE / f"test_{s}.py").exists()} + undeclared = sorted(s for s in reg if s not in paired and s not in set(unp)) + ported = sorted(s for s in unp if s in paired) + phantom = sorted(set(unp) - set(reg)) + expect.text(", ".join(undeclared) or "none", "none", + "every registered suite without a pytest twin is declared unported") + expect.text(", ".join(ported) or "none", "none", + "and a suite that gained its twin is no longer declared unported") + expect.text(", ".join(phantom) or "none", "none", + "and every declared unported stem is a registered suite") + + def test_the_ported_suites_in_this_tree_are_graded_one_for_one(expect): """THE STANDING ARM, and the reason this file is not only about fixtures. diff --git a/test/pytest/unported_suites.txt b/test/pytest/unported_suites.txt new file mode 100644 index 000000000..c5a84717b --- /dev/null +++ b/test/pytest/unported_suites.txt @@ -0,0 +1,176 @@ +# Registered shell suites that have no pytest twin yet (#1341, #432). +# +# One stem per line, sorted. test_compare_to_bash.py holds this list in both +# directions: a registered suite with no test_.py must be listed here, and a +# listed stem that gains its twin must be removed and declared COMPLETE or INCOMPLETE +# in that file. A port PR removes its own line, so two ports never conflict here. +# +# The parity goal (#432) is this file empty. +alter_column_type +analyze_stats +arrow_export +arrow_nested +arrow_nested_import +audit +autovacuum +autovacuum_yield +avro_manifest +batch_fold_explain +bench_guards +bloom_lazy +bloom_setting +bloom_sizing +cancel_decode +catalog_natts +column_projection +concurrency +concurrent_diff +cost_written_geometry +decode_interrupts +decode_skip_interrupts +dependency_estimate +doc_parallel_premise +docs_style +eager_ordering_record +encode_effort +encode_invariants +estimate_deleted +export_sink +fsst_margin +fsst_verdict_cache +fuzz +fuzz_arrow +fuzz_avro +fuzz_listing +fuzz_parquet +generated_columns +groupagg_table_sizing +hardening +harness_selftest +hilbert_curve +iceberg_catalog +iceberg_deletes +iceberg_dv +iceberg_fdw_estimate +iceberg_fdw_projection +iceberg_malformed +iceberg_name_mapping +iceberg_name_mapping_memory +iceberg_objstore +iceberg_rest +iceberg_rest_scan +iceberg_rest_server +iceberg_rest_vended +iceberg_scan +import_deferred +import_exclusion +index_delete_liveness +index_only +int8_agg_int128 +isolation +local_open_race_free +logical_decoding_cdc_recipe +logical_decoding_source +logical_subscriber +maintenance_due +native_agg_addcolumn +native_agg_deletes +native_bloom +native_cancel +native_cluster +native_compact +native_ctas +native_encoding +native_exact_selection +native_fastdecode +native_fetch_group_memo +native_fetch_interrupt +native_fetch_sort_context +native_fold_deferral +native_fold_skipguard +native_gap +native_groupagg_wide_cost +native_join_runtime_filter +native_late_materialization +native_lazy_slot +native_metadata_flush +native_page_offset_bound +native_param_pushdown +native_parquet_codecs +native_parquet_fdw +native_parquet_fieldid +native_parquet_hardening +native_parquet_partition +native_parquet_projection +native_parquet_stack +native_parquet_streaming +native_parquet_units +native_read_parquet +native_reclaim_reconcile +native_rewrite +native_rewrite_conc +native_roundtrip +native_sort_by +native_toasted_write +native_truncate +native_varlena_bound +native_vecdecode +native_zonemap +native_zonemap_narrow +native_zonemap_session +objstore_addressing +objstore_http_read +objstore_listing +objstore_module +objstore_s3_read +objstore_sink_write +objstore_stash_recovery +objstore_tls_read +parallel +parallel_copy +parallel_copy_dedup +parallel_degree +parallel_export_parquet +parallel_flush_optin +parallel_vector_agg +parquet_count_bounds +parquet_export +parquet_export_stats +parquet_level_width +parquet_nested +parquet_nested_import +pg19_vacuum_options +phase2 +phase3 +phase4 +phase6 +planner_choice_quality +preimage_rewrite +projection_rename_restore +projection_rewrite +pushdown_report +qual_order_selectivity +read_stream +reader_buffer_reuse +recluster_extent +recluster_gate +rewrite_group_scan +row_triggers +scan_direction +server_file_privilege +smoke +sort_status +sort_status_privilege +sorted_mark_rename +tablesample +ungrouped_vector_agg +vacuum_sorted_gate +vacuum_stripe_count +vector_agg_rescan_memory +vector_agg_tlist_shape +vm_clear_on_renumber +wal_envelope +write_fsst_compressed +write_minmax_fastpath +zonemap_cost +zonemap_estimate_sample