From 9cf1e78d9df25e5ed623ba9067b0e77d0092e776 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Sat, 17 Oct 2026 15:46:59 +0000 Subject: [PATCH] test: range_pruning's twin is a port by name, and its unbounded arms can fail (#1341) The twin was an independent port: its own fixture and names, and 28 of the shell's 31 arms had no counterpart (#1339 test_range_pruning.py:1). It is now the shell's fixture and checks by name, one check per test, with every reading through query_error() (2 -> 34 tests). Porting it showed that the shell's two unbounded-range oracle arms could not fail, for three reasons, each measured: - the fingerprint sums payload, which u_col and u_heap lacked, so both sides errored and check compared "" with "" (PG17: u_col=[] u_heap=[]); - an empty row in the same unit made the unit's minimum empty, and such a unit is never pruned; - the unbounded row went in by its own INSERT, into a row group of its own. With all three fixed, and check_text refusing an empty side, a writer that ignores unbounded ranges reddens both arms and 'present but empty' on each of 15-19, in both twins. Three ledger rows dated, census 3080 -> 3077. Co-Authored-By: Claude Opus 5.5 --- test/check_ledger.tsv | 6 +- test/check_ledger_budget.txt | 16 +- test/pytest/TESTS.md | 63 +-- test/pytest/expected_tests.txt | 4 +- test/pytest/test_compare_to_bash.py | 9 +- test/pytest/test_range_pruning.py | 579 +++++++++++++++++++--------- test/range_pruning.sh | 37 +- 7 files changed, 476 insertions(+), 238 deletions(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index e2c36eb42..e4f40e129 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -3584,11 +3584,11 @@ projections projections unknown column rejected 15;16;17;18;19 never - range_pruning range_pruning a clustered overlap near the end skips the four vectors only their upper bounds rule out 15;16;17;18;19 2026-10-13 native_range_excludes never using max_upper (haveUpper forced false) range_pruning range_pruning a clustered overlap skips vectors instead of decoding them 15;16;17;18;19 never - range_pruning range_pruning a scattered column skips nothing, which is what the documentation says 15;16;17;18;19 never - -range_pruning range_pruning a unit holding an unbounded range is never pruned from above 15;16;17;18;19 never - +range_pruning range_pruning a unit holding an unbounded range is never pruned from above 15;16;17;18;19 2026-10-17 columnar_write_state.c: the writer ignores an unbounded range, so its unit records only the largest finite upper bound range_pruning range_pruning a window no row can meet reads no chunk group at all 15;16;17;18;19 2026-10-13 native_range_excludes never using max_upper (haveUpper forced false) range_pruning range_pruning a window no row can meet returns nothing, from both 15;16;17;18;19 never - range_pruning range_pruning an empty range contributes no bound, so its units record NULL 15;16;17;18;19 never - -range_pruning range_pruning and an unbounded range records a bound that is present but empty 15;16;17;18;19 never - +range_pruning range_pruning and an unbounded range records a bound that is present but empty 15;16;17;18;19 2026-10-17 columnar_write_state.c: the writer ignores an unbounded range, so its unit records only the largest finite upper bound range_pruning range_pruning and still returns the right rows 15;16;17;18;19 never - range_pruning range_pruning and that late overlap returns the heap's rows 15;16;17;18;19 never - range_pruning range_pruning and the rows themselves match, not only the count 15;16;17;18;19 never - @@ -3613,7 +3613,7 @@ range_pruning range_pruning premise: the range column is summarised in several u range_pruning range_pruning premise: the range type collates differently from its element type 15;16;17;18;19 never - range_pruning range_pruning so does a containment probe 15;16;17;18;19 never - range_pruning range_pruning the overlap qual is pushed down, where it used to be dropped 15;16;17;18;19 never - -range_pruning range_pruning the same for a containment probe far to its right 15;16;17;18;19 never - +range_pruning range_pruning the same for a containment probe far to its right 15;16;17;18;19 2026-10-17 columnar_write_state.c: the writer ignores an unbounded range, so its unit records only the largest finite upper bound recovery recovery premise: s2's INSERT ran in its open transaction before the crash 15;16;17;18;19 never - recovery recovery s1 aggregate 15;16;17;18;19 2026-10-16 columnar_storage.c: a stripe data page written with no WAL record recovery recovery s1 col count 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 3c6320fbc..12ffeb587 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -3651,4 +3651,18 @@ suites_not_covered 185 # never 3080 + 0 = 3080 # dated 1374 + 1 = 1375 # 3080 + 1375 == 4455 -checks_never_observed_red 3080 +# +# #1341, range_pruning's unbounded-range arms could not fail (jdatcmd): +# 3080 -> 3077. Both oracle arms summed a payload column their tables lacked, +# so both sides errored and `check` compared "" with "". Fixed, they still +# passed: an empty row in the unit made its minimum empty, which is never +# pruned, and the unbounded row went in by its own INSERT, into a row group +# of its own. With all three fixed, a writer that ignores unbounded ranges +# reddens both arms and "present but empty", on each of 15-19, in both twins. +# No row is added, removed or renamed. +# +# ledger rows 4455 + 0 = 4455 +# never 3080 - 3 = 3077 +# dated 1375 + 3 = 1378 +# 3077 + 1378 == 4455 +checks_never_observed_red 3077 diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 31b254092..fe85d4f83 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -6055,37 +6055,48 @@ Assertion names match. ## 75. test_range_pruning.py: a range prunes on overlap, containment, and under its own collation - SELECT id FROM rp_col WHERE span && tstzrange('2024-03-01', '2024-03-01 06:00') + SELECT count(*) FROM r_col WHERE span && tstzrange('2020-01-02 00:00+00','2020-01-02 01:00+00') A zone map recorded `minimum` and `maximum` only. Those are the range type's btree ordering, which sorts by lower bound and then upper bound, so the largest range is not the one reaching furthest right: a unit holding `[1,2)` and `[3,100)` has the same `maximum` as one holding `[1,2)` and `[3,4)`. Neither `&&` nor `@>` can be -answered from that, and both read every chunk group. `zone_map.max_upper` records -the greatest upper bound, and the scan skips a unit only when the summary proves it -cannot match. - -**THE HEAP IS THE ORACLE.** A pruning defect returns FEWER rows rather than an -error, so every predicate is answered by a heap table built from the same generator -and the row IDENTIFIERS are compared, not the counts. - -**THE SCATTERED ZERO IS AN ASSERTION, NOT AN OMISSION.** On a column whose spans are -not clustered, every group holds a bound near the maximum and no summary can exclude -any of them, so `Columnar Chunk Groups Read` equals `Columnar Chunk Groups Total`. - -**AND A RANGE COMPARES UNDER THE COLLATION IT WAS DECLARED WITH**, which the type -cache carries as `rng_collation`. Reading the ELEMENT type's `typcollation` instead -made the writer summarise under one ordering and the reader prune under another, and -the scan MISSED ROWS. Not reachable with a built-in range type, so it needs a -user-defined range over a collatable subtype with an explicit non-default -`COLLATION`. Found by @jdatcmd reviewing #1144. - -`test_range_pruning` asserts the counters, the heap oracle and the scattered zero. -`test_a_range_prunes_under_its_declared_collation` asserts the collation property, -on a user-defined range type it creates for the purpose. - -Independent of `test/range_pruning.sh`: same public seam, own fixture, own row -counts, own observations. Neither file reads or runs the other. +answered from that. `zone_map.max_upper` records the greatest upper bound in three +states: a bound, NULL for no bound (an empty range), and present but empty for an +unbounded range. The scan skips a unit only when the summary proves it cannot match. + +The port of `test/range_pruning.sh`, by name, one check per test (#1341, #1339). The +fixture is the shell's: 50,000 rows in one stripe, which is one row group of five +vectors, so pruning shows at both levels. Module fixtures build the tables once. + +| test | what it holds | +| --- | --- | +| `test_the_clustered_fixture_holds_the_rows`, `test_the_heap_mirror_holds_the_same_rows` | both tables hold 50,000 rows | +| `test_the_range_column_is_summarised_in_several_units`, `test_every_unit_carries_an_upper_bound`, `test_a_non_range_column_carries_none` | the range column has at least 4 units, each with an upper bound; no other column has one | +| `test_the_columnar_side_returns_the_heaps_rows` | 8 heap-oracle arms, compared by count and two sums: the one-hour, week, point, never-met and late windows, the scattered table, and the two unbounded-range probes | +| `test_the_first_three_oracle_probes_are_not_vacuously_empty` | the window, week and point probes match rows on the heap | +| `test_the_overlap_qual_is_pushed_down`, `test_the_zone_map_is_probed_for_it` | the overlap qual is pushed down and probes the zone map | +| `test_a_clustered_overlap_skips_vectors`, `test_a_containment_probe_skips_vectors` | overlap and containment skip vectors | +| `test_a_late_overlap_skips_the_four_vectors_only_upper_bounds_rule_out` | the late window skips exactly the 4 vectors only upper bounds rule out | +| `test_a_window_no_row_can_meet_reads_no_chunk_group` | the never-met window reads no chunk group | +| `test_a_scattered_column_skips_nothing` | the documented zero | +| `test_the_unbounded_row_really_is_matched` | the unbounded row is matched on the heap | +| `test_the_all_empty_table_is_summarised`, `test_an_empty_range_contributes_no_bound` | the all-empty table is summarised, and records NULL | +| `test_an_unbounded_range_records_a_present_but_empty_bound` | the unbounded unit records a present but empty bound | +| `test_a_discriminating_collation_was_found`, `test_the_collated_pair`, `test_a_collated_overlap_returns_the_heaps_rows` | a range compares under its declared collation: the type's collation differs from text's, the pair holds the same rows and is summarised, four overlaps and the row ids agree with the heap | + +**THE UNBOUNDED ARMS COULD NOT FAIL UNTIL #1341.** In the shell, the unbounded tables +lacked the `payload` column the fingerprint sums, so both sides errored and `check` +compared "" with "". Once that was fixed, an empty row in the same unit kept them +green: an empty range sorts first, and a unit whose minimum is empty is never +pruned. And a separate `INSERT` put the unbounded row in a row group of its own. +With all three fixed, a writer that ignores unbounded ranges reddens both arms in +both twins. + +**THE COLLATION IS DISCOVERED, NOT ASSUMED.** A host with no collation ordering +differently from the database default declines the collation tests by name. + +Independent of `test/range_pruning.sh`: neither file reads or runs the other. ## 76. test_docs_upgrade_chain.py: the documented upgrade chain must be the one that ships diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index e8f6754e4..fe5243f4b 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -840,4 +840,6 @@ guard_tests 442 # tests) and test_latest_reading.py (1 -> 9 tests), by collection over the cluster files. # cluster_tests 837 -> 900 for #1339, one check per test in test_native_skip.py (13 -> 76 # tests), by collection over the cluster files. -cluster_tests 900 +# cluster_tests 900 -> 932 for #1341, test_range_pruning.py ported by name, one check per +# test (2 -> 34 tests), by collection over the cluster files. +cluster_tests 932 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 49584dc3a..e713319be 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -86,6 +86,7 @@ # The shape is `SHELL_REFERENCES`' in `test_harness_deps.py`, asserted in both # directions for the same reason: a one-way list rots into a permanent exemption. COMPLETE = ["advisory_lock_class", + "range_pruning", "alter_table_scans", "fdw_plan_privilege", "am_contracts", @@ -249,14 +250,6 @@ # having no SKIP verdict for an arm a run chooses not to make, does not carry. "large_values": "the 1 MB to 200 MB round trips and the TOASTed-update arms are " "ported; the opt-in chunk-over-1-GB refusal is not", - # The port asserts the engine counters, the heap oracle, the scattered zero - # and the declared-collation property. The shell suite additionally pins the - # three states of max_upper and its own fixture geometry by name, and those - # arms are not ported, so the pair does not grade to zero and is declared - # here rather than listed as complete. - "range_pruning": "counters, heap oracle, scattered zero and the declared " - "collation are ported; the three-state and fixture-geometry " - "premises of the shell suite are not", # #1137 ported the arms it added and the format-version pin. The rest of the # shell suite needs a C hook bound at test time (the metapage version) and # lib.sh's heap-mirror pair, and porting those is #432's work, not this one. diff --git a/test/pytest/test_range_pruning.py b/test/pytest/test_range_pruning.py index b5306b6e1..574e1ac07 100644 --- a/test/pytest/test_range_pruning.py +++ b/test/pytest/test_range_pruning.py @@ -1,204 +1,409 @@ -"""A range column prunes on overlap and containment, and under its own collation. +"""A range column prunes on overlap and containment, and under its own collation (#1144). A zone map recorded `minimum` and `maximum` only. Those are the range type's btree ordering, which sorts by lower bound and then upper bound, so the largest range is not the one reaching furthest right: a unit holding `[1,2)` and `[3,100)` has the same `maximum` as one holding `[1,2)` and `[3,4)`. Neither `&&` nor `@>` can be answered from that. `zone_map.max_upper` records the greatest upper bound, in three -states, and the scan skips a unit only when the summary proves it cannot match. +states (a bound; NULL for "no bound"; present but empty for "unbounded"), and the +scan skips a unit only when the summary proves it cannot match. -This file asserts the engine's own counters and a heap oracle. Independent of -test/range_pruning.sh: same public seam (a query against a columnar table and the -counters its scan prints), own fixture, own row counts, own observations. Neither -file reads or runs the other. +Both halves are asserted, because a pruning change can be wrong in two directions: +a heap mirror answers every query (a prune that removes a unit it should have kept +LOSES ROWS), and the scan's counters must move (a prune that removes nothing is +correct and worthless). -THE COUNTERS ARE READ AS TYPED JSON. `EXPLAIN (ANALYZE, FORMAT JSON)` comes back as -parsed Python, so "Columnar Chunk Groups Read" is a key holding an int rather than a -number a regex found in text. +THE PORT OF test/range_pruning.sh, BY NAME (#1341). Every check carries the shell +check's name and asks the same question of the same fixture: 50,000 rows in one +stripe, which at the default chunk_group_row_limit of 10,000 is one row group of +five vectors, so pruning shows at both levels. Independent of the shell suite: +neither file reads or runs the other. -THE HEAP IS THE ORACLE. A pruning defect returns FEWER rows, not an error, so every -predicate is answered by a heap table built from the same generator and the two row -sets are compared. Counting alone would miss a unit skipped and another wrongly -read, so the identifiers are compared, not only the count. +ONE CHECK PER TEST (#1339). Module fixtures build each data set once, and every +reading after that is read-only, so a red cannot stop the next check from running. +A failed query becomes `query_error()`, which every comparison refuses, so an error +fails under the check's own name and two failures never agree. """ +import re -ROWS = 12000 -GROUP = 1000 # set_options refuses anything smaller - - -def _plan(conn, sql): - with conn.cursor() as cur: - cur.execute(f"EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, FORMAT JSON) {sql}") - return cur.fetchone()[0] - - -def _counter(plan, key): - stack = [plan[0]["Plan"]] - while stack: - node = stack.pop(0) - if key in node: - return node[key] - stack.extend(node.get("Plans") or ()) - return None - - -def _ids(conn, sql): - with conn.cursor() as cur: - cur.execute(sql) - return sorted(r[0] for r in cur.fetchall()) - - -def _build(cur, name, order_by, am): - using = "USING pgcolumnar" if am else "" - cur.execute(f"CREATE TABLE {name} (id int, span tstzrange) {using}") - if am: - cur.execute( - f"SELECT pgcolumnar.set_options('{name}', stripe_row_limit => {GROUP}," - f" chunk_group_row_limit => {GROUP})" - ) - cur.execute( - f"INSERT INTO {name} SELECT g, tstzrange(t, t + interval '30 minutes') " - f"FROM (SELECT g, timestamptz '2024-01-01' + (g * interval '1 hour') AS t " - f"FROM generate_series(1, {ROWS}) g) s ORDER BY {order_by}" - ) - cur.execute(f"ANALYZE {name}") - - -def test_range_pruning(pgc_conn, expect): - probe = "tstzrange(timestamptz '2024-03-01', timestamptz '2024-03-01 06:00')" - with pgc_conn.cursor() as cur: - _build(cur, "rp_heap", "g", False) - _build(cur, "rp_col", "g", True) - _build(cur, "rp_scatter", "md5(g::text)", True) - - cur.execute("SELECT count(*) FROM rp_col") - expect.num(cur.fetchone()[0], ROWS, - "premise: the table holds every inserted row") - - cur.execute( - "SELECT count(*) FROM pgcolumnar.zone_map z" - " JOIN pgcolumnar.storage s USING (storage_id)" - " WHERE s.relation_oid = 'rp_col'::regclass AND z.max_upper IS NOT NULL") - expect.at_least(cur.fetchone()[0], 2, - "premise: the range column is summarised in several units") - - # THE CLUSTERED CASE. Groups hold consecutive spans, so a summary can exclude - # most of them. - sql = f"SELECT id FROM rp_col WHERE span && {probe}" - plan = _plan(pgc_conn, sql) - total = _counter(plan, "Columnar Chunk Groups Total") - read = _counter(plan, "Columnar Chunk Groups Read") - expect.at_least(total, 2, - "premise: the scan saw more than one chunk group to choose from") - expect.text("pruned" if (read or 0) < (total or 0) else f"read {read} of {total}", - "pruned", - "a clustered overlap skips chunk groups instead of reading them all") - - expect.rows(_ids(pgc_conn, sql), - _ids(pgc_conn, f"SELECT id FROM rp_heap WHERE span && {probe}"), - "and an overlap returns the heap's rows") - - # NEAR THE END (#1339). The probe above sits in group 2 of 12, so the groups after - # it are pruned on their LOWER bounds alone, and the arm above passed with - # max_upper ignored entirely. A probe in the last group leaves the eleven before - # it to the upper bound each records: exactly the groups holding a match are read. - late = "tstzrange(timestamptz '2025-04-24 04:00', timestamptz '2025-04-24 06:00')" - lsql = f"SELECT id FROM rp_col WHERE span && {late}" - lplan = _plan(pgc_conn, lsql) - lread = _counter(lplan, "Columnar Chunk Groups Read") - lheap = _ids(pgc_conn, f"SELECT id FROM rp_heap WHERE span && {late}") - holding = len({(i - 1) // GROUP for i in lheap}) - expect.text(f"read {lread} of {_counter(lplan, 'Columnar Chunk Groups Total')}, {holding} hold a match", - f"read {holding} of {ROWS // GROUP}, {holding} hold a match", - "a clustered overlap near the end reads only the groups that hold a match") - expect.rows(_ids(pgc_conn, lsql), lheap, "and that late overlap returns the heap's rows") - - contains = "SELECT id FROM %s WHERE span @> timestamptz '2024-03-01 02:15'" - expect.rows(_ids(pgc_conn, contains % "rp_col"), - _ids(pgc_conn, contains % "rp_heap"), - "containment of a point returns the heap's rows") - - # THE DOCUMENTED ZERO. Scattered, every group holds a span near the maximum, - # so no summary can exclude any of them. This is the outcome, not a shortfall. - splan = _plan(pgc_conn, f"SELECT id FROM rp_scatter WHERE span && {probe}") - stotal = _counter(splan, "Columnar Chunk Groups Total") - sread = _counter(splan, "Columnar Chunk Groups Read") - expect.num(sread, stotal, +import psycopg +import pytest + +from pgc_vacuity import query_error + +N = 50000 + +WINDOW = "tstzrange('2020-01-02 00:00+00','2020-01-02 01:00+00')" +WEEK = "tstzrange('2020-01-05 00:00+00','2020-01-12 00:00+00')" +POINT = "'2020-01-02 00:30+00'::timestamptz" +FUTURE = "tstzrange('2099-01-01 00:00+00','2099-01-02 00:00+00')" +# NEAR THE END (#1339). A window at minute 45,360 falls in the last of the group's five +# vectors. The four before it start before the window ends, so their LOWER bounds +# cannot rule them out; only the upper bound each one records can. +LATE = "tstzrange('2020-02-01 12:00+00','2020-02-01 13:00+00')" +FAR_POINT = "'2099-06-06 00:00+00'::timestamptz" + + +def _q(conn, sql): + """The first column of the first row, or a query_error naming the failure.""" + try: + return conn.execute(sql).fetchone()[0] + except psycopg.Error as exc: + return query_error(exc.diag.message_primary) + + +def _fingerprint(conn, table, pred): + """A count AND two sums: a count alone agrees whenever two wrong answers are the + same size.""" + return str(_q(conn, f"SELECT count(*) || '/' || coalesce(sum(payload),0) || '/' " + f"|| coalesce(sum(id),0) FROM {table} WHERE {pred}")) + + +def _counter(conn, table, pred, label): + """The scan's own count for `label`, or a query_error when the plan failed or has + no such line: a plan that lost the columnar scan must not read as "skipped + nothing".""" + try: + plan = "\n".join(r[0] for r in conn.execute( + f"EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF) SELECT count(*) FROM {table} " + f"WHERE {pred}").fetchall()) + except psycopg.Error as exc: + return query_error(exc.diag.message_primary) + m = re.search(re.escape(label) + r": (\d+)", plan) + return int(m.group(1)) if m else query_error(f"the plan has no '{label}' line") + + +def _zone(conn, table, where): + """Zone map rows of `table` matching `where`, counted.""" + return _q(conn, f"SELECT count(*) FROM pgcolumnar.zone_map z " + f"JOIN pgcolumnar.storage s USING (storage_id) " + f"WHERE s.relation_oid = '{table}'::regclass AND {where}") + + +def _schema_conn(pgc_cluster, schema): + conn = psycopg.connect(pgc_cluster.dsn(), autocommit=True) + conn.execute(f"DROP SCHEMA IF EXISTS {schema} CASCADE") + conn.execute(f"CREATE SCHEMA {schema}") + conn.execute(f"SET search_path = {schema}, public") + return conn + + +def _drop(conn, schema): + try: + conn.execute(f"DROP SCHEMA IF EXISTS {schema} CASCADE") + except psycopg.Error: + pass + finally: + conn.close() + + +def _build(conn, steps, what): + """Run each step; a failure is printed and leaves the table short or missing, + which the premises and the oracle then name.""" + try: + for sql in steps: + conn.execute(sql) + except psycopg.Error as exc: + print(f"-- building {what} failed: {exc.diag.message_primary}") + + +def _mk(name, columnar, clustered): + off = "g" if clustered else f"(g * 7919 % {N})" + return [f"CREATE TABLE {name} (id bigint, span tstzrange, payload int)" + + (" USING pgcolumnar" if columnar else "")] + ( + [f"SELECT pgcolumnar.set_options('{name}', stripe_row_limit => {N})"] + if columnar else []) + [ + f"INSERT INTO {name} SELECT g, " + f"tstzrange('2020-01-01'::timestamptz + ({off}||' minutes')::interval, " + f"'2020-01-01'::timestamptz + ({off}||' minutes')::interval + interval '1 hour'), " + f"g % 97 FROM generate_series(1,{N}) g", + f"ANALYZE {name}"] + + +UNBOUNDED_ROWS = ( + "SELECT g, tstzrange('2020-01-01'::timestamptz + (g||' minutes')::interval, " + "'2020-01-01'::timestamptz + (g||' minutes')::interval + interval '1 hour'), g % 97 " + "FROM generate_series(1,20000) g " + "UNION ALL SELECT 20001, tstzrange('2020-01-03'::timestamptz, NULL), 5") +# NO EMPTY RANGE IN THIS UNIT: an empty range sorts first, becomes the unit's minimum, +# and a unit whose minimum is empty is never pruned, so the unbounded arms could not +# fail. e_col holds the empty case on its own. + + +@pytest.fixture(scope="module") +def rp(pgc_cluster): + """The clustered and scattered pairs, the unbounded pair and the all-empty table.""" + c = _schema_conn(pgc_cluster, "pgc_rp") + try: + for name, columnar, clustered in (("r_col", True, True), ("r_heap", False, True), + ("s_col", True, False), ("s_heap", False, False)): + _build(c, _mk(name, columnar, clustered), name) + _build(c, [ + "CREATE TABLE u_col (id bigint, span tstzrange, payload int) USING pgcolumnar", + f"SELECT pgcolumnar.set_options('u_col', stripe_row_limit => {N})", + "CREATE TABLE u_heap (id bigint, span tstzrange, payload int)", + f"INSERT INTO u_col {UNBOUNDED_ROWS}", + f"INSERT INTO u_heap {UNBOUNDED_ROWS}", + "ANALYZE u_col"], "the unbounded pair") + # AN EMPTY RANGE NEEDS ITS OWN TABLE. In u_col it shares a unit with the + # unbounded row, and unbounded is absorbing, so that unit says nothing about + # empty. + _build(c, [ + "CREATE TABLE e_col (id bigint, span tstzrange) USING pgcolumnar", + f"SELECT pgcolumnar.set_options('e_col', stripe_row_limit => {N})", + "INSERT INTO e_col SELECT g, 'empty'::tstzrange FROM generate_series(1,2000) g", + "ANALYZE e_col"], "the all-empty table") + yield c + finally: + _drop(c, "pgc_rp") + + +# ---- premises ---------------------------------------------------------------------- + +def test_the_clustered_fixture_holds_the_rows(rp, expect): + expect.num(_q(rp, "SELECT count(*) FROM r_col"), N, + "premise: the clustered fixture holds the rows the numbers below count") + + +def test_the_heap_mirror_holds_the_same_rows(rp, expect): + expect.num(_q(rp, "SELECT count(*) FROM r_heap"), N, + "premise: the heap mirror holds the same rows") + + +def test_the_range_column_is_summarised_in_several_units(rp, expect): + expect.at_least(_zone(rp, "r_col", "z.column_index = 1 AND z.vector_index >= 0"), 4, + "premise: the range column is summarised in several units, " + "or nothing can be skipped") + + +def test_every_unit_carries_an_upper_bound(rp, expect): + expect.num(_zone(rp, "r_col", "z.column_index = 1 AND z.max_upper IS NOT NULL"), + _zone(rp, "r_col", "z.column_index = 1"), + "premise: every one of those units carries an upper bound") + + +def test_a_non_range_column_carries_none(rp, expect): + expect.num(_zone(rp, "r_col", "z.column_index <> 1 AND z.max_upper IS NOT NULL"), 0, + "premise: a non-range column carries none") + + +# ---- correctness: the heap answers the same questions ------------------------------ + +PROBES = {"WINDOW": WINDOW, "WEEK": WEEK, "POINT": POINT, "FUTURE": FUTURE, + "LATE": LATE, "FAR_POINT": FAR_POINT} +# (name, pair, operator, probe): a literal table, so the twin grader reads its names. +ORACLE = [ + ("overlap with a one-hour window returns the heap's rows", "r", "&&", "WINDOW"), + ("overlap with a week-long window returns the heap's rows", "r", "&&", "WEEK"), + ("containment of a point returns the heap's rows", "r", "@>", "POINT"), + ("a window no row can meet returns nothing, from both", "r", "&&", "FUTURE"), + ("and that late overlap returns the heap's rows", "r", "&&", "LATE"), + ("and still returns the right rows", "s", "&&", "WINDOW"), + ("a unit holding an unbounded range is never pruned from above", "u", "&&", "FUTURE"), + ("the same for a containment probe far to its right", "u", "@>", "FAR_POINT"), +] + + +@pytest.mark.parametrize("name, pair, op, probe", ORACLE) +def test_the_columnar_side_returns_the_heaps_rows(rp, expect, name, pair, op, probe): + pred = f"span {op} {PROBES[probe]}" + expect.text(_fingerprint(rp, f"{pair}_col", pred), _fingerprint(rp, f"{pair}_heap", pred), + name) + + +def test_the_first_three_oracle_probes_are_not_vacuously_empty(rp, expect): + counts = [_q(rp, f"SELECT count(*) FROM r_heap WHERE {p}") + for p in (f"span && {WINDOW}", f"span && {WEEK}", f"span @> {POINT}")] + expect.text("all three match rows" + if all(isinstance(n, int) and n > 0 for n in counts) + else f"AT LEAST ONE IS EMPTY {counts}", + "all three match rows", + "premise: the first three of those are not vacuously empty") + + +# ---- effect: the counters move ----------------------------------------------------- + +def test_the_overlap_qual_is_pushed_down(rp, expect): + expect.num(_counter(rp, "r_col", f"span && {WINDOW}", "Pushed-Down Filters"), 1, + "the overlap qual is pushed down, where it used to be dropped") + + +def test_the_zone_map_is_probed_for_it(rp, expect): + expect.at_least(_counter(rp, "r_col", f"span && {WINDOW}", "Zone Map Probes"), 1, + "and the zone map is probed for it") + + +def test_a_clustered_overlap_skips_vectors(rp, expect): + expect.at_least(_counter(rp, "r_col", f"span && {WINDOW}", "Vectors Skipped"), 1, + "a clustered overlap skips vectors instead of decoding them") + + +def test_a_late_overlap_skips_the_four_vectors_only_upper_bounds_rule_out(rp, expect): + expect.num(_counter(rp, "r_col", f"span && {LATE}", "Vectors Skipped"), 4, + "a clustered overlap near the end skips the four vectors only their " + "upper bounds rule out") + + +def test_a_containment_probe_skips_vectors(rp, expect): + expect.at_least(_counter(rp, "r_col", f"span @> {POINT}", "Vectors Skipped"), 1, + "so does a containment probe") + + +def test_a_window_no_row_can_meet_reads_no_chunk_group(rp, expect): + expect.num(_counter(rp, "r_col", f"span && {FUTURE}", "Chunk Groups Read"), 0, + "a window no row can meet reads no chunk group at all") + + +# ---- the documented zero ----------------------------------------------------------- + +def test_a_scattered_column_skips_nothing(rp, expect): + expect.num(_counter(rp, "s_col", f"span && {WINDOW}", "Vectors Skipped"), 0, "a scattered column skips nothing, which is what the documentation says") - expect.rows(_ids(pgc_conn, f"SELECT id FROM rp_scatter WHERE span && {probe}"), - _ids(pgc_conn, f"SELECT id FROM rp_heap WHERE span && {probe}"), - "and it still returns the right rows") - - -def test_a_range_prunes_under_its_declared_collation(pgc_conn, expect): - """A range compares under the collation it was DECLARED with. - - The type cache carries that as rng_collation; the element type's typcollation - is a different value. Summarising under one ordering and pruning under another - makes the scan MISS ROWS. Not reachable with a built-in range type, whose - subtypes are all non-collatable, so this needs a user-defined one. - """ - # ASKED OF THE CATALOG, NOT OF A FAILED STATEMENT. Naming a collation that - # does not exist raises, and one failed statement makes psycopg raise for - # every later one in the transaction, so a try/except here would both hide a - # real failure and poison the rest of the test. - coll = None - with pgc_conn.cursor() as cur: - cur.execute( - "SELECT collname FROM pg_collation" - " WHERE collname = ANY(%s)" - # ICU collations are listed even in a build without ICU, and naming - # one raises FeatureNotSupported rather than returning a wrong answer. - " AND collprovider <> 'i'" - " AND (collencoding = -1" - " OR collencoding = (SELECT encoding FROM pg_database" - " WHERE datname = current_database()))", - (["en_US.utf8", "en_GB.utf8", "unicode", "ucs_basic", "C"],)) - available = [r[0] for r in cur.fetchall()] + + +# ---- the two values upper() cannot tell apart -------------------------------------- + +def test_the_unbounded_row_really_is_matched(rp, expect): + expect.num(_q(rp, f"SELECT count(*) FROM u_heap WHERE span && {FUTURE}"), 1, + "premise: that unbounded row really is matched, so the arm is not vacuous") + + +def test_the_all_empty_table_is_summarised(rp, expect): + expect.at_least(_zone(rp, "e_col", "z.column_index = 1"), 1, + "premise: the all-empty table is summarised at all") + + +def test_an_empty_range_contributes_no_bound(rp, expect): + expect.num(_zone(rp, "e_col", "z.column_index = 1 AND z.max_upper IS NULL"), + _zone(rp, "e_col", "z.column_index = 1"), + "an empty range contributes no bound, so its units record NULL") + + +def test_an_unbounded_range_records_a_present_but_empty_bound(rp, expect): + n = _zone(rp, "u_col", "z.column_index = 1 AND z.max_upper IS NOT NULL " + "AND octet_length(z.max_upper) = 0") + expect.text("present and empty" if isinstance(n, int) and n >= 1 + else f"NOT RECORDED ({n})", "present and empty", + "and an unbounded range records a bound that is present but empty") + + +# ---- a range compares under ITS OWN collation, not its element type's -------------- +# +# NOT REACHABLE WITH A BUILT-IN RANGE TYPE: every built-in range is over a +# non-collatable subtype. It needs a user-defined range over text with a COLLATION +# whose ordering differs from the database default, and the collation is discovered, +# not assumed. ASKED OF THE CATALOG, NOT OF A FAILED STATEMENT: an ICU collation is +# listed even in a build without ICU, and naming one raises. + +COLLATED_PROBES = ['["A","C"]', '["a","b"]', '["B","M"]', '["m","z"]'] +NO_COLLATION = ("no available collation orders differently from this database's " + "default, so the arm cannot discriminate") +RC_GEN = ("SELECT g, tr_coll(v, v || 'zz') FROM (SELECT g, " + "(ARRAY['A','a','B','b','C','c','M','m','Y','y'])[1 + (g / 1000) % 10] " + "AS v FROM generate_series(1, 10000) g) s") + + +@pytest.fixture(scope="module") +def rc(pgc_cluster): + """(connection, collation), the collation None when no candidate discriminates.""" + c = _schema_conn(pgc_cluster, "pgc_rp_coll") + try: + coll = None + available = [r[0] for r in c.execute( + "SELECT collname FROM pg_collation WHERE collname = ANY(%s) " + "AND collprovider <> 'i' AND (collencoding = -1 OR collencoding = " + "(SELECT encoding FROM pg_database WHERE datname = current_database()))", + (["en_US.utf8", "en_GB.utf8", "unicode", "ucs_basic", "C"],)).fetchall()] for candidate in available: - cur.execute( - 'SELECT (\'B\' < \'a\' COLLATE "%s") <> (\'B\' < \'a\')' - % candidate) - if cur.fetchone()[0]: - coll = '"%s"' % candidate + if _q(c, f"SELECT ('B' < 'a' COLLATE \"{candidate}\") <> ('B' < 'a')") is True: + coll = f'"{candidate}"' break + if coll is not None: + # 1000 is the floor set_options enforces. + _build(c, [ + f"CREATE TYPE tr_coll AS RANGE (SUBTYPE = text, COLLATION = {coll})", + "CREATE TABLE rc_heap (id int, span tr_coll)", + "CREATE TABLE rc_col (id int, span tr_coll) USING pgcolumnar", + "SELECT pgcolumnar.set_options('rc_col', stripe_row_limit => 1000, " + "chunk_group_row_limit => 1000)", + f"INSERT INTO rc_heap {RC_GEN}", + f"INSERT INTO rc_col {RC_GEN}", + "ANALYZE rc_heap", + "ANALYZE rc_col"], "the collated pair") + yield c, coll + finally: + _drop(c, "pgc_rp_coll") + + +def _collated(rc, expect, name): + """The connection, or None after declaring this check unrunnable.""" + conn, coll = rc if coll is None: - expect.cannot_run( - "MISSING_DEPENDENCY", - "no available collation orders differently from this database's " - "default, so the arm cannot pose its question", - name="a range prunes under its declared collation") + expect.cannot_run("MISSING_DEPENDENCY", NO_COLLATION, name=name) + return None + return conn + + +def _differs(conn): + return str(_q(conn, "SELECT (SELECT rngcollation FROM pg_range " + "WHERE rngtypid = 'tr_coll'::regtype) " + "<> (SELECT typcollation FROM pg_type WHERE oid = 'text'::regtype)")) + + +def _same_rows(conn): + return (f"{_q(conn, 'SELECT count(*) FROM rc_heap')} " + f"{_q(conn, 'SELECT count(*) FROM rc_col')}") + + +def _summarised(conn): + n = _zone(conn, "rc_col", "z.max_upper IS NOT NULL") + return "summarised" if isinstance(n, int) and n >= 1 else f"NOT SUMMARISED ({n})" + + +def _ids_match(conn): + ids = ("SELECT string_agg(id::text, ',' ORDER BY id) FROM {} " + "WHERE span && '[\"A\",\"C\"]'::tr_coll") + col, heap = str(_q(conn, ids.format("rc_col"))), str(_q(conn, ids.format("rc_heap"))) + return "same ids" if col == heap and "QUERY_ERROR" not in col else f"{col} vs {heap}" + + +READ = {"differs": (_differs, "True"), "same_rows": (_same_rows, "10000 10000"), + "summarised": (_summarised, "summarised"), "ids": (_ids_match, "same ids")} +# (name, reading): a literal table, so the twin grader reads its names. +COLLATED = [ + ("premise: the range type collates differently from its element type", "differs"), + ("premise: both tables hold the same rows before any predicate", "same_rows"), + ("premise: the collated range column is summarised, so pruning can engage", "summarised"), + ("and the rows themselves match, not only the count", "ids"), +] + + +def test_a_discriminating_collation_was_found(rc, expect): + """The shell records this name only as its SKIP. Here it also asserts what the + skip stands for: the collation chosen really orders differently.""" + conn = _collated(rc, expect, "a range type's declared collation is the one it prunes under") + if conn is None: return + expect.text(str(_q(conn, f"SELECT ('B' < 'a' COLLATE {rc[1]}) <> ('B' < 'a')")), "True", + "a range type's declared collation is the one it prunes under") + - with pgc_conn.cursor() as cur: - cur.execute(f"CREATE TYPE rp_tr AS RANGE (SUBTYPE = text, COLLATION = {coll})") - cur.execute("CREATE TABLE rpc_heap (id int, span rp_tr)") - cur.execute("CREATE TABLE rpc_col (id int, span rp_tr) USING pgcolumnar") - cur.execute("SELECT pgcolumnar.set_options('rpc_col'," - " stripe_row_limit => 1000, chunk_group_row_limit => 1000)") - gen = ("SELECT g, rp_tr(v, v || 'zz') FROM (SELECT g," - " (ARRAY['A','a','B','b','C','c','M','m','Y','y'])[1 + (g / 1000) % 10]" - " AS v FROM generate_series(1, 10000) g) s") - cur.execute(f"INSERT INTO rpc_heap {gen}") - cur.execute(f"INSERT INTO rpc_col {gen}") - cur.execute("ANALYZE rpc_heap") - cur.execute("ANALYZE rpc_col") - - cur.execute("SELECT (SELECT rngcollation FROM pg_range" - " WHERE rngtypid = 'rp_tr'::regtype)" - " <> (SELECT typcollation FROM pg_type WHERE oid = 'text'::regtype)") - expect.text("differs" if cur.fetchone()[0] else "same", "differs", - "premise: the range collates differently from its element type") - - cur.execute("SELECT count(*) FROM pgcolumnar.zone_map z" - " JOIN pgcolumnar.storage s USING (storage_id)" - " WHERE s.relation_oid = 'rpc_col'::regclass" - " AND z.max_upper IS NOT NULL") - expect.at_least(cur.fetchone()[0], 1, - "premise: the collated range column is summarised") - - for lo, hi in (("A", "C"), ("B", "M"), ("a", "b")): - q = "SELECT id FROM %%s WHERE span && rp_tr('%s', '%s')" % (lo, hi) - expect.rows(_ids(pgc_conn, q % "rpc_col"), - _ids(pgc_conn, q % "rpc_heap"), - f"overlap [{lo},{hi}] returns the heap's rows under a declared collation") +@pytest.mark.parametrize("name, reading", COLLATED) +def test_the_collated_pair(rc, expect, name, reading): + conn = _collated(rc, expect, name) + if conn is None: + return + fn, want = READ[reading] + expect.text(fn(conn), want, name) + + +@pytest.mark.parametrize("probe", COLLATED_PROBES) +def test_a_collated_overlap_returns_the_heaps_rows(rc, expect, probe): + conn, coll = rc + if coll is None: + expect.cannot_run("MISSING_DEPENDENCY", NO_COLLATION, + name=f"overlap {probe} returns the heap's rows under a declared collation") + return + expect.text(str(_q(conn, f"SELECT count(*) FROM rc_col WHERE span && '{probe}'::tr_coll")), + str(_q(conn, f"SELECT count(*) FROM rc_heap WHERE span && '{probe}'::tr_coll")), + f"overlap {probe} returns the heap's rows under a declared collation") diff --git a/test/range_pruning.sh b/test/range_pruning.sh index 9713951eb..8455b83d8 100755 --- a/test/range_pruning.sh +++ b/test/range_pruning.sh @@ -169,25 +169,38 @@ check_text "and still returns the right rows" \ "$(fingerprint s_col "span && $WINDOW")" "$(fingerprint s_heap "span && $WINDOW")" # ---- the two values upper() cannot tell apart ------------------------------ +# +# THE PAYLOAD COLUMN IS WHAT fingerprint SUMS. These tables lacked it until #1341, +# so both sides of the two arms below errored, q turned each error into "", and +# `check` compared "" with "" and passed: measured on PG17, u_col=[] u_heap=[]. +# check_text refuses an empty side, so a failed query is now a red. +# +# AND NO EMPTY RANGE IN THIS UNIT. An empty range sorts first, so it becomes the +# unit's minimum, and a unit whose minimum is empty is never pruned at all +# (native_range_excludes returns false before reading max_upper). With the empty +# row here, a writer that ignored unbounded ranges left both arms green; e_col +# below holds the empty case on its own. +# +# AND ONE STATEMENT, SO ONE UNIT. A separate INSERT for the unbounded row wrote it +# to a row group of its own, which records no finite bound to prune on, so the +# arms passed with the unbounded state ignored by the writer. psql_run "DROP TABLE IF EXISTS u_col; DROP TABLE IF EXISTS u_heap; - CREATE TABLE u_col (id bigint, span tstzrange) USING pgcolumnar; - CREATE TABLE u_heap (id bigint, span tstzrange); + CREATE TABLE u_col (id bigint, span tstzrange, payload int) USING pgcolumnar; + CREATE TABLE u_heap (id bigint, span tstzrange, payload int); INSERT INTO u_col SELECT g, tstzrange('2020-01-01'::timestamptz + (g||' minutes')::interval, - '2020-01-01'::timestamptz + (g||' minutes')::interval + interval '1 hour') - FROM generate_series(1,20000) g; - INSERT INTO u_col VALUES (20001, tstzrange('2020-01-03'::timestamptz, NULL)); - INSERT INTO u_col VALUES (20002, 'empty'::tstzrange); + '2020-01-01'::timestamptz + (g||' minutes')::interval + interval '1 hour'), + g % 97 FROM generate_series(1,20000) g + UNION ALL SELECT 20001, tstzrange('2020-01-03'::timestamptz, NULL), 5; INSERT INTO u_heap SELECT g, tstzrange('2020-01-01'::timestamptz + (g||' minutes')::interval, - '2020-01-01'::timestamptz + (g||' minutes')::interval + interval '1 hour') - FROM generate_series(1,20000) g; - INSERT INTO u_heap VALUES (20001, tstzrange('2020-01-03'::timestamptz, NULL)); - INSERT INTO u_heap VALUES (20002, 'empty'::tstzrange); + '2020-01-01'::timestamptz + (g||' minutes')::interval + interval '1 hour'), + g % 97 FROM generate_series(1,20000) g + UNION ALL SELECT 20001, tstzrange('2020-01-03'::timestamptz, NULL), 5; ANALYZE u_col;" >/dev/null -check "a unit holding an unbounded range is never pruned from above" \ +check_text "a unit holding an unbounded range is never pruned from above" \ "$(fingerprint u_col "span && $FUTURE")" "$(fingerprint u_heap "span && $FUTURE")" -check "the same for a containment probe far to its right" \ +check_text "the same for a containment probe far to its right" \ "$(fingerprint u_col "span @> '2099-06-06 00:00+00'::timestamptz")" \ "$(fingerprint u_heap "span @> '2099-06-06 00:00+00'::timestamptz")" check "premise: that unbounded row really is matched, so the arm is not vacuous" \