From ca693a991b48b2a431a095b7563072502c780aee Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Sat, 17 Oct 2026 03:08:56 +0000 Subject: [PATCH] test: native_skip's twin runs one check per test (#1339) The last of my twins with the shape I asked OffgridwithJD to remove from #1517-#1519: 13 tests holding 76 checks, so the first red in a test hid the rest. Module fixtures build each data set once (the coercible-type pair, the scalar table, the C and default-collation LIKE pairs, the locale-C database, the non-C collation pair, the partitioned table). Every reading after that is read-only, so each test runs one query pair and asserts one thing, through parametrize tables of literal names the twin grader reads. Every reading goes through _try, with a -1 sentinel for expect.num and an error row for expect.row_set, so an error is a named red. With zone-map skipping disabled in the scan, on PG17: shell 21 red old twin 10 red (3 passed, 10 failed: the rest hidden) new twin the same 21 names as the shell 76 tests, 76 checks, 0 names missing in the grader; 68/68 shell and 76 passed on each of PG15-19. cluster_tests 837 -> 900 by collection. The partition premise also filters on the fixture's own schema. Co-Authored-By: Claude Opus 5.5 --- test/pytest/TESTS.md | 33 +- test/pytest/expected_tests.txt | 4 +- test/pytest/test_native_skip.py | 617 ++++++++++++++++++++------------ 3 files changed, 422 insertions(+), 232 deletions(-) diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index bed4cafb0..6b99a8467 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -7323,13 +7323,32 @@ The pytest twin of `test/native_skip.sh`, every check by the shell's name (#1433 | test | what it holds | | --- | --- | -| `test_a_range_on_a_coercible_type_prunes` | a range on inet, cidr, a domain over cidr, text, varchar and a domain over varchar returns heap's rows and removes chunk groups | -| `test_equality_on_a_coercible_type_prunes` | equality on cidr and on varchar, which also takes the bloom probe, returns heap's row and removes chunk groups | -| `test_scalar_ranges_and_equality_prune_and_agree` | the plan is the columnar scan; ranges and equality on int, bigint (with and without casts, #477) and domains over both (#483) return heap's rows and remove chunk groups; a non-selective scan removes none | -| `test_an_anchored_like_prunes_under_c_and_only_then` | an anchored LIKE prunes under COLLATE "C" with heap's rows, unanchored and escaped patterns and `_` wildcards included; under this cluster's non-C default it is refused (#426) | -| `test_a_default_collation_in_a_locale_c_database_derives_the_bounds` | in a database created with LC_COLLATE 'C', a default-collation column derives both bounds and returns the right row (#426) | -| `test_an_anchored_like_does_not_prune_under_a_non_c_collation` | under a discovered non-C collation a range still prunes but an anchored LIKE does not, and rows match heap; declares unrunnable when the build has none (#426) | -| `test_a_columnar_partition_takes_the_custom_scan_and_prunes` | both partitions are columnar, the partitioned table returns heap's rows, plans the columnar custom scan and prunes (#436) | +| `test_heaps_range_on_a_coercible_type_holds_121_rows` | **premise**: heap's range on each of inet, cidr, a domain over cidr, text, varchar and a domain over varchar holds 121 rows (parametrized) | +| `test_a_range_on_a_coercible_type_returns_heaps_rows` | the columnar range returns heap's rows, for each of the six types (#1433) (parametrized) | +| `test_a_range_on_a_coercible_type_skips_groups` | and removes chunk groups, for each of the six types (#1433) (parametrized) | +| `test_equality_on_a_coercible_type_returns_heaps_rows` | equality on cidr and on varchar, which also takes the bloom probe, returns heap's row (parametrized) | +| `test_heaps_equality_on_a_coercible_type_finds_the_row` | **premise**: heap's equality finds row 11009 (parametrized) | +| `test_equality_on_a_coercible_type_skips_groups` | and the columnar equality removes chunk groups (parametrized) | +| `test_the_scalar_plan_is_a_columnar_custom_scan` | the plan under test is the columnar custom scan | +| `test_scalar_rows_agree_with_heap` | ranges on int, bigint (with and without casts, #477) and domains over both (#483) return heap's rows (parametrized) | +| `test_scalar_counts_agree_with_heap` | counts under equality, one-sided and non-selective predicates equal heap's (parametrized) | +| `test_scalar_counts_are_known` | the row count, absent values, out-of-range values and a present boundary give known counts (parametrized) | +| `test_a_scalar_predicate_skips_groups` | each selective predicate, cast or not, on a column or a domain, removes chunk groups (#477, #483) (parametrized) | +| `test_a_non_selective_scan_skips_nothing` | a non-selective scan removes none | +| `test_a_like_or_range_prunes` | under COLLATE "C" an anchored LIKE, a trailing `_` and a low-end prefix prune, and the equivalent ranges do on both tables (#426) (parametrized) | +| `test_a_like_does_not_prune` | an unanchored LIKE, and an anchored one under this cluster's non-C default, remove nothing (#426) (parametrized) | +| `test_a_like_returns_heaps_rows` | every LIKE form, escaped and `_` wildcards included, returns heap's rows (#426) (parametrized) | +| `test_a_default_collation_in_a_locale_c_database_derives_the_bounds` | in a database created with LC_COLLATE 'C', a default-collation column derives both bounds (#426) | +| `test_and_it_returns_the_right_rows` | and it returns the right row (#426) | +| `test_a_range_still_prunes_under_a_non_c_collation` | **premise**: under a discovered non-C collation a range still prunes; declares unrunnable when the build has none | +| `test_an_anchored_like_does_not_prune_under_a_non_c_collation` | but an anchored LIKE does not (#426) | +| `test_a_non_c_collation_like_returns_heaps_rows` | and its rows match heap's (#426) | +| `test_both_partitions_are_columnar` | **premise**: both partitions are columnar (#436) | +| `test_a_partitioned_columnar_table_returns_heaps_rows` | the partitioned table returns heap's rows (#436) | +| `test_a_partition_takes_the_columnar_custom_scan` | it plans the columnar custom scan (#436) | +| `test_a_partition_prunes_row_groups` | and it prunes row groups (#436) | + +ONE CHECK PER TEST (#1339): module fixtures build each data set once, and each test asserts one reading. Measured with zone-map skipping disabled in the scan on PG17: the shell reddens 21 checks. The earlier twin, 13 tests holding 76 checks, reported 10. This one reports the same 21. ## 134. test_unique_conc.py: concurrent inserters of one unique key serialize diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 8c755df65..e4e07f46d 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -835,4 +835,6 @@ guard_tests 440 # five tests, by collection over the cluster files on this tree. # cluster_tests 792 -> 837 for #1339, one check per test in test_native_agg.py (3 -> 40 # tests) and test_latest_reading.py (1 -> 9 tests), by collection over the cluster files. -cluster_tests 837 +# cluster_tests 837 -> 900 for #1339, one check per test in test_native_skip.py (13 -> 76 +# tests), by collection over the cluster files. +cluster_tests 900 diff --git a/test/pytest/test_native_skip.py b/test/pytest/test_native_skip.py index 706e92c55..3f016cd3a 100644 --- a/test/pytest/test_native_skip.py +++ b/test/pytest/test_native_skip.py @@ -16,6 +16,11 @@ (#483); an anchored LIKE as a range under the C collation and its refusals (#426); and a columnar partition taking the custom scan (#436). Independent of the shell arms: different table names and values where the names allow. + +ONE CHECK PER TEST (#1339). Module fixtures build each data set once; every reading +is read-only after that, so each test runs one query pair and asserts one thing, and +a red cannot stop the next check from running. Every reading goes through `_try`, so +an error is a named red rather than an unnamed test ERROR. """ import json import re @@ -26,24 +31,29 @@ REMOVED = re.compile(r"Columnar Chunk Groups Removed by Filter: (\d+)") -@pytest.fixture -def tables(pgc_conn): - pgc_conn.execute("CREATE DOMAIN skp_cidr AS cidr") - pgc_conn.execute("CREATE DOMAIN skp_vc AS varchar(16)") - cols = "id int, ip inet, cd cidr, dcd skp_cidr, t text, vc varchar(16), dvc skp_vc" - pgc_conn.execute(f"CREATE TABLE skh ({cols})") - pgc_conn.execute(f"CREATE TABLE skn ({cols}) USING pgcolumnar") - pgc_conn.execute("SELECT pgcolumnar.set_options('skn', stripe_row_limit => 2000, " - "chunk_group_row_limit => 1000)") - gen = ("SELECT g, a::inet, a::cidr, a::cidr, s, s, s FROM " - "(SELECT g, '172.16.' || (g / 200) || '.' || (g % 200) AS a, " - "'k' || lpad(g::text, 5, '0') AS s FROM generate_series(1, 18000) g) x") - pgc_conn.execute(f"INSERT INTO skh {gen}") - pgc_conn.execute(f"INSERT INTO skn {gen}") - return pgc_conn +def _try(fn, *args, **kw): + """A reading, or the error that replaced it, so a failing query is a named red.""" + try: + return fn(*args, **kw) + except psycopg.Error as exc: + return f"error: {exc.diag.message_primary}" + +def _num(v, what=""): + """A count for expect.num, or -1 with the error printed: expect.num refuses a + non-number with a VacuityError rather than a named red.""" + if isinstance(v, int): + return v + print(f"-- {what or 'reading'}: not a count: {v}") + return -1 -def _removed(conn, sql): + +def _rowsv(v): + """Rows for expect.row_set, or one row carrying the error, so it mismatches by name.""" + return v if isinstance(v, list) else [(str(v),)] + + +def _removed_q(conn, sql): """Chunk groups the zone maps removed, or -1 when the plan reports no counter at all: a plan that lost the columnar scan must not read as "removed nothing", or every refusal arm passes on it (#1510 review).""" @@ -53,149 +63,231 @@ def _removed(conn, sql): return int(m.group(1)) if m else -1 +def _removed(conn, sql): + return _num(_try(_removed_q, conn, sql), sql) + + +def _one(conn, sql): + return conn.execute(sql).fetchone()[0] + + +def _rows(conn, sql): + return [tuple(r) for r in conn.execute(sql).fetchall()] + + def _ids(conn, table, where): - return conn.execute(f"SELECT string_agg(id::text, ',' ORDER BY id) FROM {table} " - f"WHERE {where}").fetchone()[0] + return str(_try(lambda: conn.execute(f"SELECT string_agg(id::text, ',' ORDER BY id) FROM {table} " + f"WHERE {where}").fetchone()[0])) -IP_RANGE = "BETWEEN '172.16.30.0' AND '172.16.30.120'" -S_RANGE = "BETWEEN 'k06000' AND 'k06120'" +def _schema_conn(pgc_cluster, schema): + """A connection whose private schema is first on the search_path, as pgc_conn + gives a test, but for a module fixture.""" + conn = psycopg.connect(pgc_cluster.dsn(), autocommit=True) + conn.execute(f"DROP SCHEMA IF EXISTS {schema} CASCADE") + conn.execute(f"CREATE SCHEMA {schema}") + conn.execute(f"SET search_path = {schema}, public") + return conn + + +def _drop(conn, schema): + try: + conn.execute(f"DROP SCHEMA IF EXISTS {schema} CASCADE") + finally: + conn.close() -@pytest.mark.parametrize("col, what, pred", [ +# ---- a type binary-coercible to its opclass's type prunes (#1433) ------------------- + +@pytest.fixture(scope="module") +def tables(pgc_cluster): + c = _schema_conn(pgc_cluster, "pgc_nsk_coerce") + try: + c.execute("CREATE DOMAIN skp_cidr AS cidr") + c.execute("CREATE DOMAIN skp_vc AS varchar(16)") + cols = "id int, ip inet, cd cidr, dcd skp_cidr, t text, vc varchar(16), dvc skp_vc" + c.execute(f"CREATE TABLE skh ({cols})") + c.execute(f"CREATE TABLE skn ({cols}) USING pgcolumnar") + c.execute("SELECT pgcolumnar.set_options('skn', stripe_row_limit => 2000, " + "chunk_group_row_limit => 1000)") + gen = ("SELECT g, a::inet, a::cidr, a::cidr, s, s, s FROM " + "(SELECT g, '172.16.' || (g / 200) || '.' || (g % 200) AS a, " + "'k' || lpad(g::text, 5, '0') AS s FROM generate_series(1, 18000) g) x") + _try(c.execute, f"INSERT INTO skh {gen}") + _try(c.execute, f"INSERT INTO skn {gen}") + yield c + finally: + _drop(c, "pgc_nsk_coerce") + + +IP_RANGE = "BETWEEN '172.16.30.0' AND '172.16.30.120'" +S_RANGE = "BETWEEN 'k06000' AND 'k06120'" +COERCE_RANGES = [ ("ip", "inet", IP_RANGE), ("cd", "cidr", IP_RANGE), ("dcd", "a domain over cidr", IP_RANGE), ("t", "text", S_RANGE), ("vc", "varchar", S_RANGE), ("dvc", "a domain over varchar", S_RANGE), -]) -def test_a_range_on_a_coercible_type_prunes(tables, expect, col, what, pred): +] + + +@pytest.mark.parametrize("col, what, pred", COERCE_RANGES) +def test_heaps_range_on_a_coercible_type_holds_121_rows(tables, expect, col, what, pred): heap = _ids(tables, "skh", f"{col} {pred}") - expect.num(len(heap.split(",")) if heap else 0, 121, + expect.num(len(heap.split(",")) if heap and not heap.startswith("error") else 0, 121, f"premise: heap's range on {what} holds 121 rows") - expect.text(_ids(tables, "skn", f"{col} {pred}"), heap, + + +@pytest.mark.parametrize("col, what, pred", COERCE_RANGES) +def test_a_range_on_a_coercible_type_returns_heaps_rows(tables, expect, col, what, pred): + expect.text(_ids(tables, "skn", f"{col} {pred}"), _ids(tables, "skh", f"{col} {pred}"), f"a range on {what} returns the same rows as heap (#1433)") + + +@pytest.mark.parametrize("col, what, pred", COERCE_RANGES) +def test_a_range_on_a_coercible_type_skips_groups(tables, expect, col, what, pred): expect.at_least(_removed(tables, f"SELECT id FROM skn WHERE {col} {pred}"), 1, - f"and a range on {what} skips groups (#1433)") + f"and a range on {what} skips groups (#1433)") -@pytest.mark.parametrize("col, what, val", [ +COERCE_EQ = [ ("cd", "cidr", "'172.16.55.9'"), ("vc", "varchar", "'k11009'"), -]) -def test_equality_on_a_coercible_type_prunes(tables, expect, col, what, val): +] + + +@pytest.mark.parametrize("col, what, val", COERCE_EQ) +def test_equality_on_a_coercible_type_returns_heaps_rows(tables, expect, col, what, val): """Equality also takes the bloom probe, which hashes the constant with the column's hash proc: the same proc for both types of each pair.""" expect.text(_ids(tables, "skn", f"{col} = {val}"), "11009", f"equality on {what} returns the same rows as heap (#1433)") + + +@pytest.mark.parametrize("col, what, val", COERCE_EQ) +def test_heaps_equality_on_a_coercible_type_finds_the_row(tables, expect, col, what, val): expect.text(_ids(tables, "skh", f"{col} = {val}"), "11009", f"premise: heap's equality on {what} finds row 11009") - expect.at_least(_removed(tables, f"SELECT id FROM skn WHERE {col} = {val}"), 1, - f"and equality on {what} skips groups (#1433)") -# ---- the rest of native_skip.sh, by name (#1341) ------------------------------------ - -def _one(conn, sql): - return conn.execute(sql).fetchone()[0] +@pytest.mark.parametrize("col, what, val", COERCE_EQ) +def test_equality_on_a_coercible_type_skips_groups(tables, expect, col, what, val): + expect.at_least(_removed(tables, f"SELECT id FROM skn WHERE {col} = {val}"), 1, + f"and equality on {what} skips groups (#1433)") -def _rows(conn, sql): - return [tuple(r) for r in conn.execute(sql).fetchall()] - +# ---- the rest of native_skip.sh, by name (#1341) ------------------------------------ -@pytest.fixture -def scalar(pgc_conn): +@pytest.fixture(scope="module") +def scalar(pgc_cluster): """20480 rows in 2048-row groups, ten groups with contiguous id ranges; d4 and d8 are domains over int and bigint holding id and k (#483).""" - c = pgc_conn - c.execute("CREATE DOMAIN nsq_acct4 AS int") - c.execute("CREATE DOMAIN nsq_acct8 AS bigint") - cols = "id int, k bigint, label text, d4 nsq_acct4, d8 nsq_acct8" - c.execute(f"CREATE TABLE sqh ({cols})") - c.execute(f"CREATE TABLE sqn ({cols}) USING pgcolumnar") - c.execute("SELECT pgcolumnar.set_options('sqn', stripe_row_limit => 2048, " - "chunk_group_row_limit => 1024)") - gen = ("SELECT g, (g * 10)::bigint, 'lbl-' || (g % 100), g::nsq_acct4, (g * 10)::nsq_acct8 " - "FROM generate_series(1, 20480) g") - c.execute(f"INSERT INTO sqh {gen}") - c.execute(f"INSERT INTO sqn {gen}") - return c + c = _schema_conn(pgc_cluster, "pgc_nsk_scalar") + try: + c.execute("CREATE DOMAIN nsq_acct4 AS int") + c.execute("CREATE DOMAIN nsq_acct8 AS bigint") + cols = "id int, k bigint, label text, d4 nsq_acct4, d8 nsq_acct8" + c.execute(f"CREATE TABLE sqh ({cols})") + c.execute(f"CREATE TABLE sqn ({cols}) USING pgcolumnar") + c.execute("SELECT pgcolumnar.set_options('sqn', stripe_row_limit => 2048, " + "chunk_group_row_limit => 1024)") + gen = ("SELECT g, (g * 10)::bigint, 'lbl-' || (g % 100), g::nsq_acct4, (g * 10)::nsq_acct8 " + "FROM generate_series(1, 20480) g") + _try(c.execute, f"INSERT INTO sqh {gen}") + _try(c.execute, f"INSERT INTO sqn {gen}") + yield c + finally: + _drop(c, "pgc_nsk_scalar") -def test_scalar_ranges_and_equality_prune_and_agree(scalar, expect): - c = scalar - plan = c.execute("EXPLAIN (COSTS OFF, FORMAT JSON) " - "SELECT id FROM sqn WHERE id BETWEEN 5000 AND 5100").fetchone()[0] +def test_the_scalar_plan_is_a_columnar_custom_scan(scalar, expect): + plan = _try(lambda: scalar.execute("EXPLAIN (COSTS OFF, FORMAT JSON) " + "SELECT id FROM sqn WHERE id BETWEEN 5000 AND 5100").fetchone()[0]) + if isinstance(plan, str) and plan.startswith("error"): + plan = [{"Plan": {"Node Type": plan}}] expect.plan_marker(plan if isinstance(plan, list) else json.loads(plan), "Columnar Projected Columns", name="the plan under test is a columnar custom scan") - expect.num(_one(c, "SELECT count(*) FROM sqn"), 20480, "row count") - expect.num(_one(c, "SELECT count(*) FROM sqn"), _one(c, "SELECT count(*) FROM sqh"), - "no-predicate scan returns all rows") - - expect.row_set(_rows(c, "SELECT id, k, label FROM sqn WHERE id BETWEEN 5000 AND 5100"), - _rows(c, "SELECT id, k, label FROM sqh WHERE id BETWEEN 5000 AND 5100"), - "range result parity") - expect.at_least(_removed(c, "SELECT id FROM sqn WHERE id BETWEEN 5000 AND 5100"), 1, - "range skips groups") - expect.num(_one(c, "SELECT count(*) FROM sqn WHERE id = 12345"), - _one(c, "SELECT count(*) FROM sqh WHERE id = 12345"), "equality result parity") - expect.at_least(_removed(c, "SELECT id FROM sqn WHERE id = 12345"), 1, - "equality skips groups") - expect.row_set(_rows(c, "SELECT id FROM sqn WHERE k BETWEEN 100000::bigint AND 101000::bigint"), - _rows(c, "SELECT id FROM sqh WHERE k BETWEEN 100000::bigint AND 101000::bigint"), - "bigint range parity") - expect.at_least(_removed(c, "SELECT id FROM sqn WHERE k BETWEEN 100000::bigint AND 101000::bigint"), 1, - "bigint range skips groups") + + +# (name, select list, where): the columnar rows equal heap's +SCALAR_ROWS = [ + ("range result parity", "id, k, label", "id BETWEEN 5000 AND 5100"), + ("bigint range parity", "id", "k BETWEEN 100000::bigint AND 101000::bigint"), # the same predicate without the casts (#477): parity before skipping, because the # risk in an int8 column against an int4 constant is a wrong answer - expect.row_set(_rows(c, "SELECT id FROM sqn WHERE k BETWEEN 100000 AND 101000"), - _rows(c, "SELECT id FROM sqh WHERE k BETWEEN 100000 AND 101000"), - "bigint vs integer literal returns the same rows as heap (#477)") - expect.at_least(_removed(c, "SELECT id FROM sqn WHERE k BETWEEN 100000 AND 101000"), 1, - "and it skips groups, which the cast version already did (#477)") + ("bigint vs integer literal returns the same rows as heap (#477)", "id", "k BETWEEN 100000 AND 101000"), # a domain prunes like its base type (#483), parity first - expect.row_set(_rows(c, "SELECT id FROM sqn WHERE d4 BETWEEN 5000 AND 5100"), - _rows(c, "SELECT id FROM sqh WHERE d4 BETWEEN 5000 AND 5100"), - "domain over int returns the same rows as heap (#483)") - expect.at_least(_removed(c, "SELECT id FROM sqn WHERE d4 BETWEEN 5000 AND 5100"), 1, - "and it skips groups, as the same predicate on int does (#483)") - expect.row_set(_rows(c, "SELECT id FROM sqn WHERE d8 BETWEEN 100000 AND 101000"), - _rows(c, "SELECT id FROM sqh WHERE d8 BETWEEN 100000 AND 101000"), - "domain over bigint returns the same rows as heap (#483)") - expect.at_least(_removed(c, "SELECT id FROM sqn WHERE d8 BETWEEN 100000 AND 101000"), 1, - "and it skips groups too (#483)") - expect.at_least(_removed(c, "SELECT id FROM sqn WHERE d4 > 15000"), 1, - "one-sided domain predicate skips groups (#483)") - expect.num(_one(c, "SELECT count(*) FROM sqn WHERE d4 > 15000"), - _one(c, "SELECT count(*) FROM sqh WHERE d4 > 15000"), "one-sided domain parity") - expect.at_least(_removed(c, "SELECT id FROM sqn WHERE d4 = 12345"), 1, - "domain equality skips groups (#483)") - expect.num(_one(c, "SELECT count(*) FROM sqn WHERE d4 = 12345"), - _one(c, "SELECT count(*) FROM sqh WHERE d4 = 12345"), "domain equality parity") - expect.num(_one(c, "SELECT count(*) FROM sqn WHERE d4 = 20481"), 0, - "domain equality on an absent value returns nothing") - expect.num(_one(c, "SELECT count(*) FROM sqn WHERE d4 = 20480"), 1, - "domain equality on a present boundary value finds it") - expect.at_least(_removed(c, "SELECT id FROM sqn WHERE k > 150000"), 1, - "one-sided bigint vs integer literal skips groups (#477)") - expect.num(_one(c, "SELECT count(*) FROM sqn WHERE k > 150000"), - _one(c, "SELECT count(*) FROM sqh WHERE k > 150000"), "one-sided parity") - expect.at_least(_removed(c, "SELECT id FROM sqn WHERE k = 123450"), 1, - "bigint equality vs integer literal skips groups (#477)") - expect.num(_one(c, "SELECT count(*) FROM sqn WHERE k = 123450"), - _one(c, "SELECT count(*) FROM sqh WHERE k = 123450"), "bigint equality parity") - expect.num(_one(c, "SELECT count(*) FROM sqn WHERE k = 123451"), 0, - "bigint equality on an absent value returns nothing") - expect.num(_removed(c, "SELECT id FROM sqn WHERE id > 0"), 0, - "non-selective scan skips nothing") - expect.num(_one(c, "SELECT count(*) FROM sqn WHERE id > 0"), - _one(c, "SELECT count(*) FROM sqh WHERE id > 0"), "non-selective parity") - expect.num(_one(c, "SELECT count(*) FROM sqn WHERE id > 1000000"), 0, - "out-of-range returns nothing") + ("domain over int returns the same rows as heap (#483)", "id", "d4 BETWEEN 5000 AND 5100"), + ("domain over bigint returns the same rows as heap (#483)", "id", "d8 BETWEEN 100000 AND 101000"), +] + + +@pytest.mark.parametrize("name, cols, where", SCALAR_ROWS) +def test_scalar_rows_agree_with_heap(scalar, expect, name, cols, where): + expect.row_set(_rowsv(_try(_rows, scalar, f"SELECT {cols} FROM sqn WHERE {where}")), + _rowsv(_try(_rows, scalar, f"SELECT {cols} FROM sqh WHERE {where}")), name) + + +# (name, where): the columnar count equals heap's +SCALAR_COUNTS = [ + ("no-predicate scan returns all rows", "true"), + ("equality result parity", "id = 12345"), + ("one-sided domain parity", "d4 > 15000"), + ("domain equality parity", "d4 = 12345"), + ("one-sided parity", "k > 150000"), + ("bigint equality parity", "k = 123450"), + ("non-selective parity", "id > 0"), +] + +@pytest.mark.parametrize("name, where", SCALAR_COUNTS) +def test_scalar_counts_agree_with_heap(scalar, expect, name, where): + expect.num(_num(_try(_one, scalar, f"SELECT count(*) FROM sqn WHERE {where}"), name), + _num(_try(_one, scalar, f"SELECT count(*) FROM sqh WHERE {where}"), name), name) + + +# (name, where, want): the columnar count is a known number +SCALAR_KNOWN = [ + ("row count", "true", 20480), + ("domain equality on an absent value returns nothing", "d4 = 20481", 0), + ("domain equality on a present boundary value finds it", "d4 = 20480", 1), + ("bigint equality on an absent value returns nothing", "k = 123451", 0), + ("out-of-range returns nothing", "id > 1000000", 0), +] + + +@pytest.mark.parametrize("name, where, want", SCALAR_KNOWN) +def test_scalar_counts_are_known(scalar, expect, name, where, want): + expect.num(_num(_try(_one, scalar, f"SELECT count(*) FROM sqn WHERE {where}"), name), want, name) + + +# (name, where): the zone maps remove at least one group +SCALAR_SKIPS = [ + ("range skips groups", "id BETWEEN 5000 AND 5100"), + ("equality skips groups", "id = 12345"), + ("bigint range skips groups", "k BETWEEN 100000::bigint AND 101000::bigint"), + ("and it skips groups, which the cast version already did (#477)", "k BETWEEN 100000 AND 101000"), + ("and it skips groups, as the same predicate on int does (#483)", "d4 BETWEEN 5000 AND 5100"), + ("and it skips groups too (#483)", "d8 BETWEEN 100000 AND 101000"), + ("one-sided domain predicate skips groups (#483)", "d4 > 15000"), + ("domain equality skips groups (#483)", "d4 = 12345"), + ("one-sided bigint vs integer literal skips groups (#477)", "k > 150000"), + ("bigint equality vs integer literal skips groups (#477)", "k = 123450"), +] + + +@pytest.mark.parametrize("name, where", SCALAR_SKIPS) +def test_a_scalar_predicate_skips_groups(scalar, expect, name, where): + expect.at_least(_removed(scalar, f"SELECT id FROM sqn WHERE {where}"), 1, name) + + +def test_a_non_selective_scan_skips_nothing(scalar, expect): + expect.num(_removed(scalar, "SELECT id FROM sqn WHERE id > 0"), 0, "non-selective scan skips nothing") + + +# ---- an anchored LIKE is a range under the C collation (#426) ----------------------- LGEN = "SELECT 'k' || lpad(g::text, 8, '0') FROM generate_series(1, 20480) g" @@ -205,66 +297,84 @@ def _text_pair(c, h, n, collate): c.execute(f"CREATE TABLE {n} (t text{collate}) USING pgcolumnar") c.execute(f"SELECT pgcolumnar.set_options('{n}', stripe_row_limit => 2048, " "chunk_group_row_limit => 1024)") - c.execute(f"INSERT INTO {h} {LGEN}") - c.execute(f"INSERT INTO {n} {LGEN}") - - -def test_an_anchored_like_prunes_under_c_and_only_then(pgc_conn, expect): - """An anchored LIKE is a range under the C collation (#426). Parity beside every - pruning claim, because "did not prune" and "returned the right rows" differ.""" - c = pgc_conn - _text_pair(c, "lkh", "lkn", ' COLLATE "C"') - - expect.at_least(_removed(c, "SELECT t FROM lkn WHERE t >= 'k00005000' AND t < 'k00005001'"), 1, "premise: the equivalent hand-written range prunes (#426)") - expect.row_set(_rows(c, "SELECT t FROM lkn WHERE t LIKE 'k00005000%'"), - _rows(c, "SELECT t FROM lkh WHERE t LIKE 'k00005000%'"), - "anchored LIKE result parity (#426)") - expect.at_least(_removed(c, "SELECT t FROM lkn WHERE t LIKE 'k00005000%'"), 1, - "anchored LIKE prunes groups (#426)") - expect.num(_removed(c, "SELECT t FROM lkn WHERE t LIKE '%5000%'"), 0, - "unanchored LIKE does not prune (#426)") - expect.row_set(_rows(c, "SELECT t FROM lkn WHERE t LIKE '%5000%'"), - _rows(c, "SELECT t FROM lkh WHERE t LIKE '%5000%'"), - "unanchored LIKE result parity (#426)") - expect.row_set(_rows(c, "SELECT t FROM lkn WHERE t LIKE 'k0000\\%50%'"), - _rows(c, "SELECT t FROM lkh WHERE t LIKE 'k0000\\%50%'"), - "escaped-wildcard LIKE result parity (#426)", - allow_empty="no row matches the escaped wildcard; parity is the claim") - expect.row_set(_rows(c, "SELECT t FROM lkn WHERE t LIKE 'k0000500_'"), - _rows(c, "SELECT t FROM lkh WHERE t LIKE 'k0000500_'"), - "trailing _ wildcard result parity (#426)") - expect.at_least(_removed(c, "SELECT t FROM lkn WHERE t LIKE 'k0000500_'"), 1, - "trailing _ wildcard still prunes on its prefix (#426)") - expect.row_set(_rows(c, "SELECT t FROM lkn WHERE t LIKE 'k0000____'"), - _rows(c, "SELECT t FROM lkh WHERE t LIKE 'k0000____'"), - "swallowed _ wildcards would prune away every match (#426)") - expect.row_set(_rows(c, "SELECT t FROM lkn WHERE t LIKE 'k00000001%'"), - _rows(c, "SELECT t FROM lkh WHERE t LIKE 'k00000001%'"), - "low-end prefix parity (#426)") - expect.at_least(_removed(c, "SELECT t FROM lkn WHERE t LIKE 'k00000001%'"), 1, - "low-end prefix prunes, which needs the upper bound (#426)") - - # the default collation of this cluster is not C, so the bound must not be derived - _text_pair(c, "ldh", "ldn", "") - expect.at_least(_removed(c, "SELECT t FROM ldn WHERE t >= 'k00005000' AND t < 'k00005001'"), 1, "premise: the range prunes on the default-collation table too (#426)") - expect.num(_removed(c, "SELECT t FROM ldn WHERE t LIKE 'k00005000%'"), 0, - "default collation that is not C is refused (#426)") - expect.row_set(_rows(c, "SELECT t FROM ldn WHERE t LIKE 'k00005000%'"), - _rows(c, "SELECT t FROM ldh WHERE t LIKE 'k00005000%'"), - "default-collation LIKE result parity (#426)") - - -def test_a_default_collation_in_a_locale_c_database_derives_the_bounds(pgc_cluster, expect): + _try(c.execute, f"INSERT INTO {h} {LGEN}") + _try(c.execute, f"INSERT INTO {n} {LGEN}") + + +@pytest.fixture(scope="module") +def like(pgc_cluster): + """lkh/lkn under COLLATE "C", and ldh/ldn under this cluster's default collation, + which is not C, so the bound must not be derived there.""" + c = _schema_conn(pgc_cluster, "pgc_nsk_like") + try: + _text_pair(c, "lkh", "lkn", ' COLLATE "C"') + _text_pair(c, "ldh", "ldn", "") + yield c + finally: + _drop(c, "pgc_nsk_like") + + +# (name, table, where): the zone maps remove at least one group. Parity sits beside +# every pruning claim, because "did not prune" and "returned the right rows" differ. +LIKE_SKIPS = [ + ("premise: the equivalent hand-written range prunes (#426)", "lkn", "t >= 'k00005000' AND t < 'k00005001'"), + ("anchored LIKE prunes groups (#426)", "lkn", "t LIKE 'k00005000%'"), + ("trailing _ wildcard still prunes on its prefix (#426)", "lkn", "t LIKE 'k0000500_'"), + ("low-end prefix prunes, which needs the upper bound (#426)", "lkn", "t LIKE 'k00000001%'"), + ("premise: the range prunes on the default-collation table too (#426)", "ldn", + "t >= 'k00005000' AND t < 'k00005001'"), +] + + +@pytest.mark.parametrize("name, table, where", LIKE_SKIPS) +def test_a_like_or_range_prunes(like, expect, name, table, where): + expect.at_least(_removed(like, f"SELECT t FROM {table} WHERE {where}"), 1, name) + + +# (name, table, where): the zone maps remove nothing +LIKE_NO_SKIP = [ + ("unanchored LIKE does not prune (#426)", "lkn", "t LIKE '%5000%'"), + ("default collation that is not C is refused (#426)", "ldn", "t LIKE 'k00005000%'"), +] + + +@pytest.mark.parametrize("name, table, where", LIKE_NO_SKIP) +def test_a_like_does_not_prune(like, expect, name, table, where): + expect.num(_removed(like, f"SELECT t FROM {table} WHERE {where}"), 0, name) + + +# (name, columnar table, heap table, where, allow_empty): the rows equal heap's +LIKE_ROWS = [ + ("anchored LIKE result parity (#426)", "lkn", "lkh", "t LIKE 'k00005000%'", None), + ("unanchored LIKE result parity (#426)", "lkn", "lkh", "t LIKE '%5000%'", None), + ("escaped-wildcard LIKE result parity (#426)", "lkn", "lkh", "t LIKE 'k0000\\%50%'", + "no row matches the escaped wildcard; parity is the claim"), + ("trailing _ wildcard result parity (#426)", "lkn", "lkh", "t LIKE 'k0000500_'", None), + ("swallowed _ wildcards would prune away every match (#426)", "lkn", "lkh", "t LIKE 'k0000____'", None), + ("low-end prefix parity (#426)", "lkn", "lkh", "t LIKE 'k00000001%'", None), + ("default-collation LIKE result parity (#426)", "ldn", "ldh", "t LIKE 'k00005000%'", None), +] + + +@pytest.mark.parametrize("name, n, h, where, allow_empty", LIKE_ROWS) +def test_a_like_returns_heaps_rows(like, expect, name, n, h, where, allow_empty): + expect.row_set(_rowsv(_try(_rows, like, f"SELECT t FROM {n} WHERE {where}")), + _rowsv(_try(_rows, like, f"SELECT t FROM {h} WHERE {where}")), name, + allow_empty=allow_empty) + + +@pytest.fixture(scope="module") +def locale_c(pgc_cluster): """A default-collation column in a --locale=C database is byte ordering, so the - bound is sound and must be derived (#426).""" + bound is sound and must be derived (#426). -> the readings, or the reason no such + database could be made.""" db = "pgc_nsk_cdb" with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") try: a.execute(f"CREATE DATABASE {db} LC_COLLATE 'C' LC_CTYPE 'C' TEMPLATE template0") except (psycopg.errors.InvalidParameterValue, psycopg.errors.FeatureNotSupported) as exc: - expect.cannot_run("MISSING_DEPENDENCY", - f"a --locale=C database could not be created: {exc}") + yield {"declined": f"a --locale=C database could not be created: {exc}"} return try: with psycopg.connect(pgc_cluster.dsn(dbname=db), autocommit=True) as c: @@ -272,73 +382,132 @@ def test_a_default_collation_in_a_locale_c_database_derives_the_bounds(pgc_clust c.execute("CREATE TABLE nc (t text) USING pgcolumnar") c.execute("SELECT pgcolumnar.set_options('nc', stripe_row_limit => 2048, " "chunk_group_row_limit => 1024)") - c.execute(f"INSERT INTO nc {LGEN}") - plan = "\n".join(r[0] for r in c.execute( + _try(c.execute, f"INSERT INTO nc {LGEN}") + plan = str(_try(lambda: "\n".join(r[0] for r in c.execute( "EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) " - "SELECT t FROM nc WHERE t LIKE 'k00005000%'").fetchall()) + "SELECT t FROM nc WHERE t LIKE 'k00005000%'").fetchall()))) m = re.search(r"Usable Skip Predicates: (\d+)", plan) - expect.text(m.group(1) if m else f"absent: {plan[:200]}", "2", - "a default-collation column in a --locale=C database derives " - "the bounds (#426)") - expect.num(_one(c, "SELECT count(*) FROM nc WHERE t LIKE 'k00005000%'"), 1, - "and it returns the right rows (#426)") + yield {"usable": m.group(1) if m else f"absent: {plan[:200]}", + "count": _try(_one, c, "SELECT count(*) FROM nc WHERE t LIKE 'k00005000%'")} finally: with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") -def test_an_anchored_like_does_not_prune_under_a_non_c_collation(pgc_conn, expect): +def test_a_default_collation_in_a_locale_c_database_derives_the_bounds(locale_c, expect): + if "declined" in locale_c: + expect.cannot_run("MISSING_DEPENDENCY", locale_c["declined"]) + return + expect.text(locale_c["usable"], "2", + "a default-collation column in a --locale=C database derives the bounds (#426)") + + +def test_and_it_returns_the_right_rows(locale_c, expect): + if "declined" in locale_c: + expect.cannot_run("MISSING_DEPENDENCY", locale_c["declined"]) + return + expect.num(_num(locale_c["count"], "count"), 1, "and it returns the right rows (#426)") + + +@pytest.fixture(scope="module") +def non_c(pgc_cluster): """The refusal, on a collation that is genuinely not byte ordering (#426). The collation is discovered, as the shell does, because which ones work is a property - of the build and the host.""" - c = pgc_conn - found = None - for coll in ("en_US.utf8", "en_US.UTF-8", "unicode", "pg_unicode_fast"): - try: - with c.transaction(): - ok = c.execute(f"SELECT 'x' LIKE 'x' COLLATE \"{coll}\", " - f"('b' < 'a' COLLATE \"{coll}\")").fetchone() - except (psycopg.errors.UndefinedObject, psycopg.errors.FeatureNotSupported): - continue - if ok == (True, False): - found = coll - break - if found is None: - expect.cannot_run("MISSING_DEPENDENCY", - "no usable non-C collation on this build") - return - _text_pair(c, "luh", "lun", f' COLLATE "{found}"') - expect.at_least(_removed(c, "SELECT t FROM lun WHERE t >= 'k00005000' AND t < 'k00005001'"), 1, "premise: a range still prunes under a non-C collation (#426)") - expect.num(_removed(c, "SELECT t FROM lun WHERE t LIKE 'k00005000%'"), 0, - "anchored LIKE does not prune under a non-C collation (#426)") - expect.row_set(_rows(c, "SELECT t FROM lun WHERE t LIKE 'k00005000%'"), - _rows(c, "SELECT t FROM luh WHERE t LIKE 'k00005000%'"), - "non-C collation LIKE result parity (#426)") + of the build and the host. -> (conn, None) or (None, the reason).""" + c = _schema_conn(pgc_cluster, "pgc_nsk_nonc") + try: + found = None + for coll in ("en_US.utf8", "en_US.UTF-8", "unicode", "pg_unicode_fast"): + try: + with c.transaction(): + ok = c.execute(f"SELECT 'x' LIKE 'x' COLLATE \"{coll}\", " + f"('b' < 'a' COLLATE \"{coll}\")").fetchone() + except (psycopg.errors.UndefinedObject, psycopg.errors.FeatureNotSupported): + continue + if ok == (True, False): + found = coll + break + if found is None: + yield None, "no usable non-C collation on this build" + else: + _text_pair(c, "luh", "lun", f' COLLATE "{found}"') + yield c, None + finally: + _drop(c, "pgc_nsk_nonc") + + +def _non_c(non_c, expect): + c, reason = non_c + if c is None: + expect.cannot_run("MISSING_DEPENDENCY", reason) + return c + + +def test_a_range_still_prunes_under_a_non_c_collation(non_c, expect): + c = _non_c(non_c, expect) + if c is not None: + expect.at_least(_removed(c, "SELECT t FROM lun WHERE t >= 'k00005000' AND t < 'k00005001'"), 1, + "premise: a range still prunes under a non-C collation (#426)") + + +def test_an_anchored_like_does_not_prune_under_a_non_c_collation(non_c, expect): + c = _non_c(non_c, expect) + if c is not None: + expect.num(_removed(c, "SELECT t FROM lun WHERE t LIKE 'k00005000%'"), 0, + "anchored LIKE does not prune under a non-C collation (#426)") -def test_a_columnar_partition_takes_the_custom_scan_and_prunes(pgc_conn, expect): +def test_a_non_c_collation_like_returns_heaps_rows(non_c, expect): + c = _non_c(non_c, expect) + if c is not None: + expect.row_set(_rowsv(_try(_rows, c, "SELECT t FROM lun WHERE t LIKE 'k00005000%'")), + _rowsv(_try(_rows, c, "SELECT t FROM luh WHERE t LIKE 'k00005000%'")), + "non-C collation LIKE result parity (#426)") + + +# ---- a columnar partition takes the custom scan (#436) ------------------------------ + +@pytest.fixture(scope="module") +def part(pgc_cluster): """A partition is RELOPT_OTHER_MEMBER_REL, and set_rel_pathlist_hook used to return early for it, so a partitioned columnar table lost the custom scan (#436).""" - c = pgc_conn - c.execute("CREATE TABLE ptp_h (id int, k int, label text)") - c.execute("CREATE TABLE ptp (id int, k int, label text) PARTITION BY RANGE (id)") - c.execute("CREATE TABLE ptp_1 PARTITION OF ptp FOR VALUES FROM (1) TO (10241) USING pgcolumnar") - c.execute("CREATE TABLE ptp_2 PARTITION OF ptp FOR VALUES FROM (10241) TO (20481) USING pgcolumnar") - for p in ("ptp_1", "ptp_2"): - c.execute(f"SELECT pgcolumnar.set_options('{p}', stripe_row_limit => 2048, " - "chunk_group_row_limit => 1024)") - gen = "SELECT g, g * 10, 'lbl-' || (g % 100) FROM generate_series(1, 20480) g" - c.execute(f"INSERT INTO ptp_h {gen}") - c.execute(f"INSERT INTO ptp {gen}") - expect.num(_one(c, "SELECT count(*) FROM pg_class c JOIN pg_am a ON a.oid = c.relam " - "WHERE c.relname IN ('ptp_1','ptp_2') AND a.amname = 'pgcolumnar'"), 2, + c = _schema_conn(pgc_cluster, "pgc_nsk_part") + try: + c.execute("CREATE TABLE ptp_h (id int, k int, label text)") + c.execute("CREATE TABLE ptp (id int, k int, label text) PARTITION BY RANGE (id)") + c.execute("CREATE TABLE ptp_1 PARTITION OF ptp FOR VALUES FROM (1) TO (10241) USING pgcolumnar") + c.execute("CREATE TABLE ptp_2 PARTITION OF ptp FOR VALUES FROM (10241) TO (20481) USING pgcolumnar") + for p in ("ptp_1", "ptp_2"): + c.execute(f"SELECT pgcolumnar.set_options('{p}', stripe_row_limit => 2048, " + "chunk_group_row_limit => 1024)") + gen = "SELECT g, g * 10, 'lbl-' || (g % 100) FROM generate_series(1, 20480) g" + _try(c.execute, f"INSERT INTO ptp_h {gen}") + _try(c.execute, f"INSERT INTO ptp {gen}") + yield c + finally: + _drop(c, "pgc_nsk_part") + + +def test_both_partitions_are_columnar(part, expect): + expect.num(_num(_try(_one, part, "SELECT count(*) FROM pg_class c JOIN pg_am a ON a.oid = c.relam " + "WHERE c.relname IN ('ptp_1','ptp_2') AND a.amname = 'pgcolumnar' " + "AND c.relnamespace = current_schema()::regnamespace"), "partitions"), 2, "premise: both partitions are columnar (#436)") - expect.row_set(_rows(c, "SELECT id, k, label FROM ptp WHERE id BETWEEN 5000 AND 5100"), - _rows(c, "SELECT id, k, label FROM ptp_h WHERE id BETWEEN 5000 AND 5100"), + + +def test_a_partitioned_columnar_table_returns_heaps_rows(part, expect): + expect.row_set(_rowsv(_try(_rows, part, "SELECT id, k, label FROM ptp WHERE id BETWEEN 5000 AND 5100")), + _rowsv(_try(_rows, part, "SELECT id, k, label FROM ptp_h WHERE id BETWEEN 5000 AND 5100")), "a partitioned columnar table returns the same rows as heap (#436)") - plan = "\n".join(r[0] for r in c.execute( - "EXPLAIN (COSTS OFF) SELECT id FROM ptp WHERE id BETWEEN 5000 AND 5100").fetchall()) + + +def test_a_partition_takes_the_columnar_custom_scan(part, expect): + plan = str(_try(lambda: "\n".join(r[0] for r in part.execute( + "EXPLAIN (COSTS OFF) SELECT id FROM ptp WHERE id BETWEEN 5000 AND 5100").fetchall()))) expect.num(plan.count("Custom Scan (PgColumnarScan)"), 1, "a partition takes the columnar custom scan (#436)") - expect.at_least(_removed(c, "SELECT id FROM ptp WHERE id BETWEEN 5000 AND 5100"), 1, - "and it prunes row groups (#436)") + + +def test_a_partition_prunes_row_groups(part, expect): + expect.at_least(_removed(part, "SELECT id FROM ptp WHERE id BETWEEN 5000 AND 5100"), 1, + "and it prunes row groups (#436)")