From 9705252d8148dc4e7d5021021f5210063c87ae40 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Fri, 16 Oct 2026 16:47:32 +0000 Subject: [PATCH 1/7] test: native_fetch_position gets its pytest twin, and its fetches go through the index it was written for (#1341) The suite forced an index scan with enable_seqscan=off, which leaves the columnar custom scan on. Measured: the single-row fetch, the near and far UPDATEs and the mirror join all planned as Custom Scan (PgColumnarScan), so none of them fetched by row number. The rank off by one left the single-row arm green, and the rank walking every earlier row left both timing arms green. FORCE now also sets pgcolumnar.enable_custom_scan = off, and four premises assert that each measured query's plan is an Index Scan using its index. The timing arms then still missed the walk, because they timed an UPDATE. A columnar UPDATE writes each row again into a new row group, so only the first of three passes fetched from the far end, and the median was always a later pass. They now time a read of the same range, sum(fixed) and sum(length(txt)), so every pass fetches the same positions. Measured at head on 15-19: near and far within 1.03x, small and big group within 1.1x. Measured with the rank walking: 50 ms against 779 ms, and 390 against 779. test_native_fetch_position.py carries all 12 checks by the shell's name in four tests. Under each of four mutations (rank counting rows, rank off by one, rank walking, offset walking), the twin reddens exactly the shell's set of checks. Like the shell, it reads PGC_FETCHPOS_ROWS. COMPLETE gains native_fetch_position. TESTS.md section 167, cluster_tests 722 -> 726. Co-Authored-By: Claude Opus 5.5 --- test/native_fetch_position.sh | 44 +++++- test/pytest/TESTS.md | 14 ++ test/pytest/expected_tests.txt | 4 +- test/pytest/test_compare_to_bash.py | 1 + test/pytest/test_native_fetch_position.py | 166 ++++++++++++++++++++++ 5 files changed, 225 insertions(+), 4 deletions(-) create mode 100644 test/pytest/test_native_fetch_position.py diff --git a/test/native_fetch_position.sh b/test/native_fetch_position.sh index 9e860d8e4..18adc4f98 100755 --- a/test/native_fetch_position.sh +++ b/test/native_fetch_position.sh @@ -65,8 +65,32 @@ check "the rows are in one row group" \ # Fetch by row number is what an index scan drives, so force one. Each leading # SET prints its own line ahead of the result, so these queries take the last. +# +# enable_seqscan=off does NOT turn the columnar custom scan off, and it was all +# this suite set: measured, the single-row fetch, the near and far UPDATEs and +# the mirror join all planned as Custom Scan (PgColumnarScan), so none of them +# fetched by row number. A rank off by one left the single-row arm green, and a +# walk over every earlier row left both timing arms green. The custom scan is +# off now, and each measurement asserts first that its plan reads the index. FORCE="SET enable_seqscan = off; SET enable_bitmapscan = off; - SET max_parallel_workers_per_gather = 0;" + SET max_parallel_workers_per_gather = 0; + SET pgcolumnar.enable_custom_scan = off;" + +# reads_index QUERY INDEX -> yes, or the plan's first line: the premise that a +# query fetches by row number through INDEX rather than scanning +reads_index() { + local plan + plan="$(q "$FORCE EXPLAIN (COSTS OFF) $1")" + case "$plan" in + *"Index Scan using $2 "*) echo yes ;; + *) echo "no: $(grep -m1 -E 'Scan|Update' <<<"$plan" | sed 's/^ *//')" ;; + esac +} + +check "premise: the mirror comparison fetches through the index" \ + "$(reads_index "SELECT count(*) FROM fp_c c JOIN fp_h h USING (id) WHERE c.fixed IS DISTINCT FROM h.fixed" fp_c_id)" "yes" +check "premise: a single-row fetch goes through the index" \ + "$(reads_index "SELECT fixed FROM fp_c WHERE id = 1001" fp_c_id)" "yes" # rows spread across the whole group, including the far end where the old walk # was longest @@ -103,7 +127,7 @@ FAR_HI=$ROWS upd_once() { # id-low, id-high -> milliseconds for one pass local start end start=$(date +%s%N) - psql_run "$FORCE UPDATE fp_c SET fixed = fixed + 1 + psql_run "$FORCE SELECT sum(fixed), sum(length(txt)) FROM fp_c WHERE id BETWEEN $1 AND $2;" >/dev/null 2>&1 end=$(date +%s%N) echo $(( (end - start) / 1000000 )) @@ -130,6 +154,17 @@ upd_ms() { # id-low, id-high -> median of three passes printf '%s\n%s\n%s\n' "$a" "$b" "$c" | sort -n | sed -n 2p } +# The timed pass is a SELECT, not the UPDATE it was. A columnar UPDATE writes +# each row again into a new row group, so only the first of three passes +# fetched from the far end of the big group; the second and third fetched from +# the start of the new one, and the median was always one of those. Measured +# with the rank walking every earlier row: the UPDATE form read near 122 ms, +# far 132 ms, green. A read leaves every row where it was, so all three passes +# fetch the same positions. sum(length(txt)) makes it decode a varying-length +# column, which the offset table serves. +check "premise: the timed reads fetch through the index" \ + "$(reads_index "SELECT sum(fixed), sum(length(txt)) FROM fp_c WHERE id BETWEEN $FAR_LO AND $FAR_HI" fp_c_id)" "yes" + near="$(upd_ms $NEAR_LO $NEAR_HI)" far="$(upd_ms $FAR_LO $FAR_HI)" echo "-- $N fetches, median of three: first tenth ${near} ms, last tenth ${far} ms" @@ -157,7 +192,7 @@ build() { # table, rows fetch_once() { # table, table-rows, count -> milliseconds for one pass local start end start=$(date +%s%N) - psql_run "$FORCE UPDATE $1 SET v = v + 1 WHERE id > $(( $2 - $3 ));" \ + psql_run "$FORCE SELECT sum(v), sum(length(t)) FROM $1 WHERE id > $(( $2 - $3 ));" \ >/dev/null 2>&1 end=$(date +%s%N) echo $(( (end - start) / 1000000 )) @@ -205,6 +240,9 @@ build fp_small $((ROWS / 2)) build fp_big "$ROWS" K=$((ROWS / 20)) +check "premise: the doubled group's reads fetch through its index" \ + "$(reads_index "SELECT sum(v), sum(length(t)) FROM fp_big WHERE id > $(( ROWS - K ))" fp_big_id)" "yes" + small="$(fetch_ms fp_small $((ROWS / 2)) $K)" big="$(fetch_ms fp_big "$ROWS" $K)" echo "-- $K fetches at the far end, median of three: group of $((ROWS / 2)) took ${small} ms, group of $ROWS took ${big} ms" diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index babd7df85..06ac2d401 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -212,6 +212,7 @@ behaviour, the source of that number is named. - [164. test_native_saop_pushdown.py: IN-lists and = ANY(array) prune row groups](#164-test_native_saop_pushdownpy-in-lists-and--anyarray-prune-row-groups) - [165. test_native_groupagg_batch.py: the grouped aggregate folds column-at-a-time](#165-test_native_groupagg_batchpy-the-grouped-aggregate-folds-column-at-a-time) - [166. test_native_fetch_projection.py: a fetch by row number decodes only what its caller needs](#166-test_native_fetch_projectionpy-a-fetch-by-row-number-decodes-only-what-its-caller-needs) +- [167. test_native_fetch_position.py: a fetch by row number costs the same anywhere in its group](#167-test_native_fetch_positionpy-a-fetch-by-row-number-costs-the-same-anywhere-in-its-group) ## 1. How to read a test in here @@ -7691,3 +7692,16 @@ The pytest twin of `test/native_fetch_projection.sh` (ported for #1341), every c | `test_a_projection_covering_every_column_decodes_nothing_and_is_right` | with the computed base-column set empty, every row is reconstructed with the right values | | `test_no_caller_decodes_every_column_and_index_deletion_reads_no_row` | the projection path calls no full decode, and index deletion asks no row whether it is live | | `test_every_needed_set_test_consults_the_all_columns_flag` | every needed-set membership test in the reader consults the explicit all-columns flag | + +## 167. test_native_fetch_position.py: a fetch by row number costs the same anywhere in its group + +The pytest twin of `test/native_fetch_position.sh` (ported for #1341), every check by the shell's name. A cached row group still walked to the row: it counted the validity bits below the row, then decoded and discarded that many values (#143). A rank prefix over the validity bitmap and an offset table for varying-length columns make each fetch O(1). Every fetch here runs with the columnar custom scan off, and each measurement first asserts that its plan reads the index. With only `enable_seqscan=off`, every query scanned instead and none fetched by row number. The timing ratios read the same ranges three times. An UPDATE would write its rows into a new group, so later passes would fetch elsewhere. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_every_fetched_value_matches_the_heap_mirror` | the group is one row group, and fixed, varying-length and heavily-null columns fetched through the index match a heap mirror, at every probed position | +| `test_the_cost_does_not_depend_on_the_position` | fetching the last twentieth of the group costs under three times the first | +| `test_a_bigger_group_does_not_cost_more_per_fetch` | the same far-end fetches in a group twice the size cost under 1.8 times as much | +| `test_the_indexes_that_make_it_constant_are_in_the_fetch` | `pgcolumnar_fetch_row` takes the rank from the prefix and a varying-length value from the offset table | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 1df468344..655419001 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -795,4 +795,6 @@ guard_tests 438 # eight tests, by collection over the cluster files on this tree. # cluster_tests 718 -> 722 for #1341's test_native_fetch_projection.py, the twin of native_fetch_projection.sh: # four tests, by collection over the cluster files on this tree. -cluster_tests 722 +# cluster_tests 722 -> 726 for #1341's test_native_fetch_position.py, the twin of native_fetch_position.sh: +# four tests, by collection over the cluster files on this tree. +cluster_tests 726 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 4c323a6c0..3db54a5c3 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -130,6 +130,7 @@ "native_dml", "native_encdesc_golden", "native_fetch_coalesce", + "native_fetch_position", "native_fetch_projection", "native_groupagg", "native_groupagg_batch", diff --git a/test/pytest/test_native_fetch_position.py b/test/pytest/test_native_fetch_position.py new file mode 100644 index 000000000..f3b2bece3 --- /dev/null +++ b/test/pytest/test_native_fetch_position.py @@ -0,0 +1,166 @@ +"""A fetch by row number costs the same wherever the row sits in its group (#143, #1341). +The pytest twin of `test/native_fetch_position.sh`, every check by the shell's name. + +A cached group still walked to the row: counting the validity bits below it, then +decoding and discarding that many values, so a fetch cost O(row) and fetching a whole +group was quadratic. A rank prefix over the validity bitmap and an offset table for +varying-length columns make it O(1). Values first, against a heap mirror over fixed, +varying-length and heavily-null columns, through an index scan; then two timing ratios +(far end against near end, and a doubled group), each the median of three passes; then +the two indexes in pgcolumnar_fetch_row's source. +""" +import os +import pathlib +import re +import statistics +import time + +import psycopg +import pytest + +# the shell reads PGC_FETCHPOS_ROWS, and so does this +ROWS = int(os.environ.get("PGC_FETCHPOS_ROWS", "120000")) +SRC = pathlib.Path(__file__).resolve().parents[2] / "src" +# enable_seqscan=off leaves the columnar custom scan on, and with only that every query +# here scanned rather than fetched by row number; each measurement asserts its plan +FORCE = ("SET enable_seqscan = off", "SET enable_bitmapscan = off", "SET max_parallel_workers_per_gather = 0", + "SET pgcolumnar.enable_custom_scan = off") + + +def _one(c, sql): + return c.execute(sql).fetchone()[0] + + +def _forced(c): + for s in FORCE: + c.execute(s) + + +def _reads_index(c, sql, index): + """-> 'yes' when `sql`'s plan is an Index Scan using `index`, else its first scan line.""" + plan = "\n".join(r[0] for r in c.execute(f"EXPLAIN (COSTS OFF) {sql}")) + if f"Index Scan using {index} " in plan: + return "yes" + first = next((l.strip() for l in plan.splitlines() if "Scan" in l or "Update" in l), plan) + return f"no: {first}" + + +def _rows_or_error(c, sql): + """-> the first value, or the error as a value: a corrupt fetch is a named red.""" + try: + return _one(c, sql) + except psycopg.Error as exc: + return f"error: {exc.diag.message_primary}" + + +def _median_ms(c, sql): + """-> the median of three timed passes of `sql`, in milliseconds.""" + times = [] + for _ in range(3): + t0 = time.monotonic() + c.execute(sql) + times.append((time.monotonic() - t0) * 1000.0) + return statistics.median(times) + + +@pytest.fixture(scope="module") +def fpos(pgc_cluster): + """-> a connection to a database holding fp_c (one row group) and its heap mirror.""" + db = "pgc_fetchpos" + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + a.execute(f"CREATE DATABASE {db}") + try: + with psycopg.connect(pgc_cluster.dsn(dbname=db), autocommit=True) as c: + c.execute("CREATE EXTENSION pgcolumnar") + # every fourth row null in the nullable columns, so rank and row number diverge + c.execute(f"SET pgcolumnar.stripe_row_limit = {ROWS}") + c.execute(f"SET pgcolumnar.chunk_group_row_limit = {ROWS}") + c.execute("CREATE TABLE fp_c (id int, fixed bigint, txt text, sparse int, stxt text) USING pgcolumnar") + c.execute(f"""INSERT INTO fp_c + SELECT g, g * 7, repeat('x', 1 + (g % 40)) || g, + CASE WHEN g % 4 = 0 THEN NULL ELSE g END, + CASE WHEN g % 4 = 0 THEN NULL ELSE 's' || g END + FROM generate_series(1, {ROWS}) g""") + c.execute("CREATE INDEX fp_c_id ON fp_c (id)") + c.execute("CREATE TABLE fp_h (LIKE fp_c)") + c.execute("INSERT INTO fp_h SELECT * FROM fp_c") + yield c + finally: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + + +def test_every_fetched_value_matches_the_heap_mirror(fpos, expect): + c = fpos + expect.num(_one(c, "SELECT count(*) FROM pgcolumnar.row_group r " + "JOIN pgcolumnar.storage s ON s.storage_id = r.storage_id " + "WHERE s.relation_oid = 'fp_c'::regclass"), 1, "the rows are in one row group") + _forced(c) + expect.text(_reads_index(c, "SELECT count(*) FROM fp_c c JOIN fp_h h USING (id) " + "WHERE c.fixed IS DISTINCT FROM h.fixed", "fp_c_id"), "yes", + "premise: the mirror comparison fetches through the index") + expect.text(_reads_index(c, "SELECT fixed FROM fp_c WHERE id = 1001", "fp_c_id"), "yes", + "premise: a single-row fetch goes through the index") + expect.text(str(_rows_or_error(c, """SELECT count(*) FROM fp_c c JOIN fp_h h USING (id) + WHERE c.fixed IS DISTINCT FROM h.fixed + OR c.txt IS DISTINCT FROM h.txt + OR c.sparse IS DISTINCT FROM h.sparse + OR c.stxt IS DISTINCT FROM h.stxt""")), "0", "every column matches the heap mirror") + # one row at a time, where an off-by-one in the rank would land on a neighbour + row = "fixed || '|' || txt || '|' || coalesce(sparse::text, 'N') || '|' || coalesce(stxt, 'N')" + bad = [] + for pos in (1, 2, 3, 4, 5, 999, 1000, 1001, ROWS // 2, ROWS - 1, ROWS): + got = _rows_or_error(c, f"SELECT {row} FROM fp_c WHERE id = {pos}") + want = _one(c, f"SELECT {row} FROM fp_h WHERE id = {pos}") + if got != want: + bad.append(f"id={pos}(got={got} want={want})") + expect.text(" ".join(bad) or "same", "same", "single-row fetches are right at every position") + + +def test_the_cost_does_not_depend_on_the_position(fpos, expect): + c = fpos + _forced(c) + n = ROWS // 20 + # a read, not an UPDATE: an UPDATE writes its rows into a new group, so only the first + # pass fetched from the far end and the median was always a later one + read = "SELECT sum(fixed), sum(length(txt)) FROM fp_c WHERE id BETWEEN {} AND {}" + expect.text(_reads_index(c, read.format(ROWS - n + 1, ROWS), "fp_c_id"), "yes", + "premise: the timed reads fetch through the index") + near = _median_ms(c, read.format(1, n)) + far = _median_ms(c, read.format(ROWS - n + 1, ROWS)) + print(f"-- {n} fetches, median of three: first tenth {near:.0f} ms, last tenth {far:.0f} ms") + # a ratio of 3 catches the quadratic walk (1,001,374 ms against 614 ms) with orders to spare + expect.text("yes" if near > 0 and far / near < 3 else f"no (near={near:.0f}ms far={far:.0f}ms)", "yes", + "fetching from the far end of a group costs about what the near end does") + + +def test_a_bigger_group_does_not_cost_more_per_fetch(fpos, expect): + c = fpos + for name, rows in (("fp_small", ROWS // 2), ("fp_big", ROWS)): + c.execute(f"SET pgcolumnar.stripe_row_limit = {rows}") + c.execute(f"SET pgcolumnar.chunk_group_row_limit = {rows}") + c.execute(f"CREATE TABLE {name} (id int, v bigint, t text) USING pgcolumnar") + c.execute(f"INSERT INTO {name} SELECT g, g * 3, 't' || g FROM generate_series(1, {rows}) g") + c.execute(f"CREATE INDEX {name}_id ON {name} (id)") + _forced(c) + # the rows fetched sit at the END of each group, where the old walk was longest + k = ROWS // 20 + read = "SELECT sum(v), sum(length(t)) FROM {} WHERE id > {}" + expect.text(_reads_index(c, read.format("fp_big", ROWS - k), "fp_big_id"), "yes", + "premise: the doubled group's reads fetch through its index") + small = _median_ms(c, read.format("fp_small", ROWS // 2 - k)) + big = _median_ms(c, read.format("fp_big", ROWS - k)) + print(f"-- {k} fetches at the far end, median of three: {ROWS // 2} rows took {small:.0f} ms, {ROWS} took {big:.0f} ms") + expect.text("yes" if small > 0 and big / small < 1.8 else f"no (small={small:.0f}ms big={big:.0f}ms)", "yes", + "doubling the row group does not double the cost of the same fetches") + + +def test_the_indexes_that_make_it_constant_are_in_the_fetch(expect): + m = re.search(r"^pgcolumnar_fetch_row\(.*?^}", (SRC / "columnar_reader.c").read_text(), re.S | re.M) + fetch = m.group(0) if m else "" + expect.text("yes" if fetch else "no", "yes", "premise: pgcolumnar_fetch_row was extracted, not an empty range") + expect.text("yes" if "present = pgcolumnar_rank_before" in fetch else "no", "yes", + "the rank comes from a prefix rather than a loop over earlier rows") + expect.text("yes" if "entry->valOffset[c][present]" in fetch else "no", "yes", + "a varying-length column reaches its value through an offset table") From 0adaf345a5e34ec08000d77de10c7755ee1be2d7 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Fri, 16 Oct 2026 17:07:07 +0000 Subject: [PATCH 2/7] test: native_groupagg_batch dates three heap-mirror arms by a NULL-key mutation from #1485's review (#1341) jdatcmd's mutation drops the NULL-key flag in the grouped fold's gather, so the NULL group folds into k = 0 on the batch path only. Nine checks redden, and the twin reddens the same nine by name. Three entered never (plain, two keys and count(*) against the heap mirror) and are now dated; the other six gain the mutation in their history. Census 4189 rows / 2969 never / 1220 dated. Co-Authored-By: Claude Opus 5.5 --- test/check_ledger.tsv | 18 +++++++++--------- test/check_ledger_budget.txt | 14 +++++++++++++- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 738978f03..ae2ccd49c 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2595,17 +2595,17 @@ native_groupagg_batch native_groupagg_batch <> filter: the fold declined 15;16;1 native_groupagg_batch native_groupagg_batch add column: ANALYZE reports the row-path run 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped node's ANALYZE prints the predicted fold, not what ran (the #602 shape) native_groupagg_batch native_groupagg_batch add column: plain EXPLAIN reports the prediction 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch add column: the answer is still right 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch count(*): answers match the heap mirror 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch count(*): answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch count(*): the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch count(*): the grouped node is planned 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch date key: answers match the heap mirror 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch date key: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch date key: the grouped node is planned 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch deletes: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch deletes: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask;columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch deletes: the fold still ran 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch expression key: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: an expression key's -1 attidx is not refused by the grouped fold's eligibility native_groupagg_batch native_groupagg_batch expression key: the fold declined 15;16;17;18;19 2026-10-16 columnar_vector.c: an expression key's -1 attidx is not refused by the grouped fold's eligibility -native_groupagg_batch native_groupagg_batch filtered: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold applies no scan key per row +native_groupagg_batch native_groupagg_batch filtered: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold applies no scan key per row;columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch filtered: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch filtered: the grouped node is planned 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch float8 key: answers match the heap mirror 15;16;17;18;19 never - @@ -2619,13 +2619,13 @@ native_groupagg_batch native_groupagg_batch float8 zero: the grouped node is pla native_groupagg_batch native_groupagg_batch float8 zero: the grouped sums match the heap 15;16;17;18;19 2026-10-16 columnar_vector.c: float8 admitted as a simple key, so its group keys are hashed and compared by bits native_groupagg_batch native_groupagg_batch min/max: answers match the heap mirror 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch min/max: the fold declined 15;16;17;18;19 2026-10-16 columnar_vector.c: min and max admitted as batch fold kinds -native_groupagg_batch native_groupagg_batch parallel: and they equal the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: a partial grouped node folds without the shared row-group counter;columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch parallel: and they equal the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: a partial grouped node folds without the shared row-group counter;columnar_vector.c: the grouped fold ignores the delete mask;columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch parallel: our parallel-aware grouped node is under the Gather 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch parallel: premise: the value arm's own plan launches workers 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch parallel: the answers equal the serial fold, byte for byte 15;16;17;18;19 2026-10-16 columnar_vector.c: a partial grouped node folds without the shared row-group counter native_groupagg_batch native_groupagg_batch parallel: the fold ran in the partial node (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch parallel: workers actually launched for that node 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch plain: answers match the heap mirror 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch plain: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch plain: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch plain: the grouped node is planned 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch premise: fixture loaded every row (columnar) 15;16;17;18;19 never - @@ -2646,16 +2646,16 @@ native_groupagg_batch native_groupagg_batch text key: the fold declined 15;16;17 native_groupagg_batch native_groupagg_batch timestamp key: answers match the heap mirror 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch timestamp key: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch timestamp key: the grouped node is planned 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch toggle-differential: filtered 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold applies no scan key per row;columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch toggle-differential: filtered 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold applies no scan key per row;columnar_vector.c: the grouped fold ignores the delete mask;columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch toggle-differential: filtered [premise: node fires with the path on] 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch toggle-differential: filtered [premise: the off arm is not the grouped node] 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch toggle-differential: plain 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch toggle-differential: plain 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask;columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch toggle-differential: plain [premise: node fires with the path on] 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch toggle-differential: plain [premise: the off arm is not the grouped node] 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch toggle-differential: two keys 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch toggle-differential: two keys 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask;columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch toggle-differential: two keys [premise: node fires with the path on] 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch toggle-differential: two keys [premise: the off arm is not the grouped node] 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch two keys: answers match the heap mirror 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch two keys: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch two keys: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch two keys: the grouped node is planned 15;16;17;18;19 never - native_index_projection native_index_projection a parallel build projects one column of twenty, not all twenty 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index bc4355952..f4b32e393 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -3343,4 +3343,16 @@ suites_not_covered 199 # never 2988 - 16 = 2972 # dated 1201 + 16 = 1217 # 2972 + 1217 == 4189 -checks_never_observed_red 2972 +# +# #1341, native_groupagg_batch's NULL-key mutation (OffgridwithJD, from jdatcmd's +# #1485 review): 2972 -> 2969. The grouped fold's gather dropping the NULL-key +# flag folds the NULL group into k = 0 on the batch path only. Nine checks redden, +# shell and twin alike; three of them (plain, two keys and count(*) against the +# heap mirror) entered never and are now dated, and the other six gain the +# mutation in their history. No row is added or removed. +# +# ledger rows 4189 + 0 = 4189 +# never 2972 - 3 = 2969 +# dated 1217 + 3 = 1220 +# 2969 + 1220 == 4189 +checks_never_observed_red 2969 From fde6ab6076d09476762b09d1c48325d4dd314479 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Fri, 16 Oct 2026 17:07:07 +0000 Subject: [PATCH 3/7] test: native_fetch_position seeds the ledger, 6 of 12 arms dated by four rank and offset mutations (#1341) The suite had no rows. suites_not_covered 199 -> 198; census 4201 rows / 2975 never / 1226 dated. Co-Authored-By: Claude Opus 5.5 --- test/check_ledger.tsv | 12 ++++++++++++ test/check_ledger_budget.txt | 18 ++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index ae2ccd49c..264a6fd5a 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2477,6 +2477,18 @@ native_fetch_coalesce native_fetch_coalesce premise: fetching one projected colu native_fetch_coalesce native_fetch_coalesce premise: the coalescing helper holds both the bound and the validity copy 15;16;17;18;19 never - native_fetch_coalesce native_fetch_coalesce the validity copy is bounded by the chunk length before it runs 15;16;17;18;19 never - native_fetch_coalesce native_fetch_coalesce the wide fetch returns the projected values 15;16;17;18;19 2026-10-14 every column of a coalesced fetch is copied from the start of the span (off = 0) +native_fetch_position native_fetch_position a varying-length column reaches its value through an offset table 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row reaches a varying-length value by decoding every earlier one (the #143 walk) +native_fetch_position native_fetch_position doubling the row group does not double the cost of the same fetches 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row counts the validity bits from the group's start for every fetch (the #143 walk);columnar_reader.c: pgcolumnar_fetch_row reaches a varying-length value by decoding every earlier one (the #143 walk) +native_fetch_position native_fetch_position every column matches the heap mirror 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row takes the row number as the rank, counting rows rather than present values;columnar_reader.c: pgcolumnar_fetch_row's rank is one past the row's +native_fetch_position native_fetch_position fetching from the far end of a group costs about what the near end does 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row counts the validity bits from the group's start for every fetch (the #143 walk);columnar_reader.c: pgcolumnar_fetch_row reaches a varying-length value by decoding every earlier one (the #143 walk) +native_fetch_position native_fetch_position premise: a single-row fetch goes through the index 15;16;17;18;19 never - +native_fetch_position native_fetch_position premise: pgcolumnar_fetch_row was extracted, not an empty range 15;16;17;18;19 never - +native_fetch_position native_fetch_position premise: the doubled group's reads fetch through its index 15;16;17;18;19 never - +native_fetch_position native_fetch_position premise: the mirror comparison fetches through the index 15;16;17;18;19 never - +native_fetch_position native_fetch_position premise: the timed reads fetch through the index 15;16;17;18;19 never - +native_fetch_position native_fetch_position single-row fetches are right at every position 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row takes the row number as the rank, counting rows rather than present values;columnar_reader.c: pgcolumnar_fetch_row's rank is one past the row's +native_fetch_position native_fetch_position the rank comes from a prefix rather than a loop over earlier rows 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row counts the validity bits from the group's start for every fetch (the #143 walk);columnar_reader.c: pgcolumnar_fetch_row takes the row number as the rank, counting rows rather than present values +native_fetch_position native_fetch_position the rows are in one row group 15;16;17;18;19 never - native_fetch_projection native_fetch_projection a deleted row stops being read through the projection 15;16;17;18;19 2026-10-16 columnar_projection.c: read_projection ignores the base row's liveness native_fetch_projection native_fetch_projection an uncovered column still reads correctly after a delete 15;16;17;18;19 2026-10-16 columnar_projection.c: reconstruct's uncovered set misses the table's last column native_fetch_projection native_fetch_projection and does not decode the row to find out 15;16;17;18;19 2026-10-16 columnar_tableam.c: index deletion decodes each bottom-up row to decide diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index f4b32e393..9a0a36773 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 199 +suites_not_covered 198 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -3355,4 +3355,18 @@ suites_not_covered 199 # never 2972 - 3 = 2969 # dated 1217 + 3 = 1220 # 2969 + 1220 == 4189 -checks_never_observed_red 2969 +# +# #1341, native_fetch_position seeded, with its twin (OffgridwithJD): 2969 -> 2975, +# and suites_not_covered 199 -> 198. The suite had no rows. It enters with all 12 +# checks from five-major runs (12/12 on each of 15-19). 6 are dated by four +# mutations of pgcolumnar_fetch_row: the rank counting rows, the rank one too far, +# the rank walking the validity bits, and a varying-length value reached by +# decoding every earlier one. The two timing arms are dated by the walks only now +# that the fetches go through the index and the timed pass is a read. Six arrive +# never: the one-row-group fixture fact and five premises. +# +# ledger rows 4189 + 12 = 4201 +# never 2969 + 6 = 2975 +# dated 1220 + 6 = 1226 +# 2975 + 1226 == 4201 +checks_never_observed_red 2975 From 989fd32b1b0288487a70f72d103d94b629c2af76 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Fri, 16 Oct 2026 17:00:29 +0000 Subject: [PATCH 4/7] test: native_fetch_cache gets its pytest twin, and section 1's comment says why its UPDATE fetches by row number (#1341) test_native_fetch_cache.py carries all 26 checks by the shell's name in four tests, rebuilding each fixture before every reading as the shell does (#801). Under each of nine mutations, the twin reddens exactly the shell's set of checks: the cache never hitting, the key without the storage id, no earlier-command reject, each of three recheck fields dropped, the discard dropped from executor end and from transaction end, and the cap dropping every column rather than the one that crossed it. Section 1 sets only enable_seqscan/enable_bitmapscan off, and its UPDATE plans as Custom Scan (PgColumnarScan). It measures the cache anyway: the UPDATE fetches its target rows by number whatever the scan. Measured with the cache missing on every lookup, the unchanged suite is red at 844 ms against 93 ms. Its comment said the opposite reason ("index-driven so every row goes through the fetch path rather than a scan") and now says this. COMPLETE gains native_fetch_cache. TESTS.md section 168, cluster_tests 726 -> 730. Co-Authored-By: Claude Opus 5.5 --- test/native_fetch_cache.sh | 7 +- test/pytest/TESTS.md | 14 ++ test/pytest/expected_tests.txt | 4 +- test/pytest/test_compare_to_bash.py | 1 + test/pytest/test_native_fetch_cache.py | 207 +++++++++++++++++++++++++ 5 files changed, 231 insertions(+), 2 deletions(-) create mode 100644 test/pytest/test_native_fetch_cache.py diff --git a/test/native_fetch_cache.sh b/test/native_fetch_cache.sh index e936e0509..bd8531f9a 100755 --- a/test/native_fetch_cache.sh +++ b/test/native_fetch_cache.sh @@ -48,7 +48,12 @@ build() { # table, rows-per-group CREATE INDEX ${1}_id ON $1 (id);" } -# index-driven so every row goes through the fetch path rather than a scan +# Every updated row goes through the fetch path whatever the scan plans: the +# UPDATE fetches its target rows by number. Measured (#1341) on PG18: this +# UPDATE plans as Custom Scan (PgColumnarScan) under these SETs, and with the +# cache made to miss on every lookup the ratio below still goes red (844 ms +# against 93 ms). So the SETs are not what makes it a fetch, and no plan +# premise is needed here, unlike the point queries further down. upd_ms() { local start end start=$(date +%s%N) diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 06ac2d401..04d6e3e0e 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -213,6 +213,7 @@ behaviour, the source of that number is named. - [165. test_native_groupagg_batch.py: the grouped aggregate folds column-at-a-time](#165-test_native_groupagg_batchpy-the-grouped-aggregate-folds-column-at-a-time) - [166. test_native_fetch_projection.py: a fetch by row number decodes only what its caller needs](#166-test_native_fetch_projectionpy-a-fetch-by-row-number-decodes-only-what-its-caller-needs) - [167. test_native_fetch_position.py: a fetch by row number costs the same anywhere in its group](#167-test_native_fetch_positionpy-a-fetch-by-row-number-costs-the-same-anywhere-in-its-group) +- [168. test_native_fetch_cache.py: the fetch-by-row-number cache](#168-test_native_fetch_cachepy-the-fetch-by-row-number-cache) ## 1. How to read a test in here @@ -7705,3 +7706,16 @@ The pytest twin of `test/native_fetch_position.sh` (ported for #1341), every che | `test_the_cost_does_not_depend_on_the_position` | fetching the last twentieth of the group costs under three times the first | | `test_a_bigger_group_does_not_cost_more_per_fetch` | the same far-end fetches in a group twice the size cost under 1.8 times as much | | `test_the_indexes_that_make_it_constant_are_in_the_fetch` | `pgcolumnar_fetch_row` takes the rank from the prefix and a varying-length value from the offset table | + +## 168. test_native_fetch_cache.py: the fetch-by-row-number cache + +The pytest twin of `test/native_fetch_cache.sh` (ported for #1341), every check by the shell's name. A fetch by row number decoded a whole row group per row (#143), and a statement-scoped cache of the decoded group removes the repeat. The cache is timed as a ratio: the same index fetches from one big row group and from ten small ones. Each reading runs on a freshly built fixture, because an UPDATE moves the rows it reads into a new group (#801). Then come the scoping guards in the source, a wide group's decode scratch (#353), and crossing the cache cap (#359). + +### Every test + +| test | what it holds | +| --- | --- | +| `test_the_cache_makes_one_big_group_cost_what_ten_small_ones_do` | each reading measured the geometry it meant to, one big group costs under three times ten small ones, and the updated rows are right | +| `test_the_scoping_guards_are_in_the_source` | the cache key includes the storage id, an earlier command's entry is rejected, a hit re-checks four fields, and the cache is discarded at executor end and transaction end | +| `test_a_wide_groups_decode_scratch_does_not_blow_the_cap` | a wide group's point query drives the fetch path, costs under five times a small group's, and answers the same | +| `test_crossing_the_cap_costs_per_column` | four and five projected columns drive the fetch path, five costs under twelve times four, and every over-cap column reads back its value | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 655419001..f15994f2d 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -797,4 +797,6 @@ guard_tests 438 # four tests, by collection over the cluster files on this tree. # cluster_tests 722 -> 726 for #1341's test_native_fetch_position.py, the twin of native_fetch_position.sh: # four tests, by collection over the cluster files on this tree. -cluster_tests 726 +# cluster_tests 726 -> 730 for #1341's test_native_fetch_cache.py, the twin of native_fetch_cache.sh: +# four tests, by collection over the cluster files on this tree. +cluster_tests 730 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 3db54a5c3..803b82bc6 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -129,6 +129,7 @@ "native_dict_underfill", "native_dml", "native_encdesc_golden", + "native_fetch_cache", "native_fetch_coalesce", "native_fetch_position", "native_fetch_projection", diff --git a/test/pytest/test_native_fetch_cache.py b/test/pytest/test_native_fetch_cache.py new file mode 100644 index 000000000..5b55beaf2 --- /dev/null +++ b/test/pytest/test_native_fetch_cache.py @@ -0,0 +1,207 @@ +"""The fetch-by-row-number cache (#143, #353, #359, #1341). The pytest twin of +`test/native_fetch_cache.sh`, every check by the shell's name. + +A fetch by row number decoded a whole row group per row; a statement-scoped cache of +the decoded group removes the repeat. The cache is timed as a ratio: the same index +fetches from one big row group and from ten small ones, each reading on a freshly +built fixture because an UPDATE moves the rows it reads into a new group (#801). The +UPDATE fetches its rows by number whatever its scan; every timed point query asserts +first that its plan is an index scan, under the SETs that make one (#797). Then the +scoping guards +in the source, a wide group's decode scratch (#353), and crossing the cache cap +costing per column rather than per entry (#359). +""" +import os +import pathlib +import re +import time + +import psycopg +import pytest + +ROWS = int(os.environ.get("PGC_FETCH_ROWS", "20000")) +UPD = ROWS // 10 +SRC = pathlib.Path(__file__).resolve().parents[2] / "src" +FETCH_PATH = ("max_parallel_workers_per_gather=0", "enable_seqscan=off", "enable_bitmapscan=off", + "pgcolumnar.enable_index_fetch_penalty=off") + + +def _one(c, sql): + return c.execute(sql).fetchone()[0] + + +class _Sets: + def __init__(self, c, sets): + self.c, self.sets = c, sets + + def __enter__(self): + for s in self.sets: + self.c.execute(f"SET {s}") + + def __exit__(self, *exc): + for s in self.sets: + self.c.execute(f"RESET {s.split('=')[0]}") + + +def _row_fetch(c, sql): + """-> 'yes' when `sql` plans an Index Scan under the fetch-path SETs, as pgc_uses_row_fetch.""" + with _Sets(c, FETCH_PATH): + plan = "\n".join(r[0] for r in c.execute(f"EXPLAIN (COSTS OFF) {sql}")) + return "yes" if "Index Scan using" in plan else "no" + + +def _ms(c, sql, sets=FETCH_PATH): + with _Sets(c, sets): + t0 = time.monotonic() + c.execute(sql) + return int((time.monotonic() - t0) * 1000) + + +def _ratio(a, b, bound): + """-> 'within' when a/b, rounded to two places, is at most `bound`; else why not, + as check_ratio words it: a zero side measured nothing.""" + if a == 0 or b == 0: + return f"a side of the ratio is zero, so nothing was measured: a=[{a}] b=[{b}]" + r = round(a / b, 2) + return "within" if r <= bound else f"{r:.2f}x exceeds the {bound}x bound (a={a} b={b})" + + +def _groups(c, table): + return _one(c, f"SELECT count(*) FROM pgcolumnar.row_group r JOIN pgcolumnar.storage s USING (storage_id) " + f"WHERE s.relation_oid = '{table}'::regclass") + + +def _build(c, table, per_group): + c.execute(f"DROP TABLE IF EXISTS {table}") + c.execute(f"SET pgcolumnar.stripe_row_limit = {per_group}") + c.execute(f"SET pgcolumnar.chunk_group_row_limit = {per_group}") + try: + c.execute(f"CREATE TABLE {table} (id int, v int, t text) USING pgcolumnar") + c.execute(f"INSERT INTO {table} SELECT g, g, 'row' || g FROM generate_series(1, {ROWS}) g") + c.execute(f"CREATE INDEX {table}_id ON {table} (id)") + finally: + c.execute("RESET pgcolumnar.stripe_row_limit") + c.execute("RESET pgcolumnar.chunk_group_row_limit") + + +def _fn(path, name): + m = re.search(rf"^{name}\(.*?^}}", (SRC / path).read_text(), re.S | re.M) + return m.group(0) if m else "" + + +@pytest.fixture(scope="module") +def fc(pgc_cluster): + db = "pgc_fetchcache" + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + a.execute(f"CREATE DATABASE {db}") + try: + with psycopg.connect(pgc_cluster.dsn(dbname=db), autocommit=True) as c: + c.execute("CREATE EXTENSION pgcolumnar") + yield c + finally: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + + +def test_the_cache_makes_one_big_group_cost_what_ten_small_ones_do(fc, expect): + c = fc + # the shell's SETs for this arm; the UPDATE fetches its target rows by number anyway + upd_sets = ("max_parallel_workers_per_gather=0", "enable_seqscan=off", "enable_bitmapscan=off") + upd = f"UPDATE {{}} SET v = id + 1 WHERE id <= {UPD}" + # rebuilt before every reading: the UPDATE moves the rows it read into a new group (#801) + best, geom = {}, {} + for table, per in (("fc_one", ROWS), ("fc_many", ROWS // 10)): + times, seen = [], [] + for _ in range(3): + _build(c, table, per) + seen.append(str(_groups(c, table))) + times.append(_ms(c, upd.format(table), upd_sets)) + best[table], geom[table] = min(times), " ".join(seen) + print(f"-- {UPD} fetches: one group of {ROWS} took {best['fc_one']} ms; ten groups took {best['fc_many']} ms") + expect.text(geom["fc_one"], "1 1 1", "every fc_one reading measured one row group (#801)") + expect.text(geom["fc_many"], "10 10 10", "every fc_many reading measured ten row groups (#801)") + expect.text(_ratio(best["fc_one"], best["fc_many"], 3), "within", + "fetching from one big group is not far dearer than from ten small ones") + expect.num(_one(c, "SELECT count(*) FROM fc_one WHERE v = id + 1"), UPD, "the updated rows are correct") + expect.num(_one(c, "SELECT count(*) FROM fc_one WHERE v = id"), ROWS - UPD, "the untouched rows are unchanged") + + +def test_the_scoping_guards_are_in_the_source(expect): + reader = (SRC / "columnar_reader.c").read_text() + slot, row = _fn("columnar_reader.c", "pgcolumnar_fetch_group_slot"), _fn("columnar_reader.c", "pgcolumnar_fetch_row") + expect.text("yes" if slot and row else "no", "yes", "premise: the fetch-cache functions were extracted, not empty ranges") + lines = reader.splitlines() + grp = sum("e->groupNumber == groupNumber" in l for l in lines) + keyed = sum("e->storageId == storageId && e->groupNumber == groupNumber" in l for l in lines) + expect.at_least(grp, 1, "premise: there is a group-number comparison to key") + expect.text(f"keyed {keyed} of {grp}", f"keyed {grp} of {grp}", "every group-number comparison is keyed by the storage id too") + expect.text("yes" if "e->cid != cid" in slot else "no", "yes", "an entry from an earlier command is rejected") + for f in ("firstRowNumber", "rowCount", "fileOffset"): + expect.text("yes" if f"entry->{f} != rg->{f}" in row else "no", "yes", + f"a hit re-checks the group's {f} against the row group") + expect.text("yes" if "entry->natts != natts" in row else "no", "yes", + "a hit re-checks the group's natts against the scan's column count") + for fn in ("pgcolumnar_executor_end", "pgcolumnar_xact_callback"): + body = _fn("columnar_tableam.c", fn) + expect.text("yes" if body else "no", "yes", f"premise: {fn} was extracted, not an empty range") + expect.text("yes" if "PgColumnarDiscardFetchCache" in body else "no", "yes", + f"the fetch cache is discarded from {fn}") + + +def test_a_wide_groups_decode_scratch_does_not_blow_the_cap(fc, expect): + c = fc + for table, stripe in (("fc_wide", 150000), ("fc_wnarrow", 20000)): + c.execute(f"DROP TABLE IF EXISTS {table}") + c.execute(f"""CREATE TABLE {table} (id int, h text, c1 int,c2 int,c3 int,c4 int,c5 int,c6 int, + c7 int,c8 int,c9 int,c10 int, u float8, + t1 text,t2 text,t3 text,t4 text,t5 text,t6 text,t7 text,t8 text) USING pgcolumnar""") + c.execute(f"SELECT pgcolumnar.set_options('{table}', stripe_row_limit => {stripe})") + c.execute(f"""INSERT INTO {table} SELECT g, 'h' || (g % 500), g,g,g,g,g,g,g,g,g,g, (g % 100)::float8, + repeat('x',40),repeat('y',40),repeat('z',40),'a'||g,'b'||g,'c'||g,'d'||g,'e'||g + FROM generate_series(1, 300000) g""") + c.execute(f"CREATE INDEX {table}_h ON {table} (h)") + point = "SELECT count(*), max(u) FROM {} WHERE h='h1'" + for table in ("fc_wide", "fc_wnarrow"): + expect.text(_row_fetch(c, point.format(table)), "yes", f"the {table} point query really drives the fetch path (#797)") + bigw = min(_ms(c, point.format("fc_wide")) for _ in range(3)) + smallw = min(_ms(c, point.format("fc_wnarrow")) for _ in range(3)) + print(f"-- #353 wide point query: default-stripe {bigw} ms, small-stripe {smallw} ms") + expect.text(_ratio(bigw, smallw, 5), "within", "a wide group's fetch is not far dearer than a small group's (#353)") + both = "SELECT count(*) || '|' || coalesce(max(u)::text,'z') FROM {} WHERE h='h1'" + expect.text(_one(c, both.format("fc_wide")), _one(c, both.format("fc_wnarrow")), + "the wide-group point query is correct (#353)") + + +def test_crossing_the_cap_costs_per_column(fc, expect): + c = fc + c.execute("DROP TABLE IF EXISTS fc_w359") + c.execute("CREATE TABLE fc_w359 (id int, h text, t1 text,t2 text,t3 text,t4 text,t5 text) USING pgcolumnar") + c.execute("SELECT pgcolumnar.set_options('fc_w359', stripe_row_limit => 50000)") + c.execute("""INSERT INTO fc_w359 SELECT g, 'h' || (g % 1000), + repeat('a',150),repeat('b',150),repeat('c',150),repeat('d',150),repeat('e',150) + FROM generate_series(1,100000) g""") + c.execute("CREATE INDEX fc_w359_h ON fc_w359 (h)") + + def query(n): + return "SELECT count(*)" + "".join(f", max(t{i})" for i in range(1, n + 1)) + " FROM fc_w359 WHERE h='h7'" + + def warm_ms(n): + _ms(c, query(n)) # warm: decode cost, not first-touch I/O + return _ms(c, query(n)) + + for n in (4, 5): + expect.text(_row_fetch(c, query(n)), "yes", f"the {n}-column projection really drives the fetch path (#797)") + under = min(warm_ms(4) for _ in range(3)) # ~30 MB decoded: fits + over = min(warm_ms(5) for _ in range(3)) # ~38 MB decoded: does not + print(f"-- #359 projection width: four columns {under} ms, five columns {over} ms") + expect.text(_ratio(over, under, 12), "within", "crossing the fetch cache cap costs proportionally, not totally (#359)") + # the overflowing columns are re-decoded, not skipped: fast and null would be wrong + expect.text(_one(c, "SELECT max(t1) || '|' || max(t4) FROM fc_w359 WHERE h='h7'"), + _one(c, "SELECT max(t1) || '|' || max(t4) FROM fc_w359 WHERE h='h7' AND t5 IS NOT NULL"), + "the over-cap projection returns the same values as the under-cap one (#359)") + expect.num(_one(c, """SELECT count(*) FROM fc_w359 WHERE h='h7' + AND t1 = repeat('a',150) AND t2 = repeat('b',150) AND t3 = repeat('c',150) + AND t4 = repeat('d',150) AND t5 = repeat('e',150)"""), + _one(c, "SELECT count(*) FROM fc_w359 WHERE h='h7'"), + "every over-cap column reads back its written value (#359)") From e23416e40557cbbd75d36378d5842eec3064b6d7 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Fri, 16 Oct 2026 17:08:37 +0000 Subject: [PATCH 5/7] test: native_fetch_cache seeds the ledger, 10 of 26 arms dated by ten cache mutations (#1341) The suite had no rows. suites_not_covered 198 -> 197; census 4227 rows / 2991 never / 1236 dated. Co-Authored-By: Claude Opus 5.5 --- test/check_ledger.tsv | 26 ++++++++++++++++++++++++++ test/check_ledger_budget.txt | 20 ++++++++++++++++++-- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 264a6fd5a..4db648adb 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2470,6 +2470,32 @@ native_encdesc_golden native_encdesc_golden int column a rawLen parses as 1200 ( native_encdesc_golden native_encdesc_golden the table reads back all rows 15;16;17;18;19 never - native_encdesc_golden native_encdesc_golden trailing sharedTableLen is 0 (u32 at offset 6+13=19) 15;16;17;18;19 never - native_encdesc_golden native_encdesc_golden vectorCount parses as 1 (at offset 2) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache a hit re-checks the group's fileOffset against the row group 15;16;17;18;19 2026-10-16 columnar_reader.c: a fetch-cache hit no longer re-checks fileOffset +native_fetch_cache native_fetch_cache a hit re-checks the group's firstRowNumber against the row group 15;16;17;18;19 2026-10-16 columnar_reader.c: a fetch-cache hit no longer re-checks firstRowNumber +native_fetch_cache native_fetch_cache a hit re-checks the group's natts against the scan's column count 15;16;17;18;19 2026-10-16 columnar_reader.c: a fetch-cache hit no longer re-checks natts +native_fetch_cache native_fetch_cache a hit re-checks the group's rowCount against the row group 15;16;17;18;19 2026-10-16 columnar_reader.c: a fetch-cache hit no longer re-checks rowCount +native_fetch_cache native_fetch_cache a wide group's fetch is not far dearer than a small group's (#353) 15;16;17;18;19 2026-10-16 columnar_reader.c: crossing the fetch-cache cap drops every column, not the one that crossed it;columnar_reader.c: every fetch-cache lookup misses, so each fetch decodes its group again +native_fetch_cache native_fetch_cache an entry from an earlier command is rejected 15;16;17;18;19 2026-10-16 columnar_reader.c: a fetch-cache entry from an earlier command is not rejected +native_fetch_cache native_fetch_cache crossing the fetch cache cap costs proportionally, not totally (#359) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache every fc_many reading measured ten row groups (#801) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache every fc_one reading measured one row group (#801) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache every group-number comparison is keyed by the storage id too 15;16;17;18;19 2026-10-16 columnar_reader.c: the fetch-cache key compares the group number without the storage id +native_fetch_cache native_fetch_cache every over-cap column reads back its written value (#359) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache fetching from one big group is not far dearer than from ten small ones 15;16;17;18;19 2026-10-16 columnar_reader.c: every fetch-cache lookup misses, so each fetch decodes its group again +native_fetch_cache native_fetch_cache premise: pgcolumnar_executor_end was extracted, not an empty range 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache premise: pgcolumnar_xact_callback was extracted, not an empty range 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache premise: the fetch-cache functions were extracted, not empty ranges 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache premise: there is a group-number comparison to key 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache the 4-column projection really drives the fetch path (#797) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache the 5-column projection really drives the fetch path (#797) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache the fc_wide point query really drives the fetch path (#797) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache the fc_wnarrow point query really drives the fetch path (#797) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache the fetch cache is discarded from pgcolumnar_executor_end 15;16;17;18;19 2026-10-16 columnar_tableam.c: pgcolumnar_executor_end no longer discards the fetch cache +native_fetch_cache native_fetch_cache the fetch cache is discarded from pgcolumnar_xact_callback 15;16;17;18;19 2026-10-16 columnar_tableam.c: the transaction callback no longer discards the fetch cache +native_fetch_cache native_fetch_cache the over-cap projection returns the same values as the under-cap one (#359) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache the untouched rows are unchanged 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache the updated rows are correct 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache the wide-group point query is correct (#353) 15;16;17;18;19 never - native_fetch_coalesce native_fetch_coalesce a wide index fetch does not pin once per column 15;16;17;18;19 2026-10-13 the coalesced fetch merging no adjacent ranges, so each column is read on its own native_fetch_coalesce native_fetch_coalesce premise: a point lookup uses the index 15;16;17;18;19 never - native_fetch_coalesce native_fetch_coalesce premise: fetching every projected column touched a measurable number of buffers 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 9a0a36773..1da87870b 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 198 +suites_not_covered 197 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -3369,4 +3369,20 @@ suites_not_covered 198 # never 2969 + 6 = 2975 # dated 1220 + 6 = 1226 # 2975 + 1226 == 4201 -checks_never_observed_red 2975 +# +# #1341, native_fetch_cache seeded, with its twin (OffgridwithJD): 2975 -> 2991, +# and suites_not_covered 198 -> 197. The suite had no rows. It enters with all 26 +# checks from five-major runs (26/26 on each of 15-19). 10 are dated by ten +# mutations: the cache never hitting, the key without the storage id, no +# earlier-command reject, each of the four recheck fields dropped, the discard +# dropped from executor end and from transaction end, and the cap dropping every +# column. Sixteen arrive never: four premises and the four #797 plan premises; +# the two #801 geometry records; the four correctness arms; and the #359 ratio, +# which the cap-drops-every-column mutation cannot redden on this fixture (it +# slows one projected column as much as five, measured 46 -> 286 ms in-suite). +# +# ledger rows 4201 + 26 = 4227 +# never 2975 + 16 = 2991 +# dated 1226 + 10 = 1236 +# 2991 + 1236 == 4227 +checks_never_observed_red 2991 From fcce69bacad8afc9e2be2b4bcf5f9cab87e1e04d Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Fri, 16 Oct 2026 17:18:28 +0000 Subject: [PATCH 6/7] test: native_parquet_pushdown gets its pytest twin (#1341) test_native_parquet_pushdown.py carries all 36 checks by the shell's name in four tests, each writing its file with pyarrow and declining under the shell's own pgc_skip name when pyarrow is absent. Under each of seven mutations of the FDW's skip decision, the twin reddens exactly the shell's set of checks: no group ever skipped, the inverted-interval guard dropped, float > and >= pushed, a NaN equality pushed, INT64 statistics not decoded, col < const skipping the groups that match, and an equality skipping every group. The last two are what date the oracle arms: a skip that only stops saving work cannot change an answer. Each test creates its own foreign server, because a server is database-wide and pgc_conn isolates tests by schema only. COMPLETE gains native_parquet_pushdown, and its literal tables join the loop-reader guard's list. TESTS.md section 169, cluster_tests 730 -> 734. Co-Authored-By: Claude Opus 5.5 --- test/pytest/TESTS.md | 14 ++ test/pytest/expected_tests.txt | 4 +- test/pytest/test_compare_to_bash.py | 3 +- test/pytest/test_native_parquet_pushdown.py | 163 ++++++++++++++++++++ 4 files changed, 182 insertions(+), 2 deletions(-) create mode 100644 test/pytest/test_native_parquet_pushdown.py diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 04d6e3e0e..139f21753 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -214,6 +214,7 @@ behaviour, the source of that number is named. - [166. test_native_fetch_projection.py: a fetch by row number decodes only what its caller needs](#166-test_native_fetch_projectionpy-a-fetch-by-row-number-decodes-only-what-its-caller-needs) - [167. test_native_fetch_position.py: a fetch by row number costs the same anywhere in its group](#167-test_native_fetch_positionpy-a-fetch-by-row-number-costs-the-same-anywhere-in-its-group) - [168. test_native_fetch_cache.py: the fetch-by-row-number cache](#168-test_native_fetch_cachepy-the-fetch-by-row-number-cache) +- [169. test_native_parquet_pushdown.py: the Parquet FDW skips row groups its statistics rule out](#169-test_native_parquet_pushdownpy-the-parquet-fdw-skips-row-groups-its-statistics-rule-out) ## 1. How to read a test in here @@ -7719,3 +7720,16 @@ The pytest twin of `test/native_fetch_cache.sh` (ported for #1341), every check | `test_the_scoping_guards_are_in_the_source` | the cache key includes the storage id, an earlier command's entry is rejected, a hit re-checks four fields, and the cache is discarded at executor end and transaction end | | `test_a_wide_groups_decode_scratch_does_not_blow_the_cap` | a wide group's point query drives the fetch path, costs under five times a small group's, and answers the same | | `test_crossing_the_cap_costs_per_column` | four and five projected columns drive the fetch path, five costs under twelve times four, and every over-cap column reads back its value | + +## 169. test_native_parquet_pushdown.py: the Parquet FDW skips row groups its statistics rule out + +The pytest twin of `test/native_parquet_pushdown.sh` (ported for #1341), every check by the shell's name. The FDW reads each row group's min/max and skips a group that a pushable col-op-const clause proves empty. The executor still rechecks every row, so a sound skip can only save work. The checks run against a heap oracle, and the skips themselves are read from EXPLAIN ANALYZE's "Row Groups Skipped". The soundness cases follow. A NaN row sits outside the statistics, so a float `>`, `>=` or `= 'NaN'` must never skip. An inverted interval must never drive a skip. An INT64-backed DECIMAL is skippable. pyarrow writes every file. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_skipping_never_changes_the_answer_and_really_skips` | eight predicates agree with the heap, each confining range skips the groups it should, float `>=` is not pushed, and `read_parquet` reads the same rows | +| `test_a_nan_row_is_never_skipped` | seven float predicates over a file with one NaN agree with the heap | +| `test_inverted_statistics_never_drive_a_skip` | a UINT_32 column whose statistics decode inverted agrees with the heap under six predicates and skips nothing | +| `test_an_int64_backed_decimal_is_skippable` | a range inside one of four DECIMAL groups skips the other three without dropping rows | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index f15994f2d..4c10d23d0 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -799,4 +799,6 @@ guard_tests 438 # four tests, by collection over the cluster files on this tree. # cluster_tests 726 -> 730 for #1341's test_native_fetch_cache.py, the twin of native_fetch_cache.sh: # four tests, by collection over the cluster files on this tree. -cluster_tests 730 +# cluster_tests 730 -> 734 for #1341's test_native_parquet_pushdown.py, the twin of native_parquet_pushdown.sh: +# four tests, by collection over the cluster files on this tree. +cluster_tests 734 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 803b82bc6..bda964b6b 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -140,6 +140,7 @@ "native_join_vector_agg", "native_ownership", "native_parquet_dict_oob", + "native_parquet_pushdown", "native_parquet_schema", "native_projection", "native_reclaim", @@ -415,7 +416,7 @@ def test_the_loop_reader_invents_nothing_in_this_corpus(expect): "came from") expect.text(", ".join(files), "build_refusal, differential, join_runtime_filter, native_groupagg_batch, " - "native_index_projection, native_reclaim, native_repack, " + "native_index_projection, native_parquet_pushdown, native_reclaim, native_repack, " "native_saop_pushdown, native_vacuum_race, native_vecskip, sorted_pathkeys", "and it is these files, so another appearing is a diff a reviewer " "sees rather than a number that moved") diff --git a/test/pytest/test_native_parquet_pushdown.py b/test/pytest/test_native_parquet_pushdown.py new file mode 100644 index 000000000..d0e2827b3 --- /dev/null +++ b/test/pytest/test_native_parquet_pushdown.py @@ -0,0 +1,163 @@ +"""The Parquet FDW skips row groups its statistics prove empty (Phase G, #1341). The +pytest twin of `test/native_parquet_pushdown.sh`, every check by the shell's name. + +Each row group's min/max is read, and a pushable col-op-const clause skips groups it +proves empty; the executor still rechecks every row, so a skip can only save work. +Correctness against a heap oracle (skipping never changes results), and the skips +themselves through EXPLAIN ANALYZE's "Row Groups Skipped". Then soundness: a NaN row +sits outside the statistics, so float > and >= and = 'NaN' must never skip; an +inverted interval (UINT_32 stats ordered unsigned, decoded signed) must never drive a +skip; and an INT64-backed DECIMAL is skippable. Every file is written by pyarrow, an +independent writer, so the reader and the writer cannot share a misunderstanding. +""" +import decimal + +import psycopg + +HASH = ("SELECT coalesce(md5(string_agg(t, chr(10) ORDER BY t)), $e$EMPTY$e$) " + "FROM (SELECT _row::text AS t FROM ({}) _row) _s") +ANALYZE = "EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) SELECT count(*) FROM {} WHERE {}" + + +def _hash(cur, sql): + """-> the shell's pgc_set_hash of `sql`, or the error: two failing queries never agree.""" + try: + cur.execute(HASH.format(sql)) + return cur.fetchone()[0] + except psycopg.Error as exc: + return f"error: {exc.diag.message_primary} [{sql}]" + + +def _counter(cur, table, where, label): + """-> the number on the plan's `label` line, or '' when there is none.""" + cur.execute(ANALYZE.format(table, where)) + for (line,) in cur.fetchall(): + if label in line: + digits = "".join(ch if ch.isdigit() else " " for ch in line.split(label, 1)[1]).split() + return digits[0] if digits else "" + return "" + + +def _write(path, table, pq, **kw): + pq.write_table(table, str(path), **kw) + path.chmod(0o644) + + +def _ft(cur, server_dir, pa, pq): + """200000 rows sorted by id in four groups of 50000: disjoint id ranges, with stats.""" + path = server_dir / "stats.parquet" + n = 200000 + _write(path, pa.table({"id": pa.array(range(n), pa.int32()), + "val": pa.array([i * 1.5 for i in range(n)], pa.float64())}), pq, row_group_size=50000) + assert pq.ParquetFile(str(path)).num_row_groups == 4 + cur.execute("CREATE TABLE h (id int, val float8)") + cur.execute("INSERT INTO h SELECT g, g*1.5 FROM generate_series(0,199999) g") + cur.execute("CREATE SERVER IF NOT EXISTS pq FOREIGN DATA WRAPPER pgcolumnar_parquet") + cur.execute(f"CREATE FOREIGN TABLE ft (id int4, val float8) SERVER pq OPTIONS (path '{path}')") + return path + + +def test_skipping_never_changes_the_answer_and_really_skips(pgc_conn, server_dir, expect): + pa = expect.require_module("pyarrow", name="pyarrow not available; predicate-pushdown suite needs it to write stats") + if pa is None: + return + pq = expect.require_module("pyarrow.parquet", + name="pyarrow not available; predicate-pushdown suite needs it to write stats") + with pgc_conn.cursor() as cur: + path = _ft(cur, server_dir, pa, pq) + expect.text(_hash(cur, "SELECT * FROM ft"), _hash(cur, "SELECT * FROM h"), "full scan == oracle") + for pred in ("id = 60000", "id BETWEEN 60000 AND 70000", "id < 100", "id >= 199900", + "id > 60000 AND id < 60010", "id <= 5 OR id >= 199995", "val > 90000 AND val < 90030"): + expect.text(_hash(cur, f"SELECT * FROM ft WHERE {pred}"), _hash(cur, f"SELECT * FROM h WHERE {pred}"), + f"predicate [{pred}] == oracle") + + expect.text(_counter(cur, "ft", "id >= 0", "Row Groups:"), "4", "reports 4 row groups") + for name, where, want in ( + ("range in one group skips 3", "id BETWEEN 60000 AND 70000", "3"), + ("point in group 0 skips 3", "id = 123", "3"), + ("id >= 199900 skips 3", "id >= 199900", "3"), + ("id < 100 skips 3", "id < 100", "3"), + # float >= and > are never pushed: Parquet stats exclude NaN, PostgreSQL sorts it last + ("float predicate skips 2 (>= not pushed, NaN-unsafe)", "val >= 90000 AND val <= 90030", "2"), + ("float < still skips", "val < 1.0", "3"), + ("spanning predicate skips 0", "id >= 0 AND id < 200000", "0"), + ("no predicate skips 0", "true", "0"), + ): + expect.text(_counter(cur, "ft", where, "Row Groups Skipped"), want, name) + + # read_parquet over the same file ignores the stats and reads everything + expect.text(_hash(cur, f"SELECT * FROM pgcolumnar.read_parquet('{path}') AS t(id int4, val float8)"), + _hash(cur, "SELECT * FROM h"), "read_parquet same file == oracle") + + +def test_a_nan_row_is_never_skipped(pgc_conn, server_dir, expect): + pa = expect.require_module("pyarrow", name="pyarrow not available; predicate-pushdown suite needs it to write stats") + if pa is None: + return + pq = expect.require_module("pyarrow.parquet", + name="pyarrow not available; predicate-pushdown suite needs it to write stats") + path = server_dir / "nan.parquet" + vals = [float(i) for i in range(4000)] + vals[500] = float("nan") # one NaN, in row group 0 + _write(path, pa.table({"id": pa.array(range(4000), pa.int32()), "val": pa.array(vals, pa.float64())}), + pq, row_group_size=1000) + assert pq.ParquetFile(str(path)).num_row_groups == 4 + with pgc_conn.cursor() as cur: + cur.execute("CREATE SERVER IF NOT EXISTS pq_nan FOREIGN DATA WRAPPER pgcolumnar_parquet") + cur.execute(f"CREATE FOREIGN TABLE ftn (id int4, val float8) SERVER pq_nan OPTIONS (path '{path}')") + cur.execute("CREATE TABLE hn (id int, val float8)") + cur.execute("INSERT INTO hn SELECT g, CASE WHEN g = 500 THEN 'NaN'::float8 ELSE g END " + "FROM generate_series(0,3999) g") + for pred in ("val > 1e300", "val >= 1e300", "val = 'NaN'::float8", "val > 3998", "val < 10", "val <= 10", + "val = 1234"): + expect.text(_hash(cur, f"SELECT * FROM ftn WHERE {pred}"), _hash(cur, f"SELECT * FROM hn WHERE {pred}"), + f"NaN-safe [{pred}] == oracle") + + +def test_inverted_statistics_never_drive_a_skip(pgc_conn, server_dir, expect): + pa = expect.require_module("pyarrow", name="pyarrow not available; predicate-pushdown suite needs it to write stats") + if pa is None: + return + pq = expect.require_module("pyarrow.parquet", + name="pyarrow not available; predicate-pushdown suite needs it to write stats") + # UINT_32 statistics are ordered unsigned and decoded signed: min=1, max=-1294967296 + path = server_dir / "inverted.parquet" + _write(path, pa.table({"c": pa.array([1, 3000000000], pa.uint32())}), pq, row_group_size=2) + st = pq.ParquetFile(str(path)).metadata.row_group(0).column(0).statistics + assert st.min == 1 and st.max == 3000000000, (st.min, st.max) + with pgc_conn.cursor() as cur: + cur.execute("CREATE SERVER IF NOT EXISTS pq_inv FOREIGN DATA WRAPPER pgcolumnar_parquet") + cur.execute(f"CREATE FOREIGN TABLE fti (c int4) SERVER pq_inv OPTIONS (path '{path}')") + cur.execute("CREATE TABLE hi (c int4)") + cur.execute("INSERT INTO hi VALUES (1), (-1294967296)") + expect.text(_hash(cur, "SELECT * FROM fti"), _hash(cur, "SELECT * FROM hi"), + "inverted stats: unfiltered == oracle") + for pred in ("c = 1", "c = -1294967296", "c >= 1", "c <= -1294967296", "c < 0", "c > 0"): + expect.text(_hash(cur, f"SELECT * FROM fti WHERE {pred}"), _hash(cur, f"SELECT * FROM hi WHERE {pred}"), + f"inverted-stats [{pred}] == oracle") + expect.text(_counter(cur, "fti", "c = 1", "Row Groups Skipped"), "0", "inverted stats skip nothing") + + +def test_an_int64_backed_decimal_is_skippable(pgc_conn, server_dir, expect): + pa = expect.require_module("pyarrow", name="pyarrow not available; predicate-pushdown suite needs it to write stats") + if pa is None: + return + pq = expect.require_module("pyarrow.parquet", + name="pyarrow not available; predicate-pushdown suite needs it to write stats") + # four row groups with disjoint ranges, stored as INT64, the type the stats decode from + path = server_dir / "dec_push.parquet" + vals = [decimal.Decimal(i) / 100 for i in range(0, 40000)] + _write(path, pa.table({"d": pa.array(vals, pa.decimal128(18, 2))}), pq, + store_decimal_as_integer=True, compression="none", row_group_size=10000) + meta = pq.ParquetFile(str(path)).metadata + assert meta.row_group(0).column(0).physical_type == "INT64" and meta.num_row_groups == 4 + with pgc_conn.cursor() as cur: + cur.execute("CREATE SERVER IF NOT EXISTS pq_dec FOREIGN DATA WRAPPER pgcolumnar_parquet") + cur.execute(f"CREATE FOREIGN TABLE ftdec (d numeric) SERVER pq_dec OPTIONS (path '{path}')") + # 0.00 .. 399.99 in groups of 100.00: a range inside the second excludes the other three + expect.text(_counter(cur, "ftdec", "d BETWEEN 120.00 AND 130.00", "Row Groups Skipped"), "3", + "INT64-backed DECIMAL: predicate skips 3 of 4 groups") + cur.execute("SELECT count(*) FROM ftdec WHERE d BETWEEN 120.00 AND 130.00") + expect.num(cur.fetchone()[0], 1001, "INT64-backed DECIMAL: skipping did not drop rows") + expect.text(_counter(cur, "ftdec", "d >= 0", "Row Groups Skipped"), "0", + "INT64-backed DECIMAL: unfiltered scan skips nothing") From 860819eca6ed9b05042dd917d68eba1c6e55804d Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Fri, 16 Oct 2026 17:18:39 +0000 Subject: [PATCH 7/7] test: native_parquet_pushdown seeds the ledger, 25 of 36 arms dated by seven skip mutations (#1341) The suite had no rows. suites_not_covered 197 -> 196; census 4263 rows / 3002 never / 1261 dated. Co-Authored-By: Claude Opus 5.5 --- test/check_ledger.tsv | 36 ++++++++++++++++++++++++++++++++++++ test/check_ledger_budget.txt | 20 ++++++++++++++++++-- 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 4db648adb..98a6b2caf 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2813,6 +2813,42 @@ native_join_vector_agg native_join_vector_agg unique join fold answer equals GUC native_join_vector_agg native_join_vector_agg unique join fold answer equals heap 15;16;17;18;19 never - native_join_vector_agg native_join_vector_agg unique join uses core Agg when GUC off 15;16;17;18;19 never - native_join_vector_agg native_join_vector_agg unique join uses vectorized agg when GUC on 15;16;17;18;19 2026-09-12 drop JOINREL fold +native_parquet_pushdown native_parquet_pushdown INT64-backed DECIMAL: predicate skips 3 of 4 groups 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: INT64 statistics are not decoded, so an INT64-backed DECIMAL never skips;columnar_parquet_reader.c: pqfdw_compute_skip never marks a row group skippable +native_parquet_pushdown native_parquet_pushdown INT64-backed DECIMAL: skipping did not drop rows 15;16;17;18;19 never - +native_parquet_pushdown native_parquet_pushdown INT64-backed DECIMAL: unfiltered scan skips nothing 15;16;17;18;19 never - +native_parquet_pushdown native_parquet_pushdown NaN-safe [val < 10] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: col < const skips the row groups that do hold matches +native_parquet_pushdown native_parquet_pushdown NaN-safe [val <= 10] == oracle 15;16;17;18;19 never - +native_parquet_pushdown native_parquet_pushdown NaN-safe [val = 'NaN'::float8] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: an equality against a NaN constant is pushed +native_parquet_pushdown native_parquet_pushdown NaN-safe [val = 1234] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: an equality skips every row group +native_parquet_pushdown native_parquet_pushdown NaN-safe [val > 1e300] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: float > and >= are pushed although the statistics exclude NaN +native_parquet_pushdown native_parquet_pushdown NaN-safe [val > 3998] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: float > and >= are pushed although the statistics exclude NaN +native_parquet_pushdown native_parquet_pushdown NaN-safe [val >= 1e300] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: float > and >= are pushed although the statistics exclude NaN +native_parquet_pushdown native_parquet_pushdown float < still skips 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: col < const skips the row groups that do hold matches;columnar_parquet_reader.c: pqfdw_compute_skip never marks a row group skippable +native_parquet_pushdown native_parquet_pushdown float predicate skips 2 (>= not pushed, NaN-unsafe) 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: float > and >= are pushed although the statistics exclude NaN;columnar_parquet_reader.c: pqfdw_compute_skip never marks a row group skippable +native_parquet_pushdown native_parquet_pushdown full scan == oracle 15;16;17;18;19 never - +native_parquet_pushdown native_parquet_pushdown id < 100 skips 3 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: col < const skips the row groups that do hold matches;columnar_parquet_reader.c: pqfdw_compute_skip never marks a row group skippable +native_parquet_pushdown native_parquet_pushdown id >= 199900 skips 3 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: pqfdw_compute_skip never marks a row group skippable +native_parquet_pushdown native_parquet_pushdown inverted stats skip nothing 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: an inverted min/max interval drives skips like any other +native_parquet_pushdown native_parquet_pushdown inverted stats: unfiltered == oracle 15;16;17;18;19 never - +native_parquet_pushdown native_parquet_pushdown inverted-stats [c < 0] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: an inverted min/max interval drives skips like any other +native_parquet_pushdown native_parquet_pushdown inverted-stats [c <= -1294967296] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: an inverted min/max interval drives skips like any other +native_parquet_pushdown native_parquet_pushdown inverted-stats [c = -1294967296] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: an inverted min/max interval drives skips like any other +native_parquet_pushdown native_parquet_pushdown inverted-stats [c = 1] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: an inverted min/max interval drives skips like any other +native_parquet_pushdown native_parquet_pushdown inverted-stats [c > 0] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: an inverted min/max interval drives skips like any other +native_parquet_pushdown native_parquet_pushdown inverted-stats [c >= 1] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: an inverted min/max interval drives skips like any other +native_parquet_pushdown native_parquet_pushdown no predicate skips 0 15;16;17;18;19 never - +native_parquet_pushdown native_parquet_pushdown point in group 0 skips 3 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: an equality skips every row group;columnar_parquet_reader.c: pqfdw_compute_skip never marks a row group skippable +native_parquet_pushdown native_parquet_pushdown predicate [id < 100] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: col < const skips the row groups that do hold matches +native_parquet_pushdown native_parquet_pushdown predicate [id <= 5 OR id >= 199995] == oracle 15;16;17;18;19 never - +native_parquet_pushdown native_parquet_pushdown predicate [id = 60000] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: an equality skips every row group +native_parquet_pushdown native_parquet_pushdown predicate [id > 60000 AND id < 60010] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: col < const skips the row groups that do hold matches +native_parquet_pushdown native_parquet_pushdown predicate [id >= 199900] == oracle 15;16;17;18;19 never - +native_parquet_pushdown native_parquet_pushdown predicate [id BETWEEN 60000 AND 70000] == oracle 15;16;17;18;19 never - +native_parquet_pushdown native_parquet_pushdown predicate [val > 90000 AND val < 90030] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: col < const skips the row groups that do hold matches +native_parquet_pushdown native_parquet_pushdown range in one group skips 3 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: pqfdw_compute_skip never marks a row group skippable +native_parquet_pushdown native_parquet_pushdown read_parquet same file == oracle 15;16;17;18;19 never - +native_parquet_pushdown native_parquet_pushdown reports 4 row groups 15;16;17;18;19 never - +native_parquet_pushdown native_parquet_pushdown spanning predicate skips 0 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: col < const skips the row groups that do hold matches native_parquet_schema native_parquet_schema OPTIONAL column reports nullable=t 15;16;17;18;19 never - native_parquet_schema native_parquet_schema REQUIRED column reports nullable=f 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: parquet_schema reports every column nullable native_parquet_schema native_parquet_schema REQUIRED int32 maps to integer 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 1da87870b..760f29c2f 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 197 +suites_not_covered 196 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -3385,4 +3385,20 @@ suites_not_covered 197 # never 2975 + 16 = 2991 # dated 1226 + 10 = 1236 # 2991 + 1236 == 4227 -checks_never_observed_red 2991 +# +# #1341, native_parquet_pushdown seeded, with its twin (OffgridwithJD): 2991 -> 3002, +# and suites_not_covered 197 -> 196. The suite had no rows. It enters with all 36 +# checks from five-major runs (36/36 on each of 15-19). 25 are dated by seven +# mutations of the FDW's skip decision: no group skipped, the inverted-interval +# guard dropped, float > and >= pushed, a NaN equality pushed, INT64 statistics +# not decoded, col < const skipping the matching groups, and an equality skipping +# every group. Eleven arrive never: the unfiltered and read_parquet oracles, which +# no pushdown reaches; four predicate oracles no wrong skip here misdirects; the +# row-group count and the no-predicate control; and the DECIMAL arm's row count +# and unfiltered control. +# +# ledger rows 4227 + 36 = 4263 +# never 2991 + 11 = 3002 +# dated 1236 + 25 = 1261 +# 3002 + 1261 == 4263 +checks_never_observed_red 3002