From 9705252d8148dc4e7d5021021f5210063c87ae40 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Fri, 16 Oct 2026 16:47:32 +0000 Subject: [PATCH 1/3] test: native_fetch_position gets its pytest twin, and its fetches go through the index it was written for (#1341) The suite forced an index scan with enable_seqscan=off, which leaves the columnar custom scan on. Measured: the single-row fetch, the near and far UPDATEs and the mirror join all planned as Custom Scan (PgColumnarScan), so none of them fetched by row number. The rank off by one left the single-row arm green, and the rank walking every earlier row left both timing arms green. FORCE now also sets pgcolumnar.enable_custom_scan = off, and four premises assert that each measured query's plan is an Index Scan using its index. The timing arms then still missed the walk, because they timed an UPDATE. A columnar UPDATE writes each row again into a new row group, so only the first of three passes fetched from the far end, and the median was always a later pass. They now time a read of the same range, sum(fixed) and sum(length(txt)), so every pass fetches the same positions. Measured at head on 15-19: near and far within 1.03x, small and big group within 1.1x. Measured with the rank walking: 50 ms against 779 ms, and 390 against 779. test_native_fetch_position.py carries all 12 checks by the shell's name in four tests. Under each of four mutations (rank counting rows, rank off by one, rank walking, offset walking), the twin reddens exactly the shell's set of checks. Like the shell, it reads PGC_FETCHPOS_ROWS. COMPLETE gains native_fetch_position. TESTS.md section 167, cluster_tests 722 -> 726. Co-Authored-By: Claude Opus 5.5 --- test/native_fetch_position.sh | 44 +++++- test/pytest/TESTS.md | 14 ++ test/pytest/expected_tests.txt | 4 +- test/pytest/test_compare_to_bash.py | 1 + test/pytest/test_native_fetch_position.py | 166 ++++++++++++++++++++++ 5 files changed, 225 insertions(+), 4 deletions(-) create mode 100644 test/pytest/test_native_fetch_position.py diff --git a/test/native_fetch_position.sh b/test/native_fetch_position.sh index 9e860d8e4..18adc4f98 100755 --- a/test/native_fetch_position.sh +++ b/test/native_fetch_position.sh @@ -65,8 +65,32 @@ check "the rows are in one row group" \ # Fetch by row number is what an index scan drives, so force one. Each leading # SET prints its own line ahead of the result, so these queries take the last. +# +# enable_seqscan=off does NOT turn the columnar custom scan off, and it was all +# this suite set: measured, the single-row fetch, the near and far UPDATEs and +# the mirror join all planned as Custom Scan (PgColumnarScan), so none of them +# fetched by row number. A rank off by one left the single-row arm green, and a +# walk over every earlier row left both timing arms green. The custom scan is +# off now, and each measurement asserts first that its plan reads the index. FORCE="SET enable_seqscan = off; SET enable_bitmapscan = off; - SET max_parallel_workers_per_gather = 0;" + SET max_parallel_workers_per_gather = 0; + SET pgcolumnar.enable_custom_scan = off;" + +# reads_index QUERY INDEX -> yes, or the plan's first line: the premise that a +# query fetches by row number through INDEX rather than scanning +reads_index() { + local plan + plan="$(q "$FORCE EXPLAIN (COSTS OFF) $1")" + case "$plan" in + *"Index Scan using $2 "*) echo yes ;; + *) echo "no: $(grep -m1 -E 'Scan|Update' <<<"$plan" | sed 's/^ *//')" ;; + esac +} + +check "premise: the mirror comparison fetches through the index" \ + "$(reads_index "SELECT count(*) FROM fp_c c JOIN fp_h h USING (id) WHERE c.fixed IS DISTINCT FROM h.fixed" fp_c_id)" "yes" +check "premise: a single-row fetch goes through the index" \ + "$(reads_index "SELECT fixed FROM fp_c WHERE id = 1001" fp_c_id)" "yes" # rows spread across the whole group, including the far end where the old walk # was longest @@ -103,7 +127,7 @@ FAR_HI=$ROWS upd_once() { # id-low, id-high -> milliseconds for one pass local start end start=$(date +%s%N) - psql_run "$FORCE UPDATE fp_c SET fixed = fixed + 1 + psql_run "$FORCE SELECT sum(fixed), sum(length(txt)) FROM fp_c WHERE id BETWEEN $1 AND $2;" >/dev/null 2>&1 end=$(date +%s%N) echo $(( (end - start) / 1000000 )) @@ -130,6 +154,17 @@ upd_ms() { # id-low, id-high -> median of three passes printf '%s\n%s\n%s\n' "$a" "$b" "$c" | sort -n | sed -n 2p } +# The timed pass is a SELECT, not the UPDATE it was. A columnar UPDATE writes +# each row again into a new row group, so only the first of three passes +# fetched from the far end of the big group; the second and third fetched from +# the start of the new one, and the median was always one of those. Measured +# with the rank walking every earlier row: the UPDATE form read near 122 ms, +# far 132 ms, green. A read leaves every row where it was, so all three passes +# fetch the same positions. sum(length(txt)) makes it decode a varying-length +# column, which the offset table serves. +check "premise: the timed reads fetch through the index" \ + "$(reads_index "SELECT sum(fixed), sum(length(txt)) FROM fp_c WHERE id BETWEEN $FAR_LO AND $FAR_HI" fp_c_id)" "yes" + near="$(upd_ms $NEAR_LO $NEAR_HI)" far="$(upd_ms $FAR_LO $FAR_HI)" echo "-- $N fetches, median of three: first tenth ${near} ms, last tenth ${far} ms" @@ -157,7 +192,7 @@ build() { # table, rows fetch_once() { # table, table-rows, count -> milliseconds for one pass local start end start=$(date +%s%N) - psql_run "$FORCE UPDATE $1 SET v = v + 1 WHERE id > $(( $2 - $3 ));" \ + psql_run "$FORCE SELECT sum(v), sum(length(t)) FROM $1 WHERE id > $(( $2 - $3 ));" \ >/dev/null 2>&1 end=$(date +%s%N) echo $(( (end - start) / 1000000 )) @@ -205,6 +240,9 @@ build fp_small $((ROWS / 2)) build fp_big "$ROWS" K=$((ROWS / 20)) +check "premise: the doubled group's reads fetch through its index" \ + "$(reads_index "SELECT sum(v), sum(length(t)) FROM fp_big WHERE id > $(( ROWS - K ))" fp_big_id)" "yes" + small="$(fetch_ms fp_small $((ROWS / 2)) $K)" big="$(fetch_ms fp_big "$ROWS" $K)" echo "-- $K fetches at the far end, median of three: group of $((ROWS / 2)) took ${small} ms, group of $ROWS took ${big} ms" diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index babd7df85..06ac2d401 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -212,6 +212,7 @@ behaviour, the source of that number is named. - [164. test_native_saop_pushdown.py: IN-lists and = ANY(array) prune row groups](#164-test_native_saop_pushdownpy-in-lists-and--anyarray-prune-row-groups) - [165. test_native_groupagg_batch.py: the grouped aggregate folds column-at-a-time](#165-test_native_groupagg_batchpy-the-grouped-aggregate-folds-column-at-a-time) - [166. test_native_fetch_projection.py: a fetch by row number decodes only what its caller needs](#166-test_native_fetch_projectionpy-a-fetch-by-row-number-decodes-only-what-its-caller-needs) +- [167. test_native_fetch_position.py: a fetch by row number costs the same anywhere in its group](#167-test_native_fetch_positionpy-a-fetch-by-row-number-costs-the-same-anywhere-in-its-group) ## 1. How to read a test in here @@ -7691,3 +7692,16 @@ The pytest twin of `test/native_fetch_projection.sh` (ported for #1341), every c | `test_a_projection_covering_every_column_decodes_nothing_and_is_right` | with the computed base-column set empty, every row is reconstructed with the right values | | `test_no_caller_decodes_every_column_and_index_deletion_reads_no_row` | the projection path calls no full decode, and index deletion asks no row whether it is live | | `test_every_needed_set_test_consults_the_all_columns_flag` | every needed-set membership test in the reader consults the explicit all-columns flag | + +## 167. test_native_fetch_position.py: a fetch by row number costs the same anywhere in its group + +The pytest twin of `test/native_fetch_position.sh` (ported for #1341), every check by the shell's name. A cached row group still walked to the row: it counted the validity bits below the row, then decoded and discarded that many values (#143). A rank prefix over the validity bitmap and an offset table for varying-length columns make each fetch O(1). Every fetch here runs with the columnar custom scan off, and each measurement first asserts that its plan reads the index. With only `enable_seqscan=off`, every query scanned instead and none fetched by row number. The timing ratios read the same ranges three times. An UPDATE would write its rows into a new group, so later passes would fetch elsewhere. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_every_fetched_value_matches_the_heap_mirror` | the group is one row group, and fixed, varying-length and heavily-null columns fetched through the index match a heap mirror, at every probed position | +| `test_the_cost_does_not_depend_on_the_position` | fetching the last twentieth of the group costs under three times the first | +| `test_a_bigger_group_does_not_cost_more_per_fetch` | the same far-end fetches in a group twice the size cost under 1.8 times as much | +| `test_the_indexes_that_make_it_constant_are_in_the_fetch` | `pgcolumnar_fetch_row` takes the rank from the prefix and a varying-length value from the offset table | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 1df468344..655419001 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -795,4 +795,6 @@ guard_tests 438 # eight tests, by collection over the cluster files on this tree. # cluster_tests 718 -> 722 for #1341's test_native_fetch_projection.py, the twin of native_fetch_projection.sh: # four tests, by collection over the cluster files on this tree. -cluster_tests 722 +# cluster_tests 722 -> 726 for #1341's test_native_fetch_position.py, the twin of native_fetch_position.sh: +# four tests, by collection over the cluster files on this tree. +cluster_tests 726 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 4c323a6c0..3db54a5c3 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -130,6 +130,7 @@ "native_dml", "native_encdesc_golden", "native_fetch_coalesce", + "native_fetch_position", "native_fetch_projection", "native_groupagg", "native_groupagg_batch", diff --git a/test/pytest/test_native_fetch_position.py b/test/pytest/test_native_fetch_position.py new file mode 100644 index 000000000..f3b2bece3 --- /dev/null +++ b/test/pytest/test_native_fetch_position.py @@ -0,0 +1,166 @@ +"""A fetch by row number costs the same wherever the row sits in its group (#143, #1341). +The pytest twin of `test/native_fetch_position.sh`, every check by the shell's name. + +A cached group still walked to the row: counting the validity bits below it, then +decoding and discarding that many values, so a fetch cost O(row) and fetching a whole +group was quadratic. A rank prefix over the validity bitmap and an offset table for +varying-length columns make it O(1). Values first, against a heap mirror over fixed, +varying-length and heavily-null columns, through an index scan; then two timing ratios +(far end against near end, and a doubled group), each the median of three passes; then +the two indexes in pgcolumnar_fetch_row's source. +""" +import os +import pathlib +import re +import statistics +import time + +import psycopg +import pytest + +# the shell reads PGC_FETCHPOS_ROWS, and so does this +ROWS = int(os.environ.get("PGC_FETCHPOS_ROWS", "120000")) +SRC = pathlib.Path(__file__).resolve().parents[2] / "src" +# enable_seqscan=off leaves the columnar custom scan on, and with only that every query +# here scanned rather than fetched by row number; each measurement asserts its plan +FORCE = ("SET enable_seqscan = off", "SET enable_bitmapscan = off", "SET max_parallel_workers_per_gather = 0", + "SET pgcolumnar.enable_custom_scan = off") + + +def _one(c, sql): + return c.execute(sql).fetchone()[0] + + +def _forced(c): + for s in FORCE: + c.execute(s) + + +def _reads_index(c, sql, index): + """-> 'yes' when `sql`'s plan is an Index Scan using `index`, else its first scan line.""" + plan = "\n".join(r[0] for r in c.execute(f"EXPLAIN (COSTS OFF) {sql}")) + if f"Index Scan using {index} " in plan: + return "yes" + first = next((l.strip() for l in plan.splitlines() if "Scan" in l or "Update" in l), plan) + return f"no: {first}" + + +def _rows_or_error(c, sql): + """-> the first value, or the error as a value: a corrupt fetch is a named red.""" + try: + return _one(c, sql) + except psycopg.Error as exc: + return f"error: {exc.diag.message_primary}" + + +def _median_ms(c, sql): + """-> the median of three timed passes of `sql`, in milliseconds.""" + times = [] + for _ in range(3): + t0 = time.monotonic() + c.execute(sql) + times.append((time.monotonic() - t0) * 1000.0) + return statistics.median(times) + + +@pytest.fixture(scope="module") +def fpos(pgc_cluster): + """-> a connection to a database holding fp_c (one row group) and its heap mirror.""" + db = "pgc_fetchpos" + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + a.execute(f"CREATE DATABASE {db}") + try: + with psycopg.connect(pgc_cluster.dsn(dbname=db), autocommit=True) as c: + c.execute("CREATE EXTENSION pgcolumnar") + # every fourth row null in the nullable columns, so rank and row number diverge + c.execute(f"SET pgcolumnar.stripe_row_limit = {ROWS}") + c.execute(f"SET pgcolumnar.chunk_group_row_limit = {ROWS}") + c.execute("CREATE TABLE fp_c (id int, fixed bigint, txt text, sparse int, stxt text) USING pgcolumnar") + c.execute(f"""INSERT INTO fp_c + SELECT g, g * 7, repeat('x', 1 + (g % 40)) || g, + CASE WHEN g % 4 = 0 THEN NULL ELSE g END, + CASE WHEN g % 4 = 0 THEN NULL ELSE 's' || g END + FROM generate_series(1, {ROWS}) g""") + c.execute("CREATE INDEX fp_c_id ON fp_c (id)") + c.execute("CREATE TABLE fp_h (LIKE fp_c)") + c.execute("INSERT INTO fp_h SELECT * FROM fp_c") + yield c + finally: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + + +def test_every_fetched_value_matches_the_heap_mirror(fpos, expect): + c = fpos + expect.num(_one(c, "SELECT count(*) FROM pgcolumnar.row_group r " + "JOIN pgcolumnar.storage s ON s.storage_id = r.storage_id " + "WHERE s.relation_oid = 'fp_c'::regclass"), 1, "the rows are in one row group") + _forced(c) + expect.text(_reads_index(c, "SELECT count(*) FROM fp_c c JOIN fp_h h USING (id) " + "WHERE c.fixed IS DISTINCT FROM h.fixed", "fp_c_id"), "yes", + "premise: the mirror comparison fetches through the index") + expect.text(_reads_index(c, "SELECT fixed FROM fp_c WHERE id = 1001", "fp_c_id"), "yes", + "premise: a single-row fetch goes through the index") + expect.text(str(_rows_or_error(c, """SELECT count(*) FROM fp_c c JOIN fp_h h USING (id) + WHERE c.fixed IS DISTINCT FROM h.fixed + OR c.txt IS DISTINCT FROM h.txt + OR c.sparse IS DISTINCT FROM h.sparse + OR c.stxt IS DISTINCT FROM h.stxt""")), "0", "every column matches the heap mirror") + # one row at a time, where an off-by-one in the rank would land on a neighbour + row = "fixed || '|' || txt || '|' || coalesce(sparse::text, 'N') || '|' || coalesce(stxt, 'N')" + bad = [] + for pos in (1, 2, 3, 4, 5, 999, 1000, 1001, ROWS // 2, ROWS - 1, ROWS): + got = _rows_or_error(c, f"SELECT {row} FROM fp_c WHERE id = {pos}") + want = _one(c, f"SELECT {row} FROM fp_h WHERE id = {pos}") + if got != want: + bad.append(f"id={pos}(got={got} want={want})") + expect.text(" ".join(bad) or "same", "same", "single-row fetches are right at every position") + + +def test_the_cost_does_not_depend_on_the_position(fpos, expect): + c = fpos + _forced(c) + n = ROWS // 20 + # a read, not an UPDATE: an UPDATE writes its rows into a new group, so only the first + # pass fetched from the far end and the median was always a later one + read = "SELECT sum(fixed), sum(length(txt)) FROM fp_c WHERE id BETWEEN {} AND {}" + expect.text(_reads_index(c, read.format(ROWS - n + 1, ROWS), "fp_c_id"), "yes", + "premise: the timed reads fetch through the index") + near = _median_ms(c, read.format(1, n)) + far = _median_ms(c, read.format(ROWS - n + 1, ROWS)) + print(f"-- {n} fetches, median of three: first tenth {near:.0f} ms, last tenth {far:.0f} ms") + # a ratio of 3 catches the quadratic walk (1,001,374 ms against 614 ms) with orders to spare + expect.text("yes" if near > 0 and far / near < 3 else f"no (near={near:.0f}ms far={far:.0f}ms)", "yes", + "fetching from the far end of a group costs about what the near end does") + + +def test_a_bigger_group_does_not_cost_more_per_fetch(fpos, expect): + c = fpos + for name, rows in (("fp_small", ROWS // 2), ("fp_big", ROWS)): + c.execute(f"SET pgcolumnar.stripe_row_limit = {rows}") + c.execute(f"SET pgcolumnar.chunk_group_row_limit = {rows}") + c.execute(f"CREATE TABLE {name} (id int, v bigint, t text) USING pgcolumnar") + c.execute(f"INSERT INTO {name} SELECT g, g * 3, 't' || g FROM generate_series(1, {rows}) g") + c.execute(f"CREATE INDEX {name}_id ON {name} (id)") + _forced(c) + # the rows fetched sit at the END of each group, where the old walk was longest + k = ROWS // 20 + read = "SELECT sum(v), sum(length(t)) FROM {} WHERE id > {}" + expect.text(_reads_index(c, read.format("fp_big", ROWS - k), "fp_big_id"), "yes", + "premise: the doubled group's reads fetch through its index") + small = _median_ms(c, read.format("fp_small", ROWS // 2 - k)) + big = _median_ms(c, read.format("fp_big", ROWS - k)) + print(f"-- {k} fetches at the far end, median of three: {ROWS // 2} rows took {small:.0f} ms, {ROWS} took {big:.0f} ms") + expect.text("yes" if small > 0 and big / small < 1.8 else f"no (small={small:.0f}ms big={big:.0f}ms)", "yes", + "doubling the row group does not double the cost of the same fetches") + + +def test_the_indexes_that_make_it_constant_are_in_the_fetch(expect): + m = re.search(r"^pgcolumnar_fetch_row\(.*?^}", (SRC / "columnar_reader.c").read_text(), re.S | re.M) + fetch = m.group(0) if m else "" + expect.text("yes" if fetch else "no", "yes", "premise: pgcolumnar_fetch_row was extracted, not an empty range") + expect.text("yes" if "present = pgcolumnar_rank_before" in fetch else "no", "yes", + "the rank comes from a prefix rather than a loop over earlier rows") + expect.text("yes" if "entry->valOffset[c][present]" in fetch else "no", "yes", + "a varying-length column reaches its value through an offset table") From 0adaf345a5e34ec08000d77de10c7755ee1be2d7 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Fri, 16 Oct 2026 17:07:07 +0000 Subject: [PATCH 2/3] test: native_groupagg_batch dates three heap-mirror arms by a NULL-key mutation from #1485's review (#1341) jdatcmd's mutation drops the NULL-key flag in the grouped fold's gather, so the NULL group folds into k = 0 on the batch path only. Nine checks redden, and the twin reddens the same nine by name. Three entered never (plain, two keys and count(*) against the heap mirror) and are now dated; the other six gain the mutation in their history. Census 4189 rows / 2969 never / 1220 dated. Co-Authored-By: Claude Opus 5.5 --- test/check_ledger.tsv | 18 +++++++++--------- test/check_ledger_budget.txt | 14 +++++++++++++- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 738978f03..ae2ccd49c 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2595,17 +2595,17 @@ native_groupagg_batch native_groupagg_batch <> filter: the fold declined 15;16;1 native_groupagg_batch native_groupagg_batch add column: ANALYZE reports the row-path run 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped node's ANALYZE prints the predicted fold, not what ran (the #602 shape) native_groupagg_batch native_groupagg_batch add column: plain EXPLAIN reports the prediction 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch add column: the answer is still right 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch count(*): answers match the heap mirror 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch count(*): answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch count(*): the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch count(*): the grouped node is planned 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch date key: answers match the heap mirror 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch date key: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch date key: the grouped node is planned 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch deletes: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch deletes: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask;columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch deletes: the fold still ran 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch expression key: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: an expression key's -1 attidx is not refused by the grouped fold's eligibility native_groupagg_batch native_groupagg_batch expression key: the fold declined 15;16;17;18;19 2026-10-16 columnar_vector.c: an expression key's -1 attidx is not refused by the grouped fold's eligibility -native_groupagg_batch native_groupagg_batch filtered: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold applies no scan key per row +native_groupagg_batch native_groupagg_batch filtered: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold applies no scan key per row;columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch filtered: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch filtered: the grouped node is planned 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch float8 key: answers match the heap mirror 15;16;17;18;19 never - @@ -2619,13 +2619,13 @@ native_groupagg_batch native_groupagg_batch float8 zero: the grouped node is pla native_groupagg_batch native_groupagg_batch float8 zero: the grouped sums match the heap 15;16;17;18;19 2026-10-16 columnar_vector.c: float8 admitted as a simple key, so its group keys are hashed and compared by bits native_groupagg_batch native_groupagg_batch min/max: answers match the heap mirror 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch min/max: the fold declined 15;16;17;18;19 2026-10-16 columnar_vector.c: min and max admitted as batch fold kinds -native_groupagg_batch native_groupagg_batch parallel: and they equal the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: a partial grouped node folds without the shared row-group counter;columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch parallel: and they equal the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: a partial grouped node folds without the shared row-group counter;columnar_vector.c: the grouped fold ignores the delete mask;columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch parallel: our parallel-aware grouped node is under the Gather 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch parallel: premise: the value arm's own plan launches workers 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch parallel: the answers equal the serial fold, byte for byte 15;16;17;18;19 2026-10-16 columnar_vector.c: a partial grouped node folds without the shared row-group counter native_groupagg_batch native_groupagg_batch parallel: the fold ran in the partial node (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch parallel: workers actually launched for that node 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch plain: answers match the heap mirror 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch plain: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch plain: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch plain: the grouped node is planned 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch premise: fixture loaded every row (columnar) 15;16;17;18;19 never - @@ -2646,16 +2646,16 @@ native_groupagg_batch native_groupagg_batch text key: the fold declined 15;16;17 native_groupagg_batch native_groupagg_batch timestamp key: answers match the heap mirror 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch timestamp key: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch timestamp key: the grouped node is planned 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch toggle-differential: filtered 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold applies no scan key per row;columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch toggle-differential: filtered 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold applies no scan key per row;columnar_vector.c: the grouped fold ignores the delete mask;columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch toggle-differential: filtered [premise: node fires with the path on] 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch toggle-differential: filtered [premise: the off arm is not the grouped node] 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch toggle-differential: plain 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch toggle-differential: plain 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask;columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch toggle-differential: plain [premise: node fires with the path on] 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch toggle-differential: plain [premise: the off arm is not the grouped node] 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch toggle-differential: two keys 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch toggle-differential: two keys 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask;columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch toggle-differential: two keys [premise: node fires with the path on] 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch toggle-differential: two keys [premise: the off arm is not the grouped node] 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch two keys: answers match the heap mirror 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch two keys: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch two keys: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch two keys: the grouped node is planned 15;16;17;18;19 never - native_index_projection native_index_projection a parallel build projects one column of twenty, not all twenty 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index bc4355952..f4b32e393 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -3343,4 +3343,16 @@ suites_not_covered 199 # never 2988 - 16 = 2972 # dated 1201 + 16 = 1217 # 2972 + 1217 == 4189 -checks_never_observed_red 2972 +# +# #1341, native_groupagg_batch's NULL-key mutation (OffgridwithJD, from jdatcmd's +# #1485 review): 2972 -> 2969. The grouped fold's gather dropping the NULL-key +# flag folds the NULL group into k = 0 on the batch path only. Nine checks redden, +# shell and twin alike; three of them (plain, two keys and count(*) against the +# heap mirror) entered never and are now dated, and the other six gain the +# mutation in their history. No row is added or removed. +# +# ledger rows 4189 + 0 = 4189 +# never 2972 - 3 = 2969 +# dated 1217 + 3 = 1220 +# 2969 + 1220 == 4189 +checks_never_observed_red 2969 From fde6ab6076d09476762b09d1c48325d4dd314479 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Fri, 16 Oct 2026 17:07:07 +0000 Subject: [PATCH 3/3] test: native_fetch_position seeds the ledger, 6 of 12 arms dated by four rank and offset mutations (#1341) The suite had no rows. suites_not_covered 199 -> 198; census 4201 rows / 2975 never / 1226 dated. Co-Authored-By: Claude Opus 5.5 --- test/check_ledger.tsv | 12 ++++++++++++ test/check_ledger_budget.txt | 18 ++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index ae2ccd49c..264a6fd5a 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2477,6 +2477,18 @@ native_fetch_coalesce native_fetch_coalesce premise: fetching one projected colu native_fetch_coalesce native_fetch_coalesce premise: the coalescing helper holds both the bound and the validity copy 15;16;17;18;19 never - native_fetch_coalesce native_fetch_coalesce the validity copy is bounded by the chunk length before it runs 15;16;17;18;19 never - native_fetch_coalesce native_fetch_coalesce the wide fetch returns the projected values 15;16;17;18;19 2026-10-14 every column of a coalesced fetch is copied from the start of the span (off = 0) +native_fetch_position native_fetch_position a varying-length column reaches its value through an offset table 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row reaches a varying-length value by decoding every earlier one (the #143 walk) +native_fetch_position native_fetch_position doubling the row group does not double the cost of the same fetches 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row counts the validity bits from the group's start for every fetch (the #143 walk);columnar_reader.c: pgcolumnar_fetch_row reaches a varying-length value by decoding every earlier one (the #143 walk) +native_fetch_position native_fetch_position every column matches the heap mirror 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row takes the row number as the rank, counting rows rather than present values;columnar_reader.c: pgcolumnar_fetch_row's rank is one past the row's +native_fetch_position native_fetch_position fetching from the far end of a group costs about what the near end does 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row counts the validity bits from the group's start for every fetch (the #143 walk);columnar_reader.c: pgcolumnar_fetch_row reaches a varying-length value by decoding every earlier one (the #143 walk) +native_fetch_position native_fetch_position premise: a single-row fetch goes through the index 15;16;17;18;19 never - +native_fetch_position native_fetch_position premise: pgcolumnar_fetch_row was extracted, not an empty range 15;16;17;18;19 never - +native_fetch_position native_fetch_position premise: the doubled group's reads fetch through its index 15;16;17;18;19 never - +native_fetch_position native_fetch_position premise: the mirror comparison fetches through the index 15;16;17;18;19 never - +native_fetch_position native_fetch_position premise: the timed reads fetch through the index 15;16;17;18;19 never - +native_fetch_position native_fetch_position single-row fetches are right at every position 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row takes the row number as the rank, counting rows rather than present values;columnar_reader.c: pgcolumnar_fetch_row's rank is one past the row's +native_fetch_position native_fetch_position the rank comes from a prefix rather than a loop over earlier rows 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row counts the validity bits from the group's start for every fetch (the #143 walk);columnar_reader.c: pgcolumnar_fetch_row takes the row number as the rank, counting rows rather than present values +native_fetch_position native_fetch_position the rows are in one row group 15;16;17;18;19 never - native_fetch_projection native_fetch_projection a deleted row stops being read through the projection 15;16;17;18;19 2026-10-16 columnar_projection.c: read_projection ignores the base row's liveness native_fetch_projection native_fetch_projection an uncovered column still reads correctly after a delete 15;16;17;18;19 2026-10-16 columnar_projection.c: reconstruct's uncovered set misses the table's last column native_fetch_projection native_fetch_projection and does not decode the row to find out 15;16;17;18;19 2026-10-16 columnar_tableam.c: index deletion decodes each bottom-up row to decide diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index f4b32e393..9a0a36773 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 199 +suites_not_covered 198 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -3355,4 +3355,18 @@ suites_not_covered 199 # never 2972 - 3 = 2969 # dated 1217 + 3 = 1220 # 2969 + 1220 == 4189 -checks_never_observed_red 2969 +# +# #1341, native_fetch_position seeded, with its twin (OffgridwithJD): 2969 -> 2975, +# and suites_not_covered 199 -> 198. The suite had no rows. It enters with all 12 +# checks from five-major runs (12/12 on each of 15-19). 6 are dated by four +# mutations of pgcolumnar_fetch_row: the rank counting rows, the rank one too far, +# the rank walking the validity bits, and a varying-length value reached by +# decoding every earlier one. The two timing arms are dated by the walks only now +# that the fetches go through the index and the timed pass is a read. Six arrive +# never: the one-row-group fixture fact and five premises. +# +# ledger rows 4189 + 12 = 4201 +# never 2969 + 6 = 2975 +# dated 1220 + 6 = 1226 +# 2975 + 1226 == 4201 +checks_never_observed_red 2975