diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 738978f03..4db648adb 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2470,6 +2470,32 @@ native_encdesc_golden native_encdesc_golden int column a rawLen parses as 1200 ( native_encdesc_golden native_encdesc_golden the table reads back all rows 15;16;17;18;19 never - native_encdesc_golden native_encdesc_golden trailing sharedTableLen is 0 (u32 at offset 6+13=19) 15;16;17;18;19 never - native_encdesc_golden native_encdesc_golden vectorCount parses as 1 (at offset 2) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache a hit re-checks the group's fileOffset against the row group 15;16;17;18;19 2026-10-16 columnar_reader.c: a fetch-cache hit no longer re-checks fileOffset +native_fetch_cache native_fetch_cache a hit re-checks the group's firstRowNumber against the row group 15;16;17;18;19 2026-10-16 columnar_reader.c: a fetch-cache hit no longer re-checks firstRowNumber +native_fetch_cache native_fetch_cache a hit re-checks the group's natts against the scan's column count 15;16;17;18;19 2026-10-16 columnar_reader.c: a fetch-cache hit no longer re-checks natts +native_fetch_cache native_fetch_cache a hit re-checks the group's rowCount against the row group 15;16;17;18;19 2026-10-16 columnar_reader.c: a fetch-cache hit no longer re-checks rowCount +native_fetch_cache native_fetch_cache a wide group's fetch is not far dearer than a small group's (#353) 15;16;17;18;19 2026-10-16 columnar_reader.c: crossing the fetch-cache cap drops every column, not the one that crossed it;columnar_reader.c: every fetch-cache lookup misses, so each fetch decodes its group again +native_fetch_cache native_fetch_cache an entry from an earlier command is rejected 15;16;17;18;19 2026-10-16 columnar_reader.c: a fetch-cache entry from an earlier command is not rejected +native_fetch_cache native_fetch_cache crossing the fetch cache cap costs proportionally, not totally (#359) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache every fc_many reading measured ten row groups (#801) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache every fc_one reading measured one row group (#801) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache every group-number comparison is keyed by the storage id too 15;16;17;18;19 2026-10-16 columnar_reader.c: the fetch-cache key compares the group number without the storage id +native_fetch_cache native_fetch_cache every over-cap column reads back its written value (#359) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache fetching from one big group is not far dearer than from ten small ones 15;16;17;18;19 2026-10-16 columnar_reader.c: every fetch-cache lookup misses, so each fetch decodes its group again +native_fetch_cache native_fetch_cache premise: pgcolumnar_executor_end was extracted, not an empty range 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache premise: pgcolumnar_xact_callback was extracted, not an empty range 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache premise: the fetch-cache functions were extracted, not empty ranges 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache premise: there is a group-number comparison to key 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache the 4-column projection really drives the fetch path (#797) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache the 5-column projection really drives the fetch path (#797) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache the fc_wide point query really drives the fetch path (#797) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache the fc_wnarrow point query really drives the fetch path (#797) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache the fetch cache is discarded from pgcolumnar_executor_end 15;16;17;18;19 2026-10-16 columnar_tableam.c: pgcolumnar_executor_end no longer discards the fetch cache +native_fetch_cache native_fetch_cache the fetch cache is discarded from pgcolumnar_xact_callback 15;16;17;18;19 2026-10-16 columnar_tableam.c: the transaction callback no longer discards the fetch cache +native_fetch_cache native_fetch_cache the over-cap projection returns the same values as the under-cap one (#359) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache the untouched rows are unchanged 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache the updated rows are correct 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache the wide-group point query is correct (#353) 15;16;17;18;19 never - native_fetch_coalesce native_fetch_coalesce a wide index fetch does not pin once per column 15;16;17;18;19 2026-10-13 the coalesced fetch merging no adjacent ranges, so each column is read on its own native_fetch_coalesce native_fetch_coalesce premise: a point lookup uses the index 15;16;17;18;19 never - native_fetch_coalesce native_fetch_coalesce premise: fetching every projected column touched a measurable number of buffers 15;16;17;18;19 never - @@ -2477,6 +2503,18 @@ native_fetch_coalesce native_fetch_coalesce premise: fetching one projected colu native_fetch_coalesce native_fetch_coalesce premise: the coalescing helper holds both the bound and the validity copy 15;16;17;18;19 never - native_fetch_coalesce native_fetch_coalesce the validity copy is bounded by the chunk length before it runs 15;16;17;18;19 never - native_fetch_coalesce native_fetch_coalesce the wide fetch returns the projected values 15;16;17;18;19 2026-10-14 every column of a coalesced fetch is copied from the start of the span (off = 0) +native_fetch_position native_fetch_position a varying-length column reaches its value through an offset table 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row reaches a varying-length value by decoding every earlier one (the #143 walk) +native_fetch_position native_fetch_position doubling the row group does not double the cost of the same fetches 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row counts the validity bits from the group's start for every fetch (the #143 walk);columnar_reader.c: pgcolumnar_fetch_row reaches a varying-length value by decoding every earlier one (the #143 walk) +native_fetch_position native_fetch_position every column matches the heap mirror 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row takes the row number as the rank, counting rows rather than present values;columnar_reader.c: pgcolumnar_fetch_row's rank is one past the row's +native_fetch_position native_fetch_position fetching from the far end of a group costs about what the near end does 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row counts the validity bits from the group's start for every fetch (the #143 walk);columnar_reader.c: pgcolumnar_fetch_row reaches a varying-length value by decoding every earlier one (the #143 walk) +native_fetch_position native_fetch_position premise: a single-row fetch goes through the index 15;16;17;18;19 never - +native_fetch_position native_fetch_position premise: pgcolumnar_fetch_row was extracted, not an empty range 15;16;17;18;19 never - +native_fetch_position native_fetch_position premise: the doubled group's reads fetch through its index 15;16;17;18;19 never - +native_fetch_position native_fetch_position premise: the mirror comparison fetches through the index 15;16;17;18;19 never - +native_fetch_position native_fetch_position premise: the timed reads fetch through the index 15;16;17;18;19 never - +native_fetch_position native_fetch_position single-row fetches are right at every position 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row takes the row number as the rank, counting rows rather than present values;columnar_reader.c: pgcolumnar_fetch_row's rank is one past the row's +native_fetch_position native_fetch_position the rank comes from a prefix rather than a loop over earlier rows 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row counts the validity bits from the group's start for every fetch (the #143 walk);columnar_reader.c: pgcolumnar_fetch_row takes the row number as the rank, counting rows rather than present values +native_fetch_position native_fetch_position the rows are in one row group 15;16;17;18;19 never - native_fetch_projection native_fetch_projection a deleted row stops being read through the projection 15;16;17;18;19 2026-10-16 columnar_projection.c: read_projection ignores the base row's liveness native_fetch_projection native_fetch_projection an uncovered column still reads correctly after a delete 15;16;17;18;19 2026-10-16 columnar_projection.c: reconstruct's uncovered set misses the table's last column native_fetch_projection native_fetch_projection and does not decode the row to find out 15;16;17;18;19 2026-10-16 columnar_tableam.c: index deletion decodes each bottom-up row to decide @@ -2595,17 +2633,17 @@ native_groupagg_batch native_groupagg_batch <> filter: the fold declined 15;16;1 native_groupagg_batch native_groupagg_batch add column: ANALYZE reports the row-path run 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped node's ANALYZE prints the predicted fold, not what ran (the #602 shape) native_groupagg_batch native_groupagg_batch add column: plain EXPLAIN reports the prediction 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch add column: the answer is still right 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch count(*): answers match the heap mirror 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch count(*): answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch count(*): the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch count(*): the grouped node is planned 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch date key: answers match the heap mirror 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch date key: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch date key: the grouped node is planned 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch deletes: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch deletes: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask;columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch deletes: the fold still ran 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch expression key: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: an expression key's -1 attidx is not refused by the grouped fold's eligibility native_groupagg_batch native_groupagg_batch expression key: the fold declined 15;16;17;18;19 2026-10-16 columnar_vector.c: an expression key's -1 attidx is not refused by the grouped fold's eligibility -native_groupagg_batch native_groupagg_batch filtered: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold applies no scan key per row +native_groupagg_batch native_groupagg_batch filtered: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold applies no scan key per row;columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch filtered: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch filtered: the grouped node is planned 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch float8 key: answers match the heap mirror 15;16;17;18;19 never - @@ -2619,13 +2657,13 @@ native_groupagg_batch native_groupagg_batch float8 zero: the grouped node is pla native_groupagg_batch native_groupagg_batch float8 zero: the grouped sums match the heap 15;16;17;18;19 2026-10-16 columnar_vector.c: float8 admitted as a simple key, so its group keys are hashed and compared by bits native_groupagg_batch native_groupagg_batch min/max: answers match the heap mirror 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch min/max: the fold declined 15;16;17;18;19 2026-10-16 columnar_vector.c: min and max admitted as batch fold kinds -native_groupagg_batch native_groupagg_batch parallel: and they equal the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: a partial grouped node folds without the shared row-group counter;columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch parallel: and they equal the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: a partial grouped node folds without the shared row-group counter;columnar_vector.c: the grouped fold ignores the delete mask;columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch parallel: our parallel-aware grouped node is under the Gather 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch parallel: premise: the value arm's own plan launches workers 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch parallel: the answers equal the serial fold, byte for byte 15;16;17;18;19 2026-10-16 columnar_vector.c: a partial grouped node folds without the shared row-group counter native_groupagg_batch native_groupagg_batch parallel: the fold ran in the partial node (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch parallel: workers actually launched for that node 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch plain: answers match the heap mirror 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch plain: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch plain: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch plain: the grouped node is planned 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch premise: fixture loaded every row (columnar) 15;16;17;18;19 never - @@ -2646,16 +2684,16 @@ native_groupagg_batch native_groupagg_batch text key: the fold declined 15;16;17 native_groupagg_batch native_groupagg_batch timestamp key: answers match the heap mirror 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch timestamp key: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch timestamp key: the grouped node is planned 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch toggle-differential: filtered 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold applies no scan key per row;columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch toggle-differential: filtered 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold applies no scan key per row;columnar_vector.c: the grouped fold ignores the delete mask;columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch toggle-differential: filtered [premise: node fires with the path on] 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch toggle-differential: filtered [premise: the off arm is not the grouped node] 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch toggle-differential: plain 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch toggle-differential: plain 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask;columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch toggle-differential: plain [premise: node fires with the path on] 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch toggle-differential: plain [premise: the off arm is not the grouped node] 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch toggle-differential: two keys 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch toggle-differential: two keys 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask;columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch toggle-differential: two keys [premise: node fires with the path on] 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch toggle-differential: two keys [premise: the off arm is not the grouped node] 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch two keys: answers match the heap mirror 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch two keys: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch two keys: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch two keys: the grouped node is planned 15;16;17;18;19 never - native_index_projection native_index_projection a parallel build projects one column of twenty, not all twenty 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index bc4355952..1da87870b 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 199 +suites_not_covered 197 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -3343,4 +3343,46 @@ suites_not_covered 199 # never 2988 - 16 = 2972 # dated 1201 + 16 = 1217 # 2972 + 1217 == 4189 -checks_never_observed_red 2972 +# +# #1341, native_groupagg_batch's NULL-key mutation (OffgridwithJD, from jdatcmd's +# #1485 review): 2972 -> 2969. The grouped fold's gather dropping the NULL-key +# flag folds the NULL group into k = 0 on the batch path only. Nine checks redden, +# shell and twin alike; three of them (plain, two keys and count(*) against the +# heap mirror) entered never and are now dated, and the other six gain the +# mutation in their history. No row is added or removed. +# +# ledger rows 4189 + 0 = 4189 +# never 2972 - 3 = 2969 +# dated 1217 + 3 = 1220 +# 2969 + 1220 == 4189 +# +# #1341, native_fetch_position seeded, with its twin (OffgridwithJD): 2969 -> 2975, +# and suites_not_covered 199 -> 198. The suite had no rows. It enters with all 12 +# checks from five-major runs (12/12 on each of 15-19). 6 are dated by four +# mutations of pgcolumnar_fetch_row: the rank counting rows, the rank one too far, +# the rank walking the validity bits, and a varying-length value reached by +# decoding every earlier one. The two timing arms are dated by the walks only now +# that the fetches go through the index and the timed pass is a read. Six arrive +# never: the one-row-group fixture fact and five premises. +# +# ledger rows 4189 + 12 = 4201 +# never 2969 + 6 = 2975 +# dated 1220 + 6 = 1226 +# 2975 + 1226 == 4201 +# +# #1341, native_fetch_cache seeded, with its twin (OffgridwithJD): 2975 -> 2991, +# and suites_not_covered 198 -> 197. The suite had no rows. It enters with all 26 +# checks from five-major runs (26/26 on each of 15-19). 10 are dated by ten +# mutations: the cache never hitting, the key without the storage id, no +# earlier-command reject, each of the four recheck fields dropped, the discard +# dropped from executor end and from transaction end, and the cap dropping every +# column. Sixteen arrive never: four premises and the four #797 plan premises; +# the two #801 geometry records; the four correctness arms; and the #359 ratio, +# which the cap-drops-every-column mutation cannot redden on this fixture (it +# slows one projected column as much as five, measured 46 -> 286 ms in-suite). +# +# ledger rows 4201 + 26 = 4227 +# never 2975 + 16 = 2991 +# dated 1226 + 10 = 1236 +# 2991 + 1236 == 4227 +checks_never_observed_red 2991 diff --git a/test/native_fetch_cache.sh b/test/native_fetch_cache.sh index e936e0509..bd8531f9a 100755 --- a/test/native_fetch_cache.sh +++ b/test/native_fetch_cache.sh @@ -48,7 +48,12 @@ build() { # table, rows-per-group CREATE INDEX ${1}_id ON $1 (id);" } -# index-driven so every row goes through the fetch path rather than a scan +# Every updated row goes through the fetch path whatever the scan plans: the +# UPDATE fetches its target rows by number. Measured (#1341) on PG18: this +# UPDATE plans as Custom Scan (PgColumnarScan) under these SETs, and with the +# cache made to miss on every lookup the ratio below still goes red (844 ms +# against 93 ms). So the SETs are not what makes it a fetch, and no plan +# premise is needed here, unlike the point queries further down. upd_ms() { local start end start=$(date +%s%N) diff --git a/test/native_fetch_position.sh b/test/native_fetch_position.sh index 9e860d8e4..18adc4f98 100755 --- a/test/native_fetch_position.sh +++ b/test/native_fetch_position.sh @@ -65,8 +65,32 @@ check "the rows are in one row group" \ # Fetch by row number is what an index scan drives, so force one. Each leading # SET prints its own line ahead of the result, so these queries take the last. +# +# enable_seqscan=off does NOT turn the columnar custom scan off, and it was all +# this suite set: measured, the single-row fetch, the near and far UPDATEs and +# the mirror join all planned as Custom Scan (PgColumnarScan), so none of them +# fetched by row number. A rank off by one left the single-row arm green, and a +# walk over every earlier row left both timing arms green. The custom scan is +# off now, and each measurement asserts first that its plan reads the index. FORCE="SET enable_seqscan = off; SET enable_bitmapscan = off; - SET max_parallel_workers_per_gather = 0;" + SET max_parallel_workers_per_gather = 0; + SET pgcolumnar.enable_custom_scan = off;" + +# reads_index QUERY INDEX -> yes, or the plan's first line: the premise that a +# query fetches by row number through INDEX rather than scanning +reads_index() { + local plan + plan="$(q "$FORCE EXPLAIN (COSTS OFF) $1")" + case "$plan" in + *"Index Scan using $2 "*) echo yes ;; + *) echo "no: $(grep -m1 -E 'Scan|Update' <<<"$plan" | sed 's/^ *//')" ;; + esac +} + +check "premise: the mirror comparison fetches through the index" \ + "$(reads_index "SELECT count(*) FROM fp_c c JOIN fp_h h USING (id) WHERE c.fixed IS DISTINCT FROM h.fixed" fp_c_id)" "yes" +check "premise: a single-row fetch goes through the index" \ + "$(reads_index "SELECT fixed FROM fp_c WHERE id = 1001" fp_c_id)" "yes" # rows spread across the whole group, including the far end where the old walk # was longest @@ -103,7 +127,7 @@ FAR_HI=$ROWS upd_once() { # id-low, id-high -> milliseconds for one pass local start end start=$(date +%s%N) - psql_run "$FORCE UPDATE fp_c SET fixed = fixed + 1 + psql_run "$FORCE SELECT sum(fixed), sum(length(txt)) FROM fp_c WHERE id BETWEEN $1 AND $2;" >/dev/null 2>&1 end=$(date +%s%N) echo $(( (end - start) / 1000000 )) @@ -130,6 +154,17 @@ upd_ms() { # id-low, id-high -> median of three passes printf '%s\n%s\n%s\n' "$a" "$b" "$c" | sort -n | sed -n 2p } +# The timed pass is a SELECT, not the UPDATE it was. A columnar UPDATE writes +# each row again into a new row group, so only the first of three passes +# fetched from the far end of the big group; the second and third fetched from +# the start of the new one, and the median was always one of those. Measured +# with the rank walking every earlier row: the UPDATE form read near 122 ms, +# far 132 ms, green. A read leaves every row where it was, so all three passes +# fetch the same positions. sum(length(txt)) makes it decode a varying-length +# column, which the offset table serves. +check "premise: the timed reads fetch through the index" \ + "$(reads_index "SELECT sum(fixed), sum(length(txt)) FROM fp_c WHERE id BETWEEN $FAR_LO AND $FAR_HI" fp_c_id)" "yes" + near="$(upd_ms $NEAR_LO $NEAR_HI)" far="$(upd_ms $FAR_LO $FAR_HI)" echo "-- $N fetches, median of three: first tenth ${near} ms, last tenth ${far} ms" @@ -157,7 +192,7 @@ build() { # table, rows fetch_once() { # table, table-rows, count -> milliseconds for one pass local start end start=$(date +%s%N) - psql_run "$FORCE UPDATE $1 SET v = v + 1 WHERE id > $(( $2 - $3 ));" \ + psql_run "$FORCE SELECT sum(v), sum(length(t)) FROM $1 WHERE id > $(( $2 - $3 ));" \ >/dev/null 2>&1 end=$(date +%s%N) echo $(( (end - start) / 1000000 )) @@ -205,6 +240,9 @@ build fp_small $((ROWS / 2)) build fp_big "$ROWS" K=$((ROWS / 20)) +check "premise: the doubled group's reads fetch through its index" \ + "$(reads_index "SELECT sum(v), sum(length(t)) FROM fp_big WHERE id > $(( ROWS - K ))" fp_big_id)" "yes" + small="$(fetch_ms fp_small $((ROWS / 2)) $K)" big="$(fetch_ms fp_big "$ROWS" $K)" echo "-- $K fetches at the far end, median of three: group of $((ROWS / 2)) took ${small} ms, group of $ROWS took ${big} ms" diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index babd7df85..04d6e3e0e 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -212,6 +212,8 @@ behaviour, the source of that number is named. - [164. test_native_saop_pushdown.py: IN-lists and = ANY(array) prune row groups](#164-test_native_saop_pushdownpy-in-lists-and--anyarray-prune-row-groups) - [165. test_native_groupagg_batch.py: the grouped aggregate folds column-at-a-time](#165-test_native_groupagg_batchpy-the-grouped-aggregate-folds-column-at-a-time) - [166. test_native_fetch_projection.py: a fetch by row number decodes only what its caller needs](#166-test_native_fetch_projectionpy-a-fetch-by-row-number-decodes-only-what-its-caller-needs) +- [167. test_native_fetch_position.py: a fetch by row number costs the same anywhere in its group](#167-test_native_fetch_positionpy-a-fetch-by-row-number-costs-the-same-anywhere-in-its-group) +- [168. test_native_fetch_cache.py: the fetch-by-row-number cache](#168-test_native_fetch_cachepy-the-fetch-by-row-number-cache) ## 1. How to read a test in here @@ -7691,3 +7693,29 @@ The pytest twin of `test/native_fetch_projection.sh` (ported for #1341), every c | `test_a_projection_covering_every_column_decodes_nothing_and_is_right` | with the computed base-column set empty, every row is reconstructed with the right values | | `test_no_caller_decodes_every_column_and_index_deletion_reads_no_row` | the projection path calls no full decode, and index deletion asks no row whether it is live | | `test_every_needed_set_test_consults_the_all_columns_flag` | every needed-set membership test in the reader consults the explicit all-columns flag | + +## 167. test_native_fetch_position.py: a fetch by row number costs the same anywhere in its group + +The pytest twin of `test/native_fetch_position.sh` (ported for #1341), every check by the shell's name. A cached row group still walked to the row: it counted the validity bits below the row, then decoded and discarded that many values (#143). A rank prefix over the validity bitmap and an offset table for varying-length columns make each fetch O(1). Every fetch here runs with the columnar custom scan off, and each measurement first asserts that its plan reads the index. With only `enable_seqscan=off`, every query scanned instead and none fetched by row number. The timing ratios read the same ranges three times. An UPDATE would write its rows into a new group, so later passes would fetch elsewhere. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_every_fetched_value_matches_the_heap_mirror` | the group is one row group, and fixed, varying-length and heavily-null columns fetched through the index match a heap mirror, at every probed position | +| `test_the_cost_does_not_depend_on_the_position` | fetching the last twentieth of the group costs under three times the first | +| `test_a_bigger_group_does_not_cost_more_per_fetch` | the same far-end fetches in a group twice the size cost under 1.8 times as much | +| `test_the_indexes_that_make_it_constant_are_in_the_fetch` | `pgcolumnar_fetch_row` takes the rank from the prefix and a varying-length value from the offset table | + +## 168. test_native_fetch_cache.py: the fetch-by-row-number cache + +The pytest twin of `test/native_fetch_cache.sh` (ported for #1341), every check by the shell's name. A fetch by row number decoded a whole row group per row (#143), and a statement-scoped cache of the decoded group removes the repeat. The cache is timed as a ratio: the same index fetches from one big row group and from ten small ones. Each reading runs on a freshly built fixture, because an UPDATE moves the rows it reads into a new group (#801). Then come the scoping guards in the source, a wide group's decode scratch (#353), and crossing the cache cap (#359). + +### Every test + +| test | what it holds | +| --- | --- | +| `test_the_cache_makes_one_big_group_cost_what_ten_small_ones_do` | each reading measured the geometry it meant to, one big group costs under three times ten small ones, and the updated rows are right | +| `test_the_scoping_guards_are_in_the_source` | the cache key includes the storage id, an earlier command's entry is rejected, a hit re-checks four fields, and the cache is discarded at executor end and transaction end | +| `test_a_wide_groups_decode_scratch_does_not_blow_the_cap` | a wide group's point query drives the fetch path, costs under five times a small group's, and answers the same | +| `test_crossing_the_cap_costs_per_column` | four and five projected columns drive the fetch path, five costs under twelve times four, and every over-cap column reads back its value | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 1df468344..f15994f2d 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -795,4 +795,8 @@ guard_tests 438 # eight tests, by collection over the cluster files on this tree. # cluster_tests 718 -> 722 for #1341's test_native_fetch_projection.py, the twin of native_fetch_projection.sh: # four tests, by collection over the cluster files on this tree. -cluster_tests 722 +# cluster_tests 722 -> 726 for #1341's test_native_fetch_position.py, the twin of native_fetch_position.sh: +# four tests, by collection over the cluster files on this tree. +# cluster_tests 726 -> 730 for #1341's test_native_fetch_cache.py, the twin of native_fetch_cache.sh: +# four tests, by collection over the cluster files on this tree. +cluster_tests 730 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 4c323a6c0..803b82bc6 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -129,7 +129,9 @@ "native_dict_underfill", "native_dml", "native_encdesc_golden", + "native_fetch_cache", "native_fetch_coalesce", + "native_fetch_position", "native_fetch_projection", "native_groupagg", "native_groupagg_batch", diff --git a/test/pytest/test_native_fetch_cache.py b/test/pytest/test_native_fetch_cache.py new file mode 100644 index 000000000..5b55beaf2 --- /dev/null +++ b/test/pytest/test_native_fetch_cache.py @@ -0,0 +1,207 @@ +"""The fetch-by-row-number cache (#143, #353, #359, #1341). The pytest twin of +`test/native_fetch_cache.sh`, every check by the shell's name. + +A fetch by row number decoded a whole row group per row; a statement-scoped cache of +the decoded group removes the repeat. The cache is timed as a ratio: the same index +fetches from one big row group and from ten small ones, each reading on a freshly +built fixture because an UPDATE moves the rows it reads into a new group (#801). The +UPDATE fetches its rows by number whatever its scan; every timed point query asserts +first that its plan is an index scan, under the SETs that make one (#797). Then the +scoping guards +in the source, a wide group's decode scratch (#353), and crossing the cache cap +costing per column rather than per entry (#359). +""" +import os +import pathlib +import re +import time + +import psycopg +import pytest + +ROWS = int(os.environ.get("PGC_FETCH_ROWS", "20000")) +UPD = ROWS // 10 +SRC = pathlib.Path(__file__).resolve().parents[2] / "src" +FETCH_PATH = ("max_parallel_workers_per_gather=0", "enable_seqscan=off", "enable_bitmapscan=off", + "pgcolumnar.enable_index_fetch_penalty=off") + + +def _one(c, sql): + return c.execute(sql).fetchone()[0] + + +class _Sets: + def __init__(self, c, sets): + self.c, self.sets = c, sets + + def __enter__(self): + for s in self.sets: + self.c.execute(f"SET {s}") + + def __exit__(self, *exc): + for s in self.sets: + self.c.execute(f"RESET {s.split('=')[0]}") + + +def _row_fetch(c, sql): + """-> 'yes' when `sql` plans an Index Scan under the fetch-path SETs, as pgc_uses_row_fetch.""" + with _Sets(c, FETCH_PATH): + plan = "\n".join(r[0] for r in c.execute(f"EXPLAIN (COSTS OFF) {sql}")) + return "yes" if "Index Scan using" in plan else "no" + + +def _ms(c, sql, sets=FETCH_PATH): + with _Sets(c, sets): + t0 = time.monotonic() + c.execute(sql) + return int((time.monotonic() - t0) * 1000) + + +def _ratio(a, b, bound): + """-> 'within' when a/b, rounded to two places, is at most `bound`; else why not, + as check_ratio words it: a zero side measured nothing.""" + if a == 0 or b == 0: + return f"a side of the ratio is zero, so nothing was measured: a=[{a}] b=[{b}]" + r = round(a / b, 2) + return "within" if r <= bound else f"{r:.2f}x exceeds the {bound}x bound (a={a} b={b})" + + +def _groups(c, table): + return _one(c, f"SELECT count(*) FROM pgcolumnar.row_group r JOIN pgcolumnar.storage s USING (storage_id) " + f"WHERE s.relation_oid = '{table}'::regclass") + + +def _build(c, table, per_group): + c.execute(f"DROP TABLE IF EXISTS {table}") + c.execute(f"SET pgcolumnar.stripe_row_limit = {per_group}") + c.execute(f"SET pgcolumnar.chunk_group_row_limit = {per_group}") + try: + c.execute(f"CREATE TABLE {table} (id int, v int, t text) USING pgcolumnar") + c.execute(f"INSERT INTO {table} SELECT g, g, 'row' || g FROM generate_series(1, {ROWS}) g") + c.execute(f"CREATE INDEX {table}_id ON {table} (id)") + finally: + c.execute("RESET pgcolumnar.stripe_row_limit") + c.execute("RESET pgcolumnar.chunk_group_row_limit") + + +def _fn(path, name): + m = re.search(rf"^{name}\(.*?^}}", (SRC / path).read_text(), re.S | re.M) + return m.group(0) if m else "" + + +@pytest.fixture(scope="module") +def fc(pgc_cluster): + db = "pgc_fetchcache" + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + a.execute(f"CREATE DATABASE {db}") + try: + with psycopg.connect(pgc_cluster.dsn(dbname=db), autocommit=True) as c: + c.execute("CREATE EXTENSION pgcolumnar") + yield c + finally: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + + +def test_the_cache_makes_one_big_group_cost_what_ten_small_ones_do(fc, expect): + c = fc + # the shell's SETs for this arm; the UPDATE fetches its target rows by number anyway + upd_sets = ("max_parallel_workers_per_gather=0", "enable_seqscan=off", "enable_bitmapscan=off") + upd = f"UPDATE {{}} SET v = id + 1 WHERE id <= {UPD}" + # rebuilt before every reading: the UPDATE moves the rows it read into a new group (#801) + best, geom = {}, {} + for table, per in (("fc_one", ROWS), ("fc_many", ROWS // 10)): + times, seen = [], [] + for _ in range(3): + _build(c, table, per) + seen.append(str(_groups(c, table))) + times.append(_ms(c, upd.format(table), upd_sets)) + best[table], geom[table] = min(times), " ".join(seen) + print(f"-- {UPD} fetches: one group of {ROWS} took {best['fc_one']} ms; ten groups took {best['fc_many']} ms") + expect.text(geom["fc_one"], "1 1 1", "every fc_one reading measured one row group (#801)") + expect.text(geom["fc_many"], "10 10 10", "every fc_many reading measured ten row groups (#801)") + expect.text(_ratio(best["fc_one"], best["fc_many"], 3), "within", + "fetching from one big group is not far dearer than from ten small ones") + expect.num(_one(c, "SELECT count(*) FROM fc_one WHERE v = id + 1"), UPD, "the updated rows are correct") + expect.num(_one(c, "SELECT count(*) FROM fc_one WHERE v = id"), ROWS - UPD, "the untouched rows are unchanged") + + +def test_the_scoping_guards_are_in_the_source(expect): + reader = (SRC / "columnar_reader.c").read_text() + slot, row = _fn("columnar_reader.c", "pgcolumnar_fetch_group_slot"), _fn("columnar_reader.c", "pgcolumnar_fetch_row") + expect.text("yes" if slot and row else "no", "yes", "premise: the fetch-cache functions were extracted, not empty ranges") + lines = reader.splitlines() + grp = sum("e->groupNumber == groupNumber" in l for l in lines) + keyed = sum("e->storageId == storageId && e->groupNumber == groupNumber" in l for l in lines) + expect.at_least(grp, 1, "premise: there is a group-number comparison to key") + expect.text(f"keyed {keyed} of {grp}", f"keyed {grp} of {grp}", "every group-number comparison is keyed by the storage id too") + expect.text("yes" if "e->cid != cid" in slot else "no", "yes", "an entry from an earlier command is rejected") + for f in ("firstRowNumber", "rowCount", "fileOffset"): + expect.text("yes" if f"entry->{f} != rg->{f}" in row else "no", "yes", + f"a hit re-checks the group's {f} against the row group") + expect.text("yes" if "entry->natts != natts" in row else "no", "yes", + "a hit re-checks the group's natts against the scan's column count") + for fn in ("pgcolumnar_executor_end", "pgcolumnar_xact_callback"): + body = _fn("columnar_tableam.c", fn) + expect.text("yes" if body else "no", "yes", f"premise: {fn} was extracted, not an empty range") + expect.text("yes" if "PgColumnarDiscardFetchCache" in body else "no", "yes", + f"the fetch cache is discarded from {fn}") + + +def test_a_wide_groups_decode_scratch_does_not_blow_the_cap(fc, expect): + c = fc + for table, stripe in (("fc_wide", 150000), ("fc_wnarrow", 20000)): + c.execute(f"DROP TABLE IF EXISTS {table}") + c.execute(f"""CREATE TABLE {table} (id int, h text, c1 int,c2 int,c3 int,c4 int,c5 int,c6 int, + c7 int,c8 int,c9 int,c10 int, u float8, + t1 text,t2 text,t3 text,t4 text,t5 text,t6 text,t7 text,t8 text) USING pgcolumnar""") + c.execute(f"SELECT pgcolumnar.set_options('{table}', stripe_row_limit => {stripe})") + c.execute(f"""INSERT INTO {table} SELECT g, 'h' || (g % 500), g,g,g,g,g,g,g,g,g,g, (g % 100)::float8, + repeat('x',40),repeat('y',40),repeat('z',40),'a'||g,'b'||g,'c'||g,'d'||g,'e'||g + FROM generate_series(1, 300000) g""") + c.execute(f"CREATE INDEX {table}_h ON {table} (h)") + point = "SELECT count(*), max(u) FROM {} WHERE h='h1'" + for table in ("fc_wide", "fc_wnarrow"): + expect.text(_row_fetch(c, point.format(table)), "yes", f"the {table} point query really drives the fetch path (#797)") + bigw = min(_ms(c, point.format("fc_wide")) for _ in range(3)) + smallw = min(_ms(c, point.format("fc_wnarrow")) for _ in range(3)) + print(f"-- #353 wide point query: default-stripe {bigw} ms, small-stripe {smallw} ms") + expect.text(_ratio(bigw, smallw, 5), "within", "a wide group's fetch is not far dearer than a small group's (#353)") + both = "SELECT count(*) || '|' || coalesce(max(u)::text,'z') FROM {} WHERE h='h1'" + expect.text(_one(c, both.format("fc_wide")), _one(c, both.format("fc_wnarrow")), + "the wide-group point query is correct (#353)") + + +def test_crossing_the_cap_costs_per_column(fc, expect): + c = fc + c.execute("DROP TABLE IF EXISTS fc_w359") + c.execute("CREATE TABLE fc_w359 (id int, h text, t1 text,t2 text,t3 text,t4 text,t5 text) USING pgcolumnar") + c.execute("SELECT pgcolumnar.set_options('fc_w359', stripe_row_limit => 50000)") + c.execute("""INSERT INTO fc_w359 SELECT g, 'h' || (g % 1000), + repeat('a',150),repeat('b',150),repeat('c',150),repeat('d',150),repeat('e',150) + FROM generate_series(1,100000) g""") + c.execute("CREATE INDEX fc_w359_h ON fc_w359 (h)") + + def query(n): + return "SELECT count(*)" + "".join(f", max(t{i})" for i in range(1, n + 1)) + " FROM fc_w359 WHERE h='h7'" + + def warm_ms(n): + _ms(c, query(n)) # warm: decode cost, not first-touch I/O + return _ms(c, query(n)) + + for n in (4, 5): + expect.text(_row_fetch(c, query(n)), "yes", f"the {n}-column projection really drives the fetch path (#797)") + under = min(warm_ms(4) for _ in range(3)) # ~30 MB decoded: fits + over = min(warm_ms(5) for _ in range(3)) # ~38 MB decoded: does not + print(f"-- #359 projection width: four columns {under} ms, five columns {over} ms") + expect.text(_ratio(over, under, 12), "within", "crossing the fetch cache cap costs proportionally, not totally (#359)") + # the overflowing columns are re-decoded, not skipped: fast and null would be wrong + expect.text(_one(c, "SELECT max(t1) || '|' || max(t4) FROM fc_w359 WHERE h='h7'"), + _one(c, "SELECT max(t1) || '|' || max(t4) FROM fc_w359 WHERE h='h7' AND t5 IS NOT NULL"), + "the over-cap projection returns the same values as the under-cap one (#359)") + expect.num(_one(c, """SELECT count(*) FROM fc_w359 WHERE h='h7' + AND t1 = repeat('a',150) AND t2 = repeat('b',150) AND t3 = repeat('c',150) + AND t4 = repeat('d',150) AND t5 = repeat('e',150)"""), + _one(c, "SELECT count(*) FROM fc_w359 WHERE h='h7'"), + "every over-cap column reads back its written value (#359)") diff --git a/test/pytest/test_native_fetch_position.py b/test/pytest/test_native_fetch_position.py new file mode 100644 index 000000000..f3b2bece3 --- /dev/null +++ b/test/pytest/test_native_fetch_position.py @@ -0,0 +1,166 @@ +"""A fetch by row number costs the same wherever the row sits in its group (#143, #1341). +The pytest twin of `test/native_fetch_position.sh`, every check by the shell's name. + +A cached group still walked to the row: counting the validity bits below it, then +decoding and discarding that many values, so a fetch cost O(row) and fetching a whole +group was quadratic. A rank prefix over the validity bitmap and an offset table for +varying-length columns make it O(1). Values first, against a heap mirror over fixed, +varying-length and heavily-null columns, through an index scan; then two timing ratios +(far end against near end, and a doubled group), each the median of three passes; then +the two indexes in pgcolumnar_fetch_row's source. +""" +import os +import pathlib +import re +import statistics +import time + +import psycopg +import pytest + +# the shell reads PGC_FETCHPOS_ROWS, and so does this +ROWS = int(os.environ.get("PGC_FETCHPOS_ROWS", "120000")) +SRC = pathlib.Path(__file__).resolve().parents[2] / "src" +# enable_seqscan=off leaves the columnar custom scan on, and with only that every query +# here scanned rather than fetched by row number; each measurement asserts its plan +FORCE = ("SET enable_seqscan = off", "SET enable_bitmapscan = off", "SET max_parallel_workers_per_gather = 0", + "SET pgcolumnar.enable_custom_scan = off") + + +def _one(c, sql): + return c.execute(sql).fetchone()[0] + + +def _forced(c): + for s in FORCE: + c.execute(s) + + +def _reads_index(c, sql, index): + """-> 'yes' when `sql`'s plan is an Index Scan using `index`, else its first scan line.""" + plan = "\n".join(r[0] for r in c.execute(f"EXPLAIN (COSTS OFF) {sql}")) + if f"Index Scan using {index} " in plan: + return "yes" + first = next((l.strip() for l in plan.splitlines() if "Scan" in l or "Update" in l), plan) + return f"no: {first}" + + +def _rows_or_error(c, sql): + """-> the first value, or the error as a value: a corrupt fetch is a named red.""" + try: + return _one(c, sql) + except psycopg.Error as exc: + return f"error: {exc.diag.message_primary}" + + +def _median_ms(c, sql): + """-> the median of three timed passes of `sql`, in milliseconds.""" + times = [] + for _ in range(3): + t0 = time.monotonic() + c.execute(sql) + times.append((time.monotonic() - t0) * 1000.0) + return statistics.median(times) + + +@pytest.fixture(scope="module") +def fpos(pgc_cluster): + """-> a connection to a database holding fp_c (one row group) and its heap mirror.""" + db = "pgc_fetchpos" + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + a.execute(f"CREATE DATABASE {db}") + try: + with psycopg.connect(pgc_cluster.dsn(dbname=db), autocommit=True) as c: + c.execute("CREATE EXTENSION pgcolumnar") + # every fourth row null in the nullable columns, so rank and row number diverge + c.execute(f"SET pgcolumnar.stripe_row_limit = {ROWS}") + c.execute(f"SET pgcolumnar.chunk_group_row_limit = {ROWS}") + c.execute("CREATE TABLE fp_c (id int, fixed bigint, txt text, sparse int, stxt text) USING pgcolumnar") + c.execute(f"""INSERT INTO fp_c + SELECT g, g * 7, repeat('x', 1 + (g % 40)) || g, + CASE WHEN g % 4 = 0 THEN NULL ELSE g END, + CASE WHEN g % 4 = 0 THEN NULL ELSE 's' || g END + FROM generate_series(1, {ROWS}) g""") + c.execute("CREATE INDEX fp_c_id ON fp_c (id)") + c.execute("CREATE TABLE fp_h (LIKE fp_c)") + c.execute("INSERT INTO fp_h SELECT * FROM fp_c") + yield c + finally: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + + +def test_every_fetched_value_matches_the_heap_mirror(fpos, expect): + c = fpos + expect.num(_one(c, "SELECT count(*) FROM pgcolumnar.row_group r " + "JOIN pgcolumnar.storage s ON s.storage_id = r.storage_id " + "WHERE s.relation_oid = 'fp_c'::regclass"), 1, "the rows are in one row group") + _forced(c) + expect.text(_reads_index(c, "SELECT count(*) FROM fp_c c JOIN fp_h h USING (id) " + "WHERE c.fixed IS DISTINCT FROM h.fixed", "fp_c_id"), "yes", + "premise: the mirror comparison fetches through the index") + expect.text(_reads_index(c, "SELECT fixed FROM fp_c WHERE id = 1001", "fp_c_id"), "yes", + "premise: a single-row fetch goes through the index") + expect.text(str(_rows_or_error(c, """SELECT count(*) FROM fp_c c JOIN fp_h h USING (id) + WHERE c.fixed IS DISTINCT FROM h.fixed + OR c.txt IS DISTINCT FROM h.txt + OR c.sparse IS DISTINCT FROM h.sparse + OR c.stxt IS DISTINCT FROM h.stxt""")), "0", "every column matches the heap mirror") + # one row at a time, where an off-by-one in the rank would land on a neighbour + row = "fixed || '|' || txt || '|' || coalesce(sparse::text, 'N') || '|' || coalesce(stxt, 'N')" + bad = [] + for pos in (1, 2, 3, 4, 5, 999, 1000, 1001, ROWS // 2, ROWS - 1, ROWS): + got = _rows_or_error(c, f"SELECT {row} FROM fp_c WHERE id = {pos}") + want = _one(c, f"SELECT {row} FROM fp_h WHERE id = {pos}") + if got != want: + bad.append(f"id={pos}(got={got} want={want})") + expect.text(" ".join(bad) or "same", "same", "single-row fetches are right at every position") + + +def test_the_cost_does_not_depend_on_the_position(fpos, expect): + c = fpos + _forced(c) + n = ROWS // 20 + # a read, not an UPDATE: an UPDATE writes its rows into a new group, so only the first + # pass fetched from the far end and the median was always a later one + read = "SELECT sum(fixed), sum(length(txt)) FROM fp_c WHERE id BETWEEN {} AND {}" + expect.text(_reads_index(c, read.format(ROWS - n + 1, ROWS), "fp_c_id"), "yes", + "premise: the timed reads fetch through the index") + near = _median_ms(c, read.format(1, n)) + far = _median_ms(c, read.format(ROWS - n + 1, ROWS)) + print(f"-- {n} fetches, median of three: first tenth {near:.0f} ms, last tenth {far:.0f} ms") + # a ratio of 3 catches the quadratic walk (1,001,374 ms against 614 ms) with orders to spare + expect.text("yes" if near > 0 and far / near < 3 else f"no (near={near:.0f}ms far={far:.0f}ms)", "yes", + "fetching from the far end of a group costs about what the near end does") + + +def test_a_bigger_group_does_not_cost_more_per_fetch(fpos, expect): + c = fpos + for name, rows in (("fp_small", ROWS // 2), ("fp_big", ROWS)): + c.execute(f"SET pgcolumnar.stripe_row_limit = {rows}") + c.execute(f"SET pgcolumnar.chunk_group_row_limit = {rows}") + c.execute(f"CREATE TABLE {name} (id int, v bigint, t text) USING pgcolumnar") + c.execute(f"INSERT INTO {name} SELECT g, g * 3, 't' || g FROM generate_series(1, {rows}) g") + c.execute(f"CREATE INDEX {name}_id ON {name} (id)") + _forced(c) + # the rows fetched sit at the END of each group, where the old walk was longest + k = ROWS // 20 + read = "SELECT sum(v), sum(length(t)) FROM {} WHERE id > {}" + expect.text(_reads_index(c, read.format("fp_big", ROWS - k), "fp_big_id"), "yes", + "premise: the doubled group's reads fetch through its index") + small = _median_ms(c, read.format("fp_small", ROWS // 2 - k)) + big = _median_ms(c, read.format("fp_big", ROWS - k)) + print(f"-- {k} fetches at the far end, median of three: {ROWS // 2} rows took {small:.0f} ms, {ROWS} took {big:.0f} ms") + expect.text("yes" if small > 0 and big / small < 1.8 else f"no (small={small:.0f}ms big={big:.0f}ms)", "yes", + "doubling the row group does not double the cost of the same fetches") + + +def test_the_indexes_that_make_it_constant_are_in_the_fetch(expect): + m = re.search(r"^pgcolumnar_fetch_row\(.*?^}", (SRC / "columnar_reader.c").read_text(), re.S | re.M) + fetch = m.group(0) if m else "" + expect.text("yes" if fetch else "no", "yes", "premise: pgcolumnar_fetch_row was extracted, not an empty range") + expect.text("yes" if "present = pgcolumnar_rank_before" in fetch else "no", "yes", + "the rank comes from a prefix rather than a loop over earlier rows") + expect.text("yes" if "entry->valOffset[c][present]" in fetch else "no", "yes", + "a varying-length column reaches its value through an offset table")