diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 738978f03..eabc7e55f 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2470,6 +2470,32 @@ native_encdesc_golden native_encdesc_golden int column a rawLen parses as 1200 ( native_encdesc_golden native_encdesc_golden the table reads back all rows 15;16;17;18;19 never - native_encdesc_golden native_encdesc_golden trailing sharedTableLen is 0 (u32 at offset 6+13=19) 15;16;17;18;19 never - native_encdesc_golden native_encdesc_golden vectorCount parses as 1 (at offset 2) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache a hit re-checks the group's fileOffset against the row group 15;16;17;18;19 2026-10-16 columnar_reader.c: a fetch-cache hit no longer re-checks fileOffset +native_fetch_cache native_fetch_cache a hit re-checks the group's firstRowNumber against the row group 15;16;17;18;19 2026-10-16 columnar_reader.c: a fetch-cache hit no longer re-checks firstRowNumber +native_fetch_cache native_fetch_cache a hit re-checks the group's natts against the scan's column count 15;16;17;18;19 2026-10-16 columnar_reader.c: a fetch-cache hit no longer re-checks natts +native_fetch_cache native_fetch_cache a hit re-checks the group's rowCount against the row group 15;16;17;18;19 2026-10-16 columnar_reader.c: a fetch-cache hit no longer re-checks rowCount +native_fetch_cache native_fetch_cache a wide group's fetch is not far dearer than a small group's (#353) 15;16;17;18;19 2026-10-16 columnar_reader.c: crossing the fetch-cache cap drops every column, not the one that crossed it;columnar_reader.c: every fetch-cache lookup misses, so each fetch decodes its group again +native_fetch_cache native_fetch_cache an entry from an earlier command is rejected 15;16;17;18;19 2026-10-16 columnar_reader.c: a fetch-cache entry from an earlier command is not rejected +native_fetch_cache native_fetch_cache crossing the fetch cache cap costs proportionally, not totally (#359) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache every fc_many reading measured ten row groups (#801) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache every fc_one reading measured one row group (#801) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache every group-number comparison is keyed by the storage id too 15;16;17;18;19 2026-10-16 columnar_reader.c: the fetch-cache key compares the group number without the storage id +native_fetch_cache native_fetch_cache every over-cap column reads back its written value (#359) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache fetching from one big group is not far dearer than from ten small ones 15;16;17;18;19 2026-10-16 columnar_reader.c: every fetch-cache lookup misses, so each fetch decodes its group again +native_fetch_cache native_fetch_cache premise: pgcolumnar_executor_end was extracted, not an empty range 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache premise: pgcolumnar_xact_callback was extracted, not an empty range 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache premise: the fetch-cache functions were extracted, not empty ranges 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache premise: there is a group-number comparison to key 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache the 4-column projection really drives the fetch path (#797) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache the 5-column projection really drives the fetch path (#797) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache the fc_wide point query really drives the fetch path (#797) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache the fc_wnarrow point query really drives the fetch path (#797) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache the fetch cache is discarded from pgcolumnar_executor_end 15;16;17;18;19 2026-10-16 columnar_tableam.c: pgcolumnar_executor_end no longer discards the fetch cache +native_fetch_cache native_fetch_cache the fetch cache is discarded from pgcolumnar_xact_callback 15;16;17;18;19 2026-10-16 columnar_tableam.c: the transaction callback no longer discards the fetch cache +native_fetch_cache native_fetch_cache the over-cap projection returns the same values as the under-cap one (#359) 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache the untouched rows are unchanged 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache the updated rows are correct 15;16;17;18;19 never - +native_fetch_cache native_fetch_cache the wide-group point query is correct (#353) 15;16;17;18;19 never - native_fetch_coalesce native_fetch_coalesce a wide index fetch does not pin once per column 15;16;17;18;19 2026-10-13 the coalesced fetch merging no adjacent ranges, so each column is read on its own native_fetch_coalesce native_fetch_coalesce premise: a point lookup uses the index 15;16;17;18;19 never - native_fetch_coalesce native_fetch_coalesce premise: fetching every projected column touched a measurable number of buffers 15;16;17;18;19 never - @@ -2477,6 +2503,18 @@ native_fetch_coalesce native_fetch_coalesce premise: fetching one projected colu native_fetch_coalesce native_fetch_coalesce premise: the coalescing helper holds both the bound and the validity copy 15;16;17;18;19 never - native_fetch_coalesce native_fetch_coalesce the validity copy is bounded by the chunk length before it runs 15;16;17;18;19 never - native_fetch_coalesce native_fetch_coalesce the wide fetch returns the projected values 15;16;17;18;19 2026-10-14 every column of a coalesced fetch is copied from the start of the span (off = 0) +native_fetch_position native_fetch_position a varying-length column reaches its value through an offset table 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row reaches a varying-length value by decoding every earlier one (the #143 walk) +native_fetch_position native_fetch_position doubling the row group does not double the cost of the same fetches 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row counts the validity bits from the group's start for every fetch (the #143 walk);columnar_reader.c: pgcolumnar_fetch_row reaches a varying-length value by decoding every earlier one (the #143 walk) +native_fetch_position native_fetch_position every column matches the heap mirror 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row takes the row number as the rank, counting rows rather than present values;columnar_reader.c: pgcolumnar_fetch_row's rank is one past the row's +native_fetch_position native_fetch_position fetching from the far end of a group costs about what the near end does 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row counts the validity bits from the group's start for every fetch (the #143 walk);columnar_reader.c: pgcolumnar_fetch_row reaches a varying-length value by decoding every earlier one (the #143 walk) +native_fetch_position native_fetch_position premise: a single-row fetch goes through the index 15;16;17;18;19 never - +native_fetch_position native_fetch_position premise: pgcolumnar_fetch_row was extracted, not an empty range 15;16;17;18;19 never - +native_fetch_position native_fetch_position premise: the doubled group's reads fetch through its index 15;16;17;18;19 never - +native_fetch_position native_fetch_position premise: the mirror comparison fetches through the index 15;16;17;18;19 never - +native_fetch_position native_fetch_position premise: the timed reads fetch through the index 15;16;17;18;19 never - +native_fetch_position native_fetch_position single-row fetches are right at every position 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row takes the row number as the rank, counting rows rather than present values;columnar_reader.c: pgcolumnar_fetch_row's rank is one past the row's +native_fetch_position native_fetch_position the rank comes from a prefix rather than a loop over earlier rows 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row counts the validity bits from the group's start for every fetch (the #143 walk);columnar_reader.c: pgcolumnar_fetch_row takes the row number as the rank, counting rows rather than present values +native_fetch_position native_fetch_position the rows are in one row group 15;16;17;18;19 never - native_fetch_projection native_fetch_projection a deleted row stops being read through the projection 15;16;17;18;19 2026-10-16 columnar_projection.c: read_projection ignores the base row's liveness native_fetch_projection native_fetch_projection an uncovered column still reads correctly after a delete 15;16;17;18;19 2026-10-16 columnar_projection.c: reconstruct's uncovered set misses the table's last column native_fetch_projection native_fetch_projection and does not decode the row to find out 15;16;17;18;19 2026-10-16 columnar_tableam.c: index deletion decodes each bottom-up row to decide @@ -2595,17 +2633,17 @@ native_groupagg_batch native_groupagg_batch <> filter: the fold declined 15;16;1 native_groupagg_batch native_groupagg_batch add column: ANALYZE reports the row-path run 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped node's ANALYZE prints the predicted fold, not what ran (the #602 shape) native_groupagg_batch native_groupagg_batch add column: plain EXPLAIN reports the prediction 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch add column: the answer is still right 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch count(*): answers match the heap mirror 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch count(*): answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch count(*): the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch count(*): the grouped node is planned 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch date key: answers match the heap mirror 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch date key: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch date key: the grouped node is planned 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch deletes: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch deletes: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask;columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch deletes: the fold still ran 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch expression key: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: an expression key's -1 attidx is not refused by the grouped fold's eligibility native_groupagg_batch native_groupagg_batch expression key: the fold declined 15;16;17;18;19 2026-10-16 columnar_vector.c: an expression key's -1 attidx is not refused by the grouped fold's eligibility -native_groupagg_batch native_groupagg_batch filtered: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold applies no scan key per row +native_groupagg_batch native_groupagg_batch filtered: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold applies no scan key per row;columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch filtered: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch filtered: the grouped node is planned 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch float8 key: answers match the heap mirror 15;16;17;18;19 never - @@ -2619,13 +2657,13 @@ native_groupagg_batch native_groupagg_batch float8 zero: the grouped node is pla native_groupagg_batch native_groupagg_batch float8 zero: the grouped sums match the heap 15;16;17;18;19 2026-10-16 columnar_vector.c: float8 admitted as a simple key, so its group keys are hashed and compared by bits native_groupagg_batch native_groupagg_batch min/max: answers match the heap mirror 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch min/max: the fold declined 15;16;17;18;19 2026-10-16 columnar_vector.c: min and max admitted as batch fold kinds -native_groupagg_batch native_groupagg_batch parallel: and they equal the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: a partial grouped node folds without the shared row-group counter;columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch parallel: and they equal the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: a partial grouped node folds without the shared row-group counter;columnar_vector.c: the grouped fold ignores the delete mask;columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch parallel: our parallel-aware grouped node is under the Gather 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch parallel: premise: the value arm's own plan launches workers 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch parallel: the answers equal the serial fold, byte for byte 15;16;17;18;19 2026-10-16 columnar_vector.c: a partial grouped node folds without the shared row-group counter native_groupagg_batch native_groupagg_batch parallel: the fold ran in the partial node (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch parallel: workers actually launched for that node 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch plain: answers match the heap mirror 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch plain: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch plain: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch plain: the grouped node is planned 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch premise: fixture loaded every row (columnar) 15;16;17;18;19 never - @@ -2646,16 +2684,16 @@ native_groupagg_batch native_groupagg_batch text key: the fold declined 15;16;17 native_groupagg_batch native_groupagg_batch timestamp key: answers match the heap mirror 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch timestamp key: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch timestamp key: the grouped node is planned 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch toggle-differential: filtered 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold applies no scan key per row;columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch toggle-differential: filtered 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold applies no scan key per row;columnar_vector.c: the grouped fold ignores the delete mask;columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch toggle-differential: filtered [premise: node fires with the path on] 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch toggle-differential: filtered [premise: the off arm is not the grouped node] 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch toggle-differential: plain 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch toggle-differential: plain 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask;columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch toggle-differential: plain [premise: node fires with the path on] 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch toggle-differential: plain [premise: the off arm is not the grouped node] 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch toggle-differential: two keys 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch toggle-differential: two keys 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask;columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch toggle-differential: two keys [premise: node fires with the path on] 15;16;17;18;19 never - native_groupagg_batch native_groupagg_batch toggle-differential: two keys [premise: the off arm is not the grouped node] 15;16;17;18;19 never - -native_groupagg_batch native_groupagg_batch two keys: answers match the heap mirror 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch two keys: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch two keys: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch two keys: the grouped node is planned 15;16;17;18;19 never - native_index_projection native_index_projection a parallel build projects one column of twenty, not all twenty 15;16;17;18;19 never - @@ -2775,6 +2813,57 @@ native_join_vector_agg native_join_vector_agg unique join fold answer equals GUC native_join_vector_agg native_join_vector_agg unique join fold answer equals heap 15;16;17;18;19 never - native_join_vector_agg native_join_vector_agg unique join uses core Agg when GUC off 15;16;17;18;19 never - native_join_vector_agg native_join_vector_agg unique join uses vectorized agg when GUC on 15;16;17;18;19 2026-09-12 drop JOINREL fold +native_parquet_flba native_parquet_flba FDW scan == source 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a FIXED_LEN_BYTE_ARRAY uuid's bytes are reversed on read +native_parquet_flba native_parquet_flba INT32-backed DECIMAL keeps its null 15;16;17;18;19 never - +native_parquet_flba native_parquet_flba INT32-backed DECIMAL reads 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: an INT32 or INT64 DECIMAL is widened byte-reversed;columnar_parquet_reader.c: pq_decimal_to_numeric reads a narrower-than-16-byte negative DECIMAL as its unsigned magnitude +native_parquet_flba native_parquet_flba INT64-backed DECIMAL reads 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: an INT32 or INT64 DECIMAL is widened byte-reversed;columnar_parquet_reader.c: pq_decimal_to_numeric reads a narrower-than-16-byte negative DECIMAL as its unsigned magnitude +native_parquet_flba native_parquet_flba an INT64 DECIMAL still binds to bigint as the unscaled integer 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a bigint target over a DECIMAL column is scaled like numeric +native_parquet_flba native_parquet_flba backend survived the crafted scale 15;16;17;18;19 never - +native_parquet_flba native_parquet_flba crafted out-of-range scale is rejected, not decoded 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: the bind guard pq_decimal_bounds_ok is dropped from pq_want_phys_for +native_parquet_flba native_parquet_flba import_parquet == source 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a FIXED_LEN_BYTE_ARRAY uuid's bytes are reversed on read +native_parquet_flba native_parquet_flba numeric round-trips through export/read 15;16;17;18;19 never - +native_parquet_flba native_parquet_flba parquet_schema advises numeric for an INT32 DECIMAL 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: parquet_schema advises an integer for an INT32 DECIMAL +native_parquet_flba native_parquet_flba parquet_schema advises numeric for an INT64 DECIMAL 15;16;17;18;19 never - +native_parquet_flba native_parquet_flba pyarrow decimal128 values are exact 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: pq_decimal_to_numeric reads a narrower-than-16-byte negative DECIMAL as its unsigned magnitude +native_parquet_flba native_parquet_flba pyarrow uuid reads as uuid 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a FIXED_LEN_BYTE_ARRAY uuid's bytes are reversed on read +native_parquet_flba native_parquet_flba uuid round-trips through export/read 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a FIXED_LEN_BYTE_ARRAY uuid's bytes are reversed on read +native_parquet_flba native_parquet_flba whole-row round-trip == source 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a FIXED_LEN_BYTE_ARRAY uuid's bytes are reversed on read +native_parquet_pushdown native_parquet_pushdown INT64-backed DECIMAL: predicate skips 3 of 4 groups 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: INT64 statistics are not decoded, so an INT64-backed DECIMAL never skips;columnar_parquet_reader.c: pqfdw_compute_skip never marks a row group skippable +native_parquet_pushdown native_parquet_pushdown INT64-backed DECIMAL: skipping did not drop rows 15;16;17;18;19 never - +native_parquet_pushdown native_parquet_pushdown INT64-backed DECIMAL: unfiltered scan skips nothing 15;16;17;18;19 never - +native_parquet_pushdown native_parquet_pushdown NaN-safe [val < 10] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: col < const skips the row groups that do hold matches +native_parquet_pushdown native_parquet_pushdown NaN-safe [val <= 10] == oracle 15;16;17;18;19 never - +native_parquet_pushdown native_parquet_pushdown NaN-safe [val = 'NaN'::float8] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: an equality against a NaN constant is pushed +native_parquet_pushdown native_parquet_pushdown NaN-safe [val = 1234] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: an equality skips every row group +native_parquet_pushdown native_parquet_pushdown NaN-safe [val > 1e300] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: float > and >= are pushed although the statistics exclude NaN +native_parquet_pushdown native_parquet_pushdown NaN-safe [val > 3998] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: float > and >= are pushed although the statistics exclude NaN +native_parquet_pushdown native_parquet_pushdown NaN-safe [val >= 1e300] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: float > and >= are pushed although the statistics exclude NaN +native_parquet_pushdown native_parquet_pushdown float < still skips 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: col < const skips the row groups that do hold matches;columnar_parquet_reader.c: pqfdw_compute_skip never marks a row group skippable +native_parquet_pushdown native_parquet_pushdown float predicate skips 2 (>= not pushed, NaN-unsafe) 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: float > and >= are pushed although the statistics exclude NaN;columnar_parquet_reader.c: pqfdw_compute_skip never marks a row group skippable +native_parquet_pushdown native_parquet_pushdown full scan == oracle 15;16;17;18;19 never - +native_parquet_pushdown native_parquet_pushdown id < 100 skips 3 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: col < const skips the row groups that do hold matches;columnar_parquet_reader.c: pqfdw_compute_skip never marks a row group skippable +native_parquet_pushdown native_parquet_pushdown id >= 199900 skips 3 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: pqfdw_compute_skip never marks a row group skippable +native_parquet_pushdown native_parquet_pushdown inverted stats skip nothing 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: an inverted min/max interval drives skips like any other +native_parquet_pushdown native_parquet_pushdown inverted stats: unfiltered == oracle 15;16;17;18;19 never - +native_parquet_pushdown native_parquet_pushdown inverted-stats [c < 0] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: an inverted min/max interval drives skips like any other +native_parquet_pushdown native_parquet_pushdown inverted-stats [c <= -1294967296] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: an inverted min/max interval drives skips like any other +native_parquet_pushdown native_parquet_pushdown inverted-stats [c = -1294967296] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: an inverted min/max interval drives skips like any other +native_parquet_pushdown native_parquet_pushdown inverted-stats [c = 1] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: an inverted min/max interval drives skips like any other +native_parquet_pushdown native_parquet_pushdown inverted-stats [c > 0] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: an inverted min/max interval drives skips like any other +native_parquet_pushdown native_parquet_pushdown inverted-stats [c >= 1] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: an inverted min/max interval drives skips like any other +native_parquet_pushdown native_parquet_pushdown no predicate skips 0 15;16;17;18;19 never - +native_parquet_pushdown native_parquet_pushdown point in group 0 skips 3 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: an equality skips every row group;columnar_parquet_reader.c: pqfdw_compute_skip never marks a row group skippable +native_parquet_pushdown native_parquet_pushdown predicate [id < 100] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: col < const skips the row groups that do hold matches +native_parquet_pushdown native_parquet_pushdown predicate [id <= 5 OR id >= 199995] == oracle 15;16;17;18;19 never - +native_parquet_pushdown native_parquet_pushdown predicate [id = 60000] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: an equality skips every row group +native_parquet_pushdown native_parquet_pushdown predicate [id > 60000 AND id < 60010] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: col < const skips the row groups that do hold matches +native_parquet_pushdown native_parquet_pushdown predicate [id >= 199900] == oracle 15;16;17;18;19 never - +native_parquet_pushdown native_parquet_pushdown predicate [id BETWEEN 60000 AND 70000] == oracle 15;16;17;18;19 never - +native_parquet_pushdown native_parquet_pushdown predicate [val > 90000 AND val < 90030] == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: col < const skips the row groups that do hold matches +native_parquet_pushdown native_parquet_pushdown range in one group skips 3 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: pqfdw_compute_skip never marks a row group skippable +native_parquet_pushdown native_parquet_pushdown read_parquet same file == oracle 15;16;17;18;19 never - +native_parquet_pushdown native_parquet_pushdown reports 4 row groups 15;16;17;18;19 never - +native_parquet_pushdown native_parquet_pushdown spanning predicate skips 0 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: col < const skips the row groups that do hold matches native_parquet_schema native_parquet_schema OPTIONAL column reports nullable=t 15;16;17;18;19 never - native_parquet_schema native_parquet_schema REQUIRED column reports nullable=f 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: parquet_schema reports every column nullable native_parquet_schema native_parquet_schema REQUIRED int32 maps to integer 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index bc4355952..cda61de08 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 199 +suites_not_covered 195 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -3343,4 +3343,77 @@ suites_not_covered 199 # never 2988 - 16 = 2972 # dated 1201 + 16 = 1217 # 2972 + 1217 == 4189 -checks_never_observed_red 2972 +# +# #1341, native_groupagg_batch's NULL-key mutation (OffgridwithJD, from jdatcmd's +# #1485 review): 2972 -> 2969. The grouped fold's gather dropping the NULL-key +# flag folds the NULL group into k = 0 on the batch path only. Nine checks redden, +# shell and twin alike; three of them (plain, two keys and count(*) against the +# heap mirror) entered never and are now dated, and the other six gain the +# mutation in their history. No row is added or removed. +# +# ledger rows 4189 + 0 = 4189 +# never 2972 - 3 = 2969 +# dated 1217 + 3 = 1220 +# 2969 + 1220 == 4189 +# +# #1341, native_fetch_position seeded, with its twin (OffgridwithJD): 2969 -> 2975, +# and suites_not_covered 199 -> 198. The suite had no rows. It enters with all 12 +# checks from five-major runs (12/12 on each of 15-19). 6 are dated by four +# mutations of pgcolumnar_fetch_row: the rank counting rows, the rank one too far, +# the rank walking the validity bits, and a varying-length value reached by +# decoding every earlier one. The two timing arms are dated by the walks only now +# that the fetches go through the index and the timed pass is a read. Six arrive +# never: the one-row-group fixture fact and five premises. +# +# ledger rows 4189 + 12 = 4201 +# never 2969 + 6 = 2975 +# dated 1220 + 6 = 1226 +# 2975 + 1226 == 4201 +# +# #1341, native_fetch_cache seeded, with its twin (OffgridwithJD): 2975 -> 2991, +# and suites_not_covered 198 -> 197. The suite had no rows. It enters with all 26 +# checks from five-major runs (26/26 on each of 15-19). 10 are dated by ten +# mutations: the cache never hitting, the key without the storage id, no +# earlier-command reject, each of the four recheck fields dropped, the discard +# dropped from executor end and from transaction end, and the cap dropping every +# column. Sixteen arrive never: four premises and the four #797 plan premises; +# the two #801 geometry records; the four correctness arms; and the #359 ratio, +# which the cap-drops-every-column mutation cannot redden on this fixture (it +# slows one projected column as much as five, measured 46 -> 286 ms in-suite). +# +# ledger rows 4201 + 26 = 4227 +# never 2975 + 16 = 2991 +# dated 1226 + 10 = 1236 +# 2991 + 1236 == 4227 +# +# #1341, native_parquet_pushdown seeded, with its twin (OffgridwithJD): 2991 -> 3002, +# and suites_not_covered 197 -> 196. The suite had no rows. It enters with all 36 +# checks from five-major runs (36/36 on each of 15-19). 25 are dated by seven +# mutations of the FDW's skip decision: no group skipped, the inverted-interval +# guard dropped, float > and >= pushed, a NaN equality pushed, INT64 statistics +# not decoded, col < const skipping the matching groups, and an equality skipping +# every group. Eleven arrive never: the unfiltered and read_parquet oracles, which +# no pushdown reaches; four predicate oracles no wrong skip here misdirects; the +# row-group count and the no-predicate control; and the DECIMAL arm's row count +# and unfiltered control. +# +# ledger rows 4227 + 36 = 4263 +# never 2991 + 11 = 3002 +# dated 1236 + 25 = 1261 +# 3002 + 1261 == 4263 +# +# #1341, native_parquet_flba seeded, with its twin (OffgridwithJD): 3002 -> 3006, +# and suites_not_covered 196 -> 195. The suite had no rows. It enters with all 15 +# checks from five-major runs (15/15 on each of 15-19). 11 are dated by six +# mutations of the FLBA and DECIMAL read path: a uuid's bytes reversed, a narrow +# negative DECIMAL read unsigned, the bind guard dropped, an integer DECIMAL +# widened byte-reversed, parquet_schema advising an integer, and bigint scaled +# like numeric. Four arrive never: the round-trip numeric arm (our export writes +# 16-byte DECIMALs, whose sign no narrow-width mutation reaches), the INT32 null, +# the INT64 schema advice, and the backend liveness after the crafted scale. +# +# ledger rows 4263 + 15 = 4278 +# never 3002 + 4 = 3006 +# dated 1261 + 11 = 1272 +# 3006 + 1272 == 4278 +checks_never_observed_red 3006 diff --git a/test/native_fetch_cache.sh b/test/native_fetch_cache.sh index e936e0509..bd8531f9a 100755 --- a/test/native_fetch_cache.sh +++ b/test/native_fetch_cache.sh @@ -48,7 +48,12 @@ build() { # table, rows-per-group CREATE INDEX ${1}_id ON $1 (id);" } -# index-driven so every row goes through the fetch path rather than a scan +# Every updated row goes through the fetch path whatever the scan plans: the +# UPDATE fetches its target rows by number. Measured (#1341) on PG18: this +# UPDATE plans as Custom Scan (PgColumnarScan) under these SETs, and with the +# cache made to miss on every lookup the ratio below still goes red (844 ms +# against 93 ms). So the SETs are not what makes it a fetch, and no plan +# premise is needed here, unlike the point queries further down. upd_ms() { local start end start=$(date +%s%N) diff --git a/test/native_fetch_position.sh b/test/native_fetch_position.sh index 9e860d8e4..18adc4f98 100755 --- a/test/native_fetch_position.sh +++ b/test/native_fetch_position.sh @@ -65,8 +65,32 @@ check "the rows are in one row group" \ # Fetch by row number is what an index scan drives, so force one. Each leading # SET prints its own line ahead of the result, so these queries take the last. +# +# enable_seqscan=off does NOT turn the columnar custom scan off, and it was all +# this suite set: measured, the single-row fetch, the near and far UPDATEs and +# the mirror join all planned as Custom Scan (PgColumnarScan), so none of them +# fetched by row number. A rank off by one left the single-row arm green, and a +# walk over every earlier row left both timing arms green. The custom scan is +# off now, and each measurement asserts first that its plan reads the index. FORCE="SET enable_seqscan = off; SET enable_bitmapscan = off; - SET max_parallel_workers_per_gather = 0;" + SET max_parallel_workers_per_gather = 0; + SET pgcolumnar.enable_custom_scan = off;" + +# reads_index QUERY INDEX -> yes, or the plan's first line: the premise that a +# query fetches by row number through INDEX rather than scanning +reads_index() { + local plan + plan="$(q "$FORCE EXPLAIN (COSTS OFF) $1")" + case "$plan" in + *"Index Scan using $2 "*) echo yes ;; + *) echo "no: $(grep -m1 -E 'Scan|Update' <<<"$plan" | sed 's/^ *//')" ;; + esac +} + +check "premise: the mirror comparison fetches through the index" \ + "$(reads_index "SELECT count(*) FROM fp_c c JOIN fp_h h USING (id) WHERE c.fixed IS DISTINCT FROM h.fixed" fp_c_id)" "yes" +check "premise: a single-row fetch goes through the index" \ + "$(reads_index "SELECT fixed FROM fp_c WHERE id = 1001" fp_c_id)" "yes" # rows spread across the whole group, including the far end where the old walk # was longest @@ -103,7 +127,7 @@ FAR_HI=$ROWS upd_once() { # id-low, id-high -> milliseconds for one pass local start end start=$(date +%s%N) - psql_run "$FORCE UPDATE fp_c SET fixed = fixed + 1 + psql_run "$FORCE SELECT sum(fixed), sum(length(txt)) FROM fp_c WHERE id BETWEEN $1 AND $2;" >/dev/null 2>&1 end=$(date +%s%N) echo $(( (end - start) / 1000000 )) @@ -130,6 +154,17 @@ upd_ms() { # id-low, id-high -> median of three passes printf '%s\n%s\n%s\n' "$a" "$b" "$c" | sort -n | sed -n 2p } +# The timed pass is a SELECT, not the UPDATE it was. A columnar UPDATE writes +# each row again into a new row group, so only the first of three passes +# fetched from the far end of the big group; the second and third fetched from +# the start of the new one, and the median was always one of those. Measured +# with the rank walking every earlier row: the UPDATE form read near 122 ms, +# far 132 ms, green. A read leaves every row where it was, so all three passes +# fetch the same positions. sum(length(txt)) makes it decode a varying-length +# column, which the offset table serves. +check "premise: the timed reads fetch through the index" \ + "$(reads_index "SELECT sum(fixed), sum(length(txt)) FROM fp_c WHERE id BETWEEN $FAR_LO AND $FAR_HI" fp_c_id)" "yes" + near="$(upd_ms $NEAR_LO $NEAR_HI)" far="$(upd_ms $FAR_LO $FAR_HI)" echo "-- $N fetches, median of three: first tenth ${near} ms, last tenth ${far} ms" @@ -157,7 +192,7 @@ build() { # table, rows fetch_once() { # table, table-rows, count -> milliseconds for one pass local start end start=$(date +%s%N) - psql_run "$FORCE UPDATE $1 SET v = v + 1 WHERE id > $(( $2 - $3 ));" \ + psql_run "$FORCE SELECT sum(v), sum(length(t)) FROM $1 WHERE id > $(( $2 - $3 ));" \ >/dev/null 2>&1 end=$(date +%s%N) echo $(( (end - start) / 1000000 )) @@ -205,6 +240,9 @@ build fp_small $((ROWS / 2)) build fp_big "$ROWS" K=$((ROWS / 20)) +check "premise: the doubled group's reads fetch through its index" \ + "$(reads_index "SELECT sum(v), sum(length(t)) FROM fp_big WHERE id > $(( ROWS - K ))" fp_big_id)" "yes" + small="$(fetch_ms fp_small $((ROWS / 2)) $K)" big="$(fetch_ms fp_big "$ROWS" $K)" echo "-- $K fetches at the far end, median of three: group of $((ROWS / 2)) took ${small} ms, group of $ROWS took ${big} ms" diff --git a/test/native_parquet_flba.sh b/test/native_parquet_flba.sh index bd70df907..ec420e596 100755 --- a/test/native_parquet_flba.sh +++ b/test/native_parquet_flba.sh @@ -77,9 +77,12 @@ pq.write_table(pa.table({"d": pa.array(ds, pa.decimal128(10, 2))}), f"{W}/pa_dec PY # uuid stored as raw bytes; declare bytea to compare, since pyarrow gives no # uuid logical type, then also read as uuid (our exporter's convention). + # Both values, in order. The arm read only the smallest, the all-zero uuid, + # which every byte order leaves unchanged: with the reader reversing a + # uuid's bytes it stayed green (#1341). check "pyarrow uuid reads as uuid" \ - "$(q "SELECT u::text FROM pgcolumnar.read_parquet('$W/pa_uuid.parquet') AS t(u uuid) ORDER BY u LIMIT 1;")" \ - "00000000-0000-0000-0000-000000000000" + "$(q "SELECT string_agg(u::text, ',' ORDER BY u) FROM pgcolumnar.read_parquet('$W/pa_uuid.parquet') AS t(u uuid);")" \ + "00000000-0000-0000-0000-000000000000,11111111-2222-3333-4444-555555555555" check "pyarrow decimal128 values are exact" \ "$(pgc_set_hash "SELECT d FROM pgcolumnar.read_parquet('$W/pa_dec.parquet') AS t(d numeric(10,2))")" \ "$(pgc_set_hash "SELECT * FROM (VALUES (123.45::numeric(10,2)),(-9999.99),(0.01),(0.00)) v(d)")" @@ -172,13 +175,18 @@ PY psql_run "CREATE FUNCTION pgc_try_flba(q text) RETURNS text LANGUAGE plpgsql AS \$\$ BEGIN EXECUTE q; RETURN 'NO ERROR'; - EXCEPTION WHEN OTHERS THEN RETURN 'REJECTED'; END \$\$;" + EXCEPTION WHEN OTHERS THEN RETURN 'REJECTED ' || SQLSTATE; END \$\$;" # Fixed: the bind guard rejects scale > precision/38 -> REJECTED. Unguarded: the # decoder accepts it and returns a wrong-but-valid numeric -> NO ERROR. So this # check distinguishes fixed from unfixed without depending on a crash. + # + # BY THE BIND, WHICH IS WHAT "not decoded" MEANS. Any error used to count. + # With the bind guard removed, the decoder's own scale check still refused + # the file, as XX001 "could not decode Parquet column 0" -- after decoding + # began -- and the arm stayed green (#1341). The bind refuses with 42804. check "crafted out-of-range scale is rejected, not decoded" \ "$(q "SELECT pgc_try_flba(\$q\$SELECT * FROM pgcolumnar.read_parquet('$W/dec_badscale.parquet') AS t(d numeric)\$q\$);")" \ - "REJECTED" + "REJECTED 42804" check "backend survived the crafted scale" "$(q 'SELECT 1;')" "1" # ---- DECIMAL held in an INT32 or INT64 ---------------------------------- diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index babd7df85..4b984cccc 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -212,6 +212,10 @@ behaviour, the source of that number is named. - [164. test_native_saop_pushdown.py: IN-lists and = ANY(array) prune row groups](#164-test_native_saop_pushdownpy-in-lists-and--anyarray-prune-row-groups) - [165. test_native_groupagg_batch.py: the grouped aggregate folds column-at-a-time](#165-test_native_groupagg_batchpy-the-grouped-aggregate-folds-column-at-a-time) - [166. test_native_fetch_projection.py: a fetch by row number decodes only what its caller needs](#166-test_native_fetch_projectionpy-a-fetch-by-row-number-decodes-only-what-its-caller-needs) +- [167. test_native_fetch_position.py: a fetch by row number costs the same anywhere in its group](#167-test_native_fetch_positionpy-a-fetch-by-row-number-costs-the-same-anywhere-in-its-group) +- [168. test_native_fetch_cache.py: the fetch-by-row-number cache](#168-test_native_fetch_cachepy-the-fetch-by-row-number-cache) +- [169. test_native_parquet_pushdown.py: the Parquet FDW skips row groups its statistics rule out](#169-test_native_parquet_pushdownpy-the-parquet-fdw-skips-row-groups-its-statistics-rule-out) +- [170. test_native_parquet_flba.py: uuid and DECIMAL read from fixed-length byte arrays](#170-test_native_parquet_flbapy-uuid-and-decimal-read-from-fixed-length-byte-arrays) ## 1. How to read a test in here @@ -7691,3 +7695,54 @@ The pytest twin of `test/native_fetch_projection.sh` (ported for #1341), every c | `test_a_projection_covering_every_column_decodes_nothing_and_is_right` | with the computed base-column set empty, every row is reconstructed with the right values | | `test_no_caller_decodes_every_column_and_index_deletion_reads_no_row` | the projection path calls no full decode, and index deletion asks no row whether it is live | | `test_every_needed_set_test_consults_the_all_columns_flag` | every needed-set membership test in the reader consults the explicit all-columns flag | + +## 167. test_native_fetch_position.py: a fetch by row number costs the same anywhere in its group + +The pytest twin of `test/native_fetch_position.sh` (ported for #1341), every check by the shell's name. A cached row group still walked to the row: it counted the validity bits below the row, then decoded and discarded that many values (#143). A rank prefix over the validity bitmap and an offset table for varying-length columns make each fetch O(1). Every fetch here runs with the columnar custom scan off, and each measurement first asserts that its plan reads the index. With only `enable_seqscan=off`, every query scanned instead and none fetched by row number. The timing ratios read the same ranges three times. An UPDATE would write its rows into a new group, so later passes would fetch elsewhere. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_every_fetched_value_matches_the_heap_mirror` | the group is one row group, and fixed, varying-length and heavily-null columns fetched through the index match a heap mirror, at every probed position | +| `test_the_cost_does_not_depend_on_the_position` | fetching the last twentieth of the group costs under three times the first | +| `test_a_bigger_group_does_not_cost_more_per_fetch` | the same far-end fetches in a group twice the size cost under 1.8 times as much | +| `test_the_indexes_that_make_it_constant_are_in_the_fetch` | `pgcolumnar_fetch_row` takes the rank from the prefix and a varying-length value from the offset table | + +## 168. test_native_fetch_cache.py: the fetch-by-row-number cache + +The pytest twin of `test/native_fetch_cache.sh` (ported for #1341), every check by the shell's name. A fetch by row number decoded a whole row group per row (#143), and a statement-scoped cache of the decoded group removes the repeat. The cache is timed as a ratio: the same index fetches from one big row group and from ten small ones. Each reading runs on a freshly built fixture, because an UPDATE moves the rows it reads into a new group (#801). Then come the scoping guards in the source, a wide group's decode scratch (#353), and crossing the cache cap (#359). + +### Every test + +| test | what it holds | +| --- | --- | +| `test_the_cache_makes_one_big_group_cost_what_ten_small_ones_do` | each reading measured the geometry it meant to, one big group costs under three times ten small ones, and the updated rows are right | +| `test_the_scoping_guards_are_in_the_source` | the cache key includes the storage id, an earlier command's entry is rejected, a hit re-checks four fields, and the cache is discarded at executor end and transaction end | +| `test_a_wide_groups_decode_scratch_does_not_blow_the_cap` | a wide group's point query drives the fetch path, costs under five times a small group's, and answers the same | +| `test_crossing_the_cap_costs_per_column` | four and five projected columns drive the fetch path, five costs under twelve times four, and every over-cap column reads back its value | + +## 169. test_native_parquet_pushdown.py: the Parquet FDW skips row groups its statistics rule out + +The pytest twin of `test/native_parquet_pushdown.sh` (ported for #1341), every check by the shell's name. The FDW reads each row group's min/max and skips a group that a pushable col-op-const clause proves empty. The executor still rechecks every row, so a sound skip can only save work. The checks run against a heap oracle, and the skips themselves are read from EXPLAIN ANALYZE's "Row Groups Skipped". The soundness cases follow. A NaN row sits outside the statistics, so a float `>`, `>=` or `= 'NaN'` must never skip. An inverted interval must never drive a skip. An INT64-backed DECIMAL is skippable. pyarrow writes every file. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_skipping_never_changes_the_answer_and_really_skips` | eight predicates agree with the heap, each confining range skips the groups it should, float `>=` is not pushed, and `read_parquet` reads the same rows | +| `test_a_nan_row_is_never_skipped` | seven float predicates over a file with one NaN agree with the heap | +| `test_inverted_statistics_never_drive_a_skip` | a UINT_32 column whose statistics decode inverted agrees with the heap under six predicates and skips nothing | +| `test_an_int64_backed_decimal_is_skippable` | a range inside one of four DECIMAL groups skips the other three without dropping rows | + +## 170. test_native_parquet_flba.py: uuid and DECIMAL read from fixed-length byte arrays + +The pytest twin of `test/native_parquet_flba.sh` (ported for #1341), every check by the shell's name. The reader handles uuid as a 16-byte FLBA, and DECIMAL as big-endian two's complement at the column's scale, in FLBA, BYTE_ARRAY or a little-endian INT32/INT64. Our own export is read back through `read_parquet`, `import_parquet` and the FDW and compared with the source table. pyarrow-written files serve as the foreign producer. A footer whose DECIMAL scale has been patched out of range must be refused at bind (42804), before any decoding. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_our_export_reads_back_through_every_surface` | uuid, numeric and whole rows round-trip through `read_parquet`, `import_parquet` and the FDW | +| `test_pyarrow_written_flba_reads_exactly` | both pyarrow uuids read in order, decimal128 values are exact, and a crafted scale is refused at bind with the backend alive | +| `test_a_decimal_held_in_an_integer_reads_as_numeric` | INT32 and INT64 DECIMALs read at their scale with their null, `parquet_schema` advises numeric, and bigint still binds to the unscaled integer | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 1df468344..007a993de 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -795,4 +795,12 @@ guard_tests 438 # eight tests, by collection over the cluster files on this tree. # cluster_tests 718 -> 722 for #1341's test_native_fetch_projection.py, the twin of native_fetch_projection.sh: # four tests, by collection over the cluster files on this tree. -cluster_tests 722 +# cluster_tests 722 -> 726 for #1341's test_native_fetch_position.py, the twin of native_fetch_position.sh: +# four tests, by collection over the cluster files on this tree. +# cluster_tests 726 -> 730 for #1341's test_native_fetch_cache.py, the twin of native_fetch_cache.sh: +# four tests, by collection over the cluster files on this tree. +# cluster_tests 730 -> 734 for #1341's test_native_parquet_pushdown.py, the twin of native_parquet_pushdown.sh: +# four tests, by collection over the cluster files on this tree. +# cluster_tests 734 -> 737 for #1341's test_native_parquet_flba.py, the twin of native_parquet_flba.sh: +# three tests, by collection over the cluster files on this tree. +cluster_tests 737 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 4c323a6c0..4af427309 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -129,7 +129,9 @@ "native_dict_underfill", "native_dml", "native_encdesc_golden", + "native_fetch_cache", "native_fetch_coalesce", + "native_fetch_position", "native_fetch_projection", "native_groupagg", "native_groupagg_batch", @@ -138,6 +140,8 @@ "native_join_vector_agg", "native_ownership", "native_parquet_dict_oob", + "native_parquet_flba", + "native_parquet_pushdown", "native_parquet_schema", "native_projection", "native_reclaim", @@ -413,7 +417,7 @@ def test_the_loop_reader_invents_nothing_in_this_corpus(expect): "came from") expect.text(", ".join(files), "build_refusal, differential, join_runtime_filter, native_groupagg_batch, " - "native_index_projection, native_reclaim, native_repack, " + "native_index_projection, native_parquet_pushdown, native_reclaim, native_repack, " "native_saop_pushdown, native_vacuum_race, native_vecskip, sorted_pathkeys", "and it is these files, so another appearing is a diff a reviewer " "sees rather than a number that moved") diff --git a/test/pytest/test_native_fetch_cache.py b/test/pytest/test_native_fetch_cache.py new file mode 100644 index 000000000..5b55beaf2 --- /dev/null +++ b/test/pytest/test_native_fetch_cache.py @@ -0,0 +1,207 @@ +"""The fetch-by-row-number cache (#143, #353, #359, #1341). The pytest twin of +`test/native_fetch_cache.sh`, every check by the shell's name. + +A fetch by row number decoded a whole row group per row; a statement-scoped cache of +the decoded group removes the repeat. The cache is timed as a ratio: the same index +fetches from one big row group and from ten small ones, each reading on a freshly +built fixture because an UPDATE moves the rows it reads into a new group (#801). The +UPDATE fetches its rows by number whatever its scan; every timed point query asserts +first that its plan is an index scan, under the SETs that make one (#797). Then the +scoping guards +in the source, a wide group's decode scratch (#353), and crossing the cache cap +costing per column rather than per entry (#359). +""" +import os +import pathlib +import re +import time + +import psycopg +import pytest + +ROWS = int(os.environ.get("PGC_FETCH_ROWS", "20000")) +UPD = ROWS // 10 +SRC = pathlib.Path(__file__).resolve().parents[2] / "src" +FETCH_PATH = ("max_parallel_workers_per_gather=0", "enable_seqscan=off", "enable_bitmapscan=off", + "pgcolumnar.enable_index_fetch_penalty=off") + + +def _one(c, sql): + return c.execute(sql).fetchone()[0] + + +class _Sets: + def __init__(self, c, sets): + self.c, self.sets = c, sets + + def __enter__(self): + for s in self.sets: + self.c.execute(f"SET {s}") + + def __exit__(self, *exc): + for s in self.sets: + self.c.execute(f"RESET {s.split('=')[0]}") + + +def _row_fetch(c, sql): + """-> 'yes' when `sql` plans an Index Scan under the fetch-path SETs, as pgc_uses_row_fetch.""" + with _Sets(c, FETCH_PATH): + plan = "\n".join(r[0] for r in c.execute(f"EXPLAIN (COSTS OFF) {sql}")) + return "yes" if "Index Scan using" in plan else "no" + + +def _ms(c, sql, sets=FETCH_PATH): + with _Sets(c, sets): + t0 = time.monotonic() + c.execute(sql) + return int((time.monotonic() - t0) * 1000) + + +def _ratio(a, b, bound): + """-> 'within' when a/b, rounded to two places, is at most `bound`; else why not, + as check_ratio words it: a zero side measured nothing.""" + if a == 0 or b == 0: + return f"a side of the ratio is zero, so nothing was measured: a=[{a}] b=[{b}]" + r = round(a / b, 2) + return "within" if r <= bound else f"{r:.2f}x exceeds the {bound}x bound (a={a} b={b})" + + +def _groups(c, table): + return _one(c, f"SELECT count(*) FROM pgcolumnar.row_group r JOIN pgcolumnar.storage s USING (storage_id) " + f"WHERE s.relation_oid = '{table}'::regclass") + + +def _build(c, table, per_group): + c.execute(f"DROP TABLE IF EXISTS {table}") + c.execute(f"SET pgcolumnar.stripe_row_limit = {per_group}") + c.execute(f"SET pgcolumnar.chunk_group_row_limit = {per_group}") + try: + c.execute(f"CREATE TABLE {table} (id int, v int, t text) USING pgcolumnar") + c.execute(f"INSERT INTO {table} SELECT g, g, 'row' || g FROM generate_series(1, {ROWS}) g") + c.execute(f"CREATE INDEX {table}_id ON {table} (id)") + finally: + c.execute("RESET pgcolumnar.stripe_row_limit") + c.execute("RESET pgcolumnar.chunk_group_row_limit") + + +def _fn(path, name): + m = re.search(rf"^{name}\(.*?^}}", (SRC / path).read_text(), re.S | re.M) + return m.group(0) if m else "" + + +@pytest.fixture(scope="module") +def fc(pgc_cluster): + db = "pgc_fetchcache" + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + a.execute(f"CREATE DATABASE {db}") + try: + with psycopg.connect(pgc_cluster.dsn(dbname=db), autocommit=True) as c: + c.execute("CREATE EXTENSION pgcolumnar") + yield c + finally: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + + +def test_the_cache_makes_one_big_group_cost_what_ten_small_ones_do(fc, expect): + c = fc + # the shell's SETs for this arm; the UPDATE fetches its target rows by number anyway + upd_sets = ("max_parallel_workers_per_gather=0", "enable_seqscan=off", "enable_bitmapscan=off") + upd = f"UPDATE {{}} SET v = id + 1 WHERE id <= {UPD}" + # rebuilt before every reading: the UPDATE moves the rows it read into a new group (#801) + best, geom = {}, {} + for table, per in (("fc_one", ROWS), ("fc_many", ROWS // 10)): + times, seen = [], [] + for _ in range(3): + _build(c, table, per) + seen.append(str(_groups(c, table))) + times.append(_ms(c, upd.format(table), upd_sets)) + best[table], geom[table] = min(times), " ".join(seen) + print(f"-- {UPD} fetches: one group of {ROWS} took {best['fc_one']} ms; ten groups took {best['fc_many']} ms") + expect.text(geom["fc_one"], "1 1 1", "every fc_one reading measured one row group (#801)") + expect.text(geom["fc_many"], "10 10 10", "every fc_many reading measured ten row groups (#801)") + expect.text(_ratio(best["fc_one"], best["fc_many"], 3), "within", + "fetching from one big group is not far dearer than from ten small ones") + expect.num(_one(c, "SELECT count(*) FROM fc_one WHERE v = id + 1"), UPD, "the updated rows are correct") + expect.num(_one(c, "SELECT count(*) FROM fc_one WHERE v = id"), ROWS - UPD, "the untouched rows are unchanged") + + +def test_the_scoping_guards_are_in_the_source(expect): + reader = (SRC / "columnar_reader.c").read_text() + slot, row = _fn("columnar_reader.c", "pgcolumnar_fetch_group_slot"), _fn("columnar_reader.c", "pgcolumnar_fetch_row") + expect.text("yes" if slot and row else "no", "yes", "premise: the fetch-cache functions were extracted, not empty ranges") + lines = reader.splitlines() + grp = sum("e->groupNumber == groupNumber" in l for l in lines) + keyed = sum("e->storageId == storageId && e->groupNumber == groupNumber" in l for l in lines) + expect.at_least(grp, 1, "premise: there is a group-number comparison to key") + expect.text(f"keyed {keyed} of {grp}", f"keyed {grp} of {grp}", "every group-number comparison is keyed by the storage id too") + expect.text("yes" if "e->cid != cid" in slot else "no", "yes", "an entry from an earlier command is rejected") + for f in ("firstRowNumber", "rowCount", "fileOffset"): + expect.text("yes" if f"entry->{f} != rg->{f}" in row else "no", "yes", + f"a hit re-checks the group's {f} against the row group") + expect.text("yes" if "entry->natts != natts" in row else "no", "yes", + "a hit re-checks the group's natts against the scan's column count") + for fn in ("pgcolumnar_executor_end", "pgcolumnar_xact_callback"): + body = _fn("columnar_tableam.c", fn) + expect.text("yes" if body else "no", "yes", f"premise: {fn} was extracted, not an empty range") + expect.text("yes" if "PgColumnarDiscardFetchCache" in body else "no", "yes", + f"the fetch cache is discarded from {fn}") + + +def test_a_wide_groups_decode_scratch_does_not_blow_the_cap(fc, expect): + c = fc + for table, stripe in (("fc_wide", 150000), ("fc_wnarrow", 20000)): + c.execute(f"DROP TABLE IF EXISTS {table}") + c.execute(f"""CREATE TABLE {table} (id int, h text, c1 int,c2 int,c3 int,c4 int,c5 int,c6 int, + c7 int,c8 int,c9 int,c10 int, u float8, + t1 text,t2 text,t3 text,t4 text,t5 text,t6 text,t7 text,t8 text) USING pgcolumnar""") + c.execute(f"SELECT pgcolumnar.set_options('{table}', stripe_row_limit => {stripe})") + c.execute(f"""INSERT INTO {table} SELECT g, 'h' || (g % 500), g,g,g,g,g,g,g,g,g,g, (g % 100)::float8, + repeat('x',40),repeat('y',40),repeat('z',40),'a'||g,'b'||g,'c'||g,'d'||g,'e'||g + FROM generate_series(1, 300000) g""") + c.execute(f"CREATE INDEX {table}_h ON {table} (h)") + point = "SELECT count(*), max(u) FROM {} WHERE h='h1'" + for table in ("fc_wide", "fc_wnarrow"): + expect.text(_row_fetch(c, point.format(table)), "yes", f"the {table} point query really drives the fetch path (#797)") + bigw = min(_ms(c, point.format("fc_wide")) for _ in range(3)) + smallw = min(_ms(c, point.format("fc_wnarrow")) for _ in range(3)) + print(f"-- #353 wide point query: default-stripe {bigw} ms, small-stripe {smallw} ms") + expect.text(_ratio(bigw, smallw, 5), "within", "a wide group's fetch is not far dearer than a small group's (#353)") + both = "SELECT count(*) || '|' || coalesce(max(u)::text,'z') FROM {} WHERE h='h1'" + expect.text(_one(c, both.format("fc_wide")), _one(c, both.format("fc_wnarrow")), + "the wide-group point query is correct (#353)") + + +def test_crossing_the_cap_costs_per_column(fc, expect): + c = fc + c.execute("DROP TABLE IF EXISTS fc_w359") + c.execute("CREATE TABLE fc_w359 (id int, h text, t1 text,t2 text,t3 text,t4 text,t5 text) USING pgcolumnar") + c.execute("SELECT pgcolumnar.set_options('fc_w359', stripe_row_limit => 50000)") + c.execute("""INSERT INTO fc_w359 SELECT g, 'h' || (g % 1000), + repeat('a',150),repeat('b',150),repeat('c',150),repeat('d',150),repeat('e',150) + FROM generate_series(1,100000) g""") + c.execute("CREATE INDEX fc_w359_h ON fc_w359 (h)") + + def query(n): + return "SELECT count(*)" + "".join(f", max(t{i})" for i in range(1, n + 1)) + " FROM fc_w359 WHERE h='h7'" + + def warm_ms(n): + _ms(c, query(n)) # warm: decode cost, not first-touch I/O + return _ms(c, query(n)) + + for n in (4, 5): + expect.text(_row_fetch(c, query(n)), "yes", f"the {n}-column projection really drives the fetch path (#797)") + under = min(warm_ms(4) for _ in range(3)) # ~30 MB decoded: fits + over = min(warm_ms(5) for _ in range(3)) # ~38 MB decoded: does not + print(f"-- #359 projection width: four columns {under} ms, five columns {over} ms") + expect.text(_ratio(over, under, 12), "within", "crossing the fetch cache cap costs proportionally, not totally (#359)") + # the overflowing columns are re-decoded, not skipped: fast and null would be wrong + expect.text(_one(c, "SELECT max(t1) || '|' || max(t4) FROM fc_w359 WHERE h='h7'"), + _one(c, "SELECT max(t1) || '|' || max(t4) FROM fc_w359 WHERE h='h7' AND t5 IS NOT NULL"), + "the over-cap projection returns the same values as the under-cap one (#359)") + expect.num(_one(c, """SELECT count(*) FROM fc_w359 WHERE h='h7' + AND t1 = repeat('a',150) AND t2 = repeat('b',150) AND t3 = repeat('c',150) + AND t4 = repeat('d',150) AND t5 = repeat('e',150)"""), + _one(c, "SELECT count(*) FROM fc_w359 WHERE h='h7'"), + "every over-cap column reads back its written value (#359)") diff --git a/test/pytest/test_native_fetch_position.py b/test/pytest/test_native_fetch_position.py new file mode 100644 index 000000000..f3b2bece3 --- /dev/null +++ b/test/pytest/test_native_fetch_position.py @@ -0,0 +1,166 @@ +"""A fetch by row number costs the same wherever the row sits in its group (#143, #1341). +The pytest twin of `test/native_fetch_position.sh`, every check by the shell's name. + +A cached group still walked to the row: counting the validity bits below it, then +decoding and discarding that many values, so a fetch cost O(row) and fetching a whole +group was quadratic. A rank prefix over the validity bitmap and an offset table for +varying-length columns make it O(1). Values first, against a heap mirror over fixed, +varying-length and heavily-null columns, through an index scan; then two timing ratios +(far end against near end, and a doubled group), each the median of three passes; then +the two indexes in pgcolumnar_fetch_row's source. +""" +import os +import pathlib +import re +import statistics +import time + +import psycopg +import pytest + +# the shell reads PGC_FETCHPOS_ROWS, and so does this +ROWS = int(os.environ.get("PGC_FETCHPOS_ROWS", "120000")) +SRC = pathlib.Path(__file__).resolve().parents[2] / "src" +# enable_seqscan=off leaves the columnar custom scan on, and with only that every query +# here scanned rather than fetched by row number; each measurement asserts its plan +FORCE = ("SET enable_seqscan = off", "SET enable_bitmapscan = off", "SET max_parallel_workers_per_gather = 0", + "SET pgcolumnar.enable_custom_scan = off") + + +def _one(c, sql): + return c.execute(sql).fetchone()[0] + + +def _forced(c): + for s in FORCE: + c.execute(s) + + +def _reads_index(c, sql, index): + """-> 'yes' when `sql`'s plan is an Index Scan using `index`, else its first scan line.""" + plan = "\n".join(r[0] for r in c.execute(f"EXPLAIN (COSTS OFF) {sql}")) + if f"Index Scan using {index} " in plan: + return "yes" + first = next((l.strip() for l in plan.splitlines() if "Scan" in l or "Update" in l), plan) + return f"no: {first}" + + +def _rows_or_error(c, sql): + """-> the first value, or the error as a value: a corrupt fetch is a named red.""" + try: + return _one(c, sql) + except psycopg.Error as exc: + return f"error: {exc.diag.message_primary}" + + +def _median_ms(c, sql): + """-> the median of three timed passes of `sql`, in milliseconds.""" + times = [] + for _ in range(3): + t0 = time.monotonic() + c.execute(sql) + times.append((time.monotonic() - t0) * 1000.0) + return statistics.median(times) + + +@pytest.fixture(scope="module") +def fpos(pgc_cluster): + """-> a connection to a database holding fp_c (one row group) and its heap mirror.""" + db = "pgc_fetchpos" + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + a.execute(f"CREATE DATABASE {db}") + try: + with psycopg.connect(pgc_cluster.dsn(dbname=db), autocommit=True) as c: + c.execute("CREATE EXTENSION pgcolumnar") + # every fourth row null in the nullable columns, so rank and row number diverge + c.execute(f"SET pgcolumnar.stripe_row_limit = {ROWS}") + c.execute(f"SET pgcolumnar.chunk_group_row_limit = {ROWS}") + c.execute("CREATE TABLE fp_c (id int, fixed bigint, txt text, sparse int, stxt text) USING pgcolumnar") + c.execute(f"""INSERT INTO fp_c + SELECT g, g * 7, repeat('x', 1 + (g % 40)) || g, + CASE WHEN g % 4 = 0 THEN NULL ELSE g END, + CASE WHEN g % 4 = 0 THEN NULL ELSE 's' || g END + FROM generate_series(1, {ROWS}) g""") + c.execute("CREATE INDEX fp_c_id ON fp_c (id)") + c.execute("CREATE TABLE fp_h (LIKE fp_c)") + c.execute("INSERT INTO fp_h SELECT * FROM fp_c") + yield c + finally: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + + +def test_every_fetched_value_matches_the_heap_mirror(fpos, expect): + c = fpos + expect.num(_one(c, "SELECT count(*) FROM pgcolumnar.row_group r " + "JOIN pgcolumnar.storage s ON s.storage_id = r.storage_id " + "WHERE s.relation_oid = 'fp_c'::regclass"), 1, "the rows are in one row group") + _forced(c) + expect.text(_reads_index(c, "SELECT count(*) FROM fp_c c JOIN fp_h h USING (id) " + "WHERE c.fixed IS DISTINCT FROM h.fixed", "fp_c_id"), "yes", + "premise: the mirror comparison fetches through the index") + expect.text(_reads_index(c, "SELECT fixed FROM fp_c WHERE id = 1001", "fp_c_id"), "yes", + "premise: a single-row fetch goes through the index") + expect.text(str(_rows_or_error(c, """SELECT count(*) FROM fp_c c JOIN fp_h h USING (id) + WHERE c.fixed IS DISTINCT FROM h.fixed + OR c.txt IS DISTINCT FROM h.txt + OR c.sparse IS DISTINCT FROM h.sparse + OR c.stxt IS DISTINCT FROM h.stxt""")), "0", "every column matches the heap mirror") + # one row at a time, where an off-by-one in the rank would land on a neighbour + row = "fixed || '|' || txt || '|' || coalesce(sparse::text, 'N') || '|' || coalesce(stxt, 'N')" + bad = [] + for pos in (1, 2, 3, 4, 5, 999, 1000, 1001, ROWS // 2, ROWS - 1, ROWS): + got = _rows_or_error(c, f"SELECT {row} FROM fp_c WHERE id = {pos}") + want = _one(c, f"SELECT {row} FROM fp_h WHERE id = {pos}") + if got != want: + bad.append(f"id={pos}(got={got} want={want})") + expect.text(" ".join(bad) or "same", "same", "single-row fetches are right at every position") + + +def test_the_cost_does_not_depend_on_the_position(fpos, expect): + c = fpos + _forced(c) + n = ROWS // 20 + # a read, not an UPDATE: an UPDATE writes its rows into a new group, so only the first + # pass fetched from the far end and the median was always a later one + read = "SELECT sum(fixed), sum(length(txt)) FROM fp_c WHERE id BETWEEN {} AND {}" + expect.text(_reads_index(c, read.format(ROWS - n + 1, ROWS), "fp_c_id"), "yes", + "premise: the timed reads fetch through the index") + near = _median_ms(c, read.format(1, n)) + far = _median_ms(c, read.format(ROWS - n + 1, ROWS)) + print(f"-- {n} fetches, median of three: first tenth {near:.0f} ms, last tenth {far:.0f} ms") + # a ratio of 3 catches the quadratic walk (1,001,374 ms against 614 ms) with orders to spare + expect.text("yes" if near > 0 and far / near < 3 else f"no (near={near:.0f}ms far={far:.0f}ms)", "yes", + "fetching from the far end of a group costs about what the near end does") + + +def test_a_bigger_group_does_not_cost_more_per_fetch(fpos, expect): + c = fpos + for name, rows in (("fp_small", ROWS // 2), ("fp_big", ROWS)): + c.execute(f"SET pgcolumnar.stripe_row_limit = {rows}") + c.execute(f"SET pgcolumnar.chunk_group_row_limit = {rows}") + c.execute(f"CREATE TABLE {name} (id int, v bigint, t text) USING pgcolumnar") + c.execute(f"INSERT INTO {name} SELECT g, g * 3, 't' || g FROM generate_series(1, {rows}) g") + c.execute(f"CREATE INDEX {name}_id ON {name} (id)") + _forced(c) + # the rows fetched sit at the END of each group, where the old walk was longest + k = ROWS // 20 + read = "SELECT sum(v), sum(length(t)) FROM {} WHERE id > {}" + expect.text(_reads_index(c, read.format("fp_big", ROWS - k), "fp_big_id"), "yes", + "premise: the doubled group's reads fetch through its index") + small = _median_ms(c, read.format("fp_small", ROWS // 2 - k)) + big = _median_ms(c, read.format("fp_big", ROWS - k)) + print(f"-- {k} fetches at the far end, median of three: {ROWS // 2} rows took {small:.0f} ms, {ROWS} took {big:.0f} ms") + expect.text("yes" if small > 0 and big / small < 1.8 else f"no (small={small:.0f}ms big={big:.0f}ms)", "yes", + "doubling the row group does not double the cost of the same fetches") + + +def test_the_indexes_that_make_it_constant_are_in_the_fetch(expect): + m = re.search(r"^pgcolumnar_fetch_row\(.*?^}", (SRC / "columnar_reader.c").read_text(), re.S | re.M) + fetch = m.group(0) if m else "" + expect.text("yes" if fetch else "no", "yes", "premise: pgcolumnar_fetch_row was extracted, not an empty range") + expect.text("yes" if "present = pgcolumnar_rank_before" in fetch else "no", "yes", + "the rank comes from a prefix rather than a loop over earlier rows") + expect.text("yes" if "entry->valOffset[c][present]" in fetch else "no", "yes", + "a varying-length column reaches its value through an offset table") diff --git a/test/pytest/test_native_parquet_flba.py b/test/pytest/test_native_parquet_flba.py new file mode 100644 index 000000000..f6206c44a --- /dev/null +++ b/test/pytest/test_native_parquet_flba.py @@ -0,0 +1,210 @@ +"""Parquet FIXED_LEN_BYTE_ARRAY reads: uuid and DECIMAL (Phase G, #1341). The pytest twin +of `test/native_parquet_flba.sh`, every check by the shell's name. + +uuid as a 16-byte FLBA, and DECIMAL as big-endian two's complement at the column's +scale, fixed (FLBA) or variable (BYTE_ARRAY) width, or as a little-endian INT32/INT64. +Our own export is read back through read_parquet, import_parquet and the FDW against +the source table; pyarrow-written files are the foreign producer. A footer whose +DECIMAL scale has been patched out of range must be refused at bind, not decoded. +""" +import decimal +import uuid + +import psycopg + +HASH = ("SELECT coalesce(md5(string_agg(t, chr(10) ORDER BY t)), $e$EMPTY$e$) " + "FROM (SELECT _row::text AS t FROM ({}) _row) _s") +SHAPE = "t(id int, u uuid, d numeric(12,4), dn numeric(12,4), fb bytea)" + + +def _hash(cur, sql): + """-> the shell's pgc_set_hash of `sql`, or the error: two failing queries never agree.""" + try: + cur.execute(HASH.format(sql)) + return cur.fetchone()[0] + except psycopg.Error as exc: + return f"error: {exc.diag.message_primary} [{sql}]" + + +def _one(cur, sql): + try: + cur.execute(sql) + return str(cur.fetchone()[0]) + except psycopg.Error as exc: + return f"error: {exc.diag.message_primary}" + + +def _patch_scale(path): + """Rewrite the DECIMAL leaf's scale (SchemaElement field 7) from 2 to 50 in place: + zigzag(2)=0x04 and zigzag(50)=0x64 are one byte each, so the footer stays sound.""" + raw = bytearray(path.read_bytes()) + flen = int.from_bytes(raw[-8:-4], "little") + pos = [len(raw) - 8 - flen] + + def varint(): + sh = out = 0 + while True: + b = raw[pos[0]] + pos[0] += 1 + out |= (b & 0x7F) << sh + if not b & 0x80: + return out + sh += 7 + + def zigzag(): + u = varint() + return (u >> 1) ^ -(u & 1) + + def field(last): + b = raw[pos[0]] + pos[0] += 1 + if b == 0: + return 0, 0, last + t, d = b & 0x0F, b >> 4 + fid = last + d if d else zigzag() + return t, fid, fid + + def skip(t): + if t in (1, 2): + return + if t == 3: + pos[0] += 1 + elif t in (4, 5, 6): + zigzag() + elif t == 7: + pos[0] += 8 + elif t == 8: + blen = varint() + pos[0] += blen + elif t in (9, 10): + b = raw[pos[0]] + pos[0] += 1 + n, et = (b >> 4) & 0x0F, b & 0x0F + if n == 0x0F: + n = varint() + for _ in range(n): + skip(et) + elif t == 12: + last = 0 + while True: + ft, _f, last = field(last) + if ft == 0: + break + skip(ft) + + last = 0 + while True: + ft, fid, last = field(last) + if ft == 0: + break + if fid == 2 and ft in (9, 10): + b = raw[pos[0]] + pos[0] += 1 + n, et = (b >> 4) & 0x0F, b & 0x0F + if n == 0x0F: + n = varint() + for _ in range(n): + elast = 0 + while True: + eft, efid, elast = field(elast) + if eft == 0: + break + if efid == 7 and eft in (5, 6, 4): + assert raw[pos[0]] == 0x04, hex(raw[pos[0]]) + raw[pos[0]] = 0x64 + path.write_bytes(bytes(raw)) + return + skip(eft) + break + skip(ft) + raise AssertionError("did not find a scale field to patch") + + +def test_our_export_reads_back_through_every_surface(pgc_conn, server_dir, expect): + exp = server_dir / "roundtrip.parquet" + with pgc_conn.cursor() as cur: + cur.execute("CREATE SERVER IF NOT EXISTS pq_flba FOREIGN DATA WRAPPER pgcolumnar_parquet") + cur.execute("CREATE TABLE src (id int, u uuid, d numeric(12,4), dn numeric(12,4), fb bytea) USING pgcolumnar") + cur.execute("""INSERT INTO src VALUES + (1, '11111111-2222-3333-4444-555555555555', 123.4500, -0.0001, '\\xdeadbeef'), + (2, 'ffffffff-ffff-ffff-ffff-ffffffffffff', -9999.9999, 0, '\\x00'), + (3, '00000000-0000-0000-0000-000000000000', 0.0000, 42.0, NULL)""") + cur.execute(f"SELECT pgcolumnar.export_parquet('src'::regclass, '{exp}')") + rp = f"pgcolumnar.read_parquet('{exp}') AS {SHAPE}" + expect.text(_hash(cur, f"SELECT id, u FROM {rp}"), _hash(cur, "SELECT id, u FROM src"), + "uuid round-trips through export/read") + expect.text(_hash(cur, f"SELECT id, d, dn FROM {rp}"), _hash(cur, "SELECT id, d, dn FROM src"), + "numeric round-trips through export/read") + expect.text(_hash(cur, f"SELECT * FROM {rp}"), _hash(cur, "SELECT * FROM src"), "whole-row round-trip == source") + # import_parquet shares the decoder + cur.execute("CREATE TABLE imp (id int, u uuid, d numeric(12,4), dn numeric(12,4), fb bytea)") + cur.execute(f"SELECT pgcolumnar.import_parquet('imp'::regclass, '{exp}')") + expect.text(_hash(cur, "SELECT * FROM imp"), _hash(cur, "SELECT * FROM src"), "import_parquet == source") + cur.execute(f"CREATE FOREIGN TABLE ft (id int, u uuid, d numeric(12,4), dn numeric(12,4), fb bytea) " + f"SERVER pq_flba OPTIONS (path '{exp}')") + expect.text(_hash(cur, "SELECT * FROM ft"), _hash(cur, "SELECT * FROM src"), "FDW scan == source") + + +def test_pyarrow_written_flba_reads_exactly(pgc_conn, server_dir, expect): + pa = expect.require_module("pyarrow", name="pyarrow uuid reads as uuid") + if pa is None: + return + pq = expect.require_module("pyarrow.parquet", name="pyarrow uuid reads as uuid") + us = [uuid.UUID("11111111-2222-3333-4444-555555555555").bytes, uuid.UUID(int=0).bytes] + pq.write_table(pa.table({"u": pa.array(us, pa.binary(16))}), str(server_dir / "pa_uuid.parquet")) + ds = [decimal.Decimal("123.45"), decimal.Decimal("-9999.99"), decimal.Decimal("0.01"), decimal.Decimal("0.00")] + pq.write_table(pa.table({"d": pa.array(ds, pa.decimal128(10, 2))}), str(server_dir / "pa_dec.parquet")) + bad = server_dir / "dec_badscale.parquet" + pq.write_table(pa.table({"d": pa.array([decimal.Decimal("1.23")], pa.decimal128(10, 2))}), str(bad)) + _patch_scale(bad) + for f in ("pa_uuid.parquet", "pa_dec.parquet", "dec_badscale.parquet"): + (server_dir / f).chmod(0o644) + with pgc_conn.cursor() as cur: + # both values in order: the all-zero uuid alone is the same in every byte order + expect.text(_one(cur, f"SELECT string_agg(u::text, ',' ORDER BY u) " + f"FROM pgcolumnar.read_parquet('{server_dir}/pa_uuid.parquet') AS t(u uuid)"), + "00000000-0000-0000-0000-000000000000,11111111-2222-3333-4444-555555555555", + "pyarrow uuid reads as uuid") + expect.text(_hash(cur, f"SELECT d FROM pgcolumnar.read_parquet('{server_dir}/pa_dec.parquet') AS t(d numeric(10,2))"), + _hash(cur, "SELECT * FROM (VALUES (123.45::numeric(10,2)),(-9999.99),(0.01),(0.00)) v(d)"), + "pyarrow decimal128 values are exact") + # scale 50: the bind must refuse it (42804); the decoder's own check refusing it + # instead (XX001) means decoding began + try: + cur.execute(f"SELECT * FROM pgcolumnar.read_parquet('{bad}') AS t(d numeric)") + cur.fetchall() + verdict = "NO ERROR" + except psycopg.Error as exc: + verdict = f"REJECTED {exc.sqlstate}" + expect.text(verdict, "REJECTED 42804", "crafted out-of-range scale is rejected, not decoded") + expect.text(_one(cur, "SELECT 1"), "1", "backend survived the crafted scale") + + +def test_a_decimal_held_in_an_integer_reads_as_numeric(pgc_conn, server_dir, expect): + pa = expect.require_module("pyarrow", name="INT32-backed DECIMAL reads") + if pa is None: + return + pq = expect.require_module("pyarrow.parquet", name="INT32-backed DECIMAL reads") + vals = [decimal.Decimal("1.25"), decimal.Decimal("-3.50"), decimal.Decimal("0.00"), None] + for name, p, s in (("i32", 9, 2), ("i64", 18, 6)): + path = server_dir / f"dec_{name}.parquet" + pq.write_table(pa.table({"d": pa.array(vals, pa.decimal128(p, s))}), str(path), + store_decimal_as_integer=True, compression="none") + path.chmod(0o644) + phys = {n: pq.ParquetFile(str(server_dir / f"dec_{n}.parquet")).metadata.row_group(0).column(0).physical_type + for n in ("i32", "i64")} + assert phys == {"i32": "INT32", "i64": "INT64"}, phys + i32, i64 = server_dir / "dec_i32.parquet", server_dir / "dec_i64.parquet" + with pgc_conn.cursor() as cur: + agg = "SELECT string_agg(d::text, ',' ORDER BY d) FROM pgcolumnar.read_parquet('{}') AS t(d {})" + expect.text(_one(cur, agg.format(i32, "numeric")), "-3.50,0.00,1.25", "INT32-backed DECIMAL reads") + expect.text(_one(cur, agg.format(i64, "numeric")), "-3.500000,0.000000,1.250000", "INT64-backed DECIMAL reads") + expect.text(_one(cur, f"SELECT count(*) - count(d) FROM pgcolumnar.read_parquet('{i32}') AS t(d numeric)"), "1", + "INT32-backed DECIMAL keeps its null") + expect.text(_one(cur, f"SELECT data_type FROM pgcolumnar.parquet_schema('{i32}')"), "numeric(9,2)", + "parquet_schema advises numeric for an INT32 DECIMAL") + expect.text(_one(cur, f"SELECT data_type FROM pgcolumnar.parquet_schema('{i64}')"), "numeric(18,6)", + "parquet_schema advises numeric for an INT64 DECIMAL") + # the integer reading is not silently reinterpreted: bigint binds to the raw integer + expect.text(_one(cur, agg.format(i64, "bigint")), "-3500000,0,1250000", + "an INT64 DECIMAL still binds to bigint as the unscaled integer") diff --git a/test/pytest/test_native_parquet_pushdown.py b/test/pytest/test_native_parquet_pushdown.py new file mode 100644 index 000000000..d0e2827b3 --- /dev/null +++ b/test/pytest/test_native_parquet_pushdown.py @@ -0,0 +1,163 @@ +"""The Parquet FDW skips row groups its statistics prove empty (Phase G, #1341). The +pytest twin of `test/native_parquet_pushdown.sh`, every check by the shell's name. + +Each row group's min/max is read, and a pushable col-op-const clause skips groups it +proves empty; the executor still rechecks every row, so a skip can only save work. +Correctness against a heap oracle (skipping never changes results), and the skips +themselves through EXPLAIN ANALYZE's "Row Groups Skipped". Then soundness: a NaN row +sits outside the statistics, so float > and >= and = 'NaN' must never skip; an +inverted interval (UINT_32 stats ordered unsigned, decoded signed) must never drive a +skip; and an INT64-backed DECIMAL is skippable. Every file is written by pyarrow, an +independent writer, so the reader and the writer cannot share a misunderstanding. +""" +import decimal + +import psycopg + +HASH = ("SELECT coalesce(md5(string_agg(t, chr(10) ORDER BY t)), $e$EMPTY$e$) " + "FROM (SELECT _row::text AS t FROM ({}) _row) _s") +ANALYZE = "EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) SELECT count(*) FROM {} WHERE {}" + + +def _hash(cur, sql): + """-> the shell's pgc_set_hash of `sql`, or the error: two failing queries never agree.""" + try: + cur.execute(HASH.format(sql)) + return cur.fetchone()[0] + except psycopg.Error as exc: + return f"error: {exc.diag.message_primary} [{sql}]" + + +def _counter(cur, table, where, label): + """-> the number on the plan's `label` line, or '' when there is none.""" + cur.execute(ANALYZE.format(table, where)) + for (line,) in cur.fetchall(): + if label in line: + digits = "".join(ch if ch.isdigit() else " " for ch in line.split(label, 1)[1]).split() + return digits[0] if digits else "" + return "" + + +def _write(path, table, pq, **kw): + pq.write_table(table, str(path), **kw) + path.chmod(0o644) + + +def _ft(cur, server_dir, pa, pq): + """200000 rows sorted by id in four groups of 50000: disjoint id ranges, with stats.""" + path = server_dir / "stats.parquet" + n = 200000 + _write(path, pa.table({"id": pa.array(range(n), pa.int32()), + "val": pa.array([i * 1.5 for i in range(n)], pa.float64())}), pq, row_group_size=50000) + assert pq.ParquetFile(str(path)).num_row_groups == 4 + cur.execute("CREATE TABLE h (id int, val float8)") + cur.execute("INSERT INTO h SELECT g, g*1.5 FROM generate_series(0,199999) g") + cur.execute("CREATE SERVER IF NOT EXISTS pq FOREIGN DATA WRAPPER pgcolumnar_parquet") + cur.execute(f"CREATE FOREIGN TABLE ft (id int4, val float8) SERVER pq OPTIONS (path '{path}')") + return path + + +def test_skipping_never_changes_the_answer_and_really_skips(pgc_conn, server_dir, expect): + pa = expect.require_module("pyarrow", name="pyarrow not available; predicate-pushdown suite needs it to write stats") + if pa is None: + return + pq = expect.require_module("pyarrow.parquet", + name="pyarrow not available; predicate-pushdown suite needs it to write stats") + with pgc_conn.cursor() as cur: + path = _ft(cur, server_dir, pa, pq) + expect.text(_hash(cur, "SELECT * FROM ft"), _hash(cur, "SELECT * FROM h"), "full scan == oracle") + for pred in ("id = 60000", "id BETWEEN 60000 AND 70000", "id < 100", "id >= 199900", + "id > 60000 AND id < 60010", "id <= 5 OR id >= 199995", "val > 90000 AND val < 90030"): + expect.text(_hash(cur, f"SELECT * FROM ft WHERE {pred}"), _hash(cur, f"SELECT * FROM h WHERE {pred}"), + f"predicate [{pred}] == oracle") + + expect.text(_counter(cur, "ft", "id >= 0", "Row Groups:"), "4", "reports 4 row groups") + for name, where, want in ( + ("range in one group skips 3", "id BETWEEN 60000 AND 70000", "3"), + ("point in group 0 skips 3", "id = 123", "3"), + ("id >= 199900 skips 3", "id >= 199900", "3"), + ("id < 100 skips 3", "id < 100", "3"), + # float >= and > are never pushed: Parquet stats exclude NaN, PostgreSQL sorts it last + ("float predicate skips 2 (>= not pushed, NaN-unsafe)", "val >= 90000 AND val <= 90030", "2"), + ("float < still skips", "val < 1.0", "3"), + ("spanning predicate skips 0", "id >= 0 AND id < 200000", "0"), + ("no predicate skips 0", "true", "0"), + ): + expect.text(_counter(cur, "ft", where, "Row Groups Skipped"), want, name) + + # read_parquet over the same file ignores the stats and reads everything + expect.text(_hash(cur, f"SELECT * FROM pgcolumnar.read_parquet('{path}') AS t(id int4, val float8)"), + _hash(cur, "SELECT * FROM h"), "read_parquet same file == oracle") + + +def test_a_nan_row_is_never_skipped(pgc_conn, server_dir, expect): + pa = expect.require_module("pyarrow", name="pyarrow not available; predicate-pushdown suite needs it to write stats") + if pa is None: + return + pq = expect.require_module("pyarrow.parquet", + name="pyarrow not available; predicate-pushdown suite needs it to write stats") + path = server_dir / "nan.parquet" + vals = [float(i) for i in range(4000)] + vals[500] = float("nan") # one NaN, in row group 0 + _write(path, pa.table({"id": pa.array(range(4000), pa.int32()), "val": pa.array(vals, pa.float64())}), + pq, row_group_size=1000) + assert pq.ParquetFile(str(path)).num_row_groups == 4 + with pgc_conn.cursor() as cur: + cur.execute("CREATE SERVER IF NOT EXISTS pq_nan FOREIGN DATA WRAPPER pgcolumnar_parquet") + cur.execute(f"CREATE FOREIGN TABLE ftn (id int4, val float8) SERVER pq_nan OPTIONS (path '{path}')") + cur.execute("CREATE TABLE hn (id int, val float8)") + cur.execute("INSERT INTO hn SELECT g, CASE WHEN g = 500 THEN 'NaN'::float8 ELSE g END " + "FROM generate_series(0,3999) g") + for pred in ("val > 1e300", "val >= 1e300", "val = 'NaN'::float8", "val > 3998", "val < 10", "val <= 10", + "val = 1234"): + expect.text(_hash(cur, f"SELECT * FROM ftn WHERE {pred}"), _hash(cur, f"SELECT * FROM hn WHERE {pred}"), + f"NaN-safe [{pred}] == oracle") + + +def test_inverted_statistics_never_drive_a_skip(pgc_conn, server_dir, expect): + pa = expect.require_module("pyarrow", name="pyarrow not available; predicate-pushdown suite needs it to write stats") + if pa is None: + return + pq = expect.require_module("pyarrow.parquet", + name="pyarrow not available; predicate-pushdown suite needs it to write stats") + # UINT_32 statistics are ordered unsigned and decoded signed: min=1, max=-1294967296 + path = server_dir / "inverted.parquet" + _write(path, pa.table({"c": pa.array([1, 3000000000], pa.uint32())}), pq, row_group_size=2) + st = pq.ParquetFile(str(path)).metadata.row_group(0).column(0).statistics + assert st.min == 1 and st.max == 3000000000, (st.min, st.max) + with pgc_conn.cursor() as cur: + cur.execute("CREATE SERVER IF NOT EXISTS pq_inv FOREIGN DATA WRAPPER pgcolumnar_parquet") + cur.execute(f"CREATE FOREIGN TABLE fti (c int4) SERVER pq_inv OPTIONS (path '{path}')") + cur.execute("CREATE TABLE hi (c int4)") + cur.execute("INSERT INTO hi VALUES (1), (-1294967296)") + expect.text(_hash(cur, "SELECT * FROM fti"), _hash(cur, "SELECT * FROM hi"), + "inverted stats: unfiltered == oracle") + for pred in ("c = 1", "c = -1294967296", "c >= 1", "c <= -1294967296", "c < 0", "c > 0"): + expect.text(_hash(cur, f"SELECT * FROM fti WHERE {pred}"), _hash(cur, f"SELECT * FROM hi WHERE {pred}"), + f"inverted-stats [{pred}] == oracle") + expect.text(_counter(cur, "fti", "c = 1", "Row Groups Skipped"), "0", "inverted stats skip nothing") + + +def test_an_int64_backed_decimal_is_skippable(pgc_conn, server_dir, expect): + pa = expect.require_module("pyarrow", name="pyarrow not available; predicate-pushdown suite needs it to write stats") + if pa is None: + return + pq = expect.require_module("pyarrow.parquet", + name="pyarrow not available; predicate-pushdown suite needs it to write stats") + # four row groups with disjoint ranges, stored as INT64, the type the stats decode from + path = server_dir / "dec_push.parquet" + vals = [decimal.Decimal(i) / 100 for i in range(0, 40000)] + _write(path, pa.table({"d": pa.array(vals, pa.decimal128(18, 2))}), pq, + store_decimal_as_integer=True, compression="none", row_group_size=10000) + meta = pq.ParquetFile(str(path)).metadata + assert meta.row_group(0).column(0).physical_type == "INT64" and meta.num_row_groups == 4 + with pgc_conn.cursor() as cur: + cur.execute("CREATE SERVER IF NOT EXISTS pq_dec FOREIGN DATA WRAPPER pgcolumnar_parquet") + cur.execute(f"CREATE FOREIGN TABLE ftdec (d numeric) SERVER pq_dec OPTIONS (path '{path}')") + # 0.00 .. 399.99 in groups of 100.00: a range inside the second excludes the other three + expect.text(_counter(cur, "ftdec", "d BETWEEN 120.00 AND 130.00", "Row Groups Skipped"), "3", + "INT64-backed DECIMAL: predicate skips 3 of 4 groups") + cur.execute("SELECT count(*) FROM ftdec WHERE d BETWEEN 120.00 AND 130.00") + expect.num(cur.fetchone()[0], 1001, "INT64-backed DECIMAL: skipping did not drop rows") + expect.text(_counter(cur, "ftdec", "d >= 0", "Row Groups Skipped"), "0", + "INT64-backed DECIMAL: unfiltered scan skips nothing")