diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cf3201a42..cf16dc30a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -185,6 +185,13 @@ jobs: - name: fetch the release tags, for the fixture-blob comparisons run: git fetch --depth=1 origin '+refs/tags/v1.0-alpha*:refs/tags/v1.0-alpha*' + - name: Resolve the object-storage suite host names + # test_objstore_addressing.py, the twin of objstore_addressing.sh, reads + # through virtual-host-style S3 addressing (pgc-bucket.s3.local) against + # its own loopback fixture server, as the suites job's shells do. Without + # the names its tests decline UNAVAILABLE_ENDPOINT and the run ends 67. + run: echo "127.0.0.1 s3.local pgc-bucket.s3.local" | sudo tee -a /etc/hosts + - name: Add the PGDG repository and refresh the package lists run: | set -euo pipefail diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 75f95e4b8..1c3183a7b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -745,6 +745,13 @@ jobs: - name: fetch the release tags, for the fixture-blob comparisons run: git fetch --depth=1 origin '+refs/tags/v1.0-alpha*:refs/tags/v1.0-alpha*' + - name: Resolve the object-storage suite host names + # test_objstore_addressing.py, the twin of objstore_addressing.sh, reads + # through virtual-host-style S3 addressing (pgc-bucket.s3.local) against + # its own loopback fixture server, as the suites job's shells do. Without + # the names its tests decline UNAVAILABLE_ENDPOINT and the run ends 67. + run: echo "127.0.0.1 s3.local pgc-bucket.s3.local" | sudo tee -a /etc/hosts + - name: Add the PGDG repository and install PostgreSQL ${{ matrix.pg }} with its headers run: | set -euo pipefail diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 8b086cb67..c5f048b61 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -4009,6 +4009,19 @@ native_fetch_sort_context native_fetch_sort_context premise: the columnar side i native_fetch_sort_context native_fetch_sort_context three-key merge cond with a filter (#720) 15;16;17;18;19 2026-10-22 columnar_tableam.c: the deferred decode builds its needed-column set in the caller's context and bms_free's it there (#720 reverted, first half) native_fetch_sort_context native_fetch_sort_context two-key merge join count is right (#720) 15;16;17;18;19 2026-10-22 columnar_tableam.c: the deferred decode builds its needed-column set in the caller's context and bms_free's it there (#720 reverted, first half) native_fetch_sort_context native_fetch_sort_context two-key merge join sum(id) is right (#720) 15;16;17;18;19 2026-10-22 columnar_tableam.c: the deferred decode builds its needed-column set in the caller's context and bms_free's it there (#720 reverted, first half) +native_fold_deferral native_fold_deferral EXPLAIN group total == stats() group count 15;16;17;18;19 2026-10-22 columnar_vector.c: the fold counts each group twice in its total +native_fold_deferral native_fold_deferral adaptive: eager groups load more than survivors alone (work measured, not assumed) 15;16;17;18;19 never - +native_fold_deferral native_fold_deferral adaptive: only the optimistic first group deferred at 99% survival 15;16;17;18;19 2026-10-22 columnar_vector.c: the fold never defers, so the payload is always loaded eagerly +native_fold_deferral native_fold_deferral deferred fold == heap mirror (selective) 15;16;17;18;19 2026-10-22 columnar_vector.c: a deferred group's first survivor is never folded +native_fold_deferral native_fold_deferral deferred fold == row path (selective) 15;16;17;18;19 2026-10-22 columnar_vector.c: a deferred group's first survivor is never folded +native_fold_deferral native_fold_deferral deferred: every group deferred at 1% survival 15;16;17;18;19 2026-10-22 columnar_vector.c: the fold never defers, so the payload is always loaded eagerly +native_fold_deferral native_fold_deferral deferred: loads == survivors x payload width (the identity) 15;16;17;18;19 2026-10-22 columnar_vector.c: a deferred group's first survivor is never folded +native_fold_deferral native_fold_deferral fold == heap mirror (high survival) 15;16;17;18;19 2026-10-22 columnar_vector.c: a deferred group's first survivor is never folded +native_fold_deferral native_fold_deferral premise: the predicate has survivors (the identity is not 0 == 0) 15;16;17;18;19 never - +native_fold_deferral native_fold_deferral premise: the selective predicate is genuinely selective (<= 2%) 15;16;17;18;19 never - +native_fold_deferral native_fold_deferral the fixture spans several groups 15;16;17;18;19 2026-10-22 columnar_write_state.c: the writer ignores a table's stripe_row_limit +native_fold_deferral native_fold_deferral the loads counter exists under ANALYZE 15;16;17;18;19 never - +native_fold_deferral native_fold_deferral the selective query takes the fold (ANALYZE, post-#602) 15;16;17;18;19 2026-10-22 columnar_vector.c: the Columnar Batch Fold ANALYZE line reports no native_fold_skipguard native_fold_skipguard and reports the 58 it skipped, on its own plan (#542) 15;16;17;18;19 2026-10-22 columnar_reader.c: the fold path stops counting the vectors decode skipped (#542 regression) native_fold_skipguard native_fold_skipguard and the scalar arm agrees 15;16;17;18;19 never - native_fold_skipguard native_fold_skipguard so decoded plus skipped is the group's vectors, on the FOLD arm 15;16;17;18;19 2026-10-22 columnar_reader.c: the fold path stops counting the vectors decode skipped (#542 regression) @@ -4333,6 +4346,105 @@ native_join_vector_agg native_join_vector_agg unique join fold answer equals GUC native_join_vector_agg native_join_vector_agg unique join fold answer equals heap 15;16;17;18;19 never - native_join_vector_agg native_join_vector_agg unique join uses core Agg when GUC off 15;16;17;18;19 never - native_join_vector_agg native_join_vector_agg unique join uses vectorized agg when GUC on 15;16;17;18;19 2026-09-12 drop JOINREL fold +native_late_materialization native_late_materialization a rejected row does not materialize its payload columns 15;16;17;18;19 2026-10-22 columnar_reader.c: a row the qual rejects is skipped but never counted +native_late_materialization native_late_materialization a volatile qual is refused the two-pass path 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar.enable_custom_scan defaults to off +native_late_materialization native_late_materialization and the answers are the same with it off 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar_tuple_insert drops the row whose id is 7 +native_late_materialization native_late_materialization and the volatile query still matches the heap mirror 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar_tuple_insert drops the row whose id is 7 +native_late_materialization native_late_materialization heap agrees on how many rows match 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar_tuple_insert drops the row whose id is 7 +native_late_materialization native_late_materialization nothing was pruned at the group level 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar.enable_custom_scan defaults to off +native_late_materialization native_late_materialization nothing was pruned at the vector level 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar.enable_custom_scan defaults to off +native_late_materialization native_late_materialization premise: the fixture loaded every row 15;16;17;18;19 never - +native_late_materialization native_late_materialization premise: the needle is selective, not most of the table 15;16;17;18;19 never - +native_late_materialization native_late_materialization the executor still reports the rows the scan filtered 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar_tuple_insert drops the row whose id is 7 +native_late_materialization native_late_materialization the query runs on the columnar custom scan 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar.enable_custom_scan defaults to off +native_late_materialization native_late_materialization the rows are identical to the heap mirror 15;16;17;18;19 2026-10-22 columnar_reader.c: the late-materialization path rejects a row the qual accepts +native_late_materialization native_late_materialization the same shape without the volatile term does use it 15;16;17;18;19 2026-10-22 columnar_reader.c: a row the qual rejects is skipped but never counted +native_late_materialization native_late_materialization with the feature off nothing is filtered early 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar.enable_custom_scan defaults to off +native_lazy_slot native_lazy_slot SELECT * through the index matches heap 15;16;17;18;19 2026-10-22 columnar_tableam.c: tuple_insert stores c40 off by one in a wide bigint table, so lz_c holds a wrong value +native_lazy_slot native_lazy_slot a range of rows sums the same as heap 15;16;17;18;19 2026-10-22 columnar_tableam.c: tuple_insert stores c40 off by one in a wide bigint table, so lz_c holds a wrong value +native_lazy_slot native_lazy_slot a row fetched before it is flushed reads correctly 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar_slot_decode_upto decodes one column fewer than the prefix asked for +native_lazy_slot native_lazy_slot a unique violation is still detected through the fetch path 15;16;17;18;19 never - +native_lazy_slot native_lazy_slot a wide table is not in a different class from a narrow one 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar_slot_decode_upto decodes the whole row whatever prefix was asked for +native_lazy_slot native_lazy_slot an ORDER BY over index-fetched rows matches heap 15;16;17;18;19 2026-10-22 columnar_tableam.c: the index fetch no longer clears the slot, so a new row keeps the previous row's decoded prefix +native_lazy_slot native_lazy_slot copying index-fetched rows into a heap table keeps every column 15;16;17;18;19 2026-10-22 columnar_tableam.c: tuple_insert stores c40 off by one in a wide bigint table, so lz_c holds a wrong value +native_lazy_slot native_lazy_slot count(*) through the index matches heap 15;16;17;18;19 never - +native_lazy_slot native_lazy_slot single-row index fetches match heap for every column position 15;16;17;18;19 2026-10-22 columnar_tableam.c: tuple_insert stores c40 off by one in a wide bigint table, so lz_c holds a wrong value +native_lazy_slot native_lazy_slot the fetch really is an index scan 15;16;17;18;19 never - +native_metadata_flush native_metadata_flush a 20-column flush opens a small constant number of metadata relations, not one per row 15;16;17;18;19 2026-10-22 columnar_metadata.c: open_columnar_table never reuses a session's open relation (#445 reverted), so each metadata row opens its catalog again +native_metadata_flush native_metadata_flush a zone-map-pruned range read matches heap (the batched zone rows are correct) 15;16;17;18;19 never - +native_metadata_flush native_metadata_flush and that range read rules out a vector by its zone row alone 15;16;17;18;19 2026-10-22 columnar_metadata.c: zone_map rows are written with a NULL minimum and maximum, so no vector can be ruled out by its zone row +native_metadata_flush native_metadata_flush and the 20-column flush opens at least the three tables every flush writes 15;16;17;18;19 2026-10-22 columnar_metadata.c: the flush session never counts an open, so the witness reads opens=0 for every flush +native_metadata_flush native_metadata_flush every column reads back correct (sum over all 20 columns matches the heap mirror) 15;16;17;18;19 2026-10-22 columnar_reader.c: PgColumnarEncodeValueByLen stores a 4-byte by-value 1777 as 1778 +native_metadata_flush native_metadata_flush narrow (3 col) and wide (20 col) flush open the same number of metadata relations 15;16;17;18;19 2026-10-22 columnar_metadata.c: open_columnar_table never reuses a session's open relation (#445 reverted), so each metadata row opens its catalog again +native_metadata_flush native_metadata_flush the flush emits the metadata-open witness 15;16;17;18;19 2026-10-22 columnar_write_state.c: the flush never begins a metadata session, so it neither batches nor emits the witness +native_metadata_flush native_metadata_flush the wide table's row count survives the batched flush 15;16;17;18;19 never - +native_page_offset_bound native_page_offset_bound alive after projected overrun 15;16;17;18;19 never - +native_page_offset_bound native_page_offset_bound alive after projected page_offset=-1 15;16;17;18;19 never - +native_page_offset_bound native_page_offset_bound alive after whole-group overrun 15;16;17;18;19 2026-10-22 columnar_reader.c: the whole-group read's chunk containment guard removed +native_page_offset_bound native_page_offset_bound alive after whole-group page_offset=-1 15;16;17;18;19 2026-10-22 columnar_reader.c: the whole-group read's chunk containment guard removed +native_page_offset_bound native_page_offset_bound alive after whole-group wrap 15;16;17;18;19 2026-10-22 columnar_reader.c: the whole-group read's chunk containment guard removed +native_page_offset_bound native_page_offset_bound an intact table reads every row and value with projection off 15;16;17;18;19 2026-10-22 columnar_reader.c: both containment guards refuse a chunk ending exactly at its group end +native_page_offset_bound native_page_offset_bound an intact table's projected read returns every id 15;16;17;18;19 2026-10-22 columnar_reader.c: both containment guards refuse every chunk +native_page_offset_bound native_page_offset_bound projected page_offset overrun is refused by the containment guard 15;16;17;18;19 2026-10-22 columnar_reader.c: the projected read's chunk containment guard removed +native_page_offset_bound native_page_offset_bound projected page_offset=-1 hits the containment guard 15;16;17;18;19 2026-10-22 columnar_reader.c: the projected read's chunk containment guard removed +native_page_offset_bound native_page_offset_bound whole-group page_offset overrun is refused by the containment guard 15;16;17;18;19 2026-10-22 columnar_reader.c: the whole-group read's chunk containment guard removed +native_page_offset_bound native_page_offset_bound whole-group page_offset=-1 hits the containment guard 15;16;17;18;19 2026-10-22 columnar_reader.c: the whole-group read's chunk containment guard removed +native_page_offset_bound native_page_offset_bound whole-group page_offset=0 is refused by the containment guard 15;16;17;18;19 2026-10-22 columnar_reader.c: the whole-group read's chunk containment guard removed +native_param_pushdown native_param_pushdown a correlated PARAM_EXEC builds no skip predicate 15;16;17;18;19 2026-10-22 columnar_customscan.c: a correlated PARAM_EXEC is frozen into a scan key too +native_param_pushdown native_param_pushdown a correlated PARAM_EXEC is not mis-pruned (results stay correct) 15;16;17;18;19 never - +native_param_pushdown native_param_pushdown a correlated PARAM_EXEC upper bound is not mis-pruned either 15;16;17;18;19 2026-10-22 columnar_customscan.c: a correlated PARAM_EXEC is frozen into a scan key too +native_param_pushdown native_param_pushdown a generic-plan parameterized qual prunes chunk groups (19 of 20 removed) 15;16;17;18;19 2026-10-22 columnar_customscan.c: an int4 scan-key operand is frozen 2000 above its value +native_param_pushdown native_param_pushdown a second execution in the same session counts with its own value 15;16;17;18;19 2026-10-22 columnar_customscan.c: an int4 scan-key operand is frozen 2000 above its value +native_param_pushdown native_param_pushdown and prunes with its own value too (18 of 20 removed) 15;16;17;18;19 2026-10-22 columnar_customscan.c: an int4 scan-key operand is frozen 2000 above its value +native_param_pushdown native_param_pushdown backend alive 15;16;17;18;19 never - +native_param_pushdown native_param_pushdown literal count is the expected value 15;16;17;18;19 never - +native_param_pushdown native_param_pushdown parameterized count matches the literal count 15;16;17;18;19 2026-10-22 columnar_customscan.c: an int4 scan-key operand is frozen 2000 above its value +native_param_pushdown native_param_pushdown the correlated arm rescans the columnar scan once per outer row 15;16;17;18;19 never - +native_parquet_codecs native_parquet_codecs backend survived the crafted header 15;16;17;18;19 2026-10-22 columnar_parquet_codec.c: a page declaring a size past MaxAllocSize aborts instead of being refused +native_parquet_codecs native_parquet_codecs crafted uncompressed size gives a clean decode error 15;16;17;18;19 2026-10-22 columnar_parquet_codec.c: a page declaring a size past MaxAllocSize aborts instead of being refused +native_parquet_codecs native_parquet_codecs gzip FDW count 15;16;17;18;19 never - +native_parquet_codecs native_parquet_codecs gzip decodes to the generator's rows 15;16;17;18;19 2026-10-22 columnar_parquet_codec.c: every decompressed page over 4 KB has one byte flipped after decompression, the same way for every codec +native_parquet_codecs native_parquet_codecs gzip sum matches the generator 15;16;17;18;19 2026-10-22 columnar_parquet_codec.c: every decompressed page over 4 KB has one byte flipped after decompression, the same way for every codec +native_parquet_codecs native_parquet_codecs lz4 FDW count 15;16;17;18;19 never - +native_parquet_codecs native_parquet_codecs lz4 decodes to the generator's rows 15;16;17;18;19 2026-10-22 columnar_parquet_codec.c: every decompressed page over 4 KB has one byte flipped after decompression, the same way for every codec +native_parquet_codecs native_parquet_codecs premise: the gzip file is GZIP-compressed 15;16;17;18;19 never - +native_parquet_codecs native_parquet_codecs premise: the lz4 file is LZ4-compressed 15;16;17;18;19 never - +native_parquet_codecs native_parquet_codecs premise: the uncompressed file is UNCOMPRESSED 15;16;17;18;19 never - +native_parquet_codecs native_parquet_codecs premise: the zstd file is ZSTD-compressed 15;16;17;18;19 never - +native_parquet_codecs native_parquet_codecs uncompressed decodes to the generator's rows 15;16;17;18;19 2026-10-22 columnar_parquet_codec.c: every decompressed page over 4 KB has one byte flipped after decompression, the same way for every codec +native_parquet_codecs native_parquet_codecs zstd FDW count 15;16;17;18;19 never - +native_parquet_codecs native_parquet_codecs zstd decodes to the generator's rows 15;16;17;18;19 2026-10-22 columnar_parquet_codec.c: every decompressed page over 4 KB has one byte flipped after decompression, the same way for every codec +native_parquet_fdw native_parquet_fdw EXPLAIN shows Foreign Scan 15;16;17;18;19 never - +native_parquet_fdw native_parquet_fdw a one-column query reads one of the eleven columns 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: projection pushdown ignored, every column decoded +native_parquet_fdw native_parquet_fdw a selective predicate skips the three row groups it cannot match 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: row-group skipping never skips a group +native_parquet_fdw native_parquet_fdw and the skipped scan still returns every matching row 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pqfdw_compute_skip moves each group's skip decision to the next group, skipping the one that matches +native_parquet_fdw native_parquet_fdw backend still up after the refusals 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the scan's missing-path refusal removed (the backend crashes) +native_parquet_fdw native_parquet_fdw fdw row count 15;16;17;18;19 never - +native_parquet_fdw native_parquet_fdw full scan: fdw == source 15;16;17;18;19 never - +native_parquet_fdw native_parquet_fdw nested export rows 15;16;17;18;19 never - +native_parquet_fdw native_parquet_fdw nested full scan: fdw == source 15;16;17;18;19 never - +native_parquet_fdw native_parquet_fdw nested-loop self-join (rescan): fdw == source 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: every row group skipped once any qual is pushable +native_parquet_fdw native_parquet_fdw predicate: fdw == source 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: every row group skipped once any qual is pushable +native_parquet_fdw native_parquet_fdw projection: fdw == source 15;16;17;18;19 never - +native_parquet_fdw native_parquet_fdw reject empty path option (HV00D) 15;16;17;18;19 never - +native_parquet_fdw native_parquet_fdw reject invalid table option (HV00D) 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the option validator accepts an unknown option +native_parquet_fdw native_parquet_fdw scalar export rows 15;16;17;18;19 never - +native_parquet_fdw native_parquet_fdw scan of non-parquet file errors (XX001) 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the short-file refusal removed, so the bad-magic guard refuses instead +native_parquet_fdw native_parquet_fdw scan with incompatible declared type errors (42804) 15;16;17;18;19 never - +native_parquet_fdw native_parquet_fdw scan without path option errors (HV00J) 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the scan's missing-path refusal removed (the backend crashes) +native_parquet_fdw native_parquet_fdw skip export rows 15;16;17;18;19 never - +native_parquet_fdw native_parquet_fdw the skip file has four row groups 15;16;17;18;19 2026-10-22 columnar_parquet.c: the exporter writes 131072-row groups instead of 65536 +native_parquet_fieldid native_parquet_fieldid a duplicate field id in the file is refused (42702) 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a requested field id binds the column whose id is one more +native_parquet_fieldid native_parquet_fieldid a duplicate requested field id is refused (22023) 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a requested field id binds the column whose id is one more +native_parquet_fieldid native_parquet_fieldid a field_ids/column-count mismatch is refused (22023) 15;16;17;18;19 never - +native_parquet_fieldid native_parquet_fieldid a file with no field ids is refused (22023) 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the 'file carries no field ids' guard removed, so an id-less file falls to the absent-id refusal +native_parquet_fieldid native_parquet_fieldid a nested/array output column is refused (0A000) 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the scalar-only guard on field-id projection removed +native_parquet_fieldid native_parquet_fieldid an absent field id is refused (22023) 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the absent-field-id refusal becomes abort() +native_parquet_fieldid native_parquet_fieldid backend still up after the refusals 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the absent-field-id refusal becomes abort() +native_parquet_fieldid native_parquet_fieldid field-id projection binds and reorders by id (g=gamma, a=alpha) 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a requested field id binds the column whose id is one more +native_parquet_fieldid native_parquet_fieldid field-id projection reads a subset (only id 3 = beta) 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a requested field id binds the column whose id is one more +native_parquet_fieldid native_parquet_fieldid positional read_parquet is unaffected 15;16;17;18;19 never - +native_parquet_fieldid native_parquet_fieldid the duplicate-requested-id error names the id, not the file 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a requested field id binds the column whose id is one more native_parquet_flba native_parquet_flba FDW scan == source 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a FIXED_LEN_BYTE_ARRAY uuid's bytes are reversed on read native_parquet_flba native_parquet_flba INT32-backed DECIMAL keeps its null 15;16;17;18;19 never - native_parquet_flba native_parquet_flba INT32-backed DECIMAL reads 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: an INT32 or INT64 DECIMAL is widened byte-reversed;columnar_parquet_reader.c: pq_decimal_to_numeric reads a narrower-than-16-byte negative DECIMAL as its unsigned magnitude @@ -4348,6 +4460,24 @@ native_parquet_flba native_parquet_flba pyarrow decimal128 values are exact 15;1 native_parquet_flba native_parquet_flba pyarrow uuid reads as uuid 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a FIXED_LEN_BYTE_ARRAY uuid's bytes are reversed on read native_parquet_flba native_parquet_flba uuid round-trips through export/read 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a FIXED_LEN_BYTE_ARRAY uuid's bytes are reversed on read native_parquet_flba native_parquet_flba whole-row round-trip == source 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a FIXED_LEN_BYTE_ARRAY uuid's bytes are reversed on read +native_parquet_hardening native_parquet_hardening backend survived short chunk list 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the chunk-count refusal becomes abort() +native_parquet_hardening native_parquet_hardening date in range still reads 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: an in-range date decodes one day late +native_parquet_hardening native_parquet_hardening date invalid -> 22008 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the date validity check removed (the offset-overflow branch kept) +native_parquet_hardening native_parquet_hardening date offset overflow -> 22008 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the whole date guard removed, so the epoch offset wraps +native_parquet_hardening native_parquet_hardening import_parquet also raises 22003 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the int2 range guard removed, so 40000 wraps to -25536 +native_parquet_hardening native_parquet_hardening import_parquet left no wrapped rows 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the int2 range guard removed, so 40000 wraps to -25536 +native_parquet_hardening native_parquet_hardening int2 in range still reads 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the int2 guard over-applied, refusing the in-range 30000 +native_parquet_hardening native_parquet_hardening int2 out of range -> 22003 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the int2 range guard removed, so 40000 wraps to -25536 +native_parquet_hardening native_parquet_hardening no wrapped -25536 appears 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the int2 range guard removed, so 40000 wraps to -25536 +native_parquet_hardening native_parquet_hardening parquet_schema still describes a bad chunk list 15;16;17;18;19 never - +native_parquet_hardening native_parquet_hardening read_parquet also raises 22003 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the int2 range guard removed, so 40000 wraps to -25536 +native_parquet_hardening native_parquet_hardening read_parquet in range still reads 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the int2 guard over-applied, refusing the in-range 30000 +native_parquet_hardening native_parquet_hardening short chunk list rejected with a specific error 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the row-group chunk-count guard removed +native_parquet_hardening native_parquet_hardening time in range still reads 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: an in-range INT64 time decodes one microsecond late +native_parquet_hardening native_parquet_hardening time out of day -> 22008 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the INT64 time range guard removed +native_parquet_hardening native_parquet_hardening timestamp in range still reads 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: an in-range timestamp decodes one microsecond late +native_parquet_hardening native_parquet_hardening timestamp overflow -> 22008 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the timestamp guard removed, so the epoch offset wraps +native_parquet_hardening native_parquet_hardening valid two-column file still reads 15;16;17;18;19 never - native_parquet_multifile native_parquet_multifile EXPLAIN reports 3 files 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension native_parquet_multifile native_parquet_multifile EXPLAIN sums row groups across files 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension native_parquet_multifile native_parquet_multifile FDW(directory) == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension @@ -4380,6 +4510,58 @@ native_parquet_multifile native_parquet_multifile predicate skips the two non-ma native_parquet_multifile native_parquet_multifile read_parquet(directory) == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension native_parquet_multifile native_parquet_multifile schema-mismatched file in directory errors 15;16;17;18;19 never - native_parquet_multifile native_parquet_multifile tree read order follows the sorted full path 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked directory's files are not sorted;columnar_parquet_reader.c: a walked entry is read whatever its extension +native_parquet_partition native_parquet_partition IN on a partition column prunes 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning never drops a file +native_parquet_partition native_parquet_partition __HIVE_DEFAULT_PARTITION__ reads as null 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition a byte invalid in the server encoding is rejected 15;16;17;18;19 never - +native_parquet_partition native_parquet_partition a file missing a declared partition component errors 15;16;17;18;19 never - +native_parquet_partition native_parquet_partition a file whose basename looks like a partition component is unaffected 15;16;17;18;19 never - +native_parquet_partition native_parquet_partition a null partition value is not the literal marker 15;16;17;18;19 never - +native_parquet_partition native_parquet_partition a null partition value prunes 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning never drops a file +native_parquet_partition native_parquet_partition a percent-encoded partition value decodes 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition a predicate matching everything prunes nothing 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition a predicate on a file column prunes no files 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition a pruned file's row groups are never counted 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning never drops a file +native_parquet_partition native_parquet_partition a stable clause still prunes 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning never drops a file +native_parquet_partition native_parquet_partition a volatile clause keeps every matching partition row 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition a volatile clause prunes nothing 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the volatile-clause guard on partition pruning removed +native_parquet_partition native_parquet_partition all four partitions read 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition all three partitions still read 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition an empty partition_columns option is rejected 15;16;17;18;19 never - +native_parquet_partition native_parquet_partition an encoded null byte in a partition value is rejected 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the explicit encoded-NUL guard removed, so the encoding check's 22021 stands in +native_parquet_partition native_parquet_partition an inequality on a partition column prunes 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning never drops a file +native_parquet_partition native_parquet_partition an unencoded value is unaffected 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition an unknown partition column errors rather than being ignored 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the unknown-partition-column guard removed, so a later column-count error stands in +native_parquet_partition native_parquet_partition and reads the other half 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning never drops a file +native_parquet_partition native_parquet_partition backend survived the bad declarations 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the unknown-partition-column refusal becomes abort() +native_parquet_partition native_parquet_partition backend survived the malformed partition values 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the unknown-partition-column refusal becomes abort() +native_parquet_partition native_parquet_partition both columns prune down to one file 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning never drops a file +native_parquet_partition native_parquet_partition declaring a file column as a partition column errors 15;16;17;18;19 never - +native_parquet_partition native_parquet_partition group by a partition column 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition one partition value prunes half the files 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning never drops a file +native_parquet_partition native_parquet_partition partition column is typed, not text 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition partition prune plus row-group skip 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition partition values materialize 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition pruned scan still returns the right rows 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition row groups are still skipped inside the surviving files 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning never drops a file +native_parquet_partition native_parquet_partition the encoded value is queryable as itself 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition the tree reads again once both are gone 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_projection native_parquet_projection after a dropped column, one column reads 1 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pqfdw_compute_needed matches the needed set by target index, not by attribute number +native_parquet_projection native_parquet_projection after a dropped column, projection [d] == oracle 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pqfdw_compute_needed matches the needed set by target index, not by attribute number +native_parquet_projection native_parquet_projection count(*) decodes nothing, survives corruption 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pqfdw_compute_needed marks every column needed, so projection decodes all +native_parquet_projection native_parquet_projection count(*) reads 0 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pqfdw_compute_needed marks every column needed, so projection decodes all +native_parquet_projection native_parquet_projection good-column values are still correct 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pqfdw_compute_needed marks every column needed, so projection decodes all +native_parquet_projection native_parquet_projection one column reads 1 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pqfdw_compute_needed marks every column needed, so projection decodes all +native_parquet_projection native_parquet_projection projection [*] == oracle 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a BYTE_ARRAY text longer than 2 bytes decodes without its last byte (shared with read_parquet) +native_parquet_projection native_parquet_projection projection [a, d] == oracle 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the last target column is never decoded, even when referenced +native_parquet_projection native_parquet_projection projection [a] == oracle 15;16;17;18;19 never - +native_parquet_projection native_parquet_projection projection [b, c, d] == oracle 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a BYTE_ARRAY text longer than 2 bytes decodes without its last byte (shared with read_parquet) +native_parquet_projection native_parquet_projection projection [c] == oracle 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a BYTE_ARRAY text longer than 2 bytes decodes without its last byte (shared with read_parquet) +native_parquet_projection native_parquet_projection qual on unselected column == oracle 15;16;17;18;19 never - +native_parquet_projection native_parquet_projection select star reads 4 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the last target column is never decoded, even when referenced +native_parquet_projection native_parquet_projection selected 1 + qual 1 reads 2 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pqfdw_compute_needed marks every column needed, so projection decodes all +native_parquet_projection native_parquet_projection selecting the corrupt column does error 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the last target column is never decoded, even when referenced +native_parquet_projection native_parquet_projection selecting the good column skips the corrupt one 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pqfdw_compute_needed marks every column needed, so projection decodes all +native_parquet_projection native_parquet_projection two columns read 2 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the last target column is never decoded, even when referenced native_parquet_pushdown native_parquet_pushdown INT64-backed DECIMAL: predicate skips 3 of 4 groups 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: INT64 statistics are not decoded, so an INT64-backed DECIMAL never skips;columnar_parquet_reader.c: pqfdw_compute_skip never marks a row group skippable native_parquet_pushdown native_parquet_pushdown INT64-backed DECIMAL: skipping did not drop rows 15;16;17;18;19 never - native_parquet_pushdown native_parquet_pushdown INT64-backed DECIMAL: unfiltered scan skips nothing 15;16;17;18;19 never - @@ -4462,6 +4644,69 @@ native_parquet_schema native_parquet_schema type of c_text 15;16;17;18;19 never native_parquet_schema native_parquet_schema type of c_time 15;16;17;18;19 never - native_parquet_schema native_parquet_schema type of c_ts 15;16;17;18;19 never - native_parquet_schema native_parquet_schema type of c_uuid 15;16;17;18;19 never - +native_parquet_stack native_parquet_stack a shallow valid schema chain still reads 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: walk_schema's guard trips on any nesting, not only on stack exhaustion +native_parquet_stack native_parquet_stack backend survived read_parquet on the crash input 15;16;17;18;19 2026-10-22 columnar_thrift.c: PgColumnarThriftSkip calls abort() where its stack guard would raise +native_parquet_stack native_parquet_stack backend survived the nested-struct footer 15;16;17;18;19 2026-10-22 columnar_thrift.c: PgColumnarThriftSkip calls abort() where its stack guard would raise +native_parquet_stack native_parquet_stack backend survived the schema-chain footer 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: walk_schema calls abort() where its stack guard would raise +native_parquet_stack native_parquet_stack backend survived the valid file 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: walk_schema calls abort() on entry, so any footer that parses crashes the backend +native_parquet_stack native_parquet_stack nested-struct footer raises stack-depth, not SIGSEGV 15;16;17;18;19 2026-10-22 columnar_thrift.c: PgColumnarThriftSkip's check_stack_depth() removed, so a nested-struct footer is not bounded +native_parquet_stack native_parquet_stack read_parquet rejects the nested-struct footer the same way 15;16;17;18;19 2026-10-22 columnar_thrift.c: PgColumnarThriftSkip's check_stack_depth() removed, so a nested-struct footer is not bounded +native_parquet_stack native_parquet_stack schema-chain footer raises stack-depth, not SIGSEGV 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: walk_schema's check_stack_depth() removed, so a schema chain is not bounded +native_parquet_streaming native_parquet_streaming a second dictionary page in a chunk is rejected 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the one-dictionary-page-per-chunk guard removed +native_parquet_streaming native_parquet_streaming all-null column (empty dictionary page) still reads 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming backend survived the crafted page areas 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the second-dictionary-page refusal becomes abort() +native_parquet_streaming native_parquet_streaming backend survived the failed read 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming negative page offset is rejected by the offset check 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_read_page_header no longer refuses a negative page offset +native_parquet_streaming native_parquet_streaming page size past end of file is rejected by the size bound 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: compressed_size no longer bounded by what is left of the file +native_parquet_streaming native_parquet_streaming parquet_schema reads a file over the 1GB palloc ceiling 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming parquet_schema reports the columns of the oversized file 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming parquet_schema still reads a normal file 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming read_parquet still reads a normal file 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming streaming: LIMIT 10 (inside group 0) decodes one row group 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming streaming: LIMIT 1010 (into group 1) decodes two row groups 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming streaming: a full scan decodes all four row groups 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming streaming: full scan content is correct 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming streaming: rescan under a nested loop is correct 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming the uncompressed source file still reads 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming unparseable page header is rejected 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming v2 level lengths past the page are rejected 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming value read of the sparse file fails cleanly 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming zero-filled page area is rejected, not walked byte by byte 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the progress guard for a data page with no values removed +native_parquet_units native_parquet_units import_parquet honours the unit 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_scale_to_usecs reads MILLIS as micros (no x1000) +native_parquet_units native_parquet_units millis == micros 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_scale_to_usecs reads MILLIS as micros (no x1000) +native_parquet_units native_parquet_units millis pushdown result == oracle 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_scale_to_usecs reads MILLIS as micros (no x1000) +native_parquet_units native_parquet_units millis-scaled stats skip the non-matching group 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_scale_to_usecs reads MILLIS as micros (no x1000) +native_parquet_units native_parquet_units millis-to-micros overflow raises 22008 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_scale_to_usecs reads MILLIS as micros (no x1000) +native_parquet_units native_parquet_units nanos == micros 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_scale_to_usecs reads NANOS as micros (no /1000) +native_parquet_units native_parquet_units nanos read as bigint is exact 15;16;17;18;19 never - +native_parquet_units native_parquet_units nanos sub-us truncates toward zero [frac] 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_scale_to_usecs reads NANOS as micros (no /1000) +native_parquet_units native_parquet_units nanos sub-us truncates toward zero [neg] 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_scale_to_usecs reads NANOS as micros (no /1000) +native_parquet_units native_parquet_units parquet_schema advises bigint for nanos (lossless) 15;16;17;18;19 never - +native_parquet_units native_parquet_units parquet_schema advises time for TIME_MILLIS 15;16;17;18;19 never - +native_parquet_units native_parquet_units parquet_schema advises timestamp for micros 15;16;17;18;19 never - +native_parquet_units native_parquet_units parquet_schema advises timestamp for millis 15;16;17;18;19 never - +native_parquet_units native_parquet_units read_parquet honours the unit 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_scale_to_usecs reads MILLIS as micros (no x1000) +native_parquet_units native_parquet_units the millis file reads back exactly as written 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_scale_to_usecs reads MILLIS as micros (no x1000) +native_parquet_units native_parquet_units time micros decodes to the right time 15;16;17;18;19 never - +native_parquet_units native_parquet_units time millis decodes to the right time 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_scale_to_usecs reads MILLIS as micros (no x1000) +native_parquet_units native_parquet_units ts_ms decodes to the right instant 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_scale_to_usecs reads MILLIS as micros (no x1000) +native_parquet_units native_parquet_units ts_ns decodes to the right instant 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_scale_to_usecs reads NANOS as micros (no /1000) +native_parquet_units native_parquet_units ts_us decodes to the right instant 15;16;17;18;19 never - +native_read_parquet native_read_parquet a role without pg_read_server_files is refused by read_parquet's own gate 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: read_parquet's own pg_read_server_files gate removed +native_read_parquet native_read_parquet and a member of pg_read_server_files reads every row 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a scalar's definition level ignored, every entry read as present +native_read_parquet native_read_parquet backend still up after the refusals 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a scalar's definition level ignored, every entry read as present +native_read_parquet native_read_parquet import parity rows 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a scalar's definition level ignored, every entry read as present +native_read_parquet native_read_parquet nested export rows 15;16;17;18;19 never - +native_read_parquet native_read_parquet nested round-trip: read_parquet == the generated rows 15;16;17;18;19 2026-10-22 columnar_reader.c: the columnar reader returns int4 777 as 778 (store_att_byval), so the source table and the file agree on the wrong value +native_read_parquet native_read_parquet premise: reject missing column definition list 15;16;17;18;19 never - +native_read_parquet native_read_parquet projected column round-trips 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a scalar's definition level ignored, every entry read as present +native_read_parquet native_read_parquet read_parquet == import_parquet 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a scalar's definition level ignored, every entry read as present +native_read_parquet native_read_parquet reject fewer columns than file 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the leaf-count guard removed +native_read_parquet native_read_parquet reject incompatible declared type 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the scalar physical-type guard removed +native_read_parquet native_read_parquet reject non-parquet file 15;16;17;18;19 never - +native_read_parquet native_read_parquet scalar export rows 15;16;17;18;19 never - +native_read_parquet native_read_parquet scalar read_parquet row count 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a scalar's definition level ignored, every entry read as present +native_read_parquet native_read_parquet scalar round-trip: read_parquet == the generated rows 15;16;17;18;19 2026-10-22 columnar_reader.c: the columnar reader returns int4 777 as 778 (store_att_byval), so the source table and the file agree on the wrong value native_reclaim native_reclaim and every live row is still there, once 15;16;17;18;19 never - native_reclaim native_reclaim compact_rewrite ACCEPTS the boundary threshold 0.0 15;16;17;18;19 2026-10-16 columnar_vacuum.c: compact_rewrite's guard refuses minFrac <= 0.0 native_reclaim native_reclaim compact_rewrite ACCEPTS the boundary threshold 1.0 15;16;17;18;19 2026-10-16 columnar_vacuum.c: compact_rewrite's guard refuses minFrac >= 1.0 @@ -4499,6 +4744,19 @@ native_reclaim native_reclaim repeated recluster reuses space (file plateaus) 15 native_reclaim native_reclaim reused-block data matches heap mirror 15;16;17;18;19 never - native_reclaim native_reclaim row count correct after cycle 15;16;17;18;19 never - native_reclaim native_reclaim ten row groups, every one half deleted 15;16;17;18;19 2026-10-14 the write state ignores stripe_row_limit +native_reclaim_reconcile native_reclaim_reconcile compact_rewrite reconciled the stale row 15;16;17;18;19 2026-10-22 columnar_metadata.c: PgColumnarReconcileFreeList never drops a free row, overlapping or not +native_reclaim_reconcile native_reclaim_reconcile compact_rewrite recorded the free ranges it left 15;16;17;18;19 2026-10-22 columnar_metadata.c: record_free_space drops its insert, so every freed range leaks +native_reclaim_reconcile native_reclaim_reconcile data intact after compact reconcile 15;16;17;18;19 never - +native_reclaim_reconcile native_reclaim_reconcile data intact after reconcile + rewrite 15;16;17;18;19 never - +native_reclaim_reconcile native_reclaim_reconcile data intact after truncate reconcile 15;16;17;18;19 never - +native_reclaim_reconcile native_reclaim_reconcile parity after reuse cycle 1 15;16;17;18;19 never - +native_reclaim_reconcile native_reclaim_reconcile parity after reuse cycle 2 15;16;17;18;19 2026-10-22 columnar_metadata.c: PgColumnarReconcileFreeList never drops a free row, overlapping or not +native_reclaim_reconcile native_reclaim_reconcile parity before 15;16;17;18;19 never - +native_reclaim_reconcile native_reclaim_reconcile plain compact kept the legitimate free ranges, which overlap no live group 15;16;17;18;19 2026-10-22 columnar_metadata.c: PgColumnarReconcileFreeList drops every free row, not only those overlapping a live group +native_reclaim_reconcile native_reclaim_reconcile plain compact reconciled the stale row 15;16;17;18;19 2026-10-22 columnar_metadata.c: PgColumnarReconcileFreeList never drops a free row, overlapping or not +native_reclaim_reconcile native_reclaim_reconcile row count intact 15;16;17;18;19 never - +native_reclaim_reconcile native_reclaim_reconcile stale overlapping free row injected 15;16;17;18;19 never - +native_reclaim_reconcile native_reclaim_reconcile truncate reconciled the stale row 15;16;17;18;19 2026-10-22 columnar_metadata.c: PgColumnarReconcileFreeList never drops a free row, overlapping or not native_recluster native_recluster VACUUM set the visibility map, so the index-only count answers from the index before the recluster 15;16;17;18;19 never - native_recluster native_recluster box parity (pre-recluster) 15;16;17;18;19 never - native_recluster native_recluster full-table parity after recluster 15;16;17;18;19 2026-10-16 columnar_vacuum.c: recluster_online retires none of the old groups (PgColumnarRetireGroup removed) @@ -4535,6 +4793,37 @@ native_repack native_repack the columnar table is unharmed: content hash 19 2026 native_repack native_repack the columnar table is unharmed: row count 19 2026-10-16 columnar_tableam.c: relation_copy_for_cluster returns without raising, so a rewrite copies nothing native_repack native_repack the columnar table is unharmed: storage options 19 never - native_repack native_repack without an identity index it is refused by PostgreSQL, not by us 19 never - +native_rewrite native_rewrite a group deleted below the threshold is not rewritten 15;16;17;18;19 2026-10-22 columnar_vacuum.c: every group with a live row is a rewrite candidate, deleted or not +native_rewrite native_rewrite a rewrite plus later writes reuse no group number 15;16;17;18;19 never - +native_rewrite native_rewrite a second rewrite with nothing deleted rewrites no group 15;16;17;18;19 2026-10-22 columnar_vacuum.c: every group with a live row is a rewrite candidate, deleted or not +native_rewrite native_rewrite an index range read after vacuum returns the rows asked for 15;16;17;18;19 2026-10-22 columnar_vacuum.c: rewrite_one_group inserts no index entries for the rows it moves +native_rewrite native_rewrite compact_rewrite holds ShareUpdateExclusiveLock 15;16;17;18;19 never - +native_rewrite native_rewrite compact_rewrite holds no AccessExclusiveLock 15;16;17;18;19 never - +native_rewrite native_rewrite dead-row space reclaimed 15;16;17;18;19 never - +native_rewrite native_rewrite deleted id absent via index 15;16;17;18;19 never - +native_rewrite native_rewrite every live row still reads back after rewrite and vacuum 15;16;17;18;19 2026-10-22 columnar_vacuum.c: rewrite_one_group drops the first live row of each group it rewrites +native_rewrite native_rewrite group numbers issued after a rewrite are past every earlier one 15;16;17;18;19 never - +native_rewrite native_rewrite index scan returns each live row once 15;16;17;18;19 2026-10-22 columnar_vacuum.c: rewrite_one_group drops the first live row of each group it rewrites +native_rewrite native_rewrite live id present via index 15;16;17;18;19 2026-10-22 columnar_vacuum.c: rewrite_one_group drops the first live row of each group it rewrites +native_rewrite native_rewrite parity after delete 15;16;17;18;19 never - +native_rewrite native_rewrite parity after rewrite 15;16;17;18;19 2026-10-22 columnar_vacuum.c: rewrite_one_group drops the first live row of each group it rewrites +native_rewrite native_rewrite rewrote some groups 15;16;17;18;19 never - +native_rewrite native_rewrite row count after delete 15;16;17;18;19 never - +native_rewrite native_rewrite row count after rewrite 15;16;17;18;19 2026-10-22 columnar_vacuum.c: rewrite_one_group drops the first live row of each group it rewrites +native_rewrite native_rewrite there are deleted rows to reclaim 15;16;17;18;19 never - +native_rewrite native_rewrite unique constraint still enforced 15;16;17;18;19 2026-10-22 columnar_vacuum.c: rewrite_one_group drops the first live row of each group it rewrites +native_rewrite native_rewrite vacuum rewrites into a fresh storage id 15;16;17;18;19 never - +native_rewrite_conc native_rewrite_conc a compact_rewrite holds its lock on t_col mid-verb 15;16;17;18;19 2026-10-22 columnar_vacuum.c: compact_rewrite no longer honours pgcolumnar.maintenance_hold_ms +native_rewrite_conc native_rewrite_conc a reader with a 3 s lock_timeout finishes while the rewrite holds its lock 15;16;17;18;19 2026-10-22 columnar_vacuum.c: compact_rewrite opens the table with AccessExclusiveLock instead of ShareUpdateExclusiveLock +native_rewrite_conc native_rewrite_conc a writer with a 3 s lock_timeout inserts while the rewrite holds its lock 15;16;17;18;19 2026-10-22 columnar_vacuum.c: compact_rewrite opens the table with AccessExclusiveLock instead of ShareUpdateExclusiveLock +native_rewrite_conc native_rewrite_conc all workers succeeded (retried past serialization failures) 15;16;17;18;19 2026-10-22 columnar_metadata.c: every delete flush is refused as a concurrent compaction (over-apply) +native_rewrite_conc native_rewrite_conc and the rewrite still held its lock when both had finished 15;16;17;18;19 2026-10-22 columnar_vacuum.c: compact_rewrite no longer honours pgcolumnar.maintenance_hold_ms +native_rewrite_conc native_rewrite_conc conc-rewrite count/sum 15;16;17;18;19 2026-10-22 columnar_metadata.c: the F3b conflict check never raises, so a delete racing a committed rewrite lands on the retired group +native_rewrite_conc native_rewrite_conc conc-rewrite range 15;16;17;18;19 2026-10-22 columnar_metadata.c: the F3b conflict check never raises, so a delete racing a committed rewrite lands on the retired group +native_rewrite_conc native_rewrite_conc conc-rewrite whole-row 15;16;17;18;19 2026-10-22 columnar_metadata.c: the F3b conflict check never raises, so a delete racing a committed rewrite lands on the retired group +native_rewrite_conc native_rewrite_conc row count sane 15;16;17;18;19 never - +native_rewrite_conc native_rewrite_conc the compactor rewrote groups while the workers were writing 15;16;17;18;19 2026-10-22 columnar_metadata.c: every delete flush is refused as a concurrent compaction (over-apply) +native_rewrite_conc native_rewrite_conc the held rewrite itself completed 15;16;17;18;19 never - native_roundtrip native_roundtrip native aggregate (sum over a scanned column) 15;16;17;18;19 2026-10-17 columnar_reader.c: the native reader never loads the last row group native_roundtrip native_roundtrip native count with filter 15;16;17;18;19 2026-10-17 columnar_reader.c: the native reader never loads the last row group native_roundtrip native_roundtrip native filtered scan 15;16;17;18;19 2026-10-17 columnar_reader.c: a null by-reference column reads back as a present empty value;columnar_reader.c: zone-map pruning skips the groups that can match and keeps the ones that cannot @@ -4597,6 +4886,29 @@ native_saop_pushdown native_saop_pushdown the predicate builder's reject express native_saop_pushdown native_saop_pushdown the predicate builder's reject expression contains SK_SEARCHNULL 15;16;17;18;19 2026-10-16 columnar_reader.c: the reject expression keeps only SK_ISNULL native_saop_pushdown native_saop_pushdown vector-agg count over an IN-list is exact (conservative keys not folded) 15;16;17;18;19 2026-10-16 columnar_reader.c: a set key never probes the bloom filter (it probes zero values and rules the group out) native_saop_pushdown native_saop_pushdown vector-agg path engaged for the guard arm (plan shape) 15;16;17;18;19 never - +native_sort_by native_sort_by backend still up after the refusals 15;16;17;18;19 2026-10-22 columnar_vacuum.c: vacuum_sorted with no key declared aborts the backend in place of its refusal +native_sort_by native_sort_by declared sort_by round-trips 15;16;17;18;19 never - +native_sort_by native_sort_by empty table sorts without error 15;16;17;18;19 2026-10-22 columnar_vacuum.c: the sorted rewrite orders each key descending +native_sort_by native_sort_by equality parity (post-sort) 15;16;17;18;19 never - +native_sort_by native_sort_by equality parity (pre-sort) 15;16;17;18;19 never - +native_sort_by native_sort_by explicit columns lay the rows out as the declared key does 15;16;17;18;19 2026-10-22 columnar_vacuum.c: the declared key loses its first column, so the rewrite sorts by ts alone +native_sort_by native_sort_by explicit-column sort parity 15;16;17;18;19 never - +native_sort_by native_sort_by full-table parity after sort 15;16;17;18;19 never - +native_sort_by native_sort_by premise: before sorting, the scan order does not follow (host, ts) 15;16;17;18;19 never - +native_sort_by native_sort_by row count 15;16;17;18;19 never - +native_sort_by native_sort_by row count after sort 15;16;17;18;19 never - +native_sort_by native_sort_by set_options rejects a nonexistent sort_by column 15;16;17;18;19 never - +native_sort_by native_sort_by set_options rejects a virtual generated sort_by column 18;19 never - +native_sort_by native_sort_by single-row parity 15;16;17;18;19 never - +native_sort_by native_sort_by single-row table sorts without error 15;16;17;18;19 2026-10-22 columnar_vacuum.c: the sorted rewrite orders each key descending +native_sort_by native_sort_by sort_by cleared 15;16;17;18;19 never - +native_sort_by native_sort_by sorting a text key enables group skipping 15;16;17;18;19 2026-10-22 columnar_vacuum.c: vacuum_sorted rewrites in arrival order (no sort keys) +native_sort_by native_sort_by the plan under test is a columnar custom scan 15;16;17;18;19 never - +native_sort_by native_sort_by the scan order follows the declared key (host, ts) 15;16;17;18;19 2026-10-22 columnar_vacuum.c: the sorted rewrite applies only the first sort key (host), leaving ts unordered +native_sort_by native_sort_by vacuum_sorted rejects an explicit virtual generated column 18;19 2026-10-22 columnar_vacuum.c: the virtual-column refusal aborts the backend in place of its error (PG18+) +native_sort_by native_sort_by virtual generated sort keys 15;16;17 never - +native_sort_by native_sort_by zero-arg errors on a dropped declared column 15;16;17;18;19 2026-10-22 columnar_vacuum.c: a dropped declared sort_by column gets the generic column-does-not-exist message +native_sort_by native_sort_by zero-arg errors when no key declared 15;16;17;18;19 2026-10-22 columnar_vacuum.c: vacuum_sorted with no key declared aborts the backend in place of its refusal native_toasted_write native_toasted_write equality on a toasted value (bloom + recheck) matches heap 15;16;17;18;19 2026-10-17 columnar_write_state.c: the bloom hash of a varlena is taken from a fixed value, not the row's flattened one;columnar_write_state.c: the running maximum is never raised past a chunk's first value native_toasted_write native_toasted_write premise: each range selects some rows and not all 15;16;17;18;19 never - native_toasted_write native_toasted_write premise: the source values are toasted (pglz) 15;16;17;18;19 never - @@ -4634,6 +4946,34 @@ native_varlena_bound native_varlena_bound backend survived the corrupt minimum 1 native_varlena_bound native_varlena_bound backend survived the external-tagged minimum 15;16;17;18;19 never - native_varlena_bound native_varlena_bound the txt column has a whole-chunk zone map to corrupt 15;16;17;18;19 never - native_varlena_bound native_varlena_bound with the minimum restored, the range read is clean 15;16;17;18;19 never - +native_vecdecode native_vecdecode EXPLAIN reports Columnar Vectors Decoded 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar.enable_custom_scan defaults to off +native_vecdecode native_vecdecode a nullable aggregate reaches the fold 15;16;17;18;19 2026-10-22 columnar_vector.c: pgcolumnar_batch_shape_eligible refuses every shape, so no aggregate reaches the batch fold +native_vecdecode native_vecdecode a range matching no row at all is still correct 15;16;17;18;19 never - +native_vecdecode native_vecdecode a ruled-out vector is not decoded 15;16;17;18;19 2026-10-22 columnar_reader.c: the zone maps never build a skip mask, so no vector is ruled out +native_vecdecode native_vecdecode aggregate over a range spanning many skipped vectors matches heap 15;16;17;18;19 never - +native_vecdecode native_vecdecode aggregate over a skipping range matches heap 15;16;17;18;19 never - +native_vecdecode native_vecdecode and EVERY ruled-out vector is: decoded plus skipped is the group's vectors 15;16;17;18;19 2026-10-22 columnar_reader.c: decode ignores the skip mask and decodes every vector +native_vecdecode native_vecdecode and it agrees with heap after many leading skipped vectors 15;16;17;18;19 never - +native_vecdecode native_vecdecode and it is not stuck at zero when a full scan decodes everything 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar.enable_custom_scan defaults to off +native_vecdecode native_vecdecode boundary range parity, a range starting exactly on a vector edge 15;16;17;18;19 never - +native_vecdecode native_vecdecode control: and the same counter reads every vector when nothing is skipped 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar.enable_custom_scan defaults to off +native_vecdecode native_vecdecode cross-vector parity of a nullable column, nulls included 15;16;17;18;19 never - +native_vecdecode native_vecdecode cross-vector range parity 15;16;17;18;19 never - +native_vecdecode native_vecdecode narrow range parity with heap 15;16;17;18;19 never - +native_vecdecode native_vecdecode no whole group is pruned on the narrow query 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar.enable_custom_scan defaults to off +native_vecdecode native_vecdecode one row group, so every difference is per-vector 15;16;17;18;19 2026-10-22 columnar_write_state.c: the writer divides the table's stripe_row_limit by 100 +native_vecdecode native_vecdecode premise: the nullable column has nulls inside the ranges read below 15;16;17;18;19 never - +native_vecdecode native_vecdecode the FOLD ARM really decodes only the vectors it needs 15;16;17;18;19 2026-10-22 columnar_reader.c: the zone maps never build a skip mask, so no vector is ruled out +native_vecdecode native_vecdecode the aggregate really reaches the batch fold 15;16;17;18;19 2026-10-22 columnar_vector.c: pgcolumnar_batch_shape_eligible refuses every shape, so no aggregate reaches the batch fold +native_vecdecode native_vecdecode the deep range reaches the fold too 15;16;17;18;19 2026-10-22 columnar_vector.c: pgcolumnar_batch_shape_eligible refuses every shape, so no aggregate reaches the batch fold +native_vecdecode native_vecdecode the fold answers correctly over a range that skips vectors 15;16;17;18;19 never - +native_vecdecode native_vecdecode the fold answers count and sum of a nullable column over a skipping range 15;16;17;18;19 2026-10-22 columnar_vector.c: the fold ignores the validity bitmap, so a null row is read as a present value +native_vecdecode native_vecdecode the fold does not count rows from a vector it never decoded 15;16;17;18;19 2026-10-22 columnar_vector.c: the fold ignores the skip mask and reads a skipped vector's hole as values (the hole zero-filled so the read is deterministic) +native_vecdecode native_vecdecode the narrow plan is a columnar custom scan 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar.enable_custom_scan defaults to off +native_vecdecode native_vecdecode the narrow range skips vectors 15;16;17;18;19 2026-10-22 columnar_reader.c: the zone maps never build a skip mask, so no vector is ruled out +native_vecdecode native_vecdecode the poison-accepting range reaches the fold 15;16;17;18;19 2026-10-22 columnar_vector.c: pgcolumnar_batch_shape_eligible refuses every shape, so no aggregate reaches the batch fold +native_vecdecode native_vecdecode the wide plan is a columnar custom scan 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar.enable_custom_scan defaults to off +native_vecdecode native_vecdecode the wide range skips none, so the arms really differ 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar.enable_custom_scan defaults to off native_vecskip native_vecskip and the aggregate skips the same seven vectors 15;16;17;18;19 2026-10-16 columnar_reader.c: no vector skip at all (zone-map rule off, refine_skipvec and qual_skipvec return at once) native_vecskip native_vecskip cross-vector range parity 15;16;17;18;19 never - native_vecskip native_vecskip empty range parity 15;16;17;18;19 never - @@ -4659,6 +4999,48 @@ native_writer native_writer native row groups (5000 rows / 1000 limit) 15;16;17; native_writer native_writer native row-group row total 15;16;17;18;19 never - native_writer native_writer native storage row written 15;16;17;18;19 never - native_writer native_writer native vector length recorded 15;16;17;18;19 2026-10-16 columnar.h: COLUMNAR_NATIVE_VECTOR_LENGTH 2048 +native_zonemap native_zonemap every vector's maximum of id, k and v is the heap's 15;16;17;18;19 2026-10-22 columnar_write_state.c: no vector carries a minimum or maximum, and so no chunk does either +native_zonemap native_zonemap every vector's minimum of id, k and v is the heap's 15;16;17;18;19 2026-10-22 columnar_write_state.c: no vector carries a minimum or maximum, and so no chunk does either +native_zonemap native_zonemap every whole-chunk maximum of id, k and v is the heap's 15;16;17;18;19 2026-10-22 columnar_write_state.c: no vector carries a minimum or maximum, and so no chunk does either +native_zonemap native_zonemap every whole-chunk minimum of id, k and v is the heap's 15;16;17;18;19 2026-10-22 columnar_write_state.c: no vector carries a minimum or maximum, and so no chunk does either +native_zonemap native_zonemap id (int4) min width = 4 bytes 15;16;17;18;19 never - +native_zonemap native_zonemap k (int8) max width = 8 bytes 15;16;17;18;19 never - +native_zonemap native_zonemap min/max present for all columns 15;16;17;18;19 2026-10-22 columnar_write_state.c: no vector carries a minimum or maximum, and so no chunk does either +native_zonemap native_zonemap null_count sums (v, nullable) 15;16;17;18;19 never - +native_zonemap native_zonemap null_count zero (id, no nulls) 15;16;17;18;19 never - +native_zonemap native_zonemap per-vector value+null = 1024 15;16;17;18;19 never - +native_zonemap native_zonemap per-vector value_count folds into chunk (k) 15;16;17;18;19 never - +native_zonemap native_zonemap per-vector zone rows 15;16;17;18;19 never - +native_zonemap native_zonemap premise: v is not ascending within a vector, so a stale minimum would show 15;16;17;18;19 never - +native_zonemap native_zonemap read parity (no regression) 15;16;17;18;19 never - +native_zonemap native_zonemap row count 15;16;17;18;19 never - +native_zonemap native_zonemap sum null for non-int4 columns 15;16;17;18;19 never - +native_zonemap native_zonemap sum present for int4 columns 15;16;17;18;19 never - +native_zonemap native_zonemap value_count sums (id, no nulls) 15;16;17;18;19 never - +native_zonemap native_zonemap value_count sums (v, nullable) 15;16;17;18;19 never - +native_zonemap native_zonemap whole-chunk sum(id) matches heap 15;16;17;18;19 never - +native_zonemap native_zonemap whole-chunk value+null = group rows 15;16;17;18;19 never - +native_zonemap native_zonemap whole-chunk zone rows 15;16;17;18;19 never - +native_zonemap native_zonemap zone maps cleaned on drop 15;16;17;18;19 never - +native_zonemap_narrow native_zonemap_narrow a pruning predicate still prunes on the wide table 15;16;17;18;19 2026-10-22 columnar_reader.c: group skipping excludes every group +native_zonemap_narrow native_zonemap_narrow and on the narrow table 15;16;17;18;19 2026-10-22 columnar_reader.c: group skipping reads the predicate column's zone map but never excludes a group +native_zonemap_narrow native_zonemap_narrow narrow scan result is correct 15;16;17;18;19 2026-10-22 columnar_reader.c: group skipping excludes every group +native_zonemap_narrow native_zonemap_narrow the zone map removes the nine groups c1 > 19000 cannot match, on the wide table 15;16;17;18;19 2026-10-22 columnar_reader.c: group skipping reads the predicate column's zone map but never excludes a group +native_zonemap_narrow native_zonemap_narrow wide scan result is correct 15;16;17;18;19 2026-10-22 columnar_reader.c: group skipping excludes every group +native_zonemap_narrow native_zonemap_narrow zone_map reads scale with predicates, not table width (wide ~= narrow) 15;16;17;18;19 2026-10-22 columnar_reader.c: every attribute's zone map is read up front (#314 reverted) +native_zonemap_session native_zonemap_session a second predicate column still resolves zone_map once 15;16;17;18;19 2026-10-22 columnar_metadata.c: the session opens zone_map afresh on every probe (#744 reverted inside the session) +native_zonemap_session native_zonemap_session and each of them opened exactly once 15;16;17;18;19 2026-10-22 columnar_reader.c: the scan's zone-map session is one static shared across scans and reset at the end hook (the first implementation of #744) +native_zonemap_session native_zonemap_session and each of them probed exactly its own groups 15;16;17;18;19 2026-10-22 columnar_reader.c: the scan's zone-map session is one static shared across scans and reset at the end hook (the first implementation of #744) +native_zonemap_session native_zonemap_session and the short circuit means it adds one probe, not one per group 15;16;17;18;19 2026-10-22 columnar_reader.c: every zone-map fetch is made twice +native_zonemap_session native_zonemap_session every scan around an aborted one reports its own session (#744) 15;16;17;18;19 2026-10-22 columnar_reader.c: the scan's zone-map session is one static shared across scans and reset at the end hook (the first implementation of #744) +native_zonemap_session native_zonemap_session nested columnar scans each hold their own handle 15;16;17;18;19 2026-10-22 columnar_reader.c: the scan's zone-map session is one static shared across scans and reset at the end hook (the first implementation of #744) +native_zonemap_session native_zonemap_session nested columnar scans return the right answer 15;16;17;18;19 2026-10-22 columnar_reader.c: every chunk group is ruled out +native_zonemap_session native_zonemap_session the fixture has more than one chunk group 15;16;17;18;19 2026-10-22 columnar_write_state.c: the writer multiplies the table's stripe_row_limit by 100 +native_zonemap_session native_zonemap_session the middle scan really did error inside the subtransaction 15;16;17;18;19 2026-10-22 columnar_reader.c: every chunk group is ruled out +native_zonemap_session native_zonemap_session the scan after the subtransaction abort returns the right answer 15;16;17;18;19 2026-10-22 columnar_reader.c: the scan's zone-map session is one static shared across scans and reset at the end hook (the first implementation of #744) +native_zonemap_session native_zonemap_session the scan really probed the zone maps, once per group 15;16;17;18;19 2026-10-22 columnar_reader.c: every zone-map fetch is made twice +native_zonemap_session native_zonemap_session the scan resolves zone_map ONCE, not once per group (#744) 15;16;17;18;19 2026-10-22 columnar_metadata.c: the session opens zone_map afresh on every probe (#744 reverted inside the session) +native_zonemap_session native_zonemap_session zs2 has more than one chunk group 15;16;17;18;19 2026-10-22 columnar_write_state.c: the writer ignores the table's stripe_row_limit nologin_workers nologin_workers a NOLOGIN role's parallel flush encodes every column in a worker 15;16;17;18;19 2026-10-21 the workers connect as the calling role, as before #1585, so PG15 and PG16 refuse a NOLOGIN caller nologin_workers nologin_workers a NOLOGIN role's parallel_copy loads every row 15;16;17;18;19 2026-10-21 the workers connect as the calling role, as before #1585, so PG15 and PG16 refuse a NOLOGIN caller nologin_workers nologin_workers a NOLOGIN role's parallel_export_parquet exports every row 15;16;17;18;19 2026-10-21 the workers connect as the calling role, as before #1585, so PG15 and PG16 refuse a NOLOGIN caller @@ -4674,6 +5056,22 @@ nologin_workers nologin_workers premise: SET ROLE to the NOLOGIN role takes 15;1 nologin_workers nologin_workers premise: the NOLOGIN role cannot open a session 15;16;17;18;19 never - nologin_workers nologin_workers premise: the server log records the NOLOGIN role's refused login 15;16;17;18;19 never - nologin_workers nologin_workers the flush is dispatched to workers 15;16;17;18;19 never - +objstore_addressing objstore_addressing a key with a space, '+' and '=' reads back (path-style) 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: os_uriencode_path encodes a space as '+' +objstore_addressing objstore_addressing a key with a space, '+' and '=' reads back (virtual-host) 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: os_uriencode_path encodes a space as '+' +objstore_addressing objstore_addressing and its path-style request path is the key URI-encoded once 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: virtual-host addressing always (objstore_s3_addressing='path' ignored) +objstore_addressing objstore_addressing and its virtual-host request path is the key alone, URI-encoded once 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: path-style addressing always (objstore_s3_addressing='virtual' ignored) +objstore_addressing objstore_addressing and the request path carried the bucket again 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: virtual-host addressing always (objstore_s3_addressing='path' ignored) +objstore_addressing objstore_addressing gs:// read round-trips (scheme handled, HMAC creds reused) 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the AWS_ENDPOINT_URL override is dropped (ep = NULL) +objstore_addressing objstore_addressing gs:// read-back of the written object 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the AWS_ENDPOINT_URL override is dropped (ep = NULL) +objstore_addressing objstore_addressing gs:// with no endpoint goes to storage.googleapis.com (the allow-list refuses that host by name) 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: gs:// demands AWS_ENDPOINT_URL like s3:// instead of defaulting to storage.googleapis.com +objstore_addressing objstore_addressing gs:// write round-trips 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the AWS_ENDPOINT_URL override is dropped (ep = NULL) +objstore_addressing objstore_addressing path-style put the bucket in the request path 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: virtual-host addressing always (objstore_s3_addressing='path' ignored) +objstore_addressing objstore_addressing premise: fixture up 15;16;17;18;19 never - +objstore_addressing objstore_addressing removal proof: with addressing=path the bucket is back in the path 15;16;17;18;19 never - +objstore_addressing objstore_addressing s3:// path-style read 15;16;17;18;19 never - +objstore_addressing objstore_addressing s3:// with no endpoint demands one (names AWS_ENDPOINT_URL) 15;16;17;18;19 never - +objstore_addressing objstore_addressing virtual-host put the KEY ALONE in the path (no bucket segment) 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: path-style addressing always (objstore_s3_addressing='virtual' ignored) +objstore_addressing objstore_addressing virtual-host read == path-style read (same object, different addressing) 15;16;17;18;19 never - objstore_allowlist objstore_allowlist a multi-entry list matches any entry 15;16;17;18;19 never - objstore_allowlist objstore_allowlist a non-superuser cannot widen the list (42501) 15;16;17;18;19 2026-10-14 objstore_allowed_endpoints defined PGC_SIGHUP, so no session may SET it objstore_allowlist objstore_allowlist a superuser session SET works (admin override) 15;16;17;18;19 2026-10-14 objstore_allowed_endpoints defined PGC_SIGHUP, so no session may SET it @@ -4722,6 +5120,56 @@ objstore_crlf objstore_crlf a CR/LF in the URL path is rejected before the reque objstore_crlf objstore_crlf backend alive 15;16;17;18;19 never - objstore_crlf objstore_crlf backend alive after the REST attempt 15;16;17;18;19 never - objstore_crlf objstore_crlf no injected line reached the listener (no request splitting) 15;16;17;18;19 2026-10-14 the request-line guard on the URL path removed +objstore_listing objstore_listing FDW over a prefix == read_parquet over the prefix 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the listing stops after its first page +objstore_listing objstore_listing Hive over a remote prefix stamps the partition column 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the last key of every page is dropped +objstore_listing objstore_listing a LIST that returns HTTP 500 is refused, not read as an empty prefix 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: a non-2xx LIST read as an empty, untruncated page +objstore_listing objstore_listing a non-advancing continuation token is refused, not an infinite paging loop 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the non-advancing continuation token refusal removed +objstore_listing objstore_listing a partition predicate prunes a remote file (Files Pruned = 1) 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the last key of every page is dropped +objstore_listing objstore_listing a truncated page with no keys is refused, not looped 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the truncated-page-with-no-keys refusal removed +objstore_listing objstore_listing an exact-key read issues no ListObjectsV2 15;16;17;18;19 never - +objstore_listing objstore_listing and the request log recorded the exact-key read 15;16;17;18;19 never - +objstore_listing objstore_listing backend still up after the listing refusals 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: abort() in place of the non-advancing token refusal (the backend crashes) +objstore_listing objstore_listing exact-key read returns its rows 15;16;17;18;19 never - +objstore_listing objstore_listing glob s3://.../lst/*.parquet matches the 3 top-level parts, not sub/ 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the listing stops after its first page +objstore_listing objstore_listing hidden (_SUCCESS) and non-parquet (notes.txt) objects are excluded 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the listing stops after its first page +objstore_listing objstore_listing premise: fixture up 15;16;17;18;19 never - +objstore_listing objstore_listing read_parquet over a prefix reads every object under it (4 parts, recursive) 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the listing stops after its first page +objstore_listing objstore_listing the prefix listing paged (more than one list-type=2 request) 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the listing stops after its first page +objstore_s3_read objstore_s3_read 403 ordering premise: verified requests still reach 404 = 58P01 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: a 404 on open is refused 28000 instead of 58P01 +objstore_s3_read objstore_s3_read 403: tamper bucket surfaces 28000 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: a 403 on open is read as an empty object +objstore_s3_read objstore_s3_read A: count(*) over s3 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: AWS_DEFAULT_REGION outranks AWS_REGION +objstore_s3_read objstore_s3_read A: full scan s3 == local 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: AWS_DEFAULT_REGION outranks AWS_REGION +objstore_s3_read objstore_s3_read A: percent-encoded key s3 == local 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: AWS_DEFAULT_REGION outranks AWS_REGION +objstore_s3_read objstore_s3_read A: projection s3 == local 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: AWS_DEFAULT_REGION outranks AWS_REGION +objstore_s3_read objstore_s3_read B: signing left the request arithmetic unchanged 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: AWS_DEFAULT_REGION outranks AWS_REGION +objstore_s3_read objstore_s3_read backend still up after the s3 refusals 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: abort cleanup of an open handle calls abort() after the ERROR reached the client +objstore_s3_read objstore_s3_read config: s3 read with no AWS_SECRET_ACCESS_KEY is refused naming it 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: a missing AWS_SECRET_ACCESS_KEY signs with an empty secret +objstore_s3_read objstore_s3_read config: s3 read with no environment is 28000 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: s3:// with no AWS_ENDPOINT_URL falls back to a default host instead of refusing +objstore_s3_read objstore_s3_read config: s3 read with no region is refused naming it 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: with no region configured the signer uses us-east-1 instead of refusing +objstore_s3_read objstore_s3_read https endpoint is not downgraded 15;16;17;18;19 never - +objstore_s3_read objstore_s3_read premise: fixture written 15;16;17;18;19 never - +objstore_s3_read objstore_s3_read premise: signed run logged requests 15;16;17;18;19 never - +objstore_s3_read objstore_s3_read premise: sigv4 fixture server is up 15;16;17;18;19 never - +objstore_s3_read objstore_s3_read region: AWS_DEFAULT_REGION is the fallback 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: AWS_DEFAULT_REGION is never consulted +objstore_s3_read objstore_s3_read region: AWS_REGION outranks AWS_DEFAULT_REGION 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: AWS_DEFAULT_REGION outranks AWS_REGION +objstore_stash_recovery objstore_stash_recovery and the installed module is still the one that was there, not the debris 15;16;17;18;19 2026-10-22 objstore_module.sh start guard with no debris branch: every stash restored over the installed module +objstore_stash_recovery objstore_stash_recovery and the module left installed is still valid 15;16;17;18;19 never - +objstore_stash_recovery objstore_stash_recovery and the stash is cleared 15;16;17;18;19 2026-10-22 objstore_module.sh start guard refuses on any stash, as before #393's fix +objstore_stash_recovery objstore_stash_recovery and the stash is not left behind for the next run to trip on 15;16;17;18;19 2026-10-22 objstore_module.sh start guard refuses on any stash, as before #393's fix +objstore_stash_recovery objstore_stash_recovery premise: a 19-byte stand-in really is installed as the module 15;16;17;18;19 never - +objstore_stash_recovery objstore_stash_recovery premise: a debris stash really is in place for the run below 15;16;17;18;19 never - +objstore_stash_recovery objstore_stash_recovery premise: and the debris is a valid module that differs from the installed one 15;16;17;18;19 never - +objstore_stash_recovery objstore_stash_recovery premise: and the real module is the thing parked at the stash 15;16;17;18;19 never - +objstore_stash_recovery objstore_stash_recovery premise: and the stash beside it is the real module 15;16;17;18;19 never - +objstore_stash_recovery objstore_stash_recovery premise: nothing is installed at the module's path for this slice 15;16;17;18;19 never - +objstore_stash_recovery objstore_stash_recovery premise: the installed module is valid before this slice arranges anything 15;16;17;18;19 never - +objstore_stash_recovery objstore_stash_recovery premise: which is not a module 15;16;17;18;19 never - +objstore_stash_recovery objstore_stash_recovery the debris is gone afterwards 15;16;17;18;19 2026-10-22 objstore_module.sh start guard refuses on any stash, as before #393's fix +objstore_stash_recovery objstore_stash_recovery the module is installed again afterwards, and valid 15;16;17;18;19 2026-10-22 objstore_module.sh start guard decides on presence alone: every stash is debris and is deleted +objstore_stash_recovery objstore_stash_recovery the real module survived, rather than being deleted with the stash 15;16;17;18;19 2026-10-22 objstore_module.sh start guard decides on presence alone: every stash is debris and is deleted +objstore_stash_recovery objstore_stash_recovery the suite recovers the only surviving copy and runs to completion 15;16;17;18;19 2026-10-22 objstore_module.sh start guard walks .away only, so a .probe that is the only copy is never restored +objstore_stash_recovery objstore_stash_recovery the suite replaces the stand-in with the real module and completes 15;16;17;18;19 2026-10-22 objstore_module.sh start guard decides on presence alone: every stash is debris and is deleted +objstore_stash_recovery objstore_stash_recovery the suite runs to completion with debris beside a valid module 15;16;17;18;19 2026-10-22 objstore_module.sh start guard refuses on any stash, as before #393's fix parallel parallel par count 15;16;17;18;19 2026-10-22 columnar_reader.c: a parallel custom scan claims row groups from its own rowGroupIndex, not the shared counter, so every participant reads every group (D6e) parallel parallel par filtered scan 15;16;17;18;19 2026-10-22 columnar_reader.c: a parallel custom scan claims row groups from its own rowGroupIndex, not the shared counter, so every participant reads every group (D6e) parallel parallel par no-filter 15;16;17;18;19 2026-10-22 columnar_reader.c: a parallel custom scan claims row groups from its own rowGroupIndex, not the shared counter, so every participant reads every group (D6e) diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index dbcc9fa8f..e146394c3 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 69 +suites_not_covered 41 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -6112,4 +6112,542 @@ suites_not_covered 69 # never 4176 + 16 = 4192 # dated 2914 + 38 = 2952 # 4192 + 2952 == 7144 -checks_never_observed_red 4192 +# +# #1341, native_fold_deferral seeded, with its twin (OffgridwithJD): 4192 -> 4196, and +# suites_not_covered 69 -> 68. The suite had no rows. It enters with 13 checks from +# five-major runs (all green on each of 15-19, shell and twin alike), after three fixes. +# The heap mirror was a CTAS from the columnar table, so it read through the scan under +# test: with the writer storing one value off by one, the fold, the row path and the +# mirror all agreed and main passed 13/13. The mirror is now written from the +# generator. The three answer checks compared readings taken with tail -1 using check: +# with the server stopped all three passed on empty strings. Readings now fail empty +# and the checks are check_text. And three premises a one-line pgcolumnar change +# reddens (the fold's ANALYZE line, the EXPLAIN group total, the writer's stripe limit) +# lost the prefix. 9 are dated by eight mutations, each reddening the same names in +# both twins. 4 arrive never: the two remaining premises, the counter's presence, and +# the eager-load bound, which no measured mutation reaches: with every group deferring, +# the loads equal survivors times width exactly, which still clears its 99% floor. +# +# ledger rows 7144 + 13 = 7157 +# never 4192 + 4 = 4196 +# dated 2952 + 9 = 2961 +# 4196 + 2961 == 7157 +# +# #1341, native_lazy_slot seeded, with its twin (OffgridwithJD): 4196 -> 4199, +# and suites_not_covered 68 -> 67. The suite had no rows. It enters with its 10 +# checks from five-major runs (all green on each of 15-19, shell and twin alike), after two +# fixes. The heap mirror was a copy of lz_c, read through the reader under test, so a value +# the writer stored wrong was wrong on both sides: with the writer storing c40 off by one, +# main passed 10/10. It is now written from the generator, and that mutant reddens four +# checks. And five comparisons read `q ... | tail -1` and compared with check, so two failed +# reads agreed, and a failed timing read as fast: with the server stopped after setup, six +# checks passed on main. Each side's failure now carries its side's name, and a failed +# timing is no reading. 7 are dated by four mutations, each reddening the same names in +# both twins. 3 arrive never. +# +# ledger rows 7157 + 10 = 7167 +# never 4196 + 3 = 4199 +# dated 2961 + 7 = 2968 +# 4199 + 2968 == 7167 +# +# #1341, native_metadata_flush seeded, with its twin (OffgridwithJD): 4199 -> 4201, +# and suites_not_covered 67 -> 66. The suite had no rows. It enters with 8 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after four fixes, +# each measured on main's shell, which passed all six under it. A witness that counted +# nothing (zerocount, opens=0 for every flush) passed both open-count checks, since 0 equals +# 0 and 0 is small: a floor now wants at least the three tables every flush writes. The heap +# mirror was copied from the columnar table, so a writer storing a wrong value +# (writecorrupt) agreed with itself: it is now written from the generator. Nothing showed +# the zone rows were read, since the reader also rules vectors out by value after decoding: +# a check now wants a vector ruled out by its zone row alone (Skipped minus Ruled Out by +# Value), and zone rows with no min/max (zonenull) redden it. And the witness check was +# labelled a premise though a flush that never begins its session (nosession) reddens it, +# so it lost the prefix (#1339). 6 are dated by five mutations, each the same names in +# both twins. 2 arrive never: the row count and the range read against the heap mirror. +# +# ledger rows 7167 + 8 = 7175 +# never 4199 + 2 = 4201 +# dated 2968 + 6 = 2974 +# 4201 + 2974 == 7175 +# +# #1341, native_late_materialization seeded, with its twin (OffgridwithJD): 4201 -> 4203, and +# suites_not_covered 66 -> 65. The suite had no rows. It enters with 14 checks from +# five-major runs (all green on each of 15-19, shell and twin alike), after two fixes. +# Every counter defaulted to 0 when it could not be read, and four checks want exactly +# 0: with the server stopped before the feature-off arm, "with the feature off nothing +# is filtered early" and "a volatile qual is refused the two-pass path" passed on main. +# An unread counter is now "no reading". And five checks a one-line pgcolumnar change +# reddens (the custom scan off by default, the early counter dropped, a survivor +# rejected, a writer that drops one row) lost the prefix; the two fixture premises +# now read the heap mirror h, which pgcolumnar does not write (jdatcmd's #1691 +# review). 12 are dated by seven mutations, each reddening the same names in both +# twins: dropseven (pgcolumnar_tuple_insert drops the row whose id is 7) dates "heap +# agrees on how many rows match", the executor's own Rows Removed by Filter and the +# two answer checks with the feature off and refused, which no mutation of the +# late-materialization path reaches. 2 arrive never: the two fixture premises. +# Building a rejected row's payload anyway (matall) reddens nothing: no counter +# reports per-row materialization. +# +# ledger rows 7175 + 14 = 7189 +# never 4201 + 2 = 4203 +# dated 2974 + 12 = 2986 +# 4203 + 2986 == 7189 +# +# #1341, native_reclaim_reconcile seeded, with its twin (OffgridwithJD): 4203 -> 4210, +# and suites_not_covered 65 -> 64. The suite had no rows. It enters with its 13 +# checks from five-major runs (all green on each of 15-19, shell and twin alike), after two +# fixes. Every reconcile check counted only the injected stale row, so a reconcile that +# dropped EVERY free row, leaking all reusable space, passed 11/11 on main. A plain compact +# (which does not reuse) must now keep the legitimate free ranges the earlier rewrite left, +# as count and bytes, and "compact_rewrite recorded the free ranges it left" checks that +# one exists. It is not a premise (jdatcmd's #1692 review): the ranges exist only because +# pgcolumnar recorded them, and nofree (record_free_space drops its insert, one line) +# reddens it alone. And "row count intact" compared two +# readings with plain check: with the server stopped before it, main passed it on two +# empty strings. It is check_num now. The reuse-cycle loop is unrolled so its two names are +# literals. 6 are dated by three mutations, each reddening the same names in both twins. +# 7 arrive never. +# +# ledger rows 7189 + 13 = 7202 +# never 4203 + 7 = 4210 +# dated 2986 + 6 = 2992 +# 4210 + 2992 == 7202 +# +# #1341, native_page_offset_bound seeded, with its twin (OffgridwithJD): 4210 -> 4212, and +# suites_not_covered 64 -> 63. The suite had no rows. It enters with 12 checks from +# five-major runs (all green on each of 15-19, shell and twin alike), after three fixes. +# Every refusal arm counted any psql failure as refused: with the whole-group guard +# removed the backend segfaulted and the arm still passed, and with the projected +# guard removed a later XX001 passed it. Each now wants XX001 with the guard's own +# message. Survival was a bare SELECT 1, which answers again after crash recovery: +# with the probe 4s late, main passed both whole-group survival arms with the backend +# killed. It now also reads the server log's crash count. And nothing read an intact +# table, so a guard over-applied to refuse every chunk (overall) or a legal chunk +# ending exactly at its group end (offby1) passed all 10: two controls now read rows +# behind an OFFSET 0 fence, which an aggregate over an intact table would not decode. +# 10 are dated by four mutations, each reddening the same names in both twins. With +# the whole-group guard removed, whether the read segfaults or fails decoding depends +# on where it lands; its three refusal checks are red either way. 2 arrive never: +# the two projected survival arms, which no mutation crashes. +# +# ledger rows 7202 + 12 = 7214 +# never 4210 + 2 = 4212 +# dated 2992 + 10 = 3002 +# 4212 + 3002 == 7214 +# +# #1341, native_param_pushdown seeded, with its twin (OffgridwithJD): 4212 -> 4216, and +# suites_not_covered 63 -> 62. The suite had no rows. It enters with 10 checks from +# five-major runs (all green on each of 15-19, shell and twin alike), after three fixes. +# With the PARAM_EXEC guard removed (freezeexec) the correlated arm still answered 3002: +# Begin freezes before any outer row sets the param, and "ts >= " +# prunes nothing. The arm now reads the scan's Usable Skip Predicates and asks the upper +# bound too, where that frozen value prunes every group. Every arm ran one execution per +# session, so a value frozen once and reused (staleparam) passed; two executions in one +# session now each count and prune with their own value. And survival was a bare +# SELECT 1, green on main with a backend SIGKILLed; it now reads the crash count. +# 6 are dated by four mutations, each reddening the same names in both twins. 4 +# arrive never: the literal count, the rescan premise, the lower-bound correlated answer +# and survival (red only under the harness kill). +# +# ledger rows 7214 + 10 = 7224 +# never 4212 + 4 = 4216 +# dated 3002 + 6 = 3008 +# 4216 + 3008 == 7224 +# +# #1341, native_sort_by seeded, with its twin (OffgridwithJD): 4216 -> 4230, and +# suites_not_covered 62 -> 61. The suite had no rows. It enters with 23 rows: 20 checks +# run on every major, two virtual-column checks run on 18-19, and their skip recorded +# on 15-17, all green in shell and twin alike on each of 15-19, after two fixes. +# The physical order was never read: group skipping on host was the only evidence of a +# sort, and a rewrite that applied only the first key (ts left unordered) passed 16/16 on +# main. The rows are now read in scan order and every adjacent pair is compared on (host, +# ts), with a premise that the inserted order is not already sorted, and the explicit- +# column sort must lay the rows out as the declared key does. And every refusal counted +# any psql failure: a backend aborting in place of the no-key refusal passed its own +# check, a crash in place of the virtual-column refusal passed 18/18 on PG18, and a +# dropped declared column given the generic message passed. Each refusal now wants its +# guard's SQLSTATE and message, and a new check reads the server log's crash count. 9 +# are dated by the mutations, each reddening the same names in both twins. 14 arrive +# never: the premise, the virtual arm's skip on 15-17, and the checks no measured +# mutation reaches. +# +# ledger rows 7224 + 23 = 7247 +# never 4216 + 14 = 4230 +# dated 3008 + 9 = 3017 +# 4230 + 3017 == 7247 +# +# #1341, native_rewrite seeded, with its twin (OffgridwithJD): 4230 -> 4241, +# and suites_not_covered 61 -> 60. The suite had no rows. It enters with its 20 +# checks from five-major runs (all green on each of 15-19, shell and twin alike), after four +# fixes. Rewriting every undeleted group, or ignoring min_deleted_fraction, passed all 17 on +# main: two checks now hold the threshold from both sides. vacuum without its index rebuild +# passed too: stale entries land on every live row once over the whole table, so a count +# and a hash through the index both matched; a narrow index range read now catches it +# (id 4097 read back as 6146). The four counts compared two readings with plain check, and +# "unique constraint still enforced" took any psql failure: with the server stopped after +# the rewrite, four checks passed on main. Counts are check_num and the refusal wants +# 23505 and n_id_uq's message. 9 are dated by five mutations of the rewrite, each +# reddening the same names in both twins. 11 arrive never. +# +# ledger rows 7247 + 20 = 7267 +# never 4230 + 11 = 4241 +# dated 3017 + 9 = 3026 +# 4241 + 3026 == 7267 +# +# #1341, native_zonemap seeded, with its twin (OffgridwithJD): 4241 -> 4259, and +# suites_not_covered 60 -> 59. The suite had no rows. It enters with 23 checks from +# five-major runs (all green on each of 15-19, shell and twin alike), after two fixes. +# The bounds were checked for presence and width only, and every column ascended, so a +# vector's first value was always its minimum: with a vector max never raised past its +# first value, a min never lowered, or a chunk max never folded past its first vector, +# main passed 18/18. v is now a permutation (values, nulls and counts unchanged), and +# four checks decode each bound of id, k and v and compare it with the heap's for its +# group and vector. And three checks compared two readings with check: with the server +# stopped they passed on two empty strings; they are now check_num. 5 are dated by +# four mutations, each reddening the same names in both twins. 18 arrive never. +# +# ledger rows 7267 + 23 = 7290 +# never 4241 + 18 = 4259 +# dated 3026 + 5 = 3031 +# 4259 + 3031 == 7290 +# +# #1341, native_zonemap_session seeded, with its twin (OffgridwithJD): 4259 -> 4259, +# and suites_not_covered 59 -> 58. The suite had no rows. It enters with 13 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after two fixes. +# "nested columnar scans each hold their own handle" compared only the join's count, which +# a shared handle still answers correctly: with one static session shared across scans, +# the first implementation's defect, main passed it. It is now the witness, each scan +# reporting its own groups opened once, and the count is a check of its own. The three scans +# around the aborted one must also each probe exactly their own groups. Three premises that +# a one-line pgcolumnar change reddens lost the prefix (#1339), and a new check holds zs2 +# to more than one group, which a writer ignoring stripe_row_limit fails. The third is +# "the fixture has more than one chunk group" (jdatcmd's #1698 review): it reads the +# writer's row_group catalog, and optx100 (a writer that multiplies stripe_row_limit by +# 100) reddens it. All 13 are dated by seven mutations, each reddening the same names in +# both twins; none arrives never. The re-resolved index oid columnar_metadata.c documents +# is not counted by any witness here: with it restored the suite stays 13/13, stated +# rather than claimed. +# +# ledger rows 7290 + 13 = 7303 +# never 4259 + 0 = 4259 +# dated 3031 + 13 = 3044 +# 4259 + 3044 == 7303 +# +# #1341, native_zonemap_narrow seeded, with its twin (OffgridwithJD): 4259 -> 4259, +# and suites_not_covered 58 -> 57. The suite had no rows. It enters with 6 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after one fix. +# Nothing showed a group was skipped: the pruning check read only the answer, so a scan +# that read the zone maps but never excluded a group, or never consulted them, returned +# the same 1000 rows, and main passed all four checks under each. Two checks now read the +# scan's Columnar Chunk Groups Removed by Filter (9 of 10, on each table), and the width +# check refuses an unread count rather than comparing two blanks. 6 are dated by four +# mutations, each reddening the same names in both twins. 0 arrive never. +# +# ledger rows 7303 + 6 = 7309 +# never 4259 + 0 = 4259 +# dated 3044 + 6 = 3050 +# 4259 + 3050 == 7309 +# +# #1341, native_vecdecode seeded, with its twin (OffgridwithJD): 4259 -> 4269, +# and suites_not_covered 57 -> 56. The suite had no rows. It enters with 28 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after three fixes. +# The fixture had no nulls, so a fold that ignored the validity bitmap (nonull) passed all +# 24 on main: a nullable column w now runs through the fold and through a scalar parity +# check. "decoded plus skipped is the group's vectors" read missing counters as 0, so with +# the plans failing it passed as 0 + 0 = 0, and two aggregate parities compared two empty +# readings: with the server stopped before the plans main passed 3, now none. And nine +# "premise:" checks read pgcolumnar's own behaviour, so they lost the prefix (#1339): +# nofold reddens the three "reaches the fold" checks, decodeall the fold arm's decode +# count, noskip "the narrow range skips vectors"; and (jdatcmd's #1700 review) noscan, +# the custom scan off by default, reddens the two plan-shape checks, "no whole group is +# pruned" and "the wide range skips none", while optdiv100, a writer that divides the +# table's stripe_row_limit by 100, reddens "one row group". 18 are dated by seven +# mutations, each the same names in both twins; noscan also dates "EXPLAIN reports +# Columnar Vectors Decoded", "and it is not stuck at zero" and the control. The poison +# check is dated by holeread, whose hole is zero-filled so a fold that reads it counts +# rows that are not there on a non-assert build too. 10 arrive never: the heap-side +# nulls premise and nine parity checks with heap. +# +# ledger rows 7309 + 28 = 7337 +# never 4259 + 10 = 4269 +# dated 3050 + 18 = 3068 +# 4269 + 3068 == 7337 +# +# #1341, native_parquet_fieldid seeded, with its twin (OffgridwithJD): 4269 -> 4271, +# and suites_not_covered 56 -> 55. The suite had no rows. It enters with 11 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after two fixes. +# The refusal arms read only the SQLSTATE, and four refusals share 22023: with the 'no field +# ids' guard removed (noanyid) an id-less file fell to the absent-id refusal and main passed +# 11/11. Each arm now wants its own guard's message. And survival was a bare SELECT 1: with +# the absent-id refusal turned into abort() it passed after crash recovery, and the crash took +# five later refusal arms with it. Survival now reads the server log's count of killed +# backends, and every reading waits for the server first, so a crash fails only its own arm +# and survival. 9 are dated by five mutations, each the same names in both twins. 2 +# arrive never. +# +# ledger rows 7337 + 11 = 7348 +# never 4269 + 2 = 4271 +# dated 3068 + 9 = 3077 +# 4271 + 3077 == 7348 +# +# #1341, native_parquet_codecs seeded, with its twin (OffgridwithJD): 4271 -> 4278, +# and suites_not_covered 55 -> 54. The suite had no rows. It enters with 14 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after three fixes. +# Every codec was compared with c_none.parquet read back through read_parquet, the reader +# under test: with one decoded byte flipped in every page over 4 KB, the same way for +# every codec, main passed all nine checks. Each file now decodes to the generator's rows, +# built in SQL. Each file's codec is read from its own metadata (a crafter that wrote every +# file uncompressed passed main 9/9). And survival reads the server log's crash count: with +# the crafted-size guard aborting and the probe 3 s late, main's bare SELECT 1 passed. 7 +# are dated by four mutations, each reddening the same names in both twins. 7 arrive +# never: the four codec premises read pyarrow's metadata, not pgcolumnar, and the FDW counts +# hold under each mutation. +# +# ledger rows 7348 + 14 = 7362 +# never 4271 + 7 = 4278 +# dated 3077 + 7 = 3084 +# 4278 + 3084 == 7362 +# +# #1341, native_parquet_units seeded, with its twin (OffgridwithJD): 4278 -> 4285, +# and suites_not_covered 54 -> 53. The suite had no rows. It enters with 20 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after two fixes. +# The pushdown oracle was the two-group millis file read back through read_parquet, the +# reader under test: with every row after a page's first taking the next dictionary +# entry, the scan and the mirror shifted alike and main passed 19/19. The oracle is now +# the generator's hourly series, and a new check compares the whole file with it. The two +# nanos truncation checks compared the file with a second file read the same way, so +# with the server stopped both passed on two empty reads; they now want the toward-zero +# microseconds as literals. 13 are dated by four mutations, each reddening the same +# names in both twins. 7 arrive never. +# +# ledger rows 7362 + 20 = 7382 +# never 4278 + 7 = 4285 +# dated 3084 + 13 = 3097 +# 4285 + 3097 == 7382 +# +# #1341, native_parquet_projection seeded, with its twin (OffgridwithJD): 4285 -> 4287, +# and suites_not_covered 53 -> 52. The suite had no rows. It enters with 17 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after two fixes. +# The oracle h was read_parquet of the same file, and read_parquet shares the scan's +# decoders: with a text decoder dropping a byte, or an INT64 decoder off by one, main passed +# 15/15. h is now built in SQL from the crafter's generator. And no table had a dropped +# column, so a projection matching the needed set by target index rather than attribute +# number passed 15/15; ftd drops a column ahead of the others and must still read exactly +# the column asked for. 15 are dated by five mutations, each reddening the same names in +# both twins. 2 arrive never: projection [a] and the unselected-qual hash, which no +# mutation reached. +# +# ledger rows 7382 + 17 = 7399 +# never 4285 + 2 = 4287 +# dated 3097 + 15 = 3112 +# 4287 + 3112 == 7399 +# +# #1341, native_parquet_hardening seeded, with its twin (OffgridwithJD): 4287 -> 4289, +# and suites_not_covered 52 -> 51. The suite had no rows. It enters with 18 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after four fixes, +# each measured on main's shell on PG17. The short-chunk verdict matched any 'has a malformed +# row group', which five guards raise: with the chunk-count guard removed a later guard +# rejected the file and main passed 18/18; it now wants this guard's detail. The 'still +# reads' arms counted rows: a date, time or timestamp decoded one unit off passed 18/18; they +# now compare the decoded values. The no-wrap arm counted -25536 in the in-range file, which +# never holds 40000, so it could not fail; it now reads the out-of-range file, which must +# refuse. And survival was a bare SELECT 1: with the chunk refusal turned into abort() and the +# probe 5 s late it passed; it now reads the server log's count of killed backends, and every +# reading waits for the server. The range refusals also want their own type's message, since +# four share 22008. 16 are dated by eleven mutations, each the same names in both twins. 2 +# arrive never. +# +# ledger rows 7399 + 18 = 7417 +# never 4287 + 2 = 4289 +# dated 3112 + 16 = 3128 +# 4289 + 3128 == 7417 +# +# #1341, native_parquet_stack seeded, with its twin (OffgridwithJD): 4289 -> 4289, +# and suites_not_covered 51 -> 50. The suite had no rows. It enters with 8 +# checks from five-major runs (all green on each of 15-19, shell and twin alike). The footer +# parser has two independent unbounded recursions, PgColumnarThriftSkip and walk_schema, +# each guarded by check_stack_depth(); the fixture pins max_stack_depth so a ~20 KB crafted +# footer trips the guard without overflowing the real C stack, so removing a guard fails the +# check by a wrong SQLSTATE rather than by crashing. The two refusal checks pin 54001, not +# any error, and read_parquet is held to the same refusal. 4 are dated: the nested-struct +# and read_parquet refusals by removing the thrift guard, the schema-chain refusal by removing +# the walk guard, and the valid-file control by a guard that trips on any nesting. The four +# survival checks were a bare SELECT 1, which answers again once crash recovery is over, and +# the twin's read_parquet survival test never read the file (jdatcmd's #1708 review). Each +# arm now takes a crash baseline right before its trigger, every reading waits for the +# server, and survival wants SELECT 1 and an unchanged count of "terminated by signal" in the +# server log, so a crash fails only its own arm. An abort() at each input's decode site dates +# them: thriftabort (PgColumnarThriftSkip aborts where its guard would raise) the +# nested-struct and read_parquet survivals, walkabort (the same in walk_schema) the +# schema-chain survival, validabort (walk_schema aborts on entry) the valid-file survival. +# All 8 are dated; none arrives never. +# +# ledger rows 7417 + 8 = 7425 +# never 4289 + 0 = 4289 +# dated 3128 + 8 = 3136 +# 4289 + 3136 == 7425 +# +# #1341, native_parquet_streaming seeded, with its twin (OffgridwithJD): 4289 -> 4304, +# and suites_not_covered 50 -> 49. The suite had no rows. It enters with 20 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after one fix measured +# on main's shell on PG17: survival was a bare SELECT 1, so with the second-dictionary refusal +# turned into abort() 'backend survived the crafted page areas' passed, and the crash also +# failed the v2 arm read during recovery. Survival now reads the server log's count of killed +# backends since its section began, and every reading waits for the server first. As hardening, +# the two refusals that accepted any ERROR now want the page loop's own decode refusal; no +# mutant here separated them. The v2 level guard stays unisolated, as the suite's header +# argues. 5 are dated by five mutations, each the same names in both twins. 15 arrive +# never. +# +# ledger rows 7425 + 20 = 7445 +# never 4289 + 15 = 4304 +# dated 3136 + 5 = 3141 +# 4304 + 3141 == 7445 +# +# #1341, native_parquet_partition seeded, with its twin (OffgridwithJD): 4304 -> 4310, +# and suites_not_covered 49 -> 48. The suite had no rows. It enters with 35 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after three fixes. +# The pruning reader defaulted to 0 on any failure, so with the server stopped the three +# 'prunes nothing' checks passed on no reading; 'Files Pruned' prints only above 0, so a +# missing line now counts as 0 only beside the plan's Files line. The refusals took any +# error: with the unknown-column guard removed (nocolcheck) a later 42804 stood in, and with +# the explicit encoded-NUL guard removed (nonul) the encoding check's 22021 did, main 35/35 +# each; every refusal now wants its own message. And survival was a bare SELECT 1, which +# an abort() in the unknown-column refusal passed with the probe 5 s late; it now reads the +# server log's crash count. 29 are dated by six mutations, each the same names in both +# twins. 6 arrive never. +# +# ledger rows 7445 + 35 = 7480 +# never 4304 + 6 = 4310 +# dated 3141 + 29 = 3170 +# 4310 + 3170 == 7480 +# +# #1341, native_parquet_fdw seeded, with its twin (OffgridwithJD): 4310 -> 4320, +# and suites_not_covered 48 -> 47. The suite had no rows. It enters with 20 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after four fixes. +# Projection and row-group skipping were never read: every arm compared answers, which the +# executor's recheck and a discarding projection keep right, so with every column decoded +# (noproj) or no group skipped (noskip) main passed 14/14. Two checks now read the scan's +# EXPLAIN ANALYZE counters, over a 200,000-row file of four row groups. The refusal arms +# accepted any psql failure: with the scan's missing-path guard removed (nopathguard) the +# backend crashed and main passed, and with the short-file guard removed (nolen) the +# bad-magic guard refused instead and main passed. Each arm now wants its own guard's +# SQLSTATE and message, and a survival check reads the server log's count of killed +# backends. The oracle is a heap table written from the generator, and the self-join counts +# are check_num, since two failed reads agreed. The skipped scan's row count read a plain +# int4 constant, a cross-type comparison that skips nothing (#1712), so it came from a full +# scan (jdatcmd's #1711 review); it now runs the plan's own int8 query, and skipshift (each +# group's skip decision moved to the next group, still 3 skipped) dates it. 10 are dated by +# eight mutations, each the same names in both twins. 10 arrive never. +# +# ledger rows 7480 + 20 = 7500 +# never 4310 + 10 = 4320 +# dated 3170 + 10 = 3180 +# 4320 + 3180 == 7500 +# +# #1341, native_read_parquet seeded, with its twin (OffgridwithJD): 4320 -> 4324, +# and suites_not_covered 47 -> 46. The suite had no rows. It enters with 15 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after three fixes. +# The round trips compared read_parquet with the source table, which the columnar scan +# reads both to write the file and as the oracle: with the columnar reader returning +# int4 777 as 778 (columnar_reader.c, store_att_byval), main passed 12/12. Every round trip now hashes the generator's own rows. The refusals +# counted any psql failure: with the scalar physical-type guard removed, main passed +# 12/12; each refusal now requires its SQLSTATE and its own message, and a survival check +# reads the crash count. read_parquet's own pg_read_server_files gate was never tested: +# with it removed, main passed 12/12; a role holding USAGE but not the role is now refused +# by it, beside a member that reads every row. 11 are dated by five mutations, each +# reddening the same names in both twins. 4 arrive never, among them "premise: reject +# missing column definition list", which reads PostgreSQL's parser (42601), not pgcolumnar. +# +# ledger rows 7500 + 15 = 7515 +# never 4320 + 4 = 4324 +# dated 3180 + 11 = 3191 +# 4324 + 3191 == 7515 +# +# #1341, objstore_listing seeded, with its twin (OffgridwithJD): 4324 -> 4328, +# and suites_not_covered 46 -> 45. The suite had no rows. It enters with 15 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after three fixes. +# The two non-progress refusals share 08P01 and read only the SQLSTATE: the no-keys page +# replays its token, so with the progress check removed (noprogress) the token check refused +# it on the next page and main passed 12/12. Each refusal now wants its own guard's message. +# No check made a LIST fail, so a non-2xx read as an empty prefix (non200empty) passed 12/12; +# the fixture server gained a __listing_status__ hook and a check wants the HTTP status +# refusal (08006). The exact-key 'no LIST' count read a log nothing proved alive; a control +# now wants the key's own GET in it. A survival check reads the server log's count of killed +# backends, and every refusal waits for the server first. 11 are dated by seven mutations, +# each the same names in both twins. 4 arrive never. +# +# ledger rows 7515 + 15 = 7530 +# never 4324 + 4 = 4328 +# dated 3191 + 11 = 3202 +# 4328 + 3202 == 7530 +# +# #1341, native_rewrite_conc seeded, with its twin (OffgridwithJD): 4328 -> 4330, +# and suites_not_covered 45 -> 44. The suite had no rows. It enters with 11 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after two fixes. +# Nothing tested that the online rewrite does not block: with compact_rewrite under +# AccessExclusiveLock (ael) the workers simply waited and main passed 5/5. A compact_rewrite +# now holds its lock mid-verb (pgcolumnar.maintenance_hold_ms) while a reader and a writer, +# each with a 3 s lock_timeout, must finish. And the race was assumed, not shown: a new check +# requires the compactor to have moved rows while the workers wrote. 9 are dated by four +# mutations, each the same names in both twins (noconflict on 3 of 3 runs). 2 arrive never. +# +# ledger rows 7530 + 11 = 7541 +# never 4328 + 2 = 4330 +# dated 3202 + 9 = 3211 +# 4330 + 3211 == 7541 +# +# #1341, objstore_addressing seeded, with its twin (OffgridwithJD): 4330 -> 4335, +# and suites_not_covered 44 -> 43. The suite had no rows. It enters with 16 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after two fixes. +# The gs:// no-endpoint check passed whenever the message did not name AWS_ENDPOINT_URL, so a +# mistyped default host (gswrongdef) passed 12/12; it now requires the allow-list refusal naming +# storage.googleapis.com:443. No key needed URI-encoding, so a space sent as '+' (plusspace) or +# '+'/'=' left bare (plusraw) passed 12/12; a key "dir sp/a+b=c.parquet" is now read under both +# addressings and its request path checked in the fixture's log. 11 are dated by seven +# mutations, each the same names in both twins. 5 arrive never. Neither pgc_skip name +# (pyarrow, the virtual-host name) gets a row: CI emits neither. +# +# ledger rows 7541 + 16 = 7557 +# never 4330 + 5 = 4335 +# dated 3211 + 11 = 3222 +# 4335 + 3222 == 7557 +# +# #1341, objstore_stash_recovery seeded, with its twin (OffgridwithJD): 4335 -> 4344, +# and suites_not_covered 43 -> 42. The suite had no rows. It enters with 18 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after three fixes. +# "and it did not stop at the start guard" counted 'interrupted mid-move', the refusal's +# text before #393's fix, which nothing prints any more: with the guard turned back into a +# refusal, eight checks went red and it stayed green. The debris was a byte-identical copy, +# so a guard that restored every stash over the installed module passed 17/17; the debris +# is now a different valid module and the installed one is compared with the pristine copy. +# Both restore slices parked the module at .away, so a guard that walked .away alone passed +# 17/17; the only-copy slice now parks it at .probe. Each slice also starts from the +# pristine copy, so one slice's damage no longer reddens the next one's premises. +# 9 are dated by four mutations of objstore_module.sh's guard, each reddening the same +# names in both twins. 9 arrive never. +# +# ledger rows 7557 + 18 = 7575 +# never 4335 + 9 = 4344 +# dated 3222 + 9 = 3231 +# 4344 + 3231 == 7575 +# +# #1341, objstore_s3_read seeded, with its twin (OffgridwithJD): 4344 -> 4348, +# and suites_not_covered 42 -> 41. The suite had no rows. It enters with 17 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after these fixes. +# The no-environment and 403 refusals read a bare 28000, which five guards share: with s3:// +# falling back to a default host (noepreq) the region guard refused instead and main passed +# 12/12. Each refusal now names its cause. Region handling was never tested: dropping the +# AWS_DEFAULT_REGION fallback, swapping its precedence, signing us-east-1 when no region is set, +# or signing with an empty secret each passed 12/12, and four checks now pin them. Nothing +# read the crash count, and an abort() in abort cleanup passed every SQLSTATE check (main's +# shell exited FATAL on the restart with no red): survival now reads the server log. The https +# check's name interpolated the build's want, so the grader could not read it. 13 are dated +# by twelve mutations, each the same names in both twins. 4 arrive never. Neither the +# pyarrow pgc_skip name nor the optional real-S3 check gets a row: CI emits neither. +# +# ledger rows 7575 + 17 = 7592 +# never 4344 + 4 = 4348 +# dated 3231 + 13 = 3244 +# 4348 + 3244 == 7592 +checks_never_observed_red 4348 diff --git a/test/craft_parquet_codecs.py b/test/craft_parquet_codecs.py new file mode 100644 index 000000000..724419473 --- /dev/null +++ b/test/craft_parquet_codecs.py @@ -0,0 +1,56 @@ +"""Write the native_parquet_codecs fixtures into DIR (argv[1]), for both twins. + +c_none / c_gzip / c_zstd / c_lz4.parquet hold the same 20000 rows, each written with one +codec; codecs.txt records the compression pyarrow actually stored for each file's first +column (one " " line), so a test can prove which codec it read. +badhdr.parquet is a gzip page whose header declares uncompressed_page_size -1. + +The rows are the generator both twins also compute in SQL: + id = g, grp = g % 50, val = g * 1.5, s = 'row-' || (g % 100), g in 0 .. 19999 +Written fresh for pgColumnar. +""" +import sys + +import pyarrow as pa +import pyarrow.parquet as pq + +W = sys.argv[1] +N = 20000 +tbl = pa.table({ + "id": pa.array(range(N), pa.int32()), + "grp": pa.array([i % 50 for i in range(N)], pa.int32()), + "val": pa.array([i * 1.5 for i in range(N)], pa.float64()), + "s": pa.array([f"row-{i % 100}" for i in range(N)]), +}) +with open(f"{W}/codecs.txt", "w") as out: + for codec in ["none", "gzip", "zstd", "lz4"]: + pq.write_table(tbl, f"{W}/c_{codec}.parquet", compression=codec) + c = pq.ParquetFile(f"{W}/c_{codec}.parquet").metadata.row_group(0).column(0).compression + out.write(f"{codec} {c}\n") + +# A gzip page whose header's uncompressed_page_size (field 2) is rewritten to zigzag(-1). +pq.write_table(pa.table({"a": pa.array([1, 2, 3], pa.int32())}), + f"{W}/badhdr.parquet", compression="gzip", use_dictionary=False) +raw = bytearray(open(f"{W}/badhdr.parquet", "rb").read()) +pos = 4 # the first PageHeader follows the "PAR1" magic +last = 0 +patched = False +for _ in range(6): + b = raw[pos] + pos += 1 + t, d = b & 0x0F, b >> 4 + if t == 0: + break + fid = last + d if d else None + last = fid + start = pos + while raw[pos] & 0x80: + pos += 1 + pos += 1 + if fid == 2: + assert pos - start == 1, (start, pos) + raw[start] = 0x01 # zigzag(-1) + patched = True + break +assert patched, "did not find uncompressed_page_size" +open(f"{W}/badhdr.parquet", "wb").write(bytes(raw)) diff --git a/test/craft_parquet_hardening.py b/test/craft_parquet_hardening.py new file mode 100644 index 000000000..d0570f8d7 --- /dev/null +++ b/test/craft_parquet_hardening.py @@ -0,0 +1,145 @@ +"""craft_parquet_hardening.py DIR: write the Parquet files test/native_parquet_hardening.sh +and its pytest twin read: the narrowing binds (out of range and in range) and a +two-column file whose row group carries one column chunk. One writer for both +twins, so neither can drift from the other. Needs pyarrow.""" +import sys + +import pyarrow as pa +import pyarrow.parquet as pq + +W = sys.argv[1] + +# ---- narrowing: out-of-range values must raise, not wrap -------------------- +# int2 bound over INT32: 40000 does not fit int16 (would wrap to -25536) +pq.write_table(pa.table({"c": pa.array([30000, 40000], pa.int32())}), + f"{W}/i2_bad.parquet") +# same bind, all values in range: must keep working +pq.write_table(pa.table({"c": pa.array([-30000, 0, 30000], pa.int32())}), + f"{W}/i2_ok.parquet") +# time bound over INT64 micros: beyond one day is not a time +pq.write_table(pa.table({"c": pa.array([0, 86400000000 + 1], pa.int64())}), + f"{W}/time_bad.parquet") +pq.write_table(pa.table({"c": pa.array([0, 3600000000], pa.int64())}), + f"{W}/time_ok.parquet") +# date bound over INT32 days since epoch. INT32 max does NOT overflow the epoch +# offset (2147483647 - 10957 stays in range); it is rejected by IS_VALID_DATE. +pq.write_table(pa.table({"c": pa.array([0, 2147483647], pa.int32())}), + f"{W}/date_bad.parquet") +# INT32 min is what overflows the subtraction itself, exercising the other branch +pq.write_table(pa.table({"c": pa.array([0, -2147483648], pa.int32())}), + f"{W}/date_ovf.parquet") +pq.write_table(pa.table({"c": pa.array([0, 19000], pa.int32())}), + f"{W}/date_ok.parquet") +# timestamp bound over INT64 micros: INT64 min overflows the epoch-offset +# subtraction. INT64 max does NOT -- it still lands inside PostgreSQL's timestamp +# range (year ~294276), so it is deliberately not the case tested here. +pq.write_table(pa.table({"c": pa.array([0, -9223372036854775808], pa.int64())}), + f"{W}/ts_bad.parquet") +pq.write_table(pa.table({"c": pa.array([0, 1700000000000000], pa.int64())}), + f"{W}/ts_ok.parquet") + +# ---- malformed row-group chunk list ---------------------------------------- +pq.write_table(pa.table({"a": pa.array([1, 2, 3], pa.int32()), + "b": pa.array([4, 5, 6], pa.int32())}), + f"{W}/two.parquet") + +# Walk the Thrift-compact footer to the exact byte holding the row group's +# `columns` list header, then claim it holds one fewer chunk than it does. +# Blind byte-searching is not reliable here: the same byte value occurs in the +# data pages, so the header has to be located structurally. +raw = bytearray(open(f"{W}/two.parquet", "rb").read()) +flen = int.from_bytes(raw[-8:-4], "little") +base = len(raw) - 8 - flen # start of FileMetaData +pos = base + +def varint(): + global pos + shift = 0; out = 0 + while True: + b = raw[pos]; pos += 1 + out |= (b & 0x7F) << shift + if not b & 0x80: + return out + shift += 7 + +def zigzag(): + u = varint() + return (u >> 1) ^ -(u & 1) + +def list_hdr(): + global pos + b = raw[pos]; pos += 1 + size = (b >> 4) & 0x0F + et = b & 0x0F + if size == 0x0F: + size = varint() + return size, et + +def skip(t): + global pos + if t in (1, 2): return # bool: value is in the header + if t == 3: pos += 1 # byte + elif t in (4, 5, 6): zigzag() # i16/i32/i64 + elif t == 7: pos += 8 # double + elif t == 8: pos += varint() # binary + elif t in (9, 10): + n, et = list_hdr() + for _ in range(n): skip(et) + elif t == 12: + last = 0 + while True: + ft, _fid, last = field(last) + if ft == 0: break + skip(ft) + +def field(last): + global pos + b = raw[pos]; pos += 1 + if b == 0: + return 0, 0, last + t = b & 0x0F + delta = b >> 4 + fid = last + delta if delta else zigzag() + return t, fid, fid + +# FileMetaData -> field 4 (row_groups) -> first RowGroup -> field 1 (columns) +target = None +last = 0 +while True: + ft, fid, last = field(last) + if ft == 0: + break + if fid == 4 and ft == 9: # row_groups: list + nrg, _et = list_hdr() + assert nrg >= 1, nrg + rlast = 0 + while True: + rft, rfid, rlast = field(rlast) + if rft == 0: + break + if rfid == 1 and rft == 9: # columns: list + target = pos # byte holding this list's header + n, et = list_hdr() + assert n == 2, f"expected 2 column chunks, got {n}" + skip(et) # past ColumnChunk[0] + start2 = pos + skip(et) # past ColumnChunk[1] + end2 = pos + break + skip(rft) + break + skip(ft) + +assert target is not None, "row group columns list not found" +# Remove ColumnChunk[1] outright and say so in the header, so the footer stays a +# well-formed Thrift struct. Decrementing the count alone would desync the parser +# and fail for an unrelated reason; the only defect must be that the row group +# carries fewer chunks than the schema has leaves. +assert (raw[target] >> 4) == 2, hex(raw[target]) +raw[target] = (1 << 4) | (raw[target] & 0x0F) +removed = end2 - start2 +del raw[start2:end2] +raw[-8:-4] = (flen - removed).to_bytes(4, "little") +open(f"{W}/short_chunks.parquet", "wb").write(bytes(raw)) +print(f" columns list at {target}: 2 chunks -> 1, removed {removed} footer bytes", + file=sys.stderr) diff --git a/test/craft_parquet_projection.py b/test/craft_parquet_projection.py new file mode 100644 index 000000000..ebc780474 --- /dev/null +++ b/test/craft_parquet_projection.py @@ -0,0 +1,39 @@ +"""Write the files test/native_parquet_projection.sh and its pytest twin read. + + python3 test/craft_parquet_projection.py DIR + +DIR/proj.parquet a, b, c, d over 5000 rows in row groups of 1000, from the same + generator the suites rebuild in SQL as their oracle: + a = i, b = 2i, c = 's' || (i % 20), d = i / 3.0 +DIR/corrupt.parquet g, bad over 1000 rows, with the second column's data page + smashed, so decoding `bad` fails while `g` and the footer stay + intact. + +Both harnesses run this one crafter, so a file can never differ between them. +""" +import sys + +import pyarrow as pa +import pyarrow.parquet as pq + +W = sys.argv[1] +n = 5000 +pq.write_table(pa.table({ + "a": pa.array(range(n), pa.int32()), + "b": pa.array([i * 2 for i in range(n)], pa.int64()), + "c": pa.array([f"s{i % 20}" for i in range(n)]), + "d": pa.array([i / 3.0 for i in range(n)], pa.float64()), +}), f"{W}/proj.parquet", row_group_size=1000) + +pq.write_table(pa.table({"g": pa.array(range(1000), pa.int32()), + "bad": pa.array(range(1000), pa.int32())}), + f"{W}/corrupt.parquet", row_group_size=1000, use_dictionary=False) +md = pq.ParquetFile(f"{W}/corrupt.parquet").metadata.row_group(0).column(1) +off = md.data_page_offset +raw = bytearray(open(f"{W}/corrupt.parquet", "rb").read()) +# smash the second column's page header and the start of its data, leaving the +# first column and the footer intact +for i in range(off, off + 32): + raw[i] = 0xFF +open(f"{W}/corrupt.parquet", "wb").write(bytes(raw)) +print(f" corrupted column 'bad' at offset {off}", file=sys.stderr) diff --git a/test/craft_parquet_streaming.py b/test/craft_parquet_streaming.py new file mode 100644 index 000000000..dc1267651 --- /dev/null +++ b/test/craft_parquet_streaming.py @@ -0,0 +1,166 @@ +"""Write every Parquet file native_parquet_streaming.sh and its twin read, into DIR. + +Usage: craft_parquet_streaming.py DIR + +Exit status, as the shell reads it: + 0 every file was written + 3 no room even for the oversized sparse file's metadata + 4 the sparse file did not reach its logical size (no sparse files here) + 5 the footer offset to patch is ambiguous; refusing to guess + 6 a crafted file is missing after writing + 7 the source file has no dictionary page to duplicate + +3 and 4 mean the box cannot build the oversized file, which the shell records as a +named skip; any other failure means the crafted files are not all there, which it +records as a failed check rather than letting each refusal pass on "could not open +file". The construction of each file, and why, is described beside it below. +""" +import os +import sys + +import pyarrow as pa +import pyarrow.parquet as pq + +W = sys.argv[1] + + +def parts(path): + raw = open(path, "rb").read() + n = int.from_bytes(raw[-8:-4], "little") + return raw[-8 - n:-8], n, raw[4:-8 - n] # footer, its length, page area + + +# ---- a real small file, and an oversized sparse one carrying its footer ------------ +src = os.path.join(W, "small.parquet") +pq.write_table(pa.table({"id": pa.array([1, 2, 3], pa.int32()), + "v": pa.array(["a", "b", "c"])}), src) +data = open(src, "rb").read() +metalen = int.from_bytes(data[-8:-4], "little") +meta = data[-8 - metalen:-8] + +# comfortably over MaxAllocSize (1GB - 1), which is what palloc() rejected +SIZE = 1600 * 1024 * 1024 +st = os.statvfs(W) +if st.f_bavail * st.f_frsize < 64 * 1024 * 1024: + sys.exit(3) +big = os.path.join(W, "huge_sparse.parquet") +with open(big, "wb") as f: + f.write(b"PAR1") + f.truncate(SIZE) # the hole; never read by the footer path + f.seek(SIZE - 8 - metalen) + f.write(meta) + f.write(metalen.to_bytes(4, "little")) + f.write(b"PAR1") +if os.path.getsize(big) != SIZE: + sys.exit(4) + +# ---- crafted files: each keeps a valid footer and breaks one thing the page loop +# must not trust. The footer comes from a real uncompressed file, so a zeroed page +# area really parses as page headers rather than failing in the codec first. +plain = os.path.join(W, "plain.parquet") +pq.write_table(pa.table({"id": pa.array(list(range(200)), pa.int32())}), + plain, compression="none") +meta, metalen, body = parts(plain) + + +def build(name, body_bytes, m=None, ml=None): + with open(os.path.join(W, name), "wb") as f: + f.write(b"PAR1") + f.write(body_bytes) + f.write(m if m is not None else meta) + f.write((ml if ml is not None else metalen).to_bytes(4, "little")) + f.write(b"PAR1") + + +# 1. a large zero-filled page area of a REQUIRED column (no definition levels, so the +# zero-value page is reached), sparse: each zero byte parses as an empty v1 data +# page, so without the progress guard the loop walks it one byte at a time. +req = os.path.join(W, "plain_req.parquet") +pq.write_table(pa.table({"id": pa.array(list(range(200)), pa.int32())}, + schema=pa.schema([pa.field("id", pa.int32(), nullable=False)])), + req, compression="none") +rmeta, rmetalen, rbody = parts(req) +ZSIZE = 1600 * 1024 * 1024 +with open(os.path.join(W, "zero_pages.parquet"), "wb") as f: + f.write(b"PAR1") + f.truncate(ZSIZE - 8 - rmetalen) + f.seek(ZSIZE - 8 - rmetalen) + f.write(rmeta) + f.write(rmetalen.to_bytes(4, "little")) + f.write(b"PAR1") + + +# 2. a page header (built by hand, Thrift compact) declaring a compressed_size far +# past end of file, written over the start of the chunk the footer points at. +def zz(v): + u = (v << 1) ^ (v >> 63) if v < 0 else (v << 1) + out = bytearray() + while True: + b = u & 0x7F + u >>= 7 + out.append(b | (0x80 if u else 0)) + if not u: + return bytes(out) + + +def fh(delta, t): + return bytes([(delta << 4) | t]) + + +I32, STRUCT, STOP = 5, 12, b"\x00" +BIG = 500 * 1024 * 1024 # past end of file, but under MaxAllocSize +synth = (fh(1, I32) + zz(0) + fh(1, I32) + zz(64) + fh(1, I32) + zz(BIG) + + fh(2, STRUCT) + fh(1, I32) + zz(200) + fh(1, I32) + zz(0) + + fh(1, I32) + zz(3) + fh(1, I32) + zz(3) + STOP + STOP) +build("bad_size.parquet", synth + body[len(synth):]) + +# 3. the page area is noise, so the page header does not parse at all. +build("noise_pages.parquet", bytes((i * 37 + 11) & 0xFF for i in range(len(body)))) + +# 4. the footer points the SECOND column chunk's start offset to -2^20: negative, so +# it passes the loop's "pos < len" and reaches the header read's offset check. +two = os.path.join(W, "two.parquet") +pq.write_table(pa.table({"id": pa.array(list(range(200)), pa.int32()), + "v": pa.array(["r%04d" % i for i in range(200)])}), + two, compression="none", use_dictionary=False) +tmeta, tmetalen, tbody = parts(two) +tcol = pq.ParquetFile(two).metadata.row_group(0).column(1) +tstart = tcol.dictionary_page_offset or tcol.data_page_offset +pat = zz(tstart) +if tmeta.count(pat) != 1: + sys.exit(5) +patched = tmeta.replace(pat, zz(-(1 << 20))) +build("bad_offset.parquet", tbody, patched, len(patched)) + +# 5. two dictionary pages in one chunk (the format allows at most one). +dmd = pq.ParquetFile(plain).metadata.row_group(0).column(0) +if not dmd.dictionary_page_offset or dmd.data_page_offset <= dmd.dictionary_page_offset: + sys.exit(7) +dlen = dmd.data_page_offset - dmd.dictionary_page_offset +doff = dmd.dictionary_page_offset - 4 +build("two_dicts.parquet", body[:doff] + body[doff:doff + dlen] * 2 + body[doff + dlen:]) + +# 6. a DATA_PAGE_V2 whose declared level lengths exceed the page (see the shell for +# why this rejection does not isolate the v2 guard). +synth_v2 = (fh(1, I32) + zz(3) + fh(1, I32) + zz(64) + fh(1, I32) + zz(32) + + fh(5, STRUCT) + fh(1, I32) + zz(200) + fh(1, I32) + zz(0) + + fh(1, I32) + zz(200) + fh(1, I32) + zz(0) + fh(1, I32) + zz(64) + + fh(1, I32) + zz(0) + fh(1, 2) + STOP + STOP) +build("v2_levels.parquet", synth_v2 + body[len(synth_v2):]) + +# an all-null column: pyarrow writes it with an EMPTY dictionary page +pq.write_table(pa.table({"c": pa.array([None] * 100, pa.int32())}), + os.path.join(W, "allnull.parquet"), use_dictionary=True, compression="none") + +# #620: four row groups of 1000 rows, for the FDW's Row Groups Decoded counter +n = 4000 +pq.write_table(pa.table({"id": pa.array(range(n), pa.int32()), + "v": pa.array([i * 2 for i in range(n)], pa.int64())}), + os.path.join(W, "stream_groups.parquet"), row_group_size=1000) +if pq.ParquetFile(os.path.join(W, "stream_groups.parquet")).num_row_groups != 4: + sys.exit(6) + +for name in ("zero_pages", "bad_size", "noise_pages", "bad_offset", "two_dicts", + "v2_levels", "allnull", "stream_groups"): + if not os.path.exists(os.path.join(W, name + ".parquet")): + sys.exit(6) diff --git a/test/lib.sh b/test/lib.sh index 39c017da6..d1d0567c1 100755 --- a/test/lib.sh +++ b/test/lib.sh @@ -401,10 +401,24 @@ pgc_setup() { esac echo "-- initdb" - pgc_pg "initdb -D '$PGC_PGDATA' -A trust" >/dev/null 2>&1 + # -U postgres, because every psql below connects as postgres (#1341). initdb's + # default superuser is whoever runs it, which is postgres only under the + # runuser above; a non-root run -- the pytest-cluster job, which chowns + # pkglibdir precisely so that it can stay non-root, driving a shell suite as + # test_objstore_stash_recovery.py does -- got a cluster with no postgres role, + # and every start attempt "failed". pgc_cluster.initdb has always passed it. + pgc_pg "initdb -D '$PGC_PGDATA' -A trust -U postgres" >/dev/null 2>&1 { echo "port=$PGC_PORT" echo "listen_addresses='127.0.0.1'" + # THE SOCKET DIRECTORY, pinned to this cluster's own datadir, as + # pgc_cluster.py pins it. A packaged postgres defaults it to + # /var/run/postgresql, which only postgres can write: under runuser that is + # fine, but a non-root, non-postgres run -- the pytest-cluster job driving + # objstore_module.sh from test_objstore_stash_recovery.py -- FATALs on the + # lock file, and pg_ctl says only "could not start server" (#1717). Every + # connection here is TCP on 127.0.0.1, so nothing reads the socket's path. + echo "unix_socket_directories='$PGC_PGDATA'" echo "shared_preload_libraries='pgcolumnar'" # Deterministic text output so heap and columnar hashes match. echo "extra_float_digits=3" diff --git a/test/lossy_arms.tsv b/test/lossy_arms.tsv index 925a8e6fb..ac504acf2 100644 --- a/test/lossy_arms.tsv +++ b/test/lossy_arms.tsv @@ -133,8 +133,6 @@ native_rewrite group numbers issued after a rewrite are past every earlier one native_truncate file shrank native_truncate GUC on: file shrank native_upgrade_converge fresh $TARGET install has objects to compare -native_vecdecode a ruled-out vector is not decoded -native_zonemap_narrow zone_map reads scale with predicates, not table width (wide ~= narrow) objstore_listing the prefix listing paged (more than one list-type=2 request) objstore_module premise: the real module was restored afterwards objstore_sink_write parallel: every worker PUT went to the prefix diff --git a/test/native_fold_deferral.sh b/test/native_fold_deferral.sh index 1df9062fd..2d7b0810c 100755 --- a/test/native_fold_deferral.sh +++ b/test/native_fold_deferral.sh @@ -29,9 +29,23 @@ GUC="SET pgcolumnar.enable_ungrouped_vector_agg = on;" NP=4 # payload columns ROWS=120000 -sq() { # scalar under this suite's GUC; tail -1 drops the SET tag - env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ - -d "$PGC_DB" -At -c "$GUC $1" 2>/dev/null | tail -1 +# sq SQL -> the result under this suite's GUC, or nothing when the statement failed. +# It used to be `... | tail -1`, which read a failed statement as whatever line came +# last; with ON_ERROR_STOP and the exit status, a failure is empty, and every +# comparison below refuses an empty side (check_text) or a non-number (check_num). +sq() { + local out + out="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ + -d "$PGC_DB" -At -v ON_ERROR_STOP=1 -c "$GUC $1" 2>/dev/null)" || return 0 + tail -1 <<<"$out" +} +# rq SQL -> the result on the row path (the vectorized aggregate off), the same way. +rq() { + local out + out="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ + -d "$PGC_DB" -At -v ON_ERROR_STOP=1 \ + -c "SET pgcolumnar.enable_ungrouped_vector_agg = off; $1" 2>/dev/null)" || return 0 + tail -1 <<<"$out" } explain_line() { # explain_line