diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cf3201a42..cf16dc30a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -185,6 +185,13 @@ jobs: - name: fetch the release tags, for the fixture-blob comparisons run: git fetch --depth=1 origin '+refs/tags/v1.0-alpha*:refs/tags/v1.0-alpha*' + - name: Resolve the object-storage suite host names + # test_objstore_addressing.py, the twin of objstore_addressing.sh, reads + # through virtual-host-style S3 addressing (pgc-bucket.s3.local) against + # its own loopback fixture server, as the suites job's shells do. Without + # the names its tests decline UNAVAILABLE_ENDPOINT and the run ends 67. + run: echo "127.0.0.1 s3.local pgc-bucket.s3.local" | sudo tee -a /etc/hosts + - name: Add the PGDG repository and refresh the package lists run: | set -euo pipefail diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 75f95e4b8..1c3183a7b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -745,6 +745,13 @@ jobs: - name: fetch the release tags, for the fixture-blob comparisons run: git fetch --depth=1 origin '+refs/tags/v1.0-alpha*:refs/tags/v1.0-alpha*' + - name: Resolve the object-storage suite host names + # test_objstore_addressing.py, the twin of objstore_addressing.sh, reads + # through virtual-host-style S3 addressing (pgc-bucket.s3.local) against + # its own loopback fixture server, as the suites job's shells do. Without + # the names its tests decline UNAVAILABLE_ENDPOINT and the run ends 67. + run: echo "127.0.0.1 s3.local pgc-bucket.s3.local" | sudo tee -a /etc/hosts + - name: Add the PGDG repository and install PostgreSQL ${{ matrix.pg }} with its headers run: | set -euo pipefail diff --git a/objstore/columnar_objstore_module.c b/objstore/columnar_objstore_module.c index 69b90bdff..2244cb8e9 100644 --- a/objstore/columnar_objstore_module.c +++ b/objstore/columnar_objstore_module.c @@ -245,8 +245,15 @@ os_wait(PgColumnarObjHandle *h, uint32 sockEvents, int *quiet) { int rc; + /* + * WL_TIMEOUT, or the slice is not a slice: without it WaitLatchOrSocket + * ignores the timeout and waits for the socket or the latch alone, so a + * silent peer never counts a quiet slice. Under proc_exit (a terminated + * worker aborting its upload, #1727) interrupts are held and the latch + * alone never ends the wait either. + */ rc = WaitLatchOrSocket(MyLatch, - WL_LATCH_SET | WL_EXIT_ON_PM_DEATH | sockEvents, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH | sockEvents, h->fd, OS_WAIT_SLICE_MS, PG_WAIT_EXTENSION); if (rc & WL_LATCH_SET) ResetLatch(MyLatch); @@ -2002,8 +2009,8 @@ os_flush_part(PgColumnarObjSink *s) s->h->url, resp.status))); /* The part's ETag is the server's word for its bytes; CompleteMultipart * validates the list against it on a real S3, so echo it verbatim (already - * quoted by the server). The fixture ignores it and concatenates by part - * number, so both are satisfied. */ + * quoted by the server). The fixture checks it the same way and assembles + * the listed parts, so both are satisfied. */ etag = resp.etag[0] ? resp.etag : "\"\""; appendStringInfo(&s->completeXml, "%d%s", @@ -2170,6 +2177,14 @@ objstore_sink_abort(PgColumnarObjSink *s) char *encId = os_uriencode_query(s->uploadId); char *query = psprintf("uploadId=%s", encId); + /* + * The abort can land mid-request (a cancel or a SIGTERM inside + * os_wait), with a part's body still on this connection: the + * DELETE would be read as more of that body, or its answer + * confused with the part's. A fresh connection makes it a request + * of its own (#1722). + */ + os_disconnect(s->h); os_write_request(s->h, "DELETE", query, NULL, 0, &resp, NULL); pfree(query); pfree(encId); diff --git a/src/columnar_sink.c b/src/columnar_sink.c index ea7a2ddba..5d1e2fc51 100644 --- a/src/columnar_sink.c +++ b/src/columnar_sink.c @@ -16,8 +16,11 @@ #include +#include "access/xact.h" +#include "lib/ilist.h" #include "miscadmin.h" #include "storage/fd.h" +#include "storage/ipc.h" #include "columnar_objstore.h" #include "columnar_sink.h" @@ -34,8 +37,62 @@ struct PqSink /* remote (#394): NULL api => local sink */ const PgColumnarObjStoreApi *api; PgColumnarObjSink *rsink; + dlist_node openNode; /* on sink_open_remote while rsink is live */ }; +/* + * Remote sinks with an upload in progress (#1722). A FATAL exit (SIGTERM, as + * pexport_cleanup sends each parallel_export worker on cancel, or + * pg_terminate_backend) calls proc_exit without unwinding, so the writers' + * PG_CATCH never reaches PgColumnarSinkAbort and the multipart upload stays + * open on the store, billed until a lifecycle rule reaps it. A + * before_shmem_exit callback aborts whatever is still on this list. It is + * registered after InitPostgres's ShutdownPostgres, so it runs first, while + * the transaction and the module's handles are still intact. Transaction end + * empties the list: every writer finishes or aborts its sink inside its own + * statement, so an entry still listed then belongs to memory being released. + */ +static dlist_head sink_open_remote = DLIST_STATIC_INIT(sink_open_remote); +static bool sink_exit_registered = false; + +static void +sink_abort_open_at_exit(int code, Datum arg) +{ + dlist_mutable_iter it; + + /* + * The module's abort never raises through PG_TRY, but under proc_exit an + * ERROR is promoted to FATAL, which no CATCH sees. A failed DELETE then + * ends this callback with one more FATAL line; shmem_exit has already + * popped it, so the remaining callbacks still run. + */ + dlist_foreach_modify(it, &sink_open_remote) + { + PqSink *snk = dlist_container(PqSink, openNode, it.cur); + + dlist_delete(&snk->openNode); + snk->api->sink_abort(snk->rsink); + snk->rsink = NULL; + } +} + +static void +sink_xact_callback(XactEvent event, void *arg) +{ + switch (event) + { + case XACT_EVENT_COMMIT: + case XACT_EVENT_PARALLEL_COMMIT: + case XACT_EVENT_ABORT: + case XACT_EVENT_PARALLEL_ABORT: + case XACT_EVENT_PREPARE: + dlist_init(&sink_open_remote); + break; + default: + break; + } +} + PqSink * PgColumnarSinkOpenLocal(const char *path) { @@ -83,6 +140,13 @@ PgColumnarSinkOpen(const char *path) snk->finalPath = pstrdup(path); snk->api = api; snk->rsink = api->sink_create(path, NULL); + if (!sink_exit_registered) + { + before_shmem_exit(sink_abort_open_at_exit, (Datum) 0); + RegisterXactCallback(sink_xact_callback, NULL); + sink_exit_registered = true; + } + dlist_push_head(&sink_open_remote, &snk->openNode); return snk; } return PgColumnarSinkOpenLocal(path); @@ -140,6 +204,8 @@ PgColumnarSinkFinish(PqSink *snk) if (snk->api != NULL) { snk->api->sink_finish(snk->rsink); + /* only now: a complete that raised leaves it listed for the abort */ + dlist_delete(&snk->openNode); snk->rsink = NULL; return; } @@ -173,6 +239,7 @@ PgColumnarSinkAbort(PqSink *snk) { if (snk->rsink != NULL) { + dlist_delete(&snk->openNode); snk->api->sink_abort(snk->rsink); snk->rsink = NULL; } diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 8b086cb67..fb1965c7a 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -3972,6 +3972,9 @@ native_fetch_group_memo native_fetch_group_memo scattered index-fetched values a native_fetch_group_memo native_fetch_group_memo the fetch count is exact (3000 rows found) 15;16;17;18;19 2026-10-22 columnar_reader.c: the memo's last-group fast path serves rows up to 1000 past that group's end native_fetch_group_memo native_fetch_group_memo the measured plan is a nested loop of index probes (fetch path engaged) 15;16;17;18;19 never - native_fetch_group_memo native_fetch_group_memo the table is exact after the in-statement rewrite 15;16;17;18;19 never - +native_fetch_interrupt native_fetch_interrupt pgcolumnar_fetch_row carries an interrupt check 15;16;17;18;19 2026-10-22 columnar_reader.c: pgcolumnar_fetch_row's CHECK_FOR_INTERRUPTS commented out where it stood +native_fetch_interrupt native_fetch_interrupt the interrupt check precedes the row-group resolution 15;16;17;18;19 2026-10-22 columnar_reader.c: pgcolumnar_fetch_row's CHECK_FOR_INTERRUPTS commented out where it stood +native_fetch_interrupt native_fetch_interrupt the row-group resolution is where the catalog read lives 15;16;17;18;19 2026-10-22 columnar_reader.c: pgcolumnar_lookup_row_group's uncached path no longer reads the row-group list, a comment naming the read native_fetch_position native_fetch_position a varying-length column reaches its value through an offset table 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row reaches a varying-length value by decoding every earlier one (the #143 walk) native_fetch_position native_fetch_position doubling the row group does not double the cost of the same fetches 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row counts the validity bits from the group's start for every fetch (the #143 walk);columnar_reader.c: pgcolumnar_fetch_row reaches a varying-length value by decoding every earlier one (the #143 walk) native_fetch_position native_fetch_position every column matches the heap mirror 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row takes the row number as the rank, counting rows rather than present values;columnar_reader.c: pgcolumnar_fetch_row's rank is one past the row's @@ -4009,6 +4012,19 @@ native_fetch_sort_context native_fetch_sort_context premise: the columnar side i native_fetch_sort_context native_fetch_sort_context three-key merge cond with a filter (#720) 15;16;17;18;19 2026-10-22 columnar_tableam.c: the deferred decode builds its needed-column set in the caller's context and bms_free's it there (#720 reverted, first half) native_fetch_sort_context native_fetch_sort_context two-key merge join count is right (#720) 15;16;17;18;19 2026-10-22 columnar_tableam.c: the deferred decode builds its needed-column set in the caller's context and bms_free's it there (#720 reverted, first half) native_fetch_sort_context native_fetch_sort_context two-key merge join sum(id) is right (#720) 15;16;17;18;19 2026-10-22 columnar_tableam.c: the deferred decode builds its needed-column set in the caller's context and bms_free's it there (#720 reverted, first half) +native_fold_deferral native_fold_deferral EXPLAIN group total == stats() group count 15;16;17;18;19 2026-10-22 columnar_vector.c: the fold counts each group twice in its total +native_fold_deferral native_fold_deferral adaptive: eager groups load more than survivors alone (work measured, not assumed) 15;16;17;18;19 never - +native_fold_deferral native_fold_deferral adaptive: only the optimistic first group deferred at 99% survival 15;16;17;18;19 2026-10-22 columnar_vector.c: the fold never defers, so the payload is always loaded eagerly +native_fold_deferral native_fold_deferral deferred fold == heap mirror (selective) 15;16;17;18;19 2026-10-22 columnar_vector.c: a deferred group's first survivor is never folded +native_fold_deferral native_fold_deferral deferred fold == row path (selective) 15;16;17;18;19 2026-10-22 columnar_vector.c: a deferred group's first survivor is never folded +native_fold_deferral native_fold_deferral deferred: every group deferred at 1% survival 15;16;17;18;19 2026-10-22 columnar_vector.c: the fold never defers, so the payload is always loaded eagerly +native_fold_deferral native_fold_deferral deferred: loads == survivors x payload width (the identity) 15;16;17;18;19 2026-10-22 columnar_vector.c: a deferred group's first survivor is never folded +native_fold_deferral native_fold_deferral fold == heap mirror (high survival) 15;16;17;18;19 2026-10-22 columnar_vector.c: a deferred group's first survivor is never folded +native_fold_deferral native_fold_deferral premise: the predicate has survivors (the identity is not 0 == 0) 15;16;17;18;19 never - +native_fold_deferral native_fold_deferral premise: the selective predicate is genuinely selective (<= 2%) 15;16;17;18;19 never - +native_fold_deferral native_fold_deferral the fixture spans several groups 15;16;17;18;19 2026-10-22 columnar_write_state.c: the writer ignores a table's stripe_row_limit +native_fold_deferral native_fold_deferral the loads counter exists under ANALYZE 15;16;17;18;19 never - +native_fold_deferral native_fold_deferral the selective query takes the fold (ANALYZE, post-#602) 15;16;17;18;19 2026-10-22 columnar_vector.c: the Columnar Batch Fold ANALYZE line reports no native_fold_skipguard native_fold_skipguard and reports the 58 it skipped, on its own plan (#542) 15;16;17;18;19 2026-10-22 columnar_reader.c: the fold path stops counting the vectors decode skipped (#542 regression) native_fold_skipguard native_fold_skipguard and the scalar arm agrees 15;16;17;18;19 never - native_fold_skipguard native_fold_skipguard so decoded plus skipped is the group's vectors, on the FOLD arm 15;16;17;18;19 2026-10-22 columnar_reader.c: the fold path stops counting the vectors decode skipped (#542 regression) @@ -4333,6 +4349,105 @@ native_join_vector_agg native_join_vector_agg unique join fold answer equals GUC native_join_vector_agg native_join_vector_agg unique join fold answer equals heap 15;16;17;18;19 never - native_join_vector_agg native_join_vector_agg unique join uses core Agg when GUC off 15;16;17;18;19 never - native_join_vector_agg native_join_vector_agg unique join uses vectorized agg when GUC on 15;16;17;18;19 2026-09-12 drop JOINREL fold +native_late_materialization native_late_materialization a rejected row does not materialize its payload columns 15;16;17;18;19 2026-10-22 columnar_reader.c: a row the qual rejects is skipped but never counted +native_late_materialization native_late_materialization a volatile qual is refused the two-pass path 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar.enable_custom_scan defaults to off +native_late_materialization native_late_materialization and the answers are the same with it off 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar_tuple_insert drops the row whose id is 7 +native_late_materialization native_late_materialization and the volatile query still matches the heap mirror 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar_tuple_insert drops the row whose id is 7 +native_late_materialization native_late_materialization heap agrees on how many rows match 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar_tuple_insert drops the row whose id is 7 +native_late_materialization native_late_materialization nothing was pruned at the group level 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar.enable_custom_scan defaults to off +native_late_materialization native_late_materialization nothing was pruned at the vector level 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar.enable_custom_scan defaults to off +native_late_materialization native_late_materialization premise: the fixture loaded every row 15;16;17;18;19 never - +native_late_materialization native_late_materialization premise: the needle is selective, not most of the table 15;16;17;18;19 never - +native_late_materialization native_late_materialization the executor still reports the rows the scan filtered 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar_tuple_insert drops the row whose id is 7 +native_late_materialization native_late_materialization the query runs on the columnar custom scan 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar.enable_custom_scan defaults to off +native_late_materialization native_late_materialization the rows are identical to the heap mirror 15;16;17;18;19 2026-10-22 columnar_reader.c: the late-materialization path rejects a row the qual accepts +native_late_materialization native_late_materialization the same shape without the volatile term does use it 15;16;17;18;19 2026-10-22 columnar_reader.c: a row the qual rejects is skipped but never counted +native_late_materialization native_late_materialization with the feature off nothing is filtered early 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar.enable_custom_scan defaults to off +native_lazy_slot native_lazy_slot SELECT * through the index matches heap 15;16;17;18;19 2026-10-22 columnar_tableam.c: tuple_insert stores c40 off by one in a wide bigint table, so lz_c holds a wrong value +native_lazy_slot native_lazy_slot a range of rows sums the same as heap 15;16;17;18;19 2026-10-22 columnar_tableam.c: tuple_insert stores c40 off by one in a wide bigint table, so lz_c holds a wrong value +native_lazy_slot native_lazy_slot a row fetched before it is flushed reads correctly 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar_slot_decode_upto decodes one column fewer than the prefix asked for +native_lazy_slot native_lazy_slot a unique violation is still detected through the fetch path 15;16;17;18;19 never - +native_lazy_slot native_lazy_slot a wide table is not in a different class from a narrow one 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar_slot_decode_upto decodes the whole row whatever prefix was asked for +native_lazy_slot native_lazy_slot an ORDER BY over index-fetched rows matches heap 15;16;17;18;19 2026-10-22 columnar_tableam.c: the index fetch no longer clears the slot, so a new row keeps the previous row's decoded prefix +native_lazy_slot native_lazy_slot copying index-fetched rows into a heap table keeps every column 15;16;17;18;19 2026-10-22 columnar_tableam.c: tuple_insert stores c40 off by one in a wide bigint table, so lz_c holds a wrong value +native_lazy_slot native_lazy_slot count(*) through the index matches heap 15;16;17;18;19 never - +native_lazy_slot native_lazy_slot single-row index fetches match heap for every column position 15;16;17;18;19 2026-10-22 columnar_tableam.c: tuple_insert stores c40 off by one in a wide bigint table, so lz_c holds a wrong value +native_lazy_slot native_lazy_slot the fetch really is an index scan 15;16;17;18;19 never - +native_metadata_flush native_metadata_flush a 20-column flush opens a small constant number of metadata relations, not one per row 15;16;17;18;19 2026-10-22 columnar_metadata.c: open_columnar_table never reuses a session's open relation (#445 reverted), so each metadata row opens its catalog again +native_metadata_flush native_metadata_flush a zone-map-pruned range read matches heap (the batched zone rows are correct) 15;16;17;18;19 never - +native_metadata_flush native_metadata_flush and that range read rules out a vector by its zone row alone 15;16;17;18;19 2026-10-22 columnar_metadata.c: zone_map rows are written with a NULL minimum and maximum, so no vector can be ruled out by its zone row +native_metadata_flush native_metadata_flush and the 20-column flush opens at least the three tables every flush writes 15;16;17;18;19 2026-10-22 columnar_metadata.c: the flush session never counts an open, so the witness reads opens=0 for every flush +native_metadata_flush native_metadata_flush every column reads back correct (sum over all 20 columns matches the heap mirror) 15;16;17;18;19 2026-10-22 columnar_reader.c: PgColumnarEncodeValueByLen stores a 4-byte by-value 1777 as 1778 +native_metadata_flush native_metadata_flush narrow (3 col) and wide (20 col) flush open the same number of metadata relations 15;16;17;18;19 2026-10-22 columnar_metadata.c: open_columnar_table never reuses a session's open relation (#445 reverted), so each metadata row opens its catalog again +native_metadata_flush native_metadata_flush the flush emits the metadata-open witness 15;16;17;18;19 2026-10-22 columnar_write_state.c: the flush never begins a metadata session, so it neither batches nor emits the witness +native_metadata_flush native_metadata_flush the wide table's row count survives the batched flush 15;16;17;18;19 never - +native_page_offset_bound native_page_offset_bound alive after projected overrun 15;16;17;18;19 never - +native_page_offset_bound native_page_offset_bound alive after projected page_offset=-1 15;16;17;18;19 never - +native_page_offset_bound native_page_offset_bound alive after whole-group overrun 15;16;17;18;19 2026-10-22 columnar_reader.c: the whole-group read's chunk containment guard removed +native_page_offset_bound native_page_offset_bound alive after whole-group page_offset=-1 15;16;17;18;19 2026-10-22 columnar_reader.c: the whole-group read's chunk containment guard removed +native_page_offset_bound native_page_offset_bound alive after whole-group wrap 15;16;17;18;19 2026-10-22 columnar_reader.c: the whole-group read's chunk containment guard removed +native_page_offset_bound native_page_offset_bound an intact table reads every row and value with projection off 15;16;17;18;19 2026-10-22 columnar_reader.c: both containment guards refuse a chunk ending exactly at its group end +native_page_offset_bound native_page_offset_bound an intact table's projected read returns every id 15;16;17;18;19 2026-10-22 columnar_reader.c: both containment guards refuse every chunk +native_page_offset_bound native_page_offset_bound projected page_offset overrun is refused by the containment guard 15;16;17;18;19 2026-10-22 columnar_reader.c: the projected read's chunk containment guard removed +native_page_offset_bound native_page_offset_bound projected page_offset=-1 hits the containment guard 15;16;17;18;19 2026-10-22 columnar_reader.c: the projected read's chunk containment guard removed +native_page_offset_bound native_page_offset_bound whole-group page_offset overrun is refused by the containment guard 15;16;17;18;19 2026-10-22 columnar_reader.c: the whole-group read's chunk containment guard removed +native_page_offset_bound native_page_offset_bound whole-group page_offset=-1 hits the containment guard 15;16;17;18;19 2026-10-22 columnar_reader.c: the whole-group read's chunk containment guard removed +native_page_offset_bound native_page_offset_bound whole-group page_offset=0 is refused by the containment guard 15;16;17;18;19 2026-10-22 columnar_reader.c: the whole-group read's chunk containment guard removed +native_param_pushdown native_param_pushdown a correlated PARAM_EXEC builds no skip predicate 15;16;17;18;19 2026-10-22 columnar_customscan.c: a correlated PARAM_EXEC is frozen into a scan key too +native_param_pushdown native_param_pushdown a correlated PARAM_EXEC is not mis-pruned (results stay correct) 15;16;17;18;19 never - +native_param_pushdown native_param_pushdown a correlated PARAM_EXEC upper bound is not mis-pruned either 15;16;17;18;19 2026-10-22 columnar_customscan.c: a correlated PARAM_EXEC is frozen into a scan key too +native_param_pushdown native_param_pushdown a generic-plan parameterized qual prunes chunk groups (19 of 20 removed) 15;16;17;18;19 2026-10-22 columnar_customscan.c: an int4 scan-key operand is frozen 2000 above its value +native_param_pushdown native_param_pushdown a second execution in the same session counts with its own value 15;16;17;18;19 2026-10-22 columnar_customscan.c: an int4 scan-key operand is frozen 2000 above its value +native_param_pushdown native_param_pushdown and prunes with its own value too (18 of 20 removed) 15;16;17;18;19 2026-10-22 columnar_customscan.c: an int4 scan-key operand is frozen 2000 above its value +native_param_pushdown native_param_pushdown backend alive 15;16;17;18;19 never - +native_param_pushdown native_param_pushdown literal count is the expected value 15;16;17;18;19 never - +native_param_pushdown native_param_pushdown parameterized count matches the literal count 15;16;17;18;19 2026-10-22 columnar_customscan.c: an int4 scan-key operand is frozen 2000 above its value +native_param_pushdown native_param_pushdown the correlated arm rescans the columnar scan once per outer row 15;16;17;18;19 never - +native_parquet_codecs native_parquet_codecs backend survived the crafted header 15;16;17;18;19 2026-10-22 columnar_parquet_codec.c: a page declaring a size past MaxAllocSize aborts instead of being refused +native_parquet_codecs native_parquet_codecs crafted uncompressed size gives a clean decode error 15;16;17;18;19 2026-10-22 columnar_parquet_codec.c: a page declaring a size past MaxAllocSize aborts instead of being refused +native_parquet_codecs native_parquet_codecs gzip FDW count 15;16;17;18;19 never - +native_parquet_codecs native_parquet_codecs gzip decodes to the generator's rows 15;16;17;18;19 2026-10-22 columnar_parquet_codec.c: every decompressed page over 4 KB has one byte flipped after decompression, the same way for every codec +native_parquet_codecs native_parquet_codecs gzip sum matches the generator 15;16;17;18;19 2026-10-22 columnar_parquet_codec.c: every decompressed page over 4 KB has one byte flipped after decompression, the same way for every codec +native_parquet_codecs native_parquet_codecs lz4 FDW count 15;16;17;18;19 never - +native_parquet_codecs native_parquet_codecs lz4 decodes to the generator's rows 15;16;17;18;19 2026-10-22 columnar_parquet_codec.c: every decompressed page over 4 KB has one byte flipped after decompression, the same way for every codec +native_parquet_codecs native_parquet_codecs premise: the gzip file is GZIP-compressed 15;16;17;18;19 never - +native_parquet_codecs native_parquet_codecs premise: the lz4 file is LZ4-compressed 15;16;17;18;19 never - +native_parquet_codecs native_parquet_codecs premise: the uncompressed file is UNCOMPRESSED 15;16;17;18;19 never - +native_parquet_codecs native_parquet_codecs premise: the zstd file is ZSTD-compressed 15;16;17;18;19 never - +native_parquet_codecs native_parquet_codecs uncompressed decodes to the generator's rows 15;16;17;18;19 2026-10-22 columnar_parquet_codec.c: every decompressed page over 4 KB has one byte flipped after decompression, the same way for every codec +native_parquet_codecs native_parquet_codecs zstd FDW count 15;16;17;18;19 never - +native_parquet_codecs native_parquet_codecs zstd decodes to the generator's rows 15;16;17;18;19 2026-10-22 columnar_parquet_codec.c: every decompressed page over 4 KB has one byte flipped after decompression, the same way for every codec +native_parquet_fdw native_parquet_fdw EXPLAIN shows Foreign Scan 15;16;17;18;19 never - +native_parquet_fdw native_parquet_fdw a one-column query reads one of the eleven columns 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: projection pushdown ignored, every column decoded +native_parquet_fdw native_parquet_fdw a selective predicate skips the three row groups it cannot match 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: row-group skipping never skips a group +native_parquet_fdw native_parquet_fdw and the skipped scan still returns every matching row 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pqfdw_compute_skip moves each group's skip decision to the next group, skipping the one that matches +native_parquet_fdw native_parquet_fdw backend still up after the refusals 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the scan's missing-path refusal removed (the backend crashes) +native_parquet_fdw native_parquet_fdw fdw row count 15;16;17;18;19 never - +native_parquet_fdw native_parquet_fdw full scan: fdw == source 15;16;17;18;19 never - +native_parquet_fdw native_parquet_fdw nested export rows 15;16;17;18;19 never - +native_parquet_fdw native_parquet_fdw nested full scan: fdw == source 15;16;17;18;19 never - +native_parquet_fdw native_parquet_fdw nested-loop self-join (rescan): fdw == source 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: every row group skipped once any qual is pushable +native_parquet_fdw native_parquet_fdw predicate: fdw == source 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: every row group skipped once any qual is pushable +native_parquet_fdw native_parquet_fdw projection: fdw == source 15;16;17;18;19 never - +native_parquet_fdw native_parquet_fdw reject empty path option (HV00D) 15;16;17;18;19 never - +native_parquet_fdw native_parquet_fdw reject invalid table option (HV00D) 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the option validator accepts an unknown option +native_parquet_fdw native_parquet_fdw scalar export rows 15;16;17;18;19 never - +native_parquet_fdw native_parquet_fdw scan of non-parquet file errors (XX001) 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the short-file refusal removed, so the bad-magic guard refuses instead +native_parquet_fdw native_parquet_fdw scan with incompatible declared type errors (42804) 15;16;17;18;19 never - +native_parquet_fdw native_parquet_fdw scan without path option errors (HV00J) 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the scan's missing-path refusal removed (the backend crashes) +native_parquet_fdw native_parquet_fdw skip export rows 15;16;17;18;19 never - +native_parquet_fdw native_parquet_fdw the skip file has four row groups 15;16;17;18;19 2026-10-22 columnar_parquet.c: the exporter writes 131072-row groups instead of 65536 +native_parquet_fieldid native_parquet_fieldid a duplicate field id in the file is refused (42702) 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a requested field id binds the column whose id is one more +native_parquet_fieldid native_parquet_fieldid a duplicate requested field id is refused (22023) 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a requested field id binds the column whose id is one more +native_parquet_fieldid native_parquet_fieldid a field_ids/column-count mismatch is refused (22023) 15;16;17;18;19 never - +native_parquet_fieldid native_parquet_fieldid a file with no field ids is refused (22023) 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the 'file carries no field ids' guard removed, so an id-less file falls to the absent-id refusal +native_parquet_fieldid native_parquet_fieldid a nested/array output column is refused (0A000) 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the scalar-only guard on field-id projection removed +native_parquet_fieldid native_parquet_fieldid an absent field id is refused (22023) 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the absent-field-id refusal becomes abort() +native_parquet_fieldid native_parquet_fieldid backend still up after the refusals 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the absent-field-id refusal becomes abort() +native_parquet_fieldid native_parquet_fieldid field-id projection binds and reorders by id (g=gamma, a=alpha) 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a requested field id binds the column whose id is one more +native_parquet_fieldid native_parquet_fieldid field-id projection reads a subset (only id 3 = beta) 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a requested field id binds the column whose id is one more +native_parquet_fieldid native_parquet_fieldid positional read_parquet is unaffected 15;16;17;18;19 never - +native_parquet_fieldid native_parquet_fieldid the duplicate-requested-id error names the id, not the file 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a requested field id binds the column whose id is one more native_parquet_flba native_parquet_flba FDW scan == source 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a FIXED_LEN_BYTE_ARRAY uuid's bytes are reversed on read native_parquet_flba native_parquet_flba INT32-backed DECIMAL keeps its null 15;16;17;18;19 never - native_parquet_flba native_parquet_flba INT32-backed DECIMAL reads 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: an INT32 or INT64 DECIMAL is widened byte-reversed;columnar_parquet_reader.c: pq_decimal_to_numeric reads a narrower-than-16-byte negative DECIMAL as its unsigned magnitude @@ -4348,6 +4463,24 @@ native_parquet_flba native_parquet_flba pyarrow decimal128 values are exact 15;1 native_parquet_flba native_parquet_flba pyarrow uuid reads as uuid 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a FIXED_LEN_BYTE_ARRAY uuid's bytes are reversed on read native_parquet_flba native_parquet_flba uuid round-trips through export/read 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a FIXED_LEN_BYTE_ARRAY uuid's bytes are reversed on read native_parquet_flba native_parquet_flba whole-row round-trip == source 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a FIXED_LEN_BYTE_ARRAY uuid's bytes are reversed on read +native_parquet_hardening native_parquet_hardening backend survived short chunk list 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the chunk-count refusal becomes abort() +native_parquet_hardening native_parquet_hardening date in range still reads 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: an in-range date decodes one day late +native_parquet_hardening native_parquet_hardening date invalid -> 22008 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the date validity check removed (the offset-overflow branch kept) +native_parquet_hardening native_parquet_hardening date offset overflow -> 22008 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the whole date guard removed, so the epoch offset wraps +native_parquet_hardening native_parquet_hardening import_parquet also raises 22003 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the int2 range guard removed, so 40000 wraps to -25536 +native_parquet_hardening native_parquet_hardening import_parquet left no wrapped rows 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the int2 range guard removed, so 40000 wraps to -25536 +native_parquet_hardening native_parquet_hardening int2 in range still reads 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the int2 guard over-applied, refusing the in-range 30000 +native_parquet_hardening native_parquet_hardening int2 out of range -> 22003 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the int2 range guard removed, so 40000 wraps to -25536 +native_parquet_hardening native_parquet_hardening no wrapped -25536 appears 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the int2 range guard removed, so 40000 wraps to -25536 +native_parquet_hardening native_parquet_hardening parquet_schema still describes a bad chunk list 15;16;17;18;19 never - +native_parquet_hardening native_parquet_hardening read_parquet also raises 22003 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the int2 range guard removed, so 40000 wraps to -25536 +native_parquet_hardening native_parquet_hardening read_parquet in range still reads 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the int2 guard over-applied, refusing the in-range 30000 +native_parquet_hardening native_parquet_hardening short chunk list rejected with a specific error 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the row-group chunk-count guard removed +native_parquet_hardening native_parquet_hardening time in range still reads 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: an in-range INT64 time decodes one microsecond late +native_parquet_hardening native_parquet_hardening time out of day -> 22008 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the INT64 time range guard removed +native_parquet_hardening native_parquet_hardening timestamp in range still reads 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: an in-range timestamp decodes one microsecond late +native_parquet_hardening native_parquet_hardening timestamp overflow -> 22008 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the timestamp guard removed, so the epoch offset wraps +native_parquet_hardening native_parquet_hardening valid two-column file still reads 15;16;17;18;19 never - native_parquet_multifile native_parquet_multifile EXPLAIN reports 3 files 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension native_parquet_multifile native_parquet_multifile EXPLAIN sums row groups across files 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension native_parquet_multifile native_parquet_multifile FDW(directory) == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension @@ -4380,6 +4513,58 @@ native_parquet_multifile native_parquet_multifile predicate skips the two non-ma native_parquet_multifile native_parquet_multifile read_parquet(directory) == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension native_parquet_multifile native_parquet_multifile schema-mismatched file in directory errors 15;16;17;18;19 never - native_parquet_multifile native_parquet_multifile tree read order follows the sorted full path 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked directory's files are not sorted;columnar_parquet_reader.c: a walked entry is read whatever its extension +native_parquet_partition native_parquet_partition IN on a partition column prunes 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning never drops a file +native_parquet_partition native_parquet_partition __HIVE_DEFAULT_PARTITION__ reads as null 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition a byte invalid in the server encoding is rejected 15;16;17;18;19 never - +native_parquet_partition native_parquet_partition a file missing a declared partition component errors 15;16;17;18;19 never - +native_parquet_partition native_parquet_partition a file whose basename looks like a partition component is unaffected 15;16;17;18;19 never - +native_parquet_partition native_parquet_partition a null partition value is not the literal marker 15;16;17;18;19 never - +native_parquet_partition native_parquet_partition a null partition value prunes 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning never drops a file +native_parquet_partition native_parquet_partition a percent-encoded partition value decodes 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition a predicate matching everything prunes nothing 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition a predicate on a file column prunes no files 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition a pruned file's row groups are never counted 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning never drops a file +native_parquet_partition native_parquet_partition a stable clause still prunes 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning never drops a file +native_parquet_partition native_parquet_partition a volatile clause keeps every matching partition row 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition a volatile clause prunes nothing 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the volatile-clause guard on partition pruning removed +native_parquet_partition native_parquet_partition all four partitions read 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition all three partitions still read 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition an empty partition_columns option is rejected 15;16;17;18;19 never - +native_parquet_partition native_parquet_partition an encoded null byte in a partition value is rejected 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the explicit encoded-NUL guard removed, so the encoding check's 22021 stands in +native_parquet_partition native_parquet_partition an inequality on a partition column prunes 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning never drops a file +native_parquet_partition native_parquet_partition an unencoded value is unaffected 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition an unknown partition column errors rather than being ignored 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the unknown-partition-column guard removed, so a later column-count error stands in +native_parquet_partition native_parquet_partition and reads the other half 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning never drops a file +native_parquet_partition native_parquet_partition backend survived the bad declarations 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the unknown-partition-column refusal becomes abort() +native_parquet_partition native_parquet_partition backend survived the malformed partition values 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the unknown-partition-column refusal becomes abort() +native_parquet_partition native_parquet_partition both columns prune down to one file 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning never drops a file +native_parquet_partition native_parquet_partition declaring a file column as a partition column errors 15;16;17;18;19 never - +native_parquet_partition native_parquet_partition group by a partition column 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition one partition value prunes half the files 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning never drops a file +native_parquet_partition native_parquet_partition partition column is typed, not text 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition partition prune plus row-group skip 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition partition values materialize 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition pruned scan still returns the right rows 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition row groups are still skipped inside the surviving files 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning never drops a file +native_parquet_partition native_parquet_partition the encoded value is queryable as itself 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_partition native_parquet_partition the tree reads again once both are gone 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: partition pruning drops the files that match instead of those that do not +native_parquet_projection native_parquet_projection after a dropped column, one column reads 1 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pqfdw_compute_needed matches the needed set by target index, not by attribute number +native_parquet_projection native_parquet_projection after a dropped column, projection [d] == oracle 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pqfdw_compute_needed matches the needed set by target index, not by attribute number +native_parquet_projection native_parquet_projection count(*) decodes nothing, survives corruption 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pqfdw_compute_needed marks every column needed, so projection decodes all +native_parquet_projection native_parquet_projection count(*) reads 0 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pqfdw_compute_needed marks every column needed, so projection decodes all +native_parquet_projection native_parquet_projection good-column values are still correct 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pqfdw_compute_needed marks every column needed, so projection decodes all +native_parquet_projection native_parquet_projection one column reads 1 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pqfdw_compute_needed marks every column needed, so projection decodes all +native_parquet_projection native_parquet_projection projection [*] == oracle 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a BYTE_ARRAY text longer than 2 bytes decodes without its last byte (shared with read_parquet) +native_parquet_projection native_parquet_projection projection [a, d] == oracle 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the last target column is never decoded, even when referenced +native_parquet_projection native_parquet_projection projection [a] == oracle 15;16;17;18;19 never - +native_parquet_projection native_parquet_projection projection [b, c, d] == oracle 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a BYTE_ARRAY text longer than 2 bytes decodes without its last byte (shared with read_parquet) +native_parquet_projection native_parquet_projection projection [c] == oracle 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a BYTE_ARRAY text longer than 2 bytes decodes without its last byte (shared with read_parquet) +native_parquet_projection native_parquet_projection qual on unselected column == oracle 15;16;17;18;19 never - +native_parquet_projection native_parquet_projection select star reads 4 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the last target column is never decoded, even when referenced +native_parquet_projection native_parquet_projection selected 1 + qual 1 reads 2 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pqfdw_compute_needed marks every column needed, so projection decodes all +native_parquet_projection native_parquet_projection selecting the corrupt column does error 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the last target column is never decoded, even when referenced +native_parquet_projection native_parquet_projection selecting the good column skips the corrupt one 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pqfdw_compute_needed marks every column needed, so projection decodes all +native_parquet_projection native_parquet_projection two columns read 2 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the last target column is never decoded, even when referenced native_parquet_pushdown native_parquet_pushdown INT64-backed DECIMAL: predicate skips 3 of 4 groups 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: INT64 statistics are not decoded, so an INT64-backed DECIMAL never skips;columnar_parquet_reader.c: pqfdw_compute_skip never marks a row group skippable native_parquet_pushdown native_parquet_pushdown INT64-backed DECIMAL: skipping did not drop rows 15;16;17;18;19 never - native_parquet_pushdown native_parquet_pushdown INT64-backed DECIMAL: unfiltered scan skips nothing 15;16;17;18;19 never - @@ -4462,6 +4647,69 @@ native_parquet_schema native_parquet_schema type of c_text 15;16;17;18;19 never native_parquet_schema native_parquet_schema type of c_time 15;16;17;18;19 never - native_parquet_schema native_parquet_schema type of c_ts 15;16;17;18;19 never - native_parquet_schema native_parquet_schema type of c_uuid 15;16;17;18;19 never - +native_parquet_stack native_parquet_stack a shallow valid schema chain still reads 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: walk_schema's guard trips on any nesting, not only on stack exhaustion +native_parquet_stack native_parquet_stack backend survived read_parquet on the crash input 15;16;17;18;19 2026-10-22 columnar_thrift.c: PgColumnarThriftSkip calls abort() where its stack guard would raise +native_parquet_stack native_parquet_stack backend survived the nested-struct footer 15;16;17;18;19 2026-10-22 columnar_thrift.c: PgColumnarThriftSkip calls abort() where its stack guard would raise +native_parquet_stack native_parquet_stack backend survived the schema-chain footer 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: walk_schema calls abort() where its stack guard would raise +native_parquet_stack native_parquet_stack backend survived the valid file 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: walk_schema calls abort() on entry, so any footer that parses crashes the backend +native_parquet_stack native_parquet_stack nested-struct footer raises stack-depth, not SIGSEGV 15;16;17;18;19 2026-10-22 columnar_thrift.c: PgColumnarThriftSkip's check_stack_depth() removed, so a nested-struct footer is not bounded +native_parquet_stack native_parquet_stack read_parquet rejects the nested-struct footer the same way 15;16;17;18;19 2026-10-22 columnar_thrift.c: PgColumnarThriftSkip's check_stack_depth() removed, so a nested-struct footer is not bounded +native_parquet_stack native_parquet_stack schema-chain footer raises stack-depth, not SIGSEGV 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: walk_schema's check_stack_depth() removed, so a schema chain is not bounded +native_parquet_streaming native_parquet_streaming a second dictionary page in a chunk is rejected 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the one-dictionary-page-per-chunk guard removed +native_parquet_streaming native_parquet_streaming all-null column (empty dictionary page) still reads 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming backend survived the crafted page areas 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the second-dictionary-page refusal becomes abort() +native_parquet_streaming native_parquet_streaming backend survived the failed read 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming negative page offset is rejected by the offset check 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_read_page_header no longer refuses a negative page offset +native_parquet_streaming native_parquet_streaming page size past end of file is rejected by the size bound 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: compressed_size no longer bounded by what is left of the file +native_parquet_streaming native_parquet_streaming parquet_schema reads a file over the 1GB palloc ceiling 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming parquet_schema reports the columns of the oversized file 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming parquet_schema still reads a normal file 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming read_parquet still reads a normal file 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming streaming: LIMIT 10 (inside group 0) decodes one row group 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming streaming: LIMIT 1010 (into group 1) decodes two row groups 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming streaming: a full scan decodes all four row groups 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming streaming: full scan content is correct 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming streaming: rescan under a nested loop is correct 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming the uncompressed source file still reads 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming unparseable page header is rejected 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming v2 level lengths past the page are rejected 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming value read of the sparse file fails cleanly 15;16;17;18;19 never - +native_parquet_streaming native_parquet_streaming zero-filled page area is rejected, not walked byte by byte 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the progress guard for a data page with no values removed +native_parquet_units native_parquet_units import_parquet honours the unit 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_scale_to_usecs reads MILLIS as micros (no x1000) +native_parquet_units native_parquet_units millis == micros 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_scale_to_usecs reads MILLIS as micros (no x1000) +native_parquet_units native_parquet_units millis pushdown result == oracle 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_scale_to_usecs reads MILLIS as micros (no x1000) +native_parquet_units native_parquet_units millis-scaled stats skip the non-matching group 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_scale_to_usecs reads MILLIS as micros (no x1000) +native_parquet_units native_parquet_units millis-to-micros overflow raises 22008 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_scale_to_usecs reads MILLIS as micros (no x1000) +native_parquet_units native_parquet_units nanos == micros 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_scale_to_usecs reads NANOS as micros (no /1000) +native_parquet_units native_parquet_units nanos read as bigint is exact 15;16;17;18;19 never - +native_parquet_units native_parquet_units nanos sub-us truncates toward zero [frac] 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_scale_to_usecs reads NANOS as micros (no /1000) +native_parquet_units native_parquet_units nanos sub-us truncates toward zero [neg] 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_scale_to_usecs reads NANOS as micros (no /1000) +native_parquet_units native_parquet_units parquet_schema advises bigint for nanos (lossless) 15;16;17;18;19 never - +native_parquet_units native_parquet_units parquet_schema advises time for TIME_MILLIS 15;16;17;18;19 never - +native_parquet_units native_parquet_units parquet_schema advises timestamp for micros 15;16;17;18;19 never - +native_parquet_units native_parquet_units parquet_schema advises timestamp for millis 15;16;17;18;19 never - +native_parquet_units native_parquet_units read_parquet honours the unit 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_scale_to_usecs reads MILLIS as micros (no x1000) +native_parquet_units native_parquet_units the millis file reads back exactly as written 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_scale_to_usecs reads MILLIS as micros (no x1000) +native_parquet_units native_parquet_units time micros decodes to the right time 15;16;17;18;19 never - +native_parquet_units native_parquet_units time millis decodes to the right time 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_scale_to_usecs reads MILLIS as micros (no x1000) +native_parquet_units native_parquet_units ts_ms decodes to the right instant 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_scale_to_usecs reads MILLIS as micros (no x1000) +native_parquet_units native_parquet_units ts_ns decodes to the right instant 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: pq_scale_to_usecs reads NANOS as micros (no /1000) +native_parquet_units native_parquet_units ts_us decodes to the right instant 15;16;17;18;19 never - +native_read_parquet native_read_parquet a role without pg_read_server_files is refused by read_parquet's own gate 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: read_parquet's own pg_read_server_files gate removed +native_read_parquet native_read_parquet and a member of pg_read_server_files reads every row 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a scalar's definition level ignored, every entry read as present +native_read_parquet native_read_parquet backend still up after the refusals 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a scalar's definition level ignored, every entry read as present +native_read_parquet native_read_parquet import parity rows 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a scalar's definition level ignored, every entry read as present +native_read_parquet native_read_parquet nested export rows 15;16;17;18;19 never - +native_read_parquet native_read_parquet nested round-trip: read_parquet == the generated rows 15;16;17;18;19 2026-10-22 columnar_reader.c: the columnar reader returns int4 777 as 778 (store_att_byval), so the source table and the file agree on the wrong value +native_read_parquet native_read_parquet premise: reject missing column definition list 15;16;17;18;19 never - +native_read_parquet native_read_parquet projected column round-trips 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a scalar's definition level ignored, every entry read as present +native_read_parquet native_read_parquet read_parquet == import_parquet 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a scalar's definition level ignored, every entry read as present +native_read_parquet native_read_parquet reject fewer columns than file 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the leaf-count guard removed +native_read_parquet native_read_parquet reject incompatible declared type 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: the scalar physical-type guard removed +native_read_parquet native_read_parquet reject non-parquet file 15;16;17;18;19 never - +native_read_parquet native_read_parquet scalar export rows 15;16;17;18;19 never - +native_read_parquet native_read_parquet scalar read_parquet row count 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a scalar's definition level ignored, every entry read as present +native_read_parquet native_read_parquet scalar round-trip: read_parquet == the generated rows 15;16;17;18;19 2026-10-22 columnar_reader.c: the columnar reader returns int4 777 as 778 (store_att_byval), so the source table and the file agree on the wrong value native_reclaim native_reclaim and every live row is still there, once 15;16;17;18;19 never - native_reclaim native_reclaim compact_rewrite ACCEPTS the boundary threshold 0.0 15;16;17;18;19 2026-10-16 columnar_vacuum.c: compact_rewrite's guard refuses minFrac <= 0.0 native_reclaim native_reclaim compact_rewrite ACCEPTS the boundary threshold 1.0 15;16;17;18;19 2026-10-16 columnar_vacuum.c: compact_rewrite's guard refuses minFrac >= 1.0 @@ -4499,6 +4747,19 @@ native_reclaim native_reclaim repeated recluster reuses space (file plateaus) 15 native_reclaim native_reclaim reused-block data matches heap mirror 15;16;17;18;19 never - native_reclaim native_reclaim row count correct after cycle 15;16;17;18;19 never - native_reclaim native_reclaim ten row groups, every one half deleted 15;16;17;18;19 2026-10-14 the write state ignores stripe_row_limit +native_reclaim_reconcile native_reclaim_reconcile compact_rewrite reconciled the stale row 15;16;17;18;19 2026-10-22 columnar_metadata.c: PgColumnarReconcileFreeList never drops a free row, overlapping or not +native_reclaim_reconcile native_reclaim_reconcile compact_rewrite recorded the free ranges it left 15;16;17;18;19 2026-10-22 columnar_metadata.c: record_free_space drops its insert, so every freed range leaks +native_reclaim_reconcile native_reclaim_reconcile data intact after compact reconcile 15;16;17;18;19 never - +native_reclaim_reconcile native_reclaim_reconcile data intact after reconcile + rewrite 15;16;17;18;19 never - +native_reclaim_reconcile native_reclaim_reconcile data intact after truncate reconcile 15;16;17;18;19 never - +native_reclaim_reconcile native_reclaim_reconcile parity after reuse cycle 1 15;16;17;18;19 never - +native_reclaim_reconcile native_reclaim_reconcile parity after reuse cycle 2 15;16;17;18;19 2026-10-22 columnar_metadata.c: PgColumnarReconcileFreeList never drops a free row, overlapping or not +native_reclaim_reconcile native_reclaim_reconcile parity before 15;16;17;18;19 never - +native_reclaim_reconcile native_reclaim_reconcile plain compact kept the legitimate free ranges, which overlap no live group 15;16;17;18;19 2026-10-22 columnar_metadata.c: PgColumnarReconcileFreeList drops every free row, not only those overlapping a live group +native_reclaim_reconcile native_reclaim_reconcile plain compact reconciled the stale row 15;16;17;18;19 2026-10-22 columnar_metadata.c: PgColumnarReconcileFreeList never drops a free row, overlapping or not +native_reclaim_reconcile native_reclaim_reconcile row count intact 15;16;17;18;19 never - +native_reclaim_reconcile native_reclaim_reconcile stale overlapping free row injected 15;16;17;18;19 never - +native_reclaim_reconcile native_reclaim_reconcile truncate reconciled the stale row 15;16;17;18;19 2026-10-22 columnar_metadata.c: PgColumnarReconcileFreeList never drops a free row, overlapping or not native_recluster native_recluster VACUUM set the visibility map, so the index-only count answers from the index before the recluster 15;16;17;18;19 never - native_recluster native_recluster box parity (pre-recluster) 15;16;17;18;19 never - native_recluster native_recluster full-table parity after recluster 15;16;17;18;19 2026-10-16 columnar_vacuum.c: recluster_online retires none of the old groups (PgColumnarRetireGroup removed) @@ -4535,6 +4796,37 @@ native_repack native_repack the columnar table is unharmed: content hash 19 2026 native_repack native_repack the columnar table is unharmed: row count 19 2026-10-16 columnar_tableam.c: relation_copy_for_cluster returns without raising, so a rewrite copies nothing native_repack native_repack the columnar table is unharmed: storage options 19 never - native_repack native_repack without an identity index it is refused by PostgreSQL, not by us 19 never - +native_rewrite native_rewrite a group deleted below the threshold is not rewritten 15;16;17;18;19 2026-10-22 columnar_vacuum.c: every group with a live row is a rewrite candidate, deleted or not +native_rewrite native_rewrite a rewrite plus later writes reuse no group number 15;16;17;18;19 never - +native_rewrite native_rewrite a second rewrite with nothing deleted rewrites no group 15;16;17;18;19 2026-10-22 columnar_vacuum.c: every group with a live row is a rewrite candidate, deleted or not +native_rewrite native_rewrite an index range read after vacuum returns the rows asked for 15;16;17;18;19 2026-10-22 columnar_vacuum.c: rewrite_one_group inserts no index entries for the rows it moves +native_rewrite native_rewrite compact_rewrite holds ShareUpdateExclusiveLock 15;16;17;18;19 never - +native_rewrite native_rewrite compact_rewrite holds no AccessExclusiveLock 15;16;17;18;19 never - +native_rewrite native_rewrite dead-row space reclaimed 15;16;17;18;19 never - +native_rewrite native_rewrite deleted id absent via index 15;16;17;18;19 never - +native_rewrite native_rewrite every live row still reads back after rewrite and vacuum 15;16;17;18;19 2026-10-22 columnar_vacuum.c: rewrite_one_group drops the first live row of each group it rewrites +native_rewrite native_rewrite group numbers issued after a rewrite are past every earlier one 15;16;17;18;19 never - +native_rewrite native_rewrite index scan returns each live row once 15;16;17;18;19 2026-10-22 columnar_vacuum.c: rewrite_one_group drops the first live row of each group it rewrites +native_rewrite native_rewrite live id present via index 15;16;17;18;19 2026-10-22 columnar_vacuum.c: rewrite_one_group drops the first live row of each group it rewrites +native_rewrite native_rewrite parity after delete 15;16;17;18;19 never - +native_rewrite native_rewrite parity after rewrite 15;16;17;18;19 2026-10-22 columnar_vacuum.c: rewrite_one_group drops the first live row of each group it rewrites +native_rewrite native_rewrite rewrote some groups 15;16;17;18;19 never - +native_rewrite native_rewrite row count after delete 15;16;17;18;19 never - +native_rewrite native_rewrite row count after rewrite 15;16;17;18;19 2026-10-22 columnar_vacuum.c: rewrite_one_group drops the first live row of each group it rewrites +native_rewrite native_rewrite there are deleted rows to reclaim 15;16;17;18;19 never - +native_rewrite native_rewrite unique constraint still enforced 15;16;17;18;19 2026-10-22 columnar_vacuum.c: rewrite_one_group drops the first live row of each group it rewrites +native_rewrite native_rewrite vacuum rewrites into a fresh storage id 15;16;17;18;19 never - +native_rewrite_conc native_rewrite_conc a compact_rewrite holds its lock on t_col mid-verb 15;16;17;18;19 2026-10-22 columnar_vacuum.c: compact_rewrite no longer honours pgcolumnar.maintenance_hold_ms +native_rewrite_conc native_rewrite_conc a reader with a 3 s lock_timeout finishes while the rewrite holds its lock 15;16;17;18;19 2026-10-22 columnar_vacuum.c: compact_rewrite opens the table with AccessExclusiveLock instead of ShareUpdateExclusiveLock +native_rewrite_conc native_rewrite_conc a writer with a 3 s lock_timeout inserts while the rewrite holds its lock 15;16;17;18;19 2026-10-22 columnar_vacuum.c: compact_rewrite opens the table with AccessExclusiveLock instead of ShareUpdateExclusiveLock +native_rewrite_conc native_rewrite_conc all workers succeeded (retried past serialization failures) 15;16;17;18;19 2026-10-22 columnar_metadata.c: every delete flush is refused as a concurrent compaction (over-apply) +native_rewrite_conc native_rewrite_conc and the rewrite still held its lock when both had finished 15;16;17;18;19 2026-10-22 columnar_vacuum.c: compact_rewrite no longer honours pgcolumnar.maintenance_hold_ms +native_rewrite_conc native_rewrite_conc conc-rewrite count/sum 15;16;17;18;19 2026-10-22 columnar_metadata.c: the F3b conflict check never raises, so a delete racing a committed rewrite lands on the retired group +native_rewrite_conc native_rewrite_conc conc-rewrite range 15;16;17;18;19 2026-10-22 columnar_metadata.c: the F3b conflict check never raises, so a delete racing a committed rewrite lands on the retired group +native_rewrite_conc native_rewrite_conc conc-rewrite whole-row 15;16;17;18;19 2026-10-22 columnar_metadata.c: the F3b conflict check never raises, so a delete racing a committed rewrite lands on the retired group +native_rewrite_conc native_rewrite_conc row count sane 15;16;17;18;19 never - +native_rewrite_conc native_rewrite_conc the compactor rewrote groups while the workers were writing 15;16;17;18;19 2026-10-22 columnar_metadata.c: every delete flush is refused as a concurrent compaction (over-apply) +native_rewrite_conc native_rewrite_conc the held rewrite itself completed 15;16;17;18;19 never - native_roundtrip native_roundtrip native aggregate (sum over a scanned column) 15;16;17;18;19 2026-10-17 columnar_reader.c: the native reader never loads the last row group native_roundtrip native_roundtrip native count with filter 15;16;17;18;19 2026-10-17 columnar_reader.c: the native reader never loads the last row group native_roundtrip native_roundtrip native filtered scan 15;16;17;18;19 2026-10-17 columnar_reader.c: a null by-reference column reads back as a present empty value;columnar_reader.c: zone-map pruning skips the groups that can match and keeps the ones that cannot @@ -4597,6 +4889,29 @@ native_saop_pushdown native_saop_pushdown the predicate builder's reject express native_saop_pushdown native_saop_pushdown the predicate builder's reject expression contains SK_SEARCHNULL 15;16;17;18;19 2026-10-16 columnar_reader.c: the reject expression keeps only SK_ISNULL native_saop_pushdown native_saop_pushdown vector-agg count over an IN-list is exact (conservative keys not folded) 15;16;17;18;19 2026-10-16 columnar_reader.c: a set key never probes the bloom filter (it probes zero values and rules the group out) native_saop_pushdown native_saop_pushdown vector-agg path engaged for the guard arm (plan shape) 15;16;17;18;19 never - +native_sort_by native_sort_by backend still up after the refusals 15;16;17;18;19 2026-10-22 columnar_vacuum.c: vacuum_sorted with no key declared aborts the backend in place of its refusal +native_sort_by native_sort_by declared sort_by round-trips 15;16;17;18;19 never - +native_sort_by native_sort_by empty table sorts without error 15;16;17;18;19 2026-10-22 columnar_vacuum.c: the sorted rewrite orders each key descending +native_sort_by native_sort_by equality parity (post-sort) 15;16;17;18;19 never - +native_sort_by native_sort_by equality parity (pre-sort) 15;16;17;18;19 never - +native_sort_by native_sort_by explicit columns lay the rows out as the declared key does 15;16;17;18;19 2026-10-22 columnar_vacuum.c: the declared key loses its first column, so the rewrite sorts by ts alone +native_sort_by native_sort_by explicit-column sort parity 15;16;17;18;19 never - +native_sort_by native_sort_by full-table parity after sort 15;16;17;18;19 never - +native_sort_by native_sort_by premise: before sorting, the scan order does not follow (host, ts) 15;16;17;18;19 never - +native_sort_by native_sort_by row count 15;16;17;18;19 never - +native_sort_by native_sort_by row count after sort 15;16;17;18;19 never - +native_sort_by native_sort_by set_options rejects a nonexistent sort_by column 15;16;17;18;19 never - +native_sort_by native_sort_by set_options rejects a virtual generated sort_by column 18;19 never - +native_sort_by native_sort_by single-row parity 15;16;17;18;19 never - +native_sort_by native_sort_by single-row table sorts without error 15;16;17;18;19 2026-10-22 columnar_vacuum.c: the sorted rewrite orders each key descending +native_sort_by native_sort_by sort_by cleared 15;16;17;18;19 never - +native_sort_by native_sort_by sorting a text key enables group skipping 15;16;17;18;19 2026-10-22 columnar_vacuum.c: vacuum_sorted rewrites in arrival order (no sort keys) +native_sort_by native_sort_by the plan under test is a columnar custom scan 15;16;17;18;19 never - +native_sort_by native_sort_by the scan order follows the declared key (host, ts) 15;16;17;18;19 2026-10-22 columnar_vacuum.c: the sorted rewrite applies only the first sort key (host), leaving ts unordered +native_sort_by native_sort_by vacuum_sorted rejects an explicit virtual generated column 18;19 2026-10-22 columnar_vacuum.c: the virtual-column refusal aborts the backend in place of its error (PG18+) +native_sort_by native_sort_by virtual generated sort keys 15;16;17 never - +native_sort_by native_sort_by zero-arg errors on a dropped declared column 15;16;17;18;19 2026-10-22 columnar_vacuum.c: a dropped declared sort_by column gets the generic column-does-not-exist message +native_sort_by native_sort_by zero-arg errors when no key declared 15;16;17;18;19 2026-10-22 columnar_vacuum.c: vacuum_sorted with no key declared aborts the backend in place of its refusal native_toasted_write native_toasted_write equality on a toasted value (bloom + recheck) matches heap 15;16;17;18;19 2026-10-17 columnar_write_state.c: the bloom hash of a varlena is taken from a fixed value, not the row's flattened one;columnar_write_state.c: the running maximum is never raised past a chunk's first value native_toasted_write native_toasted_write premise: each range selects some rows and not all 15;16;17;18;19 never - native_toasted_write native_toasted_write premise: the source values are toasted (pglz) 15;16;17;18;19 never - @@ -4634,6 +4949,34 @@ native_varlena_bound native_varlena_bound backend survived the corrupt minimum 1 native_varlena_bound native_varlena_bound backend survived the external-tagged minimum 15;16;17;18;19 never - native_varlena_bound native_varlena_bound the txt column has a whole-chunk zone map to corrupt 15;16;17;18;19 never - native_varlena_bound native_varlena_bound with the minimum restored, the range read is clean 15;16;17;18;19 never - +native_vecdecode native_vecdecode EXPLAIN reports Columnar Vectors Decoded 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar.enable_custom_scan defaults to off +native_vecdecode native_vecdecode a nullable aggregate reaches the fold 15;16;17;18;19 2026-10-22 columnar_vector.c: pgcolumnar_batch_shape_eligible refuses every shape, so no aggregate reaches the batch fold +native_vecdecode native_vecdecode a range matching no row at all is still correct 15;16;17;18;19 never - +native_vecdecode native_vecdecode a ruled-out vector is not decoded 15;16;17;18;19 2026-10-22 columnar_reader.c: the zone maps never build a skip mask, so no vector is ruled out +native_vecdecode native_vecdecode aggregate over a range spanning many skipped vectors matches heap 15;16;17;18;19 never - +native_vecdecode native_vecdecode aggregate over a skipping range matches heap 15;16;17;18;19 never - +native_vecdecode native_vecdecode and EVERY ruled-out vector is: decoded plus skipped is the group's vectors 15;16;17;18;19 2026-10-22 columnar_reader.c: decode ignores the skip mask and decodes every vector +native_vecdecode native_vecdecode and it agrees with heap after many leading skipped vectors 15;16;17;18;19 never - +native_vecdecode native_vecdecode and it is not stuck at zero when a full scan decodes everything 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar.enable_custom_scan defaults to off +native_vecdecode native_vecdecode boundary range parity, a range starting exactly on a vector edge 15;16;17;18;19 never - +native_vecdecode native_vecdecode control: and the same counter reads every vector when nothing is skipped 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar.enable_custom_scan defaults to off +native_vecdecode native_vecdecode cross-vector parity of a nullable column, nulls included 15;16;17;18;19 never - +native_vecdecode native_vecdecode cross-vector range parity 15;16;17;18;19 never - +native_vecdecode native_vecdecode narrow range parity with heap 15;16;17;18;19 never - +native_vecdecode native_vecdecode no whole group is pruned on the narrow query 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar.enable_custom_scan defaults to off +native_vecdecode native_vecdecode one row group, so every difference is per-vector 15;16;17;18;19 2026-10-22 columnar_write_state.c: the writer divides the table's stripe_row_limit by 100 +native_vecdecode native_vecdecode premise: the nullable column has nulls inside the ranges read below 15;16;17;18;19 never - +native_vecdecode native_vecdecode the FOLD ARM really decodes only the vectors it needs 15;16;17;18;19 2026-10-22 columnar_reader.c: the zone maps never build a skip mask, so no vector is ruled out +native_vecdecode native_vecdecode the aggregate really reaches the batch fold 15;16;17;18;19 2026-10-22 columnar_vector.c: pgcolumnar_batch_shape_eligible refuses every shape, so no aggregate reaches the batch fold +native_vecdecode native_vecdecode the deep range reaches the fold too 15;16;17;18;19 2026-10-22 columnar_vector.c: pgcolumnar_batch_shape_eligible refuses every shape, so no aggregate reaches the batch fold +native_vecdecode native_vecdecode the fold answers correctly over a range that skips vectors 15;16;17;18;19 never - +native_vecdecode native_vecdecode the fold answers count and sum of a nullable column over a skipping range 15;16;17;18;19 2026-10-22 columnar_vector.c: the fold ignores the validity bitmap, so a null row is read as a present value +native_vecdecode native_vecdecode the fold does not count rows from a vector it never decoded 15;16;17;18;19 2026-10-22 columnar_vector.c: the fold ignores the skip mask and reads a skipped vector's hole as values (the hole zero-filled so the read is deterministic) +native_vecdecode native_vecdecode the narrow plan is a columnar custom scan 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar.enable_custom_scan defaults to off +native_vecdecode native_vecdecode the narrow range skips vectors 15;16;17;18;19 2026-10-22 columnar_reader.c: the zone maps never build a skip mask, so no vector is ruled out +native_vecdecode native_vecdecode the poison-accepting range reaches the fold 15;16;17;18;19 2026-10-22 columnar_vector.c: pgcolumnar_batch_shape_eligible refuses every shape, so no aggregate reaches the batch fold +native_vecdecode native_vecdecode the wide plan is a columnar custom scan 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar.enable_custom_scan defaults to off +native_vecdecode native_vecdecode the wide range skips none, so the arms really differ 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar.enable_custom_scan defaults to off native_vecskip native_vecskip and the aggregate skips the same seven vectors 15;16;17;18;19 2026-10-16 columnar_reader.c: no vector skip at all (zone-map rule off, refine_skipvec and qual_skipvec return at once) native_vecskip native_vecskip cross-vector range parity 15;16;17;18;19 never - native_vecskip native_vecskip empty range parity 15;16;17;18;19 never - @@ -4659,6 +5002,48 @@ native_writer native_writer native row groups (5000 rows / 1000 limit) 15;16;17; native_writer native_writer native row-group row total 15;16;17;18;19 never - native_writer native_writer native storage row written 15;16;17;18;19 never - native_writer native_writer native vector length recorded 15;16;17;18;19 2026-10-16 columnar.h: COLUMNAR_NATIVE_VECTOR_LENGTH 2048 +native_zonemap native_zonemap every vector's maximum of id, k and v is the heap's 15;16;17;18;19 2026-10-22 columnar_write_state.c: no vector carries a minimum or maximum, and so no chunk does either +native_zonemap native_zonemap every vector's minimum of id, k and v is the heap's 15;16;17;18;19 2026-10-22 columnar_write_state.c: no vector carries a minimum or maximum, and so no chunk does either +native_zonemap native_zonemap every whole-chunk maximum of id, k and v is the heap's 15;16;17;18;19 2026-10-22 columnar_write_state.c: no vector carries a minimum or maximum, and so no chunk does either +native_zonemap native_zonemap every whole-chunk minimum of id, k and v is the heap's 15;16;17;18;19 2026-10-22 columnar_write_state.c: no vector carries a minimum or maximum, and so no chunk does either +native_zonemap native_zonemap id (int4) min width = 4 bytes 15;16;17;18;19 never - +native_zonemap native_zonemap k (int8) max width = 8 bytes 15;16;17;18;19 never - +native_zonemap native_zonemap min/max present for all columns 15;16;17;18;19 2026-10-22 columnar_write_state.c: no vector carries a minimum or maximum, and so no chunk does either +native_zonemap native_zonemap null_count sums (v, nullable) 15;16;17;18;19 never - +native_zonemap native_zonemap null_count zero (id, no nulls) 15;16;17;18;19 never - +native_zonemap native_zonemap per-vector value+null = 1024 15;16;17;18;19 never - +native_zonemap native_zonemap per-vector value_count folds into chunk (k) 15;16;17;18;19 never - +native_zonemap native_zonemap per-vector zone rows 15;16;17;18;19 never - +native_zonemap native_zonemap premise: v is not ascending within a vector, so a stale minimum would show 15;16;17;18;19 never - +native_zonemap native_zonemap read parity (no regression) 15;16;17;18;19 never - +native_zonemap native_zonemap row count 15;16;17;18;19 never - +native_zonemap native_zonemap sum null for non-int4 columns 15;16;17;18;19 never - +native_zonemap native_zonemap sum present for int4 columns 15;16;17;18;19 never - +native_zonemap native_zonemap value_count sums (id, no nulls) 15;16;17;18;19 never - +native_zonemap native_zonemap value_count sums (v, nullable) 15;16;17;18;19 never - +native_zonemap native_zonemap whole-chunk sum(id) matches heap 15;16;17;18;19 never - +native_zonemap native_zonemap whole-chunk value+null = group rows 15;16;17;18;19 never - +native_zonemap native_zonemap whole-chunk zone rows 15;16;17;18;19 never - +native_zonemap native_zonemap zone maps cleaned on drop 15;16;17;18;19 never - +native_zonemap_narrow native_zonemap_narrow a pruning predicate still prunes on the wide table 15;16;17;18;19 2026-10-22 columnar_reader.c: group skipping excludes every group +native_zonemap_narrow native_zonemap_narrow and on the narrow table 15;16;17;18;19 2026-10-22 columnar_reader.c: group skipping reads the predicate column's zone map but never excludes a group +native_zonemap_narrow native_zonemap_narrow narrow scan result is correct 15;16;17;18;19 2026-10-22 columnar_reader.c: group skipping excludes every group +native_zonemap_narrow native_zonemap_narrow the zone map removes the nine groups c1 > 19000 cannot match, on the wide table 15;16;17;18;19 2026-10-22 columnar_reader.c: group skipping reads the predicate column's zone map but never excludes a group +native_zonemap_narrow native_zonemap_narrow wide scan result is correct 15;16;17;18;19 2026-10-22 columnar_reader.c: group skipping excludes every group +native_zonemap_narrow native_zonemap_narrow zone_map reads scale with predicates, not table width (wide ~= narrow) 15;16;17;18;19 2026-10-22 columnar_reader.c: every attribute's zone map is read up front (#314 reverted) +native_zonemap_session native_zonemap_session a second predicate column still resolves zone_map once 15;16;17;18;19 2026-10-22 columnar_metadata.c: the session opens zone_map afresh on every probe (#744 reverted inside the session) +native_zonemap_session native_zonemap_session and each of them opened exactly once 15;16;17;18;19 2026-10-22 columnar_reader.c: the scan's zone-map session is one static shared across scans and reset at the end hook (the first implementation of #744) +native_zonemap_session native_zonemap_session and each of them probed exactly its own groups 15;16;17;18;19 2026-10-22 columnar_reader.c: the scan's zone-map session is one static shared across scans and reset at the end hook (the first implementation of #744) +native_zonemap_session native_zonemap_session and the short circuit means it adds one probe, not one per group 15;16;17;18;19 2026-10-22 columnar_reader.c: every zone-map fetch is made twice +native_zonemap_session native_zonemap_session every scan around an aborted one reports its own session (#744) 15;16;17;18;19 2026-10-22 columnar_reader.c: the scan's zone-map session is one static shared across scans and reset at the end hook (the first implementation of #744) +native_zonemap_session native_zonemap_session nested columnar scans each hold their own handle 15;16;17;18;19 2026-10-22 columnar_reader.c: the scan's zone-map session is one static shared across scans and reset at the end hook (the first implementation of #744) +native_zonemap_session native_zonemap_session nested columnar scans return the right answer 15;16;17;18;19 2026-10-22 columnar_reader.c: every chunk group is ruled out +native_zonemap_session native_zonemap_session the fixture has more than one chunk group 15;16;17;18;19 2026-10-22 columnar_write_state.c: the writer multiplies the table's stripe_row_limit by 100 +native_zonemap_session native_zonemap_session the middle scan really did error inside the subtransaction 15;16;17;18;19 2026-10-22 columnar_reader.c: every chunk group is ruled out +native_zonemap_session native_zonemap_session the scan after the subtransaction abort returns the right answer 15;16;17;18;19 2026-10-22 columnar_reader.c: the scan's zone-map session is one static shared across scans and reset at the end hook (the first implementation of #744) +native_zonemap_session native_zonemap_session the scan really probed the zone maps, once per group 15;16;17;18;19 2026-10-22 columnar_reader.c: every zone-map fetch is made twice +native_zonemap_session native_zonemap_session the scan resolves zone_map ONCE, not once per group (#744) 15;16;17;18;19 2026-10-22 columnar_metadata.c: the session opens zone_map afresh on every probe (#744 reverted inside the session) +native_zonemap_session native_zonemap_session zs2 has more than one chunk group 15;16;17;18;19 2026-10-22 columnar_write_state.c: the writer ignores the table's stripe_row_limit nologin_workers nologin_workers a NOLOGIN role's parallel flush encodes every column in a worker 15;16;17;18;19 2026-10-21 the workers connect as the calling role, as before #1585, so PG15 and PG16 refuse a NOLOGIN caller nologin_workers nologin_workers a NOLOGIN role's parallel_copy loads every row 15;16;17;18;19 2026-10-21 the workers connect as the calling role, as before #1585, so PG15 and PG16 refuse a NOLOGIN caller nologin_workers nologin_workers a NOLOGIN role's parallel_export_parquet exports every row 15;16;17;18;19 2026-10-21 the workers connect as the calling role, as before #1585, so PG15 and PG16 refuse a NOLOGIN caller @@ -4674,6 +5059,22 @@ nologin_workers nologin_workers premise: SET ROLE to the NOLOGIN role takes 15;1 nologin_workers nologin_workers premise: the NOLOGIN role cannot open a session 15;16;17;18;19 never - nologin_workers nologin_workers premise: the server log records the NOLOGIN role's refused login 15;16;17;18;19 never - nologin_workers nologin_workers the flush is dispatched to workers 15;16;17;18;19 never - +objstore_addressing objstore_addressing a key with a space, '+' and '=' reads back (path-style) 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: os_uriencode_path encodes a space as '+' +objstore_addressing objstore_addressing a key with a space, '+' and '=' reads back (virtual-host) 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: os_uriencode_path encodes a space as '+' +objstore_addressing objstore_addressing and its path-style request path is the key URI-encoded once 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: virtual-host addressing always (objstore_s3_addressing='path' ignored) +objstore_addressing objstore_addressing and its virtual-host request path is the key alone, URI-encoded once 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: path-style addressing always (objstore_s3_addressing='virtual' ignored) +objstore_addressing objstore_addressing and the request path carried the bucket again 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: virtual-host addressing always (objstore_s3_addressing='path' ignored) +objstore_addressing objstore_addressing gs:// read round-trips (scheme handled, HMAC creds reused) 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the AWS_ENDPOINT_URL override is dropped (ep = NULL) +objstore_addressing objstore_addressing gs:// read-back of the written object 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the AWS_ENDPOINT_URL override is dropped (ep = NULL) +objstore_addressing objstore_addressing gs:// with no endpoint goes to storage.googleapis.com (the allow-list refuses that host by name) 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: gs:// demands AWS_ENDPOINT_URL like s3:// instead of defaulting to storage.googleapis.com +objstore_addressing objstore_addressing gs:// write round-trips 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the AWS_ENDPOINT_URL override is dropped (ep = NULL) +objstore_addressing objstore_addressing path-style put the bucket in the request path 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: virtual-host addressing always (objstore_s3_addressing='path' ignored) +objstore_addressing objstore_addressing premise: fixture up 15;16;17;18;19 never - +objstore_addressing objstore_addressing removal proof: with addressing=path the bucket is back in the path 15;16;17;18;19 never - +objstore_addressing objstore_addressing s3:// path-style read 15;16;17;18;19 never - +objstore_addressing objstore_addressing s3:// with no endpoint demands one (names AWS_ENDPOINT_URL) 15;16;17;18;19 never - +objstore_addressing objstore_addressing virtual-host put the KEY ALONE in the path (no bucket segment) 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: path-style addressing always (objstore_s3_addressing='virtual' ignored) +objstore_addressing objstore_addressing virtual-host read == path-style read (same object, different addressing) 15;16;17;18;19 never - objstore_allowlist objstore_allowlist a multi-entry list matches any entry 15;16;17;18;19 never - objstore_allowlist objstore_allowlist a non-superuser cannot widen the list (42501) 15;16;17;18;19 2026-10-14 objstore_allowed_endpoints defined PGC_SIGHUP, so no session may SET it objstore_allowlist objstore_allowlist a superuser session SET works (admin override) 15;16;17;18;19 2026-10-14 objstore_allowed_endpoints defined PGC_SIGHUP, so no session may SET it @@ -4722,6 +5123,177 @@ objstore_crlf objstore_crlf a CR/LF in the URL path is rejected before the reque objstore_crlf objstore_crlf backend alive 15;16;17;18;19 never - objstore_crlf objstore_crlf backend alive after the REST attempt 15;16;17;18;19 never - objstore_crlf objstore_crlf no injected line reached the listener (no request splitting) 15;16;17;18;19 2026-10-14 the request-line guard on the URL path removed +objstore_http_read objstore_http_read A: count(*) over http 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the Range header's end is one past the last byte wanted +objstore_http_read objstore_http_read A: full scan http == local 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the Range header's end is one past the last byte wanted +objstore_http_read objstore_http_read A: projection http == local 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the Range header's end is one past the last byte wanted +objstore_http_read objstore_http_read B: buffered 2-col requests (K = 4 + G*2) 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the Range header's end is one past the last byte wanted +objstore_http_read objstore_http_read B: buffered full-width requests (K = 4 + G*NCOLS) 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the Range header's end is one past the last byte wanted +objstore_http_read objstore_http_read B: buffered/unbuffered request ratio 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the Range header's end is one past the last byte wanted +objstore_http_read objstore_http_read B: every GET carried a Range header 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: no Range header sent on a GET +objstore_http_read objstore_http_read B: unbuffered >= 10x K 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the Range header's end is one past the last byte wanted +objstore_http_read objstore_http_read C: EOF mid-body is refused 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: os_read_exact ends the read on EOF mid-body instead of raising +objstore_http_read objstore_http_read C: a dropped keep-alive connection is retried (http == local) 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: os_request never retries a stale keep-alive connection +objstore_http_read objstore_http_read C: a server that hangs up on a fresh connection is refused 15;16;17;18;19 never - +objstore_http_read objstore_http_read C: a short 206 body is refused 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: os_read_body accepts a 206 body shorter than the range (cl > want only) +objstore_http_read objstore_http_read C: and that fresh connection is not retried (one request logged) 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: os_request retries a fresh connection that failed before a response +objstore_http_read objstore_http_read C: http 404 SQLSTATE 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: a HEAD 404 read as an empty object +objstore_http_read objstore_http_read C: range-ignoring server SQLSTATE 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the Range-ignored guard dropped (a 200 falls through to unexpected HTTP status) +objstore_http_read objstore_http_read C: unreachable endpoint SQLSTATE 15;16;17;18;19 never - +objstore_http_read objstore_http_read D: cancel latency bounded (<=10s) 15;16;17;18;19 never - +objstore_http_read objstore_http_read D: stalled transfer cancelled with query_canceled 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the Range header's end is one past the last byte wanted +objstore_http_read objstore_http_read premise: buffered run logged requests 15;16;17;18;19 never - +objstore_http_read objstore_http_read premise: fixture server is up 15;16;17;18;19 never - +objstore_http_read objstore_http_read premise: fixture written 15;16;17;18;19 never - +objstore_http_read objstore_http_read premise: the range-audit phase logged GETs 15;16;17;18;19 never - +objstore_http_read objstore_http_read premise: unbuffered run logged requests 15;16;17;18;19 never - +objstore_http_read objstore_http_read survival: the server answers and no backend crashed 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: abort() in place of the HEAD 404 refusal +objstore_listing objstore_listing FDW over a prefix == read_parquet over the prefix 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the listing stops after its first page +objstore_listing objstore_listing Hive over a remote prefix stamps the partition column 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the last key of every page is dropped +objstore_listing objstore_listing a LIST that returns HTTP 500 is refused, not read as an empty prefix 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: a non-2xx LIST read as an empty, untruncated page +objstore_listing objstore_listing a non-advancing continuation token is refused, not an infinite paging loop 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the non-advancing continuation token refusal removed +objstore_listing objstore_listing a partition predicate prunes a remote file (Files Pruned = 1) 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the last key of every page is dropped +objstore_listing objstore_listing a truncated page with no keys is refused, not looped 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the truncated-page-with-no-keys refusal removed +objstore_listing objstore_listing an exact-key read issues no ListObjectsV2 15;16;17;18;19 never - +objstore_listing objstore_listing and the request log recorded the exact-key read 15;16;17;18;19 never - +objstore_listing objstore_listing backend still up after the listing refusals 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: abort() in place of the non-advancing token refusal (the backend crashes) +objstore_listing objstore_listing exact-key read returns its rows 15;16;17;18;19 never - +objstore_listing objstore_listing glob s3://.../lst/*.parquet matches the 3 top-level parts, not sub/ 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the listing stops after its first page +objstore_listing objstore_listing hidden (_SUCCESS) and non-parquet (notes.txt) objects are excluded 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the listing stops after its first page +objstore_listing objstore_listing premise: fixture up 15;16;17;18;19 never - +objstore_listing objstore_listing read_parquet over a prefix reads every object under it (4 parts, recursive) 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the listing stops after its first page +objstore_listing objstore_listing the prefix listing paged (more than one list-type=2 request) 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the listing stops after its first page +objstore_module objstore_module a broken module does NOT masquerade as an unhandled scheme 15;16;17;18;19 never - +objstore_module objstore_module a gs URL reports an object-storage error, not a missing file 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the module stops handling gs:// URLs +objstore_module objstore_module a https URL reports an object-storage error, not a missing file 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: a module built with OpenSSL stops handling https:// URLs +objstore_module objstore_module a local path still works 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: every local read_parquet path is refused as a missing file +objstore_module objstore_module a relative path is not mistaken for a URL 15;16;17;18;19 never - +objstore_module objstore_module a remote glob (*) is handled remotely (an object-storage error, not a local one) 15;16;17;18;19 never - +objstore_module objstore_module a remote glob (?) is handled remotely (an object-storage error, not a local one) 15;16;17;18;19 never - +objstore_module objstore_module a remote glob ([0-9]) is handled remotely (an object-storage error, not a local one) 15;16;17;18;19 never - +objstore_module objstore_module a s3 URL reports an object-storage error, not a missing file 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the module stops handling s3:// URLs +objstore_module objstore_module and neither claims the module is missing 15;16;17;18;19 never - +objstore_module objstore_module and neither downgrades it to an unhandled scheme 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a missing object-store module is reported as an unhandled scheme +objstore_module objstore_module and the * glob is NOT reported as a local filesystem miss 15;16;17;18;19 never - +objstore_module objstore_module and the ? glob is NOT reported as a local filesystem miss 15;16;17;18;19 never - +objstore_module objstore_module and the [0-9] glob is NOT reported as a local filesystem miss 15;16;17;18;19 never - +objstore_module objstore_module and the gs URL is NOT reported as a missing file 15;16;17;18;19 never - +objstore_module objstore_module and the https URL is NOT reported as a missing file 15;16;17;18;19 never - +objstore_module objstore_module and the loader's own reason survives 15;16;17;18;19 2026-10-22 columnar_objstore.c: a module that is present but fails to load is reported as not installed +objstore_module objstore_module and the s3 URL is NOT reported as a missing file 15;16;17;18;19 never - +objstore_module objstore_module both reads name the missing module, identically 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: a missing object-store module is reported as an unhandled scheme +objstore_module objstore_module nor as a module that was never installed 15;16;17;18;19 2026-10-22 columnar_objstore.c: a module that is present but fails to load is reported as not installed +objstore_module objstore_module positive control: it IS defined in the module, so nm really looked 15;16;17;18;19 never - +objstore_module objstore_module premise: and the real module is back afterwards 15;16;17;18;19 never - +objstore_module objstore_module premise: the module really is corrupt for this run, not merely intended to be 15;16;17;18;19 never - +objstore_module objstore_module premise: the module really was absent while those two reads ran 15;16;17;18;19 never - +objstore_module objstore_module premise: the real module was restored afterwards 15;16;17;18;19 never - +objstore_module objstore_module the main library is installed 15;16;17;18;19 never - +objstore_module objstore_module the module's entry point is NOT inside the preloaded library 15;16;17;18;19 never - +objstore_module objstore_module the object-store module is installed BESIDE it, not inside it 15;16;17;18;19 never - +objstore_module objstore_module with the module absent, neither read leaks the loader's own error 15;16;17;18;19 never - +objstore_module objstore_module with the module present, both reads report an unhandled scheme 15;16;17;18;19 never - +objstore_s3_read objstore_s3_read 403 ordering premise: verified requests still reach 404 = 58P01 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: a 404 on open is refused 28000 instead of 58P01 +objstore_s3_read objstore_s3_read 403: tamper bucket surfaces 28000 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: a 403 on open is read as an empty object +objstore_s3_read objstore_s3_read A: count(*) over s3 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: AWS_DEFAULT_REGION outranks AWS_REGION +objstore_s3_read objstore_s3_read A: full scan s3 == local 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: AWS_DEFAULT_REGION outranks AWS_REGION +objstore_s3_read objstore_s3_read A: percent-encoded key s3 == local 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: AWS_DEFAULT_REGION outranks AWS_REGION +objstore_s3_read objstore_s3_read A: projection s3 == local 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: AWS_DEFAULT_REGION outranks AWS_REGION +objstore_s3_read objstore_s3_read B: signing left the request arithmetic unchanged 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: AWS_DEFAULT_REGION outranks AWS_REGION +objstore_s3_read objstore_s3_read backend still up after the s3 refusals 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: abort cleanup of an open handle calls abort() after the ERROR reached the client +objstore_s3_read objstore_s3_read config: s3 read with no AWS_SECRET_ACCESS_KEY is refused naming it 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: a missing AWS_SECRET_ACCESS_KEY signs with an empty secret +objstore_s3_read objstore_s3_read config: s3 read with no environment is 28000 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: s3:// with no AWS_ENDPOINT_URL falls back to a default host instead of refusing +objstore_s3_read objstore_s3_read config: s3 read with no region is refused naming it 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: with no region configured the signer uses us-east-1 instead of refusing +objstore_s3_read objstore_s3_read https endpoint is not downgraded 15;16;17;18;19 never - +objstore_s3_read objstore_s3_read premise: fixture written 15;16;17;18;19 never - +objstore_s3_read objstore_s3_read premise: signed run logged requests 15;16;17;18;19 never - +objstore_s3_read objstore_s3_read premise: sigv4 fixture server is up 15;16;17;18;19 never - +objstore_s3_read objstore_s3_read region: AWS_DEFAULT_REGION is the fallback 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: AWS_DEFAULT_REGION is never consulted +objstore_s3_read objstore_s3_read region: AWS_REGION outranks AWS_DEFAULT_REGION 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: AWS_DEFAULT_REGION outranks AWS_REGION +objstore_sink_write objstore_sink_write all four workers' uploads were open before the cancel 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the sink never flushes a part, so no export opens a multipart upload +objstore_sink_write objstore_sink_write allow-list: an unlisted endpoint refuses 42501 before any write 15;16;17;18;19 never - +objstore_sink_write objstore_sink_write and completed it exactly once (POST with uploadId, no partNumber) 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: CompleteMultipartUpload is never sent and read as done +objstore_sink_write objstore_sink_write arrow export to s3:// also round-trips 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: a single-PUT object loses its last byte +objstore_sink_write objstore_sink_write cancel: no _SUCCESS marker at the prefix 15;16;17;18;19 never - +objstore_sink_write objstore_sink_write cancel: no completed object remains at the prefix 15;16;17;18;19 never - +objstore_sink_write objstore_sink_write cancel: the dispatcher issued DELETE over the known keys 15;16;17;18;19 never - +objstore_sink_write objstore_sink_write cancel: the server holds no open upload under the prefix 15;16;17;18;19 2026-10-22 the before_shmem_exit registration in PgColumnarSinkOpen dropped (noreg) +objstore_sink_write objstore_sink_write inject: every upload started was aborted (no billable orphan) 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: a failed export never aborts its multipart upload +objstore_sink_write objstore_sink_write inject: mid-upload failure surfaces as disk-full (53100) 15;16;17;18;19 never - +objstore_sink_write objstore_sink_write inject: the key does not exist 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: no multipart, the growing object is PUT to the final key as it grows +objstore_sink_write objstore_sink_write inject: the server holds no open upload for the key 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the abort DELETE names an upload that is not this one +objstore_sink_write objstore_sink_write large export to s3:// succeeds 15;16;17;18;19 never - +objstore_sink_write objstore_sink_write large round trip: s3 read-back == source 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the last part is left out of the CompleteMultipartUpload list +objstore_sink_write objstore_sink_write parallel export to an s3:// prefix succeeds 15;16;17;18;19 never - +objstore_sink_write objstore_sink_write parallel remote: a _SUCCESS marker object exists at the prefix 15;16;17;18;19 never - +objstore_sink_write objstore_sink_write parallel remote: the marker went to the prefix as a PUT 15;16;17;18;19 never - +objstore_sink_write objstore_sink_write parallel: a re-run into the same prefix succeeds (overwrite) 15;16;17;18;19 never - +objstore_sink_write objstore_sink_write parallel: every worker PUT went to the prefix 15;16;17;18;19 never - +objstore_sink_write objstore_sink_write parallel: one object per worker was written (4) 15;16;17;18;19 never - +objstore_sink_write objstore_sink_write parallel: the union of the per-worker objects == source 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: a single-PUT object loses its last byte +objstore_sink_write objstore_sink_write part failure: a refused part fails the export (58030) 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: a part upload the server refused is read as stored +objstore_sink_write objstore_sink_write part failure: the key does not exist 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: no multipart, the growing object is PUT to the final key as it grows +objstore_sink_write objstore_sink_write part failure: the server holds no open upload for the key 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the abort DELETE names an upload that is not this one +objstore_sink_write objstore_sink_write premise: a worker was uploading before the cancel 15;16;17;18;19 never - +objstore_sink_write objstore_sink_write premise: sigv4 fixture server is up 15;16;17;18;19 never - +objstore_sink_write objstore_sink_write premise: the export was cancelled, not completed 15;16;17;18;19 never - +objstore_sink_write objstore_sink_write premise: the fault landed mid-multipart (an upload was started) 15;16;17;18;19 never - +objstore_sink_write objstore_sink_write premise: the large export took the multipart protocol 15;16;17;18;19 never - +objstore_sink_write objstore_sink_write premise: the part failure landed mid-multipart (an upload was started) 15;16;17;18;19 never - +objstore_sink_write objstore_sink_write shrink: _SUCCESS certifies the reconciled prefix 15;16;17;18;19 never - +objstore_sink_write objstore_sink_write shrink: a smaller re-run leaves exactly its own 2 parts (stale tail dropped) 15;16;17;18;19 never - +objstore_sink_write objstore_sink_write shrink: parts 0002/0003 from the larger run are gone 15;16;17;18;19 never - +objstore_sink_write objstore_sink_write shrink: the 4-worker run wrote 4 parts + marker 15;16;17;18;19 never - +objstore_sink_write objstore_sink_write small export to s3:// succeeds 15;16;17;18;19 never - +objstore_sink_write objstore_sink_write small object went as ONE plain PUT (no multipart) 15;16;17;18;19 never - +objstore_sink_write objstore_sink_write small round trip: s3 read-back == source 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: a single-PUT object loses its last byte +objstore_sink_write objstore_sink_write terminate: export workers exit within 75 s while the store is silent 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: os_wait without WL_TIMEOUT, so a silent store never counts a quiet slice +objstore_stash_recovery objstore_stash_recovery and the installed module is still the one that was there, not the debris 15;16;17;18;19 2026-10-22 objstore_module.sh start guard with no debris branch: every stash restored over the installed module +objstore_stash_recovery objstore_stash_recovery and the module left installed is still valid 15;16;17;18;19 never - +objstore_stash_recovery objstore_stash_recovery and the stash is cleared 15;16;17;18;19 2026-10-22 objstore_module.sh start guard refuses on any stash, as before #393's fix +objstore_stash_recovery objstore_stash_recovery and the stash is not left behind for the next run to trip on 15;16;17;18;19 2026-10-22 objstore_module.sh start guard refuses on any stash, as before #393's fix +objstore_stash_recovery objstore_stash_recovery premise: a 19-byte stand-in really is installed as the module 15;16;17;18;19 never - +objstore_stash_recovery objstore_stash_recovery premise: a debris stash really is in place for the run below 15;16;17;18;19 never - +objstore_stash_recovery objstore_stash_recovery premise: and the debris is a valid module that differs from the installed one 15;16;17;18;19 never - +objstore_stash_recovery objstore_stash_recovery premise: and the real module is the thing parked at the stash 15;16;17;18;19 never - +objstore_stash_recovery objstore_stash_recovery premise: and the stash beside it is the real module 15;16;17;18;19 never - +objstore_stash_recovery objstore_stash_recovery premise: nothing is installed at the module's path for this slice 15;16;17;18;19 never - +objstore_stash_recovery objstore_stash_recovery premise: the installed module is valid before this slice arranges anything 15;16;17;18;19 never - +objstore_stash_recovery objstore_stash_recovery premise: which is not a module 15;16;17;18;19 never - +objstore_stash_recovery objstore_stash_recovery the debris is gone afterwards 15;16;17;18;19 2026-10-22 objstore_module.sh start guard refuses on any stash, as before #393's fix +objstore_stash_recovery objstore_stash_recovery the module is installed again afterwards, and valid 15;16;17;18;19 2026-10-22 objstore_module.sh start guard decides on presence alone: every stash is debris and is deleted +objstore_stash_recovery objstore_stash_recovery the real module survived, rather than being deleted with the stash 15;16;17;18;19 2026-10-22 objstore_module.sh start guard decides on presence alone: every stash is debris and is deleted +objstore_stash_recovery objstore_stash_recovery the suite recovers the only surviving copy and runs to completion 15;16;17;18;19 2026-10-22 objstore_module.sh start guard walks .away only, so a .probe that is the only copy is never restored +objstore_stash_recovery objstore_stash_recovery the suite replaces the stand-in with the real module and completes 15;16;17;18;19 2026-10-22 objstore_module.sh start guard decides on presence alone: every stash is debris and is deleted +objstore_stash_recovery objstore_stash_recovery the suite runs to completion with debris beside a valid module 15;16;17;18;19 2026-10-22 objstore_module.sh start guard refuses on any stash, as before #393's fix +objstore_tls_read objstore_tls_read A2: s3 over https differential == local 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: trust from SSL_CERT_DIR only (SSL_CTX_set_default_verify_dir in place of set_default_verify_paths) +objstore_tls_read objstore_tls_read A: count(*) over https 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: trust from SSL_CERT_DIR only (SSL_CTX_set_default_verify_dir in place of set_default_verify_paths) +objstore_tls_read objstore_tls_read A: https differential == local 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: trust from SSL_CERT_DIR only (SSL_CTX_set_default_verify_dir in place of set_default_verify_paths) +objstore_tls_read objstore_tls_read C: expired names its reason 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: abort() in place of the certificate-verification refusal +objstore_tls_read objstore_tls_read C: expired refusal is the TLS one 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: abort() in place of the certificate-verification refusal +objstore_tls_read objstore_tls_read C: expired refused with 08006 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: abort() in place of the certificate-verification refusal +objstore_tls_read objstore_tls_read C: expired returned no rows 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: SSL_VERIFY_NONE in place of SSL_VERIFY_PEER +objstore_tls_read objstore_tls_read C: self-signed names its reason 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: abort() in place of the certificate-verification refusal +objstore_tls_read objstore_tls_read C: self-signed refusal is the TLS one 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: abort() in place of the certificate-verification refusal +objstore_tls_read objstore_tls_read C: self-signed refused with 08006 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: abort() in place of the certificate-verification refusal +objstore_tls_read objstore_tls_read C: self-signed returned no rows 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: SSL_VERIFY_NONE in place of SSL_VERIFY_PEER +objstore_tls_read objstore_tls_read C: untrusted-ca names its reason 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: abort() in place of the certificate-verification refusal +objstore_tls_read objstore_tls_read C: untrusted-ca refusal is the TLS one 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: abort() in place of the certificate-verification refusal +objstore_tls_read objstore_tls_read C: untrusted-ca refused with 08006 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: abort() in place of the certificate-verification refusal +objstore_tls_read objstore_tls_read C: untrusted-ca returned no rows 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: SSL_VERIFY_NONE in place of SSL_VERIFY_PEER +objstore_tls_read objstore_tls_read C: wrong-host names its reason 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: abort() in place of the certificate-verification refusal +objstore_tls_read objstore_tls_read C: wrong-host refusal is the TLS one 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: abort() in place of the certificate-verification refusal +objstore_tls_read objstore_tls_read C: wrong-host refused with 08006 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: abort() in place of the certificate-verification refusal +objstore_tls_read objstore_tls_read C: wrong-host returned no rows 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: the IP-literal endpoint is never bound (X509_VERIFY_PARAM_set1_ip_asc skipped) +objstore_tls_read objstore_tls_read D: cancel latency bounded (<=10s) 15;16;17;18;19 never - +objstore_tls_read objstore_tls_read D: stalled TLS transfer cancelled with query_canceled 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: trust from SSL_CERT_DIR only (SSL_CTX_set_default_verify_dir in place of set_default_verify_paths) +objstore_tls_read objstore_tls_read E: SSL_CERT_DIR alone trusts the CA (count(*) over https) 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: trust from SSL_CERT_FILE only (SSL_CTX_set_default_verify_file in place of set_default_verify_paths) +objstore_tls_read objstore_tls_read premise: expired certificate is genuinely expired 15;16;17;18;19 never - +objstore_tls_read objstore_tls_read premise: expired handshake was attempted 15;16;17;18;19 never - +objstore_tls_read objstore_tls_read premise: self-signed handshake was attempted 15;16;17;18;19 never - +objstore_tls_read objstore_tls_read premise: six TLS servers are up 15;16;17;18;19 never - +objstore_tls_read objstore_tls_read premise: untrusted-ca handshake was attempted 15;16;17;18;19 never - +objstore_tls_read objstore_tls_read premise: wrong-host handshake was attempted 15;16;17;18;19 never - +objstore_tls_read objstore_tls_read survival: no backend was terminated by a signal 15;16;17;18;19 2026-10-22 columnar_objstore_module.c: abort() in place of the certificate-verification refusal parallel parallel par count 15;16;17;18;19 2026-10-22 columnar_reader.c: a parallel custom scan claims row groups from its own rowGroupIndex, not the shared counter, so every participant reads every group (D6e) parallel parallel par filtered scan 15;16;17;18;19 2026-10-22 columnar_reader.c: a parallel custom scan claims row groups from its own rowGroupIndex, not the shared counter, so every participant reads every group (D6e) parallel parallel par no-filter 15;16;17;18;19 2026-10-22 columnar_reader.c: a parallel custom scan claims row groups from its own rowGroupIndex, not the shared counter, so every participant reads every group (D6e) @@ -5298,6 +5870,90 @@ pg_dump_roundtrip pg_dump_roundtrip plain: row count survives 15;16;17;18;19 nev pg_dump_roundtrip pg_dump_roundtrip plain: row data survives (checksum) 15;16;17;18;19 never - pg_dump_roundtrip pg_dump_roundtrip plain: the projection declaration IS carried 15;16;17;18;19 2026-10-16 columnar_projection.c: drop_projection keeps the projection's declaration;pgcolumnar--1.0-alpha5.sql: pgcolumnar.projection_declaration no longer registered with pg_extension_config_dump pg_dump_roundtrip pg_dump_roundtrip the base table has a projection to preserve 15;16;17;18;19 never - +phase2 phase2 between count 15;16;17;18;19 2026-10-22 columnar_metadata.c: a zone map's maximum written as its minimum +phase2 phase2 count where a<50 15;16;17;18;19 never - +phase2 phase2 count(*) 15;16;17;18;19 never - +phase2 phase2 empty range 15;16;17;18;19 never - +phase2 phase2 fallback codec=0 15;16;17;18;19 2026-10-22 columnar_compression.c: zstd output kept when it is no smaller than its input +phase2 phase2 fallback value 15;16;17;18;19 never - +phase2 phase2 lz4 block codec 15;16;17;18;19 2026-10-22 columnar_write_state.c: the session's pgcolumnar.compression ignored, every write uses the boot default zstd +phase2 phase2 lz4 count 15;16;17;18;19 never - +phase2 phase2 lz4 distinct pl 15;16;17;18;19 never - +phase2 phase2 lz4 payload ok 15;16;17;18;19 2026-10-22 columnar_metadata.c: a zone map's maximum written as its minimum +phase2 phase2 lz4 sum(id) 15;16;17;18;19 never - +phase2 phase2 minmax present 15;16;17;18;19 never - +phase2 phase2 no nulls t.b 15;16;17;18;19 never - +phase2 phase2 none block codec 15;16;17;18;19 2026-10-22 columnar_write_state.c: the session's pgcolumnar.compression ignored, every write uses the boot default zstd +phase2 phase2 none count 15;16;17;18;19 never - +phase2 phase2 none distinct pl 15;16;17;18;19 never - +phase2 phase2 none payload ok 15;16;17;18;19 2026-10-22 columnar_metadata.c: a zone map's maximum written as its minimum +phase2 phase2 none sum(id) 15;16;17;18;19 never - +phase2 phase2 nonnull val 15;16;17;18;19 2026-10-22 columnar_metadata.c: a zone map's maximum written as its minimum +phase2 phase2 null count 15;16;17;18;19 never - +phase2 phase2 null rows 15;16;17;18;19 never - +phase2 phase2 null value 15;16;17;18;19 2026-10-22 columnar_metadata.c: a zone map's maximum written as its minimum +phase2 phase2 order by a limit3 15;16;17;18;19 never - +phase2 phase2 orphan chunks 15;16;17;18;19 2026-10-22 columnar_metadata.c: a dropped table's column_chunk rows kept +phase2 phase2 orphan row groups 15;16;17;18;19 2026-10-22 columnar_metadata.c: a dropped table's row_group rows kept +phase2 phase2 pglz block codec 15;16;17;18;19 2026-10-22 columnar_write_state.c: the session's pgcolumnar.compression ignored, every write uses the boot default zstd +phase2 phase2 pglz count 15;16;17;18;19 never - +phase2 phase2 pglz distinct pl 15;16;17;18;19 never - +phase2 phase2 pglz payload ok 15;16;17;18;19 2026-10-22 columnar_metadata.c: a zone map's maximum written as its minimum +phase2 phase2 pglz sum(id) 15;16;17;18;19 never - +phase2 phase2 point lookup 15;16;17;18;19 2026-10-22 columnar_metadata.c: a zone map's maximum written as its minimum +phase2 phase2 project a 15;16;17;18;19 2026-10-22 columnar_metadata.c: a zone map's maximum written as its minimum +phase2 phase2 project b 15;16;17;18;19 2026-10-22 columnar_metadata.c: a zone map's maximum written as its minimum +phase2 phase2 project b,c 15;16;17;18;19 never - +phase2 phase2 project c 15;16;17;18;19 2026-10-22 columnar_metadata.c: a zone map's maximum written as its minimum +phase2 phase2 project sum 15;16;17;18;19 never - +phase2 phase2 range decodes 1 of 3 vectors 15;16;17;18;19 2026-10-22 columnar_reader.c: the zone maps rule out no vector +phase2 phase2 range min/max 15;16;17;18;19 2026-10-22 columnar_metadata.c: a zone map's maximum written as its minimum +phase2 phase2 v0 max=10000 15;16;17;18;19 2026-10-22 columnar_metadata.c: a zone map's maximum written as its minimum +phase2 phase2 v0 min=1 15;16;17;18;19 never - +phase2 phase2 v2 max=25000 15;16;17;18;19 2026-10-22 columnar_metadata.c: a zone map's maximum written as its minimum +phase2 phase2 vectors 15;16;17;18;19 never - +phase2 phase2 zstd block codec 15;16;17;18;19 2026-10-22 columnar_write_state.c: the block codec never applied, every chunk stored with codec 0 +phase2 phase2 zstd count 15;16;17;18;19 never - +phase2 phase2 zstd distinct pl 15;16;17;18;19 never - +phase2 phase2 zstd payload ok 15;16;17;18;19 2026-10-22 columnar_metadata.c: a zone map's maximum written as its minimum +phase2 phase2 zstd sum(id) 15;16;17;18;19 never - +phase3 phase3 bulk update rows 15;16;17;18;19 never - +phase3 phase3 bulk update sfx 15;16;17;18;19 never - +phase3 phase3 delete boundaries 15;16;17;18;19 never - +phase3 phase3 delete count 15;16;17;18;19 never - +phase3 phase3 delete gone 15;16;17;18;19 never - +phase3 phase3 delete neighbors 15;16;17;18;19 never - +phase3 phase3 delete tail count 15;16;17;18;19 never - +phase3 phase3 delete tail max 15;16;17;18;19 never - +phase3 phase3 delete_vector cleaned 15;16;17;18;19 2026-10-22 columnar_metadata.c: a dropped table's delete_vector rows kept +phase3 phase3 delete_vector deleted 15;16;17;18;19 never - +phase3 phase3 delete_vector rows>0 15;16;17;18;19 never - +phase3 phase3 durable odd only 15;16;17;18;19 never - +phase3 phase3 durable survivors 15;16;17;18;19 never - +phase3 phase3 rb after count 15;16;17;18;19 never - +phase3 phase3 rb after range 15;16;17;18;19 never - +phase3 phase3 rb before 15;16;17;18;19 never - +phase3 phase3 reg count 15;16;17;18;19 never - +phase3 phase3 reg filter a<100 15;16;17;18;19 never - +phase3 phase3 row group cleaned 15;16;17;18;19 2026-10-22 columnar_metadata.c: a dropped table's row_group rows kept +phase3 phase3 ryw count 15;16;17;18;19 never - +phase3 phase3 ryw persisted 15;16;17;18;19 never - +phase3 phase3 ryw sum 15;16;17;18;19 never - +phase3 phase3 savepoint failed delete discarded 15;16;17;18;19 2026-10-22 columnar_tableam.c: a savepoint's abort keeps its pending delete-vector changes +phase3 phase3 savepoint failed final 15;16;17;18;19 never - +phase3 phase3 savepoint failed insert discarded 15;16;17;18;19 2026-10-22 columnar_tableam.c: a savepoint's abort keeps its pending writes +phase3 phase3 savepoint final 15;16;17;18;19 never - +phase3 phase3 savepoint in-txn 15;16;17;18;19 never - +phase3 phase3 savepoint nested read in a delete sees its deleted rows, as on heap 15;16;17;18;19 2026-10-22 columnar_tableam.c: a savepoint's commit does not hand its delete buffers and written marks to the parent +phase3 phase3 temp deleted 15;16;17;18;19 never - +phase3 phase3 temp survivors 15;16;17;18;19 never - +phase3 phase3 the deleted range straddles vectors 0 and 1 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar.chunk_group_row_limit boots at 16384, not 10000 +phase3 phase3 update count 15;16;17;18;19 never - +phase3 phase3 update key gone 15;16;17;18;19 never - +phase3 phase3 update key new 15;16;17;18;19 never - +phase3 phase3 update key total 15;16;17;18;19 never - +phase3 phase3 update one row 15;16;17;18;19 never - +phase3 phase3 update value 15;16;17;18;19 never - preimage_rewrite preimage_rewrite ...and prunes as well as the equivalent explicit range 15;16;17;18;19 2026-10-21 columnar_customscan.c: the date_trunc preimage keys are derived and then dropped, so no rewrite happens preimage_rewrite preimage_rewrite ...and still answers correctly with the fold enabled 15;16;17;18;19 never - preimage_rewrite preimage_rewrite ...and the heap agrees it is unsatisfiable 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index dbcc9fa8f..ed89477fe 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 69 +suites_not_covered 34 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -6112,4 +6112,696 @@ suites_not_covered 69 # never 4176 + 16 = 4192 # dated 2914 + 38 = 2952 # 4192 + 2952 == 7144 -checks_never_observed_red 4192 +# +# #1341, native_fold_deferral seeded, with its twin (OffgridwithJD): 4192 -> 4196, and +# suites_not_covered 69 -> 68. The suite had no rows. It enters with 13 checks from +# five-major runs (all green on each of 15-19, shell and twin alike), after three fixes. +# The heap mirror was a CTAS from the columnar table, so it read through the scan under +# test: with the writer storing one value off by one, the fold, the row path and the +# mirror all agreed and main passed 13/13. The mirror is now written from the +# generator. The three answer checks compared readings taken with tail -1 using check: +# with the server stopped all three passed on empty strings. Readings now fail empty +# and the checks are check_text. And three premises a one-line pgcolumnar change +# reddens (the fold's ANALYZE line, the EXPLAIN group total, the writer's stripe limit) +# lost the prefix. 9 are dated by eight mutations, each reddening the same names in +# both twins. 4 arrive never: the two remaining premises, the counter's presence, and +# the eager-load bound, which no measured mutation reaches: with every group deferring, +# the loads equal survivors times width exactly, which still clears its 99% floor. +# +# ledger rows 7144 + 13 = 7157 +# never 4192 + 4 = 4196 +# dated 2952 + 9 = 2961 +# 4196 + 2961 == 7157 +# +# #1341, native_lazy_slot seeded, with its twin (OffgridwithJD): 4196 -> 4199, +# and suites_not_covered 68 -> 67. The suite had no rows. It enters with its 10 +# checks from five-major runs (all green on each of 15-19, shell and twin alike), after two +# fixes. The heap mirror was a copy of lz_c, read through the reader under test, so a value +# the writer stored wrong was wrong on both sides: with the writer storing c40 off by one, +# main passed 10/10. It is now written from the generator, and that mutant reddens four +# checks. And five comparisons read `q ... | tail -1` and compared with check, so two failed +# reads agreed, and a failed timing read as fast: with the server stopped after setup, six +# checks passed on main. Each side's failure now carries its side's name, and a failed +# timing is no reading. 7 are dated by four mutations, each reddening the same names in +# both twins. 3 arrive never. +# +# ledger rows 7157 + 10 = 7167 +# never 4196 + 3 = 4199 +# dated 2961 + 7 = 2968 +# 4199 + 2968 == 7167 +# +# #1341, native_metadata_flush seeded, with its twin (OffgridwithJD): 4199 -> 4201, +# and suites_not_covered 67 -> 66. The suite had no rows. It enters with 8 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after four fixes, +# each measured on main's shell, which passed all six under it. A witness that counted +# nothing (zerocount, opens=0 for every flush) passed both open-count checks, since 0 equals +# 0 and 0 is small: a floor now wants at least the three tables every flush writes. The heap +# mirror was copied from the columnar table, so a writer storing a wrong value +# (writecorrupt) agreed with itself: it is now written from the generator. Nothing showed +# the zone rows were read, since the reader also rules vectors out by value after decoding: +# a check now wants a vector ruled out by its zone row alone (Skipped minus Ruled Out by +# Value), and zone rows with no min/max (zonenull) redden it. And the witness check was +# labelled a premise though a flush that never begins its session (nosession) reddens it, +# so it lost the prefix (#1339). 6 are dated by five mutations, each the same names in +# both twins. 2 arrive never: the row count and the range read against the heap mirror. +# +# ledger rows 7167 + 8 = 7175 +# never 4199 + 2 = 4201 +# dated 2968 + 6 = 2974 +# 4201 + 2974 == 7175 +# +# #1341, native_late_materialization seeded, with its twin (OffgridwithJD): 4201 -> 4203, and +# suites_not_covered 66 -> 65. The suite had no rows. It enters with 14 checks from +# five-major runs (all green on each of 15-19, shell and twin alike), after two fixes. +# Every counter defaulted to 0 when it could not be read, and four checks want exactly +# 0: with the server stopped before the feature-off arm, "with the feature off nothing +# is filtered early" and "a volatile qual is refused the two-pass path" passed on main. +# An unread counter is now "no reading". And five checks a one-line pgcolumnar change +# reddens (the custom scan off by default, the early counter dropped, a survivor +# rejected, a writer that drops one row) lost the prefix; the two fixture premises +# now read the heap mirror h, which pgcolumnar does not write (jdatcmd's #1691 +# review). 12 are dated by seven mutations, each reddening the same names in both +# twins: dropseven (pgcolumnar_tuple_insert drops the row whose id is 7) dates "heap +# agrees on how many rows match", the executor's own Rows Removed by Filter and the +# two answer checks with the feature off and refused, which no mutation of the +# late-materialization path reaches. 2 arrive never: the two fixture premises. +# Building a rejected row's payload anyway (matall) reddens nothing: no counter +# reports per-row materialization. +# +# ledger rows 7175 + 14 = 7189 +# never 4201 + 2 = 4203 +# dated 2974 + 12 = 2986 +# 4203 + 2986 == 7189 +# +# #1341, native_reclaim_reconcile seeded, with its twin (OffgridwithJD): 4203 -> 4210, +# and suites_not_covered 65 -> 64. The suite had no rows. It enters with its 13 +# checks from five-major runs (all green on each of 15-19, shell and twin alike), after two +# fixes. Every reconcile check counted only the injected stale row, so a reconcile that +# dropped EVERY free row, leaking all reusable space, passed 11/11 on main. A plain compact +# (which does not reuse) must now keep the legitimate free ranges the earlier rewrite left, +# as count and bytes, and "compact_rewrite recorded the free ranges it left" checks that +# one exists. It is not a premise (jdatcmd's #1692 review): the ranges exist only because +# pgcolumnar recorded them, and nofree (record_free_space drops its insert, one line) +# reddens it alone. And "row count intact" compared two +# readings with plain check: with the server stopped before it, main passed it on two +# empty strings. It is check_num now. The reuse-cycle loop is unrolled so its two names are +# literals. 6 are dated by three mutations, each reddening the same names in both twins. +# 7 arrive never. +# +# ledger rows 7189 + 13 = 7202 +# never 4203 + 7 = 4210 +# dated 2986 + 6 = 2992 +# 4210 + 2992 == 7202 +# +# #1341, native_page_offset_bound seeded, with its twin (OffgridwithJD): 4210 -> 4212, and +# suites_not_covered 64 -> 63. The suite had no rows. It enters with 12 checks from +# five-major runs (all green on each of 15-19, shell and twin alike), after three fixes. +# Every refusal arm counted any psql failure as refused: with the whole-group guard +# removed the backend segfaulted and the arm still passed, and with the projected +# guard removed a later XX001 passed it. Each now wants XX001 with the guard's own +# message. Survival was a bare SELECT 1, which answers again after crash recovery: +# with the probe 4s late, main passed both whole-group survival arms with the backend +# killed. It now also reads the server log's crash count. And nothing read an intact +# table, so a guard over-applied to refuse every chunk (overall) or a legal chunk +# ending exactly at its group end (offby1) passed all 10: two controls now read rows +# behind an OFFSET 0 fence, which an aggregate over an intact table would not decode. +# 10 are dated by four mutations, each reddening the same names in both twins. With +# the whole-group guard removed, whether the read segfaults or fails decoding depends +# on where it lands; its three refusal checks are red either way. 2 arrive never: +# the two projected survival arms, which no mutation crashes. +# +# ledger rows 7202 + 12 = 7214 +# never 4210 + 2 = 4212 +# dated 2992 + 10 = 3002 +# 4212 + 3002 == 7214 +# +# #1341, native_param_pushdown seeded, with its twin (OffgridwithJD): 4212 -> 4216, and +# suites_not_covered 63 -> 62. The suite had no rows. It enters with 10 checks from +# five-major runs (all green on each of 15-19, shell and twin alike), after three fixes. +# With the PARAM_EXEC guard removed (freezeexec) the correlated arm still answered 3002: +# Begin freezes before any outer row sets the param, and "ts >= " +# prunes nothing. The arm now reads the scan's Usable Skip Predicates and asks the upper +# bound too, where that frozen value prunes every group. Every arm ran one execution per +# session, so a value frozen once and reused (staleparam) passed; two executions in one +# session now each count and prune with their own value. And survival was a bare +# SELECT 1, green on main with a backend SIGKILLed; it now reads the crash count. +# 6 are dated by four mutations, each reddening the same names in both twins. 4 +# arrive never: the literal count, the rescan premise, the lower-bound correlated answer +# and survival (red only under the harness kill). +# +# ledger rows 7214 + 10 = 7224 +# never 4212 + 4 = 4216 +# dated 3002 + 6 = 3008 +# 4216 + 3008 == 7224 +# +# #1341, native_sort_by seeded, with its twin (OffgridwithJD): 4216 -> 4230, and +# suites_not_covered 62 -> 61. The suite had no rows. It enters with 23 rows: 20 checks +# run on every major, two virtual-column checks run on 18-19, and their skip recorded +# on 15-17, all green in shell and twin alike on each of 15-19, after two fixes. +# The physical order was never read: group skipping on host was the only evidence of a +# sort, and a rewrite that applied only the first key (ts left unordered) passed 16/16 on +# main. The rows are now read in scan order and every adjacent pair is compared on (host, +# ts), with a premise that the inserted order is not already sorted, and the explicit- +# column sort must lay the rows out as the declared key does. And every refusal counted +# any psql failure: a backend aborting in place of the no-key refusal passed its own +# check, a crash in place of the virtual-column refusal passed 18/18 on PG18, and a +# dropped declared column given the generic message passed. Each refusal now wants its +# guard's SQLSTATE and message, and a new check reads the server log's crash count. 9 +# are dated by the mutations, each reddening the same names in both twins. 14 arrive +# never: the premise, the virtual arm's skip on 15-17, and the checks no measured +# mutation reaches. +# +# ledger rows 7224 + 23 = 7247 +# never 4216 + 14 = 4230 +# dated 3008 + 9 = 3017 +# 4230 + 3017 == 7247 +# +# #1341, native_rewrite seeded, with its twin (OffgridwithJD): 4230 -> 4241, +# and suites_not_covered 61 -> 60. The suite had no rows. It enters with its 20 +# checks from five-major runs (all green on each of 15-19, shell and twin alike), after four +# fixes. Rewriting every undeleted group, or ignoring min_deleted_fraction, passed all 17 on +# main: two checks now hold the threshold from both sides. vacuum without its index rebuild +# passed too: stale entries land on every live row once over the whole table, so a count +# and a hash through the index both matched; a narrow index range read now catches it +# (id 4097 read back as 6146). The four counts compared two readings with plain check, and +# "unique constraint still enforced" took any psql failure: with the server stopped after +# the rewrite, four checks passed on main. Counts are check_num and the refusal wants +# 23505 and n_id_uq's message. 9 are dated by five mutations of the rewrite, each +# reddening the same names in both twins. 11 arrive never. +# +# ledger rows 7247 + 20 = 7267 +# never 4230 + 11 = 4241 +# dated 3017 + 9 = 3026 +# 4241 + 3026 == 7267 +# +# #1341, native_zonemap seeded, with its twin (OffgridwithJD): 4241 -> 4259, and +# suites_not_covered 60 -> 59. The suite had no rows. It enters with 23 checks from +# five-major runs (all green on each of 15-19, shell and twin alike), after two fixes. +# The bounds were checked for presence and width only, and every column ascended, so a +# vector's first value was always its minimum: with a vector max never raised past its +# first value, a min never lowered, or a chunk max never folded past its first vector, +# main passed 18/18. v is now a permutation (values, nulls and counts unchanged), and +# four checks decode each bound of id, k and v and compare it with the heap's for its +# group and vector. And three checks compared two readings with check: with the server +# stopped they passed on two empty strings; they are now check_num. 5 are dated by +# four mutations, each reddening the same names in both twins. 18 arrive never. +# +# ledger rows 7267 + 23 = 7290 +# never 4241 + 18 = 4259 +# dated 3026 + 5 = 3031 +# 4259 + 3031 == 7290 +# +# #1341, native_zonemap_session seeded, with its twin (OffgridwithJD): 4259 -> 4259, +# and suites_not_covered 59 -> 58. The suite had no rows. It enters with 13 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after two fixes. +# "nested columnar scans each hold their own handle" compared only the join's count, which +# a shared handle still answers correctly: with one static session shared across scans, +# the first implementation's defect, main passed it. It is now the witness, each scan +# reporting its own groups opened once, and the count is a check of its own. The three scans +# around the aborted one must also each probe exactly their own groups. Three premises that +# a one-line pgcolumnar change reddens lost the prefix (#1339), and a new check holds zs2 +# to more than one group, which a writer ignoring stripe_row_limit fails. The third is +# "the fixture has more than one chunk group" (jdatcmd's #1698 review): it reads the +# writer's row_group catalog, and optx100 (a writer that multiplies stripe_row_limit by +# 100) reddens it. All 13 are dated by seven mutations, each reddening the same names in +# both twins; none arrives never. The re-resolved index oid columnar_metadata.c documents +# is not counted by any witness here: with it restored the suite stays 13/13, stated +# rather than claimed. +# +# ledger rows 7290 + 13 = 7303 +# never 4259 + 0 = 4259 +# dated 3031 + 13 = 3044 +# 4259 + 3044 == 7303 +# +# #1341, native_zonemap_narrow seeded, with its twin (OffgridwithJD): 4259 -> 4259, +# and suites_not_covered 58 -> 57. The suite had no rows. It enters with 6 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after one fix. +# Nothing showed a group was skipped: the pruning check read only the answer, so a scan +# that read the zone maps but never excluded a group, or never consulted them, returned +# the same 1000 rows, and main passed all four checks under each. Two checks now read the +# scan's Columnar Chunk Groups Removed by Filter (9 of 10, on each table), and the width +# check refuses an unread count rather than comparing two blanks. 6 are dated by four +# mutations, each reddening the same names in both twins. 0 arrive never. +# +# ledger rows 7303 + 6 = 7309 +# never 4259 + 0 = 4259 +# dated 3044 + 6 = 3050 +# 4259 + 3050 == 7309 +# +# #1341, native_vecdecode seeded, with its twin (OffgridwithJD): 4259 -> 4269, +# and suites_not_covered 57 -> 56. The suite had no rows. It enters with 28 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after three fixes. +# The fixture had no nulls, so a fold that ignored the validity bitmap (nonull) passed all +# 24 on main: a nullable column w now runs through the fold and through a scalar parity +# check. "decoded plus skipped is the group's vectors" read missing counters as 0, so with +# the plans failing it passed as 0 + 0 = 0, and two aggregate parities compared two empty +# readings: with the server stopped before the plans main passed 3, now none. And nine +# "premise:" checks read pgcolumnar's own behaviour, so they lost the prefix (#1339): +# nofold reddens the three "reaches the fold" checks, decodeall the fold arm's decode +# count, noskip "the narrow range skips vectors"; and (jdatcmd's #1700 review) noscan, +# the custom scan off by default, reddens the two plan-shape checks, "no whole group is +# pruned" and "the wide range skips none", while optdiv100, a writer that divides the +# table's stripe_row_limit by 100, reddens "one row group". 18 are dated by seven +# mutations, each the same names in both twins; noscan also dates "EXPLAIN reports +# Columnar Vectors Decoded", "and it is not stuck at zero" and the control. The poison +# check is dated by holeread, whose hole is zero-filled so a fold that reads it counts +# rows that are not there on a non-assert build too. 10 arrive never: the heap-side +# nulls premise and nine parity checks with heap. +# +# ledger rows 7309 + 28 = 7337 +# never 4259 + 10 = 4269 +# dated 3050 + 18 = 3068 +# 4269 + 3068 == 7337 +# +# #1341, native_parquet_fieldid seeded, with its twin (OffgridwithJD): 4269 -> 4271, +# and suites_not_covered 56 -> 55. The suite had no rows. It enters with 11 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after two fixes. +# The refusal arms read only the SQLSTATE, and four refusals share 22023: with the 'no field +# ids' guard removed (noanyid) an id-less file fell to the absent-id refusal and main passed +# 11/11. Each arm now wants its own guard's message. And survival was a bare SELECT 1: with +# the absent-id refusal turned into abort() it passed after crash recovery, and the crash took +# five later refusal arms with it. Survival now reads the server log's count of killed +# backends, and every reading waits for the server first, so a crash fails only its own arm +# and survival. 9 are dated by five mutations, each the same names in both twins. 2 +# arrive never. +# +# ledger rows 7337 + 11 = 7348 +# never 4269 + 2 = 4271 +# dated 3068 + 9 = 3077 +# 4271 + 3077 == 7348 +# +# #1341, native_parquet_codecs seeded, with its twin (OffgridwithJD): 4271 -> 4278, +# and suites_not_covered 55 -> 54. The suite had no rows. It enters with 14 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after three fixes. +# Every codec was compared with c_none.parquet read back through read_parquet, the reader +# under test: with one decoded byte flipped in every page over 4 KB, the same way for +# every codec, main passed all nine checks. Each file now decodes to the generator's rows, +# built in SQL. Each file's codec is read from its own metadata (a crafter that wrote every +# file uncompressed passed main 9/9). And survival reads the server log's crash count: with +# the crafted-size guard aborting and the probe 3 s late, main's bare SELECT 1 passed. 7 +# are dated by four mutations, each reddening the same names in both twins. 7 arrive +# never: the four codec premises read pyarrow's metadata, not pgcolumnar, and the FDW counts +# hold under each mutation. +# +# ledger rows 7348 + 14 = 7362 +# never 4271 + 7 = 4278 +# dated 3077 + 7 = 3084 +# 4278 + 3084 == 7362 +# +# #1341, native_parquet_units seeded, with its twin (OffgridwithJD): 4278 -> 4285, +# and suites_not_covered 54 -> 53. The suite had no rows. It enters with 20 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after two fixes. +# The pushdown oracle was the two-group millis file read back through read_parquet, the +# reader under test: with every row after a page's first taking the next dictionary +# entry, the scan and the mirror shifted alike and main passed 19/19. The oracle is now +# the generator's hourly series, and a new check compares the whole file with it. The two +# nanos truncation checks compared the file with a second file read the same way, so +# with the server stopped both passed on two empty reads; they now want the toward-zero +# microseconds as literals. 13 are dated by four mutations, each reddening the same +# names in both twins. 7 arrive never. +# +# ledger rows 7362 + 20 = 7382 +# never 4278 + 7 = 4285 +# dated 3084 + 13 = 3097 +# 4285 + 3097 == 7382 +# +# #1341, native_parquet_projection seeded, with its twin (OffgridwithJD): 4285 -> 4287, +# and suites_not_covered 53 -> 52. The suite had no rows. It enters with 17 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after two fixes. +# The oracle h was read_parquet of the same file, and read_parquet shares the scan's +# decoders: with a text decoder dropping a byte, or an INT64 decoder off by one, main passed +# 15/15. h is now built in SQL from the crafter's generator. And no table had a dropped +# column, so a projection matching the needed set by target index rather than attribute +# number passed 15/15; ftd drops a column ahead of the others and must still read exactly +# the column asked for. 15 are dated by five mutations, each reddening the same names in +# both twins. 2 arrive never: projection [a] and the unselected-qual hash, which no +# mutation reached. +# +# ledger rows 7382 + 17 = 7399 +# never 4285 + 2 = 4287 +# dated 3097 + 15 = 3112 +# 4287 + 3112 == 7399 +# +# #1341, native_parquet_hardening seeded, with its twin (OffgridwithJD): 4287 -> 4289, +# and suites_not_covered 52 -> 51. The suite had no rows. It enters with 18 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after four fixes, +# each measured on main's shell on PG17. The short-chunk verdict matched any 'has a malformed +# row group', which five guards raise: with the chunk-count guard removed a later guard +# rejected the file and main passed 18/18; it now wants this guard's detail. The 'still +# reads' arms counted rows: a date, time or timestamp decoded one unit off passed 18/18; they +# now compare the decoded values. The no-wrap arm counted -25536 in the in-range file, which +# never holds 40000, so it could not fail; it now reads the out-of-range file, which must +# refuse. And survival was a bare SELECT 1: with the chunk refusal turned into abort() and the +# probe 5 s late it passed; it now reads the server log's count of killed backends, and every +# reading waits for the server. The range refusals also want their own type's message, since +# four share 22008. 16 are dated by eleven mutations, each the same names in both twins. 2 +# arrive never. +# +# ledger rows 7399 + 18 = 7417 +# never 4287 + 2 = 4289 +# dated 3112 + 16 = 3128 +# 4289 + 3128 == 7417 +# +# #1341, native_parquet_stack seeded, with its twin (OffgridwithJD): 4289 -> 4289, +# and suites_not_covered 51 -> 50. The suite had no rows. It enters with 8 +# checks from five-major runs (all green on each of 15-19, shell and twin alike). The footer +# parser has two independent unbounded recursions, PgColumnarThriftSkip and walk_schema, +# each guarded by check_stack_depth(); the fixture pins max_stack_depth so a ~20 KB crafted +# footer trips the guard without overflowing the real C stack, so removing a guard fails the +# check by a wrong SQLSTATE rather than by crashing. The two refusal checks pin 54001, not +# any error, and read_parquet is held to the same refusal. 4 are dated: the nested-struct +# and read_parquet refusals by removing the thrift guard, the schema-chain refusal by removing +# the walk guard, and the valid-file control by a guard that trips on any nesting. The four +# survival checks were a bare SELECT 1, which answers again once crash recovery is over, and +# the twin's read_parquet survival test never read the file (jdatcmd's #1708 review). Each +# arm now takes a crash baseline right before its trigger, every reading waits for the +# server, and survival wants SELECT 1 and an unchanged count of "terminated by signal" in the +# server log, so a crash fails only its own arm. An abort() at each input's decode site dates +# them: thriftabort (PgColumnarThriftSkip aborts where its guard would raise) the +# nested-struct and read_parquet survivals, walkabort (the same in walk_schema) the +# schema-chain survival, validabort (walk_schema aborts on entry) the valid-file survival. +# All 8 are dated; none arrives never. +# +# ledger rows 7417 + 8 = 7425 +# never 4289 + 0 = 4289 +# dated 3128 + 8 = 3136 +# 4289 + 3136 == 7425 +# +# #1341, native_parquet_streaming seeded, with its twin (OffgridwithJD): 4289 -> 4304, +# and suites_not_covered 50 -> 49. The suite had no rows. It enters with 20 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after one fix measured +# on main's shell on PG17: survival was a bare SELECT 1, so with the second-dictionary refusal +# turned into abort() 'backend survived the crafted page areas' passed, and the crash also +# failed the v2 arm read during recovery. Survival now reads the server log's count of killed +# backends since its section began, and every reading waits for the server first. As hardening, +# the two refusals that accepted any ERROR now want the page loop's own decode refusal; no +# mutant here separated them. The v2 level guard stays unisolated, as the suite's header +# argues. 5 are dated by five mutations, each the same names in both twins. 15 arrive +# never. +# +# ledger rows 7425 + 20 = 7445 +# never 4289 + 15 = 4304 +# dated 3136 + 5 = 3141 +# 4304 + 3141 == 7445 +# +# #1341, native_parquet_partition seeded, with its twin (OffgridwithJD): 4304 -> 4310, +# and suites_not_covered 49 -> 48. The suite had no rows. It enters with 35 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after three fixes. +# The pruning reader defaulted to 0 on any failure, so with the server stopped the three +# 'prunes nothing' checks passed on no reading; 'Files Pruned' prints only above 0, so a +# missing line now counts as 0 only beside the plan's Files line. The refusals took any +# error: with the unknown-column guard removed (nocolcheck) a later 42804 stood in, and with +# the explicit encoded-NUL guard removed (nonul) the encoding check's 22021 did, main 35/35 +# each; every refusal now wants its own message. And survival was a bare SELECT 1, which +# an abort() in the unknown-column refusal passed with the probe 5 s late; it now reads the +# server log's crash count. 29 are dated by six mutations, each the same names in both +# twins. 6 arrive never. +# +# ledger rows 7445 + 35 = 7480 +# never 4304 + 6 = 4310 +# dated 3141 + 29 = 3170 +# 4310 + 3170 == 7480 +# +# #1341, native_parquet_fdw seeded, with its twin (OffgridwithJD): 4310 -> 4320, +# and suites_not_covered 48 -> 47. The suite had no rows. It enters with 20 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after four fixes. +# Projection and row-group skipping were never read: every arm compared answers, which the +# executor's recheck and a discarding projection keep right, so with every column decoded +# (noproj) or no group skipped (noskip) main passed 14/14. Two checks now read the scan's +# EXPLAIN ANALYZE counters, over a 200,000-row file of four row groups. The refusal arms +# accepted any psql failure: with the scan's missing-path guard removed (nopathguard) the +# backend crashed and main passed, and with the short-file guard removed (nolen) the +# bad-magic guard refused instead and main passed. Each arm now wants its own guard's +# SQLSTATE and message, and a survival check reads the server log's count of killed +# backends. The oracle is a heap table written from the generator, and the self-join counts +# are check_num, since two failed reads agreed. The skipped scan's row count read a plain +# int4 constant, a cross-type comparison that skips nothing (#1712), so it came from a full +# scan (jdatcmd's #1711 review); it now runs the plan's own int8 query, and skipshift (each +# group's skip decision moved to the next group, still 3 skipped) dates it. 10 are dated by +# eight mutations, each the same names in both twins. 10 arrive never. +# +# ledger rows 7480 + 20 = 7500 +# never 4310 + 10 = 4320 +# dated 3170 + 10 = 3180 +# 4320 + 3180 == 7500 +# +# #1341, native_read_parquet seeded, with its twin (OffgridwithJD): 4320 -> 4324, +# and suites_not_covered 47 -> 46. The suite had no rows. It enters with 15 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after three fixes. +# The round trips compared read_parquet with the source table, which the columnar scan +# reads both to write the file and as the oracle: with the columnar reader returning +# int4 777 as 778 (columnar_reader.c, store_att_byval), main passed 12/12. Every round trip now hashes the generator's own rows. The refusals +# counted any psql failure: with the scalar physical-type guard removed, main passed +# 12/12; each refusal now requires its SQLSTATE and its own message, and a survival check +# reads the crash count. read_parquet's own pg_read_server_files gate was never tested: +# with it removed, main passed 12/12; a role holding USAGE but not the role is now refused +# by it, beside a member that reads every row. 11 are dated by five mutations, each +# reddening the same names in both twins. 4 arrive never, among them "premise: reject +# missing column definition list", which reads PostgreSQL's parser (42601), not pgcolumnar. +# +# ledger rows 7500 + 15 = 7515 +# never 4320 + 4 = 4324 +# dated 3180 + 11 = 3191 +# 4324 + 3191 == 7515 +# +# #1341, objstore_listing seeded, with its twin (OffgridwithJD): 4324 -> 4328, +# and suites_not_covered 46 -> 45. The suite had no rows. It enters with 15 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after three fixes. +# The two non-progress refusals share 08P01 and read only the SQLSTATE: the no-keys page +# replays its token, so with the progress check removed (noprogress) the token check refused +# it on the next page and main passed 12/12. Each refusal now wants its own guard's message. +# No check made a LIST fail, so a non-2xx read as an empty prefix (non200empty) passed 12/12; +# the fixture server gained a __listing_status__ hook and a check wants the HTTP status +# refusal (08006). The exact-key 'no LIST' count read a log nothing proved alive; a control +# now wants the key's own GET in it. A survival check reads the server log's count of killed +# backends, and every refusal waits for the server first. 11 are dated by seven mutations, +# each the same names in both twins. 4 arrive never. +# +# ledger rows 7515 + 15 = 7530 +# never 4324 + 4 = 4328 +# dated 3191 + 11 = 3202 +# 4328 + 3202 == 7530 +# +# #1341, native_rewrite_conc seeded, with its twin (OffgridwithJD): 4328 -> 4330, +# and suites_not_covered 45 -> 44. The suite had no rows. It enters with 11 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after two fixes. +# Nothing tested that the online rewrite does not block: with compact_rewrite under +# AccessExclusiveLock (ael) the workers simply waited and main passed 5/5. A compact_rewrite +# now holds its lock mid-verb (pgcolumnar.maintenance_hold_ms) while a reader and a writer, +# each with a 3 s lock_timeout, must finish. And the race was assumed, not shown: a new check +# requires the compactor to have moved rows while the workers wrote. 9 are dated by four +# mutations, each the same names in both twins (noconflict on 3 of 3 runs). 2 arrive never. +# +# ledger rows 7530 + 11 = 7541 +# never 4328 + 2 = 4330 +# dated 3202 + 9 = 3211 +# 4330 + 3211 == 7541 +# +# #1341, objstore_addressing seeded, with its twin (OffgridwithJD): 4330 -> 4335, +# and suites_not_covered 44 -> 43. The suite had no rows. It enters with 16 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after two fixes. +# The gs:// no-endpoint check passed whenever the message did not name AWS_ENDPOINT_URL, so a +# mistyped default host (gswrongdef) passed 12/12; it now requires the allow-list refusal naming +# storage.googleapis.com:443. No key needed URI-encoding, so a space sent as '+' (plusspace) or +# '+'/'=' left bare (plusraw) passed 12/12; a key "dir sp/a+b=c.parquet" is now read under both +# addressings and its request path checked in the fixture's log. 11 are dated by seven +# mutations, each the same names in both twins. 5 arrive never. Neither pgc_skip name +# (pyarrow, the virtual-host name) gets a row: CI emits neither. +# +# ledger rows 7541 + 16 = 7557 +# never 4330 + 5 = 4335 +# dated 3211 + 11 = 3222 +# 4335 + 3222 == 7557 +# +# #1341, objstore_stash_recovery seeded, with its twin (OffgridwithJD): 4335 -> 4344, +# and suites_not_covered 43 -> 42. The suite had no rows. It enters with 18 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after three fixes. +# "and it did not stop at the start guard" counted 'interrupted mid-move', the refusal's +# text before #393's fix, which nothing prints any more: with the guard turned back into a +# refusal, eight checks went red and it stayed green. The debris was a byte-identical copy, +# so a guard that restored every stash over the installed module passed 17/17; the debris +# is now a different valid module and the installed one is compared with the pristine copy. +# Both restore slices parked the module at .away, so a guard that walked .away alone passed +# 17/17; the only-copy slice now parks it at .probe. Each slice also starts from the +# pristine copy, so one slice's damage no longer reddens the next one's premises. +# 9 are dated by four mutations of objstore_module.sh's guard, each reddening the same +# names in both twins. 9 arrive never. +# +# ledger rows 7557 + 18 = 7575 +# never 4335 + 9 = 4344 +# dated 3222 + 9 = 3231 +# 4344 + 3231 == 7575 +# +# #1341, objstore_s3_read seeded, with its twin (OffgridwithJD): 4344 -> 4348, +# and suites_not_covered 42 -> 41. The suite had no rows. It enters with 17 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after these fixes. +# The no-environment and 403 refusals read a bare 28000, which five guards share: with s3:// +# falling back to a default host (noepreq) the region guard refused instead and main passed +# 12/12. Each refusal now names its cause. Region handling was never tested: dropping the +# AWS_DEFAULT_REGION fallback, swapping its precedence, signing us-east-1 when no region is set, +# or signing with an empty secret each passed 12/12, and four checks now pin them. Nothing +# read the crash count, and an abort() in abort cleanup passed every SQLSTATE check (main's +# shell exited FATAL on the restart with no red): survival now reads the server log. The https +# check's name interpolated the build's want, so the grader could not read it. 13 are dated +# by twelve mutations, each the same names in both twins. 4 arrive never. Neither the +# pyarrow pgc_skip name nor the optional real-S3 check gets a row: CI emits neither. +# +# ledger rows 7575 + 17 = 7592 +# never 4344 + 4 = 4348 +# dated 3231 + 13 = 3244 +# 4348 + 3244 == 7592 +# +# #1341, objstore_tls_read seeded, with its twin (OffgridwithJD): 4348 -> 4355, +# and suites_not_covered 41 -> 40. The suite had no rows. It enters with 29 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after four fixes. +# A refusal was any 08006 plus a reason grepped anywhere in the output: with the module +# speaking cleartext to the TLS port (httpfall), main's three "refused with 08006" checks +# passed; each refusal now also wants the module's own TLS message naming the URL, and the +# whole DETAIL, OpenSSL's reason for that certificate. A self-signed leaf had no arm; it has +# one now. Trust from SSL_CERT_DIR was never read: with only SSL_CERT_FILE honoured +# (nocadir), main passed 19/19; arm E restarts with SSL_CERT_DIR alone and reads the good +# server. Nothing read the server log: with abort() in place of the verification refusal, +# the crash also reddened the next arm's premise and D, and "returned no rows" passed; each +# refusal now waits for SELECT 1, and a survival check counts "terminated by signal". +# 22 are dated by six mutations, each the same names in both twins. 7 arrive never. +# Neither pgc_skip name (pyarrow, openssl) gets a row: CI emits neither. +# +# ledger rows 7592 + 29 = 7621 +# never 4348 + 7 = 4355 +# dated 3244 + 22 = 3266 +# 4355 + 3266 == 7621 +# +# #1341, objstore_http_read seeded, with its twin (OffgridwithJD): 4355 -> 4363, +# and suites_not_covered 40 -> 39. The suite had no rows. It enters with 24 checks +# from five-major runs (all green on each of 15-19, shell and twin alike). Main's arm C took +# the SQLSTATE alone, so a dropped Range-ignored guard (noguard200) passed on 08P01 from +# "unexpected HTTP status 200"; each refusal now also needs its own guard's message. Short +# 206 bodies, EOF mid-body and the keep-alive retry had no fixture: noshort, eofok, noretry +# and alwaysretry each passed 18/18 on main. The fixture server gains /short/, /trunc/, +# /dropka/ and /hangup/ modes, inert unless a path names them, and a survival check reads +# the crash count after waiting for the server. 16 are dated by ten mutations, each the +# same names in both twins. 8 arrive never. The pgc_skip name (pyarrow) gets no row: +# CI does not emit it. +# +# ledger rows 7621 + 24 = 7645 +# never 4355 + 8 = 4363 +# dated 3266 + 16 = 3282 +# 4363 + 3282 == 7645 +# +# #1341, objstore_sink_write seeded, with its twin (OffgridwithJD): 4363 -> 4387, +# and suites_not_covered 39 -> 38. The suite had no rows. It enters with 35 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after five fixes. +# The fixture server completed an upload from every stored part, whatever the complete listed, +# so dropping the last part from the list (droplist) passed 30/30; it now assembles the listed +# parts, each checked against its ETag, as S3 does. No check made a part fail, so a refused +# part read as stored (partok) passed 30/30; a __part_status__ hook and four checks want the +# part check's own 58030, no key and no open upload. Abort was read from the DELETE count, so +# an abort naming the wrong upload (abortwrongid) passed 30/30; the server now keeps its open +# uploads and two checks want none. The Arrow arm read only export_arrow > 0 (green under +# shortput); the object is imported back. Every oracle is the generator, not the scanned +# table, and each refusal wants its own guard's message. 11 are dated by eight mutations, +# each the same names in both twins. 24 arrive never. +# +# ledger rows 7645 + 35 = 7680 +# never 4363 + 24 = 4387 +# dated 3282 + 11 = 3293 +# 4387 + 3293 == 7680 +# +# #1722, a cancelled remote parallel export aborts its workers' uploads (OffgridwithJD): +# 3907 -> 3907. One check joins objstore_sink_write, red first on all five majors in both +# twins: after the cancel the fixture server still held 4 open multipart uploads under cx/, +# because SIGTERM ends each worker in FATAL, which never unwinds to the writers' PG_CATCH +# and its sink abort. It is dated by dropping the new before_shmem_exit registration, which +# reddens it by name in both twins. +# +# ledger rows 7680 + 1 = 7681 +# never 4387 + 0 = 4387 +# dated 3293 + 1 = 3294 +# 4387 + 3294 == 7681 +# +# #1727's review (jdatcmd): two more checks. os_wait passed no WL_TIMEOUT, so +# WaitLatchOrSocket ignored its 5 s slice and a silent store never counted a quiet slice; +# under proc_exit interrupts are held, so a worker SIGTERM ended stayed in its abort until +# the store answered (all four alive at +92 s). "terminate: export workers exit within +# 75 s while the store is silent" stops the fixture mid-upload and times the workers' +# exit: red without WL_TIMEOUT (4 left after 91 s, both twins), 61 s with it; that +# pre-fix wait dates it. And the cancel arm now cancels only once all four uploads are +# registered on the server, "all four workers' uploads were open before the cancel": an +# upload whose POST ?uploads was in flight is #1743's window, and it flaked the +# open-upload check in CI. It is not a premise: a sink that never flushes a part (no +# export opens a multipart upload, one line) reddens it, which dates it. +# +# ledger rows 7681 + 2 = 7683 +# never 4387 + 0 = 4387 +# dated 3294 + 2 = 3296 +# 4387 + 3296 == 7683 +# +# #1341, native_fetch_interrupt seeded, with its twin (OffgridwithJD): 4387 -> 4387, +# and suites_not_covered 38 -> 37. The suite had no rows. It enters with 3 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after one fix: the +# checks read the function bodies with their comments in them, so the guard written as +# /* CHECK_FOR_INTERRUPTS(); */, or folded into the comment above it, passed 3/3 on main, and so +# did the lookup's catalog read commented out. Both bodies now have their comments blanked, +# newlines kept. All 3 are dated by three mutations, each the same names in both twins; none arrive never. +# +# ledger rows 7683 + 3 = 7686 +# never 4387 + 0 = 4387 +# dated 3296 + 3 = 3299 +# 4387 + 3299 == 7686 +# +# #1341, objstore_module seeded, with its twin (OffgridwithJD): 4387 -> 4409, +# and suites_not_covered 37 -> 36. The suite had no rows. It enters with 30 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after two fixes. +# The remote-read checks took any of seven phrases, "is not supported" among them: with the +# module no longer handling s3://, gs:// or (built with OpenSSL) https://, main passed 30/30. +# Each URL's refusal is now pinned by SQLSTATE and message, the https one by whether the +# module links libssl. "a local path still works" counted a columnar table, which never +# reaches read_parquet's dispatch: with every local read_parquet refused, main passed 30/30. +# It now reads back a file export_parquet wrote. 8 are dated by seven mutations, each the +# same names in both twins. 22 arrive never. The unmovable-module skip gets no row: CI +# emits none. Nor does the start guard's failure, now under a literal name the twin +# reads every run (one extra test): green runs never record it. +# +# ledger rows 7686 + 30 = 7716 +# never 4387 + 22 = 4409 +# dated 3299 + 8 = 3307 +# 4409 + 3307 == 7716 +# +# #1341, phase2 seeded, with its twin (OffgridwithJD): 4409 -> 4434, +# and suites_not_covered 36 -> 35. The suite had no rows. It enters with 47 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after two fixes. +# The codec loop round-tripped each codec and never read which one was stored, since the +# native conversion dropped the 2.2 catalog's check and left $code computed for nothing: +# with the session's setting ignored, or no block codec ever applied, all 42 passed. Each +# codec now reads its block_codec (four checks). The filter checks held with the zone +# maps ruling out nothing, because exact selection returns the same rows; a range inside +# the last vector must now decode one of the three (one check). +# 22 are dated by seven mutations of the write path, the compression fallback, the zone +# maps and the drop cleanup, each reddening the same names in both twins. 25 arrive never. +# +# ledger rows 7716 + 47 = 7763 +# never 4409 + 25 = 4434 +# dated 3307 + 22 = 3329 +# 4434 + 3329 == 7763 +# +# #1341, phase3 seeded, with its twin (OffgridwithJD): 4434 -> 4465, +# and suites_not_covered 35 -> 34. The suite had no rows. It enters with 37 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after three fixes. +# Its savepoint arm rolled back only completed statements, whose buffers executor end had +# already flushed under the subtransaction, so the abort's xmin verdict hid them (#1441): +# with the savepoint's discard of pending writes or pending deletes removed, all 32 passed. +# An INSERT and a DELETE that fail partway inside a savepoint must now leave nothing, in +# the transaction and after it (three checks). The delete arm said its range straddled a +# vector boundary and nothing read one: with 16384-row vectors all 32 passed; the zone +# maps' maxima are now read (one check). And a DELETE whose RETURNING reads the table in +# a block that commits into one that aborts must see the rows deleted so far, as heap +# does: with the commit not handing the read's written marks to the parent, no later +# read wrote them again, and the reads summed 19999 against heap's 14950 (one check). +# 6 are dated by six mutations of the subtransaction callback, the vector size and the +# drop cleanup, each reddening the same names in both twins. 31 arrive never. +# +# ledger rows 7763 + 37 = 7800 +# never 4434 + 31 = 4465 +# dated 3329 + 6 = 3335 +# 4465 + 3335 == 7800 +checks_never_observed_red 4465 diff --git a/test/craft_parquet_codecs.py b/test/craft_parquet_codecs.py new file mode 100644 index 000000000..724419473 --- /dev/null +++ b/test/craft_parquet_codecs.py @@ -0,0 +1,56 @@ +"""Write the native_parquet_codecs fixtures into DIR (argv[1]), for both twins. + +c_none / c_gzip / c_zstd / c_lz4.parquet hold the same 20000 rows, each written with one +codec; codecs.txt records the compression pyarrow actually stored for each file's first +column (one " " line), so a test can prove which codec it read. +badhdr.parquet is a gzip page whose header declares uncompressed_page_size -1. + +The rows are the generator both twins also compute in SQL: + id = g, grp = g % 50, val = g * 1.5, s = 'row-' || (g % 100), g in 0 .. 19999 +Written fresh for pgColumnar. +""" +import sys + +import pyarrow as pa +import pyarrow.parquet as pq + +W = sys.argv[1] +N = 20000 +tbl = pa.table({ + "id": pa.array(range(N), pa.int32()), + "grp": pa.array([i % 50 for i in range(N)], pa.int32()), + "val": pa.array([i * 1.5 for i in range(N)], pa.float64()), + "s": pa.array([f"row-{i % 100}" for i in range(N)]), +}) +with open(f"{W}/codecs.txt", "w") as out: + for codec in ["none", "gzip", "zstd", "lz4"]: + pq.write_table(tbl, f"{W}/c_{codec}.parquet", compression=codec) + c = pq.ParquetFile(f"{W}/c_{codec}.parquet").metadata.row_group(0).column(0).compression + out.write(f"{codec} {c}\n") + +# A gzip page whose header's uncompressed_page_size (field 2) is rewritten to zigzag(-1). +pq.write_table(pa.table({"a": pa.array([1, 2, 3], pa.int32())}), + f"{W}/badhdr.parquet", compression="gzip", use_dictionary=False) +raw = bytearray(open(f"{W}/badhdr.parquet", "rb").read()) +pos = 4 # the first PageHeader follows the "PAR1" magic +last = 0 +patched = False +for _ in range(6): + b = raw[pos] + pos += 1 + t, d = b & 0x0F, b >> 4 + if t == 0: + break + fid = last + d if d else None + last = fid + start = pos + while raw[pos] & 0x80: + pos += 1 + pos += 1 + if fid == 2: + assert pos - start == 1, (start, pos) + raw[start] = 0x01 # zigzag(-1) + patched = True + break +assert patched, "did not find uncompressed_page_size" +open(f"{W}/badhdr.parquet", "wb").write(bytes(raw)) diff --git a/test/craft_parquet_hardening.py b/test/craft_parquet_hardening.py new file mode 100644 index 000000000..d0570f8d7 --- /dev/null +++ b/test/craft_parquet_hardening.py @@ -0,0 +1,145 @@ +"""craft_parquet_hardening.py DIR: write the Parquet files test/native_parquet_hardening.sh +and its pytest twin read: the narrowing binds (out of range and in range) and a +two-column file whose row group carries one column chunk. One writer for both +twins, so neither can drift from the other. Needs pyarrow.""" +import sys + +import pyarrow as pa +import pyarrow.parquet as pq + +W = sys.argv[1] + +# ---- narrowing: out-of-range values must raise, not wrap -------------------- +# int2 bound over INT32: 40000 does not fit int16 (would wrap to -25536) +pq.write_table(pa.table({"c": pa.array([30000, 40000], pa.int32())}), + f"{W}/i2_bad.parquet") +# same bind, all values in range: must keep working +pq.write_table(pa.table({"c": pa.array([-30000, 0, 30000], pa.int32())}), + f"{W}/i2_ok.parquet") +# time bound over INT64 micros: beyond one day is not a time +pq.write_table(pa.table({"c": pa.array([0, 86400000000 + 1], pa.int64())}), + f"{W}/time_bad.parquet") +pq.write_table(pa.table({"c": pa.array([0, 3600000000], pa.int64())}), + f"{W}/time_ok.parquet") +# date bound over INT32 days since epoch. INT32 max does NOT overflow the epoch +# offset (2147483647 - 10957 stays in range); it is rejected by IS_VALID_DATE. +pq.write_table(pa.table({"c": pa.array([0, 2147483647], pa.int32())}), + f"{W}/date_bad.parquet") +# INT32 min is what overflows the subtraction itself, exercising the other branch +pq.write_table(pa.table({"c": pa.array([0, -2147483648], pa.int32())}), + f"{W}/date_ovf.parquet") +pq.write_table(pa.table({"c": pa.array([0, 19000], pa.int32())}), + f"{W}/date_ok.parquet") +# timestamp bound over INT64 micros: INT64 min overflows the epoch-offset +# subtraction. INT64 max does NOT -- it still lands inside PostgreSQL's timestamp +# range (year ~294276), so it is deliberately not the case tested here. +pq.write_table(pa.table({"c": pa.array([0, -9223372036854775808], pa.int64())}), + f"{W}/ts_bad.parquet") +pq.write_table(pa.table({"c": pa.array([0, 1700000000000000], pa.int64())}), + f"{W}/ts_ok.parquet") + +# ---- malformed row-group chunk list ---------------------------------------- +pq.write_table(pa.table({"a": pa.array([1, 2, 3], pa.int32()), + "b": pa.array([4, 5, 6], pa.int32())}), + f"{W}/two.parquet") + +# Walk the Thrift-compact footer to the exact byte holding the row group's +# `columns` list header, then claim it holds one fewer chunk than it does. +# Blind byte-searching is not reliable here: the same byte value occurs in the +# data pages, so the header has to be located structurally. +raw = bytearray(open(f"{W}/two.parquet", "rb").read()) +flen = int.from_bytes(raw[-8:-4], "little") +base = len(raw) - 8 - flen # start of FileMetaData +pos = base + +def varint(): + global pos + shift = 0; out = 0 + while True: + b = raw[pos]; pos += 1 + out |= (b & 0x7F) << shift + if not b & 0x80: + return out + shift += 7 + +def zigzag(): + u = varint() + return (u >> 1) ^ -(u & 1) + +def list_hdr(): + global pos + b = raw[pos]; pos += 1 + size = (b >> 4) & 0x0F + et = b & 0x0F + if size == 0x0F: + size = varint() + return size, et + +def skip(t): + global pos + if t in (1, 2): return # bool: value is in the header + if t == 3: pos += 1 # byte + elif t in (4, 5, 6): zigzag() # i16/i32/i64 + elif t == 7: pos += 8 # double + elif t == 8: pos += varint() # binary + elif t in (9, 10): + n, et = list_hdr() + for _ in range(n): skip(et) + elif t == 12: + last = 0 + while True: + ft, _fid, last = field(last) + if ft == 0: break + skip(ft) + +def field(last): + global pos + b = raw[pos]; pos += 1 + if b == 0: + return 0, 0, last + t = b & 0x0F + delta = b >> 4 + fid = last + delta if delta else zigzag() + return t, fid, fid + +# FileMetaData -> field 4 (row_groups) -> first RowGroup -> field 1 (columns) +target = None +last = 0 +while True: + ft, fid, last = field(last) + if ft == 0: + break + if fid == 4 and ft == 9: # row_groups: list + nrg, _et = list_hdr() + assert nrg >= 1, nrg + rlast = 0 + while True: + rft, rfid, rlast = field(rlast) + if rft == 0: + break + if rfid == 1 and rft == 9: # columns: list + target = pos # byte holding this list's header + n, et = list_hdr() + assert n == 2, f"expected 2 column chunks, got {n}" + skip(et) # past ColumnChunk[0] + start2 = pos + skip(et) # past ColumnChunk[1] + end2 = pos + break + skip(rft) + break + skip(ft) + +assert target is not None, "row group columns list not found" +# Remove ColumnChunk[1] outright and say so in the header, so the footer stays a +# well-formed Thrift struct. Decrementing the count alone would desync the parser +# and fail for an unrelated reason; the only defect must be that the row group +# carries fewer chunks than the schema has leaves. +assert (raw[target] >> 4) == 2, hex(raw[target]) +raw[target] = (1 << 4) | (raw[target] & 0x0F) +removed = end2 - start2 +del raw[start2:end2] +raw[-8:-4] = (flen - removed).to_bytes(4, "little") +open(f"{W}/short_chunks.parquet", "wb").write(bytes(raw)) +print(f" columns list at {target}: 2 chunks -> 1, removed {removed} footer bytes", + file=sys.stderr) diff --git a/test/craft_parquet_projection.py b/test/craft_parquet_projection.py new file mode 100644 index 000000000..ebc780474 --- /dev/null +++ b/test/craft_parquet_projection.py @@ -0,0 +1,39 @@ +"""Write the files test/native_parquet_projection.sh and its pytest twin read. + + python3 test/craft_parquet_projection.py DIR + +DIR/proj.parquet a, b, c, d over 5000 rows in row groups of 1000, from the same + generator the suites rebuild in SQL as their oracle: + a = i, b = 2i, c = 's' || (i % 20), d = i / 3.0 +DIR/corrupt.parquet g, bad over 1000 rows, with the second column's data page + smashed, so decoding `bad` fails while `g` and the footer stay + intact. + +Both harnesses run this one crafter, so a file can never differ between them. +""" +import sys + +import pyarrow as pa +import pyarrow.parquet as pq + +W = sys.argv[1] +n = 5000 +pq.write_table(pa.table({ + "a": pa.array(range(n), pa.int32()), + "b": pa.array([i * 2 for i in range(n)], pa.int64()), + "c": pa.array([f"s{i % 20}" for i in range(n)]), + "d": pa.array([i / 3.0 for i in range(n)], pa.float64()), +}), f"{W}/proj.parquet", row_group_size=1000) + +pq.write_table(pa.table({"g": pa.array(range(1000), pa.int32()), + "bad": pa.array(range(1000), pa.int32())}), + f"{W}/corrupt.parquet", row_group_size=1000, use_dictionary=False) +md = pq.ParquetFile(f"{W}/corrupt.parquet").metadata.row_group(0).column(1) +off = md.data_page_offset +raw = bytearray(open(f"{W}/corrupt.parquet", "rb").read()) +# smash the second column's page header and the start of its data, leaving the +# first column and the footer intact +for i in range(off, off + 32): + raw[i] = 0xFF +open(f"{W}/corrupt.parquet", "wb").write(bytes(raw)) +print(f" corrupted column 'bad' at offset {off}", file=sys.stderr) diff --git a/test/craft_parquet_streaming.py b/test/craft_parquet_streaming.py new file mode 100644 index 000000000..dc1267651 --- /dev/null +++ b/test/craft_parquet_streaming.py @@ -0,0 +1,166 @@ +"""Write every Parquet file native_parquet_streaming.sh and its twin read, into DIR. + +Usage: craft_parquet_streaming.py DIR + +Exit status, as the shell reads it: + 0 every file was written + 3 no room even for the oversized sparse file's metadata + 4 the sparse file did not reach its logical size (no sparse files here) + 5 the footer offset to patch is ambiguous; refusing to guess + 6 a crafted file is missing after writing + 7 the source file has no dictionary page to duplicate + +3 and 4 mean the box cannot build the oversized file, which the shell records as a +named skip; any other failure means the crafted files are not all there, which it +records as a failed check rather than letting each refusal pass on "could not open +file". The construction of each file, and why, is described beside it below. +""" +import os +import sys + +import pyarrow as pa +import pyarrow.parquet as pq + +W = sys.argv[1] + + +def parts(path): + raw = open(path, "rb").read() + n = int.from_bytes(raw[-8:-4], "little") + return raw[-8 - n:-8], n, raw[4:-8 - n] # footer, its length, page area + + +# ---- a real small file, and an oversized sparse one carrying its footer ------------ +src = os.path.join(W, "small.parquet") +pq.write_table(pa.table({"id": pa.array([1, 2, 3], pa.int32()), + "v": pa.array(["a", "b", "c"])}), src) +data = open(src, "rb").read() +metalen = int.from_bytes(data[-8:-4], "little") +meta = data[-8 - metalen:-8] + +# comfortably over MaxAllocSize (1GB - 1), which is what palloc() rejected +SIZE = 1600 * 1024 * 1024 +st = os.statvfs(W) +if st.f_bavail * st.f_frsize < 64 * 1024 * 1024: + sys.exit(3) +big = os.path.join(W, "huge_sparse.parquet") +with open(big, "wb") as f: + f.write(b"PAR1") + f.truncate(SIZE) # the hole; never read by the footer path + f.seek(SIZE - 8 - metalen) + f.write(meta) + f.write(metalen.to_bytes(4, "little")) + f.write(b"PAR1") +if os.path.getsize(big) != SIZE: + sys.exit(4) + +# ---- crafted files: each keeps a valid footer and breaks one thing the page loop +# must not trust. The footer comes from a real uncompressed file, so a zeroed page +# area really parses as page headers rather than failing in the codec first. +plain = os.path.join(W, "plain.parquet") +pq.write_table(pa.table({"id": pa.array(list(range(200)), pa.int32())}), + plain, compression="none") +meta, metalen, body = parts(plain) + + +def build(name, body_bytes, m=None, ml=None): + with open(os.path.join(W, name), "wb") as f: + f.write(b"PAR1") + f.write(body_bytes) + f.write(m if m is not None else meta) + f.write((ml if ml is not None else metalen).to_bytes(4, "little")) + f.write(b"PAR1") + + +# 1. a large zero-filled page area of a REQUIRED column (no definition levels, so the +# zero-value page is reached), sparse: each zero byte parses as an empty v1 data +# page, so without the progress guard the loop walks it one byte at a time. +req = os.path.join(W, "plain_req.parquet") +pq.write_table(pa.table({"id": pa.array(list(range(200)), pa.int32())}, + schema=pa.schema([pa.field("id", pa.int32(), nullable=False)])), + req, compression="none") +rmeta, rmetalen, rbody = parts(req) +ZSIZE = 1600 * 1024 * 1024 +with open(os.path.join(W, "zero_pages.parquet"), "wb") as f: + f.write(b"PAR1") + f.truncate(ZSIZE - 8 - rmetalen) + f.seek(ZSIZE - 8 - rmetalen) + f.write(rmeta) + f.write(rmetalen.to_bytes(4, "little")) + f.write(b"PAR1") + + +# 2. a page header (built by hand, Thrift compact) declaring a compressed_size far +# past end of file, written over the start of the chunk the footer points at. +def zz(v): + u = (v << 1) ^ (v >> 63) if v < 0 else (v << 1) + out = bytearray() + while True: + b = u & 0x7F + u >>= 7 + out.append(b | (0x80 if u else 0)) + if not u: + return bytes(out) + + +def fh(delta, t): + return bytes([(delta << 4) | t]) + + +I32, STRUCT, STOP = 5, 12, b"\x00" +BIG = 500 * 1024 * 1024 # past end of file, but under MaxAllocSize +synth = (fh(1, I32) + zz(0) + fh(1, I32) + zz(64) + fh(1, I32) + zz(BIG) + + fh(2, STRUCT) + fh(1, I32) + zz(200) + fh(1, I32) + zz(0) + + fh(1, I32) + zz(3) + fh(1, I32) + zz(3) + STOP + STOP) +build("bad_size.parquet", synth + body[len(synth):]) + +# 3. the page area is noise, so the page header does not parse at all. +build("noise_pages.parquet", bytes((i * 37 + 11) & 0xFF for i in range(len(body)))) + +# 4. the footer points the SECOND column chunk's start offset to -2^20: negative, so +# it passes the loop's "pos < len" and reaches the header read's offset check. +two = os.path.join(W, "two.parquet") +pq.write_table(pa.table({"id": pa.array(list(range(200)), pa.int32()), + "v": pa.array(["r%04d" % i for i in range(200)])}), + two, compression="none", use_dictionary=False) +tmeta, tmetalen, tbody = parts(two) +tcol = pq.ParquetFile(two).metadata.row_group(0).column(1) +tstart = tcol.dictionary_page_offset or tcol.data_page_offset +pat = zz(tstart) +if tmeta.count(pat) != 1: + sys.exit(5) +patched = tmeta.replace(pat, zz(-(1 << 20))) +build("bad_offset.parquet", tbody, patched, len(patched)) + +# 5. two dictionary pages in one chunk (the format allows at most one). +dmd = pq.ParquetFile(plain).metadata.row_group(0).column(0) +if not dmd.dictionary_page_offset or dmd.data_page_offset <= dmd.dictionary_page_offset: + sys.exit(7) +dlen = dmd.data_page_offset - dmd.dictionary_page_offset +doff = dmd.dictionary_page_offset - 4 +build("two_dicts.parquet", body[:doff] + body[doff:doff + dlen] * 2 + body[doff + dlen:]) + +# 6. a DATA_PAGE_V2 whose declared level lengths exceed the page (see the shell for +# why this rejection does not isolate the v2 guard). +synth_v2 = (fh(1, I32) + zz(3) + fh(1, I32) + zz(64) + fh(1, I32) + zz(32) + + fh(5, STRUCT) + fh(1, I32) + zz(200) + fh(1, I32) + zz(0) + + fh(1, I32) + zz(200) + fh(1, I32) + zz(0) + fh(1, I32) + zz(64) + + fh(1, I32) + zz(0) + fh(1, 2) + STOP + STOP) +build("v2_levels.parquet", synth_v2 + body[len(synth_v2):]) + +# an all-null column: pyarrow writes it with an EMPTY dictionary page +pq.write_table(pa.table({"c": pa.array([None] * 100, pa.int32())}), + os.path.join(W, "allnull.parquet"), use_dictionary=True, compression="none") + +# #620: four row groups of 1000 rows, for the FDW's Row Groups Decoded counter +n = 4000 +pq.write_table(pa.table({"id": pa.array(range(n), pa.int32()), + "v": pa.array([i * 2 for i in range(n)], pa.int64())}), + os.path.join(W, "stream_groups.parquet"), row_group_size=1000) +if pq.ParquetFile(os.path.join(W, "stream_groups.parquet")).num_row_groups != 4: + sys.exit(6) + +for name in ("zero_pages", "bad_size", "noise_pages", "bad_offset", "two_dicts", + "v2_levels", "allnull", "stream_groups"): + if not os.path.exists(os.path.join(W, name + ".parquet")): + sys.exit(6) diff --git a/test/lib.sh b/test/lib.sh index 39c017da6..d1d0567c1 100755 --- a/test/lib.sh +++ b/test/lib.sh @@ -401,10 +401,24 @@ pgc_setup() { esac echo "-- initdb" - pgc_pg "initdb -D '$PGC_PGDATA' -A trust" >/dev/null 2>&1 + # -U postgres, because every psql below connects as postgres (#1341). initdb's + # default superuser is whoever runs it, which is postgres only under the + # runuser above; a non-root run -- the pytest-cluster job, which chowns + # pkglibdir precisely so that it can stay non-root, driving a shell suite as + # test_objstore_stash_recovery.py does -- got a cluster with no postgres role, + # and every start attempt "failed". pgc_cluster.initdb has always passed it. + pgc_pg "initdb -D '$PGC_PGDATA' -A trust -U postgres" >/dev/null 2>&1 { echo "port=$PGC_PORT" echo "listen_addresses='127.0.0.1'" + # THE SOCKET DIRECTORY, pinned to this cluster's own datadir, as + # pgc_cluster.py pins it. A packaged postgres defaults it to + # /var/run/postgresql, which only postgres can write: under runuser that is + # fine, but a non-root, non-postgres run -- the pytest-cluster job driving + # objstore_module.sh from test_objstore_stash_recovery.py -- FATALs on the + # lock file, and pg_ctl says only "could not start server" (#1717). Every + # connection here is TCP on 127.0.0.1, so nothing reads the socket's path. + echo "unix_socket_directories='$PGC_PGDATA'" echo "shared_preload_libraries='pgcolumnar'" # Deterministic text output so heap and columnar hashes match. echo "extra_float_digits=3" diff --git a/test/lossy_arms.tsv b/test/lossy_arms.tsv index 925a8e6fb..ac504acf2 100644 --- a/test/lossy_arms.tsv +++ b/test/lossy_arms.tsv @@ -133,8 +133,6 @@ native_rewrite group numbers issued after a rewrite are past every earlier one native_truncate file shrank native_truncate GUC on: file shrank native_upgrade_converge fresh $TARGET install has objects to compare -native_vecdecode a ruled-out vector is not decoded -native_zonemap_narrow zone_map reads scale with predicates, not table width (wide ~= narrow) objstore_listing the prefix listing paged (more than one list-type=2 request) objstore_module premise: the real module was restored afterwards objstore_sink_write parallel: every worker PUT went to the prefix diff --git a/test/native_fetch_interrupt.sh b/test/native_fetch_interrupt.sh index 613dbebf9..a68c16f7a 100755 --- a/test/native_fetch_interrupt.sh +++ b/test/native_fetch_interrupt.sh @@ -31,8 +31,16 @@ pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" SRC="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/src" -# The body of pgcolumnar_fetch_row, from its definition to its closing brace. -body="$(awk '/^pgcolumnar_fetch_row\(/{p=1} p{print} p&&/^}/{exit}' "$SRC/columnar_reader.c")" +# The body of a function in columnar_reader.c, from its definition to its closing +# brace, with its comments blanked, multi-line ones included: a guard commented out +# of the code is not a guard, and every check below passed with the call written +# as /* CHECK_FOR_INTERRUPTS(); */ (#1341). Each line keeps its place, so the +# placement check still compares line numbers of the code itself. +fn_body() { + awk -v fn="$1" '$0 ~ "^" fn "\\(" {p=1} p{print} p&&/^}/{exit}' "$SRC/columnar_reader.c" | + perl -0777 -pe 's{/\*.*?\*/}{ my $c = $&; $c =~ tr/\n//cd; $c }gse; s{//[^\n]*}{}g' +} +body="$(fn_body pgcolumnar_fetch_row)" # The guard has to be in this function, not merely in the file. The interrupt # checks elsewhere in columnar_reader.c are on the scan/decode path, which the @@ -50,14 +58,14 @@ check "pgcolumnar_fetch_row carries an interrupt check" \ # the catalog read moved, so this arm cannot rot into comparing against a # call that no longer does the work. cfi_line="$(printf '%s\n' "$body" | grep -n 'CHECK_FOR_INTERRUPTS' | head -1 | cut -d: -f1)" -# match the CALL, with its paren -- the bare name also appears in a comment -# above the call, and a comment must not satisfy a placement check +# match the CALL, with its paren; comments are already blanked, so a comment +# that names it cannot satisfy a placement check rgl_line="$(printf '%s\n' "$body" | grep -n 'pgcolumnar_lookup_row_group(' | head -1 | cut -d: -f1)" check "the interrupt check precedes the row-group resolution" \ "$( [ -n "$cfi_line" ] && [ -n "$rgl_line" ] && [ "$cfi_line" -lt "$rgl_line" ] && echo yes || echo no )" \ "yes" -lookup_body="$(awk '/^pgcolumnar_lookup_row_group\(/{p=1} p{print} p&&/^}/{exit}' "$SRC/columnar_reader.c")" +lookup_body="$(fn_body pgcolumnar_lookup_row_group)" check "the row-group resolution is where the catalog read lives" \ "$(printf '%s\n' "$lookup_body" | grep -c 'PgColumnarReadRowGroupList' | awk '{print ($1>=1)?"yes":"no"}')" \ "yes" diff --git a/test/native_fold_deferral.sh b/test/native_fold_deferral.sh index 1df9062fd..2d7b0810c 100755 --- a/test/native_fold_deferral.sh +++ b/test/native_fold_deferral.sh @@ -29,9 +29,23 @@ GUC="SET pgcolumnar.enable_ungrouped_vector_agg = on;" NP=4 # payload columns ROWS=120000 -sq() { # scalar under this suite's GUC; tail -1 drops the SET tag - env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ - -d "$PGC_DB" -At -c "$GUC $1" 2>/dev/null | tail -1 +# sq SQL -> the result under this suite's GUC, or nothing when the statement failed. +# It used to be `... | tail -1`, which read a failed statement as whatever line came +# last; with ON_ERROR_STOP and the exit status, a failure is empty, and every +# comparison below refuses an empty side (check_text) or a non-number (check_num). +sq() { + local out + out="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ + -d "$PGC_DB" -At -v ON_ERROR_STOP=1 -c "$GUC $1" 2>/dev/null)" || return 0 + tail -1 <<<"$out" +} +# rq SQL -> the result on the row path (the vectorized aggregate off), the same way. +rq() { + local out + out="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ + -d "$PGC_DB" -At -v ON_ERROR_STOP=1 \ + -c "SET pgcolumnar.enable_ungrouped_vector_agg = off; $1" 2>/dev/null)" || return 0 + tail -1 <<<"$out" } explain_line() { # explain_line