diff --git a/CHANGELOG.md b/CHANGELOG.md index d2b779a6d..c0dffb95f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -400,6 +400,18 @@ true until the next version shipped. Reported by @OffgridwithJD. ### Fixed +- **In the upgrade window, a refusal for a missing catalog table names `ALTER + EXTENSION UPDATE`.** After the new library is installed and the server restarted, + but before `ALTER EXTENSION pgcolumnar UPDATE`, `DROP TABLE`, `TRUNCATE` and + `pgcolumnar.vacuum()` on a columnar table fail, because the 1.0-alpha5 library + needs `pgcolumnar.retired_group` and a 1.0-alpha4 catalog lacks it. They change + nothing. The error used to name only the missing table, and it now carries a hint: + `The pgcolumnar catalog in this database may be older than the installed library. + Run ALTER EXTENSION pgcolumnar UPDATE in this database.` Reads and writes work + throughout, and after the update every one of these succeeds on the same rows + (#1341 item 66). `test/native_upgrade_converge.sh` and + `test_native_upgrade_converge.py`. + - **A join whose runtime-filter build is empty no longer reads the fact table.** When a large dimension filtered to no row was joined to a columnar table, the hash join diff --git a/docs/installation.md b/docs/installation.md index 2618c47bf..6cf867108 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -152,6 +152,18 @@ The fix is step 3. Run `ALTER EXTENSION pgcolumnar UPDATE;` in that database and the error goes away. **Your data is not affected.** The tables are intact and no conversion happens. Only the catalog entry is stale. +Between releases that add a catalog table, the new library can also need a table +the old catalog does not have. Upgrading from `1.0-alpha4` to `1.0-alpha5`, +`DROP TABLE`, `TRUNCATE` and `pgcolumnar.vacuum()` on a columnar table fail until +step 3, and change nothing; reads and writes keep working: + +``` +ERROR: columnar metadata table "pgcolumnar.retired_group" does not exist +HINT: The pgcolumnar catalog in this database may be older than the installed library. Run ALTER EXTENSION pgcolumnar UPDATE in this database. +``` + +The fix is the same: step 3. + Do not use `DROP EXTENSION` instead. A plain `DROP EXTENSION pgcolumnar` fails while columnar tables exist, because they depend on the access method. The form that succeeds is `DROP EXTENSION pgcolumnar CASCADE`, and it drops every diff --git a/src/columnar_metadata.c b/src/columnar_metadata.c index bd0120efb..eb5bdc53f 100644 --- a/src/columnar_metadata.c +++ b/src/columnar_metadata.c @@ -239,11 +239,22 @@ open_columnar_table(const char *name, LOCKMODE lockmode) nspOid = pgcolumnar_schema_oid(); relOid = get_relname_relid(name, nspOid); + /* + * NAME THE REMEDY (#1341 item 66). The usual way here is the upgrade window: + * the new library is installed and the server restarted (docs/installation.md + * steps 1 and 2), but this database's catalog is still the previous release's + * until ALTER EXTENSION UPDATE (step 3). A catalog table this library added, + * such as retired_group in 1.0-alpha5, is then missing, and DROP TABLE, + * TRUNCATE and pgcolumnar.vacuum() stop here. The error alone named only the + * table, which says neither why nor what to do. + */ if (!OidIsValid(relOid)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), errmsg("columnar metadata table \"%s.%s\" does not exist", - COLUMNAR_SCHEMA_NAME, name))); + COLUMNAR_SCHEMA_NAME, name), + errhint("The pgcolumnar catalog in this database may be older than the installed library. " + "Run ALTER EXTENSION pgcolumnar UPDATE in this database."))); rel = table_open(relOid, lockmode); diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 74fcd205f..c0267df50 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -804,6 +804,34 @@ fdw_plan_privilege fdw_plan_privilege and a metadata_path that does not exist pl fdw_plan_privilege fdw_plan_privilege premise: the Parquet file was written 15;16;17;18;19 never - fdw_plan_privilege fdw_plan_privilege premise: the reader is not a member of pg_read_server_files 15;16;17;18;19 never - fdw_plan_privilege fdw_plan_privilege the unprivileged Iceberg scan is still refused (42501) 15;16;17;18;19 never - +fk_referencing fk_referencing ALTER TABLE ADD CONSTRAINT is refused by the same rule 15;16;17;18;19 2026-10-16 columnar_tableam.c: a foreign key referencing a columnar table no longer refused;columnar_tableam.c: the foreign-key guard keyed on the referencing table, not the referenced one +fk_referencing fk_referencing a columnar partition of an FK-referenced parent is refused 15;16;17;18;19 2026-10-16 columnar_tableam.c: a foreign key referencing a columnar table no longer refused;columnar_tableam.c: the foreign-key guard keyed on the referencing table, not the referenced one +fk_referencing fk_referencing a columnar table may reference a heap table 15;16;17;18;19 2026-10-16 columnar_tableam.c: the foreign-key guard keyed on the referencing table, not the referenced one +fk_referencing fk_referencing a foreign key between two heap tables is unaffected 15;16;17;18;19 never - +fk_referencing fk_referencing a foreign key referencing a columnar table is refused 15;16;17;18;19 2026-10-16 columnar_tableam.c: a foreign key referencing a columnar table no longer refused;columnar_tableam.c: the foreign-key guard keyed on the referencing table, not the referenced one +fk_referencing fk_referencing a partitioned table with no foreign key still converts 17;18;19 never - +fk_referencing fk_referencing a row that violates it is still rejected 15;16;17;18;19 2026-10-16 columnar_tableam.c: the foreign-key guard keyed on the referencing table, not the referenced one +fk_referencing fk_referencing a table with no foreign key still converts 15;16;17;18;19 never - +fk_referencing fk_referencing an FK-referenced table may still be set to a row store 15;16;17;18;19 2026-10-16 columnar_tableam.c: converting an FK-referenced table refused whatever the target access method +fk_referencing fk_referencing an unrelated ALTER TABLE on a referenced table is unaffected 15;16;17;18;19 never - +fk_referencing fk_referencing and a partition can still be created the ordinary way 15;16;17;18;19 2026-10-16 columnar_tableam.c: converting an FK-referenced table to columnar no longer refused +fk_referencing fk_referencing and its partitions inherit the columnar access method 17;18;19 never - +fk_referencing fk_referencing and rows that satisfy it are accepted 15;16;17;18;19 2026-10-16 columnar_tableam.c: the foreign-key guard keyed on the referencing table, not the referenced one +fk_referencing fk_referencing and setting the parent's access method is refused as well 15;16;17;18;19 2026-10-16 columnar_tableam.c: converting an FK-referenced table to columnar no longer refused +fk_referencing fk_referencing and the error names the constraint 15;16;17;18;19 2026-10-16 columnar_tableam.c: converting an FK-referenced table to columnar no longer refused +fk_referencing fk_referencing and the error says why 15;16;17;18;19 2026-10-16 columnar_tableam.c: a foreign key referencing a columnar table no longer refused;columnar_tableam.c: the foreign-key guard keyed on the referencing table, not the referenced one +fk_referencing fk_referencing and they still work 15;16;17;18;19 never - +fk_referencing fk_referencing before 17, core refuses this for every partitioned table 15;16 never - +fk_referencing fk_referencing converting an FK-referenced table to columnar is refused 15;16;17;18;19 2026-10-16 columnar_tableam.c: converting an FK-referenced table to columnar no longer refused +fk_referencing fk_referencing converting back to heap is unaffected 15;16;17;18;19 never - +fk_referencing fk_referencing converting the referencing side is still allowed 15;16;17;18;19 never - +fk_referencing fk_referencing primary key, unique, check and not-null on a columnar table are unaffected 15;16;17;18;19 never - +fk_referencing fk_referencing so the parent did not become columnar 15;16;17;18;19 2026-10-16 columnar_tableam.c: converting an FK-referenced table to columnar no longer refused +fk_referencing fk_referencing the child is still writable afterwards 15;16;17;18;19 2026-10-16 columnar_tableam.c: converting an FK-referenced table to columnar no longer refused +fk_referencing fk_referencing the conversion helper is refused identically 15;16;17;18;19 2026-10-16 columnar_tableam.c: converting an FK-referenced table to columnar no longer refused +fk_referencing fk_referencing the parent is still a row store 15;16;17;18;19 2026-10-16 columnar_tableam.c: converting an FK-referenced table to columnar no longer refused +fk_referencing fk_referencing the primary key is still enforced 15;16;17;18;19 never - +fk_referencing fk_referencing the refused table was not left behind 15;16;17;18;19 2026-10-16 columnar_tableam.c: a foreign key referencing a columnar table no longer refused;columnar_tableam.c: the foreign-key guard keyed on the referencing table, not the referenced one harness_selftest 030-assertions nothing leaked into the squatter 15;16;17;18;19 never - harness_selftest 030-assertions pgc_port_free says the squatter's port is busy 15;16;17;18;19 never - harness_selftest 030-assertions squatter survived untouched 15;16;17;18;19 never - @@ -2280,6 +2308,70 @@ native_chunk_length_bound native_chunk_length_bound backend survived the sequent native_chunk_length_bound native_chunk_length_bound backend survived the truncated-length fetch 15;16;17;18;19 never - native_chunk_length_bound native_chunk_length_bound premise: a point lookup uses the index, not a sequential columnar scan 15;16;17;18;19 never - native_chunk_length_bound native_chunk_length_bound that fetch returns the row 15;16;17;18;19 2026-10-16 the index fetch treated a flushed row as not live, so the lookup returned no row (mutation, #1339) +native_decode_gate_width native_decode_gate_width control, gate forced on: a matches-everything narrow qual skips no vector (full decode) 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width control, gate forced on: a matches-everything wide qual skips no vector (full decode) 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width correctness: narrow needle matches the heap either way 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width correctness: the full scan matches the heap 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width correctness: wide needle, gate OFF, returns the identical rows (output-invariant) 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width correctness: wide needle, gate ON, matches the heap oracle 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width narrow, default: a zero-match LIKE decodes EVERY projected column of every vector -> NOT gated 15;16;17;18;19 2026-10-16 columnar_reader.c: phase-2 gating runs whatever the projection's payload width +native_decode_gate_width native_decode_gate_width narrow, gate off (GUC=0): the SAME narrow projection now gates -> qual column alone 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width premise: EXPLAIN reports Columnar Vector Decodes (the observable) 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width premise: back to the shipped default 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width premise: gate suppressed for the invariance arm 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width premise: no vector is ruled out by VALUE (that is 1b-ii's path, not phase 2) 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width premise: no whole row group is pruned 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width premise: one row group, so every count below is one group's 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width premise: still at the default before arm 1 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width premise: the LIKE is unprunable (no usable skip predicate) 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width premise: the control matches every row 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width premise: the narrow plan is a columnar custom scan with projection 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width premise: the narrow query takes the late-materialization path 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width premise: the needle matches exactly one vector's worth of rows 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width premise: the nomatch predicate matches no row 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width premise: the rejected probe GUC qual_skipvec_min_skip_pct was NOT shipped 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width premise: the rejected probe GUC qual_skipvec_probe_vecs was NOT shipped 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width premise: the wide plan is a columnar custom scan with projection 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width premise: the wide query takes the late-materialization path 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width premise: the width gate is now disabled (0) 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width premise: the width threshold is now above the projection width 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width premise: the width threshold now equals the projection width 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width premise: the width-gate GUC exists and defaults to the shipped 20 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width wide, default: a one-vector LIKE decodes the qual column plus payload for that vector only 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width wide, default: a zero-match LIKE decodes the qual column and NO payload -> gated 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width wide, default: and reports the payload vectors skipped 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width wide, default: skips exactly the vectors holding no match 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width wide, gate AT the width (>= boundary): gates again -> qual column alone 15;16;17;18;19 2026-10-16 columnar_reader.c: the width gate requires more payload columns than the threshold, not at least as many +native_decode_gate_width native_decode_gate_width wide, gate raised above width: the SAME wide SELECT * stops gating -> full decode 15;16;17;18;19 2026-10-16 columnar_reader.c: phase-2 gating runs whatever the projection's payload width;columnar_reader.c: the payload-width count includes one column too many +native_decode_gating native_decode_gating EXPLAIN reports Columnar Vector Decodes 15;16;17;18;19 never - +native_decode_gating native_decode_gating a one-vector LIKE decodes the qual column plus payload for that vector only 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluation between the decode passes never runs;columnar_reader.c: the per-vector qual evaluator marks every vector skipped +native_decode_gating native_decode_gating a zero-matching LIKE decodes the qual column and no payload column 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluation between the decode passes never runs +native_decode_gating native_decode_gating and reports them all filtered before materialization 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluator marks every vector skipped +native_decode_gating native_decode_gating and skips every vector 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluation between the decode passes never runs +native_decode_gating native_decode_gating and the full scan across every group's mask matches too 15;16;17;18;19 never - +native_decode_gating native_decode_gating deletes + multiple groups + unprunable qual: columnar matches the heap 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluator marks every vector skipped +native_decode_gating native_decode_gating premise: no vector is ruled out by VALUE either (that is 1b-ii's path) 15;16;17;18;19 never - +native_decode_gating native_decode_gating premise: no whole row group is pruned 15;16;17;18;19 never - +native_decode_gating native_decode_gating premise: one row group, so every count below is one group's 15;16;17;18;19 never - +native_decode_gating native_decode_gating premise: rows really were deleted (so the mask is non-NULL) 15;16;17;18;19 never - +native_decode_gating native_decode_gating premise: the columnar table spans more than one row group 15;16;17;18;19 never - +native_decode_gating native_decode_gating premise: the control matches every row 15;16;17;18;19 never - +native_decode_gating native_decode_gating premise: the half-needle survives in one vector, half of its rows 15;16;17;18;19 never - +native_decode_gating native_decode_gating premise: the needle plan is a columnar custom scan 15;16;17;18;19 never - +native_decode_gating native_decode_gating premise: the needle query takes the late-materialization path 15;16;17;18;19 never - +native_decode_gating native_decode_gating premise: the needle really matches one vector's worth of rows 15;16;17;18;19 never - +native_decode_gating native_decode_gating premise: the nomatch predicate matches no row 15;16;17;18;19 never - +native_decode_gating native_decode_gating premise: the reader has no usable skip predicate for this LIKE 15;16;17;18;19 never - +native_decode_gating native_decode_gating premise: the width gate is pinned off, so this narrow projection still gates 15;16;17;18;19 never - +native_decode_gating native_decode_gating rejections inside a surviving vector are counted once 15;16;17;18;19 2026-10-16 columnar_reader.c: the producer counts a gated group's rejections a second time +native_decode_gating native_decode_gating rows removed by filter across deleted rows count only live rows 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluator counts a deleted row as filtered;columnar_reader.c: the producer counts a gated group's rejections a second time +native_decode_gating native_decode_gating the control decodes every column of every vector 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluator marks every vector skipped +native_decode_gating native_decode_gating the control returns every row, values intact 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluator marks every vector skipped +native_decode_gating native_decode_gating the needle query returns exactly the matching rows 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluator marks every vector skipped +native_decode_gating native_decode_gating the needle removes exactly the non-matching rows, counted once 15;16;17;18;19 never - +native_decode_gating native_decode_gating the needle skips exactly the vectors holding no match 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluation between the decode passes never runs;columnar_reader.c: the per-vector qual evaluator marks every vector skipped +native_decode_gating native_decode_gating the nomatch query returns nothing 15;16;17;18;19 never - +native_decode_gating native_decode_gating the zero-match query removes every row by filter, counted once 15;16;17;18;19 never - native_delete_visibility_paths native_delete_visibility_paths a DELETE whose qual deletes another row of the table succeeds and leaves heap's rows (#1394) 15;16;17;18;19 2026-10-14 PgColumnarUpsertDeleteVector without the command advance (main before #1394's fix) native_delete_visibility_paths native_delete_visibility_paths a deleted row is invisible via the index 15;16;17;18;19 never - native_delete_visibility_paths native_delete_visibility_paths a live row is visible via the index 15;16;17;18;19 never - @@ -2288,6 +2380,25 @@ native_delete_visibility_paths native_delete_visibility_paths aggregate over the native_delete_visibility_paths native_delete_visibility_paths index / bitmap scan sees the live set 15;16;17;18;19 never - native_delete_visibility_paths native_delete_visibility_paths index-only scan sees the live set 15;16;17;18;19 never - native_delete_visibility_paths native_delete_visibility_paths sequential scan sees the live set (row-emitting path) 15;16;17;18;19 never - +native_dict_underfill native_dict_underfill backend survives the under-filled DICT chunk 15;16;17;18;19 2026-10-16 columnar_encoding.c: the DICT fill refusal replaced by a backend crash +native_dict_underfill native_dict_underfill premise: the inflation rewrote the DICT chunk's descriptor 15;16;17;18;19 never - +native_dict_underfill native_dict_underfill the text column is DICT-encoded (precondition) 15;16;17;18;19 never - +native_dict_underfill native_dict_underfill under-filled DICT chunk is rejected with a clean error 15;16;17;18;19 2026-10-16 columnar_encoding.c: a DICT chunk whose codes do not fill the raw length is no longer refused;columnar_encoding.c: the DICT fill refusal replaced by a backend crash +native_dml native_dml PK still enforced after DML 15;16;17;18;19 never - +native_dml native_dml count after delete 15;16;17;18;19 2026-10-16 columnar_vector.c: count(*) from metadata no longer subtracts a group's deleted rows +native_dml native_dml count after merge 15;16;17;18;19 2026-10-16 columnar_tableam.c: the index fetch no longer checks the delete vector;columnar_vector.c: count(*) from metadata no longer subtracts a group's deleted rows +native_dml native_dml count after update 15;16;17;18;19 2026-10-16 columnar_vector.c: count(*) from metadata no longer subtracts a group's deleted rows +native_dml native_dml deleted rows gone 15;16;17;18;19 never - +native_dml native_dml index finds live row 15;16;17;18;19 never - +native_dml native_dml index skips deleted row 15;16;17;18;19 2026-10-16 columnar_tableam.c: the index fetch no longer checks the delete vector +native_dml native_dml min/max after delete 15;16;17;18;19 2026-10-16 columnar_vector.c: a row group holding deletes still answers min and max from its zone maps +native_dml native_dml premise: the deletes moved the smallest and largest id and k 15;16;17;18;19 never - +native_dml native_dml premise: the lookups take the primary-key index 15;16;17;18;19 never - +native_dml native_dml rows after delete match heap 15;16;17;18;19 never - +native_dml native_dml rows after merge match heap 15;16;17;18;19 2026-10-16 columnar_tableam.c: the index fetch no longer checks the delete vector +native_dml native_dml rows after update match heap 15;16;17;18;19 2026-10-16 columnar_tableam.c: the index fetch no longer checks the delete vector +native_dml native_dml sum(k) after delete 15;16;17;18;19 never - +native_dml native_dml sum(k) after update 15;16;17;18;19 2026-10-16 columnar_tableam.c: the index fetch no longer checks the delete vector native_fetch_coalesce native_fetch_coalesce a wide index fetch does not pin once per column 15;16;17;18;19 2026-10-13 the coalesced fetch merging no adjacent ranges, so each column is read on its own native_fetch_coalesce native_fetch_coalesce premise: a point lookup uses the index 15;16;17;18;19 never - native_fetch_coalesce native_fetch_coalesce premise: fetching every projected column touched a measurable number of buffers 15;16;17;18;19 never - @@ -2666,6 +2777,46 @@ partman_columnar partman_columnar premise: with default_table_access_method ever partman_columnar partman_columnar premise: without default_table_access_method pg_partman makes heap partitions 15;16;17;18;19 never - partman_columnar partman_columnar retention leaves no pgcolumnar storage row behind 15;16;17;18;19 2026-10-12 the drop hook stops deleting the storage row, so a partition pg_partman drops leaves it behind partman_columnar partman_columnar the rows went in 15;16;17;18;19 2026-10-12 a sequential scan drops the last row group from its list, so every one-group partition reads empty +pg_dump_roundtrip pg_dump_roundtrip a rebuild now builds nothing 15;16;17;18;19 2026-10-16 columnar_projection.c: drop_projection keeps the projection's declaration;pgcolumnar--1.0-alpha5.sql: rebuild_projections() skips the declarations whose projection is missing +pg_dump_roundtrip pg_dump_roundtrip a second projection is declared 15;16;17;18;19 never - +pg_dump_roundtrip pg_dump_roundtrip and a declaration recorded for it 15;16;17;18;19 never - +pg_dump_roundtrip pg_dump_roundtrip and dropping it forgets the declaration, so a rebuild cannot resurrect it 15;16;17;18;19 2026-10-16 columnar_projection.c: drop_projection keeps the projection's declaration +pg_dump_roundtrip pg_dump_roundtrip custom: a second rebuild is a no-op 15;16;17;18;19 never - +pg_dump_roundtrip pg_dump_roundtrip custom: access method is pgcolumnar 15;16;17;18;19 never - +pg_dump_roundtrip pg_dump_roundtrip custom: and it reads the same rows as the base table 15;16;17;18;19 2026-10-16 pgcolumnar--1.0-alpha5.sql: pgcolumnar.projection_declaration no longer registered with pg_extension_config_dump;pgcolumnar--1.0-alpha5.sql: rebuild_projections() skips the declarations whose projection is missing +pg_dump_roundtrip pg_dump_roundtrip custom: and the projection exists again after the rebuild 15;16;17;18;19 2026-10-16 columnar_projection.c: drop_projection keeps the projection's declaration;pgcolumnar--1.0-alpha5.sql: pgcolumnar.projection_declaration no longer registered with pg_extension_config_dump;pgcolumnar--1.0-alpha5.sql: rebuild_projections() skips the declarations whose projection is missing +pg_dump_roundtrip pg_dump_roundtrip custom: declared sort_by survives restore 15;16;17;18;19 2026-10-16 pgcolumnar--1.0-alpha5.sql: pgcolumnar.options no longer registered with pg_extension_config_dump +pg_dump_roundtrip pg_dump_roundtrip custom: per-table options survive 15;16;17;18;19 2026-10-16 pgcolumnar--1.0-alpha5.sql: pgcolumnar.options no longer registered with pg_extension_config_dump +pg_dump_roundtrip pg_dump_roundtrip custom: pg_dump -Fc exited 0 15;16;17;18;19 never - +pg_dump_roundtrip pg_dump_roundtrip custom: pg_restore exited 0 (--exit-on-error) 15;16;17;18;19 never - +pg_dump_roundtrip pg_dump_roundtrip custom: premise: the point lookup takes the restored index 15;16;17;18;19 never - +pg_dump_roundtrip pg_dump_roundtrip custom: premise: the restored ds is not yet in its sort_by order 15;16;17;18;19 never - +pg_dump_roundtrip pg_dump_roundtrip custom: projection storage is NOT carried by pg_dump (see #266) 15;16;17;18;19 never - +pg_dump_roundtrip pg_dump_roundtrip custom: rebuild_projections() reports it rebuilt one 15;16;17;18;19 2026-10-16 columnar_projection.c: drop_projection keeps the projection's declaration;pgcolumnar--1.0-alpha5.sql: pgcolumnar.projection_declaration no longer registered with pg_extension_config_dump;pgcolumnar--1.0-alpha5.sql: rebuild_projections() skips the declarations whose projection is missing +pg_dump_roundtrip pg_dump_roundtrip custom: restored index answers a point lookup 15;16;17;18;19 never - +pg_dump_roundtrip pg_dump_roundtrip custom: restored sort_by resolves and applies 15;16;17;18;19 2026-10-16 columnar_vacuum.c: vacuum_sorted sorts on column 1 instead of the resolved sort keys;pgcolumnar--1.0-alpha5.sql: pgcolumnar.options no longer registered with pg_extension_config_dump +pg_dump_roundtrip pg_dump_roundtrip custom: row count survives 15;16;17;18;19 never - +pg_dump_roundtrip pg_dump_roundtrip custom: row data survives (checksum) 15;16;17;18;19 never - +pg_dump_roundtrip pg_dump_roundtrip custom: the projection declaration IS carried 15;16;17;18;19 2026-10-16 columnar_projection.c: drop_projection keeps the projection's declaration;pgcolumnar--1.0-alpha5.sql: pgcolumnar.projection_declaration no longer registered with pg_extension_config_dump +pg_dump_roundtrip pg_dump_roundtrip plain: a second rebuild is a no-op 15;16;17;18;19 never - +pg_dump_roundtrip pg_dump_roundtrip plain: access method is pgcolumnar 15;16;17;18;19 never - +pg_dump_roundtrip pg_dump_roundtrip plain: and it reads the same rows as the base table 15;16;17;18;19 2026-10-16 pgcolumnar--1.0-alpha5.sql: pgcolumnar.projection_declaration no longer registered with pg_extension_config_dump;pgcolumnar--1.0-alpha5.sql: rebuild_projections() skips the declarations whose projection is missing +pg_dump_roundtrip pg_dump_roundtrip plain: and the projection exists again after the rebuild 15;16;17;18;19 2026-10-16 columnar_projection.c: drop_projection keeps the projection's declaration;pgcolumnar--1.0-alpha5.sql: pgcolumnar.projection_declaration no longer registered with pg_extension_config_dump;pgcolumnar--1.0-alpha5.sql: rebuild_projections() skips the declarations whose projection is missing +pg_dump_roundtrip pg_dump_roundtrip plain: declared sort_by survives restore 15;16;17;18;19 2026-10-16 pgcolumnar--1.0-alpha5.sql: pgcolumnar.options no longer registered with pg_extension_config_dump +pg_dump_roundtrip pg_dump_roundtrip plain: dump recreates the table on the columnar AM 15;16;17;18;19 never - +pg_dump_roundtrip pg_dump_roundtrip plain: per-table options survive 15;16;17;18;19 2026-10-16 pgcolumnar--1.0-alpha5.sql: pgcolumnar.options no longer registered with pg_extension_config_dump +pg_dump_roundtrip pg_dump_roundtrip plain: pg_dump exited 0 15;16;17;18;19 never - +pg_dump_roundtrip pg_dump_roundtrip plain: premise: the point lookup takes the restored index 15;16;17;18;19 never - +pg_dump_roundtrip pg_dump_roundtrip plain: premise: the restored ds is not yet in its sort_by order 15;16;17;18;19 never - +pg_dump_roundtrip pg_dump_roundtrip plain: projection storage is NOT carried by pg_dump (see #266) 15;16;17;18;19 never - +pg_dump_roundtrip pg_dump_roundtrip plain: rebuild_projections() reports it rebuilt one 15;16;17;18;19 2026-10-16 columnar_projection.c: drop_projection keeps the projection's declaration;pgcolumnar--1.0-alpha5.sql: pgcolumnar.projection_declaration no longer registered with pg_extension_config_dump;pgcolumnar--1.0-alpha5.sql: rebuild_projections() skips the declarations whose projection is missing +pg_dump_roundtrip pg_dump_roundtrip plain: restore exited 0 (ON_ERROR_STOP) 15;16;17;18;19 never - +pg_dump_roundtrip pg_dump_roundtrip plain: restored index answers a point lookup 15;16;17;18;19 never - +pg_dump_roundtrip pg_dump_roundtrip plain: restored sort_by resolves and applies 15;16;17;18;19 2026-10-16 columnar_vacuum.c: vacuum_sorted sorts on column 1 instead of the resolved sort keys;pgcolumnar--1.0-alpha5.sql: pgcolumnar.options no longer registered with pg_extension_config_dump +pg_dump_roundtrip pg_dump_roundtrip plain: row count survives 15;16;17;18;19 never - +pg_dump_roundtrip pg_dump_roundtrip plain: row data survives (checksum) 15;16;17;18;19 never - +pg_dump_roundtrip pg_dump_roundtrip plain: the projection declaration IS carried 15;16;17;18;19 2026-10-16 columnar_projection.c: drop_projection keeps the projection's declaration;pgcolumnar--1.0-alpha5.sql: pgcolumnar.projection_declaration no longer registered with pg_extension_config_dump +pg_dump_roundtrip pg_dump_roundtrip the base table has a projection to preserve 15;16;17;18;19 never - prepared_across_sessions prepared_across_sessions ROLLBACK PREPARED of a DELETE leaves every row as it was 15;16;17;18;19 never - prepared_across_sessions prepared_across_sessions ROLLBACK PREPARED of an UPDATE leaves every row as it was, none duplicated 15;16;17;18;19 never - prepared_across_sessions prepared_across_sessions a prepared insert is invisible to another session 15;16;17;18;19 never - @@ -3341,6 +3492,31 @@ timeseries_recipes timeseries_recipes the how-to gives the LOCF recipe 15;16;17; timeseries_recipes timeseries_recipes the how-to gives the gap-fill recipe 15;16;17;18;19 2026-10-13 the how-to's 'Fill empty buckets' heading renamed, so the recipe is no longer found timeseries_recipes timeseries_recipes the how-to gives the interpolation recipe 15;16;17;18;19 2026-10-13 the how-to's 'Interpolate between known values' heading renamed, so the recipe is no longer found timeseries_recipes timeseries_recipes the interpolation recipe fills a gap linearly, and leaves both ends NULL 15;16;17;18;19 2026-10-13 the how-to's 'Interpolate between known values' heading renamed, so the recipe is no longer found;the how-to's interpolation recipe uses prev_v in place of next_v, so every gap is filled flat;the how-to's recipes join the series to the aggregate with JOIN instead of LEFT JOIN +truncate_cleanup truncate_cleanup DROP of a loaded projected table leaves nothing 15;16;17;18;19 2026-10-16 columnar_tableam.c: a rewrite no longer deletes the retired storage's catalog rows;columnar_tableam.c: deleting a storage tree no longer reaches its projections' storages +truncate_cleanup truncate_cleanup DROP of a loaded table after TRUNCATE leaves nothing 15;16;17;18;19 2026-10-16 columnar_tableam.c: a rewrite no longer deletes the retired storage's catalog rows +truncate_cleanup truncate_cleanup INSERT, TRUNCATE, INSERT in one transaction commits and keeps the new rows 15;16;17;18;19 2026-10-16 columnar_tableam.c: a rewrite no longer drops the cached write state of the retired storage +truncate_cleanup truncate_cleanup PREMISE the in-place transaction committed 15;16;17;18;19 2026-10-16 columnar_tableam.c: the in-place truncate no longer clears a projection's own storage +truncate_cleanup truncate_cleanup PREMISE the projection has its own storage, distinct from the base 15;16;17;18;19 2026-10-16 columnar_tableam.c: the in-place truncate no longer clears a projection's own storage +truncate_cleanup truncate_cleanup PREMISE the table exists after it 15;16;17;18;19 2026-10-16 columnar_tableam.c: the in-place truncate no longer clears a projection's own storage +truncate_cleanup truncate_cleanup TRUNCATE leaves no projection row under a retired storage id 15;16;17;18;19 2026-10-16 columnar_tableam.c: a rewrite no longer deletes the retired storage's catalog rows +truncate_cleanup truncate_cleanup TRUNCATE leaves no rows 15;16;17;18;19 never - +truncate_cleanup truncate_cleanup TRUNCATE returns the catalog to the empty baseline 15;16;17;18;19 2026-10-16 columnar_tableam.c: a rewrite no longer deletes the retired storage's catalog rows +truncate_cleanup truncate_cleanup a full rewrite keeps every row, and they are still readable 15;16;17;18;19 never - +truncate_cleanup truncate_cleanup and dropping it afterwards still leaves nothing 15;16;17;18;19 2026-10-16 columnar_tableam.c: a rewrite no longer deletes the retired storage's catalog rows +truncate_cleanup truncate_cleanup and that group describes only the rows written after it 15;16;17;18;19 2026-10-16 columnar_tableam.c: the in-place truncate no longer clears a projection's own storage +truncate_cleanup truncate_cleanup and the declaration survives the TRUNCATE, so the projection is rebuildable 15;16;17;18;19 never - +truncate_cleanup truncate_cleanup and the projection reads them back 15;16;17;18;19 2026-10-16 columnar_tableam.c: the in-place truncate no longer clears a projection's own storage +truncate_cleanup truncate_cleanup insert did add metadata 15;16;17;18;19 never - +truncate_cleanup truncate_cleanup premise: five thousand rows before the rewrite 15;16;17;18;19 never - +truncate_cleanup truncate_cleanup premise: the projected table carries metadata again before the DROP 15;16;17;18;19 never - +truncate_cleanup truncate_cleanup premise: the table carries metadata again, so the DROP has work to do 15;16;17;18;19 never - +truncate_cleanup truncate_cleanup premise: the table holds its first ten rows 15;16;17;18;19 never - +truncate_cleanup truncate_cleanup ten insert-and-truncate cycles on one table leave nothing 15;16;17;18;19 2026-10-16 columnar_tableam.c: a rewrite no longer deletes the retired storage's catalog rows +truncate_cleanup truncate_cleanup the projection agrees with the base table 15;16;17;18;19 2026-10-16 columnar_tableam.c: the in-place truncate no longer clears a projection's own storage +truncate_cleanup truncate_cleanup the projection did add metadata 15;16;17;18;19 never - +truncate_cleanup truncate_cleanup the projection storage holds exactly one row group after the truncate 15;16;17;18;19 2026-10-16 columnar_tableam.c: the in-place truncate no longer clears a projection's own storage +truncate_cleanup truncate_cleanup the rows written after the in-place truncate are all there 15;16;17;18;19 2026-10-16 columnar_tableam.c: the in-place truncate no longer clears a projection's own storage +truncate_cleanup truncate_cleanup three truncate-and-reload cycles still sit at the baseline 15;16;17;18;19 2026-10-16 columnar_tableam.c: a rewrite no longer deletes the retired storage's catalog rows ttl_expire ttl_expire NO row still inside the retention was dropped (#403 item 5a) 15;16;17;18;19 never - ttl_expire ttl_expire NO row still inside the retention was dropped, on a date column 15;16;17;18;19 never - ttl_expire ttl_expire a compaction's retired groups are not recorded as retention 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 3f8e29cde..94865b834 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 219 +suites_not_covered 212 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -2943,4 +2943,108 @@ suites_not_covered 219 # never 2676 - 1 = 2675 # dated 943 + 1 = 944 # 2675 + 944 == 3619 -checks_never_observed_red 2675 +# +# #1341, fk_referencing seeded, with its twin (OffgridwithJD): 2675 -> 2686, and +# suites_not_covered 219 -> 218. The suite had no rows. It enters with 28 rows +# from five-major runs: 27 checks on PG17-19 and 26 on 15-16, where the +# partitioned-control branch asserts core's refusal instead of a conversion. 17 are +# dated by four mutations of the foreign-key guards: the referenced-side refusal +# removed, the guard keyed on the referencing table, the conversion refusal +# removed, and the conversion refused whatever the target access method (the new +# referenced-table-to-row-store arm). Eleven arrive never: the over-rejection +# controls that no single guard reaches (another constraint, a heap-to-heap key, +# an unreferenced conversion, an unrelated ALTER, a conversion back to heap, the +# referencing side, the partitioned control on either side of 17) and the primary +# key's own arms. +# +# ledger rows 3619 + 28 = 3647 +# never 2675 + 11 = 2686 +# dated 944 + 17 = 961 +# 2686 + 961 == 3647 +# +# #1341, truncate_cleanup seeded, with its twin (OffgridwithJD): 2686 -> 2695, and +# suites_not_covered 218 -> 217. The suite had no rows. It enters with all 25 checks +# from five-major runs (25/25 on each of 15-19). 16 are dated by four mutations of +# the truncate paths: a rewrite that no longer deletes the retired storage's rows +# (7), nor drops the stale write state (1), an in-place truncate that skips the +# projections' storages (8), and a storage-tree delete that skips them (1). Nine +# arrive never: the premises that each fixture holds rows or metadata, the +# surviving projection declaration, the empty table after TRUNCATE, and the +# rewrite that keeps every row. +# +# ledger rows 3647 + 25 = 3672 +# never 2686 + 9 = 2695 +# dated 961 + 16 = 977 +# 2695 + 977 == 3672 +# +# #1341, native_dml seeded, with its twin (OffgridwithJD): 2695 -> 2702, and +# suites_not_covered 217 -> 216. The suite had no rows. It enters with all 15 checks +# from five-major runs (15/15 on each of 15-19). 8 are dated by three mutations: a +# row group holding deletes still answering min and max from its zone maps (the +# min/max arm, which could not see it before this change), count(*) from metadata +# not subtracting deleted rows (3), and the index fetch ignoring the delete vector +# (5). Seven arrive never: the two premises, the rows and sum after the delete, the +# deleted rows being gone, the live row's index lookup, and the primary key arm. +# +# ledger rows 3672 + 15 = 3687 +# never 2695 + 7 = 2702 +# dated 977 + 8 = 985 +# 2702 + 985 == 3687 +# +# #1341, native_dict_underfill seeded, with its twin (OffgridwithJD): 2702 -> 2704, +# and suites_not_covered 216 -> 215. The suite had no rows. It enters with all 4 +# checks from five-major runs (4/4 on each of 15-19). 2 are dated by two mutations +# of the DICT fill check: removed, so the garbage tail is read back with no error, +# and replaced by a backend crash, which the rejection and the log-counted survival +# arm both catch. The two premises arrive never. +# +# ledger rows 3687 + 4 = 3691 +# never 2702 + 2 = 2704 +# dated 985 + 2 = 987 +# 2704 + 987 == 3691 +# +# #1341, native_decode_gating seeded, with its twin (OffgridwithJD): 2704 -> 2722, and +# suites_not_covered 215 -> 214. The suite had no rows. It enters with all 29 checks +# from five-major runs (29/29 on each of 15-19). 11 are dated by four mutations of +# #452 phase 2: the per-vector evaluation never run (4), a deleted row counted as +# filtered (1, the new arm), every vector marked skipped (7), and the producer +# counting a gated group's rejections twice (2, one of them new). Eighteen arrive +# never: the thirteen premises, the observable's presence, the full scan across +# every group, the nomatch query's empty answer, and two exact counters that no +# single mutation here moves. +# +# ledger rows 3691 + 29 = 3720 +# never 2704 + 18 = 2722 +# dated 987 + 11 = 998 +# 2722 + 998 == 3720 +# +# #1341, native_decode_gate_width seeded, with its twin (OffgridwithJD): 2722 -> 2754, +# and suites_not_covered 214 -> 213. The suite had no rows. It enters with all 35 +# checks from five-major runs (35/35 on each of 15-19). 3 are dated by three +# mutations of the #595 width gate: gating whatever the width (the two "not gated" +# arms), > for >= (the boundary arm), and a payload count one too high (the +# raised-threshold arm). Thirty-two arrive never: the twenty-one premises, and the arms +# the width policy does not move -- the gated counts, the gate-off and controls, the +# correctness arms -- which the sibling native_decode_gating dates. +# +# ledger rows 3720 + 35 = 3755 +# never 2722 + 32 = 2754 +# dated 998 + 3 = 1001 +# 2754 + 1001 == 3755 +# +# #1341, pg_dump_roundtrip seeded, with its twin (OffgridwithJD): 2754 -> 2778, and +# suites_not_covered 213 -> 212. The suite had no rows. It enters with all 40 checks +# from five-major runs (40/40 on each of 15-19). 16 are dated by five mutations: the +# options and the projection-declaration catalogs no longer registered with +# pg_extension_config_dump, rebuild_projections() skipping a missing projection, +# drop_projection keeping its declaration, and vacuum_sorted sorting on column 1 +# (the strengthened sort arm). Twenty-four arrive never: the dump and restore exit +# codes and the dump's columnar AM, the access method, the row count and checksum, +# the index premise and its lookup, the pinned loss of projection storage, the +# inversion premise, the second rebuild's no-op, and the source's own premises. +# +# ledger rows 3755 + 40 = 3795 +# never 2754 + 24 = 2778 +# dated 1001 + 16 = 1017 +# 2778 + 1017 == 3795 +checks_never_observed_red 2778 diff --git a/test/fk_referencing.sh b/test/fk_referencing.sh index 73b59c41d..cd49dcecd 100755 --- a/test/fk_referencing.sh +++ b/test/fk_referencing.sh @@ -220,6 +220,16 @@ check "converting back to heap is unaffected" \ "$(case "$back" in *ERROR*) echo "no ($back)" ;; *) echo yes ;; esac)" \ "yes" +# AND ON THE REFERENCED TABLE ITSELF (#1341). The control above converts a table +# with no foreign key, so it cannot see a rule that refuses every SET ACCESS METHOD +# on a referenced table whatever the target: with the guard's columnar-target test +# removed, this file stayed 26/26. Only a columnar target needs the row lock the +# constraint cannot get. +refheap="$(psql_run "ALTER TABLE fk_conv_parent SET ACCESS METHOD heap;" 2>&1 || true)" +check "an FK-referenced table may still be set to a row store" \ + "$(case "$refheap" in *ERROR*) echo "no ($refheap)" ;; *) echo yes ;; esac)" \ + "yes" + # --- 7. a partitioned parent is refused too (#201) ----------------------------- # A partitioned table has no storage, so setting its access method breaks nothing diff --git a/test/native_decode_gate_width.sh b/test/native_decode_gate_width.sh index 94bcd8189..176830fd6 100755 --- a/test/native_decode_gate_width.sh +++ b/test/native_decode_gate_width.sh @@ -106,10 +106,17 @@ check "premise: the width-gate GUC exists and defaults to the shipped $DEFAULT_G # unknown parameter errors; q swallows the error and returns empty, so an empty # result is the proof the parameter does not exist. The paired positive premise # above (min_payload_cols == 20) rules out an empty result from a dead connection. +# By SQLSTATE rather than by an empty result (#1341): q returns empty for a dead +# connection as well, and an empty expectation is the shape the pytest layer refuses. +show_state() { # show_state PARAM -> the SQLSTATE SHOW raised, or "shown" + env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" \ + -qAtX -v VERBOSITY=verbose -c "SHOW $1;" >/dev/null 2>"$PGC_WORKDIR/show.err" && { echo shown; return; } + sed -nE 's/^ERROR: ([0-9A-Z]{5}): .*/\1/p' "$PGC_WORKDIR/show.err" | head -1 +} check "premise: the rejected probe GUC qual_skipvec_probe_vecs was NOT shipped" \ - "$(q "SHOW pgcolumnar.qual_skipvec_probe_vecs;")" "" + "$(show_state pgcolumnar.qual_skipvec_probe_vecs)" "42704" check "premise: the rejected probe GUC qual_skipvec_min_skip_pct was NOT shipped" \ - "$(q "SHOW pgcolumnar.qual_skipvec_min_skip_pct;")" "" + "$(show_state pgcolumnar.qual_skipvec_min_skip_pct)" "42704" check "premise: one row group, so every count below is one group's" \ "$(q "SELECT count(*) FROM pgcolumnar.row_group WHERE storage_id = pgcolumnar.get_storage_id('nw');")" \ "1" diff --git a/test/native_decode_gating.sh b/test/native_decode_gating.sh index fb18ac03c..2b8f9fc7b 100755 --- a/test/native_decode_gating.sh +++ b/test/native_decode_gating.sh @@ -172,6 +172,20 @@ check "the zero-match query removes every row by filter, counted once" \ check "and skips every vector" \ "$(counter_in "$PLAN_NOMATCH" 'Columnar Vectors Skipped')" "$VECTORS" +# A SURVIVING VECTOR WITH REJECTS IN IT (#1341). Every row of the needle's +# surviving vector matches, so the producer, re-testing that vector to find its +# rows, finds nothing to reject: a producer that counted those rejections a second +# time read exactly right above. Measured, with the producer's non-counting filter +# disabled, this file stayed 26/26. Half of vector 0 passes this qual (p1 % 2 = 0 is +# an expression, so it is no skip predicate either), so its 512 rejections are +# counted twice by that mistake. +HALFQ="SELECT * FROM n WHERE tag LIKE '%needle%' AND p1 % 2 = 0" +PLAN_HALF="$(explain_of "$HALFQ")" +check "premise: the half-needle survives in one vector, half of its rows" \ + "$(q "SELECT count(*) || '|' || count(DISTINCT (p1 - 1) / 1024) FROM n WHERE tag LIKE '%needle%' AND p1 % 2 = 0;")" "512|1" +check "rejections inside a surviving vector are counted once" \ + "$(counter_in "$PLAN_HALF" 'Rows Removed by Filter')" "$(( ROWS - 512 ))" + # ---- correctness ----------------------------------------------------------- # # The mask must never drop a row. Without these, an implementation that marks @@ -205,13 +219,18 @@ MGEN="SELECT 'x' || CASE WHEN g % 4096 = 7 THEN 'needle' ELSE 'miss' END || g::t psql_run "INSERT INTO hm $MGEN;" psql_run "INSERT INTO m $MGEN;" psql_run "DELETE FROM hm WHERE p1 % 17 = 0;" -psql_run "DELETE FROM m WHERE p1 % 17 = 0;" +# The fixture's DELETE must not depend on the gating under test: its WHERE is an +# unprunable qual too, so it is run with the width gate out of reach (#1341). +psql_run "SET pgcolumnar.qual_skipvec_min_payload_cols = 100000; DELETE FROM m WHERE p1 % 17 = 0;" check "premise: the columnar table spans more than one row group" \ "$(q "SELECT count(*) > 1 FROM pgcolumnar.row_group WHERE storage_id = pgcolumnar.get_storage_id('m');")" \ "t" +# count(*) rather than a count WHERE p1 % 17 = 0: that WHERE is an unprunable qual +# too, so it ran through the very gating under test, and a gating mutant broke this +# premise rather than the arms below (#1341). check "premise: rows really were deleted (so the mask is non-NULL)" \ - "$(q "SELECT count(*) FROM m WHERE p1 % 17 = 0;")" "0" + "$(q "SELECT count(*) FROM m;")" "$(( MROWS - MROWS / 17 ))" check "deletes + multiple groups + unprunable qual: columnar matches the heap" \ "$(pgc_set_hash "SELECT * FROM m WHERE tag LIKE '%needle%'")" \ @@ -220,4 +239,13 @@ check "and the full scan across every group's mask matches too" \ "$(pgc_set_hash "SELECT * FROM m")" \ "$(pgc_set_hash "SELECT * FROM hm")" +# AND THE COUNTER SKIPS DELETED ROWS (#1341). The evaluator must not count a deleted +# row as filtered: it is invisible. Only the single-group table above had its +# counters checked, and it has no deletes, so with the evaluator's deleted-row test +# removed this file stayed 26/26. +PLAN_M="$(explain_of "SELECT * FROM m WHERE tag LIKE '%needle%'")" +check "rows removed by filter across deleted rows count only live rows" \ + "$(counter_in "$PLAN_M" 'Rows Removed by Filter')" \ + "$(q "SELECT count(*) FROM hm WHERE tag NOT LIKE '%needle%';")" + pgc_summary diff --git a/test/native_dict_underfill.sh b/test/native_dict_underfill.sh index bf32d082c..b5cf5bf8e 100755 --- a/test/native_dict_underfill.sh +++ b/test/native_dict_underfill.sh @@ -27,23 +27,31 @@ echo "-- v-column vector0 encoding type byte (DICT=6): $ENC" check "the text column is DICT-encoded (precondition)" "$ENC" "6" # read old rawLen (uint32 LE at descriptor offset 11), inflate by 64, write back -q "UPDATE pgcolumnar.column_chunk c SET encoding_descriptor = +crashes() { + # grep -c prints 0 itself when nothing matches, and then exits 1 + pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null || true +} +crashes0="$(crashes)" +q "WITH u AS (UPDATE pgcolumnar.column_chunk c SET encoding_descriptor = set_byte(set_byte(set_byte(set_byte(encoding_descriptor, 11, ( (get_byte(encoding_descriptor,11) + (get_byte(encoding_descriptor,12)<<8) + (get_byte(encoding_descriptor,13)<<16) + (get_byte(encoding_descriptor,14)<<24)) + 64 ) & 255), 12, (((get_byte(encoding_descriptor,11) + (get_byte(encoding_descriptor,12)<<8) + (get_byte(encoding_descriptor,13)<<16) + (get_byte(encoding_descriptor,14)<<24)) + 64) >> 8) & 255), 13, (((get_byte(encoding_descriptor,11) + (get_byte(encoding_descriptor,12)<<8) + (get_byte(encoding_descriptor,13)<<16) + (get_byte(encoding_descriptor,14)<<24)) + 64) >>16) & 255), 14, (((get_byte(encoding_descriptor,11) + (get_byte(encoding_descriptor,12)<<8) + (get_byte(encoding_descriptor,13)<<16) + (get_byte(encoding_descriptor,14)<<24)) + 64) >>24) & 255) - WHERE c.storage_id=$SID AND c.column_index=1" >/dev/null + WHERE c.storage_id=$SID AND c.column_index=1 RETURNING 1) SELECT count(*) FROM u" > "$PGC_WORKDIR/uf.n" +check "premise: the inflation rewrote the DICT chunk's descriptor" "$(cat "$PGC_WORKDIR/uf.n")" "1" env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" \ - -c "SELECT v FROM dt" >/dev/null 2>"$PGC_WORKDIR/uf.err" + -v VERBOSITY=verbose -c "SELECT v FROM dt" >/dev/null 2>"$PGC_WORKDIR/uf.err" echo "-- select stderr:"; sed 's/^/ /' "$PGC_WORKDIR/uf.err" | head -3 -rejected=no -grep -qiE 'does not match|corrupt|invalid|out of' "$PGC_WORKDIR/uf.err" && rejected=yes -crash=no -grep -qiE 'terminated by signal|segmentation|server closed' "$PGC_WORKDIR/uf.err" && crash=yes -grep -qiE 'terminated by signal|segmentation' "$PGC_LOGFILE" 2>/dev/null && crash=yes -echo "-- rejected=$rejected crash=$crash" -check "under-filled DICT chunk is rejected with a clean error" "$rejected" "yes" -check "backend survives the under-filled DICT chunk" "$(q 'SELECT 1')" "1" +# A CLEAN ERROR IS XX001, AND A CRASH IS READ FROM THE SERVER LOG (#1341). The +# rejection used to match 'does not match|corrupt|invalid|out of' in any message, +# which "out of memory" satisfies too; and survival was a fresh SELECT 1, which a +# crashed backend passes once crash recovery has finished. The crash flag below was +# computed and never checked. +rejected="$(sed -nE 's/^ERROR: ([0-9A-Z]{5}): .*/\1/p' "$PGC_WORKDIR/uf.err" | head -1 | grep . || echo "no SQLSTATE ($(head -1 "$PGC_WORKDIR/uf.err"))")" +echo "-- rejected=$rejected crashes=$(( $(crashes) - crashes0 ))" +check "under-filled DICT chunk is rejected with a clean error" "$rejected" "XX001" +check "backend survives the under-filled DICT chunk" \ + "$(q 'SELECT 1')|$(( $(crashes) - crashes0 )) backend crashes" "1|0 backend crashes" pgc_summary diff --git a/test/native_dml.sh b/test/native_dml.sh index 9e67578b6..ae68bc317 100755 --- a/test/native_dml.sh +++ b/test/native_dml.sh @@ -21,9 +21,15 @@ GEN="SELECT g, (g * 10)::bigint, 'row-' || g FROM generate_series(1, 5000) g" psql_run "INSERT INTO h $GEN;" psql_run "INSERT INTO n $GEN;" -# DELETE across several row groups. -psql_run "DELETE FROM h WHERE id % 7 = 0;" -psql_run "DELETE FROM n WHERE id % 7 = 0;" +# DELETE across several row groups, and the table's smallest and largest ids with +# them. Without those two, min and max are the same whether or not the deleted rows +# count, so the min/max arm below could not see a zone-map answer that ignored the +# deletes (#1341): measured, with groups that hold deletes left on the zone-map path, +# this file stayed 13/13. +psql_run "DELETE FROM h WHERE id % 7 = 0 OR id IN (1, 5000);" +psql_run "DELETE FROM n WHERE id % 7 = 0 OR id IN (1, 5000);" +check "premise: the deletes moved the smallest and largest id and k" \ + "$(q 'SELECT min(id) || '"'"'|'"'"' || max(id) || '"'"'|'"'"' || min(k) || '"'"'|'"'"' || max(k) FROM h;')" "2|4999|20|49990" check "count after delete" "$(q 'SELECT count(*) FROM n;')" "$(q 'SELECT count(*) FROM h;')" check "rows after delete match heap" \ @@ -32,14 +38,20 @@ check "rows after delete match heap" \ # Aggregates after delete: the zone-map path is bypassed (deletes present), so this # exercises the delete-applying scan fold. check "sum(k) after delete" "$(q 'SELECT sum(k) FROM n;')" "$(q 'SELECT sum(k) FROM h;')" +# k as well as id: measured, a zone-map answer that ignored the deletes still gave +# min/max(id) right and min/max(k) wrong (10|50000 for 20|49990). check "min/max after delete" \ - "$(q 'SELECT min(id), max(id) FROM n;')" \ - "$(q 'SELECT min(id), max(id) FROM h;')" + "$(q 'SELECT min(id), max(id), min(k), max(k) FROM n;')" \ + "$(q 'SELECT min(id), max(id), min(k), max(k) FROM h;')" check "deleted rows gone" "$(q 'SELECT count(*) FROM n WHERE id % 7 = 0;')" "0" # Index lookup of a deleted row returns nothing; a live row still resolves. idx() { env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -q -At -c "SET enable_seqscan=off" -c "$1" 2>/dev/null; } # id 4444 is live (4444 % 7 = 6); id 4445 was deleted (4445 % 7 = 0). +# The lookups must take the primary-key index, or these arms test the scan instead +# of the index fetch's delete check. +check "premise: the lookups take the primary-key index" \ + "$(idx 'EXPLAIN (COSTS off) SELECT count(*) FROM n WHERE id = 4445;' | grep -c 'Index.* using n_pkey')" "1" check "index finds live row" "$(idx 'SELECT v FROM n WHERE id = 4444;')" "$(q 'SELECT v FROM h WHERE id = 4444;')" check "index skips deleted row" "$(idx 'SELECT count(*) FROM n WHERE id = 4445;')" "0" diff --git a/test/native_upgrade_converge.sh b/test/native_upgrade_converge.sh index 0249d5655..454900137 100755 --- a/test/native_upgrade_converge.sh +++ b/test/native_upgrade_converge.sh @@ -310,4 +310,53 @@ done rm -f "$REF" P -d postgres -c "DROP DATABASE IF EXISTS conv_fresh;" >/dev/null +# ---- the window: this library over the previous release's catalog (#1341 item 66) ---- +# +# docs/installation.md steps 1 and 2 put the new library in place; step 3, +# ALTER EXTENSION UPDATE, brings the catalog up to it. Between them a database +# runs this library over its previous release's catalog. The library deletes +# from retired_group unconditionally (columnar_metadata.c, PgColumnarDeleteMetadata), +# which that catalog does not have, so DROP TABLE, TRUNCATE and pgcolumnar.vacuum() +# fail there: measured, with 42P01 'columnar metadata table +# "pgcolumnar.retired_group" does not exist', which names neither the cause nor step 3. +# +# The arms hold what the window must keep: reads and writes work, a refused +# operation changes nothing and names the remedy, and step 3 makes every one of +# them work on the same rows. PREV is the newest fixture, so these follow the +# release they describe rather than naming it. +PREV="${_NUC_FIXTURES[${#_NUC_FIXTURES[@]}-1]}" +WIN=conv_window +P -d postgres -c "DROP DATABASE IF EXISTS $WIN;" >/dev/null +P -d postgres -c "CREATE DATABASE $WIN;" >/dev/null +P -d "$WIN" -c "CREATE EXTENSION pgcolumnar VERSION '$PREV';" >/dev/null +check "premise: the window database runs this library over the previous release's catalog" \ + "$(P -d "$WIN" -c "SELECT extversion FROM pg_extension WHERE extname='pgcolumnar';")" "$PREV" +check "premise: that catalog has no retired_group, which this library deletes from" \ + "$(P -d "$WIN" -c "SELECT count(*) FROM pg_class WHERE relname = 'retired_group' AND relnamespace = 'pgcolumnar'::regnamespace;")" "0" +P -d "$WIN" -c "CREATE TABLE w (a int, b text) USING pgcolumnar;" >/dev/null +P -d "$WIN" -c "INSERT INTO w SELECT g, 'x' || g FROM generate_series(1, 20000) g;" >/dev/null +P -d "$WIN" -c "DELETE FROM w WHERE a % 3 = 0;" >/dev/null +win_rows() { P -d "$WIN" -c "SELECT count(*) || '|' || sum(a) FROM w;"; } +check "in the window, a columnar table is written, deleted from and read" "$(win_rows)" "13334|133346667" +win_refusal() { # win_refusal SQL -> SQLSTATE|whether the HINT names step 3, or what happened instead + local out + out="$(P -d "$WIN" -v VERBOSITY=verbose -c "$1" 2>&1)" && { echo accepted; return; } + echo "$(sed -nE 's/^ERROR: ([0-9A-Z]{5}): .*/\1/p' <<<"$out" | head -1)|$(grep -q '^HINT: .*ALTER EXTENSION pgcolumnar UPDATE' <<<"$out" && echo names-the-update || echo no-hint)" +} +check "in the window, DROP TABLE is refused and names ALTER EXTENSION UPDATE" \ + "$(win_refusal "DROP TABLE w;")" "42P01|names-the-update" +check "in the window, TRUNCATE is refused and names ALTER EXTENSION UPDATE" \ + "$(win_refusal "TRUNCATE w;")" "42P01|names-the-update" +check "in the window, pgcolumnar.vacuum() is refused and names ALTER EXTENSION UPDATE" \ + "$(win_refusal "SELECT pgcolumnar.vacuum('w');")" "42P01|names-the-update" +check "in the window, the refused operations changed nothing" "$(win_rows)" "13334|133346667" + +P -d "$WIN" -c "ALTER EXTENSION pgcolumnar UPDATE;" >/dev/null +check "after ALTER EXTENSION UPDATE, the same rows are there" "$(win_rows)" "13334|133346667" +check "after ALTER EXTENSION UPDATE, pgcolumnar.vacuum() succeeds and keeps them" \ + "$(P -d "$WIN" -c "SELECT pgcolumnar.vacuum('w');" >/dev/null 2>&1 && win_rows || echo refused)" "13334|133346667" +check "after ALTER EXTENSION UPDATE, TRUNCATE and DROP TABLE succeed" \ + "$(P -d "$WIN" -c "TRUNCATE w;" >/dev/null 2>&1 && P -d "$WIN" -c "DROP TABLE w;" >/dev/null 2>&1 && echo succeeded || echo refused)" "succeeded" +P -d postgres -c "DROP DATABASE IF EXISTS $WIN;" >/dev/null + pgc_summary diff --git a/test/pg_dump_roundtrip.sh b/test/pg_dump_roundtrip.sh index 50e4ec894..60cf10d8a 100755 --- a/test/pg_dump_roundtrip.sh +++ b/test/pg_dump_roundtrip.sh @@ -95,6 +95,11 @@ verify() { # label db check "$label: row data survives (checksum)" "$(on "$db" "$sum_sql;")" "$before_sum" check "$label: access method is pgcolumnar" "$(on "$db" "$am_sql;")" "pgcolumnar" # tail -1: the two SET statements each emit a command tag before the result. + # The plan premise is what makes this arm about the INDEX (#1341): the columnar + # scan is not disabled by enable_seqscan, and with the index dropped from the + # restored database the same lookup still answered 12345. + check "$label: premise: the point lookup takes the restored index" \ + "$(on "$db" "SET enable_seqscan=off; SET enable_bitmapscan=off; EXPLAIN (COSTS off) SELECT id FROM dt WHERE id = 12345;" | grep -c 'Index.* using dt_id_idx')" "1" check "$label: restored index answers a point lookup" "$(on "$db" "$idx_sql;" | tail -1)" "12345" # Per-table options survive the round trip (#248). They did not until the # extension registered pgcolumnar.options with pg_extension_config_dump: @@ -119,8 +124,15 @@ verify() { # label db # the wrong or a missing column here. check "$label: declared sort_by survives restore" \ "$(on "$db" "SELECT sort_by::text FROM pgcolumnar.options WHERE regclass='ds'::regclass;")" "{host,ts}" + # APPLIES means the rows end up in (host, ts) order, not merely that the call + # returns (#1341): the row count alone is 20000 whichever columns it sorted on. + # Inversions are counted in scan order, with a premise that there are some + # before the call, so a no-op cannot pass. + inversions="SELECT count(*) FROM (SELECT host, ts, lag(host) OVER () AS lh, lag(ts) OVER () AS lt FROM ds) x WHERE (lh, lt) > (host, ts)" + check "$label: premise: the restored ds is not yet in its sort_by order" \ + "$(on "$db" "SELECT ($inversions) > 0;")" "t" check "$label: restored sort_by resolves and applies" \ - "$(on "$db" "SELECT pgcolumnar.vacuum_sorted('ds'); SELECT count(*) FROM ds;" 2>/dev/null | tail -1)" "20000" + "$(on "$db" "SELECT pgcolumnar.vacuum_sorted('ds'); SELECT count(*) || '|' || ($inversions) FROM ds;" 2>/dev/null | tail -1)" "20000|0" # The DECLARATION is carried, which is the half #266 fixed. It is keyed by # regclass and stores column names, so config_dump can restore it, and the diff --git a/test/pg_upgrade.sh b/test/pg_upgrade.sh index bb4e5c226..ebf7fdcea 100755 --- a/test/pg_upgrade.sh +++ b/test/pg_upgrade.sh @@ -231,8 +231,20 @@ check "per-table options preserved" \ # Compared against the value captured on the OLD cluster, not against another # query on the new one -- the first version of this line compared the new answer # with itself and would have passed whatever the index did. +# The plan premise is what makes the next arm about the INDEX (#1341): the columnar +# scan is not disabled by enable_seqscan, and with the index dropped after the +# upgrade the same range count still came back right. +# Right after pg_upgrade there are no planner statistics, and the planner then took +# the columnar scan for this range even with seqscan off: measured, the arm below +# never touched the index. So the custom scan is turned off for the premise and the +# lookup alike, and any index scan of c_id (a bitmap one too) counts. +IDXOFF="SET enable_seqscan=off; SET pgcolumnar.enable_custom_scan=off;" +up_plan="$(nq "$IDXOFF EXPLAIN (COSTS off) SELECT count(*) FROM c WHERE id BETWEEN 100 AND 200")" +echo "-- range plan after the upgrade:"; sed 's/^/ /' <<<"$up_plan" +check "premise: the range lookup takes the upgraded index" \ + "$(grep -cE 'Index (Only )?Scan (using|on) c_id' <<<"$up_plan")" "1" check "the index still answers the same range" \ - "$(nq "SET enable_seqscan=off; SELECT count(*) FROM c WHERE id BETWEEN 100 AND 200" | tail -1)" \ + "$(nq "$IDXOFF SELECT count(*) FROM c WHERE id BETWEEN 100 AND 200" | tail -1)" \ "$old_range" # The upgraded relation must still be writable, not merely readable: a table that diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 9212ee1a5..610c707b0 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -188,6 +188,15 @@ behaviour, the source of that number is named. - [140. test_parquet_import.py: a partitioned columnar parent is refused by import_parquet](#140-test_parquet_importpy-a-partitioned-columnar-parent-is-refused-by-import_parquet) - [141. test_phase5.py: set_options and reset_options agree](#141-test_phase5py-set_options-and-reset_options-agree) - [142. test_drop_cleanup.py: a columnar matview's options row goes with its DROP](#142-test_drop_cleanuppy-a-columnar-matviews-options-row-goes-with-its-drop) +- [143. test_native_upgrade_converge.py: the upgrade window, before ALTER EXTENSION UPDATE](#143-test_native_upgrade_convergepy-the-upgrade-window-before-alter-extension-update) +- [144. test_fk_referencing.py: a foreign key referencing a columnar table is refused where it is chosen](#144-test_fk_referencingpy-a-foreign-key-referencing-a-columnar-table-is-refused-where-it-is-chosen) +- [145. test_truncate_cleanup.py: TRUNCATE leaves no catalog rows behind, and keeps what it must](#145-test_truncate_cleanuppy-truncate-leaves-no-catalog-rows-behind-and-keeps-what-it-must) +- [146. test_native_dml.py: DELETE, UPDATE and MERGE on a native table agree with heap](#146-test_native_dmlpy-delete-update-and-merge-on-a-native-table-agree-with-heap) +- [147. test_native_dict_underfill.py: an under-filled DICT chunk is refused and no backend dies](#147-test_native_dict_underfillpy-an-under-filled-dict-chunk-is-refused-and-no-backend-dies) +- [148. test_native_decode_gating.py: an unprunable qual gates payload decode per vector](#148-test_native_decode_gatingpy-an-unprunable-qual-gates-payload-decode-per-vector) +- [149. test_native_decode_gate_width.py: decode gating runs only on a projection wide enough to pay](#149-test_native_decode_gate_widthpy-decode-gating-runs-only-on-a-projection-wide-enough-to-pay) +- [150. test_pg_dump_roundtrip.py: a columnar table survives pg_dump and restore](#150-test_pg_dump_roundtrippy-a-columnar-table-survives-pg_dump-and-restore) +- [151. test_pg_upgrade.py: a columnar table survives pg_upgrade](#151-test_pg_upgradepy-a-columnar-table-survives-pg_upgrade) ## 1. How to read a test in here @@ -7377,3 +7386,107 @@ Part of the pytest twin of `test/drop_cleanup.sh` (#1265, ported for #1341): its | `test_a_columnar_matview_takes_its_options_row_with_it` | a planted options row is gone after DROP MATERIALIZED VIEW | | `test_refresh_keeps_a_matviews_options_row` | a REFRESH that rewrote the matview leaves its set_options row in place | | `test_an_unrelated_options_row_survives` | another table's row survives a matview's DROP, and an ordinary columnar table still takes its own | + +## 143. test_native_upgrade_converge.py: the upgrade window, before ALTER EXTENSION UPDATE + +Part of the pytest twin of `test/native_upgrade_converge.sh` (#1341 item 66): its upgrade-window arms only, by the shell's names, and the pair is declared INCOMPLETE. Between installing a new library and running `ALTER EXTENSION pgcolumnar UPDATE`, a database runs the new library over its previous release's catalog. Reads and writes must work there. `DROP TABLE`, `TRUNCATE` and `pgcolumnar.vacuum()`, which need a catalog table the old one lacks, must be refused with 42P01, change nothing, and carry a hint naming the update; after the update all three succeed on the same rows. The previous release is the version the shipped upgrade script to the current `default_version` starts from; its base script is staged from `test/fixtures` for the test and removed after. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_the_upgrade_window` | at the previous catalog: rows written, deleted and read; the three operations refused with the hint and nothing changed; after the update the rows are intact and all three succeed | + +## 144. test_fk_referencing.py: a foreign key referencing a columnar table is refused where it is chosen + +The pytest twin of `test/fk_referencing.sh` (ported for #1341), every check by the shell's name. Enforcing a foreign key locks the referenced row, and columnar tables cannot lock rows, so a constraint referencing one would make every insert into the referencing table fail. It is refused at DDL by either route into that state: creating the constraint against a columnar table, or converting a referenced table (or partitioned parent) to columnar. Only the referenced side is refused, and the controls hold everything else steady, including converting the referenced table itself to a row store. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_the_referenced_side_is_refused_at_ddl` | CREATE TABLE and ALTER TABLE ADD CONSTRAINT referencing a columnar table are refused, saying why, and nothing is left behind | +| `test_the_referencing_side_still_works` | a columnar table referencing a heap table is created, accepts satisfying rows and rejects a violating one | +| `test_constraints_that_need_no_row_lock_are_unaffected` | primary key, unique, check and not-null on a columnar table, and a heap-to-heap foreign key, are unaffected | +| `test_converting_a_referenced_table_is_refused_and_nothing_else_is` | SET ACCESS METHOD pgcolumnar on a referenced table and the helper are refused, the table stays heap and its child writable; an unreferenced table, the referencing side, an unrelated ALTER, a conversion back to heap and the referenced table to a row store all still work | +| `test_a_partitioned_referenced_parent` | a columnar partition and setting the parent's access method are refused, the parent does not become columnar, an ordinary partition still works; an unreferenced partitioned table converts from PG17, and before 17 core refuses it | + +## 145. test_truncate_cleanup.py: TRUNCATE leaves no catalog rows behind, and keeps what it must + +The pytest twin of `test/truncate_cleanup.sh` (ported for #1341), every check by the shell's name. TRUNCATE installs a new storage id, and the old storage's catalog rows used to stay behind, so each truncate-and-reload cycle leaked one storage's worth and a later DROP removed only the empty current one. Retiring them made a cached write state stale, which is dropped with them. A relation created in the current transaction is truncated in place, on a different path, which must clear each projection's storage too. A private database, since the snapshot counts every pgcolumnar catalog row. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_truncate_and_drop_leave_the_catalog_as_they_found_it` | one TRUNCATE and three truncate-and-reload cycles return the catalog to the baseline, and a DROP of a reloaded table leaves nothing | +| `test_a_projection_is_not_leaked_by_truncate` | no projection row names a retired storage id, the declaration survives, and a DROP of the reloaded table leaves nothing | +| `test_repeated_truncates_accumulate_nothing` | ten transactional truncates of one table, and its DROP, leave the baseline | +| `test_insert_truncate_insert_in_one_transaction` | the transaction commits and holds exactly the rows written after the TRUNCATE | +| `test_a_rewrite_that_is_not_a_truncate_loses_nothing` | a type-changing rewrite keeps all 5000 rows readable | +| `test_an_in_place_truncate_clears_the_projection_too` | the in-place truncate commits, the later rows are all there, the projection reads them back and its storage holds one group of exactly them | + +## 146. test_native_dml.py: DELETE, UPDATE and MERGE on a native table agree with heap + +The pytest twin of `test/native_dml.sh` (ported for #1341), every check by the shell's name. Each statement runs on a heap table and on a columnar one with 1000-row stripes, and every result is compared. After deletes an aggregate must not come from zone maps that still describe the deleted rows: the deletes include the smallest and largest ids, so min and max move, and they are taken over `k`, a column the primary key does not order. A deleted row's index lookup finds nothing, and the lookups are asserted to take the index. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_delete_update_and_merge_match_heap` | after DELETE, UPDATE and MERGE, counts, rows, sum and min/max match heap; the index finds a live row and skips a deleted one; the primary key still rejects a duplicate | + +## 147. test_native_dict_underfill.py: an under-filled DICT chunk is refused and no backend dies + +The pytest twin of `test/native_dict_underfill.sh` (ported for #1341), every check by the shell's name. A DICT chunk's `value_raw_length` is inflated in its descriptor, so the codes decode to fewer bytes than declared; without the fill check the raw buffer's tail is uninitialized and a varlena length is read out of it. The read must raise XX001, and survival is read from the server log's crash count, not from a fresh connection, which succeeds once crash recovery has finished. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_an_under_filled_dict_chunk_is_refused` | the column is DICT-encoded, the inflation rewrites its descriptor, the read is refused with XX001, and no backend died | + +## 148. test_native_decode_gating.py: an unprunable qual gates payload decode per vector + +The pytest twin of `test/native_decode_gating.sh` (#452 phase 2, ported for #1341), every check by the shell's name. A leading-wildcard LIKE is no skip predicate, so phase 2 evaluates the qual per 1024-row vector between the decode passes, and a vector no row passes is never decoded for its payload columns; "Columnar Vector Decodes" makes the saving exact arithmetic. The filter counters stay exact: a rejection is counted once, inside a surviving vector too, and a deleted row is not counted. The width gate is set off for the session. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_an_unprunable_qual_gates_payload_decode` | the premises (no skip predicate, the late-materialization path); the control decodes all 192 (vector, column) pairs, a zero-match LIKE 32 and the needle 37; every filter counter is exact, including rejections inside the surviving vector; the answers match heap | +| `test_deleted_rows_across_several_groups` | with deletes over several groups the answers match heap, and Rows Removed by Filter counts only live rows | + +## 149. test_native_decode_gate_width.py: decode gating runs only on a projection wide enough to pay + +The pytest twin of `test/native_decode_gate_width.sh` (#595, ported for #1341), every check by the shell's name. Phase-2 gating runs only when the projection has at least `pgcolumnar.qual_skipvec_min_payload_cols` non-qual columns (default 20). The policy is pinned by moving the threshold across a fixed projection: a 24-payload `SELECT *` gates at the default, stops above 24 and gates again at exactly 24, and a three-column projection gates only once the setting is 0. The gate is output-invariant: the answers match heap with it on or off. The setting is held per session, where the shell sets it per database. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_the_width_gate` | the premises, including that the two rejected probe settings are undefined (42704); the decode counts at the default, with the gate off, above the width and at it; and correctness against heap with the gate on and off | + +## 150. test_pg_dump_roundtrip.py: a columnar table survives pg_dump and restore + +The pytest twin of `test/pg_dump_roundtrip.sh` (ported for #1341), every check by the shell's name. A source database holds a columnar table with data, an option, an index and a projection, and a second table whose declared `sort_by` names columns after a dropped one (#288). It is dumped as plain SQL and as a custom archive and restored into fresh databases, where the rows, the access method, the option, the `sort_by` and the projection declaration survive, and `rebuild_projections()` recovers the projection whose storage pg_dump cannot carry (#266). The point lookup must take the restored index, and the restored `sort_by` must leave the rows in its order. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_dropping_a_projection_forgets_its_declaration` | a dropped projection's declaration goes with it, so a rebuild builds nothing | +| `test_a_plain_dump_round_trips` | pg_dump and psql exit 0, and the restored database passes every survival arm | +| `test_a_custom_archive_round_trips` | the same through `pg_dump -Fc` and `pg_restore --exit-on-error` | + +## 151. test_pg_upgrade.py: a columnar table survives pg_upgrade + +The pytest twin of `test/pg_upgrade.sh` (ported for #1341), every check by the shell's name. A cluster on an older major gets a columnar table with non-default options, deletes and an index; it is upgraded with pg_upgrade into an empty cluster on the new major, and the row count, the content byte for byte, the options, the index and writability must all survive. The index arm runs with the columnar scan off and must take the index: right after pg_upgrade there are no planner statistics, the planner took the columnar scan for the range, and the arm passed with the index dropped. + +Like the shell it needs TWO majors: the new one is `--pg-config`, the old one `PGC_UPGRADE_FROM_PG_CONFIG`, and the test builds and installs the extension for the old one from a copy of the tree. The matrix runs one major at a time, so there it declines, and `expected_unrunnable.txt` lists it for every major. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_pg_upgrade_preserves_a_columnar_table` | pg_upgrade exits 0, and the upgraded table keeps its rows, content, options and index, and takes writes | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index a6a77ccba..22f9726d0 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -747,4 +747,22 @@ guard_tests 438 # seven tests, by collection over the cluster files on this tree. # cluster_tests 653 -> 660 for #1341 item 29's partial twins: test_inheritance.py (2), # test_parquet_import.py (1), test_phase5.py (1) and test_drop_cleanup.py (3), by collection. -cluster_tests 660 +# cluster_tests 660 -> 661 for #1341 item 66's test_native_upgrade_converge.py, one test, +# by collection over the cluster files on this tree. +# cluster_tests 661 -> 666 for #1341's test_fk_referencing.py, the twin of fk_referencing.sh: +# five tests, by collection over the cluster files on this tree. +# cluster_tests 666 -> 672 for #1341's test_truncate_cleanup.py, the twin of truncate_cleanup.sh: +# six tests, by collection over the cluster files on this tree. +# cluster_tests 672 -> 673 for #1341's test_native_dml.py, the twin of native_dml.sh: one test, +# by collection over the cluster files on this tree. +# cluster_tests 673 -> 674 for #1341's test_native_dict_underfill.py, the twin of +# native_dict_underfill.sh: one test, by collection over the cluster files on this tree. +# cluster_tests 674 -> 676 for #1341's test_native_decode_gating.py, the twin of +# native_decode_gating.sh: two tests, by collection over the cluster files on this tree. +# cluster_tests 676 -> 677 for #1341's test_native_decode_gate_width.py, the twin of +# native_decode_gate_width.sh: one test, by collection over the cluster files on this tree. +# cluster_tests 677 -> 680 for #1341's test_pg_dump_roundtrip.py, the twin of pg_dump_roundtrip.sh: +# three tests, by collection over the cluster files on this tree. +# cluster_tests 680 -> 681 for #1341's test_pg_upgrade.py, the twin of pg_upgrade.sh: +# one test, by collection over the cluster files on this tree. +cluster_tests 681 diff --git a/test/pytest/expected_unrunnable.txt b/test/pytest/expected_unrunnable.txt index 135205390..3259220b0 100644 --- a/test/pytest/expected_unrunnable.txt +++ b/test/pytest/expected_unrunnable.txt @@ -78,3 +78,12 @@ 16 test_inheritance.py::test_a_partitioned_parent_is_refused_and_its_leaf_is_not 15 test_parquet_import.py::test_a_partitioned_parent_is_refused 16 test_parquet_import.py::test_a_partitioned_parent_is_refused + +# Every major: pg_upgrade needs a SECOND, older major, named by +# PGC_UPGRADE_FROM_PG_CONFIG, and the matrix runs one major at a time. A run that +# sets it is an upgrade run, not a matrix run, and this file would refuse the row. +15 test_pg_upgrade.py::test_pg_upgrade_preserves_a_columnar_table +16 test_pg_upgrade.py::test_pg_upgrade_preserves_a_columnar_table +17 test_pg_upgrade.py::test_pg_upgrade_preserves_a_columnar_table +18 test_pg_upgrade.py::test_pg_upgrade_preserves_a_columnar_table +19 test_pg_upgrade.py::test_pg_upgrade_preserves_a_columnar_table diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 4140fbf32..8437604a1 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -106,6 +106,7 @@ "differential", "encode_post_codec", "entry_point_privilege", + "fk_referencing", "format_downgrade", "hilbert_cluster", "hilbert_locality", @@ -119,8 +120,12 @@ "maintenance_rollback", "native_backend_crash", "native_chunk_length_bound", + "native_decode_gate_width", + "native_decode_gating", "native_delete_vector_index", "native_delete_visibility_paths", + "native_dict_underfill", + "native_dml", "native_fetch_coalesce", "native_groupagg", "native_index_fetch_stripe_cost", @@ -139,6 +144,8 @@ "partition_lifecycle", "partition_rows", "partman_columnar", + "pg_dump_roundtrip", + "pg_upgrade", "prepared_across_sessions", "prepared_writes", "projection_drop_column", @@ -166,6 +173,7 @@ "tid_range_scan", "tier_partition", "timeseries_recipes", + "truncate_cleanup", "ttl_expire", "type_matrix", "unique_conc", @@ -179,6 +187,10 @@ # about the PORT rather than a licence: the standing arm does not grade it, so the # reason is the only thing standing between a declared gap and a forgotten one. INCOMPLETE = { + # #1341 item 66 ported the arms it added: the upgrade window, this library over + # the previous release's catalog before ALTER EXTENSION UPDATE. + "native_upgrade_converge": "the upgrade-window arms are ported; the fixture-blob, " + "staging and catalog-convergence arms are not", # #1341 item 29 ported the arms four fixes added with shell arms only. Each # pair below holds that fix's arms and nothing else of its shell suite. "inheritance": "the #1259 arms (a partitioned columnar parent is refused with " diff --git a/test/pytest/test_fk_referencing.py b/test/pytest/test_fk_referencing.py new file mode 100644 index 000000000..3e7a60274 --- /dev/null +++ b/test/pytest/test_fk_referencing.py @@ -0,0 +1,158 @@ +"""A foreign key that references a columnar table is refused where it is chosen +(#1341). The pytest twin of `test/fk_referencing.sh`, every check by the shell's name. + +Referential integrity reads the referenced row with FOR KEY SHARE, and row locking is +not implemented for columnar tables, so a constraint referencing one would make every +insert into the referencing table fail. It is refused at DDL instead, by either route +into that state: creating the constraint against a columnar table, or converting a +referenced table to columnar. Only the REFERENCED side is refused: a columnar table may +reference a heap table, and the controls hold everything else steady -- other +constraints, heap-to-heap keys, conversions of an unreferenced table, of the +referencing side, back to heap, and of the referenced table to a row store. +""" +import psycopg + + +def _err(conn, sql): + """-> '' when sql succeeds, else the error's message, detail and hint, joined.""" + try: + with conn.cursor() as cur: + cur.execute(sql) + return "" + except psycopg.Error as exc: + d = exc.diag + return " ".join(x for x in (d.message_primary, d.message_detail, d.message_hint) if x) or str(exc) + + +def _has(text, needle): + return "yes" if needle in text else f"no ({text})" + + +def _ok(text): + return "yes" if not text else f"no ({text})" + + +def _one(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetchone()[0] + + +def _run(conn, *sqls): + with conn.cursor() as cur: + for sql in sqls: + cur.execute(sql) + + +def test_the_referenced_side_is_refused_at_ddl(pgc_conn, expect): + conn = pgc_conn + _run(conn, "CREATE TABLE fk_parent (id int PRIMARY KEY) USING pgcolumnar", + "INSERT INTO fk_parent VALUES (1), (2)") + err = _err(conn, "CREATE TABLE fk_child (id int REFERENCES fk_parent(id))") + expect.text(_has(err, "cannot create a foreign key referencing columnar table"), "yes", + "a foreign key referencing a columnar table is refused") + expect.text(_has(err, "row locking is not supported"), "yes", "and the error says why") + expect.num(_one(conn, "SELECT count(*) FROM pg_class WHERE relname = 'fk_child' " + "AND relnamespace = current_schema()::regnamespace"), 0, + "the refused table was not left behind") + + _run(conn, "CREATE TABLE fk_child2 (id int)") + err2 = _err(conn, "ALTER TABLE fk_child2 ADD CONSTRAINT fk_c2 FOREIGN KEY (id) REFERENCES fk_parent(id)") + expect.text(_has(err2, "cannot create a foreign key referencing columnar table"), "yes", + "ALTER TABLE ADD CONSTRAINT is refused by the same rule") + + +def test_the_referencing_side_still_works(pgc_conn, expect): + conn = pgc_conn + _run(conn, "CREATE TABLE fk_h_parent (id int PRIMARY KEY)", "INSERT INTO fk_h_parent VALUES (1), (2), (3)") + expect.text(_ok(_err(conn, "CREATE TABLE fk_c_child (id int REFERENCES fk_h_parent(id), v text) " + "USING pgcolumnar")), "yes", + "a columnar table may reference a heap table") + _err(conn, "INSERT INTO fk_c_child VALUES (1, 'a'), (2, 'b')") + expect.num(_one(conn, "SELECT count(*) FROM fk_c_child"), 2, "and rows that satisfy it are accepted") + expect.text(_has(_err(conn, "INSERT INTO fk_c_child VALUES (99, 'nope')"), "violates foreign key constraint"), + "yes", "a row that violates it is still rejected") + + +def test_constraints_that_need_no_row_lock_are_unaffected(pgc_conn, expect): + conn = pgc_conn + expect.text(_ok(_err(conn, "CREATE TABLE fk_other (id int PRIMARY KEY, k int UNIQUE, v int CHECK (v > 0), " + "w int NOT NULL) USING pgcolumnar")), "yes", + "primary key, unique, check and not-null on a columnar table are unaffected") + _err(conn, "INSERT INTO fk_other VALUES (1, 10, 5, 7)") + expect.num(_one(conn, "SELECT count(*) FROM fk_other"), 1, "and they still work") + expect.text(_has(_err(conn, "INSERT INTO fk_other VALUES (1, 11, 5, 7)"), "duplicate key"), "yes", + "the primary key is still enforced") + _run(conn, "CREATE TABLE fk_hh_parent (id int PRIMARY KEY)") + expect.text(_ok(_err(conn, "CREATE TABLE fk_hh_child (id int REFERENCES fk_hh_parent(id))")), "yes", + "a foreign key between two heap tables is unaffected") + + +def test_converting_a_referenced_table_is_refused_and_nothing_else_is(pgc_conn, expect): + conn = pgc_conn + _run(conn, "CREATE TABLE fk_conv_parent (id int PRIMARY KEY)", + "CREATE TABLE fk_conv_child (id int REFERENCES fk_conv_parent(id))", + "INSERT INTO fk_conv_parent VALUES (1)", "INSERT INTO fk_conv_child VALUES (1)") + conv = _err(conn, "ALTER TABLE fk_conv_parent SET ACCESS METHOD pgcolumnar") + expect.text(_has(conv, "referenced by a foreign key"), "yes", + "converting an FK-referenced table to columnar is refused") + expect.text(_has(conv, "row locking is not supported"), "yes", "and the error names the constraint") + expect.text(_one(conn, "SELECT am.amname::text FROM pg_class c JOIN pg_am am ON am.oid = c.relam " + "WHERE c.oid = 'fk_conv_parent'::regclass"), "heap", "the parent is still a row store") + _err(conn, "INSERT INTO fk_conv_parent VALUES (2)") + expect.text(_ok(_err(conn, "INSERT INTO fk_conv_child VALUES (2)")), "yes", + "the child is still writable afterwards") + expect.text(_has(_err(conn, "SELECT pgcolumnar.alter_table_set_access_method('fk_conv_parent', 'pgcolumnar')"), + "referenced by a foreign key"), "yes", "the conversion helper is refused identically") + + # The over-rejection controls: a rule keyed on the wrong side, or firing for any + # ALTER TABLE or any target, passes every arm above and fails one of these. + _run(conn, "CREATE TABLE fk_conv_plain (id int PRIMARY KEY)", "INSERT INTO fk_conv_plain VALUES (1)") + expect.text(_ok(_err(conn, "ALTER TABLE fk_conv_plain SET ACCESS METHOD pgcolumnar")), "yes", + "a table with no foreign key still converts") + expect.text(_ok(_err(conn, "ALTER TABLE fk_conv_child SET ACCESS METHOD pgcolumnar")), "yes", + "converting the referencing side is still allowed") + expect.text(_ok(_err(conn, "ALTER TABLE fk_conv_parent ADD COLUMN note text")), "yes", + "an unrelated ALTER TABLE on a referenced table is unaffected") + expect.text(_ok(_err(conn, "ALTER TABLE fk_conv_plain SET ACCESS METHOD heap")), "yes", + "converting back to heap is unaffected") + # The control above has no foreign key; this one is the referenced table itself. + expect.text(_ok(_err(conn, "ALTER TABLE fk_conv_parent SET ACCESS METHOD heap")), "yes", + "an FK-referenced table may still be set to a row store") + + +def test_a_partitioned_referenced_parent(pgc_conn, expect): + """Setting a partitioned parent's access method chooses what every later partition + inherits, and core clones the foreign key to each, so it is refused too (#201).""" + conn = pgc_conn + _run(conn, "CREATE TABLE fk_pp (id int PRIMARY KEY) PARTITION BY RANGE (id)", + "CREATE TABLE fk_pp1 PARTITION OF fk_pp FOR VALUES FROM (1) TO (100)", + "CREATE TABLE fk_pc (id int REFERENCES fk_pp(id))") + expect.text(_has(_err(conn, "CREATE TABLE fk_pp2 PARTITION OF fk_pp FOR VALUES FROM (100) TO (200) " + "USING pgcolumnar"), "cannot create a foreign key referencing columnar table"), + "yes", "a columnar partition of an FK-referenced parent is refused") + expect.text(_has(_err(conn, "ALTER TABLE fk_pp SET ACCESS METHOD pgcolumnar"), "cannot convert table"), "yes", + "and setting the parent's access method is refused as well") + # "did it become columnar", not "is it heap": a partitioned table never given an + # access method has relam = 0, which joins to no pg_am row. + expect.num(_one(conn, "SELECT count(*) FROM pg_class c JOIN pg_am a ON a.oid = c.relam " + "WHERE c.oid = 'fk_pp'::regclass AND a.amname = 'pgcolumnar'"), 0, + "so the parent did not become columnar") + expect.num(0 if not _err(conn, "CREATE TABLE fk_pp3 PARTITION OF fk_pp FOR VALUES FROM (200) TO (300)") else 1, + 0, "and a partition can still be created the ordinary way") + + _run(conn, "CREATE TABLE fk_free (id int) PARTITION BY RANGE (id)") + major = int(_one(conn, "SHOW server_version_num")) // 10000 + if major >= 17: + expect.num(0 if not _err(conn, "ALTER TABLE fk_free SET ACCESS METHOD pgcolumnar") else 1, 0, + "a partitioned table with no foreign key still converts") + _err(conn, "CREATE TABLE fk_free1 PARTITION OF fk_free FOR VALUES FROM (1) TO (100)") + expect.text(_one(conn, "SELECT amname::text FROM pg_class c JOIN pg_am a ON a.oid = c.relam " + "WHERE c.oid = 'fk_free1'::regclass"), "pgcolumnar", + "and its partitions inherit the columnar access method") + else: + # Before 17 core refuses SET ACCESS METHOD on every partitioned table, so the + # refusal must be core's, by core's message. + expect.text(_has(_err(conn, "ALTER TABLE fk_free SET ACCESS METHOD pgcolumnar"), + "cannot change access method of a partitioned table"), "yes", + "before 17, core refuses this for every partitioned table") diff --git a/test/pytest/test_native_decode_gate_width.py b/test/pytest/test_native_decode_gate_width.py new file mode 100644 index 000000000..56e3620eb --- /dev/null +++ b/test/pytest/test_native_decode_gate_width.py @@ -0,0 +1,185 @@ +"""Phase-2 decode gating runs only on a projection wide enough to pay for it (#595, +#1341). The pytest twin of `test/native_decode_gate_width.sh`, every check by the +shell's name. + +#452 phase 2 evaluates an unprunable qual per vector so a vector no row passes skips +its payload decode; it pays only when there is payload to spare. So it runs only when +the projection has at least pgcolumnar.qual_skipvec_min_payload_cols non-qual columns +(default 20). The policy is pinned by moving the threshold across a fixed projection: +a wide SELECT * (24 payload columns) gates at the default and stops once the setting is +above 24, and gates again at exactly 24 (the >= boundary); a narrow three-column +projection does not gate at the default and does once the setting is 0. "Columnar +Vector Decodes" is the observable: VECTORS when gated, every column of every vector +when not. The shell sets the threshold per database because each helper opens a new +connection; this side holds one connection and sets it for the session. +""" +import re + +import psycopg + +ROWS = 32768 +VECTORS = 32 +PCOLS = 24 +NCOLS_WIDE = PCOLS + 1 +NARROW_NCOLS = 3 +SURVIVING_VECTORS = 1 +DEFAULT_GUC = 20 + +COLDEFS = ", ".join(f"p{i} int" for i in range(1, PCOLS + 1)) +COLVALS = ", ".join(f"g + {i - 1}" for i in range(1, PCOLS + 1)) +WIDE_NOMATCH = "SELECT * FROM nw WHERE tag LIKE '%zzzzz%'" +WIDE_NEEDLE = "SELECT * FROM nw WHERE tag LIKE '%needle%'" +WIDE_CONTROL = "SELECT * FROM nw WHERE tag LIKE '%row%'" +NARROW_NOMATCH = "SELECT tag, p1, p2 FROM nw WHERE tag LIKE '%zzzzz%'" +NARROW_CONTROL = "SELECT tag, p1, p2 FROM nw WHERE tag LIKE '%row%'" +NARROW_NEEDLE = "SELECT tag, p1, p2 FROM nw WHERE tag LIKE '%needle%'" + + +def _one(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetchone()[0] + + +def _rows(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetchall() + + +def _show_state(conn, name): + """-> the SQLSTATE SHOW raised, or 'shown'. 42704 is undefined_object.""" + try: + _one(conn, f"SHOW {name}") + return "shown" + except psycopg.Error as exc: + return exc.sqlstate or "no SQLSTATE" + + +def _plan(conn, sql): + return "\n".join(r[0] for r in _rows(conn, f"EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) {sql}")) + + +def _counter(plan, label): + for line in plan.splitlines(): + if label in line: + m = re.search(r"\d+", line[line.index(label) + len(label):]) + return int(m.group(0)) if m else None + return None + + +def _has(plan, label): + return "yes" if label in plan else "no" + + +def _set_gate(conn, v): + with conn.cursor() as cur: + cur.execute(f"SET pgcolumnar.qual_skipvec_min_payload_cols = {v}") + + +def _reset_gate(conn): + with conn.cursor() as cur: + cur.execute("RESET pgcolumnar.qual_skipvec_min_payload_cols") + + +def _gate_now(conn): + return int(_one(conn, "SHOW pgcolumnar.qual_skipvec_min_payload_cols")) + + +def test_the_width_gate(pgc_conn, expect): + conn = pgc_conn + gen = (f"SELECT 'row' || CASE WHEN g <= 1024 THEN 'needle' ELSE 'miss' END, {COLVALS} " + f"FROM generate_series(1, {ROWS}) g") + with conn.cursor() as cur: + cur.execute(f"CREATE TABLE hw (tag text, {COLDEFS})") + cur.execute(f"CREATE TABLE nw (tag text, {COLDEFS}) USING pgcolumnar") + cur.execute("SELECT pgcolumnar.set_options('nw', stripe_row_limit => 65536, chunk_group_row_limit => 1024)") + cur.execute(f"INSERT INTO hw {gen}") + cur.execute(f"INSERT INTO nw {gen}") + + expect.num(_gate_now(conn), DEFAULT_GUC, + f"premise: the width-gate GUC exists and defaults to the shipped {DEFAULT_GUC}") + expect.text(_show_state(conn, "pgcolumnar.qual_skipvec_probe_vecs"), "42704", + "premise: the rejected probe GUC qual_skipvec_probe_vecs was NOT shipped") + expect.text(_show_state(conn, "pgcolumnar.qual_skipvec_min_skip_pct"), "42704", + "premise: the rejected probe GUC qual_skipvec_min_skip_pct was NOT shipped") + expect.num(_one(conn, "SELECT count(*) FROM pgcolumnar.row_group " + "WHERE storage_id = pgcolumnar.get_storage_id('nw')"), 1, + "premise: one row group, so every count below is one group's") + + wide_needle = _plan(conn, WIDE_NEEDLE) + narrow_needle = _plan(conn, NARROW_NEEDLE) + narrow_nomatch = _plan(conn, NARROW_NOMATCH) + expect.text(_has(wide_needle, "Columnar Projected Columns"), "yes", + "premise: the wide plan is a columnar custom scan with projection") + expect.text(_has(narrow_nomatch, "Columnar Projected Columns"), "yes", + "premise: the narrow plan is a columnar custom scan with projection") + expect.num(_counter(wide_needle, "Columnar Usable Skip Predicates"), 0, + "premise: the LIKE is unprunable (no usable skip predicate)") + expect.num(_counter(wide_needle, "Columnar Chunk Groups Removed by Filter"), 0, + "premise: no whole row group is pruned") + expect.num(_counter(wide_needle, "Columnar Vectors Ruled Out by Value"), 0, + "premise: no vector is ruled out by VALUE (that is 1b-ii's path, not phase 2)") + expect.text(_has(wide_needle, "Columnar Rows Filtered Before Materialization"), "yes", + "premise: the wide query takes the late-materialization path") + expect.text(_has(narrow_needle, "Columnar Rows Filtered Before Materialization"), "yes", + "premise: the narrow query takes the late-materialization path") + expect.text(_has(narrow_nomatch, "Columnar Vector Decodes"), "yes", + "premise: EXPLAIN reports Columnar Vector Decodes (the observable)") + expect.num(_one(conn, "SELECT count(*) FROM nw WHERE tag LIKE '%needle%'"), 1024, + "premise: the needle matches exactly one vector's worth of rows") + expect.num(_one(conn, "SELECT count(*) FROM nw WHERE tag LIKE '%zzzzz%'"), 0, + "premise: the nomatch predicate matches no row") + expect.num(_one(conn, "SELECT count(*) FROM nw WHERE tag LIKE '%row%'"), ROWS, + "premise: the control matches every row") + + # arm 1: at the shipped default, wide gates and narrow does not + expect.num(_gate_now(conn), DEFAULT_GUC, "premise: still at the default before arm 1") + wide_nomatch = _plan(conn, WIDE_NOMATCH) + expect.num(_counter(wide_nomatch, "Columnar Vector Decodes"), VECTORS, + "wide, default: a zero-match LIKE decodes the qual column and NO payload -> gated") + expect.num(_counter(wide_nomatch, "Columnar Vectors Skipped"), VECTORS, + "wide, default: and reports the payload vectors skipped") + wide_needle_d = _plan(conn, WIDE_NEEDLE) + expect.num(_counter(wide_needle_d, "Columnar Vector Decodes"), VECTORS + (NCOLS_WIDE - 1) * SURVIVING_VECTORS, + "wide, default: a one-vector LIKE decodes the qual column plus payload for that vector only") + expect.num(_counter(wide_needle_d, "Columnar Vectors Skipped"), VECTORS - SURVIVING_VECTORS, + "wide, default: skips exactly the vectors holding no match") + expect.num(_counter(_plan(conn, NARROW_NOMATCH), "Columnar Vector Decodes"), VECTORS * NARROW_NCOLS, + "narrow, default: a zero-match LIKE decodes EVERY projected column of every vector -> NOT gated") + + # arm 2: the gate off, and the same narrow projection gates + _set_gate(conn, 0) + expect.num(_gate_now(conn), 0, "premise: the width gate is now disabled (0)") + expect.num(_counter(_plan(conn, NARROW_NOMATCH), "Columnar Vector Decodes"), VECTORS, + "narrow, gate off (GUC=0): the SAME narrow projection now gates -> qual column alone") + expect.num(_counter(_plan(conn, WIDE_CONTROL), "Columnar Vector Decodes"), VECTORS * NCOLS_WIDE, + "control, gate forced on: a matches-everything wide qual skips no vector (full decode)") + expect.num(_counter(_plan(conn, NARROW_CONTROL), "Columnar Vector Decodes"), VECTORS * NARROW_NCOLS, + "control, gate forced on: a matches-everything narrow qual skips no vector (full decode)") + + # arm 3: above the wide projection's width it stops; at exactly its width it gates + _set_gate(conn, NCOLS_WIDE) + expect.num(_gate_now(conn), NCOLS_WIDE, "premise: the width threshold is now above the projection width") + expect.num(_counter(_plan(conn, WIDE_NOMATCH), "Columnar Vector Decodes"), VECTORS * NCOLS_WIDE, + "wide, gate raised above width: the SAME wide SELECT * stops gating -> full decode") + _set_gate(conn, PCOLS) + expect.num(_gate_now(conn), PCOLS, "premise: the width threshold now equals the projection width") + expect.num(_counter(_plan(conn, WIDE_NOMATCH), "Columnar Vector Decodes"), VECTORS, + "wide, gate AT the width (>= boundary): gates again -> qual column alone") + + # correctness: the gate never changes a result + _reset_gate(conn) + expect.num(_gate_now(conn), DEFAULT_GUC, "premise: back to the shipped default") + gated = _rows(conn, WIDE_NEEDLE) + expect.row_set(gated, _rows(conn, "SELECT * FROM hw WHERE tag LIKE '%needle%'"), + "correctness: wide needle, gate ON, matches the heap oracle") + _set_gate(conn, NCOLS_WIDE) + expect.num(_gate_now(conn), NCOLS_WIDE, "premise: gate suppressed for the invariance arm") + expect.row_set(_rows(conn, WIDE_NEEDLE), gated, + "correctness: wide needle, gate OFF, returns the identical rows (output-invariant)") + expect.row_set(_rows(conn, NARROW_NEEDLE), _rows(conn, "SELECT tag, p1, p2 FROM hw WHERE tag LIKE '%needle%'"), + "correctness: narrow needle matches the heap either way") + expect.row_set(_rows(conn, "SELECT * FROM nw"), _rows(conn, "SELECT * FROM hw"), + "correctness: the full scan matches the heap") + _reset_gate(conn) diff --git a/test/pytest/test_native_decode_gating.py b/test/pytest/test_native_decode_gating.py new file mode 100644 index 000000000..e2a6e5148 --- /dev/null +++ b/test/pytest/test_native_decode_gating.py @@ -0,0 +1,158 @@ +"""An unprunable qual gates payload-column decode per vector (#452 phase 2, #1341). +The pytest twin of `test/native_decode_gating.sh`, every check by the shell's name. + +A leading-wildcard LIKE is never a skip predicate, so the reader cannot rule a vector +out by value; phase 2 evaluates the executor qual per 1024-row vector between the two +decode passes, and a vector no row passes is never decoded for its payload columns. +"Columnar Vector Decodes" counts (vector, column) decodes, so the saving is exact +arithmetic. The filter counters must stay exact too: a rejection is counted once, +inside a surviving vector as well as a skipped one, and a deleted row is not counted at +all. The width gate (#595) is set off for this session, as the shell pins it off for +its cluster, so the five-column projection still gates. +""" +import re + +ROWS = 32768 +VECTORS = 32 # ROWS / 1024 +NCOLS = 6 # tag plus five payload columns +SURVIVING_VECTORS = 1 # the needle lives only in the first vector +MATCHES = 1024 +MROWS = 49152 + + +def _one(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetchone()[0] + + +def _rows(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetchall() + + +def _run(conn, *sqls): + with conn.cursor() as cur: + for sql in sqls: + cur.execute(sql) + + +def _plan(conn, sql): + return "\n".join(r[0] for r in _rows(conn, f"EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) {sql}")) + + +def _counter(plan, label): + """-> the first integer on the plan's line holding label, or None.""" + for line in plan.splitlines(): + if label in line: + m = re.search(r"\d+", line[line.index(label) + len(label):]) or re.search(r"\d+", line) + return int(m.group(0)) if m else None + return None + + +def _has(plan, label): + return "yes" if label in plan else "no" + + +NEEDLEQ = "SELECT * FROM n WHERE tag LIKE '%needle%'" +NOMATCHQ = "SELECT * FROM n WHERE tag LIKE '%zzzzz%'" +CONTROLQ = "SELECT * FROM n WHERE tag LIKE '%row%'" +HALFQ = "SELECT * FROM n WHERE tag LIKE '%needle%' AND p1 % 2 = 0" + + +def test_an_unprunable_qual_gates_payload_decode(pgc_conn, expect): + conn = pgc_conn + gen = (f"SELECT 'row' || CASE WHEN g <= 1024 THEN 'needle' ELSE 'miss' END, " + f"g, g + 1, g + 2, g + 3, g + 4 FROM generate_series(1, {ROWS}) g") + _run(conn, "SET pgcolumnar.qual_skipvec_min_payload_cols = 0", + "CREATE TABLE h (tag text, p1 int, p2 int, p3 int, p4 int, p5 int)", + "CREATE TABLE n (tag text, p1 int, p2 int, p3 int, p4 int, p5 int) USING pgcolumnar", + "SELECT pgcolumnar.set_options('n', stripe_row_limit => 65536, chunk_group_row_limit => 1024)", + f"INSERT INTO h {gen}", f"INSERT INTO n {gen}") + needle, nomatch, control = _plan(conn, NEEDLEQ), _plan(conn, NOMATCHQ), _plan(conn, CONTROLQ) + + expect.text(_has(needle, "Columnar Projected Columns"), "yes", "premise: the needle plan is a columnar custom scan") + expect.num(_one(conn, "SELECT count(*) FROM pgcolumnar.row_group " + "WHERE storage_id = pgcolumnar.get_storage_id('n')"), 1, + "premise: one row group, so every count below is one group's") + expect.text(_one(conn, "SHOW pgcolumnar.qual_skipvec_min_payload_cols"), "0", + "premise: the width gate is pinned off, so this narrow projection still gates") + expect.num(_counter(needle, "Columnar Usable Skip Predicates"), 0, + "premise: the reader has no usable skip predicate for this LIKE") + expect.num(_counter(needle, "Columnar Chunk Groups Removed by Filter"), 0, "premise: no whole row group is pruned") + expect.num(_counter(needle, "Columnar Vectors Ruled Out by Value"), 0, + "premise: no vector is ruled out by VALUE either (that is 1b-ii's path)") + expect.text(_has(needle, "Columnar Rows Filtered Before Materialization"), "yes", + "premise: the needle query takes the late-materialization path") + expect.num(_one(conn, "SELECT count(*) FROM n WHERE tag LIKE '%needle%'"), MATCHES, + "premise: the needle really matches one vector's worth of rows") + expect.num(_one(conn, "SELECT count(*) FROM n WHERE tag LIKE '%zzzzz%'"), 0, + "premise: the nomatch predicate matches no row") + expect.num(_one(conn, "SELECT count(*) FROM n WHERE tag LIKE '%row%'"), ROWS, + "premise: the control matches every row") + + expect.text(_has(control, "Columnar Vector Decodes"), "yes", "EXPLAIN reports Columnar Vector Decodes") + expect.num(_counter(control, "Columnar Vector Decodes"), VECTORS * NCOLS, + "the control decodes every column of every vector") + expect.num(_counter(nomatch, "Columnar Vector Decodes"), VECTORS, + "a zero-matching LIKE decodes the qual column and no payload column") + expect.num(_counter(needle, "Columnar Vector Decodes"), VECTORS + (NCOLS - 1) * SURVIVING_VECTORS, + "a one-vector LIKE decodes the qual column plus payload for that vector only") + + expect.num(_counter(needle, "Rows Removed by Filter"), ROWS - MATCHES, + "the needle removes exactly the non-matching rows, counted once") + expect.num(_counter(needle, "Columnar Rows Filtered Before Materialization"), ROWS - MATCHES, + "and reports them all filtered before materialization") + expect.num(_counter(needle, "Columnar Vectors Skipped"), VECTORS - SURVIVING_VECTORS, + "the needle skips exactly the vectors holding no match") + expect.num(_counter(nomatch, "Rows Removed by Filter"), ROWS, + "the zero-match query removes every row by filter, counted once") + expect.num(_counter(nomatch, "Columnar Vectors Skipped"), VECTORS, "and skips every vector") + + # Every row of the needle's surviving vector matches, so a producer that counted + # its re-test's rejections twice reads right above; half of vector 0 passes here. + expect.text(_one(conn, "SELECT count(*) || '|' || count(DISTINCT (p1 - 1) / 1024) FROM n " + "WHERE tag LIKE '%needle%' AND p1 % 2 = 0"), "512|1", + "premise: the half-needle survives in one vector, half of its rows") + expect.num(_counter(_plan(conn, HALFQ), "Rows Removed by Filter"), ROWS - 512, + "rejections inside a surviving vector are counted once") + + expect.row_set(_rows(conn, NEEDLEQ), _rows(conn, "SELECT * FROM h WHERE tag LIKE '%needle%'"), + "the needle query returns exactly the matching rows") + expect.num(_one(conn, "SELECT count(*) FROM n WHERE tag LIKE '%zzzzz%'"), 0, "the nomatch query returns nothing") + expect.rows(_rows(conn, "SELECT count(*), sum(p1), sum(p5) FROM n WHERE tag LIKE '%row%'"), + _rows(conn, "SELECT count(*), sum(p1), sum(p5) FROM h WHERE tag LIKE '%row%'"), + "the control returns every row, values intact") + + +def test_deleted_rows_across_several_groups(pgc_conn, expect): + """The evaluator reads the CURRENT group's delete mask; an earlier version read the + previous group's, freed one load earlier. It needs more than one group, deletes, + and the unprunable qual at once. A deleted row is not counted as filtered.""" + conn = pgc_conn + mgen = (f"SELECT 'x' || CASE WHEN g % 4096 = 7 THEN 'needle' ELSE 'miss' END || g::text, g, g * 2 " + f"FROM generate_series(1, {MROWS}) g") + _run(conn, "SET pgcolumnar.qual_skipvec_min_payload_cols = 0", + "CREATE TABLE hm (tag text, p1 int, p2 int)", + "CREATE TABLE m (tag text, p1 int, p2 int) USING pgcolumnar", + "SELECT pgcolumnar.set_options('m', stripe_row_limit => 8192, chunk_group_row_limit => 1024)", + f"INSERT INTO hm {mgen}", f"INSERT INTO m {mgen}", + "DELETE FROM hm WHERE p1 % 17 = 0", + # the fixture's DELETE must not depend on the gating under test + "SET pgcolumnar.qual_skipvec_min_payload_cols = 100000", "DELETE FROM m WHERE p1 % 17 = 0", + "SET pgcolumnar.qual_skipvec_min_payload_cols = 0") + expect.text(str(_one(conn, "SELECT count(*) > 1 FROM pgcolumnar.row_group " + "WHERE storage_id = pgcolumnar.get_storage_id('m')")), "True", + "premise: the columnar table spans more than one row group") + # count(*), not a count WHERE p1 % 17 = 0: that WHERE would run through the gating + expect.num(_one(conn, "SELECT count(*) FROM m"), MROWS - MROWS // 17, + "premise: rows really were deleted (so the mask is non-NULL)") + expect.row_set(_rows(conn, "SELECT * FROM m WHERE tag LIKE '%needle%'"), + _rows(conn, "SELECT * FROM hm WHERE tag LIKE '%needle%'"), + "deletes + multiple groups + unprunable qual: columnar matches the heap") + expect.row_set(_rows(conn, "SELECT * FROM m"), _rows(conn, "SELECT * FROM hm"), + "and the full scan across every group's mask matches too") + expect.num(_counter(_plan(conn, "SELECT * FROM m WHERE tag LIKE '%needle%'"), "Rows Removed by Filter"), + _one(conn, "SELECT count(*) FROM hm WHERE tag NOT LIKE '%needle%'"), + "rows removed by filter across deleted rows count only live rows") diff --git a/test/pytest/test_native_dict_underfill.py b/test/pytest/test_native_dict_underfill.py new file mode 100644 index 000000000..98c3f2c4f --- /dev/null +++ b/test/pytest/test_native_dict_underfill.py @@ -0,0 +1,70 @@ +"""An under-filled DICT chunk is refused cleanly, and no backend dies (#1341). The pytest +twin of `test/native_dict_underfill.sh`, every check by the shell's name. + +The test inflates a DICT chunk's value_raw_length in its encoding descriptor, so the +codes decode to fewer bytes than declared. Without the fill check the raw buffer's +tail is left uninitialized and a varlena length is read out of it. The read must raise +XX001, and the crash is read from the server log, since a fresh connection succeeds +again once crash recovery has finished. + +The post-codec encoding choice (#1132) is turned off for this session, as the shell +does for its database, so the fixture is DICT-encoded whether or not DICT beats zstd. +""" +import psycopg + + +def _crashes(pgc_cluster): + try: + return (pgc_cluster.datadir / "server.log").read_text(errors="replace").count("terminated by signal") + except OSError: + return 0 + + +def _one(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetchone()[0] + + +RAW = ("(get_byte(encoding_descriptor, 11) + (get_byte(encoding_descriptor, 12) << 8) + " + "(get_byte(encoding_descriptor, 13) << 16) + (get_byte(encoding_descriptor, 14) << 24) + 64)") + + +def test_an_under_filled_dict_chunk_is_refused(pgc_conn, pgc_cluster, expect): + conn = pgc_conn + with conn.cursor() as cur: + cur.execute("SET pgcolumnar.enable_post_codec_encoding_choice = off") + cur.execute("CREATE TABLE dt (id int, v text) USING pgcolumnar") + # low-cardinality short text: a DICT-encoded varlena column, one small chunk + cur.execute("INSERT INTO dt SELECT g, (ARRAY['aa', 'bb', 'cc', 'dd'])[1 + (g % 4)] " + "FROM generate_series(1, 300) g") + sid = _one(conn, "SELECT pgcolumnar.get_storage_id('dt')") + # column_index is 0-based: v is attribute 2, index 1; byte 6 is vector 0's encoding type + expect.num(_one(conn, f"SELECT get_byte(encoding_descriptor, 6) FROM pgcolumnar.column_chunk " + f"WHERE storage_id = {sid} AND column_index = 1 LIMIT 1"), 6, + "the text column is DICT-encoded (precondition)") + + crashes = _crashes(pgc_cluster) + # rawLen is the uint32 LE at descriptor offset 11: raise it by 64 + rewrote = _one(conn, f"""WITH u AS (UPDATE pgcolumnar.column_chunk c SET encoding_descriptor = + set_byte(set_byte(set_byte(set_byte(encoding_descriptor, + 11, {RAW} & 255), 12, ({RAW} >> 8) & 255), 13, ({RAW} >> 16) & 255), 14, ({RAW} >> 24) & 255) + WHERE c.storage_id = {sid} AND c.column_index = 1 RETURNING 1) SELECT count(*) FROM u""") + expect.num(rewrote, 1, "premise: the inflation rewrote the DICT chunk's descriptor") + + try: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as c: + c.execute(f"SET search_path TO {conn.execute('SELECT current_schema()').fetchone()[0]}") + c.execute("SELECT v FROM dt").fetchall() + rejected = "no error" + except psycopg.Error as exc: + rejected = exc.sqlstate or f"no SQLSTATE ({str(exc).splitlines()[0]})" + expect.text(rejected, "XX001", "under-filled DICT chunk is rejected with a clean error") + + try: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as c: + alive = c.execute("SELECT 1").fetchone()[0] == 1 + except psycopg.Error: + alive = False + expect.text(f"{'1' if alive else ''}|{_crashes(pgc_cluster) - crashes} backend crashes", "1|0 backend crashes", + "backend survives the under-filled DICT chunk") diff --git a/test/pytest/test_native_dml.py b/test/pytest/test_native_dml.py new file mode 100644 index 000000000..3a97bde8a --- /dev/null +++ b/test/pytest/test_native_dml.py @@ -0,0 +1,92 @@ +"""DELETE, UPDATE and MERGE on a native-format columnar table agree with a heap mirror +(#1341). The pytest twin of `test/native_dml.sh`, every check by the shell's name. + +Each statement runs on a heap table and on a columnar one with stripes of 1000 rows, +and every result is compared: the surviving rows, counts, aggregates and index lookups. +After deletes, an aggregate must not be answered from zone maps that still describe the +deleted rows. The deletes include the table's smallest and largest ids, so min and max +move and an answer that ignored the deletes differs from heap's; min and max are taken +over k too, a column the primary key does not order. An index lookup of a deleted row +must find nothing, and the lookups are asserted to take the index. +""" + + +def _one(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetchone()[0] + + +def _row(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetchone() + + +def _rows(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetchall() + + +def _run(conn, *sqls): + with conn.cursor() as cur: + for sql in sqls: + cur.execute(sql) + + +GEN = "SELECT g, (g * 10)::bigint, 'row-' || g FROM generate_series(1, 5000) g" + + +def test_delete_update_and_merge_match_heap(pgc_conn, expect): + conn = pgc_conn + _run(conn, "CREATE TABLE h (id int PRIMARY KEY, k bigint, v text)", + "CREATE TABLE n (id int PRIMARY KEY, k bigint, v text) USING pgcolumnar", + "SELECT pgcolumnar.set_options('n', stripe_row_limit => 1000)", + f"INSERT INTO h {GEN}", f"INSERT INTO n {GEN}") + + for t in ("h", "n"): + _run(conn, f"DELETE FROM {t} WHERE id % 7 = 0 OR id IN (1, 5000)") + expect.text(_one(conn, "SELECT min(id) || '|' || max(id) || '|' || min(k) || '|' || max(k) FROM h"), + "2|4999|20|49990", "premise: the deletes moved the smallest and largest id and k") + expect.num(_one(conn, "SELECT count(*) FROM n"), _one(conn, "SELECT count(*) FROM h"), "count after delete") + expect.row_set(_rows(conn, "SELECT id, k, v FROM n"), _rows(conn, "SELECT id, k, v FROM h"), + "rows after delete match heap") + expect.num(_one(conn, "SELECT sum(k) FROM n"), _one(conn, "SELECT sum(k) FROM h"), "sum(k) after delete") + expect.rows([_row(conn, "SELECT min(id), max(id), min(k), max(k) FROM n")], + [_row(conn, "SELECT min(id), max(id), min(k), max(k) FROM h")], "min/max after delete") + expect.num(_one(conn, "SELECT count(*) FROM n WHERE id % 7 = 0"), 0, "deleted rows gone") + + # id 4444 is live (4444 % 7 = 6); id 4445 was deleted (4445 % 7 = 0) + _run(conn, "SET enable_seqscan = off") + plan = "\n".join(r[0] for r in _rows(conn, "EXPLAIN (COSTS off) SELECT count(*) FROM n WHERE id = 4445")) + expect.num(sum(1 for line in plan.splitlines() if "Index" in line and "using n_pkey" in line), 1, + "premise: the lookups take the primary-key index") + expect.text(_one(conn, "SELECT v FROM n WHERE id = 4444"), _one(conn, "SELECT v FROM h WHERE id = 4444"), + "index finds live row") + expect.num(_one(conn, "SELECT count(*) FROM n WHERE id = 4445"), 0, "index skips deleted row") + _run(conn, "RESET enable_seqscan") + + for t in ("h", "n"): + _run(conn, f"UPDATE {t} SET k = k + 1 WHERE id % 5 = 0") + expect.num(_one(conn, "SELECT count(*) FROM n"), _one(conn, "SELECT count(*) FROM h"), "count after update") + expect.row_set(_rows(conn, "SELECT id, k, v FROM n"), _rows(conn, "SELECT id, k, v FROM h"), + "rows after update match heap") + expect.num(_one(conn, "SELECT sum(k) FROM n"), _one(conn, "SELECT sum(k) FROM h"), "sum(k) after update") + + import psycopg + try: + _run(conn, "INSERT INTO n VALUES (4444, 0, 'x')") + dup = "accepted" + except psycopg.Error as exc: + dup = exc.sqlstate + expect.text(dup, "23505", "PK still enforced after DML") + + _run(conn, "CREATE TABLE src (id int, k bigint)", "INSERT INTO src VALUES (1, 999), (999001, 5), (999002, 6)") + for t in ("h", "n"): + _run(conn, f"MERGE INTO {t} t USING src s ON t.id = s.id " + "WHEN MATCHED THEN UPDATE SET k = s.k " + "WHEN NOT MATCHED THEN INSERT (id, k, v) VALUES (s.id, s.k, 'm')") + expect.num(_one(conn, "SELECT count(*) FROM n"), _one(conn, "SELECT count(*) FROM h"), "count after merge") + expect.row_set(_rows(conn, "SELECT id, k, v FROM n"), _rows(conn, "SELECT id, k, v FROM h"), + "rows after merge match heap") diff --git a/test/pytest/test_native_upgrade_converge.py b/test/pytest/test_native_upgrade_converge.py new file mode 100644 index 000000000..5a2c1d54b --- /dev/null +++ b/test/pytest/test_native_upgrade_converge.py @@ -0,0 +1,126 @@ +"""The upgrade window: this library over the previous release's catalog, before ALTER +EXTENSION UPDATE (#1341 item 66). Part of the pytest twin of +`test/native_upgrade_converge.sh`: these arms only, by the shell's names, and the pair +is declared INCOMPLETE. + +docs/installation.md steps 1 and 2 put a new library in place; step 3, ALTER EXTENSION +UPDATE, brings each database's catalog up to it. Between them a database runs the new +library over the old catalog, and the library deletes from catalog tables the old one +does not have (retired_group, new in 1.0-alpha5). Reads and writes must work there; an +operation that needs a missing table must be refused cleanly, change nothing and name +step 3; and after step 3 every one of them must work on the same rows. + +The previous release is the version the shipped upgrade script to the current +default_version starts from, and its base script is staged from test/fixtures into the +extension directory for the test, as the shell does, then removed. +""" +import pathlib +import re +import shutil +import subprocess + +import psycopg +import pytest + +REPO = pathlib.Path(__file__).resolve().parents[2] + + +def _target_and_previous(): + target = re.search(r"^default_version\s*=\s*'([^']+)'", + (REPO / "pgcolumnar.control").read_text(), re.M).group(1) + prev = [p.name[len("pgcolumnar--"):-len(f"--{target}.sql")] + for p in REPO.glob(f"pgcolumnar--*--{target}.sql")] + return target, prev + + +@pytest.fixture +def window_db(pgc_cluster): + """-> (dsn, previous version) of a database at the previous release's catalog.""" + target, prev = _target_and_previous() + assert len(prev) == 1, f"one upgrade script should lead to {target}, found {prev}" + prev = prev[0] + sharedir = subprocess.run([pgc_cluster.pg_config, "--sharedir"], capture_output=True, + text=True, check=True).stdout.strip() + dst = pathlib.Path(sharedir) / "extension" / f"pgcolumnar--{prev}.sql" + backup = dst.with_name(dst.name + ".pytest-bak") + if dst.exists(): + shutil.copy2(dst, backup) + shutil.copy2(REPO / "test" / "fixtures" / f"pgcolumnar--{prev}.sql", dst) + + def admin(sql): + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as c: + c.execute(sql) + + admin("DROP DATABASE IF EXISTS pgc_upgrade_window") + admin("CREATE DATABASE pgc_upgrade_window") + dsn = pgc_cluster.dsn(dbname="pgc_upgrade_window") + try: + with psycopg.connect(dsn, autocommit=True) as c: + c.execute(f"CREATE EXTENSION pgcolumnar VERSION '{prev}'") + yield dsn, prev + finally: + admin("DROP DATABASE IF EXISTS pgc_upgrade_window WITH (FORCE)") + if backup.exists(): + shutil.move(backup, dst) + else: + dst.unlink(missing_ok=True) + + +def _one(dsn, sql): + with psycopg.connect(dsn, autocommit=True) as c: + return c.execute(sql).fetchone()[0] + + +def _refusal(dsn, sql): + """-> 'SQLSTATE|names-the-update' or '|no-hint', or 'accepted'.""" + try: + with psycopg.connect(dsn, autocommit=True) as c: + c.execute(sql) + return "accepted" + except psycopg.Error as exc: + hint = exc.diag.message_hint or "" + return f"{exc.sqlstate}|{'names-the-update' if 'ALTER EXTENSION pgcolumnar UPDATE' in hint else 'no-hint'}" + + +def _succeeds(dsn, *sqls): + try: + with psycopg.connect(dsn, autocommit=True) as c: + for sql in sqls: + c.execute(sql) + return "succeeded" + except psycopg.Error as exc: + return f"refused ({exc.sqlstate}: {str(exc).splitlines()[0]})" + + +def test_the_upgrade_window(window_db, expect): + dsn, prev = window_db + expect.text(_one(dsn, "SELECT extversion FROM pg_extension WHERE extname = 'pgcolumnar'"), prev, + "premise: the window database runs this library over the previous release's catalog") + expect.num(_one(dsn, "SELECT count(*) FROM pg_class WHERE relname = 'retired_group' " + "AND relnamespace = 'pgcolumnar'::regnamespace"), 0, + "premise: that catalog has no retired_group, which this library deletes from") + with psycopg.connect(dsn, autocommit=True) as c: + c.execute("CREATE TABLE w (a int, b text) USING pgcolumnar") + c.execute("INSERT INTO w SELECT g, 'x' || g FROM generate_series(1, 20000) g") + c.execute("DELETE FROM w WHERE a % 3 = 0") + + def rows(): + return _one(dsn, "SELECT count(*) || '|' || sum(a) FROM w") + + expect.text(rows(), "13334|133346667", "in the window, a columnar table is written, deleted from and read") + expect.text(_refusal(dsn, "DROP TABLE w"), "42P01|names-the-update", + "in the window, DROP TABLE is refused and names ALTER EXTENSION UPDATE") + expect.text(_refusal(dsn, "TRUNCATE w"), "42P01|names-the-update", + "in the window, TRUNCATE is refused and names ALTER EXTENSION UPDATE") + expect.text(_refusal(dsn, "SELECT pgcolumnar.vacuum('w')"), "42P01|names-the-update", + "in the window, pgcolumnar.vacuum() is refused and names ALTER EXTENSION UPDATE") + expect.text(rows(), "13334|133346667", "in the window, the refused operations changed nothing") + + with psycopg.connect(dsn, autocommit=True) as c: + c.execute("ALTER EXTENSION pgcolumnar UPDATE") + expect.text(rows(), "13334|133346667", "after ALTER EXTENSION UPDATE, the same rows are there") + vacuumed = _succeeds(dsn, "SELECT pgcolumnar.vacuum('w')") + expect.text(rows() if vacuumed == "succeeded" else vacuumed, "13334|133346667", + "after ALTER EXTENSION UPDATE, pgcolumnar.vacuum() succeeds and keeps them") + expect.text(_succeeds(dsn, "TRUNCATE w", "DROP TABLE w"), "succeeded", + "after ALTER EXTENSION UPDATE, TRUNCATE and DROP TABLE succeed") diff --git a/test/pytest/test_pg_dump_roundtrip.py b/test/pytest/test_pg_dump_roundtrip.py new file mode 100644 index 000000000..0d776de10 --- /dev/null +++ b/test/pytest/test_pg_dump_roundtrip.py @@ -0,0 +1,165 @@ +"""A columnar table survives pg_dump and restore, in both formats (#1341). The pytest +twin of `test/pg_dump_roundtrip.sh`, every check by the shell's name. + +A source database holds a columnar table with data, a per-table option, an index and a +projection, and a second table whose declared sort_by names columns after a dropped +one, so a restore renumbers their attnums (#288). It is dumped as plain SQL and as a +custom archive, and each is restored into a fresh database, where the rows, the access +method, the option, the declared sort_by and the projection declaration must all +survive. The projection's storage is not carried (#266) but rebuild_projections() +recovers it. + +Two arms are about more than their call returning: the point lookup must take the +restored INDEX, since the columnar scan answers it too with the index gone; and the +restored sort_by must leave the rows in (host, ts) order, counted as inversions in scan +order, which is why the readers run with parallel workers off. +""" +import pathlib +import subprocess +import tempfile + +import psycopg +import pytest + +SRC = "pgc_dumprt" +SUM_SQL = ("SELECT coalesce(sum(hashtextextended(id::text || '|' || kind::text || '|' || payload, 0)), 0) " + "FROM dt") +AM_SQL = ("SELECT a.amname::text FROM pg_class c JOIN pg_am a ON a.oid = c.relam " + "WHERE c.oid = 'dt'::regclass") +OPT_SQL = ("SELECT coalesce((SELECT encode_effort FROM pgcolumnar.options WHERE regclass = 'dt'::regclass), " + "'')") +PROJ_SQL = ("SELECT count(*) FROM pgcolumnar.projection WHERE storage_id = pgcolumnar.get_storage_id('dt') " + "AND projection_id > 0") +DECL_SQL = "SELECT count(*) FROM pgcolumnar.projection_declaration WHERE rel = 'dt'::regclass" +INVERSIONS = ("SELECT count(*) FROM (SELECT host, ts, lag(host) OVER () AS lh, lag(ts) OVER () AS lt FROM ds) x " + "WHERE (lh, lt) > (host, ts)") + + +def _conn(cluster, db): + return psycopg.connect(cluster.dsn(dbname=db), autocommit=True, + options="-c max_parallel_workers_per_gather=0") + + +def _one(cluster, db, sql): + with _conn(cluster, db) as c: + return c.execute(sql).fetchone()[0] + + +def _admin(cluster, sql): + with psycopg.connect(cluster.dsn(), autocommit=True) as c: + c.execute(sql) + + +def _tool(cluster, name, *args): + return subprocess.run([str(pathlib.Path(cluster.bindir) / name), "-h", "127.0.0.1", "-p", str(cluster.port), + "-U", "postgres", *args], capture_output=True, text=True).returncode + + +@pytest.fixture(scope="module") +def src(pgc_cluster): + """-> (cluster, the values the source holds before any dump).""" + _admin(pgc_cluster, f"DROP DATABASE IF EXISTS {SRC}") + _admin(pgc_cluster, f"CREATE DATABASE {SRC}") + try: + with _conn(pgc_cluster, SRC) as c: + for sql in ("CREATE EXTENSION pgcolumnar", + "CREATE TABLE dt (id bigint, kind int, payload text) USING pgcolumnar", + "SELECT pgcolumnar.set_options('dt', encode_effort => 'fast')", + "INSERT INTO dt SELECT g, g % 7, 'p' || g FROM generate_series(1, 50000) g", + "CREATE INDEX dt_id_idx ON dt (id)", + "SELECT pgcolumnar.add_projection('dt', 'dt_proj', ARRAY['id', 'payload'], ARRAY['id'])", + # #288: sort_by names columns that sit after a dropped one + "CREATE TABLE ds (id bigint, doomed int, host text, ts timestamptz) USING pgcolumnar", + "INSERT INTO ds (id, host, ts) SELECT g, 'h' || (g % 20), " + "'2024-01-01'::timestamptz + (g % 100) * interval '1 minute' FROM generate_series(1, 20000) g", + "ALTER TABLE ds DROP COLUMN doomed", + "SELECT pgcolumnar.set_options('ds', sort_by => ARRAY['host', 'ts'])"): + c.execute(sql) + before = {k: _one(pgc_cluster, SRC, sql) for k, sql in + (("count", "SELECT count(*) FROM dt"), ("sum", SUM_SQL), ("opt", OPT_SQL), + ("proj", PROJ_SQL), ("decl", DECL_SQL))} + yield pgc_cluster, before + finally: + _admin(pgc_cluster, f"DROP DATABASE IF EXISTS {SRC} WITH (FORCE)") + + +def test_dropping_a_projection_forgets_its_declaration(src, expect): + cluster, before = src + expect.num(before["proj"], 1, "the base table has a projection to preserve") + expect.num(before["decl"], 1, "and a declaration recorded for it") + # If DROP kept the declaration, a rebuild would resurrect what the operator removed. + with _conn(cluster, SRC) as c: + c.execute("SELECT pgcolumnar.add_projection('dt', 'dt_tmp', ARRAY['id'], ARRAY['id'])") + expect.num(_one(cluster, SRC, DECL_SQL), 2, "a second projection is declared") + with _conn(cluster, SRC) as c: + c.execute("SELECT pgcolumnar.drop_projection('dt', 'dt_tmp')") + expect.num(_one(cluster, SRC, DECL_SQL), 1, + "and dropping it forgets the declaration, so a rebuild cannot resurrect it") + expect.num(_one(cluster, SRC, "SELECT pgcolumnar.rebuild_projections()"), 0, "a rebuild now builds nothing") + + +def _verify(cluster, db, label, before, expect): + expect.num(_one(cluster, db, "SELECT count(*) FROM dt"), before["count"], f"{label}: row count survives") + expect.num(_one(cluster, db, SUM_SQL), before["sum"], f"{label}: row data survives (checksum)") + expect.text(_one(cluster, db, AM_SQL), "pgcolumnar", f"{label}: access method is pgcolumnar") + with _conn(cluster, db) as c: + c.execute("SET enable_seqscan = off") + c.execute("SET enable_bitmapscan = off") + plan = "\n".join(r[0] for r in c.execute("EXPLAIN (COSTS off) SELECT id FROM dt WHERE id = 12345")) + expect.num(sum(1 for line in plan.splitlines() if "Index" in line and "using dt_id_idx" in line), 1, + f"{label}: premise: the point lookup takes the restored index") + expect.num(c.execute("SELECT id FROM dt WHERE id = 12345").fetchone()[0], 12345, + f"{label}: restored index answers a point lookup") + expect.text(_one(cluster, db, OPT_SQL), before["opt"], f"{label}: per-table options survive") + # #266: the projection's storage is keyed by a storage id a restore reassigns + expect.num(_one(cluster, db, PROJ_SQL), 0, f"{label}: projection storage is NOT carried by pg_dump (see #266)") + expect.text(_one(cluster, db, "SELECT sort_by::text FROM pgcolumnar.options WHERE regclass = 'ds'::regclass"), + "{host,ts}", f"{label}: declared sort_by survives restore") + expect.text(str(_one(cluster, db, f"SELECT ({INVERSIONS}) > 0")), "True", + f"{label}: premise: the restored ds is not yet in its sort_by order") + with _conn(cluster, db) as c: + c.execute("SELECT pgcolumnar.vacuum_sorted('ds')") + applied = c.execute(f"SELECT count(*) || '|' || ({INVERSIONS}) FROM ds").fetchone()[0] + expect.text(applied, "20000|0", f"{label}: restored sort_by resolves and applies") + expect.num(_one(cluster, db, DECL_SQL), before["decl"], f"{label}: the projection declaration IS carried") + expect.num(_one(cluster, db, "SELECT pgcolumnar.rebuild_projections()"), 1, + f"{label}: rebuild_projections() reports it rebuilt one") + expect.num(_one(cluster, db, PROJ_SQL), before["proj"], f"{label}: and the projection exists again after the rebuild") + expect.num(_one(cluster, db, "SELECT count(*) FROM pgcolumnar.read_projection('dt', 'dt_proj')"), + _one(cluster, db, "SELECT count(*) FROM dt"), f"{label}: and it reads the same rows as the base table") + expect.num(_one(cluster, db, "SELECT pgcolumnar.rebuild_projections()"), 0, f"{label}: a second rebuild is a no-op") + + +def test_a_plain_dump_round_trips(src, expect): + cluster, before = src + db = f"{SRC}_p" + with tempfile.TemporaryDirectory() as d: + path = str(pathlib.Path(d) / "rt.sql") + expect.num(_tool(cluster, "pg_dump", "-d", SRC, "-f", path), 0, "plain: pg_dump exited 0") + text = pathlib.Path(path).read_text() + expect.text("yes" if ("USING pgcolumnar" in text or "default_table_access_method = pgcolumnar" in text) + else "no", "yes", "plain: dump recreates the table on the columnar AM") + _admin(cluster, f"DROP DATABASE IF EXISTS {db}") + _admin(cluster, f"CREATE DATABASE {db}") + try: + expect.num(_tool(cluster, "psql", "-d", db, "-v", "ON_ERROR_STOP=1", "-q", "-f", path), 0, + "plain: restore exited 0 (ON_ERROR_STOP)") + _verify(cluster, db, "plain", before, expect) + finally: + _admin(cluster, f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + + +def test_a_custom_archive_round_trips(src, expect): + cluster, before = src + db = f"{SRC}_c" + with tempfile.TemporaryDirectory() as d: + path = str(pathlib.Path(d) / "rt.dump") + expect.num(_tool(cluster, "pg_dump", "-Fc", "-d", SRC, "-f", path), 0, "custom: pg_dump -Fc exited 0") + _admin(cluster, f"DROP DATABASE IF EXISTS {db}") + _admin(cluster, f"CREATE DATABASE {db}") + try: + expect.num(_tool(cluster, "pg_restore", "--exit-on-error", "-d", db, path), 0, + "custom: pg_restore exited 0 (--exit-on-error)") + _verify(cluster, db, "custom", before, expect) + finally: + _admin(cluster, f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") diff --git a/test/pytest/test_pg_upgrade.py b/test/pytest/test_pg_upgrade.py new file mode 100644 index 000000000..e920ae1b1 --- /dev/null +++ b/test/pytest/test_pg_upgrade.py @@ -0,0 +1,191 @@ +"""Columnar tables survive pg_upgrade across majors (#239, #1341). The pytest twin of +`test/pg_upgrade.sh`, every check by the shell's name. + +pg_upgrade carries relation files and the pgcolumnar.* catalogs to a new major. A +cluster on the OLD major gets a columnar table with non-default options, deletes and an +index; an empty cluster on the NEW major, the extension installed on both, is upgraded +in place of it; and the rows, their content, the options, the index and writability +must all survive. The index arm runs with the columnar scan off: right after pg_upgrade +there are no planner statistics, and the planner took the columnar scan for the range, +so without that the arm never touched the index. + +TWO MAJORS AT ONCE, like the shell, which is why neither runs in the matrix: the new +major is --pg-config and the old one is PGC_UPGRADE_FROM_PG_CONFIG. Without it, or +with an old major not below the new one, the test declines. +""" +import os +import pathlib +import re +import shutil +import subprocess +import tempfile + +import psycopg + +REPO = pathlib.Path(__file__).resolve().parents[2] + + +def _cfg(pg_config, flag): + return subprocess.run([pg_config, flag], capture_output=True, text=True, check=True).stdout.strip() + + +def _major(pg_config): + # "PostgreSQL 19rc1" is 19: the leading digits, not every digit + return int(re.match(r"\d+", _cfg(pg_config, "--version").split()[1]).group(0)) + + +def _sh(cmd, cwd=None): + return subprocess.run(cmd, cwd=cwd, capture_output=True, text=True) + + +def _server_tool(bindir, argv, own=(), cwd=None, check=True): + """Run initdb, pg_ctl or pg_upgrade, as postgres when we are root: each refuses + root, which is why pgc_cluster._asroot and lib.sh's runuser exist. A refused step + raises naming itself and its stderr, so it cannot surface a step later as a + missing file (jdatcmd, review of #1461). `own` is chowned to postgres first.""" + cmd = [str(pathlib.Path(bindir) / argv[0])] + list(argv[1:]) + if os.geteuid() == 0: + for top in own: + for path in [top, *pathlib.Path(top).rglob("*")]: + shutil.chown(path, user="postgres") + cmd = ["runuser", "-u", "postgres", "--"] + cmd + proc = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True) + if check and proc.returncode != 0: + raise RuntimeError(f"{argv[0]} failed rc={proc.returncode}: {(proc.stderr or proc.stdout).strip()[-2000:]}") + return proc + + +def _q(port, sql): + """-> the first column of the first row, or '' when the query cannot run.""" + try: + with psycopg.connect(f"host=127.0.0.1 port={port} user=postgres dbname=postgres", autocommit=True, + connect_timeout=10) as c: + row = c.execute(sql).fetchone() + return "" if row is None else row[0] + except psycopg.Error: + return "" + + +def _run(port, *sqls): + with psycopg.connect(f"host=127.0.0.1 port={port} user=postgres dbname=postgres", autocommit=True) as c: + for sql in sqls: + c.execute(sql) + + +def _cluster(bindir, data, port): + """initdb -k (data checksums on both sides, or pg_upgrade refuses the pair) and conf. + The socket directory is the datadir, as pgc_cluster pins it: a packaged build's + default is a directory only postgres can write.""" + data.mkdir() + _server_tool(bindir, ["initdb", "-k", "-D", str(data), "-A", "trust", "-U", "postgres"], own=[data]) + with open(data / "postgresql.conf", "a") as fh: + fh.write(f"\nport={port}\nlisten_addresses='127.0.0.1'\nunix_socket_directories='{data}'\n" + "shared_preload_libraries='pgcolumnar'\n") + + +def _ctl(bindir, data, action, log=None): + argv = ["pg_ctl", "-D", str(data), "-w"] + argv += (["-l", str(log), "start"] if action == "start" else ["-m", "fast", "stop"]) + return _server_tool(bindir, argv, own=[data]).returncode + + +def test_pg_upgrade_preserves_a_columnar_table(pgc_cluster, expect): + from pgc_cluster import pick_port + + old_pgc = os.environ.get("PGC_UPGRADE_FROM_PG_CONFIG", "") + new_pgc = pgc_cluster.pg_config + if not old_pgc or not os.access(old_pgc, os.X_OK): + expect.cannot_run("UNMET_PRECONDITION", + "PGC_UPGRADE_FROM_PG_CONFIG names no older major's pg_config, so there is nothing to " + "upgrade from; the matrix runs one major at a time", + name="pg_upgrade exited cleanly") + return + old_maj, new_maj = _major(old_pgc), _major(new_pgc) + if new_maj <= old_maj: + expect.cannot_run("UNMET_PRECONDITION", f"PG{old_maj} -> PG{new_maj} is not an upgrade", + name="pg_upgrade exited cleanly") + return + old_bin, new_bin = _cfg(old_pgc, "--bindir"), _cfg(new_pgc, "--bindir") + + work = pathlib.Path(tempfile.mkdtemp(prefix="pgc-upgrade-")) + work.chmod(0o777) + old_data, new_data = work / "old", work / "new" + old_port, new_port = pick_port(17), pick_port(18) + try: + # The extension on the OLD major, built from a copy so the two majors' objects + # never mix; conftest already installed it for the new one. + build = work / "src" + shutil.copytree(REPO, build, ignore=shutil.ignore_patterns(".git", "*.o", "*.so", "*.bc", "__pycache__")) + _sh(["make", "-C", str(build), "clean", f"PG_CONFIG={old_pgc}"]) + for target in ([], ["install"]): + made = _sh(["make", "-C", str(build), *target, f"PG_CONFIG={old_pgc}"]) + assert made.returncode == 0, \ + f"make {' '.join(target)} for the old major failed: {made.stderr.strip()[-2000:]}" + + _cluster(old_bin, old_data, old_port) + _ctl(old_bin, old_data, "start", work / "old.log") + _run(old_port, "CREATE EXTENSION pgcolumnar", + "CREATE TABLE c (id int, v text, n numeric) USING pgcolumnar", + # non-default options, so the options catalog row is carried, not only the pages + "SELECT pgcolumnar.set_options('c', stripe_row_limit => 4000, chunk_group_row_limit => 800, " + "compression => 'zstd')", + "INSERT INTO c SELECT g, md5(g::text), (g::numeric) / 7 FROM generate_series(1, 50000) g", + "DELETE FROM c WHERE id % 13 = 0", + "CREATE INDEX c_id ON c (id)") + content = "SELECT md5(string_agg(t::text, '' ORDER BY t::text)) FROM c t" + opts = ("SELECT stripe_row_limit || '/' || chunk_group_row_limit || '/' || compression " + "FROM pgcolumnar.options WHERE regclass = 'c'::regclass") + old_hash, old_count, old_opts = _q(old_port, content), _q(old_port, "SELECT count(*) FROM c"), _q(old_port, opts) + old_range = _q(old_port, "SELECT count(*) FROM c WHERE id BETWEEN 100 AND 200") + expect.text("yes" if isinstance(old_count, int) and old_count > 0 else "no", "yes", + "old cluster has rows to preserve") + expect.text(old_opts, "4000/800/zstd", "old cluster recorded non-default options") + _ctl(old_bin, old_data, "stop") + + _cluster(new_bin, new_data, new_port) + _ctl(new_bin, new_data, "start", work / "new-pre.log") + # a pass must not come from the table having been made here + expect.num(_q(new_port, "SELECT count(*) FROM pg_class WHERE relname = 'c'"), 0, + "the new cluster does NOT have the table before the upgrade") + expect.num(int(_q(new_port, "SHOW server_version_num")) // 10000, new_maj, "and is genuinely the new major") + _ctl(new_bin, new_data, "stop") + + # pg_upgrade refuses root too, and writes its logs and socket into cwd + up = _server_tool(new_bin, ["pg_upgrade", "-b", old_bin, "-B", new_bin, "-d", str(old_data), + "-D", str(new_data), "-p", str(old_port), "-P", str(new_port), + "-U", "postgres"], own=[old_data, new_data], cwd=work, check=False) + if up.returncode != 0: + print(f"-- pg_upgrade said:\n{up.stdout[-3000:]}\n{up.stderr[-2000:]}") + expect.num(up.returncode, 0, "pg_upgrade exited cleanly") + _ctl(new_bin, new_data, "start", work / "new.log") + + expect.num(_q(new_port, "SELECT 1"), 1, "the new cluster starts after the upgrade") + expect.num(int(_q(new_port, "SHOW server_version_num") or 0) // 10000, new_maj, "we are on the new major") + expect.num(_q(new_port, "SELECT count(*) FROM pg_extension WHERE extname = 'pgcolumnar'"), 1, + "the extension is present") + expect.text(_q(new_port, "SELECT a.amname::text FROM pg_class c JOIN pg_am a ON a.oid = c.relam " + "WHERE c.relname = 'c'"), "pgcolumnar", "the table is still a columnar table") + expect.num(_q(new_port, "SELECT count(*) FROM c"), old_count, "row count preserved") + expect.text(_q(new_port, content), old_hash, "content preserved byte for byte") + expect.text(_q(new_port, opts), old_opts, "per-table options preserved") + with psycopg.connect(f"host=127.0.0.1 port={new_port} user=postgres dbname=postgres", autocommit=True) as c: + c.execute("SET enable_seqscan = off") + c.execute("SET pgcolumnar.enable_custom_scan = off") + plan = "\n".join(r[0] for r in c.execute( + "EXPLAIN (COSTS off) SELECT count(*) FROM c WHERE id BETWEEN 100 AND 200")) + uses = sum(1 for line in plan.splitlines() + if ("Index Scan" in line or "Index Only Scan" in line) and "c_id" in line) + expect.num(uses, 1, "premise: the range lookup takes the upgraded index") + # compared with the OLD cluster's answer, not with another query on the new one + expect.num(c.execute("SELECT count(*) FROM c WHERE id BETWEEN 100 AND 200").fetchone()[0], old_range, + "the index still answers the same range") + _run(new_port, "INSERT INTO c SELECT g, 'post', 1 FROM generate_series(900001, 900100) g") + expect.num(_q(new_port, "SELECT count(*) FROM c WHERE v = 'post'"), 100, + "the upgraded table still accepts writes") + expect.num(_q(new_port, "SELECT count(*) FROM c"), old_count + 100, + "and reads back the new rows with the old ones") + finally: + for b, d in ((old_bin, old_data), (new_bin, new_data)): + if (d / "postmaster.pid").exists(): + _server_tool(b, ["pg_ctl", "-D", str(d), "-m", "immediate", "-w", "stop"], check=False) + shutil.rmtree(work, ignore_errors=True) diff --git a/test/pytest/test_truncate_cleanup.py b/test/pytest/test_truncate_cleanup.py new file mode 100644 index 000000000..c3c34ab9e --- /dev/null +++ b/test/pytest/test_truncate_cleanup.py @@ -0,0 +1,181 @@ +"""TRUNCATE leaves no catalog rows behind, and keeps what it must (#1341). The pytest +twin of `test/truncate_cleanup.sh`, every check by the shell's name. + +TRUNCATE installs a new relfilenode and a new storage id. It used to leave the old +storage's row_group, column_chunk, zone_map and bloom rows behind, keyed by the retired +id, so each truncate-and-reload cycle leaked one storage's worth, and a DROP after it +removed only the current, empty one. Retiring them made a cached write state stale, so +a transaction that wrote, truncated and wrote again committed into storage nothing +reads; the state is now dropped with the rows. A relation created in the current +transaction is truncated in place instead, on a different path, which must clear each +projection's storage as well as the base's. + +A PRIVATE DATABASE, because the snapshot counts every pgcolumnar catalog row, and the +shared one holds every other test's. +""" +import psycopg + + +def _one(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetchone()[0] + + +def _run(conn, *sqls): + with conn.cursor() as cur: + for sql in sqls: + cur.execute(sql) + + +def _snapshot(conn): + return _one(conn, "SELECT (SELECT count(*) FROM pgcolumnar.storage) || '/' || " + "(SELECT count(*) FROM pgcolumnar.projection) || '/' || " + "(SELECT count(*) FROM pgcolumnar.row_group) || '/' || " + "(SELECT count(*) FROM pgcolumnar.column_chunk) || '/' || " + "(SELECT count(*) FROM pgcolumnar.zone_map) || '/' || " + "(SELECT count(*) FROM pgcolumnar.bloom) || '/' || " + "(SELECT count(*) FROM pgcolumnar.delete_vector)") + + +def _grew(base, now): + return "no" if base == now else "yes" + + +LOAD = "INSERT INTO {t} SELECT g, 'x' || g FROM generate_series(1, {n}) g" + + +def test_truncate_and_drop_leave_the_catalog_as_they_found_it(pgc_own_db, expect): + conn = pgc_own_db + base = _snapshot(conn) + _run(conn, "CREATE TABLE tcl_plain (id int, v text) USING pgcolumnar", LOAD.format(t="tcl_plain", n=5000)) + expect.text(_grew(base, _snapshot(conn)), "yes", "insert did add metadata") + _run(conn, "TRUNCATE tcl_plain") + expect.text(_snapshot(conn), base, "TRUNCATE returns the catalog to the empty baseline") + expect.num(_one(conn, "SELECT count(*) FROM tcl_plain"), 0, "TRUNCATE leaves no rows") + for _ in range(2): + _run(conn, LOAD.format(t="tcl_plain", n=5000), "TRUNCATE tcl_plain") + expect.text(_snapshot(conn), base, "three truncate-and-reload cycles still sit at the baseline") + # loaded first, so the DROP has work to do: a DROP of an empty table could not fail + _run(conn, LOAD.format(t="tcl_plain", n=5000)) + expect.text(_grew(base, _snapshot(conn)), "yes", + "premise: the table carries metadata again, so the DROP has work to do") + _run(conn, "DROP TABLE tcl_plain") + expect.text(_snapshot(conn), base, "DROP of a loaded table after TRUNCATE leaves nothing") + + +def test_a_projection_is_not_leaked_by_truncate(pgc_own_db, expect): + """A projection has its own storage id. What must hold is that no projection row + names a storage id the table no longer has, not that projection is empty.""" + conn = pgc_own_db + base = _snapshot(conn) + _run(conn, "CREATE TABLE tcl_proj (id int, a int, b text) USING pgcolumnar", + "SELECT pgcolumnar.add_projection('tcl_proj', 'tcl_p', ARRAY['a', 'b'], ARRAY['a'])", + "INSERT INTO tcl_proj SELECT g, g % 50, 'b' || g FROM generate_series(1, 5000) g") + expect.text(_grew(base, _snapshot(conn)), "yes", "the projection did add metadata") + _run(conn, "TRUNCATE tcl_proj") + expect.num(_one(conn, "SELECT count(*) FROM pgcolumnar.projection p " + "WHERE p.storage_id <> pgcolumnar.get_storage_id('tcl_proj'::regclass)"), 0, + "TRUNCATE leaves no projection row under a retired storage id") + expect.num(_one(conn, "SELECT count(*) FROM pgcolumnar.projection_declaration " + "WHERE rel = 'tcl_proj'::regclass"), 1, + "and the declaration survives the TRUNCATE, so the projection is rebuildable") + _run(conn, "INSERT INTO tcl_proj SELECT g, g % 50, 'b' || g FROM generate_series(1, 5000) g") + expect.text(_grew(base, _snapshot(conn)), "yes", "premise: the projected table carries metadata again before the DROP") + _run(conn, "DROP TABLE tcl_proj") + expect.text(_snapshot(conn), base, "DROP of a loaded projected table leaves nothing") + + +def test_repeated_truncates_accumulate_nothing(pgc_own_db, expect): + """Created once outside the loop and one statement at a time, so every TRUNCATE + takes the transactional path and any leak accumulates.""" + conn = pgc_own_db + base = _snapshot(conn) + _run(conn, "CREATE TABLE tcl_rep (id int, v text) USING pgcolumnar") + for _ in range(10): + _run(conn, LOAD.format(t="tcl_rep", n=2000), "TRUNCATE tcl_rep") + expect.text(_snapshot(conn), base, "ten insert-and-truncate cycles on one table leave nothing") + _run(conn, "DROP TABLE tcl_rep") + expect.text(_snapshot(conn), base, "and dropping it afterwards still leaves nothing") + + +def test_insert_truncate_insert_in_one_transaction(pgc_own_db, expect): + """The cached write state named the storage the TRUNCATE retired, so the second + INSERT flushed into storage nothing reads, and the transaction committed an empty + table. The status and the surviving rows are one arm: split, a commit that keeps + nothing passes the status half, and 1..10 passes a bare count of 10.""" + conn = pgc_own_db + _run(conn, "CREATE TABLE tc_txn (id int) USING pgcolumnar", + "INSERT INTO tc_txn SELECT g FROM generate_series(1, 10) g") + expect.num(_one(conn, "SELECT count(*) FROM tc_txn"), 10, "premise: the table holds its first ten rows") + rc = 0 + try: + _run(conn, "BEGIN", "INSERT INTO tc_txn SELECT g FROM generate_series(11, 20) g", "TRUNCATE tc_txn", + "INSERT INTO tc_txn SELECT g FROM generate_series(21, 30) g", "COMMIT") + except psycopg.Error: + rc = 1 + _run(conn, "ROLLBACK") + expect.text(f"rc={rc} " + _one(conn, "SELECT coalesce(min(id), -1) || '..' || coalesce(max(id), -1) || " + "' n=' || count(*) FROM tc_txn"), + "rc=0 21..30 n=10", "INSERT, TRUNCATE, INSERT in one transaction commits and keeps the new rows") + + +def test_a_rewrite_that_is_not_a_truncate_loses_nothing(pgc_own_db, expect): + """The retire runs from relation_set_new_filelocator, which every rewrite calls.""" + conn = pgc_own_db + _run(conn, "CREATE TABLE tc_rw (id int, v text) USING pgcolumnar", + "INSERT INTO tc_rw SELECT g, 'v' || g FROM generate_series(1, 5000) g") + expect.num(_one(conn, "SELECT count(*) FROM tc_rw"), 5000, "premise: five thousand rows before the rewrite") + _run(conn, "ALTER TABLE tc_rw ALTER COLUMN v TYPE varchar(64)") + expect.text(_one(conn, "SELECT count(*) || '/' || count(v) || '/' || max(id) FROM tc_rw"), "5000/5000/5000", + "a full rewrite keeps every row, and they are still readable") + + +def test_an_in_place_truncate_clears_the_projection_too(pgc_own_db, expect): + """A relation created in the current transaction is truncated in place, through + nontransactional_truncate, which cleared the base storage and left the + projection's. The write after the truncate is what makes the leak observable: it + collided on row_group_pkey (#896).""" + conn = pgc_own_db + _run(conn, "CREATE TABLE tcl_inplace_pre (id int)") + committed = "committed" + try: + _run(conn, "BEGIN", + "CREATE TABLE tcl_inplace (id int, a int, b text) USING pgcolumnar", + "INSERT INTO tcl_inplace SELECT g, g % 50, 'b' || g FROM generate_series(1, 2000) g", + "SELECT pgcolumnar.add_projection('tcl_inplace', 'tcl_ip', ARRAY['a', 'b'], ARRAY['a'])", + "TRUNCATE tcl_inplace", + "INSERT INTO tcl_inplace SELECT g, g % 50, 'c' || g FROM generate_series(1, 500) g", + "COMMIT") + except psycopg.Error as exc: + committed = f"rolled back ({str(exc).splitlines()[0]})" + _run(conn, "ROLLBACK") + # vacuous without these: a rolled-back transaction leaves no table to count + expect.num(0 if committed == "committed" else 1, 0, "PREMISE the in-place transaction committed") + expect.num(_one(conn, "SELECT count(*) FROM pg_class WHERE relname = 'tcl_inplace'"), 1, + "PREMISE the table exists after it") + if committed != "committed": + return + expect.num(_one(conn, "SELECT count(*) FROM tcl_inplace"), 500, + "the rows written after the in-place truncate are all there") + expect.num(_one(conn, "SELECT count(*) FROM pgcolumnar.read_projection('tcl_inplace', 'tcl_ip')"), 500, + "and the projection reads them back") + with conn.cursor() as cur: + cur.execute("SELECT pgcolumnar.read_projection('tcl_inplace', 'tcl_ip')") + proj = cur.fetchall() + cur.execute("SELECT a::text || '|' || b FROM tcl_inplace") + base = cur.fetchall() + expect.row_set(proj, base, "the projection agrees with the base table") + proj_storage = _one(conn, "SELECT proj_storage_id FROM pgcolumnar.projection " + "WHERE storage_id = pgcolumnar.get_storage_id('tcl_inplace'::regclass) " + "AND projection_id > 0") + base_storage = _one(conn, "SELECT pgcolumnar.get_storage_id('tcl_inplace'::regclass)") + expect.text("yes" if proj_storage is not None and proj_storage != base_storage else "no", "yes", + "PREMISE the projection has its own storage, distinct from the base") + # one group of 500 rows, not a group NUMBER: the base's counter is reset and the + # projection's has no metapage to reset, so the numbers differ by design + expect.num(_one(conn, f"SELECT count(*) FROM pgcolumnar.row_group WHERE storage_id = {proj_storage}"), 1, + "the projection storage holds exactly one row group after the truncate") + expect.num(_one(conn, f"SELECT coalesce(sum(row_count), 0) FROM pgcolumnar.row_group " + f"WHERE storage_id = {proj_storage}"), 500, + "and that group describes only the rows written after it")