diff --git a/CHANGELOG.md b/CHANGELOG.md index d2b779a6d..c0dffb95f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -400,6 +400,18 @@ true until the next version shipped. Reported by @OffgridwithJD. ### Fixed +- **In the upgrade window, a refusal for a missing catalog table names `ALTER + EXTENSION UPDATE`.** After the new library is installed and the server restarted, + but before `ALTER EXTENSION pgcolumnar UPDATE`, `DROP TABLE`, `TRUNCATE` and + `pgcolumnar.vacuum()` on a columnar table fail, because the 1.0-alpha5 library + needs `pgcolumnar.retired_group` and a 1.0-alpha4 catalog lacks it. They change + nothing. The error used to name only the missing table, and it now carries a hint: + `The pgcolumnar catalog in this database may be older than the installed library. + Run ALTER EXTENSION pgcolumnar UPDATE in this database.` Reads and writes work + throughout, and after the update every one of these succeeds on the same rows + (#1341 item 66). `test/native_upgrade_converge.sh` and + `test_native_upgrade_converge.py`. + - **A join whose runtime-filter build is empty no longer reads the fact table.** When a large dimension filtered to no row was joined to a columnar table, the hash join diff --git a/docs/installation.md b/docs/installation.md index 2618c47bf..6cf867108 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -152,6 +152,18 @@ The fix is step 3. Run `ALTER EXTENSION pgcolumnar UPDATE;` in that database and the error goes away. **Your data is not affected.** The tables are intact and no conversion happens. Only the catalog entry is stale. +Between releases that add a catalog table, the new library can also need a table +the old catalog does not have. Upgrading from `1.0-alpha4` to `1.0-alpha5`, +`DROP TABLE`, `TRUNCATE` and `pgcolumnar.vacuum()` on a columnar table fail until +step 3, and change nothing; reads and writes keep working: + +``` +ERROR: columnar metadata table "pgcolumnar.retired_group" does not exist +HINT: The pgcolumnar catalog in this database may be older than the installed library. Run ALTER EXTENSION pgcolumnar UPDATE in this database. +``` + +The fix is the same: step 3. + Do not use `DROP EXTENSION` instead. A plain `DROP EXTENSION pgcolumnar` fails while columnar tables exist, because they depend on the access method. The form that succeeds is `DROP EXTENSION pgcolumnar CASCADE`, and it drops every diff --git a/src/columnar_metadata.c b/src/columnar_metadata.c index bd0120efb..eb5bdc53f 100644 --- a/src/columnar_metadata.c +++ b/src/columnar_metadata.c @@ -239,11 +239,22 @@ open_columnar_table(const char *name, LOCKMODE lockmode) nspOid = pgcolumnar_schema_oid(); relOid = get_relname_relid(name, nspOid); + /* + * NAME THE REMEDY (#1341 item 66). The usual way here is the upgrade window: + * the new library is installed and the server restarted (docs/installation.md + * steps 1 and 2), but this database's catalog is still the previous release's + * until ALTER EXTENSION UPDATE (step 3). A catalog table this library added, + * such as retired_group in 1.0-alpha5, is then missing, and DROP TABLE, + * TRUNCATE and pgcolumnar.vacuum() stop here. The error alone named only the + * table, which says neither why nor what to do. + */ if (!OidIsValid(relOid)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), errmsg("columnar metadata table \"%s.%s\" does not exist", - COLUMNAR_SCHEMA_NAME, name))); + COLUMNAR_SCHEMA_NAME, name), + errhint("The pgcolumnar catalog in this database may be older than the installed library. " + "Run ALTER EXTENSION pgcolumnar UPDATE in this database."))); rel = table_open(relOid, lockmode); diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 74fcd205f..f5f554373 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -804,6 +804,34 @@ fdw_plan_privilege fdw_plan_privilege and a metadata_path that does not exist pl fdw_plan_privilege fdw_plan_privilege premise: the Parquet file was written 15;16;17;18;19 never - fdw_plan_privilege fdw_plan_privilege premise: the reader is not a member of pg_read_server_files 15;16;17;18;19 never - fdw_plan_privilege fdw_plan_privilege the unprivileged Iceberg scan is still refused (42501) 15;16;17;18;19 never - +fk_referencing fk_referencing ALTER TABLE ADD CONSTRAINT is refused by the same rule 15;16;17;18;19 2026-10-16 columnar_tableam.c: a foreign key referencing a columnar table no longer refused;columnar_tableam.c: the foreign-key guard keyed on the referencing table, not the referenced one +fk_referencing fk_referencing a columnar partition of an FK-referenced parent is refused 15;16;17;18;19 2026-10-16 columnar_tableam.c: a foreign key referencing a columnar table no longer refused;columnar_tableam.c: the foreign-key guard keyed on the referencing table, not the referenced one +fk_referencing fk_referencing a columnar table may reference a heap table 15;16;17;18;19 2026-10-16 columnar_tableam.c: the foreign-key guard keyed on the referencing table, not the referenced one +fk_referencing fk_referencing a foreign key between two heap tables is unaffected 15;16;17;18;19 never - +fk_referencing fk_referencing a foreign key referencing a columnar table is refused 15;16;17;18;19 2026-10-16 columnar_tableam.c: a foreign key referencing a columnar table no longer refused;columnar_tableam.c: the foreign-key guard keyed on the referencing table, not the referenced one +fk_referencing fk_referencing a partitioned table with no foreign key still converts 17;18;19 never - +fk_referencing fk_referencing a row that violates it is still rejected 15;16;17;18;19 2026-10-16 columnar_tableam.c: the foreign-key guard keyed on the referencing table, not the referenced one +fk_referencing fk_referencing a table with no foreign key still converts 15;16;17;18;19 never - +fk_referencing fk_referencing an FK-referenced table may still be set to a row store 15;16;17;18;19 2026-10-16 columnar_tableam.c: converting an FK-referenced table refused whatever the target access method +fk_referencing fk_referencing an unrelated ALTER TABLE on a referenced table is unaffected 15;16;17;18;19 never - +fk_referencing fk_referencing and a partition can still be created the ordinary way 15;16;17;18;19 2026-10-16 columnar_tableam.c: converting an FK-referenced table to columnar no longer refused +fk_referencing fk_referencing and its partitions inherit the columnar access method 17;18;19 never - +fk_referencing fk_referencing and rows that satisfy it are accepted 15;16;17;18;19 2026-10-16 columnar_tableam.c: the foreign-key guard keyed on the referencing table, not the referenced one +fk_referencing fk_referencing and setting the parent's access method is refused as well 15;16;17;18;19 2026-10-16 columnar_tableam.c: converting an FK-referenced table to columnar no longer refused +fk_referencing fk_referencing and the error names the constraint 15;16;17;18;19 2026-10-16 columnar_tableam.c: converting an FK-referenced table to columnar no longer refused +fk_referencing fk_referencing and the error says why 15;16;17;18;19 2026-10-16 columnar_tableam.c: a foreign key referencing a columnar table no longer refused;columnar_tableam.c: the foreign-key guard keyed on the referencing table, not the referenced one +fk_referencing fk_referencing and they still work 15;16;17;18;19 never - +fk_referencing fk_referencing before 17, core refuses this for every partitioned table 15;16 never - +fk_referencing fk_referencing converting an FK-referenced table to columnar is refused 15;16;17;18;19 2026-10-16 columnar_tableam.c: converting an FK-referenced table to columnar no longer refused +fk_referencing fk_referencing converting back to heap is unaffected 15;16;17;18;19 never - +fk_referencing fk_referencing converting the referencing side is still allowed 15;16;17;18;19 never - +fk_referencing fk_referencing primary key, unique, check and not-null on a columnar table are unaffected 15;16;17;18;19 never - +fk_referencing fk_referencing so the parent did not become columnar 15;16;17;18;19 2026-10-16 columnar_tableam.c: converting an FK-referenced table to columnar no longer refused +fk_referencing fk_referencing the child is still writable afterwards 15;16;17;18;19 2026-10-16 columnar_tableam.c: converting an FK-referenced table to columnar no longer refused +fk_referencing fk_referencing the conversion helper is refused identically 15;16;17;18;19 2026-10-16 columnar_tableam.c: converting an FK-referenced table to columnar no longer refused +fk_referencing fk_referencing the parent is still a row store 15;16;17;18;19 2026-10-16 columnar_tableam.c: converting an FK-referenced table to columnar no longer refused +fk_referencing fk_referencing the primary key is still enforced 15;16;17;18;19 never - +fk_referencing fk_referencing the refused table was not left behind 15;16;17;18;19 2026-10-16 columnar_tableam.c: a foreign key referencing a columnar table no longer refused;columnar_tableam.c: the foreign-key guard keyed on the referencing table, not the referenced one harness_selftest 030-assertions nothing leaked into the squatter 15;16;17;18;19 never - harness_selftest 030-assertions pgc_port_free says the squatter's port is busy 15;16;17;18;19 never - harness_selftest 030-assertions squatter survived untouched 15;16;17;18;19 never - @@ -2280,6 +2308,35 @@ native_chunk_length_bound native_chunk_length_bound backend survived the sequent native_chunk_length_bound native_chunk_length_bound backend survived the truncated-length fetch 15;16;17;18;19 never - native_chunk_length_bound native_chunk_length_bound premise: a point lookup uses the index, not a sequential columnar scan 15;16;17;18;19 never - native_chunk_length_bound native_chunk_length_bound that fetch returns the row 15;16;17;18;19 2026-10-16 the index fetch treated a flushed row as not live, so the lookup returned no row (mutation, #1339) +native_decode_gating native_decode_gating EXPLAIN reports Columnar Vector Decodes 15;16;17;18;19 never - +native_decode_gating native_decode_gating a one-vector LIKE decodes the qual column plus payload for that vector only 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluation between the decode passes never runs;columnar_reader.c: the per-vector qual evaluator marks every vector skipped +native_decode_gating native_decode_gating a zero-matching LIKE decodes the qual column and no payload column 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluation between the decode passes never runs +native_decode_gating native_decode_gating and reports them all filtered before materialization 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluator marks every vector skipped +native_decode_gating native_decode_gating and skips every vector 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluation between the decode passes never runs +native_decode_gating native_decode_gating and the full scan across every group's mask matches too 15;16;17;18;19 never - +native_decode_gating native_decode_gating deletes + multiple groups + unprunable qual: columnar matches the heap 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluator marks every vector skipped +native_decode_gating native_decode_gating premise: no vector is ruled out by VALUE either (that is 1b-ii's path) 15;16;17;18;19 never - +native_decode_gating native_decode_gating premise: no whole row group is pruned 15;16;17;18;19 never - +native_decode_gating native_decode_gating premise: one row group, so every count below is one group's 15;16;17;18;19 never - +native_decode_gating native_decode_gating premise: rows really were deleted (so the mask is non-NULL) 15;16;17;18;19 never - +native_decode_gating native_decode_gating premise: the columnar table spans more than one row group 15;16;17;18;19 never - +native_decode_gating native_decode_gating premise: the control matches every row 15;16;17;18;19 never - +native_decode_gating native_decode_gating premise: the half-needle survives in one vector, half of its rows 15;16;17;18;19 never - +native_decode_gating native_decode_gating premise: the needle plan is a columnar custom scan 15;16;17;18;19 never - +native_decode_gating native_decode_gating premise: the needle query takes the late-materialization path 15;16;17;18;19 never - +native_decode_gating native_decode_gating premise: the needle really matches one vector's worth of rows 15;16;17;18;19 never - +native_decode_gating native_decode_gating premise: the nomatch predicate matches no row 15;16;17;18;19 never - +native_decode_gating native_decode_gating premise: the reader has no usable skip predicate for this LIKE 15;16;17;18;19 never - +native_decode_gating native_decode_gating premise: the width gate is pinned off, so this narrow projection still gates 15;16;17;18;19 never - +native_decode_gating native_decode_gating rejections inside a surviving vector are counted once 15;16;17;18;19 2026-10-16 columnar_reader.c: the producer counts a gated group's rejections a second time +native_decode_gating native_decode_gating rows removed by filter across deleted rows count only live rows 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluator counts a deleted row as filtered;columnar_reader.c: the producer counts a gated group's rejections a second time +native_decode_gating native_decode_gating the control decodes every column of every vector 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluator marks every vector skipped +native_decode_gating native_decode_gating the control returns every row, values intact 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluator marks every vector skipped +native_decode_gating native_decode_gating the needle query returns exactly the matching rows 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluator marks every vector skipped +native_decode_gating native_decode_gating the needle removes exactly the non-matching rows, counted once 15;16;17;18;19 never - +native_decode_gating native_decode_gating the needle skips exactly the vectors holding no match 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluation between the decode passes never runs;columnar_reader.c: the per-vector qual evaluator marks every vector skipped +native_decode_gating native_decode_gating the nomatch query returns nothing 15;16;17;18;19 never - +native_decode_gating native_decode_gating the zero-match query removes every row by filter, counted once 15;16;17;18;19 never - native_delete_visibility_paths native_delete_visibility_paths a DELETE whose qual deletes another row of the table succeeds and leaves heap's rows (#1394) 15;16;17;18;19 2026-10-14 PgColumnarUpsertDeleteVector without the command advance (main before #1394's fix) native_delete_visibility_paths native_delete_visibility_paths a deleted row is invisible via the index 15;16;17;18;19 never - native_delete_visibility_paths native_delete_visibility_paths a live row is visible via the index 15;16;17;18;19 never - @@ -2288,6 +2345,25 @@ native_delete_visibility_paths native_delete_visibility_paths aggregate over the native_delete_visibility_paths native_delete_visibility_paths index / bitmap scan sees the live set 15;16;17;18;19 never - native_delete_visibility_paths native_delete_visibility_paths index-only scan sees the live set 15;16;17;18;19 never - native_delete_visibility_paths native_delete_visibility_paths sequential scan sees the live set (row-emitting path) 15;16;17;18;19 never - +native_dict_underfill native_dict_underfill backend survives the under-filled DICT chunk 15;16;17;18;19 2026-10-16 columnar_encoding.c: the DICT fill refusal replaced by a backend crash +native_dict_underfill native_dict_underfill premise: the inflation rewrote the DICT chunk's descriptor 15;16;17;18;19 never - +native_dict_underfill native_dict_underfill the text column is DICT-encoded (precondition) 15;16;17;18;19 never - +native_dict_underfill native_dict_underfill under-filled DICT chunk is rejected with a clean error 15;16;17;18;19 2026-10-16 columnar_encoding.c: a DICT chunk whose codes do not fill the raw length is no longer refused;columnar_encoding.c: the DICT fill refusal replaced by a backend crash +native_dml native_dml PK still enforced after DML 15;16;17;18;19 never - +native_dml native_dml count after delete 15;16;17;18;19 2026-10-16 columnar_vector.c: count(*) from metadata no longer subtracts a group's deleted rows +native_dml native_dml count after merge 15;16;17;18;19 2026-10-16 columnar_tableam.c: the index fetch no longer checks the delete vector;columnar_vector.c: count(*) from metadata no longer subtracts a group's deleted rows +native_dml native_dml count after update 15;16;17;18;19 2026-10-16 columnar_vector.c: count(*) from metadata no longer subtracts a group's deleted rows +native_dml native_dml deleted rows gone 15;16;17;18;19 never - +native_dml native_dml index finds live row 15;16;17;18;19 never - +native_dml native_dml index skips deleted row 15;16;17;18;19 2026-10-16 columnar_tableam.c: the index fetch no longer checks the delete vector +native_dml native_dml min/max after delete 15;16;17;18;19 2026-10-16 columnar_vector.c: a row group holding deletes still answers min and max from its zone maps +native_dml native_dml premise: the deletes moved the smallest and largest id and k 15;16;17;18;19 never - +native_dml native_dml premise: the lookups take the primary-key index 15;16;17;18;19 never - +native_dml native_dml rows after delete match heap 15;16;17;18;19 never - +native_dml native_dml rows after merge match heap 15;16;17;18;19 2026-10-16 columnar_tableam.c: the index fetch no longer checks the delete vector +native_dml native_dml rows after update match heap 15;16;17;18;19 2026-10-16 columnar_tableam.c: the index fetch no longer checks the delete vector +native_dml native_dml sum(k) after delete 15;16;17;18;19 never - +native_dml native_dml sum(k) after update 15;16;17;18;19 2026-10-16 columnar_tableam.c: the index fetch no longer checks the delete vector native_fetch_coalesce native_fetch_coalesce a wide index fetch does not pin once per column 15;16;17;18;19 2026-10-13 the coalesced fetch merging no adjacent ranges, so each column is read on its own native_fetch_coalesce native_fetch_coalesce premise: a point lookup uses the index 15;16;17;18;19 never - native_fetch_coalesce native_fetch_coalesce premise: fetching every projected column touched a measurable number of buffers 15;16;17;18;19 never - @@ -3341,6 +3417,31 @@ timeseries_recipes timeseries_recipes the how-to gives the LOCF recipe 15;16;17; timeseries_recipes timeseries_recipes the how-to gives the gap-fill recipe 15;16;17;18;19 2026-10-13 the how-to's 'Fill empty buckets' heading renamed, so the recipe is no longer found timeseries_recipes timeseries_recipes the how-to gives the interpolation recipe 15;16;17;18;19 2026-10-13 the how-to's 'Interpolate between known values' heading renamed, so the recipe is no longer found timeseries_recipes timeseries_recipes the interpolation recipe fills a gap linearly, and leaves both ends NULL 15;16;17;18;19 2026-10-13 the how-to's 'Interpolate between known values' heading renamed, so the recipe is no longer found;the how-to's interpolation recipe uses prev_v in place of next_v, so every gap is filled flat;the how-to's recipes join the series to the aggregate with JOIN instead of LEFT JOIN +truncate_cleanup truncate_cleanup DROP of a loaded projected table leaves nothing 15;16;17;18;19 2026-10-16 columnar_tableam.c: a rewrite no longer deletes the retired storage's catalog rows;columnar_tableam.c: deleting a storage tree no longer reaches its projections' storages +truncate_cleanup truncate_cleanup DROP of a loaded table after TRUNCATE leaves nothing 15;16;17;18;19 2026-10-16 columnar_tableam.c: a rewrite no longer deletes the retired storage's catalog rows +truncate_cleanup truncate_cleanup INSERT, TRUNCATE, INSERT in one transaction commits and keeps the new rows 15;16;17;18;19 2026-10-16 columnar_tableam.c: a rewrite no longer drops the cached write state of the retired storage +truncate_cleanup truncate_cleanup PREMISE the in-place transaction committed 15;16;17;18;19 2026-10-16 columnar_tableam.c: the in-place truncate no longer clears a projection's own storage +truncate_cleanup truncate_cleanup PREMISE the projection has its own storage, distinct from the base 15;16;17;18;19 2026-10-16 columnar_tableam.c: the in-place truncate no longer clears a projection's own storage +truncate_cleanup truncate_cleanup PREMISE the table exists after it 15;16;17;18;19 2026-10-16 columnar_tableam.c: the in-place truncate no longer clears a projection's own storage +truncate_cleanup truncate_cleanup TRUNCATE leaves no projection row under a retired storage id 15;16;17;18;19 2026-10-16 columnar_tableam.c: a rewrite no longer deletes the retired storage's catalog rows +truncate_cleanup truncate_cleanup TRUNCATE leaves no rows 15;16;17;18;19 never - +truncate_cleanup truncate_cleanup TRUNCATE returns the catalog to the empty baseline 15;16;17;18;19 2026-10-16 columnar_tableam.c: a rewrite no longer deletes the retired storage's catalog rows +truncate_cleanup truncate_cleanup a full rewrite keeps every row, and they are still readable 15;16;17;18;19 never - +truncate_cleanup truncate_cleanup and dropping it afterwards still leaves nothing 15;16;17;18;19 2026-10-16 columnar_tableam.c: a rewrite no longer deletes the retired storage's catalog rows +truncate_cleanup truncate_cleanup and that group describes only the rows written after it 15;16;17;18;19 2026-10-16 columnar_tableam.c: the in-place truncate no longer clears a projection's own storage +truncate_cleanup truncate_cleanup and the declaration survives the TRUNCATE, so the projection is rebuildable 15;16;17;18;19 never - +truncate_cleanup truncate_cleanup and the projection reads them back 15;16;17;18;19 2026-10-16 columnar_tableam.c: the in-place truncate no longer clears a projection's own storage +truncate_cleanup truncate_cleanup insert did add metadata 15;16;17;18;19 never - +truncate_cleanup truncate_cleanup premise: five thousand rows before the rewrite 15;16;17;18;19 never - +truncate_cleanup truncate_cleanup premise: the projected table carries metadata again before the DROP 15;16;17;18;19 never - +truncate_cleanup truncate_cleanup premise: the table carries metadata again, so the DROP has work to do 15;16;17;18;19 never - +truncate_cleanup truncate_cleanup premise: the table holds its first ten rows 15;16;17;18;19 never - +truncate_cleanup truncate_cleanup ten insert-and-truncate cycles on one table leave nothing 15;16;17;18;19 2026-10-16 columnar_tableam.c: a rewrite no longer deletes the retired storage's catalog rows +truncate_cleanup truncate_cleanup the projection agrees with the base table 15;16;17;18;19 2026-10-16 columnar_tableam.c: the in-place truncate no longer clears a projection's own storage +truncate_cleanup truncate_cleanup the projection did add metadata 15;16;17;18;19 never - +truncate_cleanup truncate_cleanup the projection storage holds exactly one row group after the truncate 15;16;17;18;19 2026-10-16 columnar_tableam.c: the in-place truncate no longer clears a projection's own storage +truncate_cleanup truncate_cleanup the rows written after the in-place truncate are all there 15;16;17;18;19 2026-10-16 columnar_tableam.c: the in-place truncate no longer clears a projection's own storage +truncate_cleanup truncate_cleanup three truncate-and-reload cycles still sit at the baseline 15;16;17;18;19 2026-10-16 columnar_tableam.c: a rewrite no longer deletes the retired storage's catalog rows ttl_expire ttl_expire NO row still inside the retention was dropped (#403 item 5a) 15;16;17;18;19 never - ttl_expire ttl_expire NO row still inside the retention was dropped, on a date column 15;16;17;18;19 never - ttl_expire ttl_expire a compaction's retired groups are not recorded as retention 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 3f8e29cde..ef3e3c5d1 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 219 +suites_not_covered 214 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -2943,4 +2943,78 @@ suites_not_covered 219 # never 2676 - 1 = 2675 # dated 943 + 1 = 944 # 2675 + 944 == 3619 -checks_never_observed_red 2675 +# +# #1341, fk_referencing seeded, with its twin (OffgridwithJD): 2675 -> 2686, and +# suites_not_covered 219 -> 218. The suite had no rows. It enters with 28 rows +# from five-major runs: 27 checks on PG17-19 and 26 on 15-16, where the +# partitioned-control branch asserts core's refusal instead of a conversion. 17 are +# dated by four mutations of the foreign-key guards: the referenced-side refusal +# removed, the guard keyed on the referencing table, the conversion refusal +# removed, and the conversion refused whatever the target access method (the new +# referenced-table-to-row-store arm). Eleven arrive never: the over-rejection +# controls that no single guard reaches (another constraint, a heap-to-heap key, +# an unreferenced conversion, an unrelated ALTER, a conversion back to heap, the +# referencing side, the partitioned control on either side of 17) and the primary +# key's own arms. +# +# ledger rows 3619 + 28 = 3647 +# never 2675 + 11 = 2686 +# dated 944 + 17 = 961 +# 2686 + 961 == 3647 +# +# #1341, truncate_cleanup seeded, with its twin (OffgridwithJD): 2686 -> 2695, and +# suites_not_covered 218 -> 217. The suite had no rows. It enters with all 25 checks +# from five-major runs (25/25 on each of 15-19). 16 are dated by four mutations of +# the truncate paths: a rewrite that no longer deletes the retired storage's rows +# (7), nor drops the stale write state (1), an in-place truncate that skips the +# projections' storages (8), and a storage-tree delete that skips them (1). Nine +# arrive never: the premises that each fixture holds rows or metadata, the +# surviving projection declaration, the empty table after TRUNCATE, and the +# rewrite that keeps every row. +# +# ledger rows 3647 + 25 = 3672 +# never 2686 + 9 = 2695 +# dated 961 + 16 = 977 +# 2695 + 977 == 3672 +# +# #1341, native_dml seeded, with its twin (OffgridwithJD): 2695 -> 2702, and +# suites_not_covered 217 -> 216. The suite had no rows. It enters with all 15 checks +# from five-major runs (15/15 on each of 15-19). 8 are dated by three mutations: a +# row group holding deletes still answering min and max from its zone maps (the +# min/max arm, which could not see it before this change), count(*) from metadata +# not subtracting deleted rows (3), and the index fetch ignoring the delete vector +# (5). Seven arrive never: the two premises, the rows and sum after the delete, the +# deleted rows being gone, the live row's index lookup, and the primary key arm. +# +# ledger rows 3672 + 15 = 3687 +# never 2695 + 7 = 2702 +# dated 977 + 8 = 985 +# 2702 + 985 == 3687 +# +# #1341, native_dict_underfill seeded, with its twin (OffgridwithJD): 2702 -> 2704, +# and suites_not_covered 216 -> 215. The suite had no rows. It enters with all 4 +# checks from five-major runs (4/4 on each of 15-19). 2 are dated by two mutations +# of the DICT fill check: removed, so the garbage tail is read back with no error, +# and replaced by a backend crash, which the rejection and the log-counted survival +# arm both catch. The two premises arrive never. +# +# ledger rows 3687 + 4 = 3691 +# never 2702 + 2 = 2704 +# dated 985 + 2 = 987 +# 2704 + 987 == 3691 +# +# #1341, native_decode_gating seeded, with its twin (OffgridwithJD): 2704 -> 2722, and +# suites_not_covered 215 -> 214. The suite had no rows. It enters with all 29 checks +# from five-major runs (29/29 on each of 15-19). 11 are dated by four mutations of +# #452 phase 2: the per-vector evaluation never run (4), a deleted row counted as +# filtered (1, the new arm), every vector marked skipped (7), and the producer +# counting a gated group's rejections twice (2, one of them new). Eighteen arrive +# never: the thirteen premises, the observable's presence, the full scan across +# every group, the nomatch query's empty answer, and two exact counters that no +# single mutation here moves. +# +# ledger rows 3691 + 29 = 3720 +# never 2704 + 18 = 2722 +# dated 987 + 11 = 998 +# 2722 + 998 == 3720 +checks_never_observed_red 2722 diff --git a/test/fk_referencing.sh b/test/fk_referencing.sh index 73b59c41d..cd49dcecd 100755 --- a/test/fk_referencing.sh +++ b/test/fk_referencing.sh @@ -220,6 +220,16 @@ check "converting back to heap is unaffected" \ "$(case "$back" in *ERROR*) echo "no ($back)" ;; *) echo yes ;; esac)" \ "yes" +# AND ON THE REFERENCED TABLE ITSELF (#1341). The control above converts a table +# with no foreign key, so it cannot see a rule that refuses every SET ACCESS METHOD +# on a referenced table whatever the target: with the guard's columnar-target test +# removed, this file stayed 26/26. Only a columnar target needs the row lock the +# constraint cannot get. +refheap="$(psql_run "ALTER TABLE fk_conv_parent SET ACCESS METHOD heap;" 2>&1 || true)" +check "an FK-referenced table may still be set to a row store" \ + "$(case "$refheap" in *ERROR*) echo "no ($refheap)" ;; *) echo yes ;; esac)" \ + "yes" + # --- 7. a partitioned parent is refused too (#201) ----------------------------- # A partitioned table has no storage, so setting its access method breaks nothing diff --git a/test/native_decode_gating.sh b/test/native_decode_gating.sh index fb18ac03c..2b8f9fc7b 100755 --- a/test/native_decode_gating.sh +++ b/test/native_decode_gating.sh @@ -172,6 +172,20 @@ check "the zero-match query removes every row by filter, counted once" \ check "and skips every vector" \ "$(counter_in "$PLAN_NOMATCH" 'Columnar Vectors Skipped')" "$VECTORS" +# A SURVIVING VECTOR WITH REJECTS IN IT (#1341). Every row of the needle's +# surviving vector matches, so the producer, re-testing that vector to find its +# rows, finds nothing to reject: a producer that counted those rejections a second +# time read exactly right above. Measured, with the producer's non-counting filter +# disabled, this file stayed 26/26. Half of vector 0 passes this qual (p1 % 2 = 0 is +# an expression, so it is no skip predicate either), so its 512 rejections are +# counted twice by that mistake. +HALFQ="SELECT * FROM n WHERE tag LIKE '%needle%' AND p1 % 2 = 0" +PLAN_HALF="$(explain_of "$HALFQ")" +check "premise: the half-needle survives in one vector, half of its rows" \ + "$(q "SELECT count(*) || '|' || count(DISTINCT (p1 - 1) / 1024) FROM n WHERE tag LIKE '%needle%' AND p1 % 2 = 0;")" "512|1" +check "rejections inside a surviving vector are counted once" \ + "$(counter_in "$PLAN_HALF" 'Rows Removed by Filter')" "$(( ROWS - 512 ))" + # ---- correctness ----------------------------------------------------------- # # The mask must never drop a row. Without these, an implementation that marks @@ -205,13 +219,18 @@ MGEN="SELECT 'x' || CASE WHEN g % 4096 = 7 THEN 'needle' ELSE 'miss' END || g::t psql_run "INSERT INTO hm $MGEN;" psql_run "INSERT INTO m $MGEN;" psql_run "DELETE FROM hm WHERE p1 % 17 = 0;" -psql_run "DELETE FROM m WHERE p1 % 17 = 0;" +# The fixture's DELETE must not depend on the gating under test: its WHERE is an +# unprunable qual too, so it is run with the width gate out of reach (#1341). +psql_run "SET pgcolumnar.qual_skipvec_min_payload_cols = 100000; DELETE FROM m WHERE p1 % 17 = 0;" check "premise: the columnar table spans more than one row group" \ "$(q "SELECT count(*) > 1 FROM pgcolumnar.row_group WHERE storage_id = pgcolumnar.get_storage_id('m');")" \ "t" +# count(*) rather than a count WHERE p1 % 17 = 0: that WHERE is an unprunable qual +# too, so it ran through the very gating under test, and a gating mutant broke this +# premise rather than the arms below (#1341). check "premise: rows really were deleted (so the mask is non-NULL)" \ - "$(q "SELECT count(*) FROM m WHERE p1 % 17 = 0;")" "0" + "$(q "SELECT count(*) FROM m;")" "$(( MROWS - MROWS / 17 ))" check "deletes + multiple groups + unprunable qual: columnar matches the heap" \ "$(pgc_set_hash "SELECT * FROM m WHERE tag LIKE '%needle%'")" \ @@ -220,4 +239,13 @@ check "and the full scan across every group's mask matches too" \ "$(pgc_set_hash "SELECT * FROM m")" \ "$(pgc_set_hash "SELECT * FROM hm")" +# AND THE COUNTER SKIPS DELETED ROWS (#1341). The evaluator must not count a deleted +# row as filtered: it is invisible. Only the single-group table above had its +# counters checked, and it has no deletes, so with the evaluator's deleted-row test +# removed this file stayed 26/26. +PLAN_M="$(explain_of "SELECT * FROM m WHERE tag LIKE '%needle%'")" +check "rows removed by filter across deleted rows count only live rows" \ + "$(counter_in "$PLAN_M" 'Rows Removed by Filter')" \ + "$(q "SELECT count(*) FROM hm WHERE tag NOT LIKE '%needle%';")" + pgc_summary diff --git a/test/native_dict_underfill.sh b/test/native_dict_underfill.sh index bf32d082c..b5cf5bf8e 100755 --- a/test/native_dict_underfill.sh +++ b/test/native_dict_underfill.sh @@ -27,23 +27,31 @@ echo "-- v-column vector0 encoding type byte (DICT=6): $ENC" check "the text column is DICT-encoded (precondition)" "$ENC" "6" # read old rawLen (uint32 LE at descriptor offset 11), inflate by 64, write back -q "UPDATE pgcolumnar.column_chunk c SET encoding_descriptor = +crashes() { + # grep -c prints 0 itself when nothing matches, and then exits 1 + pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null || true +} +crashes0="$(crashes)" +q "WITH u AS (UPDATE pgcolumnar.column_chunk c SET encoding_descriptor = set_byte(set_byte(set_byte(set_byte(encoding_descriptor, 11, ( (get_byte(encoding_descriptor,11) + (get_byte(encoding_descriptor,12)<<8) + (get_byte(encoding_descriptor,13)<<16) + (get_byte(encoding_descriptor,14)<<24)) + 64 ) & 255), 12, (((get_byte(encoding_descriptor,11) + (get_byte(encoding_descriptor,12)<<8) + (get_byte(encoding_descriptor,13)<<16) + (get_byte(encoding_descriptor,14)<<24)) + 64) >> 8) & 255), 13, (((get_byte(encoding_descriptor,11) + (get_byte(encoding_descriptor,12)<<8) + (get_byte(encoding_descriptor,13)<<16) + (get_byte(encoding_descriptor,14)<<24)) + 64) >>16) & 255), 14, (((get_byte(encoding_descriptor,11) + (get_byte(encoding_descriptor,12)<<8) + (get_byte(encoding_descriptor,13)<<16) + (get_byte(encoding_descriptor,14)<<24)) + 64) >>24) & 255) - WHERE c.storage_id=$SID AND c.column_index=1" >/dev/null + WHERE c.storage_id=$SID AND c.column_index=1 RETURNING 1) SELECT count(*) FROM u" > "$PGC_WORKDIR/uf.n" +check "premise: the inflation rewrote the DICT chunk's descriptor" "$(cat "$PGC_WORKDIR/uf.n")" "1" env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" \ - -c "SELECT v FROM dt" >/dev/null 2>"$PGC_WORKDIR/uf.err" + -v VERBOSITY=verbose -c "SELECT v FROM dt" >/dev/null 2>"$PGC_WORKDIR/uf.err" echo "-- select stderr:"; sed 's/^/ /' "$PGC_WORKDIR/uf.err" | head -3 -rejected=no -grep -qiE 'does not match|corrupt|invalid|out of' "$PGC_WORKDIR/uf.err" && rejected=yes -crash=no -grep -qiE 'terminated by signal|segmentation|server closed' "$PGC_WORKDIR/uf.err" && crash=yes -grep -qiE 'terminated by signal|segmentation' "$PGC_LOGFILE" 2>/dev/null && crash=yes -echo "-- rejected=$rejected crash=$crash" -check "under-filled DICT chunk is rejected with a clean error" "$rejected" "yes" -check "backend survives the under-filled DICT chunk" "$(q 'SELECT 1')" "1" +# A CLEAN ERROR IS XX001, AND A CRASH IS READ FROM THE SERVER LOG (#1341). The +# rejection used to match 'does not match|corrupt|invalid|out of' in any message, +# which "out of memory" satisfies too; and survival was a fresh SELECT 1, which a +# crashed backend passes once crash recovery has finished. The crash flag below was +# computed and never checked. +rejected="$(sed -nE 's/^ERROR: ([0-9A-Z]{5}): .*/\1/p' "$PGC_WORKDIR/uf.err" | head -1 | grep . || echo "no SQLSTATE ($(head -1 "$PGC_WORKDIR/uf.err"))")" +echo "-- rejected=$rejected crashes=$(( $(crashes) - crashes0 ))" +check "under-filled DICT chunk is rejected with a clean error" "$rejected" "XX001" +check "backend survives the under-filled DICT chunk" \ + "$(q 'SELECT 1')|$(( $(crashes) - crashes0 )) backend crashes" "1|0 backend crashes" pgc_summary diff --git a/test/native_dml.sh b/test/native_dml.sh index 9e67578b6..ae68bc317 100755 --- a/test/native_dml.sh +++ b/test/native_dml.sh @@ -21,9 +21,15 @@ GEN="SELECT g, (g * 10)::bigint, 'row-' || g FROM generate_series(1, 5000) g" psql_run "INSERT INTO h $GEN;" psql_run "INSERT INTO n $GEN;" -# DELETE across several row groups. -psql_run "DELETE FROM h WHERE id % 7 = 0;" -psql_run "DELETE FROM n WHERE id % 7 = 0;" +# DELETE across several row groups, and the table's smallest and largest ids with +# them. Without those two, min and max are the same whether or not the deleted rows +# count, so the min/max arm below could not see a zone-map answer that ignored the +# deletes (#1341): measured, with groups that hold deletes left on the zone-map path, +# this file stayed 13/13. +psql_run "DELETE FROM h WHERE id % 7 = 0 OR id IN (1, 5000);" +psql_run "DELETE FROM n WHERE id % 7 = 0 OR id IN (1, 5000);" +check "premise: the deletes moved the smallest and largest id and k" \ + "$(q 'SELECT min(id) || '"'"'|'"'"' || max(id) || '"'"'|'"'"' || min(k) || '"'"'|'"'"' || max(k) FROM h;')" "2|4999|20|49990" check "count after delete" "$(q 'SELECT count(*) FROM n;')" "$(q 'SELECT count(*) FROM h;')" check "rows after delete match heap" \ @@ -32,14 +38,20 @@ check "rows after delete match heap" \ # Aggregates after delete: the zone-map path is bypassed (deletes present), so this # exercises the delete-applying scan fold. check "sum(k) after delete" "$(q 'SELECT sum(k) FROM n;')" "$(q 'SELECT sum(k) FROM h;')" +# k as well as id: measured, a zone-map answer that ignored the deletes still gave +# min/max(id) right and min/max(k) wrong (10|50000 for 20|49990). check "min/max after delete" \ - "$(q 'SELECT min(id), max(id) FROM n;')" \ - "$(q 'SELECT min(id), max(id) FROM h;')" + "$(q 'SELECT min(id), max(id), min(k), max(k) FROM n;')" \ + "$(q 'SELECT min(id), max(id), min(k), max(k) FROM h;')" check "deleted rows gone" "$(q 'SELECT count(*) FROM n WHERE id % 7 = 0;')" "0" # Index lookup of a deleted row returns nothing; a live row still resolves. idx() { env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -q -At -c "SET enable_seqscan=off" -c "$1" 2>/dev/null; } # id 4444 is live (4444 % 7 = 6); id 4445 was deleted (4445 % 7 = 0). +# The lookups must take the primary-key index, or these arms test the scan instead +# of the index fetch's delete check. +check "premise: the lookups take the primary-key index" \ + "$(idx 'EXPLAIN (COSTS off) SELECT count(*) FROM n WHERE id = 4445;' | grep -c 'Index.* using n_pkey')" "1" check "index finds live row" "$(idx 'SELECT v FROM n WHERE id = 4444;')" "$(q 'SELECT v FROM h WHERE id = 4444;')" check "index skips deleted row" "$(idx 'SELECT count(*) FROM n WHERE id = 4445;')" "0" diff --git a/test/native_upgrade_converge.sh b/test/native_upgrade_converge.sh index 0249d5655..454900137 100755 --- a/test/native_upgrade_converge.sh +++ b/test/native_upgrade_converge.sh @@ -310,4 +310,53 @@ done rm -f "$REF" P -d postgres -c "DROP DATABASE IF EXISTS conv_fresh;" >/dev/null +# ---- the window: this library over the previous release's catalog (#1341 item 66) ---- +# +# docs/installation.md steps 1 and 2 put the new library in place; step 3, +# ALTER EXTENSION UPDATE, brings the catalog up to it. Between them a database +# runs this library over its previous release's catalog. The library deletes +# from retired_group unconditionally (columnar_metadata.c, PgColumnarDeleteMetadata), +# which that catalog does not have, so DROP TABLE, TRUNCATE and pgcolumnar.vacuum() +# fail there: measured, with 42P01 'columnar metadata table +# "pgcolumnar.retired_group" does not exist', which names neither the cause nor step 3. +# +# The arms hold what the window must keep: reads and writes work, a refused +# operation changes nothing and names the remedy, and step 3 makes every one of +# them work on the same rows. PREV is the newest fixture, so these follow the +# release they describe rather than naming it. +PREV="${_NUC_FIXTURES[${#_NUC_FIXTURES[@]}-1]}" +WIN=conv_window +P -d postgres -c "DROP DATABASE IF EXISTS $WIN;" >/dev/null +P -d postgres -c "CREATE DATABASE $WIN;" >/dev/null +P -d "$WIN" -c "CREATE EXTENSION pgcolumnar VERSION '$PREV';" >/dev/null +check "premise: the window database runs this library over the previous release's catalog" \ + "$(P -d "$WIN" -c "SELECT extversion FROM pg_extension WHERE extname='pgcolumnar';")" "$PREV" +check "premise: that catalog has no retired_group, which this library deletes from" \ + "$(P -d "$WIN" -c "SELECT count(*) FROM pg_class WHERE relname = 'retired_group' AND relnamespace = 'pgcolumnar'::regnamespace;")" "0" +P -d "$WIN" -c "CREATE TABLE w (a int, b text) USING pgcolumnar;" >/dev/null +P -d "$WIN" -c "INSERT INTO w SELECT g, 'x' || g FROM generate_series(1, 20000) g;" >/dev/null +P -d "$WIN" -c "DELETE FROM w WHERE a % 3 = 0;" >/dev/null +win_rows() { P -d "$WIN" -c "SELECT count(*) || '|' || sum(a) FROM w;"; } +check "in the window, a columnar table is written, deleted from and read" "$(win_rows)" "13334|133346667" +win_refusal() { # win_refusal SQL -> SQLSTATE|whether the HINT names step 3, or what happened instead + local out + out="$(P -d "$WIN" -v VERBOSITY=verbose -c "$1" 2>&1)" && { echo accepted; return; } + echo "$(sed -nE 's/^ERROR: ([0-9A-Z]{5}): .*/\1/p' <<<"$out" | head -1)|$(grep -q '^HINT: .*ALTER EXTENSION pgcolumnar UPDATE' <<<"$out" && echo names-the-update || echo no-hint)" +} +check "in the window, DROP TABLE is refused and names ALTER EXTENSION UPDATE" \ + "$(win_refusal "DROP TABLE w;")" "42P01|names-the-update" +check "in the window, TRUNCATE is refused and names ALTER EXTENSION UPDATE" \ + "$(win_refusal "TRUNCATE w;")" "42P01|names-the-update" +check "in the window, pgcolumnar.vacuum() is refused and names ALTER EXTENSION UPDATE" \ + "$(win_refusal "SELECT pgcolumnar.vacuum('w');")" "42P01|names-the-update" +check "in the window, the refused operations changed nothing" "$(win_rows)" "13334|133346667" + +P -d "$WIN" -c "ALTER EXTENSION pgcolumnar UPDATE;" >/dev/null +check "after ALTER EXTENSION UPDATE, the same rows are there" "$(win_rows)" "13334|133346667" +check "after ALTER EXTENSION UPDATE, pgcolumnar.vacuum() succeeds and keeps them" \ + "$(P -d "$WIN" -c "SELECT pgcolumnar.vacuum('w');" >/dev/null 2>&1 && win_rows || echo refused)" "13334|133346667" +check "after ALTER EXTENSION UPDATE, TRUNCATE and DROP TABLE succeed" \ + "$(P -d "$WIN" -c "TRUNCATE w;" >/dev/null 2>&1 && P -d "$WIN" -c "DROP TABLE w;" >/dev/null 2>&1 && echo succeeded || echo refused)" "succeeded" +P -d postgres -c "DROP DATABASE IF EXISTS $WIN;" >/dev/null + pgc_summary diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 9212ee1a5..3ae974458 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -188,6 +188,12 @@ behaviour, the source of that number is named. - [140. test_parquet_import.py: a partitioned columnar parent is refused by import_parquet](#140-test_parquet_importpy-a-partitioned-columnar-parent-is-refused-by-import_parquet) - [141. test_phase5.py: set_options and reset_options agree](#141-test_phase5py-set_options-and-reset_options-agree) - [142. test_drop_cleanup.py: a columnar matview's options row goes with its DROP](#142-test_drop_cleanuppy-a-columnar-matviews-options-row-goes-with-its-drop) +- [143. test_native_upgrade_converge.py: the upgrade window, before ALTER EXTENSION UPDATE](#143-test_native_upgrade_convergepy-the-upgrade-window-before-alter-extension-update) +- [144. test_fk_referencing.py: a foreign key referencing a columnar table is refused where it is chosen](#144-test_fk_referencingpy-a-foreign-key-referencing-a-columnar-table-is-refused-where-it-is-chosen) +- [145. test_truncate_cleanup.py: TRUNCATE leaves no catalog rows behind, and keeps what it must](#145-test_truncate_cleanuppy-truncate-leaves-no-catalog-rows-behind-and-keeps-what-it-must) +- [146. test_native_dml.py: DELETE, UPDATE and MERGE on a native table agree with heap](#146-test_native_dmlpy-delete-update-and-merge-on-a-native-table-agree-with-heap) +- [147. test_native_dict_underfill.py: an under-filled DICT chunk is refused and no backend dies](#147-test_native_dict_underfillpy-an-under-filled-dict-chunk-is-refused-and-no-backend-dies) +- [148. test_native_decode_gating.py: an unprunable qual gates payload decode per vector](#148-test_native_decode_gatingpy-an-unprunable-qual-gates-payload-decode-per-vector) ## 1. How to read a test in here @@ -7377,3 +7383,73 @@ Part of the pytest twin of `test/drop_cleanup.sh` (#1265, ported for #1341): its | `test_a_columnar_matview_takes_its_options_row_with_it` | a planted options row is gone after DROP MATERIALIZED VIEW | | `test_refresh_keeps_a_matviews_options_row` | a REFRESH that rewrote the matview leaves its set_options row in place | | `test_an_unrelated_options_row_survives` | another table's row survives a matview's DROP, and an ordinary columnar table still takes its own | + +## 143. test_native_upgrade_converge.py: the upgrade window, before ALTER EXTENSION UPDATE + +Part of the pytest twin of `test/native_upgrade_converge.sh` (#1341 item 66): its upgrade-window arms only, by the shell's names, and the pair is declared INCOMPLETE. Between installing a new library and running `ALTER EXTENSION pgcolumnar UPDATE`, a database runs the new library over its previous release's catalog. Reads and writes must work there. `DROP TABLE`, `TRUNCATE` and `pgcolumnar.vacuum()`, which need a catalog table the old one lacks, must be refused with 42P01, change nothing, and carry a hint naming the update; after the update all three succeed on the same rows. The previous release is the version the shipped upgrade script to the current `default_version` starts from; its base script is staged from `test/fixtures` for the test and removed after. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_the_upgrade_window` | at the previous catalog: rows written, deleted and read; the three operations refused with the hint and nothing changed; after the update the rows are intact and all three succeed | + +## 144. test_fk_referencing.py: a foreign key referencing a columnar table is refused where it is chosen + +The pytest twin of `test/fk_referencing.sh` (ported for #1341), every check by the shell's name. Enforcing a foreign key locks the referenced row, and columnar tables cannot lock rows, so a constraint referencing one would make every insert into the referencing table fail. It is refused at DDL by either route into that state: creating the constraint against a columnar table, or converting a referenced table (or partitioned parent) to columnar. Only the referenced side is refused, and the controls hold everything else steady, including converting the referenced table itself to a row store. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_the_referenced_side_is_refused_at_ddl` | CREATE TABLE and ALTER TABLE ADD CONSTRAINT referencing a columnar table are refused, saying why, and nothing is left behind | +| `test_the_referencing_side_still_works` | a columnar table referencing a heap table is created, accepts satisfying rows and rejects a violating one | +| `test_constraints_that_need_no_row_lock_are_unaffected` | primary key, unique, check and not-null on a columnar table, and a heap-to-heap foreign key, are unaffected | +| `test_converting_a_referenced_table_is_refused_and_nothing_else_is` | SET ACCESS METHOD pgcolumnar on a referenced table and the helper are refused, the table stays heap and its child writable; an unreferenced table, the referencing side, an unrelated ALTER, a conversion back to heap and the referenced table to a row store all still work | +| `test_a_partitioned_referenced_parent` | a columnar partition and setting the parent's access method are refused, the parent does not become columnar, an ordinary partition still works; an unreferenced partitioned table converts from PG17, and before 17 core refuses it | + +## 145. test_truncate_cleanup.py: TRUNCATE leaves no catalog rows behind, and keeps what it must + +The pytest twin of `test/truncate_cleanup.sh` (ported for #1341), every check by the shell's name. TRUNCATE installs a new storage id, and the old storage's catalog rows used to stay behind, so each truncate-and-reload cycle leaked one storage's worth and a later DROP removed only the empty current one. Retiring them made a cached write state stale, which is dropped with them. A relation created in the current transaction is truncated in place, on a different path, which must clear each projection's storage too. A private database, since the snapshot counts every pgcolumnar catalog row. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_truncate_and_drop_leave_the_catalog_as_they_found_it` | one TRUNCATE and three truncate-and-reload cycles return the catalog to the baseline, and a DROP of a reloaded table leaves nothing | +| `test_a_projection_is_not_leaked_by_truncate` | no projection row names a retired storage id, the declaration survives, and a DROP of the reloaded table leaves nothing | +| `test_repeated_truncates_accumulate_nothing` | ten transactional truncates of one table, and its DROP, leave the baseline | +| `test_insert_truncate_insert_in_one_transaction` | the transaction commits and holds exactly the rows written after the TRUNCATE | +| `test_a_rewrite_that_is_not_a_truncate_loses_nothing` | a type-changing rewrite keeps all 5000 rows readable | +| `test_an_in_place_truncate_clears_the_projection_too` | the in-place truncate commits, the later rows are all there, the projection reads them back and its storage holds one group of exactly them | + +## 146. test_native_dml.py: DELETE, UPDATE and MERGE on a native table agree with heap + +The pytest twin of `test/native_dml.sh` (ported for #1341), every check by the shell's name. Each statement runs on a heap table and on a columnar one with 1000-row stripes, and every result is compared. After deletes an aggregate must not come from zone maps that still describe the deleted rows: the deletes include the smallest and largest ids, so min and max move, and they are taken over `k`, a column the primary key does not order. A deleted row's index lookup finds nothing, and the lookups are asserted to take the index. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_delete_update_and_merge_match_heap` | after DELETE, UPDATE and MERGE, counts, rows, sum and min/max match heap; the index finds a live row and skips a deleted one; the primary key still rejects a duplicate | + +## 147. test_native_dict_underfill.py: an under-filled DICT chunk is refused and no backend dies + +The pytest twin of `test/native_dict_underfill.sh` (ported for #1341), every check by the shell's name. A DICT chunk's `value_raw_length` is inflated in its descriptor, so the codes decode to fewer bytes than declared; without the fill check the raw buffer's tail is uninitialized and a varlena length is read out of it. The read must raise XX001, and survival is read from the server log's crash count, not from a fresh connection, which succeeds once crash recovery has finished. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_an_under_filled_dict_chunk_is_refused` | the column is DICT-encoded, the inflation rewrites its descriptor, the read is refused with XX001, and no backend died | + +## 148. test_native_decode_gating.py: an unprunable qual gates payload decode per vector + +The pytest twin of `test/native_decode_gating.sh` (#452 phase 2, ported for #1341), every check by the shell's name. A leading-wildcard LIKE is no skip predicate, so phase 2 evaluates the qual per 1024-row vector between the decode passes, and a vector no row passes is never decoded for its payload columns; "Columnar Vector Decodes" makes the saving exact arithmetic. The filter counters stay exact: a rejection is counted once, inside a surviving vector too, and a deleted row is not counted. The width gate is set off for the session. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_an_unprunable_qual_gates_payload_decode` | the premises (no skip predicate, the late-materialization path); the control decodes all 192 (vector, column) pairs, a zero-match LIKE 32 and the needle 37; every filter counter is exact, including rejections inside the surviving vector; the answers match heap | +| `test_deleted_rows_across_several_groups` | with deletes over several groups the answers match heap, and Rows Removed by Filter counts only live rows | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index a6a77ccba..8741e7acf 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -747,4 +747,16 @@ guard_tests 438 # seven tests, by collection over the cluster files on this tree. # cluster_tests 653 -> 660 for #1341 item 29's partial twins: test_inheritance.py (2), # test_parquet_import.py (1), test_phase5.py (1) and test_drop_cleanup.py (3), by collection. -cluster_tests 660 +# cluster_tests 660 -> 661 for #1341 item 66's test_native_upgrade_converge.py, one test, +# by collection over the cluster files on this tree. +# cluster_tests 661 -> 666 for #1341's test_fk_referencing.py, the twin of fk_referencing.sh: +# five tests, by collection over the cluster files on this tree. +# cluster_tests 666 -> 672 for #1341's test_truncate_cleanup.py, the twin of truncate_cleanup.sh: +# six tests, by collection over the cluster files on this tree. +# cluster_tests 672 -> 673 for #1341's test_native_dml.py, the twin of native_dml.sh: one test, +# by collection over the cluster files on this tree. +# cluster_tests 673 -> 674 for #1341's test_native_dict_underfill.py, the twin of +# native_dict_underfill.sh: one test, by collection over the cluster files on this tree. +# cluster_tests 674 -> 676 for #1341's test_native_decode_gating.py, the twin of +# native_decode_gating.sh: two tests, by collection over the cluster files on this tree. +cluster_tests 676 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 4140fbf32..7bd9cd799 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -106,6 +106,7 @@ "differential", "encode_post_codec", "entry_point_privilege", + "fk_referencing", "format_downgrade", "hilbert_cluster", "hilbert_locality", @@ -119,8 +120,11 @@ "maintenance_rollback", "native_backend_crash", "native_chunk_length_bound", + "native_decode_gating", "native_delete_vector_index", "native_delete_visibility_paths", + "native_dict_underfill", + "native_dml", "native_fetch_coalesce", "native_groupagg", "native_index_fetch_stripe_cost", @@ -166,6 +170,7 @@ "tid_range_scan", "tier_partition", "timeseries_recipes", + "truncate_cleanup", "ttl_expire", "type_matrix", "unique_conc", @@ -179,6 +184,10 @@ # about the PORT rather than a licence: the standing arm does not grade it, so the # reason is the only thing standing between a declared gap and a forgotten one. INCOMPLETE = { + # #1341 item 66 ported the arms it added: the upgrade window, this library over + # the previous release's catalog before ALTER EXTENSION UPDATE. + "native_upgrade_converge": "the upgrade-window arms are ported; the fixture-blob, " + "staging and catalog-convergence arms are not", # #1341 item 29 ported the arms four fixes added with shell arms only. Each # pair below holds that fix's arms and nothing else of its shell suite. "inheritance": "the #1259 arms (a partitioned columnar parent is refused with " diff --git a/test/pytest/test_fk_referencing.py b/test/pytest/test_fk_referencing.py new file mode 100644 index 000000000..3e7a60274 --- /dev/null +++ b/test/pytest/test_fk_referencing.py @@ -0,0 +1,158 @@ +"""A foreign key that references a columnar table is refused where it is chosen +(#1341). The pytest twin of `test/fk_referencing.sh`, every check by the shell's name. + +Referential integrity reads the referenced row with FOR KEY SHARE, and row locking is +not implemented for columnar tables, so a constraint referencing one would make every +insert into the referencing table fail. It is refused at DDL instead, by either route +into that state: creating the constraint against a columnar table, or converting a +referenced table to columnar. Only the REFERENCED side is refused: a columnar table may +reference a heap table, and the controls hold everything else steady -- other +constraints, heap-to-heap keys, conversions of an unreferenced table, of the +referencing side, back to heap, and of the referenced table to a row store. +""" +import psycopg + + +def _err(conn, sql): + """-> '' when sql succeeds, else the error's message, detail and hint, joined.""" + try: + with conn.cursor() as cur: + cur.execute(sql) + return "" + except psycopg.Error as exc: + d = exc.diag + return " ".join(x for x in (d.message_primary, d.message_detail, d.message_hint) if x) or str(exc) + + +def _has(text, needle): + return "yes" if needle in text else f"no ({text})" + + +def _ok(text): + return "yes" if not text else f"no ({text})" + + +def _one(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetchone()[0] + + +def _run(conn, *sqls): + with conn.cursor() as cur: + for sql in sqls: + cur.execute(sql) + + +def test_the_referenced_side_is_refused_at_ddl(pgc_conn, expect): + conn = pgc_conn + _run(conn, "CREATE TABLE fk_parent (id int PRIMARY KEY) USING pgcolumnar", + "INSERT INTO fk_parent VALUES (1), (2)") + err = _err(conn, "CREATE TABLE fk_child (id int REFERENCES fk_parent(id))") + expect.text(_has(err, "cannot create a foreign key referencing columnar table"), "yes", + "a foreign key referencing a columnar table is refused") + expect.text(_has(err, "row locking is not supported"), "yes", "and the error says why") + expect.num(_one(conn, "SELECT count(*) FROM pg_class WHERE relname = 'fk_child' " + "AND relnamespace = current_schema()::regnamespace"), 0, + "the refused table was not left behind") + + _run(conn, "CREATE TABLE fk_child2 (id int)") + err2 = _err(conn, "ALTER TABLE fk_child2 ADD CONSTRAINT fk_c2 FOREIGN KEY (id) REFERENCES fk_parent(id)") + expect.text(_has(err2, "cannot create a foreign key referencing columnar table"), "yes", + "ALTER TABLE ADD CONSTRAINT is refused by the same rule") + + +def test_the_referencing_side_still_works(pgc_conn, expect): + conn = pgc_conn + _run(conn, "CREATE TABLE fk_h_parent (id int PRIMARY KEY)", "INSERT INTO fk_h_parent VALUES (1), (2), (3)") + expect.text(_ok(_err(conn, "CREATE TABLE fk_c_child (id int REFERENCES fk_h_parent(id), v text) " + "USING pgcolumnar")), "yes", + "a columnar table may reference a heap table") + _err(conn, "INSERT INTO fk_c_child VALUES (1, 'a'), (2, 'b')") + expect.num(_one(conn, "SELECT count(*) FROM fk_c_child"), 2, "and rows that satisfy it are accepted") + expect.text(_has(_err(conn, "INSERT INTO fk_c_child VALUES (99, 'nope')"), "violates foreign key constraint"), + "yes", "a row that violates it is still rejected") + + +def test_constraints_that_need_no_row_lock_are_unaffected(pgc_conn, expect): + conn = pgc_conn + expect.text(_ok(_err(conn, "CREATE TABLE fk_other (id int PRIMARY KEY, k int UNIQUE, v int CHECK (v > 0), " + "w int NOT NULL) USING pgcolumnar")), "yes", + "primary key, unique, check and not-null on a columnar table are unaffected") + _err(conn, "INSERT INTO fk_other VALUES (1, 10, 5, 7)") + expect.num(_one(conn, "SELECT count(*) FROM fk_other"), 1, "and they still work") + expect.text(_has(_err(conn, "INSERT INTO fk_other VALUES (1, 11, 5, 7)"), "duplicate key"), "yes", + "the primary key is still enforced") + _run(conn, "CREATE TABLE fk_hh_parent (id int PRIMARY KEY)") + expect.text(_ok(_err(conn, "CREATE TABLE fk_hh_child (id int REFERENCES fk_hh_parent(id))")), "yes", + "a foreign key between two heap tables is unaffected") + + +def test_converting_a_referenced_table_is_refused_and_nothing_else_is(pgc_conn, expect): + conn = pgc_conn + _run(conn, "CREATE TABLE fk_conv_parent (id int PRIMARY KEY)", + "CREATE TABLE fk_conv_child (id int REFERENCES fk_conv_parent(id))", + "INSERT INTO fk_conv_parent VALUES (1)", "INSERT INTO fk_conv_child VALUES (1)") + conv = _err(conn, "ALTER TABLE fk_conv_parent SET ACCESS METHOD pgcolumnar") + expect.text(_has(conv, "referenced by a foreign key"), "yes", + "converting an FK-referenced table to columnar is refused") + expect.text(_has(conv, "row locking is not supported"), "yes", "and the error names the constraint") + expect.text(_one(conn, "SELECT am.amname::text FROM pg_class c JOIN pg_am am ON am.oid = c.relam " + "WHERE c.oid = 'fk_conv_parent'::regclass"), "heap", "the parent is still a row store") + _err(conn, "INSERT INTO fk_conv_parent VALUES (2)") + expect.text(_ok(_err(conn, "INSERT INTO fk_conv_child VALUES (2)")), "yes", + "the child is still writable afterwards") + expect.text(_has(_err(conn, "SELECT pgcolumnar.alter_table_set_access_method('fk_conv_parent', 'pgcolumnar')"), + "referenced by a foreign key"), "yes", "the conversion helper is refused identically") + + # The over-rejection controls: a rule keyed on the wrong side, or firing for any + # ALTER TABLE or any target, passes every arm above and fails one of these. + _run(conn, "CREATE TABLE fk_conv_plain (id int PRIMARY KEY)", "INSERT INTO fk_conv_plain VALUES (1)") + expect.text(_ok(_err(conn, "ALTER TABLE fk_conv_plain SET ACCESS METHOD pgcolumnar")), "yes", + "a table with no foreign key still converts") + expect.text(_ok(_err(conn, "ALTER TABLE fk_conv_child SET ACCESS METHOD pgcolumnar")), "yes", + "converting the referencing side is still allowed") + expect.text(_ok(_err(conn, "ALTER TABLE fk_conv_parent ADD COLUMN note text")), "yes", + "an unrelated ALTER TABLE on a referenced table is unaffected") + expect.text(_ok(_err(conn, "ALTER TABLE fk_conv_plain SET ACCESS METHOD heap")), "yes", + "converting back to heap is unaffected") + # The control above has no foreign key; this one is the referenced table itself. + expect.text(_ok(_err(conn, "ALTER TABLE fk_conv_parent SET ACCESS METHOD heap")), "yes", + "an FK-referenced table may still be set to a row store") + + +def test_a_partitioned_referenced_parent(pgc_conn, expect): + """Setting a partitioned parent's access method chooses what every later partition + inherits, and core clones the foreign key to each, so it is refused too (#201).""" + conn = pgc_conn + _run(conn, "CREATE TABLE fk_pp (id int PRIMARY KEY) PARTITION BY RANGE (id)", + "CREATE TABLE fk_pp1 PARTITION OF fk_pp FOR VALUES FROM (1) TO (100)", + "CREATE TABLE fk_pc (id int REFERENCES fk_pp(id))") + expect.text(_has(_err(conn, "CREATE TABLE fk_pp2 PARTITION OF fk_pp FOR VALUES FROM (100) TO (200) " + "USING pgcolumnar"), "cannot create a foreign key referencing columnar table"), + "yes", "a columnar partition of an FK-referenced parent is refused") + expect.text(_has(_err(conn, "ALTER TABLE fk_pp SET ACCESS METHOD pgcolumnar"), "cannot convert table"), "yes", + "and setting the parent's access method is refused as well") + # "did it become columnar", not "is it heap": a partitioned table never given an + # access method has relam = 0, which joins to no pg_am row. + expect.num(_one(conn, "SELECT count(*) FROM pg_class c JOIN pg_am a ON a.oid = c.relam " + "WHERE c.oid = 'fk_pp'::regclass AND a.amname = 'pgcolumnar'"), 0, + "so the parent did not become columnar") + expect.num(0 if not _err(conn, "CREATE TABLE fk_pp3 PARTITION OF fk_pp FOR VALUES FROM (200) TO (300)") else 1, + 0, "and a partition can still be created the ordinary way") + + _run(conn, "CREATE TABLE fk_free (id int) PARTITION BY RANGE (id)") + major = int(_one(conn, "SHOW server_version_num")) // 10000 + if major >= 17: + expect.num(0 if not _err(conn, "ALTER TABLE fk_free SET ACCESS METHOD pgcolumnar") else 1, 0, + "a partitioned table with no foreign key still converts") + _err(conn, "CREATE TABLE fk_free1 PARTITION OF fk_free FOR VALUES FROM (1) TO (100)") + expect.text(_one(conn, "SELECT amname::text FROM pg_class c JOIN pg_am a ON a.oid = c.relam " + "WHERE c.oid = 'fk_free1'::regclass"), "pgcolumnar", + "and its partitions inherit the columnar access method") + else: + # Before 17 core refuses SET ACCESS METHOD on every partitioned table, so the + # refusal must be core's, by core's message. + expect.text(_has(_err(conn, "ALTER TABLE fk_free SET ACCESS METHOD pgcolumnar"), + "cannot change access method of a partitioned table"), "yes", + "before 17, core refuses this for every partitioned table") diff --git a/test/pytest/test_native_decode_gating.py b/test/pytest/test_native_decode_gating.py new file mode 100644 index 000000000..e2a6e5148 --- /dev/null +++ b/test/pytest/test_native_decode_gating.py @@ -0,0 +1,158 @@ +"""An unprunable qual gates payload-column decode per vector (#452 phase 2, #1341). +The pytest twin of `test/native_decode_gating.sh`, every check by the shell's name. + +A leading-wildcard LIKE is never a skip predicate, so the reader cannot rule a vector +out by value; phase 2 evaluates the executor qual per 1024-row vector between the two +decode passes, and a vector no row passes is never decoded for its payload columns. +"Columnar Vector Decodes" counts (vector, column) decodes, so the saving is exact +arithmetic. The filter counters must stay exact too: a rejection is counted once, +inside a surviving vector as well as a skipped one, and a deleted row is not counted at +all. The width gate (#595) is set off for this session, as the shell pins it off for +its cluster, so the five-column projection still gates. +""" +import re + +ROWS = 32768 +VECTORS = 32 # ROWS / 1024 +NCOLS = 6 # tag plus five payload columns +SURVIVING_VECTORS = 1 # the needle lives only in the first vector +MATCHES = 1024 +MROWS = 49152 + + +def _one(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetchone()[0] + + +def _rows(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetchall() + + +def _run(conn, *sqls): + with conn.cursor() as cur: + for sql in sqls: + cur.execute(sql) + + +def _plan(conn, sql): + return "\n".join(r[0] for r in _rows(conn, f"EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) {sql}")) + + +def _counter(plan, label): + """-> the first integer on the plan's line holding label, or None.""" + for line in plan.splitlines(): + if label in line: + m = re.search(r"\d+", line[line.index(label) + len(label):]) or re.search(r"\d+", line) + return int(m.group(0)) if m else None + return None + + +def _has(plan, label): + return "yes" if label in plan else "no" + + +NEEDLEQ = "SELECT * FROM n WHERE tag LIKE '%needle%'" +NOMATCHQ = "SELECT * FROM n WHERE tag LIKE '%zzzzz%'" +CONTROLQ = "SELECT * FROM n WHERE tag LIKE '%row%'" +HALFQ = "SELECT * FROM n WHERE tag LIKE '%needle%' AND p1 % 2 = 0" + + +def test_an_unprunable_qual_gates_payload_decode(pgc_conn, expect): + conn = pgc_conn + gen = (f"SELECT 'row' || CASE WHEN g <= 1024 THEN 'needle' ELSE 'miss' END, " + f"g, g + 1, g + 2, g + 3, g + 4 FROM generate_series(1, {ROWS}) g") + _run(conn, "SET pgcolumnar.qual_skipvec_min_payload_cols = 0", + "CREATE TABLE h (tag text, p1 int, p2 int, p3 int, p4 int, p5 int)", + "CREATE TABLE n (tag text, p1 int, p2 int, p3 int, p4 int, p5 int) USING pgcolumnar", + "SELECT pgcolumnar.set_options('n', stripe_row_limit => 65536, chunk_group_row_limit => 1024)", + f"INSERT INTO h {gen}", f"INSERT INTO n {gen}") + needle, nomatch, control = _plan(conn, NEEDLEQ), _plan(conn, NOMATCHQ), _plan(conn, CONTROLQ) + + expect.text(_has(needle, "Columnar Projected Columns"), "yes", "premise: the needle plan is a columnar custom scan") + expect.num(_one(conn, "SELECT count(*) FROM pgcolumnar.row_group " + "WHERE storage_id = pgcolumnar.get_storage_id('n')"), 1, + "premise: one row group, so every count below is one group's") + expect.text(_one(conn, "SHOW pgcolumnar.qual_skipvec_min_payload_cols"), "0", + "premise: the width gate is pinned off, so this narrow projection still gates") + expect.num(_counter(needle, "Columnar Usable Skip Predicates"), 0, + "premise: the reader has no usable skip predicate for this LIKE") + expect.num(_counter(needle, "Columnar Chunk Groups Removed by Filter"), 0, "premise: no whole row group is pruned") + expect.num(_counter(needle, "Columnar Vectors Ruled Out by Value"), 0, + "premise: no vector is ruled out by VALUE either (that is 1b-ii's path)") + expect.text(_has(needle, "Columnar Rows Filtered Before Materialization"), "yes", + "premise: the needle query takes the late-materialization path") + expect.num(_one(conn, "SELECT count(*) FROM n WHERE tag LIKE '%needle%'"), MATCHES, + "premise: the needle really matches one vector's worth of rows") + expect.num(_one(conn, "SELECT count(*) FROM n WHERE tag LIKE '%zzzzz%'"), 0, + "premise: the nomatch predicate matches no row") + expect.num(_one(conn, "SELECT count(*) FROM n WHERE tag LIKE '%row%'"), ROWS, + "premise: the control matches every row") + + expect.text(_has(control, "Columnar Vector Decodes"), "yes", "EXPLAIN reports Columnar Vector Decodes") + expect.num(_counter(control, "Columnar Vector Decodes"), VECTORS * NCOLS, + "the control decodes every column of every vector") + expect.num(_counter(nomatch, "Columnar Vector Decodes"), VECTORS, + "a zero-matching LIKE decodes the qual column and no payload column") + expect.num(_counter(needle, "Columnar Vector Decodes"), VECTORS + (NCOLS - 1) * SURVIVING_VECTORS, + "a one-vector LIKE decodes the qual column plus payload for that vector only") + + expect.num(_counter(needle, "Rows Removed by Filter"), ROWS - MATCHES, + "the needle removes exactly the non-matching rows, counted once") + expect.num(_counter(needle, "Columnar Rows Filtered Before Materialization"), ROWS - MATCHES, + "and reports them all filtered before materialization") + expect.num(_counter(needle, "Columnar Vectors Skipped"), VECTORS - SURVIVING_VECTORS, + "the needle skips exactly the vectors holding no match") + expect.num(_counter(nomatch, "Rows Removed by Filter"), ROWS, + "the zero-match query removes every row by filter, counted once") + expect.num(_counter(nomatch, "Columnar Vectors Skipped"), VECTORS, "and skips every vector") + + # Every row of the needle's surviving vector matches, so a producer that counted + # its re-test's rejections twice reads right above; half of vector 0 passes here. + expect.text(_one(conn, "SELECT count(*) || '|' || count(DISTINCT (p1 - 1) / 1024) FROM n " + "WHERE tag LIKE '%needle%' AND p1 % 2 = 0"), "512|1", + "premise: the half-needle survives in one vector, half of its rows") + expect.num(_counter(_plan(conn, HALFQ), "Rows Removed by Filter"), ROWS - 512, + "rejections inside a surviving vector are counted once") + + expect.row_set(_rows(conn, NEEDLEQ), _rows(conn, "SELECT * FROM h WHERE tag LIKE '%needle%'"), + "the needle query returns exactly the matching rows") + expect.num(_one(conn, "SELECT count(*) FROM n WHERE tag LIKE '%zzzzz%'"), 0, "the nomatch query returns nothing") + expect.rows(_rows(conn, "SELECT count(*), sum(p1), sum(p5) FROM n WHERE tag LIKE '%row%'"), + _rows(conn, "SELECT count(*), sum(p1), sum(p5) FROM h WHERE tag LIKE '%row%'"), + "the control returns every row, values intact") + + +def test_deleted_rows_across_several_groups(pgc_conn, expect): + """The evaluator reads the CURRENT group's delete mask; an earlier version read the + previous group's, freed one load earlier. It needs more than one group, deletes, + and the unprunable qual at once. A deleted row is not counted as filtered.""" + conn = pgc_conn + mgen = (f"SELECT 'x' || CASE WHEN g % 4096 = 7 THEN 'needle' ELSE 'miss' END || g::text, g, g * 2 " + f"FROM generate_series(1, {MROWS}) g") + _run(conn, "SET pgcolumnar.qual_skipvec_min_payload_cols = 0", + "CREATE TABLE hm (tag text, p1 int, p2 int)", + "CREATE TABLE m (tag text, p1 int, p2 int) USING pgcolumnar", + "SELECT pgcolumnar.set_options('m', stripe_row_limit => 8192, chunk_group_row_limit => 1024)", + f"INSERT INTO hm {mgen}", f"INSERT INTO m {mgen}", + "DELETE FROM hm WHERE p1 % 17 = 0", + # the fixture's DELETE must not depend on the gating under test + "SET pgcolumnar.qual_skipvec_min_payload_cols = 100000", "DELETE FROM m WHERE p1 % 17 = 0", + "SET pgcolumnar.qual_skipvec_min_payload_cols = 0") + expect.text(str(_one(conn, "SELECT count(*) > 1 FROM pgcolumnar.row_group " + "WHERE storage_id = pgcolumnar.get_storage_id('m')")), "True", + "premise: the columnar table spans more than one row group") + # count(*), not a count WHERE p1 % 17 = 0: that WHERE would run through the gating + expect.num(_one(conn, "SELECT count(*) FROM m"), MROWS - MROWS // 17, + "premise: rows really were deleted (so the mask is non-NULL)") + expect.row_set(_rows(conn, "SELECT * FROM m WHERE tag LIKE '%needle%'"), + _rows(conn, "SELECT * FROM hm WHERE tag LIKE '%needle%'"), + "deletes + multiple groups + unprunable qual: columnar matches the heap") + expect.row_set(_rows(conn, "SELECT * FROM m"), _rows(conn, "SELECT * FROM hm"), + "and the full scan across every group's mask matches too") + expect.num(_counter(_plan(conn, "SELECT * FROM m WHERE tag LIKE '%needle%'"), "Rows Removed by Filter"), + _one(conn, "SELECT count(*) FROM hm WHERE tag NOT LIKE '%needle%'"), + "rows removed by filter across deleted rows count only live rows") diff --git a/test/pytest/test_native_dict_underfill.py b/test/pytest/test_native_dict_underfill.py new file mode 100644 index 000000000..98c3f2c4f --- /dev/null +++ b/test/pytest/test_native_dict_underfill.py @@ -0,0 +1,70 @@ +"""An under-filled DICT chunk is refused cleanly, and no backend dies (#1341). The pytest +twin of `test/native_dict_underfill.sh`, every check by the shell's name. + +The test inflates a DICT chunk's value_raw_length in its encoding descriptor, so the +codes decode to fewer bytes than declared. Without the fill check the raw buffer's +tail is left uninitialized and a varlena length is read out of it. The read must raise +XX001, and the crash is read from the server log, since a fresh connection succeeds +again once crash recovery has finished. + +The post-codec encoding choice (#1132) is turned off for this session, as the shell +does for its database, so the fixture is DICT-encoded whether or not DICT beats zstd. +""" +import psycopg + + +def _crashes(pgc_cluster): + try: + return (pgc_cluster.datadir / "server.log").read_text(errors="replace").count("terminated by signal") + except OSError: + return 0 + + +def _one(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetchone()[0] + + +RAW = ("(get_byte(encoding_descriptor, 11) + (get_byte(encoding_descriptor, 12) << 8) + " + "(get_byte(encoding_descriptor, 13) << 16) + (get_byte(encoding_descriptor, 14) << 24) + 64)") + + +def test_an_under_filled_dict_chunk_is_refused(pgc_conn, pgc_cluster, expect): + conn = pgc_conn + with conn.cursor() as cur: + cur.execute("SET pgcolumnar.enable_post_codec_encoding_choice = off") + cur.execute("CREATE TABLE dt (id int, v text) USING pgcolumnar") + # low-cardinality short text: a DICT-encoded varlena column, one small chunk + cur.execute("INSERT INTO dt SELECT g, (ARRAY['aa', 'bb', 'cc', 'dd'])[1 + (g % 4)] " + "FROM generate_series(1, 300) g") + sid = _one(conn, "SELECT pgcolumnar.get_storage_id('dt')") + # column_index is 0-based: v is attribute 2, index 1; byte 6 is vector 0's encoding type + expect.num(_one(conn, f"SELECT get_byte(encoding_descriptor, 6) FROM pgcolumnar.column_chunk " + f"WHERE storage_id = {sid} AND column_index = 1 LIMIT 1"), 6, + "the text column is DICT-encoded (precondition)") + + crashes = _crashes(pgc_cluster) + # rawLen is the uint32 LE at descriptor offset 11: raise it by 64 + rewrote = _one(conn, f"""WITH u AS (UPDATE pgcolumnar.column_chunk c SET encoding_descriptor = + set_byte(set_byte(set_byte(set_byte(encoding_descriptor, + 11, {RAW} & 255), 12, ({RAW} >> 8) & 255), 13, ({RAW} >> 16) & 255), 14, ({RAW} >> 24) & 255) + WHERE c.storage_id = {sid} AND c.column_index = 1 RETURNING 1) SELECT count(*) FROM u""") + expect.num(rewrote, 1, "premise: the inflation rewrote the DICT chunk's descriptor") + + try: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as c: + c.execute(f"SET search_path TO {conn.execute('SELECT current_schema()').fetchone()[0]}") + c.execute("SELECT v FROM dt").fetchall() + rejected = "no error" + except psycopg.Error as exc: + rejected = exc.sqlstate or f"no SQLSTATE ({str(exc).splitlines()[0]})" + expect.text(rejected, "XX001", "under-filled DICT chunk is rejected with a clean error") + + try: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as c: + alive = c.execute("SELECT 1").fetchone()[0] == 1 + except psycopg.Error: + alive = False + expect.text(f"{'1' if alive else ''}|{_crashes(pgc_cluster) - crashes} backend crashes", "1|0 backend crashes", + "backend survives the under-filled DICT chunk") diff --git a/test/pytest/test_native_dml.py b/test/pytest/test_native_dml.py new file mode 100644 index 000000000..3a97bde8a --- /dev/null +++ b/test/pytest/test_native_dml.py @@ -0,0 +1,92 @@ +"""DELETE, UPDATE and MERGE on a native-format columnar table agree with a heap mirror +(#1341). The pytest twin of `test/native_dml.sh`, every check by the shell's name. + +Each statement runs on a heap table and on a columnar one with stripes of 1000 rows, +and every result is compared: the surviving rows, counts, aggregates and index lookups. +After deletes, an aggregate must not be answered from zone maps that still describe the +deleted rows. The deletes include the table's smallest and largest ids, so min and max +move and an answer that ignored the deletes differs from heap's; min and max are taken +over k too, a column the primary key does not order. An index lookup of a deleted row +must find nothing, and the lookups are asserted to take the index. +""" + + +def _one(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetchone()[0] + + +def _row(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetchone() + + +def _rows(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetchall() + + +def _run(conn, *sqls): + with conn.cursor() as cur: + for sql in sqls: + cur.execute(sql) + + +GEN = "SELECT g, (g * 10)::bigint, 'row-' || g FROM generate_series(1, 5000) g" + + +def test_delete_update_and_merge_match_heap(pgc_conn, expect): + conn = pgc_conn + _run(conn, "CREATE TABLE h (id int PRIMARY KEY, k bigint, v text)", + "CREATE TABLE n (id int PRIMARY KEY, k bigint, v text) USING pgcolumnar", + "SELECT pgcolumnar.set_options('n', stripe_row_limit => 1000)", + f"INSERT INTO h {GEN}", f"INSERT INTO n {GEN}") + + for t in ("h", "n"): + _run(conn, f"DELETE FROM {t} WHERE id % 7 = 0 OR id IN (1, 5000)") + expect.text(_one(conn, "SELECT min(id) || '|' || max(id) || '|' || min(k) || '|' || max(k) FROM h"), + "2|4999|20|49990", "premise: the deletes moved the smallest and largest id and k") + expect.num(_one(conn, "SELECT count(*) FROM n"), _one(conn, "SELECT count(*) FROM h"), "count after delete") + expect.row_set(_rows(conn, "SELECT id, k, v FROM n"), _rows(conn, "SELECT id, k, v FROM h"), + "rows after delete match heap") + expect.num(_one(conn, "SELECT sum(k) FROM n"), _one(conn, "SELECT sum(k) FROM h"), "sum(k) after delete") + expect.rows([_row(conn, "SELECT min(id), max(id), min(k), max(k) FROM n")], + [_row(conn, "SELECT min(id), max(id), min(k), max(k) FROM h")], "min/max after delete") + expect.num(_one(conn, "SELECT count(*) FROM n WHERE id % 7 = 0"), 0, "deleted rows gone") + + # id 4444 is live (4444 % 7 = 6); id 4445 was deleted (4445 % 7 = 0) + _run(conn, "SET enable_seqscan = off") + plan = "\n".join(r[0] for r in _rows(conn, "EXPLAIN (COSTS off) SELECT count(*) FROM n WHERE id = 4445")) + expect.num(sum(1 for line in plan.splitlines() if "Index" in line and "using n_pkey" in line), 1, + "premise: the lookups take the primary-key index") + expect.text(_one(conn, "SELECT v FROM n WHERE id = 4444"), _one(conn, "SELECT v FROM h WHERE id = 4444"), + "index finds live row") + expect.num(_one(conn, "SELECT count(*) FROM n WHERE id = 4445"), 0, "index skips deleted row") + _run(conn, "RESET enable_seqscan") + + for t in ("h", "n"): + _run(conn, f"UPDATE {t} SET k = k + 1 WHERE id % 5 = 0") + expect.num(_one(conn, "SELECT count(*) FROM n"), _one(conn, "SELECT count(*) FROM h"), "count after update") + expect.row_set(_rows(conn, "SELECT id, k, v FROM n"), _rows(conn, "SELECT id, k, v FROM h"), + "rows after update match heap") + expect.num(_one(conn, "SELECT sum(k) FROM n"), _one(conn, "SELECT sum(k) FROM h"), "sum(k) after update") + + import psycopg + try: + _run(conn, "INSERT INTO n VALUES (4444, 0, 'x')") + dup = "accepted" + except psycopg.Error as exc: + dup = exc.sqlstate + expect.text(dup, "23505", "PK still enforced after DML") + + _run(conn, "CREATE TABLE src (id int, k bigint)", "INSERT INTO src VALUES (1, 999), (999001, 5), (999002, 6)") + for t in ("h", "n"): + _run(conn, f"MERGE INTO {t} t USING src s ON t.id = s.id " + "WHEN MATCHED THEN UPDATE SET k = s.k " + "WHEN NOT MATCHED THEN INSERT (id, k, v) VALUES (s.id, s.k, 'm')") + expect.num(_one(conn, "SELECT count(*) FROM n"), _one(conn, "SELECT count(*) FROM h"), "count after merge") + expect.row_set(_rows(conn, "SELECT id, k, v FROM n"), _rows(conn, "SELECT id, k, v FROM h"), + "rows after merge match heap") diff --git a/test/pytest/test_native_upgrade_converge.py b/test/pytest/test_native_upgrade_converge.py new file mode 100644 index 000000000..5a2c1d54b --- /dev/null +++ b/test/pytest/test_native_upgrade_converge.py @@ -0,0 +1,126 @@ +"""The upgrade window: this library over the previous release's catalog, before ALTER +EXTENSION UPDATE (#1341 item 66). Part of the pytest twin of +`test/native_upgrade_converge.sh`: these arms only, by the shell's names, and the pair +is declared INCOMPLETE. + +docs/installation.md steps 1 and 2 put a new library in place; step 3, ALTER EXTENSION +UPDATE, brings each database's catalog up to it. Between them a database runs the new +library over the old catalog, and the library deletes from catalog tables the old one +does not have (retired_group, new in 1.0-alpha5). Reads and writes must work there; an +operation that needs a missing table must be refused cleanly, change nothing and name +step 3; and after step 3 every one of them must work on the same rows. + +The previous release is the version the shipped upgrade script to the current +default_version starts from, and its base script is staged from test/fixtures into the +extension directory for the test, as the shell does, then removed. +""" +import pathlib +import re +import shutil +import subprocess + +import psycopg +import pytest + +REPO = pathlib.Path(__file__).resolve().parents[2] + + +def _target_and_previous(): + target = re.search(r"^default_version\s*=\s*'([^']+)'", + (REPO / "pgcolumnar.control").read_text(), re.M).group(1) + prev = [p.name[len("pgcolumnar--"):-len(f"--{target}.sql")] + for p in REPO.glob(f"pgcolumnar--*--{target}.sql")] + return target, prev + + +@pytest.fixture +def window_db(pgc_cluster): + """-> (dsn, previous version) of a database at the previous release's catalog.""" + target, prev = _target_and_previous() + assert len(prev) == 1, f"one upgrade script should lead to {target}, found {prev}" + prev = prev[0] + sharedir = subprocess.run([pgc_cluster.pg_config, "--sharedir"], capture_output=True, + text=True, check=True).stdout.strip() + dst = pathlib.Path(sharedir) / "extension" / f"pgcolumnar--{prev}.sql" + backup = dst.with_name(dst.name + ".pytest-bak") + if dst.exists(): + shutil.copy2(dst, backup) + shutil.copy2(REPO / "test" / "fixtures" / f"pgcolumnar--{prev}.sql", dst) + + def admin(sql): + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as c: + c.execute(sql) + + admin("DROP DATABASE IF EXISTS pgc_upgrade_window") + admin("CREATE DATABASE pgc_upgrade_window") + dsn = pgc_cluster.dsn(dbname="pgc_upgrade_window") + try: + with psycopg.connect(dsn, autocommit=True) as c: + c.execute(f"CREATE EXTENSION pgcolumnar VERSION '{prev}'") + yield dsn, prev + finally: + admin("DROP DATABASE IF EXISTS pgc_upgrade_window WITH (FORCE)") + if backup.exists(): + shutil.move(backup, dst) + else: + dst.unlink(missing_ok=True) + + +def _one(dsn, sql): + with psycopg.connect(dsn, autocommit=True) as c: + return c.execute(sql).fetchone()[0] + + +def _refusal(dsn, sql): + """-> 'SQLSTATE|names-the-update' or '|no-hint', or 'accepted'.""" + try: + with psycopg.connect(dsn, autocommit=True) as c: + c.execute(sql) + return "accepted" + except psycopg.Error as exc: + hint = exc.diag.message_hint or "" + return f"{exc.sqlstate}|{'names-the-update' if 'ALTER EXTENSION pgcolumnar UPDATE' in hint else 'no-hint'}" + + +def _succeeds(dsn, *sqls): + try: + with psycopg.connect(dsn, autocommit=True) as c: + for sql in sqls: + c.execute(sql) + return "succeeded" + except psycopg.Error as exc: + return f"refused ({exc.sqlstate}: {str(exc).splitlines()[0]})" + + +def test_the_upgrade_window(window_db, expect): + dsn, prev = window_db + expect.text(_one(dsn, "SELECT extversion FROM pg_extension WHERE extname = 'pgcolumnar'"), prev, + "premise: the window database runs this library over the previous release's catalog") + expect.num(_one(dsn, "SELECT count(*) FROM pg_class WHERE relname = 'retired_group' " + "AND relnamespace = 'pgcolumnar'::regnamespace"), 0, + "premise: that catalog has no retired_group, which this library deletes from") + with psycopg.connect(dsn, autocommit=True) as c: + c.execute("CREATE TABLE w (a int, b text) USING pgcolumnar") + c.execute("INSERT INTO w SELECT g, 'x' || g FROM generate_series(1, 20000) g") + c.execute("DELETE FROM w WHERE a % 3 = 0") + + def rows(): + return _one(dsn, "SELECT count(*) || '|' || sum(a) FROM w") + + expect.text(rows(), "13334|133346667", "in the window, a columnar table is written, deleted from and read") + expect.text(_refusal(dsn, "DROP TABLE w"), "42P01|names-the-update", + "in the window, DROP TABLE is refused and names ALTER EXTENSION UPDATE") + expect.text(_refusal(dsn, "TRUNCATE w"), "42P01|names-the-update", + "in the window, TRUNCATE is refused and names ALTER EXTENSION UPDATE") + expect.text(_refusal(dsn, "SELECT pgcolumnar.vacuum('w')"), "42P01|names-the-update", + "in the window, pgcolumnar.vacuum() is refused and names ALTER EXTENSION UPDATE") + expect.text(rows(), "13334|133346667", "in the window, the refused operations changed nothing") + + with psycopg.connect(dsn, autocommit=True) as c: + c.execute("ALTER EXTENSION pgcolumnar UPDATE") + expect.text(rows(), "13334|133346667", "after ALTER EXTENSION UPDATE, the same rows are there") + vacuumed = _succeeds(dsn, "SELECT pgcolumnar.vacuum('w')") + expect.text(rows() if vacuumed == "succeeded" else vacuumed, "13334|133346667", + "after ALTER EXTENSION UPDATE, pgcolumnar.vacuum() succeeds and keeps them") + expect.text(_succeeds(dsn, "TRUNCATE w", "DROP TABLE w"), "succeeded", + "after ALTER EXTENSION UPDATE, TRUNCATE and DROP TABLE succeed") diff --git a/test/pytest/test_truncate_cleanup.py b/test/pytest/test_truncate_cleanup.py new file mode 100644 index 000000000..c3c34ab9e --- /dev/null +++ b/test/pytest/test_truncate_cleanup.py @@ -0,0 +1,181 @@ +"""TRUNCATE leaves no catalog rows behind, and keeps what it must (#1341). The pytest +twin of `test/truncate_cleanup.sh`, every check by the shell's name. + +TRUNCATE installs a new relfilenode and a new storage id. It used to leave the old +storage's row_group, column_chunk, zone_map and bloom rows behind, keyed by the retired +id, so each truncate-and-reload cycle leaked one storage's worth, and a DROP after it +removed only the current, empty one. Retiring them made a cached write state stale, so +a transaction that wrote, truncated and wrote again committed into storage nothing +reads; the state is now dropped with the rows. A relation created in the current +transaction is truncated in place instead, on a different path, which must clear each +projection's storage as well as the base's. + +A PRIVATE DATABASE, because the snapshot counts every pgcolumnar catalog row, and the +shared one holds every other test's. +""" +import psycopg + + +def _one(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetchone()[0] + + +def _run(conn, *sqls): + with conn.cursor() as cur: + for sql in sqls: + cur.execute(sql) + + +def _snapshot(conn): + return _one(conn, "SELECT (SELECT count(*) FROM pgcolumnar.storage) || '/' || " + "(SELECT count(*) FROM pgcolumnar.projection) || '/' || " + "(SELECT count(*) FROM pgcolumnar.row_group) || '/' || " + "(SELECT count(*) FROM pgcolumnar.column_chunk) || '/' || " + "(SELECT count(*) FROM pgcolumnar.zone_map) || '/' || " + "(SELECT count(*) FROM pgcolumnar.bloom) || '/' || " + "(SELECT count(*) FROM pgcolumnar.delete_vector)") + + +def _grew(base, now): + return "no" if base == now else "yes" + + +LOAD = "INSERT INTO {t} SELECT g, 'x' || g FROM generate_series(1, {n}) g" + + +def test_truncate_and_drop_leave_the_catalog_as_they_found_it(pgc_own_db, expect): + conn = pgc_own_db + base = _snapshot(conn) + _run(conn, "CREATE TABLE tcl_plain (id int, v text) USING pgcolumnar", LOAD.format(t="tcl_plain", n=5000)) + expect.text(_grew(base, _snapshot(conn)), "yes", "insert did add metadata") + _run(conn, "TRUNCATE tcl_plain") + expect.text(_snapshot(conn), base, "TRUNCATE returns the catalog to the empty baseline") + expect.num(_one(conn, "SELECT count(*) FROM tcl_plain"), 0, "TRUNCATE leaves no rows") + for _ in range(2): + _run(conn, LOAD.format(t="tcl_plain", n=5000), "TRUNCATE tcl_plain") + expect.text(_snapshot(conn), base, "three truncate-and-reload cycles still sit at the baseline") + # loaded first, so the DROP has work to do: a DROP of an empty table could not fail + _run(conn, LOAD.format(t="tcl_plain", n=5000)) + expect.text(_grew(base, _snapshot(conn)), "yes", + "premise: the table carries metadata again, so the DROP has work to do") + _run(conn, "DROP TABLE tcl_plain") + expect.text(_snapshot(conn), base, "DROP of a loaded table after TRUNCATE leaves nothing") + + +def test_a_projection_is_not_leaked_by_truncate(pgc_own_db, expect): + """A projection has its own storage id. What must hold is that no projection row + names a storage id the table no longer has, not that projection is empty.""" + conn = pgc_own_db + base = _snapshot(conn) + _run(conn, "CREATE TABLE tcl_proj (id int, a int, b text) USING pgcolumnar", + "SELECT pgcolumnar.add_projection('tcl_proj', 'tcl_p', ARRAY['a', 'b'], ARRAY['a'])", + "INSERT INTO tcl_proj SELECT g, g % 50, 'b' || g FROM generate_series(1, 5000) g") + expect.text(_grew(base, _snapshot(conn)), "yes", "the projection did add metadata") + _run(conn, "TRUNCATE tcl_proj") + expect.num(_one(conn, "SELECT count(*) FROM pgcolumnar.projection p " + "WHERE p.storage_id <> pgcolumnar.get_storage_id('tcl_proj'::regclass)"), 0, + "TRUNCATE leaves no projection row under a retired storage id") + expect.num(_one(conn, "SELECT count(*) FROM pgcolumnar.projection_declaration " + "WHERE rel = 'tcl_proj'::regclass"), 1, + "and the declaration survives the TRUNCATE, so the projection is rebuildable") + _run(conn, "INSERT INTO tcl_proj SELECT g, g % 50, 'b' || g FROM generate_series(1, 5000) g") + expect.text(_grew(base, _snapshot(conn)), "yes", "premise: the projected table carries metadata again before the DROP") + _run(conn, "DROP TABLE tcl_proj") + expect.text(_snapshot(conn), base, "DROP of a loaded projected table leaves nothing") + + +def test_repeated_truncates_accumulate_nothing(pgc_own_db, expect): + """Created once outside the loop and one statement at a time, so every TRUNCATE + takes the transactional path and any leak accumulates.""" + conn = pgc_own_db + base = _snapshot(conn) + _run(conn, "CREATE TABLE tcl_rep (id int, v text) USING pgcolumnar") + for _ in range(10): + _run(conn, LOAD.format(t="tcl_rep", n=2000), "TRUNCATE tcl_rep") + expect.text(_snapshot(conn), base, "ten insert-and-truncate cycles on one table leave nothing") + _run(conn, "DROP TABLE tcl_rep") + expect.text(_snapshot(conn), base, "and dropping it afterwards still leaves nothing") + + +def test_insert_truncate_insert_in_one_transaction(pgc_own_db, expect): + """The cached write state named the storage the TRUNCATE retired, so the second + INSERT flushed into storage nothing reads, and the transaction committed an empty + table. The status and the surviving rows are one arm: split, a commit that keeps + nothing passes the status half, and 1..10 passes a bare count of 10.""" + conn = pgc_own_db + _run(conn, "CREATE TABLE tc_txn (id int) USING pgcolumnar", + "INSERT INTO tc_txn SELECT g FROM generate_series(1, 10) g") + expect.num(_one(conn, "SELECT count(*) FROM tc_txn"), 10, "premise: the table holds its first ten rows") + rc = 0 + try: + _run(conn, "BEGIN", "INSERT INTO tc_txn SELECT g FROM generate_series(11, 20) g", "TRUNCATE tc_txn", + "INSERT INTO tc_txn SELECT g FROM generate_series(21, 30) g", "COMMIT") + except psycopg.Error: + rc = 1 + _run(conn, "ROLLBACK") + expect.text(f"rc={rc} " + _one(conn, "SELECT coalesce(min(id), -1) || '..' || coalesce(max(id), -1) || " + "' n=' || count(*) FROM tc_txn"), + "rc=0 21..30 n=10", "INSERT, TRUNCATE, INSERT in one transaction commits and keeps the new rows") + + +def test_a_rewrite_that_is_not_a_truncate_loses_nothing(pgc_own_db, expect): + """The retire runs from relation_set_new_filelocator, which every rewrite calls.""" + conn = pgc_own_db + _run(conn, "CREATE TABLE tc_rw (id int, v text) USING pgcolumnar", + "INSERT INTO tc_rw SELECT g, 'v' || g FROM generate_series(1, 5000) g") + expect.num(_one(conn, "SELECT count(*) FROM tc_rw"), 5000, "premise: five thousand rows before the rewrite") + _run(conn, "ALTER TABLE tc_rw ALTER COLUMN v TYPE varchar(64)") + expect.text(_one(conn, "SELECT count(*) || '/' || count(v) || '/' || max(id) FROM tc_rw"), "5000/5000/5000", + "a full rewrite keeps every row, and they are still readable") + + +def test_an_in_place_truncate_clears_the_projection_too(pgc_own_db, expect): + """A relation created in the current transaction is truncated in place, through + nontransactional_truncate, which cleared the base storage and left the + projection's. The write after the truncate is what makes the leak observable: it + collided on row_group_pkey (#896).""" + conn = pgc_own_db + _run(conn, "CREATE TABLE tcl_inplace_pre (id int)") + committed = "committed" + try: + _run(conn, "BEGIN", + "CREATE TABLE tcl_inplace (id int, a int, b text) USING pgcolumnar", + "INSERT INTO tcl_inplace SELECT g, g % 50, 'b' || g FROM generate_series(1, 2000) g", + "SELECT pgcolumnar.add_projection('tcl_inplace', 'tcl_ip', ARRAY['a', 'b'], ARRAY['a'])", + "TRUNCATE tcl_inplace", + "INSERT INTO tcl_inplace SELECT g, g % 50, 'c' || g FROM generate_series(1, 500) g", + "COMMIT") + except psycopg.Error as exc: + committed = f"rolled back ({str(exc).splitlines()[0]})" + _run(conn, "ROLLBACK") + # vacuous without these: a rolled-back transaction leaves no table to count + expect.num(0 if committed == "committed" else 1, 0, "PREMISE the in-place transaction committed") + expect.num(_one(conn, "SELECT count(*) FROM pg_class WHERE relname = 'tcl_inplace'"), 1, + "PREMISE the table exists after it") + if committed != "committed": + return + expect.num(_one(conn, "SELECT count(*) FROM tcl_inplace"), 500, + "the rows written after the in-place truncate are all there") + expect.num(_one(conn, "SELECT count(*) FROM pgcolumnar.read_projection('tcl_inplace', 'tcl_ip')"), 500, + "and the projection reads them back") + with conn.cursor() as cur: + cur.execute("SELECT pgcolumnar.read_projection('tcl_inplace', 'tcl_ip')") + proj = cur.fetchall() + cur.execute("SELECT a::text || '|' || b FROM tcl_inplace") + base = cur.fetchall() + expect.row_set(proj, base, "the projection agrees with the base table") + proj_storage = _one(conn, "SELECT proj_storage_id FROM pgcolumnar.projection " + "WHERE storage_id = pgcolumnar.get_storage_id('tcl_inplace'::regclass) " + "AND projection_id > 0") + base_storage = _one(conn, "SELECT pgcolumnar.get_storage_id('tcl_inplace'::regclass)") + expect.text("yes" if proj_storage is not None and proj_storage != base_storage else "no", "yes", + "PREMISE the projection has its own storage, distinct from the base") + # one group of 500 rows, not a group NUMBER: the base's counter is reset and the + # projection's has no metapage to reset, so the numbers differ by design + expect.num(_one(conn, f"SELECT count(*) FROM pgcolumnar.row_group WHERE storage_id = {proj_storage}"), 1, + "the projection storage holds exactly one row group after the truncate") + expect.num(_one(conn, f"SELECT coalesce(sum(row_count), 0) FROM pgcolumnar.row_group " + f"WHERE storage_id = {proj_storage}"), 500, + "and that group describes only the rows written after it")