diff --git a/CHANGELOG.md b/CHANGELOG.md index d2b779a6d..c0dffb95f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -400,6 +400,18 @@ true until the next version shipped. Reported by @OffgridwithJD. ### Fixed +- **In the upgrade window, a refusal for a missing catalog table names `ALTER + EXTENSION UPDATE`.** After the new library is installed and the server restarted, + but before `ALTER EXTENSION pgcolumnar UPDATE`, `DROP TABLE`, `TRUNCATE` and + `pgcolumnar.vacuum()` on a columnar table fail, because the 1.0-alpha5 library + needs `pgcolumnar.retired_group` and a 1.0-alpha4 catalog lacks it. They change + nothing. The error used to name only the missing table, and it now carries a hint: + `The pgcolumnar catalog in this database may be older than the installed library. + Run ALTER EXTENSION pgcolumnar UPDATE in this database.` Reads and writes work + throughout, and after the update every one of these succeeds on the same rows + (#1341 item 66). `test/native_upgrade_converge.sh` and + `test_native_upgrade_converge.py`. + - **A join whose runtime-filter build is empty no longer reads the fact table.** When a large dimension filtered to no row was joined to a columnar table, the hash join diff --git a/docs/installation.md b/docs/installation.md index 2618c47bf..6cf867108 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -152,6 +152,18 @@ The fix is step 3. Run `ALTER EXTENSION pgcolumnar UPDATE;` in that database and the error goes away. **Your data is not affected.** The tables are intact and no conversion happens. Only the catalog entry is stale. +Between releases that add a catalog table, the new library can also need a table +the old catalog does not have. Upgrading from `1.0-alpha4` to `1.0-alpha5`, +`DROP TABLE`, `TRUNCATE` and `pgcolumnar.vacuum()` on a columnar table fail until +step 3, and change nothing; reads and writes keep working: + +``` +ERROR: columnar metadata table "pgcolumnar.retired_group" does not exist +HINT: The pgcolumnar catalog in this database may be older than the installed library. Run ALTER EXTENSION pgcolumnar UPDATE in this database. +``` + +The fix is the same: step 3. + Do not use `DROP EXTENSION` instead. A plain `DROP EXTENSION pgcolumnar` fails while columnar tables exist, because they depend on the access method. The form that succeeds is `DROP EXTENSION pgcolumnar CASCADE`, and it drops every diff --git a/src/columnar_metadata.c b/src/columnar_metadata.c index bd0120efb..eb5bdc53f 100644 --- a/src/columnar_metadata.c +++ b/src/columnar_metadata.c @@ -239,11 +239,22 @@ open_columnar_table(const char *name, LOCKMODE lockmode) nspOid = pgcolumnar_schema_oid(); relOid = get_relname_relid(name, nspOid); + /* + * NAME THE REMEDY (#1341 item 66). The usual way here is the upgrade window: + * the new library is installed and the server restarted (docs/installation.md + * steps 1 and 2), but this database's catalog is still the previous release's + * until ALTER EXTENSION UPDATE (step 3). A catalog table this library added, + * such as retired_group in 1.0-alpha5, is then missing, and DROP TABLE, + * TRUNCATE and pgcolumnar.vacuum() stop here. The error alone named only the + * table, which says neither why nor what to do. + */ if (!OidIsValid(relOid)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), errmsg("columnar metadata table \"%s.%s\" does not exist", - COLUMNAR_SCHEMA_NAME, name))); + COLUMNAR_SCHEMA_NAME, name), + errhint("The pgcolumnar catalog in this database may be older than the installed library. " + "Run ALTER EXTENSION pgcolumnar UPDATE in this database."))); rel = table_open(relOid, lockmode); diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 74fcd205f..7696edfc8 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -804,6 +804,34 @@ fdw_plan_privilege fdw_plan_privilege and a metadata_path that does not exist pl fdw_plan_privilege fdw_plan_privilege premise: the Parquet file was written 15;16;17;18;19 never - fdw_plan_privilege fdw_plan_privilege premise: the reader is not a member of pg_read_server_files 15;16;17;18;19 never - fdw_plan_privilege fdw_plan_privilege the unprivileged Iceberg scan is still refused (42501) 15;16;17;18;19 never - +fk_referencing fk_referencing ALTER TABLE ADD CONSTRAINT is refused by the same rule 15;16;17;18;19 2026-10-16 columnar_tableam.c: a foreign key referencing a columnar table no longer refused;columnar_tableam.c: the foreign-key guard keyed on the referencing table, not the referenced one +fk_referencing fk_referencing a columnar partition of an FK-referenced parent is refused 15;16;17;18;19 2026-10-16 columnar_tableam.c: a foreign key referencing a columnar table no longer refused;columnar_tableam.c: the foreign-key guard keyed on the referencing table, not the referenced one +fk_referencing fk_referencing a columnar table may reference a heap table 15;16;17;18;19 2026-10-16 columnar_tableam.c: the foreign-key guard keyed on the referencing table, not the referenced one +fk_referencing fk_referencing a foreign key between two heap tables is unaffected 15;16;17;18;19 never - +fk_referencing fk_referencing a foreign key referencing a columnar table is refused 15;16;17;18;19 2026-10-16 columnar_tableam.c: a foreign key referencing a columnar table no longer refused;columnar_tableam.c: the foreign-key guard keyed on the referencing table, not the referenced one +fk_referencing fk_referencing a partitioned table with no foreign key still converts 17;18;19 never - +fk_referencing fk_referencing a row that violates it is still rejected 15;16;17;18;19 2026-10-16 columnar_tableam.c: the foreign-key guard keyed on the referencing table, not the referenced one +fk_referencing fk_referencing a table with no foreign key still converts 15;16;17;18;19 never - +fk_referencing fk_referencing an FK-referenced table may still be set to a row store 15;16;17;18;19 2026-10-16 columnar_tableam.c: converting an FK-referenced table refused whatever the target access method +fk_referencing fk_referencing an unrelated ALTER TABLE on a referenced table is unaffected 15;16;17;18;19 never - +fk_referencing fk_referencing and a partition can still be created the ordinary way 15;16;17;18;19 2026-10-16 columnar_tableam.c: converting an FK-referenced table to columnar no longer refused +fk_referencing fk_referencing and its partitions inherit the columnar access method 17;18;19 never - +fk_referencing fk_referencing and rows that satisfy it are accepted 15;16;17;18;19 2026-10-16 columnar_tableam.c: the foreign-key guard keyed on the referencing table, not the referenced one +fk_referencing fk_referencing and setting the parent's access method is refused as well 15;16;17;18;19 2026-10-16 columnar_tableam.c: converting an FK-referenced table to columnar no longer refused +fk_referencing fk_referencing and the error names the constraint 15;16;17;18;19 2026-10-16 columnar_tableam.c: converting an FK-referenced table to columnar no longer refused +fk_referencing fk_referencing and the error says why 15;16;17;18;19 2026-10-16 columnar_tableam.c: a foreign key referencing a columnar table no longer refused;columnar_tableam.c: the foreign-key guard keyed on the referencing table, not the referenced one +fk_referencing fk_referencing and they still work 15;16;17;18;19 never - +fk_referencing fk_referencing before 17, core refuses this for every partitioned table 15;16 never - +fk_referencing fk_referencing converting an FK-referenced table to columnar is refused 15;16;17;18;19 2026-10-16 columnar_tableam.c: converting an FK-referenced table to columnar no longer refused +fk_referencing fk_referencing converting back to heap is unaffected 15;16;17;18;19 never - +fk_referencing fk_referencing converting the referencing side is still allowed 15;16;17;18;19 never - +fk_referencing fk_referencing primary key, unique, check and not-null on a columnar table are unaffected 15;16;17;18;19 never - +fk_referencing fk_referencing so the parent did not become columnar 15;16;17;18;19 2026-10-16 columnar_tableam.c: converting an FK-referenced table to columnar no longer refused +fk_referencing fk_referencing the child is still writable afterwards 15;16;17;18;19 2026-10-16 columnar_tableam.c: converting an FK-referenced table to columnar no longer refused +fk_referencing fk_referencing the conversion helper is refused identically 15;16;17;18;19 2026-10-16 columnar_tableam.c: converting an FK-referenced table to columnar no longer refused +fk_referencing fk_referencing the parent is still a row store 15;16;17;18;19 2026-10-16 columnar_tableam.c: converting an FK-referenced table to columnar no longer refused +fk_referencing fk_referencing the primary key is still enforced 15;16;17;18;19 never - +fk_referencing fk_referencing the refused table was not left behind 15;16;17;18;19 2026-10-16 columnar_tableam.c: a foreign key referencing a columnar table no longer refused;columnar_tableam.c: the foreign-key guard keyed on the referencing table, not the referenced one harness_selftest 030-assertions nothing leaked into the squatter 15;16;17;18;19 never - harness_selftest 030-assertions pgc_port_free says the squatter's port is busy 15;16;17;18;19 never - harness_selftest 030-assertions squatter survived untouched 15;16;17;18;19 never - @@ -2288,6 +2316,21 @@ native_delete_visibility_paths native_delete_visibility_paths aggregate over the native_delete_visibility_paths native_delete_visibility_paths index / bitmap scan sees the live set 15;16;17;18;19 never - native_delete_visibility_paths native_delete_visibility_paths index-only scan sees the live set 15;16;17;18;19 never - native_delete_visibility_paths native_delete_visibility_paths sequential scan sees the live set (row-emitting path) 15;16;17;18;19 never - +native_dml native_dml PK still enforced after DML 15;16;17;18;19 never - +native_dml native_dml count after delete 15;16;17;18;19 2026-10-16 columnar_vector.c: count(*) from metadata no longer subtracts a group's deleted rows +native_dml native_dml count after merge 15;16;17;18;19 2026-10-16 columnar_tableam.c: the index fetch no longer checks the delete vector;columnar_vector.c: count(*) from metadata no longer subtracts a group's deleted rows +native_dml native_dml count after update 15;16;17;18;19 2026-10-16 columnar_vector.c: count(*) from metadata no longer subtracts a group's deleted rows +native_dml native_dml deleted rows gone 15;16;17;18;19 never - +native_dml native_dml index finds live row 15;16;17;18;19 never - +native_dml native_dml index skips deleted row 15;16;17;18;19 2026-10-16 columnar_tableam.c: the index fetch no longer checks the delete vector +native_dml native_dml min/max after delete 15;16;17;18;19 2026-10-16 columnar_vector.c: a row group holding deletes still answers min and max from its zone maps +native_dml native_dml premise: the deletes moved the smallest and largest id and k 15;16;17;18;19 never - +native_dml native_dml premise: the lookups take the primary-key index 15;16;17;18;19 never - +native_dml native_dml rows after delete match heap 15;16;17;18;19 never - +native_dml native_dml rows after merge match heap 15;16;17;18;19 2026-10-16 columnar_tableam.c: the index fetch no longer checks the delete vector +native_dml native_dml rows after update match heap 15;16;17;18;19 2026-10-16 columnar_tableam.c: the index fetch no longer checks the delete vector +native_dml native_dml sum(k) after delete 15;16;17;18;19 never - +native_dml native_dml sum(k) after update 15;16;17;18;19 2026-10-16 columnar_tableam.c: the index fetch no longer checks the delete vector native_fetch_coalesce native_fetch_coalesce a wide index fetch does not pin once per column 15;16;17;18;19 2026-10-13 the coalesced fetch merging no adjacent ranges, so each column is read on its own native_fetch_coalesce native_fetch_coalesce premise: a point lookup uses the index 15;16;17;18;19 never - native_fetch_coalesce native_fetch_coalesce premise: fetching every projected column touched a measurable number of buffers 15;16;17;18;19 never - @@ -3341,6 +3384,31 @@ timeseries_recipes timeseries_recipes the how-to gives the LOCF recipe 15;16;17; timeseries_recipes timeseries_recipes the how-to gives the gap-fill recipe 15;16;17;18;19 2026-10-13 the how-to's 'Fill empty buckets' heading renamed, so the recipe is no longer found timeseries_recipes timeseries_recipes the how-to gives the interpolation recipe 15;16;17;18;19 2026-10-13 the how-to's 'Interpolate between known values' heading renamed, so the recipe is no longer found timeseries_recipes timeseries_recipes the interpolation recipe fills a gap linearly, and leaves both ends NULL 15;16;17;18;19 2026-10-13 the how-to's 'Interpolate between known values' heading renamed, so the recipe is no longer found;the how-to's interpolation recipe uses prev_v in place of next_v, so every gap is filled flat;the how-to's recipes join the series to the aggregate with JOIN instead of LEFT JOIN +truncate_cleanup truncate_cleanup DROP of a loaded projected table leaves nothing 15;16;17;18;19 2026-10-16 columnar_tableam.c: a rewrite no longer deletes the retired storage's catalog rows;columnar_tableam.c: deleting a storage tree no longer reaches its projections' storages +truncate_cleanup truncate_cleanup DROP of a loaded table after TRUNCATE leaves nothing 15;16;17;18;19 2026-10-16 columnar_tableam.c: a rewrite no longer deletes the retired storage's catalog rows +truncate_cleanup truncate_cleanup INSERT, TRUNCATE, INSERT in one transaction commits and keeps the new rows 15;16;17;18;19 2026-10-16 columnar_tableam.c: a rewrite no longer drops the cached write state of the retired storage +truncate_cleanup truncate_cleanup PREMISE the in-place transaction committed 15;16;17;18;19 2026-10-16 columnar_tableam.c: the in-place truncate no longer clears a projection's own storage +truncate_cleanup truncate_cleanup PREMISE the projection has its own storage, distinct from the base 15;16;17;18;19 2026-10-16 columnar_tableam.c: the in-place truncate no longer clears a projection's own storage +truncate_cleanup truncate_cleanup PREMISE the table exists after it 15;16;17;18;19 2026-10-16 columnar_tableam.c: the in-place truncate no longer clears a projection's own storage +truncate_cleanup truncate_cleanup TRUNCATE leaves no projection row under a retired storage id 15;16;17;18;19 2026-10-16 columnar_tableam.c: a rewrite no longer deletes the retired storage's catalog rows +truncate_cleanup truncate_cleanup TRUNCATE leaves no rows 15;16;17;18;19 never - +truncate_cleanup truncate_cleanup TRUNCATE returns the catalog to the empty baseline 15;16;17;18;19 2026-10-16 columnar_tableam.c: a rewrite no longer deletes the retired storage's catalog rows +truncate_cleanup truncate_cleanup a full rewrite keeps every row, and they are still readable 15;16;17;18;19 never - +truncate_cleanup truncate_cleanup and dropping it afterwards still leaves nothing 15;16;17;18;19 2026-10-16 columnar_tableam.c: a rewrite no longer deletes the retired storage's catalog rows +truncate_cleanup truncate_cleanup and that group describes only the rows written after it 15;16;17;18;19 2026-10-16 columnar_tableam.c: the in-place truncate no longer clears a projection's own storage +truncate_cleanup truncate_cleanup and the declaration survives the TRUNCATE, so the projection is rebuildable 15;16;17;18;19 never - +truncate_cleanup truncate_cleanup and the projection reads them back 15;16;17;18;19 2026-10-16 columnar_tableam.c: the in-place truncate no longer clears a projection's own storage +truncate_cleanup truncate_cleanup insert did add metadata 15;16;17;18;19 never - +truncate_cleanup truncate_cleanup premise: five thousand rows before the rewrite 15;16;17;18;19 never - +truncate_cleanup truncate_cleanup premise: the projected table carries metadata again before the DROP 15;16;17;18;19 never - +truncate_cleanup truncate_cleanup premise: the table carries metadata again, so the DROP has work to do 15;16;17;18;19 never - +truncate_cleanup truncate_cleanup premise: the table holds its first ten rows 15;16;17;18;19 never - +truncate_cleanup truncate_cleanup ten insert-and-truncate cycles on one table leave nothing 15;16;17;18;19 2026-10-16 columnar_tableam.c: a rewrite no longer deletes the retired storage's catalog rows +truncate_cleanup truncate_cleanup the projection agrees with the base table 15;16;17;18;19 2026-10-16 columnar_tableam.c: the in-place truncate no longer clears a projection's own storage +truncate_cleanup truncate_cleanup the projection did add metadata 15;16;17;18;19 never - +truncate_cleanup truncate_cleanup the projection storage holds exactly one row group after the truncate 15;16;17;18;19 2026-10-16 columnar_tableam.c: the in-place truncate no longer clears a projection's own storage +truncate_cleanup truncate_cleanup the rows written after the in-place truncate are all there 15;16;17;18;19 2026-10-16 columnar_tableam.c: the in-place truncate no longer clears a projection's own storage +truncate_cleanup truncate_cleanup three truncate-and-reload cycles still sit at the baseline 15;16;17;18;19 2026-10-16 columnar_tableam.c: a rewrite no longer deletes the retired storage's catalog rows ttl_expire ttl_expire NO row still inside the retention was dropped (#403 item 5a) 15;16;17;18;19 never - ttl_expire ttl_expire NO row still inside the retention was dropped, on a date column 15;16;17;18;19 never - ttl_expire ttl_expire a compaction's retired groups are not recorded as retention 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 3f8e29cde..95506368c 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 219 +suites_not_covered 216 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -2943,4 +2943,51 @@ suites_not_covered 219 # never 2676 - 1 = 2675 # dated 943 + 1 = 944 # 2675 + 944 == 3619 -checks_never_observed_red 2675 +# +# #1341, fk_referencing seeded, with its twin (OffgridwithJD): 2675 -> 2686, and +# suites_not_covered 219 -> 218. The suite had no rows. It enters with 28 rows +# from five-major runs: 27 checks on PG17-19 and 26 on 15-16, where the +# partitioned-control branch asserts core's refusal instead of a conversion. 17 are +# dated by four mutations of the foreign-key guards: the referenced-side refusal +# removed, the guard keyed on the referencing table, the conversion refusal +# removed, and the conversion refused whatever the target access method (the new +# referenced-table-to-row-store arm). Eleven arrive never: the over-rejection +# controls that no single guard reaches (another constraint, a heap-to-heap key, +# an unreferenced conversion, an unrelated ALTER, a conversion back to heap, the +# referencing side, the partitioned control on either side of 17) and the primary +# key's own arms. +# +# ledger rows 3619 + 28 = 3647 +# never 2675 + 11 = 2686 +# dated 944 + 17 = 961 +# 2686 + 961 == 3647 +# +# #1341, truncate_cleanup seeded, with its twin (OffgridwithJD): 2686 -> 2695, and +# suites_not_covered 218 -> 217. The suite had no rows. It enters with all 25 checks +# from five-major runs (25/25 on each of 15-19). 16 are dated by four mutations of +# the truncate paths: a rewrite that no longer deletes the retired storage's rows +# (7), nor drops the stale write state (1), an in-place truncate that skips the +# projections' storages (8), and a storage-tree delete that skips them (1). Nine +# arrive never: the premises that each fixture holds rows or metadata, the +# surviving projection declaration, the empty table after TRUNCATE, and the +# rewrite that keeps every row. +# +# ledger rows 3647 + 25 = 3672 +# never 2686 + 9 = 2695 +# dated 961 + 16 = 977 +# 2695 + 977 == 3672 +# +# #1341, native_dml seeded, with its twin (OffgridwithJD): 2695 -> 2702, and +# suites_not_covered 217 -> 216. The suite had no rows. It enters with all 15 checks +# from five-major runs (15/15 on each of 15-19). 8 are dated by three mutations: a +# row group holding deletes still answering min and max from its zone maps (the +# min/max arm, which could not see it before this change), count(*) from metadata +# not subtracting deleted rows (3), and the index fetch ignoring the delete vector +# (5). Seven arrive never: the two premises, the rows and sum after the delete, the +# deleted rows being gone, the live row's index lookup, and the primary key arm. +# +# ledger rows 3672 + 15 = 3687 +# never 2695 + 7 = 2702 +# dated 977 + 8 = 985 +# 2702 + 985 == 3687 +checks_never_observed_red 2702 diff --git a/test/fk_referencing.sh b/test/fk_referencing.sh index 73b59c41d..cd49dcecd 100755 --- a/test/fk_referencing.sh +++ b/test/fk_referencing.sh @@ -220,6 +220,16 @@ check "converting back to heap is unaffected" \ "$(case "$back" in *ERROR*) echo "no ($back)" ;; *) echo yes ;; esac)" \ "yes" +# AND ON THE REFERENCED TABLE ITSELF (#1341). The control above converts a table +# with no foreign key, so it cannot see a rule that refuses every SET ACCESS METHOD +# on a referenced table whatever the target: with the guard's columnar-target test +# removed, this file stayed 26/26. Only a columnar target needs the row lock the +# constraint cannot get. +refheap="$(psql_run "ALTER TABLE fk_conv_parent SET ACCESS METHOD heap;" 2>&1 || true)" +check "an FK-referenced table may still be set to a row store" \ + "$(case "$refheap" in *ERROR*) echo "no ($refheap)" ;; *) echo yes ;; esac)" \ + "yes" + # --- 7. a partitioned parent is refused too (#201) ----------------------------- # A partitioned table has no storage, so setting its access method breaks nothing diff --git a/test/native_dml.sh b/test/native_dml.sh index 9e67578b6..ae68bc317 100755 --- a/test/native_dml.sh +++ b/test/native_dml.sh @@ -21,9 +21,15 @@ GEN="SELECT g, (g * 10)::bigint, 'row-' || g FROM generate_series(1, 5000) g" psql_run "INSERT INTO h $GEN;" psql_run "INSERT INTO n $GEN;" -# DELETE across several row groups. -psql_run "DELETE FROM h WHERE id % 7 = 0;" -psql_run "DELETE FROM n WHERE id % 7 = 0;" +# DELETE across several row groups, and the table's smallest and largest ids with +# them. Without those two, min and max are the same whether or not the deleted rows +# count, so the min/max arm below could not see a zone-map answer that ignored the +# deletes (#1341): measured, with groups that hold deletes left on the zone-map path, +# this file stayed 13/13. +psql_run "DELETE FROM h WHERE id % 7 = 0 OR id IN (1, 5000);" +psql_run "DELETE FROM n WHERE id % 7 = 0 OR id IN (1, 5000);" +check "premise: the deletes moved the smallest and largest id and k" \ + "$(q 'SELECT min(id) || '"'"'|'"'"' || max(id) || '"'"'|'"'"' || min(k) || '"'"'|'"'"' || max(k) FROM h;')" "2|4999|20|49990" check "count after delete" "$(q 'SELECT count(*) FROM n;')" "$(q 'SELECT count(*) FROM h;')" check "rows after delete match heap" \ @@ -32,14 +38,20 @@ check "rows after delete match heap" \ # Aggregates after delete: the zone-map path is bypassed (deletes present), so this # exercises the delete-applying scan fold. check "sum(k) after delete" "$(q 'SELECT sum(k) FROM n;')" "$(q 'SELECT sum(k) FROM h;')" +# k as well as id: measured, a zone-map answer that ignored the deletes still gave +# min/max(id) right and min/max(k) wrong (10|50000 for 20|49990). check "min/max after delete" \ - "$(q 'SELECT min(id), max(id) FROM n;')" \ - "$(q 'SELECT min(id), max(id) FROM h;')" + "$(q 'SELECT min(id), max(id), min(k), max(k) FROM n;')" \ + "$(q 'SELECT min(id), max(id), min(k), max(k) FROM h;')" check "deleted rows gone" "$(q 'SELECT count(*) FROM n WHERE id % 7 = 0;')" "0" # Index lookup of a deleted row returns nothing; a live row still resolves. idx() { env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -q -At -c "SET enable_seqscan=off" -c "$1" 2>/dev/null; } # id 4444 is live (4444 % 7 = 6); id 4445 was deleted (4445 % 7 = 0). +# The lookups must take the primary-key index, or these arms test the scan instead +# of the index fetch's delete check. +check "premise: the lookups take the primary-key index" \ + "$(idx 'EXPLAIN (COSTS off) SELECT count(*) FROM n WHERE id = 4445;' | grep -c 'Index.* using n_pkey')" "1" check "index finds live row" "$(idx 'SELECT v FROM n WHERE id = 4444;')" "$(q 'SELECT v FROM h WHERE id = 4444;')" check "index skips deleted row" "$(idx 'SELECT count(*) FROM n WHERE id = 4445;')" "0" diff --git a/test/native_upgrade_converge.sh b/test/native_upgrade_converge.sh index 0249d5655..454900137 100755 --- a/test/native_upgrade_converge.sh +++ b/test/native_upgrade_converge.sh @@ -310,4 +310,53 @@ done rm -f "$REF" P -d postgres -c "DROP DATABASE IF EXISTS conv_fresh;" >/dev/null +# ---- the window: this library over the previous release's catalog (#1341 item 66) ---- +# +# docs/installation.md steps 1 and 2 put the new library in place; step 3, +# ALTER EXTENSION UPDATE, brings the catalog up to it. Between them a database +# runs this library over its previous release's catalog. The library deletes +# from retired_group unconditionally (columnar_metadata.c, PgColumnarDeleteMetadata), +# which that catalog does not have, so DROP TABLE, TRUNCATE and pgcolumnar.vacuum() +# fail there: measured, with 42P01 'columnar metadata table +# "pgcolumnar.retired_group" does not exist', which names neither the cause nor step 3. +# +# The arms hold what the window must keep: reads and writes work, a refused +# operation changes nothing and names the remedy, and step 3 makes every one of +# them work on the same rows. PREV is the newest fixture, so these follow the +# release they describe rather than naming it. +PREV="${_NUC_FIXTURES[${#_NUC_FIXTURES[@]}-1]}" +WIN=conv_window +P -d postgres -c "DROP DATABASE IF EXISTS $WIN;" >/dev/null +P -d postgres -c "CREATE DATABASE $WIN;" >/dev/null +P -d "$WIN" -c "CREATE EXTENSION pgcolumnar VERSION '$PREV';" >/dev/null +check "premise: the window database runs this library over the previous release's catalog" \ + "$(P -d "$WIN" -c "SELECT extversion FROM pg_extension WHERE extname='pgcolumnar';")" "$PREV" +check "premise: that catalog has no retired_group, which this library deletes from" \ + "$(P -d "$WIN" -c "SELECT count(*) FROM pg_class WHERE relname = 'retired_group' AND relnamespace = 'pgcolumnar'::regnamespace;")" "0" +P -d "$WIN" -c "CREATE TABLE w (a int, b text) USING pgcolumnar;" >/dev/null +P -d "$WIN" -c "INSERT INTO w SELECT g, 'x' || g FROM generate_series(1, 20000) g;" >/dev/null +P -d "$WIN" -c "DELETE FROM w WHERE a % 3 = 0;" >/dev/null +win_rows() { P -d "$WIN" -c "SELECT count(*) || '|' || sum(a) FROM w;"; } +check "in the window, a columnar table is written, deleted from and read" "$(win_rows)" "13334|133346667" +win_refusal() { # win_refusal SQL -> SQLSTATE|whether the HINT names step 3, or what happened instead + local out + out="$(P -d "$WIN" -v VERBOSITY=verbose -c "$1" 2>&1)" && { echo accepted; return; } + echo "$(sed -nE 's/^ERROR: ([0-9A-Z]{5}): .*/\1/p' <<<"$out" | head -1)|$(grep -q '^HINT: .*ALTER EXTENSION pgcolumnar UPDATE' <<<"$out" && echo names-the-update || echo no-hint)" +} +check "in the window, DROP TABLE is refused and names ALTER EXTENSION UPDATE" \ + "$(win_refusal "DROP TABLE w;")" "42P01|names-the-update" +check "in the window, TRUNCATE is refused and names ALTER EXTENSION UPDATE" \ + "$(win_refusal "TRUNCATE w;")" "42P01|names-the-update" +check "in the window, pgcolumnar.vacuum() is refused and names ALTER EXTENSION UPDATE" \ + "$(win_refusal "SELECT pgcolumnar.vacuum('w');")" "42P01|names-the-update" +check "in the window, the refused operations changed nothing" "$(win_rows)" "13334|133346667" + +P -d "$WIN" -c "ALTER EXTENSION pgcolumnar UPDATE;" >/dev/null +check "after ALTER EXTENSION UPDATE, the same rows are there" "$(win_rows)" "13334|133346667" +check "after ALTER EXTENSION UPDATE, pgcolumnar.vacuum() succeeds and keeps them" \ + "$(P -d "$WIN" -c "SELECT pgcolumnar.vacuum('w');" >/dev/null 2>&1 && win_rows || echo refused)" "13334|133346667" +check "after ALTER EXTENSION UPDATE, TRUNCATE and DROP TABLE succeed" \ + "$(P -d "$WIN" -c "TRUNCATE w;" >/dev/null 2>&1 && P -d "$WIN" -c "DROP TABLE w;" >/dev/null 2>&1 && echo succeeded || echo refused)" "succeeded" +P -d postgres -c "DROP DATABASE IF EXISTS $WIN;" >/dev/null + pgc_summary diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 9212ee1a5..a855303d9 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -188,6 +188,10 @@ behaviour, the source of that number is named. - [140. test_parquet_import.py: a partitioned columnar parent is refused by import_parquet](#140-test_parquet_importpy-a-partitioned-columnar-parent-is-refused-by-import_parquet) - [141. test_phase5.py: set_options and reset_options agree](#141-test_phase5py-set_options-and-reset_options-agree) - [142. test_drop_cleanup.py: a columnar matview's options row goes with its DROP](#142-test_drop_cleanuppy-a-columnar-matviews-options-row-goes-with-its-drop) +- [143. test_native_upgrade_converge.py: the upgrade window, before ALTER EXTENSION UPDATE](#143-test_native_upgrade_convergepy-the-upgrade-window-before-alter-extension-update) +- [144. test_fk_referencing.py: a foreign key referencing a columnar table is refused where it is chosen](#144-test_fk_referencingpy-a-foreign-key-referencing-a-columnar-table-is-refused-where-it-is-chosen) +- [145. test_truncate_cleanup.py: TRUNCATE leaves no catalog rows behind, and keeps what it must](#145-test_truncate_cleanuppy-truncate-leaves-no-catalog-rows-behind-and-keeps-what-it-must) +- [146. test_native_dml.py: DELETE, UPDATE and MERGE on a native table agree with heap](#146-test_native_dmlpy-delete-update-and-merge-on-a-native-table-agree-with-heap) ## 1. How to read a test in here @@ -7377,3 +7381,52 @@ Part of the pytest twin of `test/drop_cleanup.sh` (#1265, ported for #1341): its | `test_a_columnar_matview_takes_its_options_row_with_it` | a planted options row is gone after DROP MATERIALIZED VIEW | | `test_refresh_keeps_a_matviews_options_row` | a REFRESH that rewrote the matview leaves its set_options row in place | | `test_an_unrelated_options_row_survives` | another table's row survives a matview's DROP, and an ordinary columnar table still takes its own | + +## 143. test_native_upgrade_converge.py: the upgrade window, before ALTER EXTENSION UPDATE + +Part of the pytest twin of `test/native_upgrade_converge.sh` (#1341 item 66): its upgrade-window arms only, by the shell's names, and the pair is declared INCOMPLETE. Between installing a new library and running `ALTER EXTENSION pgcolumnar UPDATE`, a database runs the new library over its previous release's catalog. Reads and writes must work there. `DROP TABLE`, `TRUNCATE` and `pgcolumnar.vacuum()`, which need a catalog table the old one lacks, must be refused with 42P01, change nothing, and carry a hint naming the update; after the update all three succeed on the same rows. The previous release is the version the shipped upgrade script to the current `default_version` starts from; its base script is staged from `test/fixtures` for the test and removed after. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_the_upgrade_window` | at the previous catalog: rows written, deleted and read; the three operations refused with the hint and nothing changed; after the update the rows are intact and all three succeed | + +## 144. test_fk_referencing.py: a foreign key referencing a columnar table is refused where it is chosen + +The pytest twin of `test/fk_referencing.sh` (ported for #1341), every check by the shell's name. Enforcing a foreign key locks the referenced row, and columnar tables cannot lock rows, so a constraint referencing one would make every insert into the referencing table fail. It is refused at DDL by either route into that state: creating the constraint against a columnar table, or converting a referenced table (or partitioned parent) to columnar. Only the referenced side is refused, and the controls hold everything else steady, including converting the referenced table itself to a row store. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_the_referenced_side_is_refused_at_ddl` | CREATE TABLE and ALTER TABLE ADD CONSTRAINT referencing a columnar table are refused, saying why, and nothing is left behind | +| `test_the_referencing_side_still_works` | a columnar table referencing a heap table is created, accepts satisfying rows and rejects a violating one | +| `test_constraints_that_need_no_row_lock_are_unaffected` | primary key, unique, check and not-null on a columnar table, and a heap-to-heap foreign key, are unaffected | +| `test_converting_a_referenced_table_is_refused_and_nothing_else_is` | SET ACCESS METHOD pgcolumnar on a referenced table and the helper are refused, the table stays heap and its child writable; an unreferenced table, the referencing side, an unrelated ALTER, a conversion back to heap and the referenced table to a row store all still work | +| `test_a_partitioned_referenced_parent` | a columnar partition and setting the parent's access method are refused, the parent does not become columnar, an ordinary partition still works; an unreferenced partitioned table converts from PG17, and before 17 core refuses it | + +## 145. test_truncate_cleanup.py: TRUNCATE leaves no catalog rows behind, and keeps what it must + +The pytest twin of `test/truncate_cleanup.sh` (ported for #1341), every check by the shell's name. TRUNCATE installs a new storage id, and the old storage's catalog rows used to stay behind, so each truncate-and-reload cycle leaked one storage's worth and a later DROP removed only the empty current one. Retiring them made a cached write state stale, which is dropped with them. A relation created in the current transaction is truncated in place, on a different path, which must clear each projection's storage too. A private database, since the snapshot counts every pgcolumnar catalog row. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_truncate_and_drop_leave_the_catalog_as_they_found_it` | one TRUNCATE and three truncate-and-reload cycles return the catalog to the baseline, and a DROP of a reloaded table leaves nothing | +| `test_a_projection_is_not_leaked_by_truncate` | no projection row names a retired storage id, the declaration survives, and a DROP of the reloaded table leaves nothing | +| `test_repeated_truncates_accumulate_nothing` | ten transactional truncates of one table, and its DROP, leave the baseline | +| `test_insert_truncate_insert_in_one_transaction` | the transaction commits and holds exactly the rows written after the TRUNCATE | +| `test_a_rewrite_that_is_not_a_truncate_loses_nothing` | a type-changing rewrite keeps all 5000 rows readable | +| `test_an_in_place_truncate_clears_the_projection_too` | the in-place truncate commits, the later rows are all there, the projection reads them back and its storage holds one group of exactly them | + +## 146. test_native_dml.py: DELETE, UPDATE and MERGE on a native table agree with heap + +The pytest twin of `test/native_dml.sh` (ported for #1341), every check by the shell's name. Each statement runs on a heap table and on a columnar one with 1000-row stripes, and every result is compared. After deletes an aggregate must not come from zone maps that still describe the deleted rows: the deletes include the smallest and largest ids, so min and max move, and they are taken over `k`, a column the primary key does not order. A deleted row's index lookup finds nothing, and the lookups are asserted to take the index. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_delete_update_and_merge_match_heap` | after DELETE, UPDATE and MERGE, counts, rows, sum and min/max match heap; the index finds a live row and skips a deleted one; the primary key still rejects a duplicate | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index a6a77ccba..383f58447 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -747,4 +747,12 @@ guard_tests 438 # seven tests, by collection over the cluster files on this tree. # cluster_tests 653 -> 660 for #1341 item 29's partial twins: test_inheritance.py (2), # test_parquet_import.py (1), test_phase5.py (1) and test_drop_cleanup.py (3), by collection. -cluster_tests 660 +# cluster_tests 660 -> 661 for #1341 item 66's test_native_upgrade_converge.py, one test, +# by collection over the cluster files on this tree. +# cluster_tests 661 -> 666 for #1341's test_fk_referencing.py, the twin of fk_referencing.sh: +# five tests, by collection over the cluster files on this tree. +# cluster_tests 666 -> 672 for #1341's test_truncate_cleanup.py, the twin of truncate_cleanup.sh: +# six tests, by collection over the cluster files on this tree. +# cluster_tests 672 -> 673 for #1341's test_native_dml.py, the twin of native_dml.sh: one test, +# by collection over the cluster files on this tree. +cluster_tests 673 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 4140fbf32..42b4f5f1f 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -106,6 +106,7 @@ "differential", "encode_post_codec", "entry_point_privilege", + "fk_referencing", "format_downgrade", "hilbert_cluster", "hilbert_locality", @@ -121,6 +122,7 @@ "native_chunk_length_bound", "native_delete_vector_index", "native_delete_visibility_paths", + "native_dml", "native_fetch_coalesce", "native_groupagg", "native_index_fetch_stripe_cost", @@ -166,6 +168,7 @@ "tid_range_scan", "tier_partition", "timeseries_recipes", + "truncate_cleanup", "ttl_expire", "type_matrix", "unique_conc", @@ -179,6 +182,10 @@ # about the PORT rather than a licence: the standing arm does not grade it, so the # reason is the only thing standing between a declared gap and a forgotten one. INCOMPLETE = { + # #1341 item 66 ported the arms it added: the upgrade window, this library over + # the previous release's catalog before ALTER EXTENSION UPDATE. + "native_upgrade_converge": "the upgrade-window arms are ported; the fixture-blob, " + "staging and catalog-convergence arms are not", # #1341 item 29 ported the arms four fixes added with shell arms only. Each # pair below holds that fix's arms and nothing else of its shell suite. "inheritance": "the #1259 arms (a partitioned columnar parent is refused with " diff --git a/test/pytest/test_fk_referencing.py b/test/pytest/test_fk_referencing.py new file mode 100644 index 000000000..3e7a60274 --- /dev/null +++ b/test/pytest/test_fk_referencing.py @@ -0,0 +1,158 @@ +"""A foreign key that references a columnar table is refused where it is chosen +(#1341). The pytest twin of `test/fk_referencing.sh`, every check by the shell's name. + +Referential integrity reads the referenced row with FOR KEY SHARE, and row locking is +not implemented for columnar tables, so a constraint referencing one would make every +insert into the referencing table fail. It is refused at DDL instead, by either route +into that state: creating the constraint against a columnar table, or converting a +referenced table to columnar. Only the REFERENCED side is refused: a columnar table may +reference a heap table, and the controls hold everything else steady -- other +constraints, heap-to-heap keys, conversions of an unreferenced table, of the +referencing side, back to heap, and of the referenced table to a row store. +""" +import psycopg + + +def _err(conn, sql): + """-> '' when sql succeeds, else the error's message, detail and hint, joined.""" + try: + with conn.cursor() as cur: + cur.execute(sql) + return "" + except psycopg.Error as exc: + d = exc.diag + return " ".join(x for x in (d.message_primary, d.message_detail, d.message_hint) if x) or str(exc) + + +def _has(text, needle): + return "yes" if needle in text else f"no ({text})" + + +def _ok(text): + return "yes" if not text else f"no ({text})" + + +def _one(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetchone()[0] + + +def _run(conn, *sqls): + with conn.cursor() as cur: + for sql in sqls: + cur.execute(sql) + + +def test_the_referenced_side_is_refused_at_ddl(pgc_conn, expect): + conn = pgc_conn + _run(conn, "CREATE TABLE fk_parent (id int PRIMARY KEY) USING pgcolumnar", + "INSERT INTO fk_parent VALUES (1), (2)") + err = _err(conn, "CREATE TABLE fk_child (id int REFERENCES fk_parent(id))") + expect.text(_has(err, "cannot create a foreign key referencing columnar table"), "yes", + "a foreign key referencing a columnar table is refused") + expect.text(_has(err, "row locking is not supported"), "yes", "and the error says why") + expect.num(_one(conn, "SELECT count(*) FROM pg_class WHERE relname = 'fk_child' " + "AND relnamespace = current_schema()::regnamespace"), 0, + "the refused table was not left behind") + + _run(conn, "CREATE TABLE fk_child2 (id int)") + err2 = _err(conn, "ALTER TABLE fk_child2 ADD CONSTRAINT fk_c2 FOREIGN KEY (id) REFERENCES fk_parent(id)") + expect.text(_has(err2, "cannot create a foreign key referencing columnar table"), "yes", + "ALTER TABLE ADD CONSTRAINT is refused by the same rule") + + +def test_the_referencing_side_still_works(pgc_conn, expect): + conn = pgc_conn + _run(conn, "CREATE TABLE fk_h_parent (id int PRIMARY KEY)", "INSERT INTO fk_h_parent VALUES (1), (2), (3)") + expect.text(_ok(_err(conn, "CREATE TABLE fk_c_child (id int REFERENCES fk_h_parent(id), v text) " + "USING pgcolumnar")), "yes", + "a columnar table may reference a heap table") + _err(conn, "INSERT INTO fk_c_child VALUES (1, 'a'), (2, 'b')") + expect.num(_one(conn, "SELECT count(*) FROM fk_c_child"), 2, "and rows that satisfy it are accepted") + expect.text(_has(_err(conn, "INSERT INTO fk_c_child VALUES (99, 'nope')"), "violates foreign key constraint"), + "yes", "a row that violates it is still rejected") + + +def test_constraints_that_need_no_row_lock_are_unaffected(pgc_conn, expect): + conn = pgc_conn + expect.text(_ok(_err(conn, "CREATE TABLE fk_other (id int PRIMARY KEY, k int UNIQUE, v int CHECK (v > 0), " + "w int NOT NULL) USING pgcolumnar")), "yes", + "primary key, unique, check and not-null on a columnar table are unaffected") + _err(conn, "INSERT INTO fk_other VALUES (1, 10, 5, 7)") + expect.num(_one(conn, "SELECT count(*) FROM fk_other"), 1, "and they still work") + expect.text(_has(_err(conn, "INSERT INTO fk_other VALUES (1, 11, 5, 7)"), "duplicate key"), "yes", + "the primary key is still enforced") + _run(conn, "CREATE TABLE fk_hh_parent (id int PRIMARY KEY)") + expect.text(_ok(_err(conn, "CREATE TABLE fk_hh_child (id int REFERENCES fk_hh_parent(id))")), "yes", + "a foreign key between two heap tables is unaffected") + + +def test_converting_a_referenced_table_is_refused_and_nothing_else_is(pgc_conn, expect): + conn = pgc_conn + _run(conn, "CREATE TABLE fk_conv_parent (id int PRIMARY KEY)", + "CREATE TABLE fk_conv_child (id int REFERENCES fk_conv_parent(id))", + "INSERT INTO fk_conv_parent VALUES (1)", "INSERT INTO fk_conv_child VALUES (1)") + conv = _err(conn, "ALTER TABLE fk_conv_parent SET ACCESS METHOD pgcolumnar") + expect.text(_has(conv, "referenced by a foreign key"), "yes", + "converting an FK-referenced table to columnar is refused") + expect.text(_has(conv, "row locking is not supported"), "yes", "and the error names the constraint") + expect.text(_one(conn, "SELECT am.amname::text FROM pg_class c JOIN pg_am am ON am.oid = c.relam " + "WHERE c.oid = 'fk_conv_parent'::regclass"), "heap", "the parent is still a row store") + _err(conn, "INSERT INTO fk_conv_parent VALUES (2)") + expect.text(_ok(_err(conn, "INSERT INTO fk_conv_child VALUES (2)")), "yes", + "the child is still writable afterwards") + expect.text(_has(_err(conn, "SELECT pgcolumnar.alter_table_set_access_method('fk_conv_parent', 'pgcolumnar')"), + "referenced by a foreign key"), "yes", "the conversion helper is refused identically") + + # The over-rejection controls: a rule keyed on the wrong side, or firing for any + # ALTER TABLE or any target, passes every arm above and fails one of these. + _run(conn, "CREATE TABLE fk_conv_plain (id int PRIMARY KEY)", "INSERT INTO fk_conv_plain VALUES (1)") + expect.text(_ok(_err(conn, "ALTER TABLE fk_conv_plain SET ACCESS METHOD pgcolumnar")), "yes", + "a table with no foreign key still converts") + expect.text(_ok(_err(conn, "ALTER TABLE fk_conv_child SET ACCESS METHOD pgcolumnar")), "yes", + "converting the referencing side is still allowed") + expect.text(_ok(_err(conn, "ALTER TABLE fk_conv_parent ADD COLUMN note text")), "yes", + "an unrelated ALTER TABLE on a referenced table is unaffected") + expect.text(_ok(_err(conn, "ALTER TABLE fk_conv_plain SET ACCESS METHOD heap")), "yes", + "converting back to heap is unaffected") + # The control above has no foreign key; this one is the referenced table itself. + expect.text(_ok(_err(conn, "ALTER TABLE fk_conv_parent SET ACCESS METHOD heap")), "yes", + "an FK-referenced table may still be set to a row store") + + +def test_a_partitioned_referenced_parent(pgc_conn, expect): + """Setting a partitioned parent's access method chooses what every later partition + inherits, and core clones the foreign key to each, so it is refused too (#201).""" + conn = pgc_conn + _run(conn, "CREATE TABLE fk_pp (id int PRIMARY KEY) PARTITION BY RANGE (id)", + "CREATE TABLE fk_pp1 PARTITION OF fk_pp FOR VALUES FROM (1) TO (100)", + "CREATE TABLE fk_pc (id int REFERENCES fk_pp(id))") + expect.text(_has(_err(conn, "CREATE TABLE fk_pp2 PARTITION OF fk_pp FOR VALUES FROM (100) TO (200) " + "USING pgcolumnar"), "cannot create a foreign key referencing columnar table"), + "yes", "a columnar partition of an FK-referenced parent is refused") + expect.text(_has(_err(conn, "ALTER TABLE fk_pp SET ACCESS METHOD pgcolumnar"), "cannot convert table"), "yes", + "and setting the parent's access method is refused as well") + # "did it become columnar", not "is it heap": a partitioned table never given an + # access method has relam = 0, which joins to no pg_am row. + expect.num(_one(conn, "SELECT count(*) FROM pg_class c JOIN pg_am a ON a.oid = c.relam " + "WHERE c.oid = 'fk_pp'::regclass AND a.amname = 'pgcolumnar'"), 0, + "so the parent did not become columnar") + expect.num(0 if not _err(conn, "CREATE TABLE fk_pp3 PARTITION OF fk_pp FOR VALUES FROM (200) TO (300)") else 1, + 0, "and a partition can still be created the ordinary way") + + _run(conn, "CREATE TABLE fk_free (id int) PARTITION BY RANGE (id)") + major = int(_one(conn, "SHOW server_version_num")) // 10000 + if major >= 17: + expect.num(0 if not _err(conn, "ALTER TABLE fk_free SET ACCESS METHOD pgcolumnar") else 1, 0, + "a partitioned table with no foreign key still converts") + _err(conn, "CREATE TABLE fk_free1 PARTITION OF fk_free FOR VALUES FROM (1) TO (100)") + expect.text(_one(conn, "SELECT amname::text FROM pg_class c JOIN pg_am a ON a.oid = c.relam " + "WHERE c.oid = 'fk_free1'::regclass"), "pgcolumnar", + "and its partitions inherit the columnar access method") + else: + # Before 17 core refuses SET ACCESS METHOD on every partitioned table, so the + # refusal must be core's, by core's message. + expect.text(_has(_err(conn, "ALTER TABLE fk_free SET ACCESS METHOD pgcolumnar"), + "cannot change access method of a partitioned table"), "yes", + "before 17, core refuses this for every partitioned table") diff --git a/test/pytest/test_native_dml.py b/test/pytest/test_native_dml.py new file mode 100644 index 000000000..3a97bde8a --- /dev/null +++ b/test/pytest/test_native_dml.py @@ -0,0 +1,92 @@ +"""DELETE, UPDATE and MERGE on a native-format columnar table agree with a heap mirror +(#1341). The pytest twin of `test/native_dml.sh`, every check by the shell's name. + +Each statement runs on a heap table and on a columnar one with stripes of 1000 rows, +and every result is compared: the surviving rows, counts, aggregates and index lookups. +After deletes, an aggregate must not be answered from zone maps that still describe the +deleted rows. The deletes include the table's smallest and largest ids, so min and max +move and an answer that ignored the deletes differs from heap's; min and max are taken +over k too, a column the primary key does not order. An index lookup of a deleted row +must find nothing, and the lookups are asserted to take the index. +""" + + +def _one(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetchone()[0] + + +def _row(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetchone() + + +def _rows(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetchall() + + +def _run(conn, *sqls): + with conn.cursor() as cur: + for sql in sqls: + cur.execute(sql) + + +GEN = "SELECT g, (g * 10)::bigint, 'row-' || g FROM generate_series(1, 5000) g" + + +def test_delete_update_and_merge_match_heap(pgc_conn, expect): + conn = pgc_conn + _run(conn, "CREATE TABLE h (id int PRIMARY KEY, k bigint, v text)", + "CREATE TABLE n (id int PRIMARY KEY, k bigint, v text) USING pgcolumnar", + "SELECT pgcolumnar.set_options('n', stripe_row_limit => 1000)", + f"INSERT INTO h {GEN}", f"INSERT INTO n {GEN}") + + for t in ("h", "n"): + _run(conn, f"DELETE FROM {t} WHERE id % 7 = 0 OR id IN (1, 5000)") + expect.text(_one(conn, "SELECT min(id) || '|' || max(id) || '|' || min(k) || '|' || max(k) FROM h"), + "2|4999|20|49990", "premise: the deletes moved the smallest and largest id and k") + expect.num(_one(conn, "SELECT count(*) FROM n"), _one(conn, "SELECT count(*) FROM h"), "count after delete") + expect.row_set(_rows(conn, "SELECT id, k, v FROM n"), _rows(conn, "SELECT id, k, v FROM h"), + "rows after delete match heap") + expect.num(_one(conn, "SELECT sum(k) FROM n"), _one(conn, "SELECT sum(k) FROM h"), "sum(k) after delete") + expect.rows([_row(conn, "SELECT min(id), max(id), min(k), max(k) FROM n")], + [_row(conn, "SELECT min(id), max(id), min(k), max(k) FROM h")], "min/max after delete") + expect.num(_one(conn, "SELECT count(*) FROM n WHERE id % 7 = 0"), 0, "deleted rows gone") + + # id 4444 is live (4444 % 7 = 6); id 4445 was deleted (4445 % 7 = 0) + _run(conn, "SET enable_seqscan = off") + plan = "\n".join(r[0] for r in _rows(conn, "EXPLAIN (COSTS off) SELECT count(*) FROM n WHERE id = 4445")) + expect.num(sum(1 for line in plan.splitlines() if "Index" in line and "using n_pkey" in line), 1, + "premise: the lookups take the primary-key index") + expect.text(_one(conn, "SELECT v FROM n WHERE id = 4444"), _one(conn, "SELECT v FROM h WHERE id = 4444"), + "index finds live row") + expect.num(_one(conn, "SELECT count(*) FROM n WHERE id = 4445"), 0, "index skips deleted row") + _run(conn, "RESET enable_seqscan") + + for t in ("h", "n"): + _run(conn, f"UPDATE {t} SET k = k + 1 WHERE id % 5 = 0") + expect.num(_one(conn, "SELECT count(*) FROM n"), _one(conn, "SELECT count(*) FROM h"), "count after update") + expect.row_set(_rows(conn, "SELECT id, k, v FROM n"), _rows(conn, "SELECT id, k, v FROM h"), + "rows after update match heap") + expect.num(_one(conn, "SELECT sum(k) FROM n"), _one(conn, "SELECT sum(k) FROM h"), "sum(k) after update") + + import psycopg + try: + _run(conn, "INSERT INTO n VALUES (4444, 0, 'x')") + dup = "accepted" + except psycopg.Error as exc: + dup = exc.sqlstate + expect.text(dup, "23505", "PK still enforced after DML") + + _run(conn, "CREATE TABLE src (id int, k bigint)", "INSERT INTO src VALUES (1, 999), (999001, 5), (999002, 6)") + for t in ("h", "n"): + _run(conn, f"MERGE INTO {t} t USING src s ON t.id = s.id " + "WHEN MATCHED THEN UPDATE SET k = s.k " + "WHEN NOT MATCHED THEN INSERT (id, k, v) VALUES (s.id, s.k, 'm')") + expect.num(_one(conn, "SELECT count(*) FROM n"), _one(conn, "SELECT count(*) FROM h"), "count after merge") + expect.row_set(_rows(conn, "SELECT id, k, v FROM n"), _rows(conn, "SELECT id, k, v FROM h"), + "rows after merge match heap") diff --git a/test/pytest/test_native_upgrade_converge.py b/test/pytest/test_native_upgrade_converge.py new file mode 100644 index 000000000..5a2c1d54b --- /dev/null +++ b/test/pytest/test_native_upgrade_converge.py @@ -0,0 +1,126 @@ +"""The upgrade window: this library over the previous release's catalog, before ALTER +EXTENSION UPDATE (#1341 item 66). Part of the pytest twin of +`test/native_upgrade_converge.sh`: these arms only, by the shell's names, and the pair +is declared INCOMPLETE. + +docs/installation.md steps 1 and 2 put a new library in place; step 3, ALTER EXTENSION +UPDATE, brings each database's catalog up to it. Between them a database runs the new +library over the old catalog, and the library deletes from catalog tables the old one +does not have (retired_group, new in 1.0-alpha5). Reads and writes must work there; an +operation that needs a missing table must be refused cleanly, change nothing and name +step 3; and after step 3 every one of them must work on the same rows. + +The previous release is the version the shipped upgrade script to the current +default_version starts from, and its base script is staged from test/fixtures into the +extension directory for the test, as the shell does, then removed. +""" +import pathlib +import re +import shutil +import subprocess + +import psycopg +import pytest + +REPO = pathlib.Path(__file__).resolve().parents[2] + + +def _target_and_previous(): + target = re.search(r"^default_version\s*=\s*'([^']+)'", + (REPO / "pgcolumnar.control").read_text(), re.M).group(1) + prev = [p.name[len("pgcolumnar--"):-len(f"--{target}.sql")] + for p in REPO.glob(f"pgcolumnar--*--{target}.sql")] + return target, prev + + +@pytest.fixture +def window_db(pgc_cluster): + """-> (dsn, previous version) of a database at the previous release's catalog.""" + target, prev = _target_and_previous() + assert len(prev) == 1, f"one upgrade script should lead to {target}, found {prev}" + prev = prev[0] + sharedir = subprocess.run([pgc_cluster.pg_config, "--sharedir"], capture_output=True, + text=True, check=True).stdout.strip() + dst = pathlib.Path(sharedir) / "extension" / f"pgcolumnar--{prev}.sql" + backup = dst.with_name(dst.name + ".pytest-bak") + if dst.exists(): + shutil.copy2(dst, backup) + shutil.copy2(REPO / "test" / "fixtures" / f"pgcolumnar--{prev}.sql", dst) + + def admin(sql): + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as c: + c.execute(sql) + + admin("DROP DATABASE IF EXISTS pgc_upgrade_window") + admin("CREATE DATABASE pgc_upgrade_window") + dsn = pgc_cluster.dsn(dbname="pgc_upgrade_window") + try: + with psycopg.connect(dsn, autocommit=True) as c: + c.execute(f"CREATE EXTENSION pgcolumnar VERSION '{prev}'") + yield dsn, prev + finally: + admin("DROP DATABASE IF EXISTS pgc_upgrade_window WITH (FORCE)") + if backup.exists(): + shutil.move(backup, dst) + else: + dst.unlink(missing_ok=True) + + +def _one(dsn, sql): + with psycopg.connect(dsn, autocommit=True) as c: + return c.execute(sql).fetchone()[0] + + +def _refusal(dsn, sql): + """-> 'SQLSTATE|names-the-update' or '|no-hint', or 'accepted'.""" + try: + with psycopg.connect(dsn, autocommit=True) as c: + c.execute(sql) + return "accepted" + except psycopg.Error as exc: + hint = exc.diag.message_hint or "" + return f"{exc.sqlstate}|{'names-the-update' if 'ALTER EXTENSION pgcolumnar UPDATE' in hint else 'no-hint'}" + + +def _succeeds(dsn, *sqls): + try: + with psycopg.connect(dsn, autocommit=True) as c: + for sql in sqls: + c.execute(sql) + return "succeeded" + except psycopg.Error as exc: + return f"refused ({exc.sqlstate}: {str(exc).splitlines()[0]})" + + +def test_the_upgrade_window(window_db, expect): + dsn, prev = window_db + expect.text(_one(dsn, "SELECT extversion FROM pg_extension WHERE extname = 'pgcolumnar'"), prev, + "premise: the window database runs this library over the previous release's catalog") + expect.num(_one(dsn, "SELECT count(*) FROM pg_class WHERE relname = 'retired_group' " + "AND relnamespace = 'pgcolumnar'::regnamespace"), 0, + "premise: that catalog has no retired_group, which this library deletes from") + with psycopg.connect(dsn, autocommit=True) as c: + c.execute("CREATE TABLE w (a int, b text) USING pgcolumnar") + c.execute("INSERT INTO w SELECT g, 'x' || g FROM generate_series(1, 20000) g") + c.execute("DELETE FROM w WHERE a % 3 = 0") + + def rows(): + return _one(dsn, "SELECT count(*) || '|' || sum(a) FROM w") + + expect.text(rows(), "13334|133346667", "in the window, a columnar table is written, deleted from and read") + expect.text(_refusal(dsn, "DROP TABLE w"), "42P01|names-the-update", + "in the window, DROP TABLE is refused and names ALTER EXTENSION UPDATE") + expect.text(_refusal(dsn, "TRUNCATE w"), "42P01|names-the-update", + "in the window, TRUNCATE is refused and names ALTER EXTENSION UPDATE") + expect.text(_refusal(dsn, "SELECT pgcolumnar.vacuum('w')"), "42P01|names-the-update", + "in the window, pgcolumnar.vacuum() is refused and names ALTER EXTENSION UPDATE") + expect.text(rows(), "13334|133346667", "in the window, the refused operations changed nothing") + + with psycopg.connect(dsn, autocommit=True) as c: + c.execute("ALTER EXTENSION pgcolumnar UPDATE") + expect.text(rows(), "13334|133346667", "after ALTER EXTENSION UPDATE, the same rows are there") + vacuumed = _succeeds(dsn, "SELECT pgcolumnar.vacuum('w')") + expect.text(rows() if vacuumed == "succeeded" else vacuumed, "13334|133346667", + "after ALTER EXTENSION UPDATE, pgcolumnar.vacuum() succeeds and keeps them") + expect.text(_succeeds(dsn, "TRUNCATE w", "DROP TABLE w"), "succeeded", + "after ALTER EXTENSION UPDATE, TRUNCATE and DROP TABLE succeed") diff --git a/test/pytest/test_truncate_cleanup.py b/test/pytest/test_truncate_cleanup.py new file mode 100644 index 000000000..c3c34ab9e --- /dev/null +++ b/test/pytest/test_truncate_cleanup.py @@ -0,0 +1,181 @@ +"""TRUNCATE leaves no catalog rows behind, and keeps what it must (#1341). The pytest +twin of `test/truncate_cleanup.sh`, every check by the shell's name. + +TRUNCATE installs a new relfilenode and a new storage id. It used to leave the old +storage's row_group, column_chunk, zone_map and bloom rows behind, keyed by the retired +id, so each truncate-and-reload cycle leaked one storage's worth, and a DROP after it +removed only the current, empty one. Retiring them made a cached write state stale, so +a transaction that wrote, truncated and wrote again committed into storage nothing +reads; the state is now dropped with the rows. A relation created in the current +transaction is truncated in place instead, on a different path, which must clear each +projection's storage as well as the base's. + +A PRIVATE DATABASE, because the snapshot counts every pgcolumnar catalog row, and the +shared one holds every other test's. +""" +import psycopg + + +def _one(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetchone()[0] + + +def _run(conn, *sqls): + with conn.cursor() as cur: + for sql in sqls: + cur.execute(sql) + + +def _snapshot(conn): + return _one(conn, "SELECT (SELECT count(*) FROM pgcolumnar.storage) || '/' || " + "(SELECT count(*) FROM pgcolumnar.projection) || '/' || " + "(SELECT count(*) FROM pgcolumnar.row_group) || '/' || " + "(SELECT count(*) FROM pgcolumnar.column_chunk) || '/' || " + "(SELECT count(*) FROM pgcolumnar.zone_map) || '/' || " + "(SELECT count(*) FROM pgcolumnar.bloom) || '/' || " + "(SELECT count(*) FROM pgcolumnar.delete_vector)") + + +def _grew(base, now): + return "no" if base == now else "yes" + + +LOAD = "INSERT INTO {t} SELECT g, 'x' || g FROM generate_series(1, {n}) g" + + +def test_truncate_and_drop_leave_the_catalog_as_they_found_it(pgc_own_db, expect): + conn = pgc_own_db + base = _snapshot(conn) + _run(conn, "CREATE TABLE tcl_plain (id int, v text) USING pgcolumnar", LOAD.format(t="tcl_plain", n=5000)) + expect.text(_grew(base, _snapshot(conn)), "yes", "insert did add metadata") + _run(conn, "TRUNCATE tcl_plain") + expect.text(_snapshot(conn), base, "TRUNCATE returns the catalog to the empty baseline") + expect.num(_one(conn, "SELECT count(*) FROM tcl_plain"), 0, "TRUNCATE leaves no rows") + for _ in range(2): + _run(conn, LOAD.format(t="tcl_plain", n=5000), "TRUNCATE tcl_plain") + expect.text(_snapshot(conn), base, "three truncate-and-reload cycles still sit at the baseline") + # loaded first, so the DROP has work to do: a DROP of an empty table could not fail + _run(conn, LOAD.format(t="tcl_plain", n=5000)) + expect.text(_grew(base, _snapshot(conn)), "yes", + "premise: the table carries metadata again, so the DROP has work to do") + _run(conn, "DROP TABLE tcl_plain") + expect.text(_snapshot(conn), base, "DROP of a loaded table after TRUNCATE leaves nothing") + + +def test_a_projection_is_not_leaked_by_truncate(pgc_own_db, expect): + """A projection has its own storage id. What must hold is that no projection row + names a storage id the table no longer has, not that projection is empty.""" + conn = pgc_own_db + base = _snapshot(conn) + _run(conn, "CREATE TABLE tcl_proj (id int, a int, b text) USING pgcolumnar", + "SELECT pgcolumnar.add_projection('tcl_proj', 'tcl_p', ARRAY['a', 'b'], ARRAY['a'])", + "INSERT INTO tcl_proj SELECT g, g % 50, 'b' || g FROM generate_series(1, 5000) g") + expect.text(_grew(base, _snapshot(conn)), "yes", "the projection did add metadata") + _run(conn, "TRUNCATE tcl_proj") + expect.num(_one(conn, "SELECT count(*) FROM pgcolumnar.projection p " + "WHERE p.storage_id <> pgcolumnar.get_storage_id('tcl_proj'::regclass)"), 0, + "TRUNCATE leaves no projection row under a retired storage id") + expect.num(_one(conn, "SELECT count(*) FROM pgcolumnar.projection_declaration " + "WHERE rel = 'tcl_proj'::regclass"), 1, + "and the declaration survives the TRUNCATE, so the projection is rebuildable") + _run(conn, "INSERT INTO tcl_proj SELECT g, g % 50, 'b' || g FROM generate_series(1, 5000) g") + expect.text(_grew(base, _snapshot(conn)), "yes", "premise: the projected table carries metadata again before the DROP") + _run(conn, "DROP TABLE tcl_proj") + expect.text(_snapshot(conn), base, "DROP of a loaded projected table leaves nothing") + + +def test_repeated_truncates_accumulate_nothing(pgc_own_db, expect): + """Created once outside the loop and one statement at a time, so every TRUNCATE + takes the transactional path and any leak accumulates.""" + conn = pgc_own_db + base = _snapshot(conn) + _run(conn, "CREATE TABLE tcl_rep (id int, v text) USING pgcolumnar") + for _ in range(10): + _run(conn, LOAD.format(t="tcl_rep", n=2000), "TRUNCATE tcl_rep") + expect.text(_snapshot(conn), base, "ten insert-and-truncate cycles on one table leave nothing") + _run(conn, "DROP TABLE tcl_rep") + expect.text(_snapshot(conn), base, "and dropping it afterwards still leaves nothing") + + +def test_insert_truncate_insert_in_one_transaction(pgc_own_db, expect): + """The cached write state named the storage the TRUNCATE retired, so the second + INSERT flushed into storage nothing reads, and the transaction committed an empty + table. The status and the surviving rows are one arm: split, a commit that keeps + nothing passes the status half, and 1..10 passes a bare count of 10.""" + conn = pgc_own_db + _run(conn, "CREATE TABLE tc_txn (id int) USING pgcolumnar", + "INSERT INTO tc_txn SELECT g FROM generate_series(1, 10) g") + expect.num(_one(conn, "SELECT count(*) FROM tc_txn"), 10, "premise: the table holds its first ten rows") + rc = 0 + try: + _run(conn, "BEGIN", "INSERT INTO tc_txn SELECT g FROM generate_series(11, 20) g", "TRUNCATE tc_txn", + "INSERT INTO tc_txn SELECT g FROM generate_series(21, 30) g", "COMMIT") + except psycopg.Error: + rc = 1 + _run(conn, "ROLLBACK") + expect.text(f"rc={rc} " + _one(conn, "SELECT coalesce(min(id), -1) || '..' || coalesce(max(id), -1) || " + "' n=' || count(*) FROM tc_txn"), + "rc=0 21..30 n=10", "INSERT, TRUNCATE, INSERT in one transaction commits and keeps the new rows") + + +def test_a_rewrite_that_is_not_a_truncate_loses_nothing(pgc_own_db, expect): + """The retire runs from relation_set_new_filelocator, which every rewrite calls.""" + conn = pgc_own_db + _run(conn, "CREATE TABLE tc_rw (id int, v text) USING pgcolumnar", + "INSERT INTO tc_rw SELECT g, 'v' || g FROM generate_series(1, 5000) g") + expect.num(_one(conn, "SELECT count(*) FROM tc_rw"), 5000, "premise: five thousand rows before the rewrite") + _run(conn, "ALTER TABLE tc_rw ALTER COLUMN v TYPE varchar(64)") + expect.text(_one(conn, "SELECT count(*) || '/' || count(v) || '/' || max(id) FROM tc_rw"), "5000/5000/5000", + "a full rewrite keeps every row, and they are still readable") + + +def test_an_in_place_truncate_clears_the_projection_too(pgc_own_db, expect): + """A relation created in the current transaction is truncated in place, through + nontransactional_truncate, which cleared the base storage and left the + projection's. The write after the truncate is what makes the leak observable: it + collided on row_group_pkey (#896).""" + conn = pgc_own_db + _run(conn, "CREATE TABLE tcl_inplace_pre (id int)") + committed = "committed" + try: + _run(conn, "BEGIN", + "CREATE TABLE tcl_inplace (id int, a int, b text) USING pgcolumnar", + "INSERT INTO tcl_inplace SELECT g, g % 50, 'b' || g FROM generate_series(1, 2000) g", + "SELECT pgcolumnar.add_projection('tcl_inplace', 'tcl_ip', ARRAY['a', 'b'], ARRAY['a'])", + "TRUNCATE tcl_inplace", + "INSERT INTO tcl_inplace SELECT g, g % 50, 'c' || g FROM generate_series(1, 500) g", + "COMMIT") + except psycopg.Error as exc: + committed = f"rolled back ({str(exc).splitlines()[0]})" + _run(conn, "ROLLBACK") + # vacuous without these: a rolled-back transaction leaves no table to count + expect.num(0 if committed == "committed" else 1, 0, "PREMISE the in-place transaction committed") + expect.num(_one(conn, "SELECT count(*) FROM pg_class WHERE relname = 'tcl_inplace'"), 1, + "PREMISE the table exists after it") + if committed != "committed": + return + expect.num(_one(conn, "SELECT count(*) FROM tcl_inplace"), 500, + "the rows written after the in-place truncate are all there") + expect.num(_one(conn, "SELECT count(*) FROM pgcolumnar.read_projection('tcl_inplace', 'tcl_ip')"), 500, + "and the projection reads them back") + with conn.cursor() as cur: + cur.execute("SELECT pgcolumnar.read_projection('tcl_inplace', 'tcl_ip')") + proj = cur.fetchall() + cur.execute("SELECT a::text || '|' || b FROM tcl_inplace") + base = cur.fetchall() + expect.row_set(proj, base, "the projection agrees with the base table") + proj_storage = _one(conn, "SELECT proj_storage_id FROM pgcolumnar.projection " + "WHERE storage_id = pgcolumnar.get_storage_id('tcl_inplace'::regclass) " + "AND projection_id > 0") + base_storage = _one(conn, "SELECT pgcolumnar.get_storage_id('tcl_inplace'::regclass)") + expect.text("yes" if proj_storage is not None and proj_storage != base_storage else "no", "yes", + "PREMISE the projection has its own storage, distinct from the base") + # one group of 500 rows, not a group NUMBER: the base's counter is reset and the + # projection's has no metapage to reset, so the numbers differ by design + expect.num(_one(conn, f"SELECT count(*) FROM pgcolumnar.row_group WHERE storage_id = {proj_storage}"), 1, + "the projection storage holds exactly one row group after the truncate") + expect.num(_one(conn, f"SELECT coalesce(sum(row_count), 0) FROM pgcolumnar.row_group " + f"WHERE storage_id = {proj_storage}"), 500, + "and that group describes only the rows written after it")