test: a projection sorted on a type with no ordering is refused (0A000), not a backend crash; projections seeds the ledger (#1341)

test/check_ledger_budget.txt

@@ -15,7 +15,7 @@
 #
 #   Adding a check to a suite that is already covered does not move it, which is
 #   what makes it safe to bound.
-suites_not_covered 230
+suites_not_covered 226
 #
 # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become
 #   one.
@@ -2533,4 +2533,114 @@ suites_not_covered 230
 #     never         2344 + 1 = 2345
 #     dated          767 + 3 = 770
 #     2345 + 770 == 3115
-checks_never_observed_red 2345
+#
+# #1341 item 75, the changelog names recluster with expire, compact_rewrite and
+# VACUUM as recomputing the column extent, and only the other three had arms:
+# 2345 -> 2346. column_extent gains 3 checks from five-major runs (42/42 on each
+# of 15-19). Two are dated on PG17 by the existing mutation "PgColumnarRetireGroup
+# without marking the storage for a recompute", which also re-dates the four
+# arms it already held. "recluster rewrites the table's row groups" arrives never.
+#
+#     ledger rows   3115 + 3 = 3118
+#     never         2345 + 1 = 2346
+#     dated          770 + 2 = 772
+#     2346 + 772 == 3118
+#
+# #1341, tier_partition and partition_create_ahead on the caller's own privileges
+# (OffgridwithJD): 2346 -> 2352. Eleven new checks, all in already-covered suites.
+# Five are dated, by each function made SECURITY DEFINER: tier_partition's refusal,
+# the partition left local, no foreign table and no export file (four), and
+# partition_create_ahead's refusal (one). The two owner controls run on tables of
+# their own, so neither mutation reddens them; they and four premises arrive never.
+# Five-major runs: tier_partition 79/79 (80 on 18-19, which run the virtual column)
+# and partition_lifecycle 64/64 on 15-19.
+#
+#     ledger rows   3118 + 11 = 3129
+#     never         2346 +  6 = 2352
+#     dated          772 +  5 =  777
+#     2352 + 777 == 3129
+#
+# #1341, rls_direct_storage seeded with its pytest twin (OffgridwithJD): 2352 -> 2362,
+# and suites_not_covered 230 -> 229. The suite had no rows. It enters with all 26
+# checks from five-major runs (26/26 on each of 15-19). Sixteen are dated by three
+# mutations of the RLS guard: deleted (the eight refusals, nothing written, nothing
+# imported, and FORCE), widened to refuse RLS_NONE_ENV (the owner's read, the owner's
+# export and the owner without FORCE), and moved ahead of read_projection's ACL check
+# (the two disclosure arms). The owner's export was measured by a test both of whose
+# branches echoed "ran"; it now reads the export back, and main's suite survives the
+# widened guard on it. The other 10 arrive never: the premises and the no-policy
+# controls.
+#
+#     ledger rows   3129 + 26 = 3155
+#     never         2352 + 10 = 2362
+#     dated          777 + 16 = 793
+#     2362 + 793 == 3155
+#
+# #1341, rls_direct_storage's two scan-dependent premises (OffgridwithJD, #1397 review):
+# 2362 -> 2361. Both read the table through the columnar scan, so they are checks and
+# lose the prefix, renamed with their rows. "the policy is in force for ordinary SQL"
+# is dated by the custom scan dropping its quals (the restricted role then saw all 200
+# rows); the owner's count arrives never. Five-major runs: 26/26 on 15-19.
+#
+#     ledger rows   3155     = 3155
+#     never         2362 - 1 = 2361
+#     dated          793 + 1 = 794
+#     2361 + 794 == 3155
+#
+# #1383 review (OffgridwithJD): 2361 -> 2360. "the source files the import arms need exist"
+# loses the premise prefix: the owner's export writes those files, and an export that
+# writes nothing reddens it in both twins (the twin now fails it by name instead of
+# raising), with the owner-export arm beside it. Five-major runs: 26/26 on 15-19.
+#
+#     ledger rows   3155     = 3155
+#     never         2361 - 1 = 2360
+#     dated          794 + 1 = 795
+#     2360 + 795 == 3155
+#
+# #1341, objstore_crlf seeded, with its pytest twin (OffgridwithJD): 2360 -> 2362, and
+# suites_not_covered 229 -> 228. The suite had no rows. It enters with all 5 checks
+# from five-major runs (5/5 on each of 15-19). Three are dated by the request-line
+# guard on the URL path removed: neither refusal named CR or LF, and the listener
+# received the injected line. The two backend-alive checks arrive never.
+#
+#     ledger rows   3155 + 5 = 3160
+#     never         2360 + 2 = 2362
+#     dated          795 + 3 =  798
+#     2362 + 798 == 3160
+#
+# #1341, objstore_allowlist seeded, with its twin and a fixed admin-override arm
+# (OffgridwithJD): 2362 -> 2374, and suites_not_covered 228 -> 227. The suite had no
+# rows. It enters with all 16 checks from five-major runs (16/16 on each of 15-19).
+# Four are dated: the two default-deny arms by an empty list that admits every
+# endpoint; the non-superuser arm and the admin-override arm by the setting defined
+# PGC_SIGHUP. Main's admin-override arm survived that one, because the configured
+# list still named 127.0.0.1; it now runs with the list empty. The other 12 arrive
+# never.
+#
+#     ledger rows   3160 + 16 = 3176
+#     never         2362 + 12 = 2374
+#     dated          798 +  4 = 802
+#     2374 + 802 == 3176
+#
+# #1379 review (OffgridwithJD): 2374 -> 2371. "the list defaults to empty" loses the premise
+# prefix, dated by the default changed to 127.0.0.1; the shell now has the query name
+# "empty" itself rather than comparing with an empty string. The allow-list check that
+# never refuses also dates "unlisted host" and "host:port entry, wrong port".
+# Five-major runs: 16/16 on 15-19, the twin passing with pyarrow required.
+#
+#     ledger rows   3176     = 3176
+#     never         2374 - 3 = 2371
+#     dated          802 + 3 = 805
+#     2371 + 805 == 3176
+#
+# #1341, projections seeded, with the sort-type arms (OffgridwithJD): 2371 -> 2446, and
+# suites_not_covered 227 -> 226. The suite had no rows. It enters with all 76 checks
+# from five-major runs (76/76 on each of 15-19). One is dated by the projection
+# writer without its sort-type ordering check: a json sort key's back-fill then
+# crashed the backend instead of refusing with 0A000. The other 75 arrive never.
+#
+#     ledger rows   3176 + 76 = 3252
+#     never         2371 + 75 = 2446
+#     dated          805 +  1 =  806
+#     2446 + 806 == 3252
+checks_never_observed_red 2446

test/column_extent.sh

@@ -213,6 +213,21 @@ q "CREATE TABLE vr (ts timestamptz, v int) USING pgcolumnar;
 q "VACUUM vr;" >/dev/null
 check "after VACUUM retires a fully deleted group the extent is the live rows'" "$(ext vr ts)" "$(truth vr ts)"
 
+# recluster rewrites every group too, and the changelog names it with the three
+# above (#1341 item 75): the rows a DELETE removed are not written back, so its
+# extent must be the live rows', not wider by rows no longer stored. The deleted
+# rows hold the minimum, so a stale extent keeps it.
+q "CREATE TABLE rc (ts timestamptz, v int) USING pgcolumnar;
+   SELECT pgcolumnar.set_options('rc', stripe_row_limit => 1000, chunk_group_row_limit => 1000);
+   INSERT INTO rc SELECT '2026-01-01'::timestamptz + g * interval '1 min', (g * 7919) % 3001 FROM generate_series(1, 3000) g;
+   DELETE FROM rc WHERE ts < '2026-01-01'::timestamptz + interval '400 min';" >/dev/null
+rc_before="$(ext rc ts)"
+reclustered="$(q "SELECT pgcolumnar.recluster('rc', 'ts', 'v');")"
+check "recluster rewrites the table's row groups" "$([ "${reclustered:-0}" -ge 1 ] && echo yes || echo "no ($reclustered)")" "yes"
+check "after recluster the extent is the live rows'" "$(ext rc ts)" "$(truth rc ts)"
+check "and it did move: the deleted rows held the old minimum" \
+	"$([ -n "$rc_before" ] && [ "$rc_before" != "$(ext rc ts)" ] && echo yes || echo "no ($rc_before)")" "yes"
+
 # ---- no complete extent where the zone maps cannot give one ----------------------
 q "CREATE TABLE ac (ts timestamptz) USING pgcolumnar;
    SELECT pgcolumnar.set_options('ac', stripe_row_limit => 1000, chunk_group_row_limit => 1000);

test/objstore_allowlist.sh

@@ -69,8 +69,12 @@ URL="http://127.0.0.1:$HTTP_PORT/fixture.parquet"
 Q="SELECT count(*) FROM pgcolumnar.read_parquet('$URL') AS (a int8)"
 
 # ---- the default: deny everything remote ------------------------------------
-check "premise: the list defaults to empty" \
-	"$(q "SELECT current_setting('pgcolumnar.objstore_allowed_endpoints')")" ""
+# A check, not a premise (#1236; #1379 review): the default is the product's. And
+# "empty" is produced by the query itself, so a capture that is empty because the
+# query never ran cannot pass.
+check "the list defaults to empty" \
+	"$(q "SELECT CASE WHEN current_setting('pgcolumnar.objstore_allowed_endpoints') = '' THEN 'empty'
+	          ELSE 'set: [' || current_setting('pgcolumnar.objstore_allowed_endpoints') || ']' END")" "empty"
 check "default: an http read is refused 42501" "$(sqlstate_of "$Q")" "42501"
 check "default: the refusal names the GUC" \
 	"$([ "$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -qtA \
@@ -96,6 +100,9 @@ check "premise: the role can open a session" \
 	"$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U allow_nobody -d "$PGC_DB" -qtA -c 'SELECT 1' 2>&1)" "1"
 check "a non-superuser cannot widen the list (42501)" \
 	"$(sqlstate_of "SET pgcolumnar.objstore_allowed_endpoints = '10.0.0.1'" allow_nobody)" "42501"
+# WITH THE CONFIGURED LIST EMPTY (#1341). It was still "other.example, 127.0.0.1" here,
+# so the read worked whatever the session SET did, and the arm could not fail.
+set_list ""
 check_num "a superuser session SET works (admin override)" \
 	"$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -qtA \
 		-c "SET pgcolumnar.objstore_allowed_endpoints = '127.0.0.1'" -c "$Q" 2>/dev/null | tail -1)" "1000"

test/partition_lifecycle.sh

@@ -241,6 +241,40 @@ check "a parent with a DEFAULT partition is refused" \
 check "and nothing was created for it" \
 	"$(kids td)|$(q "SELECT count(*) FROM pg_proc WHERE proname = 'partition_create_ahead';")" "1|1"
 
+# ---- the caller's own privileges (#1341) -------------------------------------
+#
+# partition_create_ahead is not SECURITY DEFINER, so its CREATE TABLE ... PARTITION OF
+# runs as the caller, and core refuses a role that does not own the parent. Made
+# SECURITY DEFINER, it would run as the extension's owner and let any caller add
+# partitions, and the access-exclusive lock that comes with them, to a table it does
+# not own. Both roles hold USAGE and CREATE on the schema, so ownership is all that
+# separates them.
+for r in pl_own pl_peer; do
+	psql_run "DROP ROLE IF EXISTS $r; CREATE ROLE $r LOGIN NOSUPERUSER;
+	          GRANT USAGE, CREATE ON SCHEMA public TO $r; GRANT USAGE ON SCHEMA pgcolumnar TO $r;"
+done
+as_role() {	# as_role ROLE SQL -> the single value SQL returns as ROLE, or the SQLSTATE it fails with
+	local out
+	out=$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U "$1" -d "$PGC_DB" \
+		-X -At -v ON_ERROR_STOP=1 -v VERBOSITY=sqlstate -c "$2" 2>&1)
+	case "$out" in
+		*ERROR:*|*FATAL:*) sed -n 's/^\(ERROR\|FATAL\): *\([0-9A-Z]\{5\}\).*/\2/p' <<<"$out" | head -1 ;;
+		*) printf '%s\n' "$out" | tail -1 ;;
+	esac
+}
+# The owner's control has a parent of its own (pw2), so a refusal that let the peer
+# through cannot make the control fail for the peer's reason.
+q "CREATE TABLE pv (ts timestamptz NOT NULL, v int) PARTITION BY RANGE (ts); ALTER TABLE pv OWNER TO pl_own;
+   CREATE TABLE pw2 (ts timestamptz NOT NULL, v int) PARTITION BY RANGE (ts); ALTER TABLE pw2 OWNER TO pl_own;" >/dev/null
+check "premise: pl_own owns the parent, and neither role is a superuser" \
+	"$(q "SELECT string_agg(DISTINCT pg_get_userbyid(relowner), ',') FROM pg_class WHERE oid IN ('pv'::regclass, 'pw2'::regclass);")|$(q "SELECT count(*) FROM pg_roles WHERE rolname IN ('pl_own', 'pl_peer') AND rolsuper;")" \
+	"pl_own|0"
+check "premise: pl_peer can open a session" "$(as_role pl_peer "SELECT 1")" "1"
+check "a role that does not own the parent is refused by its own privileges (42501), and creates nothing" \
+	"$(as_role pl_peer "SELECT pgcolumnar.partition_create_ahead('pv', '1 day', 2)")|$(kids pv)" "42501|0"
+check "a non-superuser owner creates partitions ahead on its own table" \
+	"$(as_role pl_own "SELECT pgcolumnar.partition_create_ahead('pw2', '1 day', 2)")|$(kids pw2)" "3|3"
+
 # ==== the drop half: partition_drop_plan (#1297) ================================
 #
 #   partition_drop_plan(parent regclass, cutoff timestamptz) -> SETOF text

test/pgc_ledger.py

@@ -312,6 +312,16 @@ def read_ledger(path):
                     f"{path}:{n}: major {m!r} is neither a number nor "
                     f"{MAJOR_UNKNOWN!r}, so this row names no version it applies to")
         muts = set() if f[5] == NONE else {m for m in f[5].split(";") if m}
+        # A PIECE WITH WHITESPACE AT AN END IS A FRAGMENT, not a mutation. The writer
+        # joins the set with a bare ";", so it never produces one; a text that held
+        # "; " does, when it is split on the ";" (12 rows on the shipped ledger, 8 of
+        # them from one recovery_minimal_wal mutation, re-sorted by every later merge).
+        for m in sorted(muts):
+            if m != m.strip():
+                raise LedgerError(
+                    f"{path}:{n}: mutation {m!r} has whitespace at an end, so it is a "
+                    f"fragment of one mutation's text split on ';'. Reword that mutation "
+                    f"without ';' and keep the row's other mutations as they are.")
         key = (f[0], f[1], f[2])
         # A REPEATED KEY, REFUSED RATHER THAN OVERWRITTEN. This was
         # `rows[key] = [...]`, so a duplicated row collapsed silently and the LAST line
@@ -530,6 +540,12 @@ def cmd_merge(args):
             "--target names what a mutation killed, so it needs --mutation to say "
             "which mutation. On its own it attributes nothing.")
 
+    if args.mutation and ";" in args.mutation:
+        raise LedgerError(
+            f"--mutation {args.mutation!r} holds ';', which separates mutations in the "
+            f"ledger's mutation column, so it would be stored as several. Reword it "
+            f"without ';' (', and' reads the same).")
+
     if args.mutation:
         failed = sorted({key for _p, seen in runs for key, vs in seen.items()
                          if any(v == "FAIL" for v, _m in vs)})

test/projections.sh

@@ -448,4 +448,21 @@ psql_run "INSERT INTO dt2 SELECT g, g FROM generate_series(200, 300) g;"
 check "control: a drop in its own transaction never left one" \
 	"$(proj_orphans dt2)" "0"
 
+# ---- a sort key with no ordering is refused (#1341) --------------------------
+#
+# add_projection does not check the sort column's type. The writer does, when it
+# is built: json has no btree comparison, so the back-fill over existing rows
+# must refuse with 0A000 rather than sort by a comparison function that does not
+# exist. Last in the suite, because without the check that sort crashes the
+# backend (measured).
+psql_run "CREATE TABLE pjs (id int, j json) USING pgcolumnar;
+	  INSERT INTO pjs SELECT g, json_build_object('k', g) FROM generate_series(1, 500) g;"
+check "a projection sorted on a type with no ordering (json) is refused (0A000)" \
+	"$(proj_sqlstate "SELECT pgcolumnar.add_projection('pjs', 'pjp', ARRAY['id','j'], ARRAY['j'])")" "0A000"
+# On a server that accepts connections again: without the check the sort crashes
+# its backend (measured), and this arm must answer for the catalog, not for that.
+for _i in $(seq 1 60); do [ "$(q 'SELECT 1' 2>/dev/null)" = 1 ] && break; sleep 0.5; done
+check "and nothing of it is declared" \
+	"$(q "SELECT count(*) FROM pgcolumnar.projection_declaration WHERE name = 'pjp';")" "0"
+
 pgc_summary