diff --git a/CHANGELOG.md b/CHANGELOG.md index 803430e91..7db731a92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -401,6 +401,22 @@ true until the next version shipped. ### Fixed +- **A vectorized aggregate no longer shows a non-leakproof function rows that row-level + security hides.** Under a policy, a query such as `SELECT grp, count(*) FROM t WHERE + f(secret) GROUP BY grp` ran `f` on every row, the other owners' included, through the + grouped vectorized node (the default for a GROUP BY) and the ungrouped one + (`pgcolumnar.enable_ungrouped_vector_agg`). The answers were right; only the function + saw too much. Both nodes now leave a WHERE whose clauses could leak past the policy to + the custom scan, which orders its quals as PostgreSQL does. A WHERE with only leakproof + clauses, or a table with no policy, vectorizes as before. Found by OffgridwithJD in + review. `test/rls_query_paths.sh` and `test_rls_query_paths.py`. + +- **`pgcolumnar.column_extent` refuses a table whose row-level security policies + apply to the caller.** It returned the column's stored smallest and largest value + over every row, so a caller limited by a policy could read values from rows it + cannot see. It now raises `row-level security is in force`, as the other functions + that read storage directly do. Found by OffgridwithJD in review. + - **A concurrent `UPDATE` of the same row now fails with 40001 on a table with a unique index, as documented, not 23505 (#1421).** Two sessions updating one row of a table with a PRIMARY KEY or UNIQUE index: the second used to wait on the first's diff --git a/docs/limitations.md b/docs/limitations.md index 7b343fd3b..d64ba6fe0 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -173,7 +173,9 @@ caller every row, so the call is refused instead. The functions affected are `read_projection`, `reconstruct_via_projection`, `export_arrow`, `export_parquet`, `parallel_export_parquet`, `import_arrow`, -`import_parquet` and `parallel_copy`. +`import_parquet`, `parallel_copy` and `column_extent`. `column_extent` reports a +column's stored smallest and largest value over every row. Under a policy, that +would show values from rows the caller cannot see. A superuser, a role with `BYPASSRLS`, and a table's owner without `FORCE ROW LEVEL SECURITY` are not refused. Each already reads every row by other means. diff --git a/docs/sql-reference.md b/docs/sql-reference.md index 085275157..d16b81067 100644 --- a/docs/sql-reference.md +++ b/docs/sql-reference.md @@ -468,6 +468,9 @@ commit. Every operation that drops or rewrites row groups recomputes it before committing: `expire`, `compact_rewrite`, `VACUUM`, `recluster`. So it can be wider than the data only by rows that are deleted but still stored. +It needs `SELECT` on the table, and it refuses a table whose row-level security +policies apply to the caller (see [limitations](limitations.md#row-level-security)). + No row means the planner reads every row group's zone map instead, which gives the same answer at a cost that grows with the table. That happens for: diff --git a/src/columnar_extent.c b/src/columnar_extent.c index 1a43395e3..e47066724 100644 --- a/src/columnar_extent.c +++ b/src/columnar_extent.c @@ -527,6 +527,12 @@ pgcolumnar_column_extent(PG_FUNCTION_ARGS) ac = pg_class_aclcheck(relid, GetUserId(), ACL_SELECT); if (ac != ACLCHECK_OK) aclcheck_error(ac, OBJECT_TABLE, get_rel_name(relid)); + /* + * The stored extent covers every row, so under a policy it would show values + * from rows the caller cannot see. Refused as the other direct-storage entry + * points are (#563), after the ACL check (OffgridwithJD, review of #1429). + */ + PgColumnarRequireNoRowSecurity(relid); rel = relation_open(relid, AccessShareLock); if (!PgColumnarIsColumnarRelation(relid)) ereport(ERROR, diff --git a/src/columnar_vector.c b/src/columnar_vector.c index 7444f2e8c..3eb72b5b9 100644 --- a/src/columnar_vector.c +++ b/src/columnar_vector.c @@ -63,6 +63,7 @@ #include "nodes/nodeFuncs.h" #include "nodes/pathnodes.h" #include "nodes/plannodes.h" +#include "optimizer/clauses.h" #include "optimizer/optimizer.h" #include "optimizer/pathnode.h" #include "optimizer/paths.h" @@ -820,6 +821,38 @@ static bool pgcolumnar_batch_shape_eligible(PgColumnarAggScanState *state, * vectorized aggregate: planning * ------------------------------------------------------------------------- */ + +/* + * pgcolumnar_quals_could_leak + * True when a clause above the lowest security level could leak, so a node that + * evaluates the WHERE itself must not take the query. + * + * Row-level security puts the policy's quals at a lower security level than the + * user's. PostgreSQL lets a clause run before a lower-level one only if it is + * leakproof, and order_qual_clauses enforces that for an ordinary plan. The + * vectorized nodes build their own quals from baserestrictinfo and evaluate them + * in their own order, so a non-leakproof function in the user's WHERE was handed + * rows the policy hides (OffgridwithJD, review of #1429). Declining leaves such a + * query to the custom scan, which orders its quals correctly. + */ +static bool +pgcolumnar_quals_could_leak(List *rinfos) +{ + Index minLevel = UINT_MAX; + ListCell *lc; + + foreach(lc, rinfos) + minLevel = Min(minLevel, lfirst_node(RestrictInfo, lc)->security_level); + foreach(lc, rinfos) + { + RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); + + if (rinfo->security_level > minLevel && + contain_leaked_vars((Node *) rinfo->clause)) + return true; + } + return false; +} static Plan * PgColumnarPlanAggPath(PlannerInfo *root, RelOptInfo *rel, CustomPath *best_path, List *tlist, List *clauses, List *custom_plans) @@ -1529,6 +1562,8 @@ PgColumnarCreateUpperPaths(PlannerInfo *root, UpperRelationKind stage, * zone-map answerable. A filter, or a sum/avg over int8/float/numeric, needs * the scan-fold path instead (#289). */ + if (pgcolumnar_quals_could_leak(factRel->baserestrictinfo)) + return; quals = extract_actual_clauses(factRel->baserestrictinfo, false); needsScan = (quals != NIL); @@ -2586,8 +2621,12 @@ PgColumnarTryGroupAggPath(PlannerInfo *root, RelOptInfo *input_rel, /* * WHERE is carried whole and rechecked per row, so correctness never depends - * on which clauses become scan keys; the scan keys only prune groups. + * on which clauses become scan keys; the scan keys only prune groups. A WHERE + * whose clauses could leak past a row-level security policy is left to the + * custom scan (pgcolumnar_quals_could_leak). */ + if (pgcolumnar_quals_could_leak(input_rel->baserestrictinfo)) + return; quals = extract_actual_clauses(input_rel->baserestrictinfo, false); /* diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 40f578002..b69d6584f 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2861,6 +2861,30 @@ rls_direct_storage rls_direct_storage the policy is in force for ordinary SQL 15 rls_direct_storage rls_direct_storage the source files the import arms need exist 15;16;17;18;19 2026-10-14 export_parquet writes no file (total = 0) rls_direct_storage rls_direct_storage under FORCE the non-superuser owner IS refused, so owner-only is no substitute 15;16;17;18;19 2026-10-14 PgColumnarRequireNoRowSecurity never refuses (the RLS guard deleted) rls_direct_storage rls_direct_storage without FORCE the owner is allowed, because RLS_NONE_ENV is not RLS_ENABLED 15;16;17;18;19 2026-10-14 the RLS guard refuses anything but RLS_NONE so the owner and superuser (RLS_NONE_ENV) are refused too +rls_query_paths rls_query_paths a GROUP BY under the policy, with vectorization on, is heap's 15;16;17;18;19 2026-10-15 the grouped vectorized node's quals keep only clauses with security_level above 0, dropping the policy (src/columnar_vector.c, its baserestrictinfo walk) +rls_query_paths rls_query_paths a cheap function that is not leakproof sees only rq_alice's rows on columnar, as on heap 15;16;17;18;19 2026-10-15 the custom scan's plan qual keeps only clauses with security_level above 0, dropping the policy (src/columnar_customscan.c PgColumnarPlanCustomPath) +rls_query_paths rls_query_paths a covering projection scan under the policy is heap's 15;16;17;18;19 2026-10-15 the custom scan's plan qual keeps only clauses with security_level above 0, dropping the policy (src/columnar_customscan.c PgColumnarPlanCustomPath) +rls_query_paths rls_query_paths a non-leakproof function in a GROUP BY under the policy sees only rq_alice's rows, as on heap 15;16;17;18;19 2026-10-15 the custom scan's plan qual keeps only clauses with security_level above 0, dropping the policy (src/columnar_customscan.c PgColumnarPlanCustomPath);the vectorized nodes take a WHERE whose non-leakproof clause would run before the policy (the tree before pgcolumnar_quals_could_leak) +rls_query_paths rls_query_paths a superuser sees all 20000 rows, and rq_alice her 10000 on both tables 15;16;17;18;19 2026-10-15 the custom scan's plan qual keeps only clauses with security_level above 0, dropping the policy (src/columnar_customscan.c PgColumnarPlanCustomPath);the ungrouped vectorized aggregate's quals keep only clauses with security_level above 0, dropping the policy (src/columnar_vector.c, factRel->baserestrictinfo) +rls_query_paths rls_query_paths an index-only scan under the policy is heap's 15;16;17;18;19 never - +rls_query_paths rls_query_paths and in the ungrouped vectorized aggregate 15;16;17;18;19 2026-10-15 the custom scan's plan qual keeps only clauses with security_level above 0, dropping the policy (src/columnar_customscan.c PgColumnarPlanCustomPath);the vectorized nodes take a WHERE whose non-leakproof clause would run before the policy (the tree before pgcolumnar_quals_could_leak) +rls_query_paths rls_query_paths and the owner still reads the extent 15;16;17;18;19 never - +rls_query_paths rls_query_paths and the same with vectorization and the custom scan off 15;16;17;18;19 never - +rls_query_paths rls_query_paths and through a parallel scan 15;16;17;18;19 2026-10-15 the custom scan's plan qual keeps only clauses with security_level above 0, dropping the policy (src/columnar_customscan.c PgColumnarPlanCustomPath) +rls_query_paths rls_query_paths and with vectorization and the custom scan off 15;16;17;18;19 never - +rls_query_paths rls_query_paths column_extent refuses a caller a policy restricts 15;16;17;18;19 2026-10-15 column_extent without PgColumnarRequireNoRowSecurity, so it returns the extent of rows a policy hides +rls_query_paths rls_query_paths count and sum under the policy, by the ungrouped vectorized aggregate, are heap's 15;16;17;18;19 2026-10-15 the ungrouped vectorized aggregate's quals keep only clauses with security_level above 0, dropping the policy (src/columnar_vector.c, factRel->baserestrictinfo) +rls_query_paths rls_query_paths count, sum, min and max under the policy, by a parallel scan, are heap's 15;16;17;18;19 2026-10-15 the custom scan's plan qual keeps only clauses with security_level above 0, dropping the policy (src/columnar_customscan.c PgColumnarPlanCustomPath) +rls_query_paths rls_query_paths count, sum, min and max under the policy, by the custom scan, are heap's 15;16;17;18;19 2026-10-15 the custom scan's plan qual keeps only clauses with security_level above 0, dropping the policy (src/columnar_customscan.c PgColumnarPlanCustomPath) +rls_query_paths rls_query_paths premise: a GROUP BY with a leakproof filter under the policy is the grouped vectorized node 15;16;17;18;19 never - +rls_query_paths rls_query_paths premise: an ungrouped count and sum with a leakproof filter, with the setting on, is the ungrouped vectorized aggregate 15;16;17;18;19 never - +rls_query_paths rls_query_paths premise: the GROUP BY arm's plan is the grouped vectorized node 15;16;17;18;19 never - +rls_query_paths rls_query_paths premise: the aggregate is answered by the columnar custom scan 15;16;17;18;19 never - +rls_query_paths rls_query_paths premise: the index-only arm's plan is an index-only scan 15;16;17;18;19 never - +rls_query_paths rls_query_paths premise: the parallel arm's plan gathers from workers 15;16;17;18;19 never - +rls_query_paths rls_query_paths premise: the parallel leak arm's plan gathers from workers 15;16;17;18;19 never - +rls_query_paths rls_query_paths premise: the projection arm's plan reads the covering projection 15;16;17;18;19 never - +rls_query_paths rls_query_paths premise: the ungrouped vectorized arm's plan folds the aggregates in the scan 15;16;17;18;19 never - rollup rollup * with anything but count is refused 15;16;17;18;19 2026-10-13 create_rollup allowing * with any function rollup rollup DROP TABLE of a rollup's source is refused while the rollup exists (2BP01, #1376) 15;16;17;18;19 2026-10-14 a rollup's view, partials and source with no dependency between them, and no catalog cleanup (main before #1376's fix) rollup rollup DROP VIEW of a rollup takes its partials and its catalog rows with it (#1376) 15;16;17;18;19 2026-10-14 a rollup's view, partials and source with no dependency between them, and no catalog cleanup (main before #1376's fix) @@ -3301,6 +3325,63 @@ ttl_expire ttl_expire the retention is recorded where the other options live 15; ttl_expire ttl_expire with the extent, expire retires the same groups 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum ttl_expire ttl_expire without the extent the same expire probes every group's zone map 15;16;17;18;19 never - ttl_expire ttl_expire without the extent, expire retires the aged groups 15;16;17;18;19 2026-10-14 expire returning 0 whatever it retired +type_matrix type_matrix a range on bt finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on bt holds rows on heap, so its comparison reads something 15;16;17;18;19 2026-10-15 the range bounds cast with pg_typeof, which drops the typmod, so bt's become bit(1) and its range finds no rows +type_matrix type_matrix a range on bt lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) +type_matrix type_matrix a range on cd finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on cd holds rows on heap, so its comparison reads something 15;16;17;18;19 never - +type_matrix type_matrix a range on cd skips no chunk groups: a cidr key is treated as cross-type against inet and dropped (#1433) 15;16;17;18;19 never - +type_matrix type_matrix a range on ch finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on ch holds rows on heap, so its comparison reads something 15;16;17;18;19 never - +type_matrix type_matrix a range on ch skips no chunk groups: every row group holds every value, since the column cycles through 26 letters 15;16;17;18;19 never - +type_matrix type_matrix a range on d finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on d holds rows on heap, so its comparison reads something 15;16;17;18;19 never - +type_matrix type_matrix a range on d lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) +type_matrix type_matrix a range on dom finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on dom holds rows on heap, so its comparison reads something 15;16;17;18;19 never - +type_matrix type_matrix a range on dom lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) +type_matrix type_matrix a range on e finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on e holds rows on heap, so its comparison reads something 15;16;17;18;19 never - +type_matrix type_matrix a range on e skips no chunk groups: every row group holds every value, since the column cycles through the 4 labels 15;16;17;18;19 never - +type_matrix type_matrix a range on ip finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on ip holds rows on heap, so its comparison reads something 15;16;17;18;19 never - +type_matrix type_matrix a range on ip lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) +type_matrix type_matrix a range on mac finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on mac holds rows on heap, so its comparison reads something 15;16;17;18;19 never - +type_matrix type_matrix a range on mac lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) +type_matrix type_matrix a range on mac8 finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on mac8 holds rows on heap, so its comparison reads something 15;16;17;18;19 never - +type_matrix type_matrix a range on mac8 lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) +type_matrix type_matrix a range on mo finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on mo holds rows on heap, so its comparison reads something 15;16;17;18;19 never - +type_matrix type_matrix a range on mo lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) +type_matrix type_matrix a range on n finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on n holds rows on heap, so its comparison reads something 15;16;17;18;19 never - +type_matrix type_matrix a range on n skips no chunk groups: every row group holds NaN and both infinities, so each zone map spans every value 15;16;17;18;19 never - +type_matrix type_matrix a range on nm finds heap's rows 15;16;17;18;19 2026-10-15 a by-reference fixed-width value over 8 bytes is stored as its first 8 bytes and zero padding (PgColumnarEncodeValueByLen) +type_matrix type_matrix a range on nm holds rows on heap, so its comparison reads something 15;16;17;18;19 never - +type_matrix type_matrix a range on nm lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 a by-reference fixed-width value over 8 bytes is stored as its first 8 bytes and zero padding (PgColumnarEncodeValueByLen);the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) +type_matrix type_matrix a range on tm finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on tm holds rows on heap, so its comparison reads something 15;16;17;18;19 never - +type_matrix type_matrix a range on tm lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) +type_matrix type_matrix a range on tsi finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on tsi holds rows on heap, so its comparison reads something 15;16;17;18;19 never - +type_matrix type_matrix a range on tsi lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) +type_matrix type_matrix a range on tz finds heap's rows 15;16;17;18;19 2026-10-15 a by-reference fixed-width value over 8 bytes is stored as its first 8 bytes and zero padding (PgColumnarEncodeValueByLen) +type_matrix type_matrix a range on tz holds rows on heap, so its comparison reads something 15;16;17;18;19 never - +type_matrix type_matrix a range on tz lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) +type_matrix type_matrix a range on vb finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on vb holds rows on heap, so its comparison reads something 15;16;17;18;19 never - +type_matrix type_matrix a range on vb lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) +type_matrix type_matrix a tsvector match finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix each column's count and NULL count is heap's 15;16;17;18;19 2026-10-15 a by-reference fixed-width value over 8 bytes is stored as its first 8 bytes and zero padding (PgColumnarEncodeValueByLen) +type_matrix type_matrix equality on json, as text, finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix every row, as text, is heap's 15;16;17;18;19 2026-10-15 a by-reference fixed-width value over 8 bytes is stored as its first 8 bytes and zero padding (PgColumnarEncodeValueByLen) +type_matrix type_matrix point equality (~=) finds heap's rows 15;16;17;18;19 2026-10-15 a by-reference fixed-width value over 8 bytes is stored as its first 8 bytes and zero padding (PgColumnarEncodeValueByLen) +type_matrix type_matrix premise: both tables hold 12000 rows, the columnar one in at least twelve row groups 15;16;17;18;19 never - +type_matrix type_matrix premise: the server supports xml (built with libxml) 15;16;17;18;19 never - +type_matrix type_matrix the aggregates over numeric NaN and Infinity and timestamp and date infinity are heap's 15;16;17;18;19 never - +type_matrix type_matrix the composite's fields and the enum's order read back as heap's 15;16;17;18;19 never - vacuum_lock_privilege vacuum_lock_privilege add_projection refuses a non-owner before taking its lock (#749) 15;16;17;18;19 2026-10-14 every maintenance entry point checks ownership after taking its lock vacuum_lock_privilege vacuum_lock_privilege cluster refuses a non-owner before taking the exclusive lock 15;16;17;18;19 2026-10-14 every maintenance entry point checks ownership after taking its lock vacuum_lock_privilege vacuum_lock_privilege compact does the same, which is the behaviour drop_projection now matches 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 60dbfb047..86492b7ca 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -2741,4 +2741,43 @@ suites_not_covered 224 # never 2528 + 16 = 2544 # dated 822 + 14 = 836 # 2544 + 836 == 3380 -checks_never_observed_red 2544 +# +# #1341, row-level security holds on every columnar read path (jdatcmd): 2544 -> +# 2557. rls_query_paths enters with all 24 checks from five-major runs (24/24 on each +# of 15-19). 11 are dated on PG17 by five mutants (OffgridwithJD, reviews of +# #1429; the first three reproduce on PG18 too): +# - three that drop the policy, keeping only clauses with security_level above 0, +# one per place it is applied: the custom scan's plan qual (the counts, the +# aggregates, the projection and every leak arm, the vectorized nodes' included, +# since those fall back to it), the grouped vectorized node's quals (the GROUP BY +# arm), and the ungrouped vectorized aggregate's (its arm and the counts); +# - the tree before pgcolumnar_quals_could_leak: the two vectorized-node leak arms; +# - column_extent without its row-security guard: the column_extent arm. +# The rest arrive never: the plan premises, the owner's extent control, and arms on +# paths these mutants miss. +# +# ledger rows 3380 + 24 = 3404 +# never 2544 + 13 = 2557 +# dated 836 + 11 = 847 +# 2557 + 847 == 3404 +# +# #1341, types the differential matrix never stored answer as heap (jdatcmd): +# 2557 -> 2596. type_matrix enters with all 57 checks from five-major runs (57/57 on +# each of 15-19). 17 are dated on PG17 by two mutants (OffgridwithJD, reviews of +# #1426): +# - a by-reference fixed-width value over 8 bytes stored as its first 8 bytes and +# zero padding: every row, the counts, the tz range (its zone offset varies, so +# bytes 8-11 matter), the nm range and its pruning, and point equality; +# - the scan never skipping a row group by zone map: the 12 range arms that must +# skip chunk groups, varbit's included now that its values increase with the row. +# A third, the range bounds cast with pg_typeof (which drops the typmod), dates bt's +# heap-not-empty arm: its range then finds no rows. Each of the 16 range arms has +# one, so no comparison can pass on two empty answers (review of #1426). +# The rest arrive never, among them the four range arms pinned at 0 skipped groups, +# with the reason in each name (cidr's names #1433), and the libxml premise. +# +# ledger rows 3404 + 57 = 3461 +# never 2557 + 39 = 2596 +# dated 847 + 18 = 865 +# 2596 + 865 == 3461 +checks_never_observed_red 2596 diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 87009bb81..e3fb22797 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -175,6 +175,8 @@ behaviour, the source of that number is named. - [127. test_objstore_allowlist.py: the endpoint allow-list, empty by default](#127-test_objstore_allowlistpy-the-endpoint-allow-list-empty-by-default) - [128. test_objstore_userinfo.py: userinfo in a URL is refused on every entry point](#128-test_objstore_userinfopy-userinfo-in-a-url-is-refused-on-every-entry-point) - [129. test_savepoint_flush.py: a savepoint rolled back after a flush](#129-test_savepoint_flushpy-a-savepoint-rolled-back-after-a-flush) +- [130. test_rls_query_paths.py: row-level security holds on every read path](#130-test_rls_query_pathspy-row-level-security-holds-on-every-read-path) +- [131. test_type_matrix.py: types the differential matrix never stored answer as heap](#131-test_type_matrixpy-types-the-differential-matrix-never-stored-answer-as-heap) ## 1. How to read a test in here @@ -7164,3 +7166,23 @@ The pytest twin of `test/savepoint_flush.sh` (#1341), every check by the shell's | test | what it holds | | --- | --- | | `test_a_savepoint_rolled_back_after_a_flush` | the reader paths, the flushes, and every shape answering as heap does inside the transaction and after it | + +## 130. test_rls_query_paths.py: row-level security holds on every read path + +The pytest twin of `test/rls_query_paths.sh` (#1341), every check by the shell's name. A columnar table and a heap mirror hold the same 20000 rows, half owned by each of two roles, under a policy that shows a role only its own rows. One role, on its own login, queries both by the custom scan, the vectorized and grouped aggregates, with vectorization and the custom scan off, by a parallel scan, an index-only scan and a covering projection; each answer must be heap's, and plan premises check each path was taken. A cheap function that is not leakproof must see only that role's rows, by the custom scan, with it off, and in parallel; a notice handler counts what it saw. Each vectorized node also gets a leak arm, with a premise that the same query with a leakproof filter is answered by that node, and `column_extent`, which reads the stored extent of every row, must refuse a caller the policy restricts. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_row_level_security_holds_on_every_read_path` | the read paths against heap (the custom scan, both vectorized aggregates, parallel, index-only, the covering projection), their plan premises, and the leak arms, including one through each vectorized node | + +## 131. test_type_matrix.py: types the differential matrix never stored answer as heap + +The pytest twin of `test/type_matrix.sh` (#1341), every check by the shell's name. The encoder has fast paths for fixed widths 1, 2, 4 and 8, and every other width and every varlena takes a generic path. This matrix stores what no test had: by-reference fixed widths 6, 8, 12, 16 and 64 (macaddr, macaddr8, timetz, point, name), the 1-byte `"char"`, time, money, inet, cidr, bit, varbit, tsvector, xml, json, an enum, a composite, a domain, and numeric NaN and ±Infinity with timestamp and date ±infinity. 12000 rows in row groups of 1000, each column NULL on its own pattern, are generated once into heap and copied. Every row as text, each column's counts, a range on every ordered column with literal bounds read from heap, json, point and tsvector matches, the special-value aggregates, and the composite's fields must all equal heap's. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_types_the_matrix_never_stored_answer_as_heap` | the 21 columns against heap: rows, counts, sixteen ranges, the three non-btree matches, and the special values | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 59af69adc..968ada983 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -722,4 +722,7 @@ guard_tests 438 # cluster_tests 598 -> 599 for #1341's test_min_and_max_keep_each_columns_collation in # test_rollup.py, by collection over the cluster files on this tree. # cluster_tests 599 -> 600 for #1341's test_savepoint_flush.py, one test, by collection -cluster_tests 600 +# cluster_tests 600 -> 601 for #1341's test_rls_query_paths.py, one test, by collection +# over the cluster files on this tree. +# cluster_tests 601 -> 602 for #1341's test_type_matrix.py, one test, by collection +cluster_tests 602 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index ffa87bfdf..43943a462 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -149,6 +149,7 @@ "recovery_minimal_wal", "rewrite_storage_oid", "rls_direct_storage", + "rls_query_paths", "rollup", "savepoint_flush", "scan_decode_cost", @@ -163,6 +164,7 @@ "tier_partition", "timeseries_recipes", "ttl_expire", + "type_matrix", "vacuum_lock_privilege", "validity_elision", "vm_privilege", diff --git a/test/pytest/test_rls_query_paths.py b/test/pytest/test_rls_query_paths.py new file mode 100644 index 000000000..b817fe60a --- /dev/null +++ b/test/pytest/test_rls_query_paths.py @@ -0,0 +1,148 @@ +"""Row-level security on every columnar read path (#1341). The pytest twin of +`test/rls_query_paths.sh`, every check by the shell's name. + +A columnar table and a heap mirror hold the same 20000 rows, half owned by each of two +roles, under a policy that shows a role only its own rows. One role, on its own login, +queries both by the custom scan, the vectorized and grouped aggregates, a parallel +scan, an index-only scan and a covering projection, and each answer must be heap's; the +plan premises check each path was taken. A cheap function that is not leakproof must +see only that role's rows: its NOTICEs are counted by a notice handler. +""" +import psycopg + +ALICE, BOB = "rqp_alice", "rqp_bob" +AGG = ("SELECT count(*) || '/' || sum(v) || '/' || min(v) || '/' || max(v) || '/' || " + "count(*) FILTER (WHERE owner <> current_user) FROM %t") +GRP = ("SELECT string_agg(grp || ':' || n || ':' || s, ',' ORDER BY grp) " + "FROM (SELECT grp, count(*) n, sum(v) s FROM %t GROUP BY grp) x") +OFF = ("SET pgcolumnar.enable_vectorization = off", "SET pgcolumnar.enable_group_vectorization = off", + "SET pgcolumnar.enable_custom_scan = off") +PAR = ("SET parallel_setup_cost = 0", "SET parallel_tuple_cost = 0", "SET min_parallel_table_scan_size = 0", + "SET max_parallel_workers_per_gather = 2") +IOS = ("SET enable_seqscan = off", "SET enable_bitmapscan = off", "SET pgcolumnar.enable_custom_scan = off") +IQ = "SELECT count(*) || '/' || sum(id) FROM %t WHERE id BETWEEN 100 AND 15000" +PQ = "SELECT count(*) || '/' || sum(v) FROM %t WHERE id BETWEEN 100 AND 15000" + + +def test_row_level_security_holds_on_every_read_path(pgc_cluster, pgc_conn, expect): + with pgc_conn.cursor() as cur: + cur.execute("SELECT current_schema()") + schema = cur.fetchone()[0] + for r in (ALICE, BOB): + cur.execute(f"SELECT 1 FROM pg_roles WHERE rolname = '{r}'") + if cur.fetchone() is None: + cur.execute(f"CREATE ROLE {r} LOGIN NOSUPERUSER") + cur.execute(f'GRANT USAGE ON SCHEMA "{schema}", pgcolumnar TO {r}') + cur.execute("CREATE FUNCTION rq_leak(text) RETURNS bool LANGUAGE plpgsql COST 0.0001 PARALLEL SAFE " + "AS $$ BEGIN RAISE NOTICE 'seen:%', $1; RETURN true; END $$") + for t, am in (("rq", "pgcolumnar"), ("rh", "heap")): + cur.execute(f"CREATE TABLE {t} (id int, owner name, grp int, v int, secret text) USING {am}") + cur.execute(f"INSERT INTO {t} SELECT g, CASE WHEN g % 2 = 0 THEN '{ALICE}' ELSE '{BOB}' END, g % 10, g, " + f"'S-' || CASE WHEN g % 2 = 0 THEN '{ALICE}' ELSE '{BOB}' END || '-' || g " + f"FROM generate_series(1, 20000) g") + cur.execute(f"CREATE INDEX {t}_io ON {t} (id, owner)") + cur.execute(f"ALTER TABLE {t} ENABLE ROW LEVEL SECURITY") + cur.execute(f"CREATE POLICY {t}_own ON {t} USING (owner = current_user)") + cur.execute(f"GRANT SELECT ON {t} TO {ALICE}, {BOB}") + cur.execute(f"VACUUM ANALYZE {t}") + cur.execute("SELECT pgcolumnar.add_projection('rq', 'rq_p', ARRAY['id', 'owner', 'v'], ARRAY['id'])") + cur.execute("SELECT count(*) FROM rq") + total = cur.fetchone()[0] + + alice = psycopg.connect(pgc_cluster.dsn().replace("user=postgres", f"user={ALICE}"), autocommit=True) + notices = [] + alice.add_notice_handler(lambda d: notices.append(d.message_primary or "")) + try: + alice.execute(f'SET search_path TO "{schema}", public, pgcolumnar') + + def run(setup, sql): + try: + for s in setup: + alice.execute(s) + return alice.execute(sql).fetchone()[0] + except psycopg.Error as exc: + return f"ERROR: {exc.sqlstate}" + finally: + alice.execute("RESET ALL") + alice.execute(f'SET search_path TO "{schema}", public, pgcolumnar') + + def same(setup, sql): + c, h = run(setup, sql.replace("%t", "rq")), run(setup, sql.replace("%t", "rh")) + ok = c == h and c is not None and not str(c).startswith("ERROR") + return "same" if ok else f"columnar=[{c}] heap=[{h}]" + + def plan_has(setup, sql, needle): + for s in setup: + alice.execute(s) + plan = "\n".join(r[0] for r in alice.execute("EXPLAIN (COSTS OFF) " + sql.replace("%t", "rq")).fetchall()) + alice.execute("RESET ALL") + alice.execute(f'SET search_path TO "{schema}", public, pgcolumnar') + return sum(1 for line in plan.splitlines() if needle in line) + + def leaks(table, setup, sql="SELECT count(*) FROM %t WHERE rq_leak(secret)"): + notices.clear() + run(setup, sql.replace("%t", table)) + seen = [n for n in notices if n.startswith("seen:")] + return f"{len(seen)}|{sum(1 for n in seen if n.startswith(f'seen:S-{BOB}'))}" + + expect.text(f"{total}|{run((), 'SELECT count(*) FROM rq')}|{run((), 'SELECT count(*) FROM rh')}", + "20000|10000|10000", + "a superuser sees all 20000 rows, and rq_alice her 10000 on both tables") + expect.num(plan_has((), AGG, "Custom Scan (PgColumnarScan)"), 1, + "premise: the aggregate is answered by the columnar custom scan") + expect.text(same((), AGG), "same", "count, sum, min and max under the policy, by the custom scan, are heap's") + expect.num(plan_has((), GRP, "Columnar Vectorized Group Keys"), 1, + "premise: the GROUP BY arm's plan is the grouped vectorized node") + expect.text(same((), GRP), "same", "a GROUP BY under the policy, with vectorization on, is heap's") + # The ungrouped vectorized aggregate builds its own quals too, and any user can + # turn it on. With no qual left, count(*) alone comes from metadata, so a sum + # makes it scan. + uva = ("SET pgcolumnar.enable_ungrouped_vector_agg = on",) + uagg = "SELECT count(*) || '/' || sum(v) FROM %t" + expect.num(plan_has(uva, uagg, "Columnar Vectorized Aggregates"), 1, + "premise: the ungrouped vectorized arm's plan folds the aggregates in the scan") + expect.text(same(uva, uagg), "same", + "count and sum under the policy, by the ungrouped vectorized aggregate, are heap's") + expect.text(f"{same(OFF, GRP)}|{same(OFF, AGG)}", "same|same", "and with vectorization and the custom scan off") + expect.num(plan_has(PAR, AGG, "Gather"), 1, "premise: the parallel arm's plan gathers from workers") + expect.text(f"{same(PAR, AGG)}|{same(PAR, GRP)}", "same|same", + "count, sum, min and max under the policy, by a parallel scan, are heap's") + expect.num(plan_has(IOS, IQ, "Index Only Scan"), 1, "premise: the index-only arm's plan is an index-only scan") + expect.text(same(IOS, IQ), "same", "an index-only scan under the policy is heap's") + expect.num(plan_has((), PQ, "Columnar Projection: rq_p"), 1, + "premise: the projection arm's plan reads the covering projection") + expect.text(same((), PQ), "same", "a covering projection scan under the policy is heap's") + expect.text(f"{leaks('rq', ())}|{leaks('rh', ())}", "10000|0|10000|0", + "a cheap function that is not leakproof sees only rq_alice's rows on columnar, as on heap") + expect.text(leaks("rq", OFF), "10000|0", "and the same with vectorization and the custom scan off") + # PARALLEL SAFE, or the planner keeps the function out of the workers and + # this arm repeats the first one. + expect.num(plan_has(PAR, "SELECT count(*) FROM %t WHERE rq_leak(secret)", "Gather"), 1, + "premise: the parallel leak arm's plan gathers from workers") + expect.text(leaks("rq", PAR), "10000|0", "and through a parallel scan") + # Both vectorized nodes build their own quals, so each must also keep a + # non-leakproof function from rows the policy hides. + expect.num(plan_has((), "SELECT grp, count(*) FROM %t WHERE secret IS NOT NULL GROUP BY grp", + "Columnar Vectorized Group Keys"), 1, + "premise: a GROUP BY with a leakproof filter under the policy is the grouped vectorized node") + lgrp = "SELECT grp, count(*) FROM %t WHERE rq_leak(secret) GROUP BY grp" + expect.text(f"{leaks('rq', (), lgrp)}|{leaks('rh', (), lgrp)}", "10000|0|10000|0", + "a non-leakproof function in a GROUP BY under the policy sees only rq_alice's rows, as on heap") + expect.num(plan_has(uva, "SELECT count(*), sum(v) FROM %t WHERE secret IS NOT NULL", + "Columnar Vectorized Aggregates"), 1, + "premise: an ungrouped count and sum with a leakproof filter, with the setting on, " + "is the ungrouped vectorized aggregate") + expect.text(leaks("rq", uva, "SELECT count(*), sum(v) FROM %t WHERE rq_leak(secret)"), "10000|0", + "and in the ungrouped vectorized aggregate") + # column_extent reads the stored extent, which covers every row, so under a + # policy it would show values from rows the caller cannot see. It refuses, as + # the other direct-storage entry points do (#563); the owner still reads it. + ce = run((), "SELECT count(*) FROM pgcolumnar.column_extent('rq', 'v')") + expect.text("refused" if ce == "ERROR: 0A000" else f"not refused ({ce})", "refused", + "column_extent refuses a caller a policy restricts") + with pgc_conn.cursor() as cur: + cur.execute("SELECT count(*) FROM pgcolumnar.column_extent('rq', 'v')") + owner = cur.fetchone()[0] + expect.num(owner, 1, "and the owner still reads the extent") + finally: + alice.close() diff --git a/test/pytest/test_type_matrix.py b/test/pytest/test_type_matrix.py new file mode 100644 index 000000000..999c7bd23 --- /dev/null +++ b/test/pytest/test_type_matrix.py @@ -0,0 +1,144 @@ +"""Types the differential matrix never stored, against heap (#1341). The pytest twin +of `test/type_matrix.sh`, every check by the shell's name. + +The encoder has fast paths for fixed widths 1, 2, 4 and 8; every other width and every +varlena takes a generic path, and a zone map is built for each column whose type has a +btree ordering. These columns cover widths 6, 12, 16 and 64 by reference, the 1-byte +"char", time, money, inet, cidr, bit, varbit, tsvector, xml, json, an enum, a composite, +a domain, and numeric, timestamp and date special values. 12000 rows in row groups of +1000, each column NULL on its own pattern, generated once into heap and copied. + +Each comparison is of the rows themselves (`expect.row_set`), so a failure names the rows +that differ. The range bounds are the values heap holds at ids 3005 and 9005, written as +literals so the columnar scan can prune row groups by zone map (not 3001, which holds n's +NaN). Each range arm also says whether the zone maps skipped chunk groups, from EXPLAIN +ANALYZE; four that measure 0 are pinned at 0 with the reason. timetz's zone offset varies +with the row, so its 12-byte width is used past byte 8. +""" +import re + +import psycopg + +COLS = """id int, tz timetz, mac macaddr, mac8 macaddr8, nm name, pt point, ch "char", + tm time, mo money, ip inet, cd cidr, bt bit(16), vb varbit, tsv tsvector, x xml, + j json, e tm_mood, cmp tm_pair, dom tm_pos, n numeric, tsi timestamp, d date""" + +LOAD = """SELECT g, + CASE WHEN g % 7 = 0 THEN NULL ELSE ((('00:00'::time + (g * 7) * interval '1 s')::text || '+' || lpad((g % 13)::text, 2, '0'))::timetz) END, + CASE WHEN g % 11 = 0 THEN NULL ELSE ('08:00:2b:' || lpad(to_hex(g / 256 % 256), 2, '0') || ':' || lpad(to_hex(g % 256), 2, '0') || ':01')::macaddr END, + CASE WHEN g % 13 = 0 THEN NULL ELSE ('08:00:2b:01:02:03:' || lpad(to_hex(g / 256 % 256), 2, '0') || ':' || lpad(to_hex(g % 256), 2, '0'))::macaddr8 END, + CASE WHEN g % 17 = 0 THEN NULL ELSE ('name_' || lpad(g::text, 6, '0'))::name END, + CASE WHEN g % 19 = 0 THEN NULL ELSE point(g, -g) END, + CASE WHEN g % 23 = 0 THEN NULL ELSE chr(65 + g % 26)::"char" END, + CASE WHEN g % 29 = 0 THEN NULL ELSE ('00:00'::time + g * interval '7 s') END, + CASE WHEN g % 31 = 0 THEN NULL ELSE (g * 1.25)::numeric::money END, + CASE WHEN g % 37 = 0 THEN NULL ELSE ('10.' || g / 256 % 256 || '.' || g % 256 || '.' || g % 7 || '/32')::inet END, + CASE WHEN g % 41 = 0 THEN NULL ELSE ('10.' || g / 256 % 256 || '.' || g % 256 || '.0/24')::cidr END, + CASE WHEN g % 43 = 0 THEN NULL ELSE g::bit(16) END, + CASE WHEN g % 47 = 0 THEN NULL ELSE (g::bit(16) || (g % 7)::bit(3))::varbit END, + CASE WHEN g % 53 = 0 THEN NULL ELSE to_tsvector('simple', 'word' || g || ' common term' || g % 9) END, + CASE WHEN g % 59 = 0 THEN NULL ELSE ('' || md5(g::text) || '')::xml END, + CASE WHEN g % 61 = 0 THEN NULL ELSE json_build_object('g', g, 's', md5(g::text)) END, + CASE WHEN g % 67 = 0 THEN NULL ELSE (enum_range(NULL::tm_mood))[1 + g % 4] END, + CASE WHEN g % 71 = 0 THEN NULL ELSE ROW(g, 'p' || g)::tm_pair END, + CASE WHEN g % 73 = 0 THEN NULL ELSE (g)::tm_pos END, + CASE WHEN g % 79 = 0 THEN NULL WHEN g % 1000 = 1 THEN 'NaN'::numeric WHEN g % 1000 = 2 THEN 'Infinity'::numeric + WHEN g % 1000 = 3 THEN '-Infinity'::numeric ELSE g / 7.0 END, + CASE WHEN g % 83 = 0 THEN NULL WHEN g = 500 THEN 'infinity'::timestamp WHEN g = 501 THEN '-infinity'::timestamp + ELSE '2026-01-01'::timestamp + g * interval '1 min' END, + CASE WHEN g % 89 = 0 THEN NULL WHEN g = 600 THEN 'infinity'::date WHEN g = 601 THEN '-infinity'::date + ELSE '2026-01-01'::date + g END + FROM generate_series(1, 12000) g""" + +ALL = ("id, tz::text, mac::text, mac8::text, nm::text, pt::text, ch::text, tm::text, mo::text, ip::text, " + "cd::text, bt::text, vb::text, tsv::text, x::text, j::text, e::text, cmp::text, dom::text, n::text, " + "tsi::text, d::text") + + +def _rows(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetchall() + + +def _both(conn, sql): + return _rows(conn, sql.replace("%T", "t_col")), _rows(conn, sql.replace("%T", "t_heap")) + + +NO_PRUNE = { + "ch": "every row group holds every value, since the column cycles through 26 letters", + "e": "every row group holds every value, since the column cycles through the 4 labels", + "n": "every row group holds NaN and both infinities, so each zone map spans every value", + "cd": "a cidr key is treated as cross-type against inet and dropped (#1433)", +} + + +def _removed(conn, sql): + plan = "\n".join(r[0] for r in _rows(conn, "EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) " + + sql.replace("%T", "t_col"))) + m = re.search(r"Columnar Chunk Groups Removed by Filter: (\d+)", plan) + return m.group(1) if m else "none" + + +def test_types_the_matrix_never_stored_answer_as_heap(pgc_conn, expect): + # xml needs a server built with libxml; without it every arm after the load would + # fail for that reason, so this says so first. + try: + xml_ok = "yes" if _rows(pgc_conn, "SELECT ''::xml IS NOT NULL")[0][0] else "no" + except psycopg.Error as exc: + xml_ok = f"no ({str(exc).splitlines()[0]})" + expect.text(xml_ok, "yes", "premise: the server supports xml (built with libxml)") + with pgc_conn.cursor() as cur: + cur.execute("CREATE TYPE tm_mood AS ENUM ('sad', 'ok', 'happy', 'elated')") + cur.execute("CREATE TYPE tm_pair AS (a int, b text)") + cur.execute("CREATE DOMAIN tm_pos AS int CHECK (VALUE > 0)") + cur.execute(f"CREATE TABLE t_heap ({COLS})") + cur.execute(f"CREATE TABLE t_col ({COLS}) USING pgcolumnar") + cur.execute("SELECT pgcolumnar.set_options('t_col', stripe_row_limit => 1000)") + cur.execute(f"INSERT INTO t_heap {LOAD}") + cur.execute("INSERT INTO t_col SELECT * FROM t_heap") + groups = _rows(pgc_conn, "SELECT count(*) FROM pgcolumnar.row_group " + "WHERE storage_id = pgcolumnar.get_storage_id('t_col')")[0][0] + n_heap = _rows(pgc_conn, "SELECT count(*) FROM t_heap")[0][0] + n_col = _rows(pgc_conn, "SELECT count(*) FROM t_col")[0][0] + expect.text(f"{n_heap}|{n_col}|{'many' if groups >= 12 else groups}", "12000|12000|many", + "premise: both tables hold 12000 rows, the columnar one in at least twelve row groups") + + got, want = _both(pgc_conn, f"SELECT {ALL} FROM %T") + expect.row_set(got, want, "every row, as text, is heap's") + counts = ", ".join(f"count({c}), count(*) FILTER (WHERE {c} IS NULL)" + for c in "tz mac mac8 nm pt ch tm mo ip cd bt vb tsv x j e cmp dom n tsi d".split()) + got, want = _both(pgc_conn, f"SELECT {counts} FROM %T") + expect.row_set(got, want, "each column's count and NULL count is heap's") + + for c in ("tz", "mac", "mac8", "nm", "ch", "tm", "mo", "ip", "cd", "bt", "vb", "e", "dom", "n", "tsi", "d"): + # format_type keeps the typmod: pg_typeof would make bt's bounds bit(1). + ty = _rows(pgc_conn, f"SELECT format_type(atttypid, atttypmod) FROM pg_attribute " + f"WHERE attrelid = 't_heap'::regclass AND attname = '{c}'")[0][0] + lo, hi = (_rows(pgc_conn, f"SELECT quote_literal({c}::text) || '::' || '{ty}' " + f"FROM t_heap WHERE id = {i}")[0][0] for i in (3005, 9005)) + sql = f"SELECT id FROM %T WHERE {c} BETWEEN least({lo}, {hi}) AND greatest({lo}, {hi})" + # Heap's answer must not be empty: wrong bounds would make both answers empty. + n = _rows(pgc_conn, f"SELECT count(*) FROM ({sql.replace('%T', 't_heap')}) x")[0][0] + expect.text("yes" if n > 0 else f"no ({n} rows)", "yes", + f"a range on {c} holds rows on heap, so its comparison reads something") + got, want = _both(pgc_conn, sql) + expect.row_set(got, want, f"a range on {c} finds heap's rows") + r = _removed(pgc_conn, sql) + if c in NO_PRUNE: + expect.text(r, "0", f"a range on {c} skips no chunk groups: {NO_PRUNE[c]}") + else: + expect.text("skips" if r.isdigit() and int(r) >= 1 else f"removed {r}", "skips", + f"a range on {c} lets the zone maps skip chunk groups") + got, want = _both(pgc_conn, "SELECT id FROM %T WHERE j::text = (SELECT j::text FROM t_heap WHERE id = 4242)") + expect.row_set(got, want, "equality on json, as text, finds heap's rows") + got, want = _both(pgc_conn, "SELECT id FROM %T WHERE pt ~= point(4242, -4242)") + expect.row_set(got, want, "point equality (~=) finds heap's rows") + got, want = _both(pgc_conn, "SELECT id FROM %T WHERE tsv @@ to_tsquery('simple', 'term3')") + expect.row_set(got, want, "a tsvector match finds heap's rows") + got, want = _both(pgc_conn, "SELECT min(n)::text, max(n)::text, count(*) FILTER (WHERE n = 'NaN'), " + "count(*) FILTER (WHERE n = 'Infinity'), count(*) FILTER (WHERE n = '-Infinity'), " + "min(tsi)::text, max(tsi)::text, min(d)::text, max(d)::text FROM %T") + expect.row_set(got, want, "the aggregates over numeric NaN and Infinity and timestamp and date infinity are heap's") + got, want = _both(pgc_conn, "SELECT (cmp).a, (cmp).b, e::text, e > 'ok' FROM %T WHERE id % 97 = 0") + expect.row_set(got, want, "the composite's fields and the enum's order read back as heap's") diff --git a/test/rls_query_paths.sh b/test/rls_query_paths.sh new file mode 100755 index 000000000..e52c62a62 --- /dev/null +++ b/test/rls_query_paths.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +# +# Row-level security on every columnar read path (#1341). The columnar custom scan +# builds its quals from the restriction list, the vectorized and grouped aggregates +# and the covering projection scan answer from their own paths, and an index-only +# scan and a parallel scan read the table differently again. A policy must hold on +# each, and a function that is not leakproof must never see a row the policy hides. +# +# A table and a heap mirror hold the same 20000 rows, half owned by each of two +# roles, with a policy that shows a role only its own rows. Each arm logs in as one of +# the roles (a real login, not SET ROLE) and must answer as heap does, on the path the +# arm names: the plan premises check that path was taken. +# +# Usage: test/rls_query_paths.sh [PG_CONFIG] +# Written fresh for pgColumnar. + +set -uo pipefail +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" + +psql_run "DROP ROLE IF EXISTS rq_alice; DROP ROLE IF EXISTS rq_bob; + CREATE ROLE rq_alice LOGIN NOSUPERUSER; CREATE ROLE rq_bob LOGIN NOSUPERUSER; + GRANT USAGE ON SCHEMA pgcolumnar TO rq_alice; + CREATE FUNCTION rq_leak(text) RETURNS bool LANGUAGE plpgsql COST 0.0001 PARALLEL SAFE + AS \$\$ BEGIN RAISE NOTICE 'seen:%', \$1; RETURN true; END \$\$;" +for t in rq rh; do + am=pgcolumnar; [ $t = rh ] && am=heap + psql_run "CREATE TABLE $t (id int, owner name, grp int, v int, secret text) USING $am; + INSERT INTO $t SELECT g, CASE WHEN g % 2 = 0 THEN 'rq_alice' ELSE 'rq_bob' END, g % 10, g, + 'S-' || CASE WHEN g % 2 = 0 THEN 'rq_alice' ELSE 'rq_bob' END || '-' || g + FROM generate_series(1, 20000) g; + CREATE INDEX ${t}_io ON $t (id, owner); + ALTER TABLE $t ENABLE ROW LEVEL SECURITY; + CREATE POLICY ${t}_own ON $t USING (owner = current_user); + GRANT SELECT ON $t TO rq_alice, rq_bob;" + psql_run "VACUUM ANALYZE $t;" # not in a transaction block +done +psql_run "SELECT pgcolumnar.add_projection('rq', 'rq_p', ARRAY['id', 'owner', 'v'], ARRAY['id']);" >/dev/null + +as_alice() { # as_alice SQL -> its output, as rq_alice on her own login + env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U rq_alice -d "$PGC_DB" -XAtq \ + -c "SET search_path TO public, pgcolumnar" -c "$1" 2>&1 +} +same() { # same SETUP QUERY -> "same", or both answers; %t is the table + local c h + c="$(as_alice "$1 ${2//%t/rq}" | tail -1)"; h="$(as_alice "$1 ${2//%t/rh}" | tail -1)" + # an error on both sides is not agreement + case "$c" in *ERROR*) echo "columnar=[$c] heap=[$h]"; return ;; esac + [ "$c" = "$h" ] && [ -n "$c" ] && echo same || echo "columnar=[$c] heap=[$h]" +} +plan_has() { # plan_has SETUP QUERY PATTERN -> how many plan lines match, on the columnar table + as_alice "$1 EXPLAIN (COSTS OFF) ${2//%t/rq}" | grep -cE "$3" +} + +AGG="SELECT count(*) || '/' || sum(v) || '/' || min(v) || '/' || max(v) || '/' || count(*) FILTER (WHERE owner <> current_user) FROM %t" +GRP="SELECT string_agg(grp || ':' || n || ':' || s, ',' ORDER BY grp) FROM (SELECT grp, count(*) n, sum(v) s FROM %t GROUP BY grp) x" + +check "a superuser sees all 20000 rows, and rq_alice her 10000 on both tables" \ + "$(q "SELECT count(*) FROM rq")|$(as_alice "SELECT count(*) FROM rq")|$(as_alice "SELECT count(*) FROM rh")" "20000|10000|10000" + +# ---- the custom scan and the vectorized aggregates ----------------------------------- +check "premise: the aggregate is answered by the columnar custom scan" "$(plan_has "" "$AGG" 'Custom Scan \(PgColumnarScan\)')" "1" +check "count, sum, min and max under the policy, by the custom scan, are heap's" "$(same "" "$AGG")" "same" +check "premise: the GROUP BY arm's plan is the grouped vectorized node" \ + "$(plan_has "" "$GRP" 'Columnar Vectorized Group Keys')" "1" +check "a GROUP BY under the policy, with vectorization on, is heap's" "$(same "" "$GRP")" "same" +# The ungrouped vectorized aggregate builds its own quals from baserestrictinfo too, +# and any user can turn it on (PGC_USERSET). With no qual left, count(*) alone is +# answered from metadata, so a sum makes it scan (OffgridwithJD, review of #1429). +UVA="SET pgcolumnar.enable_ungrouped_vector_agg = on;" +UAGG="SELECT count(*) || '/' || sum(v) FROM %t" +check "premise: the ungrouped vectorized arm's plan folds the aggregates in the scan" \ + "$(plan_has "$UVA" "$UAGG" 'Columnar Vectorized Aggregates')" "1" +check "count and sum under the policy, by the ungrouped vectorized aggregate, are heap's" "$(same "$UVA" "$UAGG")" "same" +OFF="SET pgcolumnar.enable_vectorization = off; SET pgcolumnar.enable_group_vectorization = off; SET pgcolumnar.enable_custom_scan = off;" +check "and with vectorization and the custom scan off" "$(same "$OFF" "$GRP")|$(same "$OFF" "$AGG")" "same|same" + +# ---- a parallel scan ----------------------------------------------------------------- +PAR="SET parallel_setup_cost = 0; SET parallel_tuple_cost = 0; SET min_parallel_table_scan_size = 0; SET max_parallel_workers_per_gather = 2;" +check "premise: the parallel arm's plan gathers from workers" "$(plan_has "$PAR" "$AGG" 'Gather')" "1" +check "count, sum, min and max under the policy, by a parallel scan, are heap's" "$(same "$PAR" "$AGG")|$(same "$PAR" "$GRP")" "same|same" + +# ---- an index-only scan -------------------------------------------------------------- +IOS="SET enable_seqscan = off; SET enable_bitmapscan = off; SET pgcolumnar.enable_custom_scan = off;" +IQ="SELECT count(*) || '/' || sum(id) FROM %t WHERE id BETWEEN 100 AND 15000" +check "premise: the index-only arm's plan is an index-only scan" "$(plan_has "$IOS" "$IQ" 'Index Only Scan')" "1" +check "an index-only scan under the policy is heap's" "$(same "$IOS" "$IQ")" "same" + +# ---- a covering projection ----------------------------------------------------------- +PQ="SELECT count(*) || '/' || sum(v) FROM %t WHERE id BETWEEN 100 AND 15000" +check "premise: the projection arm's plan reads the covering projection" "$(plan_has "" "$PQ" 'Columnar Projection: rq_p')" "1" +check "a covering projection scan under the policy is heap's" "$(same "" "$PQ")" "same" + +# ---- a function that is not leakproof sees only visible rows -------------------------- +leaks() { # leaks TABLE SETUP [QUERY] -> "|" + local out sql="${3:-SELECT count(*) FROM %t WHERE rq_leak(secret)}" + out="$(as_alice "$2 ${sql//%t/$1}")" + echo "$(grep -c 'seen:' <<<"$out")|$(grep -c 'seen:S-rq_bob' <<<"$out")" +} +check "a cheap function that is not leakproof sees only rq_alice's rows on columnar, as on heap" \ + "$(leaks rq "")|$(leaks rh "")" "10000|0|10000|0" +check "and the same with vectorization and the custom scan off" "$(leaks rq "$OFF")" "10000|0" +# PARALLEL SAFE, or the planner keeps the function out of the workers and this arm +# repeats the first one (OffgridwithJD, review of #1429). +check "premise: the parallel leak arm's plan gathers from workers" \ + "$(plan_has "$PAR" "SELECT count(*) FROM %t WHERE rq_leak(secret)" 'Gather')" "1" +check "and through a parallel scan" "$(leaks rq "$PAR")" "10000|0" +# Both vectorized nodes build their own quals (above), so each must also keep a +# function that is not leakproof from rows the policy hides. Each premise shows the +# arm's query shape, with a leakproof filter in place of the function, is answered by +# that node (OffgridwithJD, review of #1429). +check "premise: a GROUP BY with a leakproof filter under the policy is the grouped vectorized node" \ + "$(plan_has "" "SELECT grp, count(*) FROM %t WHERE secret IS NOT NULL GROUP BY grp" 'Columnar Vectorized Group Keys')" "1" +check "a non-leakproof function in a GROUP BY under the policy sees only rq_alice's rows, as on heap" \ + "$(leaks rq "" "SELECT grp, count(*) FROM %t WHERE rq_leak(secret) GROUP BY grp")|$(leaks rh "" "SELECT grp, count(*) FROM %t WHERE rq_leak(secret) GROUP BY grp")" "10000|0|10000|0" +check "premise: an ungrouped count and sum with a leakproof filter, with the setting on, is the ungrouped vectorized aggregate" \ + "$(plan_has "$UVA" "SELECT count(*), sum(v) FROM %t WHERE secret IS NOT NULL" 'Columnar Vectorized Aggregates')" "1" +check "and in the ungrouped vectorized aggregate" "$(leaks rq "$UVA" "SELECT count(*), sum(v) FROM %t WHERE rq_leak(secret)")" "10000|0" +# pgcolumnar.column_extent reads the stored extent, which covers every row, so under a +# policy it would show values from rows the caller cannot see. It refuses, as the +# other direct-storage entry points do (#563), and the owner still reads it +# (OffgridwithJD, review of #1429). +ce="$(as_alice "SELECT count(*) FROM pgcolumnar.column_extent('rq', 'v')")" +check "column_extent refuses a caller a policy restricts" \ + "$(grep -q 'row-level security is in force' <<<"$ce" && echo refused || echo "not refused (${ce//$'\n'/ })")" "refused" +check "and the owner still reads the extent" "$(q "SELECT count(*) FROM pgcolumnar.column_extent('rq', 'v')")" "1" + +pgc_summary diff --git a/test/run_all_versions.sh b/test/run_all_versions.sh index f3961c4bf..4466359f9 100755 --- a/test/run_all_versions.sh +++ b/test/run_all_versions.sh @@ -294,6 +294,7 @@ SUITES=( rewrite_group_scan rewrite_storage_oid rls_direct_storage + rls_query_paths rollup row_triggers savepoint_flush @@ -318,6 +319,7 @@ SUITES=( timeseries_recipes truncate_cleanup ttl_expire + type_matrix ungrouped_vector_agg unique_conc update_conc diff --git a/test/type_matrix.sh b/test/type_matrix.sh new file mode 100755 index 000000000..392f2dc4e --- /dev/null +++ b/test/type_matrix.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +# +# Types the differential matrix never stored, against heap (#1341). The encoder has +# fast paths for fixed widths 1, 2, 4 and 8; every other width and every varlena +# takes a generic path, and a zone map is built for each column whose type has a +# btree ordering. None of these types had been written to a columnar table: +# +# - fixed width, by reference, at unusual widths: timetz (12), macaddr (6), +# macaddr8 (8), name (64), point (16, no btree ordering); +# - the 1-byte "char"; +# - time, money, inet, cidr, bit, varbit, tsvector, xml, json (no equality); +# - an enum, a composite, a domain over int; +# - numeric NaN and +-Infinity, timestamp and date +-infinity. +# +# 12000 rows in row groups of 1000, each column NULL on its own pattern. The same +# rows go into heap and are copied to columnar. Then: every row, as text; each +# column's count and NULL count; a range on every ordered column, with constants +# read from heap so the zone maps can prune; equality on json (as text) and point +# (~=); and the aggregates over the special values. +# +# Usage: test/type_matrix.sh [PG_CONFIG] +# Written fresh for pgColumnar. + +set -uo pipefail +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" + +psql_run "CREATE TYPE tm_mood AS ENUM ('sad', 'ok', 'happy', 'elated'); + CREATE TYPE tm_pair AS (a int, b text); + CREATE DOMAIN tm_pos AS int CHECK (VALUE > 0);" +COLS="id int, tz timetz, mac macaddr, mac8 macaddr8, nm name, pt point, ch \"char\", + tm time, mo money, ip inet, cd cidr, bt bit(16), vb varbit, tsv tsvector, x xml, + j json, e tm_mood, cmp tm_pair, dom tm_pos, n numeric, tsi timestamp, d date" +# xml needs a server built with libxml. Without it the load below fails and every +# arm after it would fail for that reason, so the suite says so and stops +# (OffgridwithJD, review of #1426). +xml_ok="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -XAtq \ + -c "SELECT ''::xml IS NOT NULL" 2>&1 | tail -1)" # 2>&1, so a missing libxml is named +check "premise: the server supports xml (built with libxml)" "$([ "$xml_ok" = t ] && echo yes || echo "no ($xml_ok)")" "yes" +[ "$xml_ok" = t ] || pgc_summary +make_pair "$COLS" +q "SELECT pgcolumnar.set_options('t_col', stripe_row_limit => 1000)" >/dev/null +# g % k = 0 makes the column NULL; each column has its own k. +load_pair "SELECT g, + CASE WHEN g % 7 = 0 THEN NULL ELSE ((('00:00'::time + (g * 7) * interval '1 s')::text || '+' || lpad((g % 13)::text, 2, '0'))::timetz) END, + CASE WHEN g % 11 = 0 THEN NULL ELSE ('08:00:2b:' || lpad(to_hex(g / 256 % 256), 2, '0') || ':' || lpad(to_hex(g % 256), 2, '0') || ':01')::macaddr END, + CASE WHEN g % 13 = 0 THEN NULL ELSE ('08:00:2b:01:02:03:' || lpad(to_hex(g / 256 % 256), 2, '0') || ':' || lpad(to_hex(g % 256), 2, '0'))::macaddr8 END, + CASE WHEN g % 17 = 0 THEN NULL ELSE ('name_' || lpad(g::text, 6, '0'))::name END, + CASE WHEN g % 19 = 0 THEN NULL ELSE point(g, -g) END, + CASE WHEN g % 23 = 0 THEN NULL ELSE chr(65 + g % 26)::\"char\" END, + CASE WHEN g % 29 = 0 THEN NULL ELSE ('00:00'::time + g * interval '7 s') END, + CASE WHEN g % 31 = 0 THEN NULL ELSE (g * 1.25)::numeric::money END, + CASE WHEN g % 37 = 0 THEN NULL ELSE ('10.' || g / 256 % 256 || '.' || g % 256 || '.' || g % 7 || '/32')::inet END, + CASE WHEN g % 41 = 0 THEN NULL ELSE ('10.' || g / 256 % 256 || '.' || g % 256 || '.0/24')::cidr END, + CASE WHEN g % 43 = 0 THEN NULL ELSE g::bit(16) END, + CASE WHEN g % 47 = 0 THEN NULL ELSE (g::bit(16) || (g % 7)::bit(3))::varbit END, + CASE WHEN g % 53 = 0 THEN NULL ELSE to_tsvector('simple', 'word' || g || ' common term' || g % 9) END, + CASE WHEN g % 59 = 0 THEN NULL ELSE ('' || md5(g::text) || '')::xml END, + CASE WHEN g % 61 = 0 THEN NULL ELSE json_build_object('g', g, 's', md5(g::text)) END, + CASE WHEN g % 67 = 0 THEN NULL ELSE (enum_range(NULL::tm_mood))[1 + g % 4] END, + CASE WHEN g % 71 = 0 THEN NULL ELSE ROW(g, 'p' || g)::tm_pair END, + CASE WHEN g % 73 = 0 THEN NULL ELSE (g)::tm_pos END, + CASE WHEN g % 79 = 0 THEN NULL WHEN g % 1000 = 1 THEN 'NaN'::numeric WHEN g % 1000 = 2 THEN 'Infinity'::numeric + WHEN g % 1000 = 3 THEN '-Infinity'::numeric ELSE g / 7.0 END, + CASE WHEN g % 83 = 0 THEN NULL WHEN g = 500 THEN 'infinity'::timestamp WHEN g = 501 THEN '-infinity'::timestamp + ELSE '2026-01-01'::timestamp + g * interval '1 min' END, + CASE WHEN g % 89 = 0 THEN NULL WHEN g = 600 THEN 'infinity'::date WHEN g = 601 THEN '-infinity'::date + ELSE '2026-01-01'::date + g END + FROM generate_series(1, 12000) g" + +check "premise: both tables hold 12000 rows, the columnar one in at least twelve row groups" \ + "$(q "SELECT count(*) FROM t_heap")|$(q "SELECT count(*) FROM t_col")|$(q "SELECT CASE WHEN count(*) >= 12 THEN 'many' ELSE count(*)::text END + FROM pgcolumnar.row_group WHERE storage_id = pgcolumnar.get_storage_id('t_col')" 2>/dev/null | tail -1)" "12000|12000|many" + +ALL="id, tz::text, mac::text, mac8::text, nm::text, pt::text, ch::text, tm::text, mo::text, ip::text, cd::text, + bt::text, vb::text, tsv::text, x::text, j::text, e::text, cmp::text, dom::text, n::text, tsi::text, d::text" +diff_query "every row, as text, is heap's" "SELECT $ALL FROM %T" +parts=() +for c in tz mac mac8 nm pt ch tm mo ip cd bt vb tsv x j e cmp dom n tsi d; do + parts+=("count($c), count(*) FILTER (WHERE $c IS NULL)") +done +COUNTS="$(IFS=,; echo "${parts[*]}")" +diff_query "each column's count and NULL count is heap's" "SELECT $COUNTS FROM %T" + +# A range on every ordered column. The bounds are the values heap holds at ids 3005 +# and 9005, as literals, so the columnar scan can prune row groups by zone map. Not +# 3001 and 9001: id 3001 holds n's NaN, which made n's "range" NaN..NaN (OffgridwithJD, +# review of #1426). Neither id is NULL in any column. +# +# Each arm also says whether the zone maps skipped chunk groups, from EXPLAIN ANALYZE, +# so an arm that should prune cannot pass on a scan that read everything. Four measure +# 0 of 12 on PG17 and PG18, and are pinned at 0 with the reason: +declare -A NO_PRUNE=( + [ch]="every row group holds every value, since the column cycles through 26 letters" + [e]="every row group holds every value, since the column cycles through the 4 labels" + [n]="every row group holds NaN and both infinities, so each zone map spans every value" + [cd]="a cidr key is treated as cross-type against inet and dropped (#1433)") +removed() { # removed SQL -> the columnar scan's "Chunk Groups Removed by Filter" + q "EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) ${1//%T/t_col}" | + sed -n 's/.*Columnar Chunk Groups Removed by Filter: \([0-9]*\).*/\1/p' | head -1 +} +for c in tz mac mac8 nm ch tm mo ip cd bt vb e dom n tsi d; do + # format_type keeps the typmod: pg_typeof would make bt's bounds bit(1) + # (OffgridwithJD, review of #1426). + ty="$(q "SELECT format_type(atttypid, atttypmod) FROM pg_attribute WHERE attrelid = 't_heap'::regclass AND attname = '$c'" | tail -1)" + lo="$(q "SELECT quote_literal($c::text) || '::' || '$ty' FROM t_heap WHERE id = 3005" | tail -1)" + hi="$(q "SELECT quote_literal($c::text) || '::' || '$ty' FROM t_heap WHERE id = 9005" | tail -1)" + sql="SELECT id FROM %T WHERE $c BETWEEN least($lo, $hi) AND greatest($lo, $hi)" + # Heap's answer must not be empty: bounds that came out wrong (a lost typmod made + # bt's BETWEEN '0' AND '0') would make both answers empty, and the comparison and + # the skip arm would pass on nothing (OffgridwithJD, review of #1426). + n="$(q "SELECT count(*) FROM (${sql//%T/t_heap}) x" | tail -1)" + check "a range on $c holds rows on heap, so its comparison reads something" \ + "$([ -n "$n" ] && [ "$n" -gt 0 ] && echo yes || echo "no (${n:-none} rows)")" "yes" + diff_query "a range on $c finds heap's rows" "$sql" + r="$(removed "$sql")" + if [ -n "${NO_PRUNE[$c]:-}" ]; then + check "a range on $c skips no chunk groups: ${NO_PRUNE[$c]}" "${r:-none}" "0" + else + check "a range on $c lets the zone maps skip chunk groups" \ + "$([ -n "$r" ] && [ "$r" -ge 1 ] && echo skips || echo "removed ${r:-none}")" "skips" + fi +done +diff_query "equality on json, as text, finds heap's rows" \ + "SELECT id FROM %T WHERE j::text = (SELECT j::text FROM t_heap WHERE id = 4242)" +diff_query "point equality (~=) finds heap's rows" "SELECT id FROM %T WHERE pt ~= point(4242, -4242)" +diff_query "a tsvector match finds heap's rows" "SELECT id FROM %T WHERE tsv @@ to_tsquery('simple', 'term3')" +diff_query "the aggregates over numeric NaN and Infinity and timestamp and date infinity are heap's" \ + "SELECT min(n)::text, max(n)::text, count(*) FILTER (WHERE n = 'NaN'), count(*) FILTER (WHERE n = 'Infinity'), + count(*) FILTER (WHERE n = '-Infinity'), min(tsi)::text, max(tsi)::text, min(d)::text, max(d)::text FROM %T" +diff_query "the composite's fields and the enum's order read back as heap's" \ + "SELECT (cmp).a, (cmp).b, e, e > 'ok' FROM %T WHERE id % 97 = 0" + +pgc_summary