diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0512c9031..71b5b7753 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,6 +93,11 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v4 + # The docs_style twin compares each dated CHANGELOG section with its release tag + # (#1341). The default checkout fetches no tags, so without this its four changelog + # arms would decline here on every run. Depth 1, as in the suites job. + - name: fetch the release tags, for the changelog section comparisons + run: git fetch --depth=1 origin '+refs/tags/v1.0-alpha*:refs/tags/v1.0-alpha*' - name: install pytest, pinned from requirements-test.txt, WITHOUT the driver run: | set -euo pipefail diff --git a/test/audit.sh b/test/audit.sh index a75405a26..ed6eadf7c 100755 --- a/test/audit.sh +++ b/test/audit.sh @@ -62,10 +62,10 @@ set -euo pipefail # not turn an abort into a pass; it stops an abort from being mistaken for a # short run that went fine. AUDIT_REACHED_END=0 -trap 'rc=$?; if [ "$AUDIT_REACHED_END" = 0 ]; then - echo; echo "AUDIT TEST ABORTED (exit $rc) before reaching its verdict."; - echo "The checks after the failing command did not run."; -fi' EXIT +# THE NOTICE LIVES IN cleanup() BELOW, NOT IN A trap OF ITS OWN. It was a trap here, +# and `trap cleanup EXIT` further down replaced it, so it never printed: with +# set_options failing at entry the suite stopped after the partitioned arm, 13 checks +# unrecorded, and the log ended on the server's error with no notice at all. # lib.sh for the check vocabulary (#965). It sources portlib.sh itself # (lib.sh:110), so this is a superset of what was here, and its top level is @@ -121,8 +121,14 @@ fi run_pg() { "${RUNPG[@]}" env PATH="$BINDIR:$PATH" bash -lc "$1"; } cleanup() { + local rc=$? + if [ "$AUDIT_REACHED_END" = 0 ]; then + echo; echo "AUDIT TEST ABORTED (exit $rc) before reaching its verdict." + echo "The checks after the failing command did not run." + fi run_pg "pg_ctl -D '$PGDATA' stop -m immediate -w" >/dev/null 2>&1 || true rm -rf "$WORKDIR" + return $rc } trap cleanup EXIT @@ -137,46 +143,36 @@ run_pg "createdb -p $PORT audit" PSQL="psql -p $PORT -d audit -qAtX -v ON_ERROR_STOP=1" q() { run_pg "$PSQL -c \"$1\""; } -fail=0 -# RECORDS RATHER THAN ONLY PRINTING (#965). This suite emitted human PASS lines -# and no RESULT records, so every mechanism built on the record vocabulary -- the -# ledger, the census, checks_never_observed_red, the red-observation record, -# duplicate-name detection -- was blind to all of them. The ledger did not merely -# return nothing on this log, it REFUSED it: "no RESULT records, so there is -# nothing to reconcile". -# -# `pgc_record` takes the DISPLAY whole, so the human output below is byte-for-byte -# what it was. NOT lib.sh's own `check`: that composes its own display and would -# drop the `: $got` suffix, which is the measured value rather than a label. +# LIB.SH'S check, NOT A LOCAL ONE (#1341). This suite used to define its own +# check() and call pgc_record from it, to keep the measured value on every PASS +# line. Nothing else here needs a local recorder, and a suite that calls +# pgc_record directly cannot have a pytest twin (test_compare_to_bash's seed +# arm), so it records through lib.sh like the rest of the corpus. A FAIL line still +# carries both sides; a PASS line carries the name. + +# expect_error NAME SQL MESSAGE -- PASS when SQL is refused WITH MESSAGE. # -# `fail` is still set, so this suite's exit logic is untouched. Its verdict line -# stays for the same reason: under `set -euo pipefail` a failing command aborts the -# suite, and the verdict is what distinguishes finished from stopped. +# ANY error used to pass. Measured on PG17 with set_options failing at entry for a +# reason that is none of its guards (a division by zero): all five refusal arms below +# passed, while the two that already read the reason went red. MESSAGE is the guard's +# own text, so a refusal must come from the guard the arm names. # -# The `checks run:` line is READ BY NOTHING YET. The matrix gates reconciliation on -# the ACCOUNTING line via `pgc_log_shows_accounting`, and this suite emits none -# because it emits no accounting line. The line is still correct and wanted -- -# it is the total the records reconcile against -- so the remaining step is a gate -# flip rather than new work (@jdatcmd, #969 review). -check() { - local name="$1" got="$2" want="$3" - if [ "$got" = "$want" ]; then - pgc_record PASS "$name" "PASS $name: $got" - else - pgc_record FAIL "$name" "FAIL $name: got [$got] want [$want]" - fail=1 - fi -} - -# run SQL that is expected to raise an error; PASS when it does. +# The name stays `expect_error`: selftest 270 recognises an out-of-range set_options +# literal as deliberate only inside this helper. expect_error() { - local name="$1" sql="$2" - if run_pg "$PSQL -c \"$sql\"" >/dev/null 2>&1; then - pgc_record FAIL "$name" "FAIL $name: statement unexpectedly succeeded" - fail=1 + local name="$1" sql="$2" want="$3" out + local got + if out="$(run_pg "$PSQL -c \"$sql\"" 2>&1)"; then + got="statement unexpectedly succeeded" + elif [ -z "$want" ]; then + got="no refusal message to require (harness)" else - pgc_record PASS "$name" "PASS $name (rejected)" + case "$out" in + *"$want"*) got="rejected" ;; + *) got="refused for another reason: $(printf '%s\n' "$out" | head -1)" ;; + esac fi + check "$name" "$got" "rejected" } q "CREATE EXTENSION pgcolumnar;" >/dev/null @@ -262,11 +258,14 @@ q "DROP TABLE col;" >/dev/null # --------------------------------------------------------------------------- q "CREATE TABLE opt (a int) USING pgcolumnar;" >/dev/null expect_error "reject chunk_group_row_limit 0" \ - "SELECT pgcolumnar.set_options('opt'::regclass, chunk_group_row_limit => 0);" + "SELECT pgcolumnar.set_options('opt'::regclass, chunk_group_row_limit => 0);" \ + "chunk_group_row_limit must be at least 100" expect_error "reject stripe_row_limit 5" \ - "SELECT pgcolumnar.set_options('opt'::regclass, stripe_row_limit => 5);" + "SELECT pgcolumnar.set_options('opt'::regclass, stripe_row_limit => 5);" \ + "stripe_row_limit must be at least 1000" expect_error "reject compression_level 99" \ - "SELECT pgcolumnar.set_options('opt'::regclass, compression_level => 99);" + "SELECT pgcolumnar.set_options('opt'::regclass, compression_level => 99);" \ + "compression_level must be between 1 and 22" # A relation that is not columnar must be refused rather than recorded. The drop # hook that clears pgcolumnar.options fires only for columnar relations, so a row @@ -276,9 +275,10 @@ expect_error "reject compression_level 99" \ # as the bare oid, while the identical sequence on a columnar table cleaned up. q "CREATE TABLE opt_heap (a int) USING heap;" >/dev/null expect_error "reject set_options on a heap table" \ - "SELECT pgcolumnar.set_options('opt_heap'::regclass, chunk_group_row_limit => 1000);" -# expect_error passes on ANY error, including a typo in this file's own SQL, so -# assert the reason too. Captured and then matched: piping an erroring psql into + "SELECT pgcolumnar.set_options('opt_heap'::regclass, chunk_group_row_limit => 1000);" \ + "is not a columnar table" +# expect_error now requires the guard's message too; this check reads the full +# reason the same way, and stays as its own record. Captured and then matched: piping an erroring psql into # grep reports on the pipeline rather than on the message. # `|| true` because this file runs under `set -e` and the call is EXPECTED to # fail: without it the assignment takes the non-zero status and aborts the suite @@ -332,13 +332,16 @@ if [ "$audit_srv" -ge 170000 ]; then "$(q "SELECT c.relkind::text || '/' || a.amname FROM pg_class c JOIN pg_am a ON a.oid = c.relam WHERE c.relname = 'opt_part';")" \ "p/pgcolumnar" expect_error "reject set_options on a partitioned parent" \ - "SELECT pgcolumnar.set_options('opt_part'::regclass, chunk_group_row_limit => 1000);" + "SELECT pgcolumnar.set_options('opt_part'::regclass, chunk_group_row_limit => 1000);" \ + "is not a columnar table" check "nothing was recorded for the partitioned parent" \ "$(q "SELECT count(*) FROM pgcolumnar.options WHERE regclass = 'opt_part'::regclass;")" "0" # And the leaf, which is an ordinary table, must still be accepted -- the guard # has to refuse the parent without refusing the thing that actually stores rows. q "CREATE TABLE opt_part_1 PARTITION OF opt_part FOR VALUES FROM (0) TO (100);" >/dev/null - q "SELECT pgcolumnar.set_options('opt_part_1'::regclass, chunk_group_row_limit => 1000);" >/dev/null + # `|| true`: a set_options that fails here is what the next check reports, by name. + # Under set -e it used to end the suite instead, with every later check unrecorded. + q "SELECT pgcolumnar.set_options('opt_part_1'::regclass, chunk_group_row_limit => 1000);" >/dev/null || true check "the partition itself is still accepted" \ "$(q "SELECT count(*) FROM pgcolumnar.options WHERE regclass = 'opt_part_1'::regclass;")" "1" q "DROP TABLE opt_part;" >/dev/null @@ -355,8 +358,9 @@ else fi # valid values are accepted, and delete/update work (no divide-by-zero) +# `|| true` for the same reason: "is recorded" below names a failed call. q "SELECT pgcolumnar.set_options('opt'::regclass, - chunk_group_row_limit => 1000, stripe_row_limit => 2000, compression_level => 9);" >/dev/null + chunk_group_row_limit => 1000, stripe_row_limit => 2000, compression_level => 9);" >/dev/null || true q "INSERT INTO opt SELECT g FROM generate_series(1,10) g;" >/dev/null q "DELETE FROM opt WHERE a=5;" >/dev/null check "valid options delete works" "$(q "SELECT count(*) FROM opt;")" "9" @@ -378,18 +382,27 @@ q "DROP TABLE opt;" >/dev/null q "SET pgcolumnar.stripe_row_limit=10000; CREATE TABLE idxleak (a int, b int) USING pgcolumnar WITH (parallel_workers=2);" >/dev/null q "INSERT INTO idxleak SELECT g, g%100 FROM generate_series(1,50000) g;" >/dev/null -run_pg "$PSQL -c \"SET max_parallel_maintenance_workers=2; SET min_parallel_table_scan_size=0; - CREATE INDEX idxleak_a_idx ON idxleak(a);\"" >/dev/null 2>&1 || true +# THE BUILD'S OUTCOME IS RECORDED, NOT DISCARDED. It was `>/dev/null 2>&1 || true`. +# Measured on PG17 with a parallel build participant raising an error: CREATE INDEX +# failed, no index existed, and "no relation leak" passed (nothing was built, so +# nothing leaked) and "rows indexed once" passed (50000, counted by a Seq Scan). +# Only the plan check went red, and it was labelled a premise. +idx_out="$(run_pg "$PSQL -c \"SET max_parallel_maintenance_workers=2; SET min_parallel_table_scan_size=0; + CREATE INDEX idxleak_a_idx ON idxleak(a);\"" 2>&1)" && idx_rc=0 || idx_rc=$? +check "the parallel CREATE INDEX completed" \ + "$([ "$idx_rc" = 0 ] && echo yes || echo "no: $(printf '%s\n' "$idx_out" | head -1)")" "yes" # the server logfile must carry no resource-leak warning LEAKS="$(run_pg "grep -c 'resource was not closed' '$LOGFILE'" || true)" check "create index no relation leak" "$LEAKS" "0" # and every row is indexed exactly once (no duplicate TIDs from a per-worker rescan) -check "premise: the indexed-once count reads idxleak_a_idx" \ - "$(reads_index idxleak_a_idx "SELECT count(*) FROM idxleak WHERE a>0")" "yes" +# NOT A PREMISE: a failed parallel build reddens it (#1339). And the count is read +# only when that plan reads the index, so a Seq Scan cannot answer for it. +idx_plan="$(reads_index idxleak_a_idx "SELECT count(*) FROM idxleak WHERE a>0")" +check "the indexed-once count reads idxleak_a_idx" "$idx_plan" "yes" check "create index rows indexed once" \ - "$(q "$IDXONLY SELECT count(*) FROM idxleak WHERE a>0;" | tail -1)" "50000" + "$([ "$idx_plan" = yes ] && q "$IDXONLY SELECT count(*) FROM idxleak WHERE a>0;" | tail -1 || echo "not read through idxleak_a_idx")" "50000" q "DROP TABLE idxleak;" >/dev/null # --------------------------------------------------------------------------- @@ -454,13 +467,13 @@ q "DROP TABLE ss_dup;" >/dev/null echo AUDIT_REACHED_END=1 -if [ "$fail" = "0" ]; then - echo "checks run: $PGC_CHECKS" +# The verdict is lib.sh's, from the counters every check updates. pgc_summary adds +# the accounting line the matrix reconciles on, which this suite never printed. +if [ "${PGC_FAIL:-0}" = "0" ]; then echo "AUDIT TEST PASSED" else - echo "checks run: $PGC_CHECKS" echo "AUDIT TEST FAILED" echo "---- server log tail ----" run_pg "tail -30 '$LOGFILE'" || true fi -exit $fail +pgc_summary diff --git a/test/bench_guards.sh b/test/bench_guards.sh index 04cf8cde6..54dd50e8c 100755 --- a/test/bench_guards.sh +++ b/test/bench_guards.sh @@ -25,28 +25,21 @@ # The argument is accepted and ignored; this suite needs no cluster. # Written fresh for pgColumnar. set -uo pipefail +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" SRCDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# No cluster needed; pgc_setup is skipped deliberately, as in wal_envelope.sh. The +# checks go through lib.sh's check() so each one is a RESULT record the ledger and +# the pytest twin can name (#1341); the suite used to keep its own tally, which +# emitted none. PGC_MAJOR comes from the pg_config every suite is handed, and +# degrades to `unknown` rather than to a guessed major when there is none. +PGC_MAJOR="$(pgc_major_of "${1:-}" 2>/dev/null)" PGC_CHECKS=0 PGC_FAIL=0 -check() { - local name="$1" got="$2" want="$3" - PGC_CHECKS=$((PGC_CHECKS + 1)) - if [ "$got" = "$want" ]; then - echo "PASS $name" - else - echo "FAIL $name: got [$got] want [$want]" - PGC_FAIL=1 - fi -} GUARDS="$SRCDIR/bench/cb_guards.sh" -if [ ! -f "$GUARDS" ]; then - echo "FAIL bench/cb_guards.sh is missing, so the benchmark's guards are untestable" - PGC_CHECKS=$((PGC_CHECKS + 1)) - PGC_FAIL=1 - echo; echo "checks run: $PGC_CHECKS"; echo "$(basename "$0"): FAILED"; exit 1 -fi +check "premise: bench/cb_guards.sh is present to be judged" "$([ -f "$GUARDS" ] && echo yes || echo no)" "yes" +[ -f "$GUARDS" ] || pgc_summary # shellcheck source=/dev/null . "$GUARDS" @@ -60,37 +53,83 @@ echo "== pgColumnar test: $(basename "$0") ==" # requirement the script does not enforce is the same shape as selftest 080 not # scanning bench/: the rule existed, the check did not. # -# Asserted on the SOURCE rather than by running the benchmarks, because running -# them takes minutes and the question is whether the guard is present and shaped -# right. -for _bp in "$(dirname "${BASH_SOURCE[0]}")/../bench/"run_*.sh; do +# Asserted by RUNNING each runner's own guard, not by grepping for its words +# (#1341). The guard is the block from the runner's --configure capture through +# the `fi` that closes its ALLOW_CASSERT test, read with comments removed, and it +# is run against a stand-in pg_config that reports an assert build, a plain one, +# or nothing. Running the whole runner is not an option: past its guard it +# builds, loads and downloads. +# +# The greps this replaces passed with the refusal gone. Every runner's header +# mentions --enable-cassert, and the cannot-tell block's `exit 1` sits within 24 +# lines of that mention, so with the assert branch's `exit 1` turned into `true`, +# with the empty-capture `exit 1` removed, with every build refused, or with the +# override ignored, main's suite passed all 60 checks. Measured on run_bench.sh. +# +# EVERY assertion still keys on shared TEXT, never on a prefix: run_profile.sh +# uses PROFILE_* where the other five use BENCH_*, so the override variable is +# read from the runner's own guard rather than spelled here. +_fake="$(mktemp -d)" +for _mode in cassert plain empty; do + { + echo '#!/bin/sh' + echo 'case "$1" in' + case "$_mode" in + cassert) echo " --configure) echo \"'--prefix=/nonexistent' '--enable-cassert'\" ;;" ;; + plain) echo " --configure) echo \"'--prefix=/nonexistent'\" ;;" ;; + empty) echo " --configure) exit 1 ;;" ;; + esac + echo ' --bindir) echo /nonexistent/bin ;;' + echo ' --version) echo "PostgreSQL 17.0" ;;' + echo ' *) echo /nonexistent ;;' + echo 'esac' + } >"$_fake/pg_config_$_mode" + chmod +x "$_fake/pg_config_$_mode" +done +guard_of() { # guard_of RUNNER -> its assert guard as code, comments removed, or nothing + # Only a COMPLETE guard is returned: the capture, an ALLOW_CASSERT test and the + # `fi` closing it, within 60 code lines. Anything else returns nothing, which + # the checks below refuse. Returning a partial block would run whatever follows + # it in the runner -- the build, the load, the download -- and with the assert + # test removed it once returned the rest of run_bench.sh. + awk '/^[[:space:]]*#/ {next} + !on && /--configure/ && /PG_CONFIG/ {on=1} + on {buf = buf $0 "\n"; n++} + on && /ALLOW_CASSERT/ {seen=1} + on && seen && /^fi$/ {printf "%s", buf; exit} + n > 60 {exit}' "$1" +} +run_guard() { # run_guard RUNNER MODE [ALLOW] -> "rc=N" then the guard's output + local block var out rc + block="$(guard_of "$1")" + var="$(grep -oE '[A-Z_]*ALLOW_CASSERT' <<<"$block" | head -1)" + out="$(env -i PATH=/usr/bin:/bin PG_CONFIG="$_fake/pg_config_$2" ${3:+"${var:-NO_OVERRIDE_FOUND}=1"} \ + bash -c "set -uo pipefail; $block" 2>&1)" + rc=$? + echo "rc=$rc" + echo "$out" +} +for _bp in "$SRCDIR/bench/"run_*.sh; do _bs="$(basename "$_bp")" + _asrt="$(run_guard "$_bp" cassert)" + _ovrd="$(run_guard "$_bp" cassert allow)" + _pln="$(run_guard "$_bp" plain)" + _none="$(run_guard "$_bp" empty)" check "[$_bs] refuses an assert build rather than documenting the requirement" \ - "$([ "$(grep -c 'enable-cassert' "$_bp")" -gt 0 ] && echo yes || echo no)" "yes" - # EVERY assertion here keys on shared TEXT, never on a prefix, and that is - # load-bearing rather than incidental: run_profile.sh uses PROFILE_* where - # the other five use BENCH_*, so a check written against `BENCH_ALLOW_CASSERT` - # would pass on five runners and silently stop covering the sixth. Proved: - # keying that one check on BENCH_ALLOW_CASSERT leaves run_profile.sh as the - # only red. The unprefixed match below is what keeps it in scope. - # - # If anyone tidies the prefixes to match, this comment says the checks need - # no change. If anyone tidies the CHECKS to match the prefixes, that is the - # moment coverage is lost with nothing going red. + "$(grep -q 'is an --enable-cassert build' <<<"$_asrt" && echo yes || echo "no: $(head -c 200 <<<"$_asrt" | tr '\n' ' ')")" "yes" check "[$_bs] and offers a named override rather than only refusing" \ - "$([ "$(grep -cE 'ALLOW_CASSERT' "$_bp")" -gt 0 ] && echo yes || echo no)" "yes" - check "[$_bs] and exits non-zero rather than warning past it" \ - "$([ "$(grep -A24 'enable-cassert' "$_bp" | grep -c 'exit 1')" -gt 0 ] && echo yes || echo no)" "yes" + "$(head -1 <<<"$_ovrd")/$(grep -c 'WARNING' <<<"$_ovrd")" "rc=0/1" + check "[$_bs] and exits non-zero rather than warning past it" "$(head -1 <<<"$_asrt")" "rc=1" + check "[$_bs] and lets a non-assert build through" \ + "$(head -1 <<<"$_pln")/$(grep -c 'FATAL' <<<"$_pln")" "rc=0/0" # "not an assert build" and "could not tell" are different answers and only - # one of them is a pass. Written as a bare `--configure | grep -c`, a missing - # or failing pg_config gives empty output, grep -c gives 0, and the guard - # returns its cleanest verdict on the least information -- so it must capture - # the output and refuse an empty one. + # one of them is a pass: a missing or failing pg_config must be refused. check "[$_bs] refuses when it CANNOT TELL, rather than treating that as a pass" \ - "$([ "$(grep -c 'produced no --configure output' "$_bp")" -gt 0 ] && echo yes || echo no)" "yes" + "$(head -1 <<<"$_none")/$(grep -c 'produced no --configure output' <<<"$_none")" "rc=1/1" check "[$_bs] and does not decide from an unbuffered pipe into grep -c" \ - "$([ "$(grep -cE '\$\("\$PG_CONFIG" --configure[^)]*\| *grep -c' "$_bp")" -eq 0 ] && echo yes || echo no)" "yes" + "$([ "$(guard_of "$_bp" | grep -cE '\$\("\$PG_CONFIG" --configure[^)]*\| *grep -c')" -eq 0 ] && echo yes || echo no)" "yes" done +rm -rf "$_fake" # The control: the scan has to be reading files that exist, or four greps over # nothing would report compliance. @@ -144,7 +183,7 @@ check "N + 1 is still refused: the coordinator needs a slot too" \ "$(cb_worker_slots_ok 9 8 && echo ok || echo refused)" "refused" check "N + 2 is accepted, the measured minimum" \ "$(cb_worker_slots_ok 10 8 && echo ok || echo refused)" "ok" -check "more than enough is accepted" \ +check "more than enough worker slots are accepted" \ "$(cb_worker_slots_ok 32 8 && echo ok || echo refused)" "ok" check "the rule holds at another worker count (4 needs 6)" \ "$(cb_worker_slots_ok 6 4 && echo ok || echo refused)" "ok" @@ -218,7 +257,7 @@ tag_none="$(cb_cold_tag none 2>/dev/null)" # "lukewarm-cold-run", and "the run was not called cold" therefore PASSES against # no implementation whatever. Seen, not reasoned about -- it passed exactly that # way on the first red run of this suite. -check "premise: the guard exists and emitted a tag to judge" \ +check "the guard exists and emitted a tag to judge" \ "$([ -n "$tag_none" ] && echo yes || echo no)" "yes" check "a host that could not drop the page cache is not called a cold run" \ "$(grep -qi 'lukewarm-cold-run' <<<"$tag_none" && echo claimed || echo not-claimed)" \ @@ -239,7 +278,7 @@ check "and the tag says plainly that the run was warm" \ # # The three numbers below are that run's q13, the smallest real case: heap # 1588.095 ms, columnar 1582.859 ms, and a warm-try scatter of 0.99 percent. -check "premise: the verdict helpers are present to be judged" \ +check "the verdict helpers are present to be judged" \ "$(type -t cb_verdict)/$(type -t cb_warm_spread)/$(type -t cb_band)" \ "function/function/function" @@ -294,11 +333,13 @@ CB="$SRCDIR/bench/run_clickbench.sh" check "premise: the benchmark script is present to be judged" \ "$([ -f "$CB" ] && echo yes || echo no)" "yes" +# Matched as a WHOLE name (#1341): `r[12]=` alone also matched `rr1=`, so with +# the variables renamed this premise passed and the check below read nothing. # Without this premise the next check is vacuous. Rename r1 and "no ratio value # decides anything" passes against a report that has gone back to deciding on one # under another name -- the grep finds nothing and nothing is what it approves. check "premise: the report still captures ratio() into a display variable" \ - "$([ "$(grep -c 'r[12]=\$(ratio' "$CB")" -ge 1 ] && echo yes || echo no)" "yes" + "$([ "$(grep -cE '(^|[^[:alnum:]_])r[12]=\$\(ratio' "$CB")" -ge 1 ] && echo yes || echo no)" "yes" check "ratio()'s output reaches printf and nothing else" \ "$(grep '\$r[12]' "$CB" | grep -vc 'printf')" "0" @@ -316,8 +357,11 @@ check "ratio()'s output reaches printf and nothing else" \ # `cb_verdict "` is the shape a call takes here and a mention almost never does. # Measured on the bypassed tree: loose 1 (approves), tight 0 (rejects); on the # good tree tight is 1 and still approves. +# Read as CODE: a comment that keeps the call's shape is not a call. With the +# verdict recomputed inline and `# was: case "$(cb_verdict "$c" ...` left behind, +# main's suite passed this check (#1341). check "and the verdict is asked of cb_verdict rather than recomputed inline" \ - "$([ "$(grep -c 'cb_verdict "' "$CB")" -ge 1 ] && echo yes || echo no)" "yes" + "$([ "$(grep -vE '^[[:space:]]*#' "$CB" | grep -c 'cb_verdict "')" -ge 1 ] && echo yes || echo no)" "yes" # ---- the accelerator claim must not call a separation "variation" (#565) ---- # @@ -369,11 +413,4 @@ echo " tuned-arm settings named on the page:$([ -z "$_miss" ] && echo ' all fou check "the tuned-arm section names all four settings and the group-cap value" \ "missing:$_miss" "missing:" -echo -echo "checks run: $PGC_CHECKS" -if [ "$PGC_FAIL" != 0 ]; then - echo "$(basename "$0"): FAILED" - exit 1 -fi -echo "$(basename "$0"): PASSED" -exit 0 +pgc_summary diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 218e6a82f..dbe6471d4 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -395,6 +395,41 @@ arrow_nested_import arrow_nested_import import rows 15;16;17;18;19 2026-10-20 ex arrow_nested_import arrow_nested_import nested import round-trips (arrays + composite) 15;16;17;18;19 2026-10-20 export: the final partial record batch never written;import: a NULL List element read as a value;import: a NULL Struct's validity bit ignored;import: an empty List read as a NULL array arrow_nested_import arrow_nested_import premise: the delete left 3999 live rows in the source 15;16;17;18;19 never - arrow_nested_import arrow_nested_import round-trips after delete 15;16;17;18;19 2026-10-20 export: deleted rows exported too;export: the final partial record batch never written;import: a NULL List element read as a value;import: a NULL Struct's validity bit ignored;import: an empty List read as a NULL array +audit audit add-column aggregate 15;16;17;18;19 2026-10-22 columnar_reader.c: a column added after the data reads NULL instead of its missing value +audit audit add-column const default 15;16;17;18;19 2026-10-22 columnar_reader.c: a column added after the data reads NULL instead of its missing value +audit audit add-column index scan 15;16;17;18;19 2026-10-22 columnar_reader.c: a column added after the data reads NULL instead of its missing value +audit audit add-column mid-transaction keeps a later UPDATE 15;16;17;18;19 2026-10-22 columnar_write_state.c: a write buffer opened before ADD COLUMN in the transaction is reused +audit audit add-column mid-transaction keeps the inserted value 15;16;17;18;19 2026-10-22 columnar_reader.c: a column added after the data reads NULL instead of its missing value +audit audit add-column mixed rows 15;16;17;18;19 2026-10-22 columnar_reader.c: a column added after the data reads NULL instead of its missing value +audit audit add-column nullable is null 15;16;17;18;19 never - +audit audit add-column projected only 15;16;17;18;19 2026-10-22 columnar_reader.c: a column added after the data reads NULL instead of its missing value +audit audit add-column text default 15;16;17;18;19 2026-10-22 columnar_reader.c: a column added after the data reads NULL instead of its missing value +audit audit add-column without a default keeps the inserted value 15;16;17;18;19 2026-10-22 columnar_write_state.c: a write buffer opened before ADD COLUMN in the transaction is reused +audit audit collation pushdown == no-pushdown 15;16;17;18;19 2026-10-22 columnar_customscan.c: a predicate under another collation is pushed into chunk-group skipping +audit audit collation result correct 15;16;17;18;19 2026-10-22 columnar_customscan.c: a predicate under another collation is pushed into chunk-group skipping +audit audit create index no relation leak 15;16;17;18;19 2026-10-22 columnar_tableam.c: index_build_range_scan does not end the scan a build participant was handed +audit audit create index rows indexed once 15;16;17;18;19 2026-10-22 columnar_tableam.c: every build participant reads through a private full-table reader +audit audit nothing was recorded for the heap table 15;16;17;18;19 never - +audit audit nothing was recorded for the partitioned parent 17;18;19 2026-10-22 pgcolumnar--1.0-alpha5.sql: set_options accepts relkind p as columnar +audit audit premise: the SQLSTATE probe reads a known code 15;16;17;18;19 never - +audit audit premise: the add-column lookup reads ac_c_idx 15;16;17;18;19 never - +audit audit premise: this major really lets a partitioned parent carry the AM 17;18;19 never - +audit audit reject chunk_group_row_limit 0 15;16;17;18;19 2026-10-22 pgcolumnar--1.0-alpha5.sql: set_options drops its chunk_group_row_limit < 100 guard +audit audit reject compression_level 99 15;16;17;18;19 2026-10-22 pgcolumnar--1.0-alpha5.sql: set_options drops its compression_level 1..22 guard +audit audit reject set_options on a heap table 15;16;17;18;19 2026-10-22 pgcolumnar--1.0-alpha5.sql: set_options raises division_by_zero at entry, before any guard +audit audit reject set_options on a partitioned parent 17;18;19 2026-10-22 pgcolumnar--1.0-alpha5.sql: set_options accepts relkind p as columnar +audit audit reject stripe_row_limit 5 15;16;17;18;19 2026-10-22 pgcolumnar--1.0-alpha5.sql: set_options drops its stripe_row_limit < 1000 guard +audit audit same-collation still pushes 15;16;17;18;19 never - +audit audit same-statement duplicate inserted nothing 15;16;17;18;19 never - +audit audit same-statement duplicate is rejected, not spun on 15;16;17;18;19 never - +audit audit set_options on a non-columnar relation raises 42809, not P0001 15;16;17;18;19 2026-10-22 pgcolumnar--1.0-alpha5.sql: set_options raises division_by_zero at entry, before any guard +audit audit the indexed-once count reads idxleak_a_idx 15;16;17;18;19 2026-10-22 columnar_tableam.c: a parallel index build participant raises an error +audit audit the parallel CREATE INDEX completed 15;16;17;18;19 2026-10-22 columnar_tableam.c: a parallel index build participant raises an error +audit audit the partition itself is still accepted 17;18;19 2026-10-22 pgcolumnar--1.0-alpha5.sql: set_options raises division_by_zero at entry, before any guard +audit audit the partitioned-parent arm 15;16 never - +audit audit the refusal says the relation is not columnar 15;16;17;18;19 2026-10-22 pgcolumnar--1.0-alpha5.sql: set_options raises division_by_zero at entry, before any guard +audit audit the same call on a columnar table is recorded 15;16;17;18;19 2026-10-22 pgcolumnar--1.0-alpha5.sql: set_options raises division_by_zero at entry, before any guard +audit audit valid options delete works 15;16;17;18;19 never - autovacuum autovacuum OFF-again: a new deleted table is left alone after disabling 15;16;17;18;19 2026-10-20 columnar_autovacuum.c: the launcher runs its passes with pgcolumnar.autovacuum off autovacuum autovacuum OFF: the daemon did not touch the table (dead rows unchanged) 15;16;17;18;19 2026-10-20 columnar_autovacuum.c: the launcher runs its passes with pgcolumnar.autovacuum off autovacuum autovacuum ON: the daemon compacted the deleted-heavy table (dead rows now 0) 15;16;17;18;19 2026-10-20 columnar_autovacuum.c: the maintenance launcher is never registered @@ -468,6 +503,97 @@ batch_fold_explain batch_fold_explain premise: control table returns the right s batch_fold_explain batch_fold_explain premise: fallback table returns the right sum 15;16;17;18;19 never - batch_fold_explain batch_fold_explain premise: fixture loaded every row 15;16;17;18;19 never - batch_fold_explain batch_fold_explain premise: with parallelism free, the planner gives the parallel partial fold 15;16;17;18;19 never - +bench_guards bench_guards N + 1 is still refused: the coordinator needs a slot too 15;16;17;18;19 2026-10-21 bench/cb_guards.sh: an N-worker arm is said to need N + 1 worker slots +bench_guards bench_guards N + 2 is accepted, the measured minimum 15;16;17;18;19 never - +bench_guards bench_guards [run_bench.sh] and does not decide from an unbuffered pipe into grep -c 15;16;17;18;19 2026-10-21 bench/run_*.sh: every runner decides from pg_config --configure piped straight into grep -c +bench_guards bench_guards [run_bench.sh] and exits non-zero rather than warning past it 15;16;17;18;19 2026-10-21 bench/run_bench.sh: the assert-build refusal warns past it (its exit 1 became true) +bench_guards bench_guards [run_bench.sh] and lets a non-assert build through 15;16;17;18;19 2026-10-21 bench/run_bench.sh: every build is refused as an assert build (over-applied) +bench_guards bench_guards [run_bench.sh] and offers a named override rather than only refusing 15;16;17;18;19 2026-10-21 bench/run_bench.sh: BENCH_ALLOW_CASSERT is ignored, so the override never runs +bench_guards bench_guards [run_bench.sh] refuses an assert build rather than documenting the requirement 15;16;17;18;19 2026-10-21 bench/run_bench.sh: the assert-build test removed outright, the header comment still naming it +bench_guards bench_guards [run_bench.sh] refuses when it CANNOT TELL, rather than treating that as a pass 15;16;17;18;19 2026-10-21 bench/run_bench.sh: an empty --configure capture is no longer refused (its exit 1 became true) +bench_guards bench_guards [run_bench_fsst.sh] and does not decide from an unbuffered pipe into grep -c 15;16;17;18;19 2026-10-21 bench/run_*.sh: every runner decides from pg_config --configure piped straight into grep -c +bench_guards bench_guards [run_bench_fsst.sh] and exits non-zero rather than warning past it 15;16;17;18;19 2026-10-21 bench/run_*.sh: in every runner, the assert-build refusal warns past it (its exit 1 became true) +bench_guards bench_guards [run_bench_fsst.sh] and lets a non-assert build through 15;16;17;18;19 2026-10-21 bench/run_*.sh: every runner refuses every build as an assert build (over-applied) +bench_guards bench_guards [run_bench_fsst.sh] and offers a named override rather than only refusing 15;16;17;18;19 2026-10-21 bench/run_*.sh: every runner ignores its ALLOW_CASSERT override +bench_guards bench_guards [run_bench_fsst.sh] refuses an assert build rather than documenting the requirement 15;16;17;18;19 never - +bench_guards bench_guards [run_bench_fsst.sh] refuses when it CANNOT TELL, rather than treating that as a pass 15;16;17;18;19 2026-10-21 bench/run_*.sh: in every runner, an empty --configure capture is no longer refused +bench_guards bench_guards [run_bench_join.sh] and does not decide from an unbuffered pipe into grep -c 15;16;17;18;19 2026-10-21 bench/run_*.sh: every runner decides from pg_config --configure piped straight into grep -c +bench_guards bench_guards [run_bench_join.sh] and exits non-zero rather than warning past it 15;16;17;18;19 2026-10-21 bench/run_*.sh: in every runner, the assert-build refusal warns past it (its exit 1 became true) +bench_guards bench_guards [run_bench_join.sh] and lets a non-assert build through 15;16;17;18;19 2026-10-21 bench/run_*.sh: every runner refuses every build as an assert build (over-applied) +bench_guards bench_guards [run_bench_join.sh] and offers a named override rather than only refusing 15;16;17;18;19 2026-10-21 bench/run_*.sh: every runner ignores its ALLOW_CASSERT override +bench_guards bench_guards [run_bench_join.sh] refuses an assert build rather than documenting the requirement 15;16;17;18;19 never - +bench_guards bench_guards [run_bench_join.sh] refuses when it CANNOT TELL, rather than treating that as a pass 15;16;17;18;19 2026-10-21 bench/run_*.sh: in every runner, an empty --configure capture is no longer refused +bench_guards bench_guards [run_bench_readstream.sh] and does not decide from an unbuffered pipe into grep -c 15;16;17;18;19 2026-10-21 bench/run_*.sh: every runner decides from pg_config --configure piped straight into grep -c +bench_guards bench_guards [run_bench_readstream.sh] and exits non-zero rather than warning past it 15;16;17;18;19 2026-10-21 bench/run_*.sh: in every runner, the assert-build refusal warns past it (its exit 1 became true) +bench_guards bench_guards [run_bench_readstream.sh] and lets a non-assert build through 15;16;17;18;19 2026-10-21 bench/run_*.sh: every runner refuses every build as an assert build (over-applied) +bench_guards bench_guards [run_bench_readstream.sh] and offers a named override rather than only refusing 15;16;17;18;19 2026-10-21 bench/run_*.sh: every runner ignores its ALLOW_CASSERT override +bench_guards bench_guards [run_bench_readstream.sh] refuses an assert build rather than documenting the requirement 15;16;17;18;19 never - +bench_guards bench_guards [run_bench_readstream.sh] refuses when it CANNOT TELL, rather than treating that as a pass 15;16;17;18;19 2026-10-21 bench/run_*.sh: in every runner, an empty --configure capture is no longer refused +bench_guards bench_guards [run_clickbench.sh] and does not decide from an unbuffered pipe into grep -c 15;16;17;18;19 2026-10-21 bench/run_*.sh: every runner decides from pg_config --configure piped straight into grep -c +bench_guards bench_guards [run_clickbench.sh] and exits non-zero rather than warning past it 15;16;17;18;19 2026-10-21 bench/run_*.sh: in every runner, the assert-build refusal warns past it (its exit 1 became true) +bench_guards bench_guards [run_clickbench.sh] and lets a non-assert build through 15;16;17;18;19 2026-10-21 bench/run_*.sh: every runner refuses every build as an assert build (over-applied) +bench_guards bench_guards [run_clickbench.sh] and offers a named override rather than only refusing 15;16;17;18;19 2026-10-21 bench/run_*.sh: every runner ignores its ALLOW_CASSERT override +bench_guards bench_guards [run_clickbench.sh] refuses an assert build rather than documenting the requirement 15;16;17;18;19 never - +bench_guards bench_guards [run_clickbench.sh] refuses when it CANNOT TELL, rather than treating that as a pass 15;16;17;18;19 2026-10-21 bench/run_*.sh: in every runner, an empty --configure capture is no longer refused +bench_guards bench_guards [run_profile.sh] and does not decide from an unbuffered pipe into grep -c 15;16;17;18;19 2026-10-21 bench/run_*.sh: every runner decides from pg_config --configure piped straight into grep -c +bench_guards bench_guards [run_profile.sh] and exits non-zero rather than warning past it 15;16;17;18;19 2026-10-21 bench/run_*.sh: in every runner, the assert-build refusal warns past it (its exit 1 became true) +bench_guards bench_guards [run_profile.sh] and lets a non-assert build through 15;16;17;18;19 2026-10-21 bench/run_*.sh: every runner refuses every build as an assert build (over-applied) +bench_guards bench_guards [run_profile.sh] and offers a named override rather than only refusing 15;16;17;18;19 2026-10-21 bench/run_*.sh: every runner ignores its ALLOW_CASSERT override +bench_guards bench_guards [run_profile.sh] refuses an assert build rather than documenting the requirement 15;16;17;18;19 never - +bench_guards bench_guards [run_profile.sh] refuses when it CANNOT TELL, rather than treating that as a pass 15;16;17;18;19 2026-10-21 bench/run_*.sh: in every runner, an empty --configure capture is no longer refused +bench_guards bench_guards a clear loss is a loss 15;16;17;18;19 never - +bench_guards bench_guards a clear win is a win 15;16;17;18;19 never - +bench_guards bench_guards a gap just outside the band is still called 15;16;17;18;19 never - +bench_guards bench_guards a host that could not drop the page cache is not called a cold run 15;16;17;18;19 2026-10-21 bench/cb_guards.sh: cb_cold_tag claims a cold run whatever happened +bench_guards bench_guards a missing count is refused rather than compared with the expectation 15;16;17;18;19 never - +bench_guards bench_guards a missing current value is refused, not compared 15;16;17;18;19 never - +bench_guards bench_guards a serial arm needs no prepared transactions 15;16;17;18;19 2026-10-21 bench/cb_guards.sh: cb_prepared_xacts_ok wants more than the worker count (-gt for -ge) +bench_guards bench_guards a serial arm needs no worker slots 15;16;17;18;19 never - +bench_guards bench_guards a short load is refused 15;16;17;18;19 never - +bench_guards bench_guards a single warm try has no measurable scatter 15;16;17;18;19 never - +bench_guards bench_guards a zero best is refused rather than divided by 15;16;17;18;19 never - +bench_guards bench_guards an empty load is refused, which is what an errored parallel arm produces 15;16;17;18;19 never - +bench_guards bench_guards an errored arm has no verdict 15;16;17;18;19 never - +bench_guards bench_guards an exact load is accepted 15;16;17;18;19 never - +bench_guards bench_guards an exact match is accepted 15;16;17;18;19 2026-10-21 bench/cb_guards.sh: cb_prepared_xacts_ok wants more than the worker count (-gt for -ge) +bench_guards bench_guards an unknown band suppresses the verdict, it does not strict-compare 15;16;17;18;19 never - +bench_guards bench_guards and a loss just outside it likewise 15;16;17;18;19 never - +bench_guards bench_guards and a missing expectation is refused too 15;16;17;18;19 never - +bench_guards bench_guards and is unknown if either arm is unknown 15;16;17;18;19 2026-10-21 bench/cb_guards.sh: cb_band is gone +bench_guards bench_guards and neither does a zero baseline 15;16;17;18;19 never - +bench_guards bench_guards and one below it does not 15;16;17;18;19 2026-10-21 bench/cb_guards.sh: an N-worker arm is said to need N + 1 worker slots +bench_guards bench_guards and says it needs a restart 15;16;17;18;19 never - +bench_guards bench_guards and says it needs a restart, which is why this runs first 15;16;17;18;19 2026-10-21 bench/cb_guards.sh: the prepared-transaction message no longer says a restart is needed +bench_guards bench_guards and the number it must reach 15;16;17;18;19 never - +bench_guards bench_guards and the tag says plainly that the run was warm 15;16;17;18;19 2026-10-21 bench/cb_guards.sh: cb_cold_tag claims a cold run whatever happened +bench_guards bench_guards and the value it must reach, not merely the worker count 15;16;17;18;19 2026-10-21 bench/cb_guards.sh: an N-worker arm is said to need N + 1 worker slots +bench_guards bench_guards and the verdict is asked of cb_verdict rather than recomputed inline 15;16;17;18;19 2026-10-21 bench/run_clickbench.sh: the verdict decided inline, a comment still carrying the cb_verdict call +bench_guards bench_guards and under a band wider than the gap it is a tie, not a win either 15;16;17;18;19 2026-10-21 bench/cb_guards.sh: cb_verdict ignores the band +bench_guards bench_guards columnar ahead by 0.33 percent is not a LOSS, which is the #531 defect 15;16;17;18;19 never - +bench_guards bench_guards more than enough is accepted 15;16;17;18;19 never - +bench_guards bench_guards more than enough worker slots are accepted 15;16;17;18;19 never - +bench_guards bench_guards no query is called run-to-run variation unless its figures are a measured tie 15;16;17;18;19 never - +bench_guards bench_guards no usable try yields no spread, not zero 15;16;17;18;19 never - +bench_guards bench_guards one short is refused, which is the off-by-one that would run 7 workers 15;16;17;18;19 never - +bench_guards bench_guards order does not matter, and ERR tries are dropped 15;16;17;18;19 2026-10-21 bench/cb_guards.sh: cb_warm_spread divides by the worst try, not the best +bench_guards bench_guards premise: bench/cb_guards.sh is present to be judged 15;16;17;18;19 never - +bench_guards bench_guards premise: every bench runner was read, not a hand-kept subset 15;16;17;18;19 never - +bench_guards bench_guards premise: the benchmark page is present to be judged 15;16;17;18;19 never - +bench_guards bench_guards premise: the benchmark script is present to be judged 15;16;17;18;19 never - +bench_guards bench_guards premise: the report still captures ratio() into a display variable 15;16;17;18;19 never - +bench_guards bench_guards ratio()'s output reaches printf and nothing else 15;16;17;18;19 never - +bench_guards bench_guards spread is the fractional range over the best 15;16;17;18;19 2026-10-21 bench/cb_guards.sh: cb_warm_spread divides by the worst try, not the best +bench_guards bench_guards the band is the wider of the two arms 15;16;17;18;19 2026-10-21 bench/cb_guards.sh: cb_band is gone +bench_guards bench_guards the guard exists and emitted a tag to judge 15;16;17;18;19 2026-10-21 bench/cb_guards.sh: cb_cold_tag prints nothing +bench_guards bench_guards the message names the setting to change 15;16;17;18;19 never - +bench_guards bench_guards the rule holds at another worker count (4 needs 6) 15;16;17;18;19 never - +bench_guards bench_guards the stock 0 is refused for an 8-worker parallel arm 15;16;17;18;19 never - +bench_guards bench_guards the stock 8 is refused for an 8-worker arm, which is the case that bit us 15;16;17;18;19 never - +bench_guards bench_guards the tuned-arm section names all four settings and the group-cap value 15;16;17;18;19 never - +bench_guards bench_guards the variation-claim extractor reads ids wrapped across two lines 15;16;17;18;19 2026-10-21 bench/cb_guards.sh: cb_doc_variation_qids reads line by line, missing a wrapped claim +bench_guards bench_guards the verdict helpers are present to be judged 15;16;17;18;19 2026-10-21 bench/cb_guards.sh: cb_band is gone +bench_guards bench_guards the worker-slot message names the setting 15;16;17;18;19 never - +bench_guards bench_guards two missing measurements are refused, not called equal 15;16;17;18;19 2026-10-21 bench/cb_guards.sh: cb_rows_ok compares without requiring numbers bloom_lazy bloom_lazy a range predicate still agrees with the heap 15;16;17;18;19 never - bloom_lazy bloom_lazy a scan that keeps every group reads more 15;16;17;18;19 2026-10-20 columnar_reader.c: the bloom filter is never fetched bloom_lazy bloom_lazy an absent value agrees with bloom filters disabled 15;16;17;18;19 never - @@ -1050,6 +1176,59 @@ doc_parallel_premise doc_parallel_premise the fixture holds several row groups, doc_parallel_premise doc_parallel_premise the group counters were read, not blank 15;16;17;18;19 2026-10-21 EXPLAIN stops printing Columnar Chunk Groups Read;pgcolumnar.enable_custom_scan boots false (columnar_tableam.c) doc_parallel_premise doc_parallel_premise the page still names a narrow query 15;16;17;18;19 2026-10-21 docs/limitations.md stops naming a narrow query doc_parallel_premise doc_parallel_premise the sel plan is exactly one columnar scan node 15;16;17;18;19 2026-10-21 pgcolumnar.enable_custom_scan boots false (columnar_tableam.c) +docs_style docs_style CHANGELOG.md carries no em or en dash 15;16;17;18;19 2026-10-21 CHANGELOG.md: a line with an em dash +docs_style docs_style CHANGELOG.md has a union merge driver, so two entries do not conflict 15;16;17;18;19 2026-10-21 .gitattributes: the CHANGELOG.md merge=union line removed +docs_style docs_style META.json provides.pgcolumnar.version is the version VERSION holds, in PGXN form 15;16;17;18;19 2026-10-21 META.json: provides.pgcolumnar.version left at 1.0.0-alpha.4 +docs_style docs_style META.json version is the version VERSION holds, in PGXN form 15;16;17;18;19 2026-10-21 META.json: version left at 1.0.0-alpha.4 +docs_style docs_style README's badge alt text names the version VERSION holds 15;16;17;18;19 2026-10-21 README.md: the badge alt text left at 1.0-alpha3 +docs_style docs_style administration.md states it in the section that says to lower the setting 15;16;17;18;19 2026-10-21 docs/administration.md: the stripe_row_limit floor written as 1000 +docs_style docs_style and CHANGELOG.md still has the union merge driver those instructions are about 15;16;17;18;19 2026-10-21 .gitattributes: the CHANGELOG.md merge=union line removed +docs_style docs_style and CONTEXT.md tells a contributor what a CONFLICTING badge means 15;16;17;18;19 2026-10-21 CONTEXT.md: the CONFLICTING-badge heading reworded +docs_style docs_style and it actually examined the documents 15;16;17;18;19 never - +docs_style docs_style and names the command rather than only the problem 15;16;17;18;19 2026-10-21 CONTEXT.md: git rebase origin/main written as git rebase main +docs_style docs_style and says WHY rebasing works where merging does not 15;16;17;18;19 2026-10-21 CONTEXT.md: merging **into** without its emphasis +docs_style docs_style and the page that nav entry points at exists 15;16;17;18;19 2026-10-21 docs/roadmap.md deleted, its nav entry kept +docs_style docs_style and warns against the button that reintroduces the conflict 15;16;17;18;19 2026-10-21 CONTEXT.md: do not click reworded +docs_style docs_style best-practices names clustering on the join key 15;16;17;18;19 2026-10-21 docs/best-practices.md: join key kept only inside an HTML comment +docs_style docs_style best-practices.md carries it with the load-sizing advice 15;16;17;18;19 2026-10-21 docs/best-practices.md: the stripe_row_limit floor written as 1000 +docs_style docs_style configuration.md states the 1024 floor where it documents the setting 15;16;17;18;19 2026-10-21 docs/configuration.md: the 1024 floor kept only inside an HTML comment +docs_style docs_style each shipped version starts exactly one upgrade script, so the chain is a walk 15;16;17;18;19 2026-10-21 an untracked pgcolumnar--1.0-alpha2--1.0-alphaX.sql, a second script starting at 1.0-alpha2 +docs_style docs_style every dated changelog section has its own section at its release tag 15;16;17;18;19 2026-10-21 CHANGELOG.md: a dated 1.0-alpha2.5 section no tag shipped +docs_style docs_style every dated changelog section holds what its tag shipped, plus only what is recorded 15;16;17;18;19 2026-10-21 CHANGELOG.md: an entry added to the released 1.0-alpha4 section +docs_style docs_style every dated changelog section still holds every entry its tag shipped 15;16;17;18;19 2026-10-21 CHANGELOG.md: an entry removed from the released 1.0-alpha4 section +docs_style docs_style every document citing VERSION quotes the version VERSION holds 15;16;17;18;19 2026-10-21 docs/limitations.md: a stale VERSION citation, the right one only in an HTML comment +docs_style docs_style every document making the claim states it in a form this rule can read 15;16;17;18;19 2026-10-21 README.md: the latest-published claim reworded out of both readable forms +docs_style docs_style every document names the same latest published pre-release 15;16;17;18;19 2026-10-21 docs/roadmap.md: the latest-published claim names v1.0-alpha1 +docs_style docs_style every document stating the upgrade chain states it in a form this rule can read 15;16;17;18;19 2026-10-21 docs/limitations.md: the upgrade-chain sentence names no destination +docs_style docs_style every pgcolumnar--*.sql in the tree is in the published distribution 15;16;17;18;19 2026-10-21 an untracked pgcolumnar--1.0-alpha2--1.0-alphaX.sql, a second script starting at 1.0-alpha2 +docs_style docs_style every recorded post-tag entry carries a reason 15;16;17;18;19 2026-10-21 test/changelog_post_tag.txt: a row with no reason +docs_style docs_style every such document names default_version as the version one UPDATE reaches 15;16;17;18;19 2026-10-21 docs/limitations.md: the upgrade-chain sentence names no destination +docs_style docs_style every such document names the versions that reach default_version in one update 15;16;17;18;19 2026-10-21 docs/limitations.md: 1.0-alpha2 dropped from the starting versions +docs_style docs_style every user-facing document meets the measurable plain-language rules 15;16;17;18;19 2026-10-21 docs/how-to.md: a sentence with an em dash +docs_style docs_style how-to names clustering on the join key for the runtime filter 15;16;17;18;19 2026-10-21 docs/how-to.md: join key reworded to join column +docs_style docs_style no document carries a merge conflict marker 15;16;17;18;19 2026-10-21 docs/user-guide.md: a merge conflict marker line +docs_style docs_style no documented query tests pg_extension while listing databases 15;16;17;18;19 2026-10-21 docs/how-to.md: a sql block testing pg_extension while listing pg_database +docs_style docs_style no recorded post-tag entry is stale, so the allowance cannot widen silently 15;16;17;18;19 2026-10-21 test/changelog_post_tag.txt: a row for 1.0-alpha9, which has no section +docs_style docs_style premise: META.json is present and parses 15;16;17;18;19 never - +docs_style docs_style premise: META.json names a base install script 15;16;17;18;19 never - +docs_style docs_style premise: and it names VERSION's version at all 15;16;17;18;19 never - +docs_style docs_style premise: at least one document cites the VERSION file 15;16;17;18;19 never - +docs_style docs_style premise: at least one document names the latest published pre-release 15;16;17;18;19 never - +docs_style docs_style premise: at least one document states the upgrade chain 15;16;17;18;19 never - +docs_style docs_style premise: some shipped version reaches default_version, so the walk found a chain 15;16;17;18;19 never - +docs_style docs_style premise: the PGXN form was derived from VERSION, not empty 15;16;17;18;19 never - +docs_style docs_style premise: the VERSION file has a version to compare against 15;16;17;18;19 never - +docs_style docs_style premise: the changelog sweep found dated release sections 15;16;17;18;19 never - +docs_style docs_style premise: the control file names a default_version to arrive at 15;16;17;18;19 never - +docs_style docs_style premise: the post-tag allowance file is present and readable 15;16;17;18;19 never - +docs_style docs_style premise: the scan read SQL statements from the documents 15;16;17;18;19 never - +docs_style docs_style premise: the tree has install scripts to check 15;16;17;18;19 never - +docs_style docs_style premise: the tree ships upgrade scripts to derive the chain from 15;16;17;18;19 never - +docs_style docs_style premise: the version badge is present 15;16;17;18;19 never - +docs_style docs_style the how-to keeps its pg_partman section under its own heading 15;16;17;18;19 2026-10-21 docs/how-to.md: a paragraph inserted between the pg_partman heading and its steps +docs_style docs_style the roadmap is in the documentation nav 15;16;17;18;19 2026-10-21 mkdocs.yml: the roadmap nav entry commented out with # +docs_style docs_style the script META.json names is in the published distribution 15;16;17;18;19 2026-10-21 META.json: provides.pgcolumnar.file names pgcolumnar--1.0-alpha3.sql +docs_style docs_style the version badge names no version other than VERSION's 15;16;17;18;19 2026-10-21 badges/version.svg: one of its three version strings left at 1.0-alpha3 eager_ordering_record eager_ordering_record an unsorted rewrite claims no ordering 15;16;17;18;19 2026-10-21 an unsorted rewrite still records a lexicographic run eager_ordering_record eager_ordering_record and it rewrote no group: the physical layout is unchanged 15;16;17;18;19 2026-10-21 the recluster self-gate never fires eager_ordering_record eager_ordering_record and records it as a lexicographic run 15;16;17;18;19 2026-10-21 eager vacuum_sorted records no key and no kind (#758) @@ -1255,6 +1434,48 @@ fsst_verdict_cache fsst_verdict_cache premise: the prose corpus KEEPS fsst, so t fsst_verdict_cache fsst_verdict_cache premise: the prose fixture loaded 15;16;17;18;19 never - fsst_verdict_cache fsst_verdict_cache premise: the three changing-column loads all produced bytes to compare 15;16;17;18;19 never - fsst_verdict_cache fsst_verdict_cache the cached load returns the same values as heap 15;16;17;18;19 2026-10-21 FSST decode writes symbol 1's bytes for symbol 0 +fuzz fuzz every iteration loaded all its rows 15;16;17;18;19 never - +fuzz fuzz every iteration's IS NULL count agrees with heap 15;16;17;18;19 never - +fuzz fuzz every iteration's compound filter agrees with heap 15;16;17;18;19 2026-10-22 columnar_reader.c: the > skip predicate also skips a group whose min is at most the constant +fuzz fuzz every iteration's counts agree with heap 15;16;17;18;19 never - +fuzz fuzz every iteration's date range agrees with heap 15;16;17;18;19 never - +fuzz fuzz every iteration's equality lookup agrees with heap 15;16;17;18;19 never - +fuzz fuzz every iteration's integer range agrees with heap 15;16;17;18;19 never - +fuzz fuzz every iteration's min and max agree with heap 15;16;17;18;19 never - +fuzz fuzz every iteration's storage options read back as set 15;16;17;18;19 2026-10-22 pgcolumnar--1.0-alpha5.sql: set_options stores NULL for chunk_group_row_limit, so the table keeps the default +fuzz fuzz every iteration's sums and averages agree with heap 15;16;17;18;19 never - +fuzz fuzz every iteration's whole rows agree with heap 15;16;17;18;19 never - +fuzz fuzz the run covered every iteration 15;16;17;18;19 never - +fuzz_arrow fuzz_arrow every mutant was generated and ran the import 15;16;17;18;19 never - +fuzz_arrow fuzz_arrow no mutant crashed the backend 15;16;17;18;19 2026-10-22 columnar_arrow.c: truncated IPC metadata crashes the backend instead of raising +fuzz_arrow fuzz_arrow no mutant hung the decode 15;16;17;18;19 never - +fuzz_arrow fuzz_arrow no mutant tripped a sanitizer 15;16;17;18;19 never - +fuzz_arrow fuzz_arrow the corpus reached the decoder at all 15;16;17;18;19 never - +fuzz_arrow fuzz_arrow the importer accepts some pristine seed 15;16;17;18;19 never - +fuzz_arrow fuzz_arrow the seed corpus generator produced files 15;16;17;18;19 never - +fuzz_avro fuzz_avro control: the valid manifest reaches the decoder (2 entries) 15;16;17;18;19 never - +fuzz_avro fuzz_avro control: the valid manifest-list reaches the decoder (1 entry) 15;16;17;18;19 never - +fuzz_avro fuzz_avro every iteration wrote a fresh mutant 15;16;17;18;19 never - +fuzz_avro fuzz_avro every mutant ran (errors + clean == iters) 15;16;17;18;19 2026-10-22 columnar_avro.c: a file with bad magic loops forever instead of raising +fuzz_avro fuzz_avro no mutant crashed the backend 15;16;17;18;19 2026-10-22 columnar_avro.c: a malformed Avro header crashes the backend instead of raising +fuzz_avro fuzz_avro no mutant hung the decoder 15;16;17;18;19 2026-10-22 columnar_avro.c: a file with bad magic loops forever instead of raising +fuzz_avro fuzz_avro no mutant tripped a sanitizer 15;16;17;18;19 never - +fuzz_avro fuzz_avro the corpus reached the decoder at all 15;16;17;18;19 never - +fuzz_listing fuzz_listing control: a valid listing reaches the parser and reads the object 15;16;17;18;19 never - +fuzz_listing fuzz_listing every iteration wrote a fresh mutant 15;16;17;18;19 never - +fuzz_listing fuzz_listing every mutant ran (errors + clean == iters) 15;16;17;18;19 2026-10-22 objstore/columnar_objstore_module.c: a non-advancing continuation token crashes the backend instead of raising +fuzz_listing fuzz_listing no mutant crashed the backend 15;16;17;18;19 2026-10-22 objstore/columnar_objstore_module.c: a non-advancing continuation token crashes the backend instead of raising +fuzz_listing fuzz_listing no mutant hung the parser 15;16;17;18;19 never - +fuzz_listing fuzz_listing no mutant tripped a sanitizer 15;16;17;18;19 never - +fuzz_listing fuzz_listing premise: fixture up 15;16;17;18;19 never - +fuzz_listing fuzz_listing the corpus reached the parser at all (some mutant errored or accepted) 15;16;17;18;19 never - +fuzz_parquet fuzz_parquet every mutant was generated and ran its three statements 15;16;17;18;19 never - +fuzz_parquet fuzz_parquet no mutant crashed the backend 15;16;17;18;19 2026-10-22 columnar_parquet_reader.c: unparseable Parquet metadata crashes the backend instead of raising +fuzz_parquet fuzz_parquet no mutant hung the decode 15;16;17;18;19 never - +fuzz_parquet fuzz_parquet no mutant tripped a sanitizer 15;16;17;18;19 never - +fuzz_parquet fuzz_parquet the corpus reached the decoder at all 15;16;17;18;19 never - +fuzz_parquet fuzz_parquet the reader renders some pristine seed's schema 15;16;17;18;19 never - +fuzz_parquet fuzz_parquet the seed corpus generator produced files 15;16;17;18;19 never - generated_columns generated_columns stored generated: aggregate matches 15;16;17;18;19 2026-10-21 the flush skips STORED generated columns too, which are materialized generated_columns generated_columns stored generated: chunk present for attr 2 15;16;17;18;19 2026-10-21 the flush skips STORED generated columns too, which are materialized generated_columns generated_columns stored generated: filtered scan matches 15;16;17;18;19 2026-10-21 the flush skips STORED generated columns too, which are materialized @@ -1612,7 +1833,7 @@ harness_selftest 320-a-check-that-could-not-run premise: the fixtures carry the harness_selftest 320-a-check-that-could-not-run premise: the harness library is where this part thinks it is 15;16;17;18;19 never - harness_selftest 320-a-check-that-could-not-run premise: the major-verdict mapping evalled out of the runner is callable 15;16;17;18;19 never - harness_selftest 320-a-check-that-could-not-run premise: the runner defines the classifier this part is about to eval 15;16;17;18;19 never - -harness_selftest 320-a-check-that-could-not-run premise: the sweep read the corpus and found sites to classify 15;16;17;18;19 never - +harness_selftest 320-a-check-that-could-not-run premise: the sweep read the corpus 15;16;17;18;19 never - harness_selftest 320-a-check-that-could-not-run the original rule flags a bump that records no outcome 15;16;17;18;19 never - harness_selftest 320-a-check-that-could-not-run the runner calls a clean exit a pass 15;16;17;18;19 never - harness_selftest 320-a-check-that-could-not-run the runner's INCOMPLETE branch calls the mapping rather than a local flag 15;16;17;18;19 never - @@ -2331,7 +2552,7 @@ harness_selftest 480-an-unrunnable-record-names-its-check premise: and it compar harness_selftest 480-an-unrunnable-record-names-its-check premise: that clean run had a refusal site to be silent ABOUT 15;16;17;18;19 never - harness_selftest 480-an-unrunnable-record-names-its-check premise: the sweep found unrunnable sites at all 15;16;17;18;19 never - harness_selftest 480-an-unrunnable-record-names-its-check premise: the sweep tool is present 15;16;17;18;19 never - -harness_selftest 480-an-unrunnable-record-names-its-check the refusal emitters are the two the tree defines 15;16;17;18;19 never - +harness_selftest 480-an-unrunnable-record-names-its-check the refusal emitters are the ones the tree defines 15;16;17;18;19 never - harness_selftest 480-an-unrunnable-record-names-its-check the sweep EXITS non-zero when it reports a mismatch 15;16;17;18;19 never - harness_selftest 480-an-unrunnable-record-names-its-check the sweep REPORTS a name the file records only in its refusal branch 15;16;17;18;19 never - harness_selftest 490-a-one-line-body-ends-at-its-own-brace a body that never closes is NAMED rather than silently swallowing the file 15;16;17;18;19 never - @@ -2537,6 +2758,190 @@ harness_selftest 640-every-suite-job-installs-the-extensions the detector names harness_selftest 640-every-suite-job-installs-the-extensions the detector names a workflow in which it finds no suite job 15;16;17;18;19 2026-10-14 640's _ej_missing without its no-suite-job guard harness_selftest 640-every-suite-job-installs-the-extensions the detector names a workflow it cannot read 15;16;17;18;19 2026-10-14 640's _ej_missing without its unreadable-file guard harness_selftest 640-every-suite-job-installs-the-extensions the suite-running jobs are exactly the five this part knows 15;16;17;18;19 2026-10-14 640's job scan no longer counts a cluster job that runs pytest +hilbert_curve hilbert_curve C0 premise: the battery encoded every case and every point 15;16;17;18;19 never - +hilbert_curve hilbert_curve C0 premise: the encoder was called once per point 15;16;17;18;19 never - +hilbert_curve hilbert_curve C1 the Hilbert index set is exactly the contiguous range 15;16;17;18;19 2026-10-21 columnar_curve.c: the transpose clears the top bit of X[0], so curve indices collide +hilbert_curve hilbert_curve C2 consecutive points in memcmp key order are unit-adjacent 15;16;17;18;19 2026-10-21 columnar_curve.c: the transpose clears the top bit of X[0], so curve indices collide +hilbert_curve hilbert_curve C3 every dyadic sub-cube occupies a contiguous run of indices 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave emits the bit rounds lowest first, a permutation whose dyadic sub-cubes are scattered +hilbert_curve hilbert_curve C4a premise: the bridge covered every case and every point 15;16;17;18;19 never - +hilbert_curve hilbert_curve C4a the full-width key prefix does not move when the low bits change 15;16;17;18;19 2026-10-21 columnar_curve.c: the lowest bit of the last transposed word is folded into the top bit of the first +hilbert_curve hilbert_curve C4a the full-width key prefixes are exactly the index range 15;16;17;18;19 2026-10-21 columnar_curve.c: the transpose clears the top bit of X[0], so curve indices collide +hilbert_curve hilbert_curve C4b consecutive keys decode to unit-adjacent points, re-encoded by the code under test 15;16;17;18;19 2026-10-21 columnar_curve.c: a correct transpose that flips bit 0 of X[0] when ncols == 8 and the low byte of X[3] is 0x5a +hilbert_curve hilbert_curve C4b control: a mutated inverse breaks every round trip the clean one made 15;16;17;18;19 2026-10-21 columnar_curve.c: a correct transpose that flips bit 0 of X[0] when ncols == 8 and the low byte of X[3] is 0x5a +hilbert_curve hilbert_curve C4b premise: 3,000 DISTINCT full-width keys per ncols were round-tripped 15;16;17;18;19 never - +hilbert_curve hilbert_curve C4b premise: consecutive key pairs were formed 15;16;17;18;19 never - +hilbert_curve hilbert_curve C4b the published inverse round-trips full-width keys 15;16;17;18;19 2026-10-21 columnar_curve.c: a correct transpose that flips bit 0 of X[0] when ncols == 8 and the low byte of X[3] is 0x5a +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=1 asym 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=1 lowbit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=1 max 15;16;17;18;19 2026-10-21 columnar_curve.c: the final mask t loses its lowest bit, so only bit 0 of each transposed word moves +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=1 mixed 15;16;17;18;19 2026-10-21 columnar_curve.c: the final mask t loses its lowest bit, so only bit 0 of each transposed word moves +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=1 tie 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=1 topbit0 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=1 unit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=1 zero 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=2 asym 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=2 lowbit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=2 max 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=2 mixed 15;16;17;18;19 2026-10-21 columnar_curve.c: the final mask t loses its lowest bit, so only bit 0 of each transposed word moves +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=2 tie 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=2 topbit0 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=2 unit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=2 zero 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=3 asym 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=3 lowbit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=3 max 15;16;17;18;19 2026-10-21 columnar_curve.c: the final mask t loses its lowest bit, so only bit 0 of each transposed word moves +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=3 mixed 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=3 tie 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=3 topbit0 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=3 unit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=3 zero 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=4 asym 15;16;17;18;19 2026-10-21 columnar_curve.c: the final mask t loses its lowest bit, so only bit 0 of each transposed word moves +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=4 lowbit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=4 max 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=4 mixed 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=4 tie 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=4 topbit0 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=4 unit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=4 zero 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=5 asym 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=5 lowbit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=5 max 15;16;17;18;19 2026-10-21 columnar_curve.c: the final mask t loses its lowest bit, so only bit 0 of each transposed word moves +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=5 mixed 15;16;17;18;19 2026-10-21 columnar_curve.c: the final mask t loses its lowest bit, so only bit 0 of each transposed word moves +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=5 tie 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=5 topbit0 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=5 unit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=5 zero 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=6 asym 15;16;17;18;19 2026-10-21 columnar_curve.c: the final mask t loses its lowest bit, so only bit 0 of each transposed word moves +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=6 lowbit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=6 max 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=6 mixed 15;16;17;18;19 2026-10-21 columnar_curve.c: the final mask t loses its lowest bit, so only bit 0 of each transposed word moves +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=6 tie 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=6 topbit0 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=6 unit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=6 zero 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=7 asym 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=7 lowbit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=7 max 15;16;17;18;19 2026-10-21 columnar_curve.c: the final mask t loses its lowest bit, so only bit 0 of each transposed word moves +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=7 mixed 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=7 tie 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=7 topbit0 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=7 unit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=7 zero 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=8 asym 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=8 lowbit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=8 max 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=8 mixed 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=8 tie 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=8 topbit0 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=8 unit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C5 golden Hilbert keys: ncols=8 zero 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C5 the pack bit order: pack(0, 2^63) starts 0x40 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 the pack bit order: pack(2^63, 0) starts 0x80 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C5 the pack establishes exactly 8*ncols bytes: ncols=1 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave writes one byte past the 8*ncols-byte key +hilbert_curve hilbert_curve C5 the pack establishes exactly 8*ncols bytes: ncols=2 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave writes one byte past the 8*ncols-byte key +hilbert_curve hilbert_curve C5 the pack establishes exactly 8*ncols bytes: ncols=3 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave writes one byte past the 8*ncols-byte key +hilbert_curve hilbert_curve C5 the pack establishes exactly 8*ncols bytes: ncols=4 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave writes one byte past the 8*ncols-byte key +hilbert_curve hilbert_curve C5 the pack establishes exactly 8*ncols bytes: ncols=5 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave writes one byte past the 8*ncols-byte key +hilbert_curve hilbert_curve C5 the pack establishes exactly 8*ncols bytes: ncols=6 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave writes one byte past the 8*ncols-byte key +hilbert_curve hilbert_curve C5 the pack establishes exactly 8*ncols bytes: ncols=7 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave writes one byte past the 8*ncols-byte key +hilbert_curve hilbert_curve C5 the pack establishes exactly 8*ncols bytes: ncols=8 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave writes one byte past the 8*ncols-byte key +hilbert_curve hilbert_curve C5 the pack sets every bit of the key for an all-ones vector: ncols=1 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave writes one byte past the 8*ncols-byte key +hilbert_curve hilbert_curve C5 the pack sets every bit of the key for an all-ones vector: ncols=2 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave writes one byte past the 8*ncols-byte key +hilbert_curve hilbert_curve C5 the pack sets every bit of the key for an all-ones vector: ncols=3 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave writes one byte past the 8*ncols-byte key +hilbert_curve hilbert_curve C5 the pack sets every bit of the key for an all-ones vector: ncols=4 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave writes one byte past the 8*ncols-byte key +hilbert_curve hilbert_curve C5 the pack sets every bit of the key for an all-ones vector: ncols=5 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave writes one byte past the 8*ncols-byte key +hilbert_curve hilbert_curve C5 the pack sets every bit of the key for an all-ones vector: ncols=6 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave writes one byte past the 8*ncols-byte key +hilbert_curve hilbert_curve C5 the pack sets every bit of the key for an all-ones vector: ncols=7 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave writes one byte past the 8*ncols-byte key +hilbert_curve hilbert_curve C5 the pack sets every bit of the key for an all-ones vector: ncols=8 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave writes one byte past the 8*ncols-byte key +hilbert_curve hilbert_curve C5 the pack wrote nothing past the key: ncols=1 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave writes one byte past the 8*ncols-byte key +hilbert_curve hilbert_curve C5 the pack wrote nothing past the key: ncols=2 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave writes one byte past the 8*ncols-byte key +hilbert_curve hilbert_curve C5 the pack wrote nothing past the key: ncols=3 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave writes one byte past the 8*ncols-byte key +hilbert_curve hilbert_curve C5 the pack wrote nothing past the key: ncols=4 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave writes one byte past the 8*ncols-byte key +hilbert_curve hilbert_curve C5 the pack wrote nothing past the key: ncols=5 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave writes one byte past the 8*ncols-byte key +hilbert_curve hilbert_curve C5 the pack wrote nothing past the key: ncols=6 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave writes one byte past the 8*ncols-byte key +hilbert_curve hilbert_curve C5 the pack wrote nothing past the key: ncols=7 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave writes one byte past the 8*ncols-byte key +hilbert_curve hilbert_curve C5 the pack wrote nothing past the key: ncols=8 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave writes one byte past the 8*ncols-byte key +hilbert_curve hilbert_curve C6 ncols == 1 agrees with Z-order over 200,000 random values 15;16;17;18;19 2026-10-21 columnar_curve.c: the final mask t loses its lowest bit, so only bit 0 of each transposed word moves +hilbert_curve hilbert_curve C6 ncols == 1 is the big-endian ordinal over 200,000 random values 15;16;17;18;19 2026-10-21 columnar_curve.c: the final mask t loses its lowest bit, so only bit 0 of each transposed word moves +hilbert_curve hilbert_curve C6 ncols == 1 is the big-endian ordinal: 0 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C6 ncols == 1 is the big-endian ordinal: 1 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C6 ncols == 1 is the big-endian ordinal: 2^63 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C6 ncols == 1 is the big-endian ordinal: UINT64_MAX 15;16;17;18;19 2026-10-21 columnar_curve.c: the final mask t loses its lowest bit, so only bit 0 of each transposed word moves +hilbert_curve hilbert_curve C6 premise: the comparison loop ran 200,000 times 15;16;17;18;19 never - +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=1 asym 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=1 lowbit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=1 max 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave stops after 32 of its 64 rounds +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=1 mixed 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave emits the bit rounds lowest first, a permutation whose dyadic sub-cubes are scattered +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=1 tie 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=1 topbit0 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=1 unit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=1 zero 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=2 asym 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=2 lowbit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=2 max 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave stops after 32 of its 64 rounds +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=2 mixed 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=2 tie 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=2 topbit0 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=2 unit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=2 zero 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=3 asym 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=3 lowbit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=3 max 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave stops after 32 of its 64 rounds +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=3 mixed 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=3 tie 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=3 topbit0 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=3 unit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=3 zero 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=4 asym 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=4 lowbit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=4 max 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave stops after 32 of its 64 rounds +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=4 mixed 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave emits the bit rounds lowest first, a permutation whose dyadic sub-cubes are scattered +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=4 tie 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=4 topbit0 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=4 unit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=4 zero 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=5 asym 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=5 lowbit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=5 max 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave stops after 32 of its 64 rounds +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=5 mixed 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave emits the bit rounds lowest first, a permutation whose dyadic sub-cubes are scattered +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=5 tie 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=5 topbit0 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=5 unit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=5 zero 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=6 asym 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=6 lowbit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=6 max 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave stops after 32 of its 64 rounds +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=6 mixed 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave emits the bit rounds lowest first, a permutation whose dyadic sub-cubes are scattered +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=6 tie 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=6 topbit0 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=6 unit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=6 zero 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes one byte short of the key +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=7 asym 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=7 lowbit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=7 max 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave stops after 32 of its 64 rounds +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=7 mixed 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=7 tie 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=7 topbit0 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=7 unit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=7 zero 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=8 asym 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=8 lowbit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=8 max 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave stops after 32 of its 64 rounds +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=8 mixed 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=8 tie 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=8 topbit0 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=8 unit 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve C7 the Z-order pack is byte-for-byte what it was: ncols=8 zero 15;16;17;18;19 2026-10-21 columnar_curve.c: cluster_pack_interleave zeroes at most 48 bytes, so a 7- or 8-column key is not fully established +hilbert_curve hilbert_curve INSTRUMENT C0 premise: the control battery encoded every case and every point 15;16;17;18;19 never - +hilbert_curve hilbert_curve INSTRUMENT C0 premise: the control encoders were called once per point 15;16;17;18;19 never - +hilbert_curve hilbert_curve INSTRUMENT C1 control: Z-order is a permutation of the index range 15;16;17;18;19 never - +hilbert_curve hilbert_curve INSTRUMENT C1 control: a truncating encoder FAILS the permutation test in every case 15;16;17;18;19 never - +hilbert_curve hilbert_curve INSTRUMENT C1 control: the serpentine is a permutation of the index range 15;16;17;18;19 never - +hilbert_curve hilbert_curve INSTRUMENT C2 control: Z-order FAILS unit-step adjacency in every multi-column case 15;16;17;18;19 never - +hilbert_curve hilbert_curve INSTRUMENT C2 control: the serpentine PASSES unit-step adjacency 15;16;17;18;19 never - +hilbert_curve hilbert_curve INSTRUMENT C3 control: Z-order PASSES dyadic contiguity 15;16;17;18;19 never - +hilbert_curve hilbert_curve INSTRUMENT C3 control: the serpentine FAILS dyadic contiguity in every nested case 15;16;17;18;19 never - +hilbert_curve hilbert_curve INSTRUMENT the control's own pack reproduces the frozen Z-order bytes 15;16;17;18;19 never - iceberg_catalog iceberg_catalog a dangling current-snapshot-id is refused (data_corrupted XX001) 15;16;17;18;19 2026-10-21 a dangling current-snapshot-id reads as no current snapshot iceberg_catalog iceberg_catalog a non-Iceberg JSON object is refused (invalid_parameter_value 22023) 15;16;17;18;19 2026-10-21 any JSON object is read as Iceberg table metadata iceberg_catalog iceberg_catalog a pg_read_server_files member who is not superuser may read the catalog 15;16;17;18;19 2026-10-21 the Iceberg SRF gate admits only superusers, refusing every pg_read_server_files member @@ -2549,6 +2954,81 @@ iceberg_catalog iceberg_catalog malformed JSON is rejected (22P02), backend surv iceberg_catalog iceberg_catalog resolves the named current snapshot, not snapshots[0] 15;16;17;18;19 2026-10-21 the resolver returns the first snapshot rather than the one named current iceberg_catalog iceberg_catalog the decoy snapshot is not returned 15;16;17;18;19 2026-10-21 the resolver returns the first snapshot rather than the one named current iceberg_catalog iceberg_catalog the first snapshot has no parent (NULL) 15;16;17;18;19 never - +iceberg_deletes iceberg_deletes ...and the boundary refusal names the escape 15;16;17;18;19 2026-10-21 columnar_iceberg.c: ice_rebase does not hold a recorded path under the recorded table location +iceberg_deletes iceberg_deletes ...and the negative-pos refusal names the negative pos 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a negative position-delete pos is not refused +iceberg_deletes iceberg_deletes ...and the null-pos refusal names pos 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a position-delete row with a null pos is not refused (the n1 || n2 guard skipped) +iceberg_deletes iceberg_deletes ...and the refusal names the type 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a timestamp equality-delete column is mapped to text instead of refused +iceberg_deletes iceberg_deletes a cross-spec partition-scoped delete applies to nothing (5 rows) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a partition-scoped equality delete applies across partition spec ids +iceberg_deletes iceberg_deletes a data file lacking partition_spec_id is refused under a partitioned delete (XX001) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a data file without partition_spec_id is not refused under a partition-scoped delete +iceberg_deletes iceberg_deletes a delete older than the data (lower seq) does not apply (5 rows) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a position delete applies whatever its sequence number +iceberg_deletes iceberg_deletes a delete scoped to an empty partition deletes nothing (5 rows) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a partition-scoped equality delete ignores the partition value +iceberg_deletes iceberg_deletes a dotdot-escaping delete path is refused (22023) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: ice_rebase does not re-check containment after canonicalize_path +iceberg_deletes iceberg_deletes a manifest list without partition_spec_id is refused for equality deletes (XX001) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: an equality delete whose manifest list carries no partition_spec_id is not refused +iceberg_deletes iceberg_deletes a never-applicable equality delete is skipped, not validated (5 rows) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: an equality delete at the oldest data file's sequence number is not skipped before validation +iceberg_deletes iceberg_deletes a never-applicable equality delete without equality_ids is skipped (5 rows) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: an equality delete at the oldest data file's sequence number is not skipped before validation +iceberg_deletes iceberg_deletes a null delete value matches only null data values 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a null equality-delete value matches any data value (comparing the null Datum kills the backend) +iceberg_deletes iceberg_deletes a partition spec without its fields key is refused (XX001) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a partition spec without a fields array is not refused (a NULL deref kills the backend) +iceberg_deletes iceberg_deletes a partition-scoped delete with an uncomparable partition value is refused (0A000) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a partition-scoped equality delete with an uncomparable partition tuple is not refused +iceberg_deletes iceberg_deletes a partition-scoped equality delete applies only to its partition 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a partition-scoped equality delete ignores the partition value +iceberg_deletes iceberg_deletes a position delete naming a different data file deletes nothing (5 rows) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a position delete applies to every data file, not only the one it names +iceberg_deletes iceberg_deletes a position delete of ordinal 0 deletes the first row (id 1 gone, 4 remain) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a position delete applies only when strictly newer than the data (> where the spec says >=) +iceberg_deletes iceberg_deletes a position delete with a negative pos is refused (XX001, not cast inert) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a negative position-delete pos is not refused +iceberg_deletes iceberg_deletes a position delete with a null pos is refused (XX001, not silently dropped) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a position-delete row with a null pos is not refused (the n1 || n2 guard skipped) +iceberg_deletes iceberg_deletes a position delete with an inherited (null-entry) sequence number applies 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a position delete applies only when strictly newer than the data (> where the spec says >=) +iceberg_deletes iceberg_deletes a v1 manifest without a sequence_number column reads (default 0, 5 rows) 15;16;17;18;19 never - +iceberg_deletes iceberg_deletes an EXISTING entry with a null sequence number is refused (XX001) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: an EXISTING (status 0) manifest entry with a null sequence number is not refused +iceberg_deletes iceberg_deletes an equality column missing from the data file errors (22023) 15;16;17;18;19 never - +iceberg_deletes iceberg_deletes an equality delete at an equal sequence number does not apply 15;16;17;18;19 2026-10-21 columnar_iceberg.c: an equality delete applies at an equal sequence number (both the pre-skip and the per-file strict < relaxed to <=) +iceberg_deletes iceberg_deletes an equality delete on an unsupported column type is refused (0A000) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a timestamp equality-delete column is mapped to text instead of refused +iceberg_deletes iceberg_deletes an equality delete without equality_ids is refused (XX001) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: an equality delete without equality_ids is not refused +iceberg_deletes iceberg_deletes an equality-delete path outside the table location is refused (22023) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: ice_rebase does not hold a recorded path under the recorded table location +iceberg_deletes iceberg_deletes an equality_ids value beyond int32 refuses the manifest (XX001) 15;16;17;18;19 2026-10-21 columnar_avro.c: an equality_ids value beyond int32 is not refused (truncated) +iceberg_deletes iceberg_deletes backend still up after the audit arms 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a partition spec without a fields array is not refused (a NULL deref kills the backend) +iceberg_deletes iceberg_deletes backend still up after the bad-sequence refusal 15;16;17;18;19 never - +iceberg_deletes iceberg_deletes backend still up after the equality applications 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a null equality-delete value matches any data value (comparing the null Datum kills the backend) +iceberg_deletes iceberg_deletes backend still up after the equality refusals 15;16;17;18;19 never - +iceberg_deletes iceberg_deletes backend still up after the negative-pos refusal 15;16;17;18;19 never - +iceberg_deletes iceberg_deletes backend still up after the null-pos refusal 15;16;17;18;19 2026-10-21 columnar_iceberg.c: abort() in place of the null-pos refusal +iceberg_deletes iceberg_deletes columns beyond equality_ids are ignored for matching 15;16;17;18;19 2026-10-21 columnar_iceberg.c: no equality-delete row ever matches +iceberg_deletes iceberg_deletes equality delete drops matching rows (survivors == oracle) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: no equality-delete row ever matches +iceberg_deletes iceberg_deletes equality deletes apply when the delete column is not projected 15;16;17;18;19 2026-10-21 columnar_iceberg.c: no equality-delete row ever matches +iceberg_deletes iceberg_deletes exactly the two deleted rows were removed (5 - 2 = 3) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a position delete applies only when strictly newer than the data (> where the spec says >=) +iceberg_deletes iceberg_deletes multi-column equality matches all columns (AND, not OR) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: an equality delete compares only its first equality column (multi-column match becomes first-column) +iceberg_deletes iceberg_deletes multiple equality-delete files apply their union 15;16;17;18;19 2026-10-21 columnar_iceberg.c: no equality-delete row ever matches +iceberg_deletes iceberg_deletes position and equality deletes combine 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a position delete applies only when strictly newer than the data (> where the spec says >=) +iceberg_deletes iceberg_deletes position deletes apply under column projection (id only -> 1,3,5) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a position delete applies only when strictly newer than the data (> where the spec says >=) +iceberg_deletes iceberg_deletes position deletes drop the listed rows (surviving == oracle) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a position delete applies only when strictly newer than the data (> where the spec says >=) +iceberg_deletes iceberg_deletes the data file has 5 rows before deletes 15;16;17;18;19 never - +iceberg_deletes iceberg_deletes the deleted ids (2, 4) are gone 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a position delete applies only when strictly newer than the data (> where the spec says >=) +iceberg_deletes iceberg_deletes the null-region row is gone, the non-null rows remain 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a null equality-delete value matches any data value (comparing the null Datum kills the backend) +iceberg_deletes iceberg_deletes the us-partition rows survive an eu-scoped delete 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a partition-scoped equality delete ignores the partition value +iceberg_deletes iceberg_deletes the would-be-deleted ids survive when the delete is older than the data 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a position delete applies whatever its sequence number +iceberg_dv iceberg_dv ...and the refusal names the compressed footer 15;16;17;18;19 2026-10-21 columnar_puffin.c: a compressed Puffin footer (flags bit 0) is not refused +iceberg_dv iceberg_dv ...and the refusal names the format version 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a deletion vector in a format-version 2 table is not refused +iceberg_dv iceberg_dv a DV blob declaring a compression codec is refused (XX001) 15;16;17;18;19 2026-10-21 columnar_puffin.c: a deletion-vector blob may declare a compression codec +iceberg_dv iceberg_dv a DV entry without referenced_data_file is refused (XX001) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a DV without referenced_data_file is not refused (a NULL path reaches strcmp and the backend dies) +iceberg_dv iceberg_dv a DV whose offset+size overflows int64 is refused, not read (XX001) 15;16;17;18;19 2026-10-21 columnar_puffin.c: a DV's offset and size are not bounded by the blob region (a wild read kills the backend) +iceberg_dv iceberg_dv a DV with run containers and a high-bucket position decodes 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a DV applies only when strictly newer than the data (> where the spec says >=) +iceberg_dv iceberg_dv a Puffin path outside the table location is refused (22023) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: ice_open_path does not hold a path under the table location +iceberg_dv iceberg_dv a bitset-container DV decodes 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a DV applies only when strictly newer than the data (> where the spec says >=) +iceberg_dv iceberg_dv a compressed Puffin footer is refused (0A000) 15;16;17;18;19 2026-10-21 columnar_puffin.c: a compressed Puffin footer (flags bit 0) is not refused +iceberg_dv iceberg_dv a corrupted DV CRC is refused (XX001) 15;16;17;18;19 2026-10-21 columnar_puffin.c: a deletion vector's CRC-32 is not checked +iceberg_dv iceberg_dv a deletion vector at an equal sequence number applies (survivors == oracle) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a DV applies only when strictly newer than the data (> where the spec says >=) +iceberg_dv iceberg_dv a deletion vector in a format-version 2 table is refused (0A000) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a deletion vector in a format-version 2 table is not refused +iceberg_dv iceberg_dv a deletion vector older than the data does not apply 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a DV applies whatever its sequence number (dv->seq >= d->seq always true) +iceberg_dv iceberg_dv a manifest/footer offset mismatch is refused (XX001) 15;16;17;18;19 2026-10-21 columnar_puffin.c: the Puffin footer's blob need not match the manifest's offset and length (the !found refusal skipped) +iceberg_dv iceberg_dv a record_count that disagrees with the DV cardinality is refused (XX001) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a manifest record_count that disagrees with the DV cardinality is not refused +iceberg_dv iceberg_dv a run container overflowing its 16-bit range is refused (XX001) 15;16;17;18;19 2026-10-21 columnar_puffin.c: a run container whose start+length passes 0x10000 is not refused +iceberg_dv iceberg_dv a run-optimized DV (cookie 12347) decodes 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a DV applies only when strictly newer than the data (> where the spec says >=) +iceberg_dv iceberg_dv a wrong DV blob magic is refused (XX001) 15;16;17;18;19 2026-10-21 columnar_puffin.c: a deletion vector's magic sequence is not checked +iceberg_dv iceberg_dv an aliased duplicate DV (scheme vs no scheme) is refused (XX001) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: the one-DV-per-data-file check compares paths without stripping the file:// scheme +iceberg_dv iceberg_dv an applicable DV supersedes position-delete files for its data file 15;16;17;18;19 2026-10-21 columnar_iceberg.c: position-delete files apply even when a DV applies (no supersede) +iceberg_dv iceberg_dv backend still up after the DV applications 15;16;17;18;19 never - +iceberg_dv iceberg_dv backend still up after the DV refusals 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a DV without referenced_data_file is not refused (a NULL path reaches strcmp and the backend dies) +iceberg_dv iceberg_dv backend still up after the audit arms 15;16;17;18;19 2026-10-21 columnar_puffin.c: a DV's offset and size are not bounded by the blob region (a wild read kills the backend) +iceberg_dv iceberg_dv supersede is per data file, not global 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a DV applies to every data file, not only the one it references +iceberg_dv iceberg_dv two DV blobs in one Puffin file each apply to their own data file 15;16;17;18;19 2026-10-21 columnar_iceberg.c: a DV applies to every data file, not only the one it references +iceberg_dv iceberg_dv two DV entries for one data file are refused (XX001) 15;16;17;18;19 2026-10-21 columnar_iceberg.c: two DVs for one data file are not refused iceberg_fdw_estimate iceberg_fdw_estimate a filtered scan estimates fewer rows than the whole table (selectivity applied) 15;16;17;18;19 2026-10-21 the Iceberg FDW sets rows directly, so the scan's selectivity never applies iceberg_fdw_estimate iceberg_fdw_estimate backend alive 15;16;17;18;19 never - iceberg_fdw_estimate iceberg_fdw_estimate the FDW estimates a positive row count from the manifests 15;16;17;18;19 never - @@ -2636,24 +3116,47 @@ iceberg_objstore iceberg_objstore name mapping reads an id-less file over s3 (5 iceberg_objstore iceberg_objstore position deletes read over s3 (surviving == filesystem oracle) 15;16;17;18;19 2026-10-21 columnar_objstore_module.c: os_check_endpoint_allowed refuses every endpoint, the listed one included iceberg_objstore iceberg_objstore premise: sigv4 fixture server is up 15;16;17;18;19 never - iceberg_objstore iceberg_objstore the position-deleted ids are gone over s3 15;16;17;18;19 2026-10-21 columnar_objstore_module.c: os_check_endpoint_allowed refuses every endpoint, the listed one included +iceberg_rest iceberg_rest a catalog endpoint off the allow-list is refused (42501) 15;16;17;18;19 2026-10-21 objstore/columnar_objstore_module.c: every address reads as link-local and the allow-list is gone +iceberg_rest iceberg_rest a link-local catalog host is refused even if allow-listed (42501) 15;16;17;18;19 2026-10-21 objstore/columnar_objstore_module.c: the allow-list refuses every endpoint and the link-local refusal is gone +iceberg_rest iceberg_rest a non-object catalog config is refused, not walked (XX001) 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: a non-object /v1/config body is not refused +iceberg_rest iceberg_rest a wrong bearer token is refused (28000) 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: a catalog 401/403 is not mapped to 28000 +iceberg_rest iceberg_rest an anonymous request to a token-protected catalog is refused (28000) 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: a catalog 401/403 is not mapped to 28000 +iceberg_rest iceberg_rest an over-cap catalog response is refused (54000) 15;16;17;18;19 2026-10-21 objstore/columnar_objstore_module.c: os_slurp_body does not refuse an advertised length over the cap +iceberg_rest iceberg_rest an unknown table surfaces a clean not-found (42P01) 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: a catalog 404 is not mapped to 42P01 +iceberg_rest iceberg_rest backend still up after the REST refusals 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: the backend aborts on a catalog 404 (abort() before the 42P01 refusal) +iceberg_rest iceberg_rest loadTable resolves the current metadata-location 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: no Authorization header is ever sent +iceberg_rest iceberg_rest premise: rest fixture server is up 15;16;17;18;19 never - +iceberg_rest iceberg_rest premise: the PG server log holds lines to search 15;16;17;18;19 never - +iceberg_rest iceberg_rest the loadTable request carried an Authorization header 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: the bearer token also travels in the request URL's query string +iceberg_rest iceberg_rest the token never appears in the server (PG) log 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: rest_get_json logs the bearer token (elog LOG) +iceberg_rest iceberg_rest the token value never appears in the catalog request log 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: the bearer token also travels in the request URL's query string +iceberg_rest_scan iceberg_rest_scan backend still up after the REST read 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: the backend aborts on a catalog 404 (abort() before the 42P01 refusal) +iceberg_rest_scan iceberg_rest_scan iceberg_rest_namespaces lists the catalog's namespaces 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: iceberg_rest_namespaces drops the catalog's last namespace +iceberg_rest_scan iceberg_rest_scan iceberg_rest_scan matches iceberg_scan of the same table (same oracle) 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: no Authorization header is ever sent +iceberg_rest_scan iceberg_rest_scan iceberg_rest_scan of an unknown table is a clean not-found (42P01) 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: a catalog 404 is not mapped to 42P01 +iceberg_rest_scan iceberg_rest_scan iceberg_rest_scan projects and reorders by field id (amount, id) 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: no Authorization header is ever sent +iceberg_rest_scan iceberg_rest_scan iceberg_rest_scan returns the five current-snapshot rows 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: no Authorization header is ever sent +iceberg_rest_scan iceberg_rest_scan iceberg_rest_tables lists a namespace's tables 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: no Authorization header is ever sent +iceberg_rest_scan iceberg_rest_scan premise: rest fixture server is up 15;16;17;18;19 never - +iceberg_rest_scan iceberg_rest_scan the catalog reports the relocated metadata location 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: no Authorization header is ever sent iceberg_rest_server iceberg_rest_server a catalog_uri option on the USER MAPPING is rejected (HV00D) 15;16;17;18;19 never - -iceberg_rest_server iceberg_rest_server a half OAuth2 credential (id, no secret) is refused before any request (28000) 15;16;17;18;19 never - -iceberg_rest_server iceberg_rest_server a link-local oauth_token_uri is refused by the allow-list (42501) 15;16;17;18;19 never - +iceberg_rest_server iceberg_rest_server a half OAuth2 credential (id, no secret) is refused before any request (28000) 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: a half OAuth2 credential (id, no secret) is not refused, so the mint runs with a NULL secret and the backend dies +iceberg_rest_server iceberg_rest_server a link-local oauth_token_uri is refused by the allow-list (42501) 15;16;17;18;19 2026-10-21 objstore/columnar_objstore_module.c: an endpoint off pgcolumnar.objstore_allowed_endpoints is not refused iceberg_rest_server iceberg_rest_server a non-superuser with no token mapping is refused (28000) despite the environment token, before any request 15;16;17;18;19 2026-10-15 the REST resolver letting every role borrow the environment token iceberg_rest_server iceberg_rest_server a server with an OAuth2 mapping reads by a minted bearer 15;16;17;18;19 never - iceberg_rest_server iceberg_rest_server a token option on the SERVER is rejected (HV00D) 15;16;17;18;19 never - iceberg_rest_server iceberg_rest_server a warehouse option on the SERVER is accepted (not HV00D) 15;16;17;18;19 never - -iceberg_rest_server iceberg_rest_server a warehouse-configured server resolves a table location 15;16;17;18;19 never - +iceberg_rest_server iceberg_rest_server a warehouse-configured server resolves a table location 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: the USER MAPPING token option is ignored iceberg_rest_server iceberg_rest_server a wrong OAuth2 client secret is refused (28000) 15;16;17;18;19 never - -iceberg_rest_server iceberg_rest_server a wrong mapping token is refused (28000) 15;16;17;18;19 never - +iceberg_rest_server iceberg_rest_server a wrong mapping token is refused (28000) 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: abort() in place of the catalog's 401/403 refusal iceberg_rest_server iceberg_rest_server an oauth option on the SERVER is rejected (HV00D) 15;16;17;18;19 never - -iceberg_rest_server iceberg_rest_server an off-allow-list oauth_token_uri host is refused (42501) 15;16;17;18;19 never - +iceberg_rest_server iceberg_rest_server an off-allow-list oauth_token_uri host is refused (42501) 15;16;17;18;19 2026-10-21 objstore/columnar_objstore_module.c: an endpoint off pgcolumnar.objstore_allowed_endpoints is not refused iceberg_rest_server iceberg_rest_server and it cannot mark its own mapping credentials_required false (42501) 15;16;17;18;19 2026-10-15 the catalog validator letting a non-superuser set credentials_required false -iceberg_rest_server iceberg_rest_server backend still up 15;16;17;18;19 never - +iceberg_rest_server iceberg_rest_server backend still up 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: a half OAuth2 credential (id, no secret) is not refused, so the mint runs with a NULL secret and the backend dies iceberg_rest_server iceberg_rest_server control: with credentials_required false set by a superuser, the same role reads by the environment token 15;16;17;18;19 never - -iceberg_rest_server iceberg_rest_server iceberg_rest_table_location('cat',...) reads via the mapping token 15;16;17;18;19 never - +iceberg_rest_server iceberg_rest_server iceberg_rest_table_location('cat',...) reads via the mapping token 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: the USER MAPPING token option is ignored iceberg_rest_server iceberg_rest_server loadTable then carried the minted bearer (Authorization present) 15;16;17;18;19 never - -iceberg_rest_server iceberg_rest_server no mapping and no env token is refused, not bypassed (28000) 15;16;17;18;19 never - +iceberg_rest_server iceberg_rest_server no mapping and no env token is refused, not bypassed (28000) 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: abort() in place of the catalog's 401/403 refusal iceberg_rest_server iceberg_rest_server oauth options are accepted on a USER MAPPING (no error) 15;16;17;18;19 never - iceberg_rest_server iceberg_rest_server premise: rest fixture up 15;16;17;18;19 never - iceberg_rest_server iceberg_rest_server premise: the PG server log holds lines to search, for the client secret 15;16;17;18;19 never - @@ -2663,8 +3166,18 @@ iceberg_rest_server iceberg_rest_server the catalog-URI form still works with th iceberg_rest_server iceberg_rest_server the client secret never appears in the PG server log 15;16;17;18;19 never - iceberg_rest_server iceberg_rest_server the client secret never appears in the fixture request log 15;16;17;18;19 never - iceberg_rest_server iceberg_rest_server the mapping token never appears in the PG server log 15;16;17;18;19 never - -iceberg_rest_server iceberg_rest_server the request carried an Authorization header 15;16;17;18;19 never - -iceberg_rest_server iceberg_rest_server the warehouse was sent on GET /v1/config (?warehouse=wh1) 15;16;17;18;19 never - +iceberg_rest_server iceberg_rest_server the request carried an Authorization header 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: the USER MAPPING token option is ignored +iceberg_rest_server iceberg_rest_server the warehouse was sent on GET /v1/config (?warehouse=wh1) 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: the warehouse server option is never sent on GET /v1/config +iceberg_rest_vended iceberg_rest_vended a table vending no creds still reads with ambient (cfg NULL path) 15;16;17;18;19 never - +iceberg_rest_vended iceberg_rest_vended a wrong vended secret is refused by S3, not bypassed (28000) 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: vended credentials are never built, so every read falls back to ambient +iceberg_rest_vended iceberg_rest_vended backend still up after the vended-credential arms 15;16;17;18;19 2026-10-21 objstore/columnar_objstore_module.c: abort() in place of the HTTP 403 refusal +iceberg_rest_vended iceberg_rest_vended iceberg_rest_scan reads via vended credentials (no ambient creds set) 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: vended credentials are never built, so every read falls back to ambient +iceberg_rest_vended iceberg_rest_vended premise: the PG server log holds lines to search 15;16;17;18;19 never - +iceberg_rest_vended iceberg_rest_vended premise: the SigV4 S3 fixture is up 15;16;17;18;19 never - +iceberg_rest_vended iceberg_rest_vended storage-credentials longest-prefix match selects the right creds 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: rest_vended_cfg never reads the storage-credentials array and signs with the flat config +iceberg_rest_vended iceberg_rest_vended the vended secret never appears in the PG server log 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: the vended secret is written to the server log +iceberg_rest_vended iceberg_rest_vended the vended secret never appears in the S3 request log 15;16;17;18;19 never - +iceberg_rest_vended iceberg_rest_vended vended creds for an off-allow-list endpoint are refused (42501) 15;16;17;18;19 2026-10-21 columnar_iceberg_rest.c: vended credentials are never built, so every read falls back to ambient iceberg_scan iceberg_scan a column renamed in the schema still reads by field id (amt2 = amount) 15;16;17;18;19 2026-10-21 iceberg_scan resolves output columns by position rather than field id iceberg_scan iceberg_scan a pg_read_server_files member who is not superuser may scan 15;16;17;18;19 2026-10-21 iceberg_scan admits only superusers, refusing every pg_read_server_files member iceberg_scan iceberg_scan an output column not in the schema is refused (42703) 15;16;17;18;19 2026-10-21 iceberg_scan does not refuse an output column the schema lacks @@ -2876,6 +3389,74 @@ local_open_race_free local_open_race_free the partition coordinator never stats local_open_race_free local_open_race_free the partition coordinator opens O_NONBLOCK (a FIFO cannot block its open) 15;16;17;18;19 2026-10-21 the partition coordinator opens without O_NONBLOCK local_open_race_free local_open_race_free the race-free opener exists 15;16;17;18;19 never - local_open_race_free local_open_race_free the racy stat-before-open helper is gone 15;16;17;18;19 2026-10-21 the racy PgColumnarRejectNonRegularFile helper comes back +logical_decoding_cdc_recipe logical_decoding_cdc_recipe CONTROL: the columnar table itself still emits nothing 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe a DELETE captures the OLD values, which are gone from the table 15;16;17;18;19 2026-10-21 docs/user-guide.md: the recipe's trigger fires on INSERT OR UPDATE only, not DELETE +logical_decoding_cdc_recipe logical_decoding_cdc_recipe a bulk load writes exactly one capture row per loaded row 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe a rolled-back transaction captures nothing 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe a row trigger on a COLUMNAR table fires once per inserted row 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe an UPDATE captures exactly one row 15;16;17;18;19 2026-10-21 columnar_tableam.c: pgcolumnar_tuple_update leaves the old row version live +logical_decoding_cdc_recipe logical_decoding_cdc_recipe and it is recorded as UPDATE with the NEW value 15;16;17;18;19 2026-10-21 columnar_tableam.c: pgcolumnar_tuple_update writes its new row version with every column null +logical_decoding_cdc_recipe logical_decoding_cdc_recipe and it saw the same values the table holds 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe and leaves no row in the columnar table either 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe and the DELETE as a DELETE 15;16;17;18;19 2026-10-21 docs/user-guide.md: the recipe's trigger fires on INSERT OR UPDATE only, not DELETE +logical_decoding_cdc_recipe logical_decoding_cdc_recipe and the rolled-back row is not in the stream 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe and the row really is gone from the columnar table 15;16;17;18;19 2026-10-21 columnar_tableam.c: pgcolumnar_tuple_delete marks nothing deleted +logical_decoding_cdc_recipe logical_decoding_cdc_recipe inside the transaction the row was captured and is in the columnar table, before the rollback 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe nor the storage's internal INSERT 15;16;17;18;19 2026-10-21 docs/user-guide.md: the recipe's capture records every non-DELETE change as an INSERT +logical_decoding_cdc_recipe logical_decoding_cdc_recipe one decoded capture row per captured change 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe premise: FOR ALL TABLES really does include pgcolumnar's internal tables 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe premise: and it is the whole block, not a fragment 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe premise: events is a COLUMNAR table 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe premise: events_cdc is a HEAP table 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe premise: the extracted recipe contains [CREATE FUNCTION events_capture] 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe premise: the extracted recipe contains [CREATE TABLE events] 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe premise: the extracted recipe contains [CREATE TABLE events_cdc] 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe premise: the extracted recipe contains [CREATE TRIGGER events_cdc_t] 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe premise: the extracted recipe contains [OLD.id] 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe premise: the extracted recipe contains [TG_OP = 'DELETE'] 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe premise: the extracted recipe contains [USING pgcolumnar] 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe premise: the extraction is not empty 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe premise: the guide is where this suite expects it 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe premise: the identical insert COMMITTED does capture, so the arm above is real 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe premise: the same matcher finds the INSERTs, so a 1 above is a real match 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe premise: the server is running at wal_level=logical 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe premise: the slot decoded something at all 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe premise: the slot exists and is logical 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe the capture table IS carried by the slot 15;16;17;18;19 never - +logical_decoding_cdc_recipe logical_decoding_cdc_recipe the decoded stream carries the UPDATE as an UPDATE 15;16;17;18;19 2026-10-21 columnar_tableam.c: pgcolumnar_tuple_update leaves the old row version live +logical_decoding_cdc_recipe logical_decoding_cdc_recipe the update did NOT leak the storage's internal DELETE 15;16;17;18;19 never - +logical_decoding_source logical_decoding_source 100x the rows does not give 100x the internal records 15;16;17;18;19 never - +logical_decoding_source logical_decoding_source CONTROL: the heap table's 50 inserts ARE decoded 15;16;17;18;19 never - +logical_decoding_source logical_decoding_source CONTROL: the second slot decodes the heap table's 5000 inserts 15;16;17;18;19 never - +logical_decoding_source logical_decoding_source a columnar table emits no decodable change 15;16;17;18;19 never - +logical_decoding_source logical_decoding_source although the rows are there 15;16;17;18;19 never - +logical_decoding_source logical_decoding_source and it too emits no decodable change of its own 15;16;17;18;19 never - +logical_decoding_source logical_decoding_source premise: both tables hold the same 50 rows 15;16;17;18;19 never - +logical_decoding_source logical_decoding_source premise: the 100x table really holds 100x the rows 15;16;17;18;19 never - +logical_decoding_source logical_decoding_source premise: the server is running at wal_level=logical 15;16;17;18;19 never - +logical_decoding_source logical_decoding_source premise: the slot decoded something at all 15;16;17;18;19 never - +logical_decoding_source logical_decoding_source premise: the slot exists and is logical 15;16;17;18;19 never - +logical_decoding_source logical_decoding_source the slot carries pgcolumnar's internal catalog instead 15;16;17;18;19 never - +logical_subscriber logical_subscriber an ordinary SELECT ... FOR NO KEY UPDATE is not told about replication 15;16;17;18;19 2026-10-21 columnar_tableam.c: pgcolumnar_tuple_lock raises the replication error for every row lock, not only the apply worker's +logical_subscriber logical_subscriber an ordinary SELECT ... FOR NO KEY UPDATE keeps the generic message 15;16;17;18;19 2026-10-21 columnar_tableam.c: pgcolumnar_tuple_lock raises the replication error for every row lock, not only the apply worker's +logical_subscriber logical_subscriber an ordinary SELECT ... FOR SHARE is not told about replication 15;16;17;18;19 2026-10-21 columnar_tableam.c: pgcolumnar_tuple_lock raises the replication error for every row lock, not only the apply worker's +logical_subscriber logical_subscriber an ordinary SELECT ... FOR SHARE keeps the generic message 15;16;17;18;19 2026-10-21 columnar_tableam.c: pgcolumnar_tuple_lock raises the replication error for every row lock, not only the apply worker's +logical_subscriber logical_subscriber an ordinary SELECT ... FOR UPDATE is not told about replication 15;16;17;18;19 2026-10-21 columnar_tableam.c: pgcolumnar_tuple_lock raises the replication error for every row lock, not only the apply worker's +logical_subscriber logical_subscriber an ordinary SELECT ... FOR UPDATE keeps the generic message 15;16;17;18;19 2026-10-21 columnar_tableam.c: pgcolumnar_tuple_lock raises the replication error for every row lock, not only the apply worker's +logical_subscriber logical_subscriber insert-only: initial sync copied every row 15;16;17;18;19 2026-10-21 columnar_tableam.c: pgcolumnar_multi_insert raises in a logical worker, so initial sync's COPY fails +logical_subscriber logical_subscriber insert-only: no apply errors 15;16;17;18;19 2026-10-21 columnar_tableam.c: pgcolumnar_tuple_insert raises in a logical worker, so streamed INSERTs fail +logical_subscriber logical_subscriber insert-only: streamed INSERTs applied 15;16;17;18;19 2026-10-21 columnar_tableam.c: pgcolumnar_tuple_insert raises in a logical worker, so streamed INSERTs fail +logical_subscriber logical_subscriber insert-only: the UPDATE is skipped by design, not applied 15;16;17;18;19 2026-10-21 columnar_tableam.c: pgcolumnar_tuple_insert raises in a logical worker, so streamed INSERTs fail +logical_subscriber logical_subscriber it retries rather than giving up, so the subscription is wedged 15;16;17;18;19 2026-10-21 columnar_tableam.c: pgcolumnar_tuple_lock never takes the IsLogicalWorker() branch, so the apply worker gets the generic row-locking error +logical_subscriber logical_subscriber premise: the lock fixture really is columnar 15;16;17;18;19 never - +logical_subscriber logical_subscriber premise: the publisher has logical decoding on 15;16;17;18;19 never - +logical_subscriber logical_subscriber premise: the subscriber cluster answers 15;16;17;18;19 never - +logical_subscriber logical_subscriber premise: the subscriber loaded pgcolumnar 15;16;17;18;19 never - +logical_subscriber logical_subscriber the error does NOT say only 'row locking is not supported' 15;16;17;18;19 2026-10-21 columnar_tableam.c: pgcolumnar_tuple_lock never takes the IsLogicalWorker() branch, so the apply worker gets the generic row-locking error +logical_subscriber logical_subscriber the error names logical replication, not row locking 15;16;17;18;19 2026-10-21 columnar_tableam.c: pgcolumnar_tuple_lock never takes the IsLogicalWorker() branch, so the apply worker gets the generic row-locking error +logical_subscriber logical_subscriber the hint names the insert-only publication 15;16;17;18;19 2026-10-21 columnar_tableam.c: the apply worker's hint stops naming CREATE PUBLICATION ... WITH (publish = 'insert') +logical_subscriber logical_subscriber the second subscription synced before the UPDATE 15;16;17;18;19 2026-10-21 columnar_tableam.c: pgcolumnar_multi_insert raises in a logical worker, so initial sync's COPY fails +logical_subscriber logical_subscriber the wedge is real: the row is still unchanged on the subscriber 15;16;17;18;19 2026-10-21 columnar_tableam.c: pgcolumnar_multi_insert raises in a logical worker, so initial sync's COPY fails maintenance_crash maintenance_crash after a crash with compact committed, the index reads the rows the scan reads 15;16;17;18;19 2026-10-16 pgcolumnar_index_fetch_tuple finds no row (columnar_tableam.c) maintenance_crash maintenance_crash after a crash with compact committed, the table is as compact left it, with the same rows 15;16;17;18;19 never - maintenance_crash maintenance_crash after a crash with compact uncommitted, the index reads the rows the scan reads 15;16;17;18;19 2026-10-16 pgcolumnar_index_fetch_tuple finds no row (columnar_tableam.c) diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 2970fed4d..87b7fef53 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 102 +suites_not_covered 85 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -5439,4 +5439,311 @@ suites_not_covered 102 # never 3883 + 1 = 3884 # dated 2282 + 4 = 2286 # 3884 + 2286 == 6170 -checks_never_observed_red 3884 +# #1341, iceberg_dv seeded, with its twin (OffgridwithJD): 3884 -> 3885, and +# suites_not_covered 102 -> 101. The suite had no rows. It enters with 26 checks from +# five-major runs (all green on each of 15-19, shell and twin alike), after two fixes. +# "a manifest/footer offset mismatch is refused (XX001)" read the SQLSTATE alone, and +# several guards here raise XX001: with the offset match removed a later XX001 refused +# the same file, and main's shell passed 26/26. Every refusal now reads its own guard's +# message. And the three "backend still up" checks read SELECT 1, which answers again +# after crash recovery: with the referenced_data_file check removed a backend died on +# the arm, and main passed the survival check. They now also read the server log's +# count of killed backends, and each refusal waits for the server first, so a crash +# reddens its own arm and its section's survival check and nothing else. 25 are dated by +# seventeen mutations, each reddening the same names in both twins. 1 arrives never: +# the survival check after the DV applications, which no mutation crashed. +# +# ledger rows 6170 + 26 = 6196 +# never 3884 + 1 = 3885 +# dated 2286 + 25 = 2311 +# 3885 + 2311 == 6196 +# +# #1341, logical_decoding_source seeded, with its twin (OffgridwithJD): 3885 -> 3897, +# and suites_not_covered 101 -> 100. The suite had no rows. It enters with its 12 +# checks, all green on each of 15-19 in shell and twin alike. The missing-plugin name +# pgc_skip records is now a literal both twins share, but it is a skip, not a check, so +# it has no row: CI has test_decoding and never emits it. After one fix. +# The second slot had no heap control: created after the 5000-row insert, it decoded +# nothing, and "it too emits no decodable change" and "100x the rows does not give 100x +# the internal records" (0 < anything) both passed, 11/11 on main. A heap table now writes +# through that slot in the same window and must decode its 5000 inserts, and the ratio +# check refuses a slot that carried no internal records. Under that harness mutation both +# are red in both twins. No source mutation reddens a check: the claim is that pgcolumnar +# writes nothing logical decoding can read, and a default chunk group of 100 rows still +# keeps the internal records following groups (114 for 50 groups, under the 10x bound). +# 0 are dated. 12 arrive never. +# +# ledger rows 6196 + 12 = 6208 +# never 3885 + 12 = 3897 +# dated 2311 + 0 = 2311 +# 3897 + 2311 == 6208 +# +# #1341, iceberg_deletes seeded, with its twin (OffgridwithJD): 3897 -> 3903, and +# suites_not_covered 100 -> 99. The suite had no rows. It enters with 49 checks from +# five-major runs (all green on each of 15-19, shell and twin alike), after one fix. +# The six "backend still up" checks read SELECT 1, which answers again after crash +# recovery: with abort() in place of the null-pos refusal, or the partition spec's +# fields guard removed (a segfault), main's shell passed the survival check once +# recovery had finished. They now also read the server log's count of killed +# backends, and every reading waits for the server first, so a crash reddens its own +# arm and its section's survival check, not the two arms after it. Every refusal now +# also reads its own guard's message (XX001 and 22023 are shared), and the oracle +# comparisons refuse an empty side. 43 are dated by twenty-three mutations, each reddening +# the same names in both twins. The equal-sequence equality check is dated only by +# relaxing both of its layered guards: the per-file strict < alone passed, because the +# pre-skip at the oldest data file's sequence number already drops that delete. +# 6 arrive never. +# +# ledger rows 6208 + 49 = 6257 +# never 3897 + 6 = 3903 +# dated 2311 + 43 = 2354 +# 3903 + 2354 == 6257 +# +# #1341, logical_subscriber seeded, with its twin (OffgridwithJD): 3903 -> 3907, +# and suites_not_covered 99 -> 98. The suite had no rows. It enters with 20 checks +# from five-major runs (all green on each of 15-19, shell and twin alike), after four fixes. +# With the insert-only subscription disabled before its UPDATE, main's shell passed 20/20: +# "skipped by design" and "no apply errors" read after a fixed sleep, and nothing applied +# is nothing applied wrongly. An INSERT after the UPDATE must now arrive first. With no +# UPDATE sent, "does NOT say only 'row locking'" and "the wedge is real" passed; the first +# now needs the replication error in the log, the second reads publisher -> subscriber. +# With the lock statements failing on a missing table, the three "is not told about +# replication" checks passed; they now need the generic error first. And the second +# subscription's sync check lost its premise prefix: COPY failing in a logical worker +# reddens it. The lock loop is unrolled so the twin grader reads each name. +# 16 are dated by five mutations, each reddening the same names in both twins. 4 +# arrive never. +# +# ledger rows 6257 + 20 = 6277 +# never 3903 + 4 = 3907 +# dated 2354 + 16 = 2370 +# 3907 + 2370 == 6277 +# +# #1341, logical_decoding_cdc_recipe seeded, with its twin (OffgridwithJD): 3907 -> 3936, +# and suites_not_covered 98 -> 97. The suite had no rows. It enters with its 36 checks from five-major +# runs (all green on each of 15-19, shell and twin alike), after two fixes. The +# missing-plugin name pgc_skip records is a skip, not a check, so it has no row: CI has +# test_decoding and never emits it. The rollback arm read the capture count +# and the row after ROLLBACK only, so a rolled-back INSERT that never ran passed both arms: +# with it made a duplicate key, main passed 35/35. A reading inside the transaction, before +# the rollback, now requires the row captured and present. And "it saw the same values the +# table holds" compared two readings with check, so with the server gone "" = "" passed; it +# is check_text. The recipe-contains premises are unrolled into literal names. Three +# table-AM mutations and two mutations of the guide's own recipe date checks, each red by the +# same names in both twins. 7 are dated. 29 arrive never. +# +# ledger rows 6277 + 36 = 6313 +# never 3907 + 29 = 3936 +# dated 2370 + 7 = 2377 +# 3936 + 2377 == 6313 +# +# #1341, iceberg_rest seeded, with its twin (OffgridwithJD): 3936 -> 3938, and +# suites_not_covered 97 -> 96. The suite had no rows. It enters with 14 checks from +# five-major runs (all green on each of 15-19, shell and twin alike), after two fixes. +# The 42501 arms read the SQLSTATE alone, and two guards raise 42501: with the allow-list +# refusing everything and the link-local refusal gone, "a link-local catalog host is +# refused" passed on the allow-list's 42501, and with every address read as link-local +# and the allow-list gone, "a catalog endpoint off the allow-list is refused" passed on +# the link-local guard's. Every refusal now reads its own guard's message. And "backend +# still up after the REST refusals" read SELECT 1, which answers again after crash +# recovery: with the backend aborting on the catalog's 404, main passed it. It now also +# reads the server log's count of killed backends. 12 are dated by eleven mutations, +# each reddening the same names in both twins. 2 arrive never: the two premises. +# +# ledger rows 6313 + 14 = 6327 +# never 3936 + 2 = 3938 +# dated 2377 + 12 = 2389 +# 3938 + 2389 == 6327 +# +# #1341, iceberg_rest_scan seeded, with its twin (OffgridwithJD): 3938 -> 3939, and +# suites_not_covered 96 -> 95. The suite had no rows. It enters with 9 checks from +# five-major runs (all green on each of 15-19, shell and twin alike), after one fix. +# "backend still up after the REST read" read SELECT 1, which answers again after crash +# recovery: with the backend aborting on the catalog's 404 and the probe 3s later, main +# passed it. It now also reads the server log's count of killed backends, and the +# not-found arm reads the catalog's own 404 message. 8 are dated by five mutations, +# each reddening the same names in both twins. 1 arrives never: the premise. +# +# ledger rows 6327 + 9 = 6336 +# never 3938 + 1 = 3939 +# dated 2389 + 8 = 2397 +# 3939 + 2397 == 6336 +# +# #1341, iceberg_rest_vended seeded, with its twin (OffgridwithJD): 3939 -> 3943, and +# suites_not_covered 95 -> 94. The suite had no rows. It enters with 10 checks from +# five-major runs (all green on each of 15-19, shell and twin alike), after three fixes. +# The longest-prefix arm passed with the storage-credentials array never read: the +# catalog's flat config carried the same right credentials, so the fixture's flat config +# is now wrong too. The wrong-secret arm read 28000 alone, which "no credentials" also +# raises: with vended credentials never built, it passed; it now needs the 403 message. +# The survival check read SELECT 1: with abort() in place of the 403 refusal it passed; +# it now also reads the server log's count of killed backends. 6 are dated by six +# mutations, each reddening the same names in both twins. 4 arrive never. +# +# ledger rows 6336 + 10 = 6346 +# never 3939 + 4 = 3943 +# dated 2397 + 6 = 2403 +# 3943 + 2403 == 6346 +# +# #1341, iceberg_rest_server's twin (OffgridwithJD): 3943 -> 3933. The suite +# already had its 29 rows; none is added. Four fixes in the shell, mirrored in +# the twin. "backend still up" read SELECT 1: with the half-credential guard removed +# (the backend segfaults) or abort() in place of the 401 refusal, it passed; it now +# also reads the server log's count of killed backends, and every reading waits for +# the server first. The two "accepted" arms read an empty SQLSTATE, which a stopped +# server also gives, so with the server stopped both passed; a statement that ran now +# reads "ok". "refused by the allow-list (42501)" read the SQLSTATE alone, and the +# link-local guard raises 42501 too: with the allow-list removed it passed. Every +# refusal now needs its own message, and the half-credential arm also counts the +# fixture's requests. 10 more rows are dated by five mutations, each reddening the +# same names in both twins. +# +# ledger rows 6346 + 0 = 6346 +# never 3943 - 10 = 3933 +# dated 2403 + 10 = 2413 +# 3933 + 2413 == 6346 +# +# #1341, hilbert_curve seeded, with its twin (OffgridwithJD): 3933 -> 3949, and +# suites_not_covered 94 -> 93. The suite had no rows. It enters with its 184 +# checks from five-major runs, 184/184 in the shell and in the twin on each of 15-19. +# Seventeen mutations of src/columnar_curve.c each reddened checks in the shell on +# main, and between them every arm but the six premises and the ten INSTRUMENT +# controls, which link nothing from src/ by design. Two shell changes. The battery +# now fills every key buffer with 0xa5 before the pack writes it: under a memset one +# byte short, which goldens went red depended on what the previous call had left in +# the buffer (121 red on main, against a stable 124), and the first use of a buffer +# read uninitialized memory. And the two list-taking helpers are written out as loops +# at their call sites, so compare_to_bash reads the suite instead of refusing it, +# which it did as the tree's one refused suite. The seventeen date 168 rows, each +# reddening the same names in the shell and the twin. 16 arrive never: the six +# premises and the ten controls. +# +# ledger rows 6346 + 184 = 6530 +# never 3933 + 16 = 3949 +# dated 2413 + 168 = 2581 +# 3949 + 2581 == 6530 +# +# #1341, bench_guards seeded, with its twin (OffgridwithJD): 3949 -> 3991, and +# suites_not_covered 93 -> 92. The suite had no rows: it kept its own tally +# and emitted no RESULT records. It now goes through lib.sh's check, with no cluster, and +# enters with 91 checks from five-major runs (all green on each of 15-19, shell and twin +# alike). Its per-runner checks grepped each bench runner for words, and the header comment +# and the cannot-tell block's exit 1 satisfied them: with the assert refusal warning past +# it, with the empty-capture refusal removed, with every build refused, or with the +# override ignored, main's suite passed. Each runner's own guard is now run against a +# stand-in pg_config, and a sixth check per runner wants a plain build let through. The +# verdict-call check counted a comment that kept the call's shape, and now reads code only. +# The ratio premise matched rr1= as r1=, and now matches a whole name. Two premises that a +# one-line change to bench/cb_guards.sh reddens lost the prefix. 49 are dated by +# twenty-two mutations, each reddening the same names in both twins. 42 arrive never. +# +# ledger rows 6530 + 91 = 6621 +# never 3949 + 42 = 3991 +# dated 2581 + 49 = 2630 +# 3991 + 2630 == 6621 +# +# #1341, docs_style seeded, with its twin (OffgridwithJD): 3991 -> 4008, and +# suites_not_covered 92 -> 91. The suite kept a private tally and emitted no RESULT +# records, so nothing could be seeded for it. It now records through lib.sh's check() and +# pgc_summary, as wal_envelope.sh does, with no cluster. Four arms read text a reader never +# sees: with the 1024 floor or the join key kept only in an HTML comment, a stale VERSION +# citation beside a hidden right one, or the roadmap nav entry commented out with #, main +# passed all 58 checks. Each arm now reads the page without its comments. The per-version +# changelog checks put the version in their names and the META.json pair came from a loop, +# so neither could be graded: three fixed names and two literal checks replace them. 36 +# are dated by documentation mutations, each reddening the same name in both twins. 17 +# arrive never: the premises, and the checker's own examined-count control. +# +# ledger rows 6621 + 53 = 6674 +# never 3991 + 17 = 4008 +# dated 2630 + 36 = 2666 +# 4008 + 2666 == 6674 +# +# #1341, audit seeded, with its twin (OffgridwithJD): 4008 -> 4018, and +# suites_not_covered 91 -> 90. The suite had no rows. It enters with 35 rows: 34 checks +# on 17-19, of which the four partitioned-parent checks run only there, and on 15 and 16 +# the one skip that arm records instead. From five-major runs (all green on each of 15-19, +# shell and twin alike), after three fixes. The refusal arms +# accepted any error: with set_options failing at entry for a reason that is none of its +# guards, all five passed; each now requires its guard's own message. The parallel CREATE +# INDEX's outcome was discarded: with a build participant raising an error, no index existed +# and both the leak check and the indexed-once count passed (the count by a Seq Scan); the +# build is now a check, the count is read only through the index, and the plan check is no +# longer a premise. And the abort notice never printed, because a second EXIT trap replaced +# it. Eleven mutations date checks, each red by the same names in both twins. 25 are +# dated. 10 arrive never, the skip row among them. +# +# ledger rows 6674 + 35 = 6709 +# never 4008 + 10 = 4018 +# dated 2666 + 25 = 2691 +# 4018 + 2691 == 6709 +# #1341, fuzz seeded, with its twin (OffgridwithJD): 4018 -> 4028, and +# suites_not_covered 90 -> 89. The suite had no rows. It enters with its 12 +# checks from five-major runs, green on each of 15-19 in the shell and the twin alike. +# The shell drew its runs from bash's $RANDOM, which changed algorithm in 5.1, and +# named each check after its iteration, so the names moved with the seed and no twin +# could draw the same run. It now draws from its own 31-bit LCG, the twin transcribes +# it, and each property is asserted once over the run. set_options' result was +# thrown away: with the chunk-group limit dropped, main passed 250/250. Its options +# now read back, which optdrop reddens, and gtmin reddens the compound filter. +# 2 are dated, each mutation reddening the same names in both twins. 10 arrive never. +# +# ledger rows 6709 + 12 = 6721 +# never 4018 + 10 = 4028 +# dated 2691 + 2 = 2693 +# 4028 + 2693 == 6721 +# +# #1341, fuzz_avro seeded, with its twin (OffgridwithJD): 4028 -> 4033, and +# suites_not_covered 89 -> 88. The suite had no rows. It enters with its 8 +# checks from five-major runs, green on each of 15-19 in the shell and the twin alike. +# A generator that wrote nothing left the previous file in place, read as an +# ERROR or a clean accept, and main passed 7/7. The mutant is now removed before it +# is generated and every iteration must write one, and a file the backend could +# not open does not count as reaching the decoder. +# 3 are dated, each mutation reddening the same names in both twins. 5 arrive never. +# +# ledger rows 6721 + 8 = 6729 +# never 4028 + 5 = 4033 +# dated 2693 + 3 = 2696 +# 4033 + 2696 == 6729 +# +# #1341, fuzz_listing seeded, with its twin (OffgridwithJD): 4033 -> 4039, and +# suites_not_covered 88 -> 87. The suite had no rows. It enters with its 8 +# checks from five-major runs, green on each of 15-19 in the shell and the twin alike. +# With no override the fixture serves the real listing, so a generator that wrote +# nothing read as a clean accept 40 times out of 40 and main passed 7/7. The override +# is now removed before each mutant is generated, and every iteration must write one. +# 2 are dated, each mutation reddening the same names in both twins. 6 arrive never. +# +# ledger rows 6729 + 8 = 6737 +# never 4033 + 6 = 4039 +# dated 2696 + 2 = 2698 +# 4039 + 2698 == 6737 +# +# #1341, fuzz_arrow seeded, with its twin (OffgridwithJD): 4039 -> 4045, and +# suites_not_covered 87 -> 86. The suite had no rows. It enters with its 7 +# checks from five-major runs, green on each of 15-19 in the shell and the twin alike. +# A mutate that failed was skipped, so a corpus run that generated half its mutants +# passed on main, 4/4. Every mutant must now be generated and imported, and an empty +# or wholly rejected seed corpus is a named check rather than an unnamed FAIL. +# 1 are dated, each mutation reddening the same names in both twins. 6 arrive never. +# +# ledger rows 6737 + 7 = 6744 +# never 4039 + 6 = 4045 +# dated 2698 + 1 = 2699 +# 4045 + 2699 == 6744 +# +# #1341, fuzz_parquet seeded, with its twin (OffgridwithJD): 4045 -> 4051, and +# suites_not_covered 86 -> 85. The suite had no rows. It enters with its 7 +# checks from five-major runs, green on each of 15-19 in the shell and the twin alike. +# A mutate that failed was skipped, so a corpus run that generated half its mutants +# passed on main, 4/4. Every mutant must now be generated and run through its three +# statements, and an empty or unrenderable seed corpus is a named check rather than +# an unnamed FAIL. +# 1 are dated, each mutation reddening the same names in both twins. 6 arrive never. +# +# ledger rows 6744 + 7 = 6751 +# never 4045 + 6 = 4051 +# dated 2699 + 1 = 2700 +# 4051 + 2700 == 6751 +checks_never_observed_red 4051 diff --git a/test/docs_style.sh b/test/docs_style.sh index 9bdb55052..0044784d7 100755 --- a/test/docs_style.sh +++ b/test/docs_style.sh @@ -53,30 +53,22 @@ # Written fresh for pgColumnar. set -uo pipefail +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" SRCDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -checks=0 -fail=0 -check() { - local name="$1" got="$2" want="$3" - checks=$((checks + 1)) - if [ "$got" = "$want" ]; then - echo "PASS $name" - else - echo "FAIL $name: got [$got] want [$want]" - fail=1 - fi -} +# lib.sh's check() AND pgc_summary, NOT A PRIVATE TALLY (#1341). This suite kept its +# own counter and emitted no RESULT records, so no ledger row could ever be seeded for +# it and its pytest twin had nothing to be compared with. It now records like every +# other suite. Like wal_envelope.sh it needs no cluster, so pgc_setup is skipped and +# the counters check() expects are provided here. PGC_MAJOR comes from the pg_config +# the runner passes every suite, so a record names the major it ran on (#1121). +PGC_MAJOR="$(pgc_major_of "${1:-}")" +PGC_CHECKS=0 +PGC_FAIL=0 -# THIS SUITE KEEPS ITS OWN TALLY, not lib.sh's, and emits none of the machine -# RESULT vocabulary. So it cannot report a `SKIP` OUTCOME: selftest 400 refuses -# `echo "SKIP` in any file that calls `check`, because a skip there is a check -# result a count and a record must see, and this suite has neither to put it in. -# -# A property this suite cannot compare is therefore a NOTE: printed for a reader, -# counted as nothing, claiming no outcome. Where the reason is itself a fact about -# the tree it is asserted with `check` instead -- see the no-baseline arm below -- -# so "this cannot be checked" is itself checked rather than asserted. +# A property this suite cannot compare in a checkout without release tags is a NOTE, +# printed for a reader and counted as nothing; see the changelog block below for why it +# is not recorded unrunnable. note() is not lib.sh's: it is defined here. # # Reaching for lib.sh's helper of that name here is `command not found`: it prints # nothing, counts nothing and fails nothing while the suite reports PASSED. @@ -87,6 +79,17 @@ note() { # note TEXT command -v python3 >/dev/null || { echo "FAIL python3 not found"; exit 1; } +# WHAT A READER SEES, NOT WHAT THE FILE HOLDS (#1341 port). Every arm below that asks +# whether a page SAYS something read the raw file, so the sentence passed from inside +# an HTML comment that no reader ever sees. Measured on main: the 1024-floor line moved +# into ``, "join key" kept only in a comment, and a stale VERSION citation with +# the right one hidden beside it each left the whole suite green. `_md` is the page as +# rendered prose: the file with its HTML comments removed. +_md() { # _md FILE -> the file's text without comments + python3 -c 'import re, sys +sys.stdout.write(re.sub(r"", "", open(sys.argv[1], encoding="utf-8").read(), flags=re.S))' "$1" +} + echo "== pgColumnar test: docs_style.sh ==" # The full rules over every user-facing document. @@ -111,7 +114,9 @@ check "and it actually examined the documents" \ # That case is also caught by "mkdocs build --strict" in docs.yml, which fails on a nav # entry pointing at nothing. Half a property here and half in a workflow is how the # missing half goes unnoticed, so both halves are stated here. -nav_roadmap=$(grep -c "roadmap.md" "$SRCDIR/mkdocs.yml" || true) +# A COMMENTED-OUT ENTRY IS NOT IN THE NAV. The raw grep counted `# - Roadmap: roadmap.md`, +# so a nav entry disabled with `#` passed: measured on main. +nav_roadmap=$(sed -E 's/(^|[[:space:]])#.*//' "$SRCDIR/mkdocs.yml" | grep -c "roadmap.md" || true) check "the roadmap is in the documentation nav" "$([ "$nav_roadmap" -ge 1 ] && echo yes || echo no)" "yes" check "and the page that nav entry points at exists" \ "$([ -f "$SRCDIR/docs/roadmap.md" ] && echo yes || echo no)" "yes" @@ -132,7 +137,7 @@ check "no document carries a merge conflict marker" \ # values and percentiles approximately". The section is found by its heading and # must open with the pg_partman paragraph its steps belong to. partman_open="$(awk '$0 == "## Manage time partitions with pg_partman" {f=1; next} - f && NF {print; exit}' "$SRCDIR/docs/how-to.md")" + f && NF {print; exit}' <(_md "$SRCDIR/docs/how-to.md"))" check "the how-to keeps its pg_partman section under its own heading" \ "$(printf '%s' "$partman_open" | grep -c '^pg_partman can create and drop columnar partitions')" "1" @@ -149,7 +154,8 @@ _howto_rf() { python3 - "$SRCDIR/docs/how-to.md" <<'PY' from pathlib import Path import sys -text = Path(sys.argv[1]).read_text(encoding="utf-8") +import re +text = re.sub(r"", "", Path(sys.argv[1]).read_text(encoding="utf-8"), flags=re.S) heading = "## Skip fact-table work under a star-schema join" start = text.find(heading) if start < 0: @@ -168,7 +174,8 @@ _practices_jk() { python3 - "$SRCDIR/docs/best-practices.md" <<'PY' from pathlib import Path import sys -text = Path(sys.argv[1]).read_text(encoding="utf-8").lower() +import re +text = re.sub(r"", "", Path(sys.argv[1]).read_text(encoding="utf-8"), flags=re.S).lower() print("yes" if "join key" in text else "no") PY } @@ -207,7 +214,8 @@ check "best-practices names clustering on the join key" \ # reader got wrong: blank lines are absent inside a markdown table and arbitrary in # prose, while a heading is declared. _floor_line() { # _floor_line FILE -- yes if one line names the setting and the floor - if grep -qE 'stripe_row_limit.*1024|1024.*stripe_row_limit' "$1"; then + # grep -c, not -q: an early-exit reader under pipefail turns a match into a failure + if [ "$(_md "$1" | grep -cE 'stripe_row_limit.*1024|1024.*stripe_row_limit')" -gt 0 ]; then echo yes else echo no @@ -220,7 +228,7 @@ _floor_same_section() { # _floor_same_section FILE ADVICE -- yes if both are und if (index(tolower($0), tolower(advice))) a[h] = 1 if (/stripe_row_limit/ && /1024/) f[h] = 1 } - END { for (k in a) if (k in f) { print "yes"; exit } print "no" }' "$1" + END { for (k in a) if (k in f) { print "yes"; exit } print "no" }' <(_md "$1") } check "configuration.md states the 1024 floor where it documents the setting" \ "$(_floor_line "$SRCDIR/docs/configuration.md")" "yes" @@ -258,7 +266,7 @@ check "premise: at least one document cites the VERSION file" \ _stale="" for _d in $_verdocs; do - grep -q "\`$_ver\`, recorded in \`VERSION\`" "$_d" || _stale="$_stale $(basename "$_d")" + [ "$(_md "$_d" | grep -c "\`$_ver\`, recorded in \`VERSION\`")" -gt 0 ] || _stale="$_stale $(basename "$_d")" done check "every document citing VERSION quotes the version VERSION holds" \ "$(printf '%s' "$_stale" | sed 's/^ //')" "" @@ -328,14 +336,14 @@ check "premise: at least one document names the latest published pre-release" \ # and yields no version is a file this rule cannot see, and it is named rather # than skipped. # shellcheck disable=SC2086 -_pubvers="$( { grep -rhoE 'latest published pre-release is `v[^`]*`' $_pubdocs 2>/dev/null - grep -rhoE '`v[^`]*` is the latest published pre-release' $_pubdocs 2>/dev/null - } | grep -oE '`v[^`]*`' | tr -d '`' | LC_ALL=C sort -u)" +_pubvers="$(for _d in $_pubdocs; do _md "$_d"; done \ + | grep -oE 'latest published pre-release is `v[^`]*`|`v[^`]*` is the latest published pre-release' \ + | grep -oE '`v[^`]*`' | tr -d '`' | LC_ALL=C sort -u)" _pubunparsed="" for _d in $_pubdocs; do - grep -qE 'latest published pre-release is `v[^`]*`|`v[^`]*` is the latest published pre-release' \ - "$_d" || _pubunparsed="$_pubunparsed $(basename "$_d")" + [ "$(_md "$_d" | grep -cE 'latest published pre-release is `v[^`]*`|`v[^`]*` is the latest published pre-release')" -gt 0 ] \ + || _pubunparsed="$_pubunparsed $(basename "$_d")" done check "every document making the claim states it in a form this rule can read" \ "$(printf '%s' "$_pubunparsed" | sed 's/^ //')" "" @@ -484,7 +492,7 @@ _upg_sentences() { # _upg_sentences FILE -> one sentence per line # by period-space, which also protects a sentence ENDING in a number and # merges it with the next one -- demonstrated by @OffgridwithJD with # `The corpus held 1444. ` injected ahead of the claim. - sed -E 's/^([0-9]+)\. /\1.@LM@/' "$1" \ + _md "$1" | sed -E 's/^([0-9]+)\. /\1.@LM@/' \ | tr '\n' ' ' \ | sed 's/\. /.\n/g' \ | sed 's/@LM@/ /g' @@ -576,14 +584,19 @@ _pgxn_ver="$(printf '%s' "$_ver" | awk -F- '{ check "premise: the PGXN form was derived from VERSION, not empty" \ "$([ -n "$_pgxn_ver" ] && echo yes || echo no)" "yes" -for _mk in version provides.pgcolumnar.version; do - check "META.json $_mk is the version VERSION holds, in PGXN form" \ - "$(python3 -c " +_meta_get() { # _meta_get DOTTED.KEY -> that value of META.json, or empty + python3 -c " import json,sys m=json.load(open(sys.argv[1])) for k in sys.argv[2].split('.'): m=m[k] -print(m)" "$_meta" "$_mk" 2>/dev/null)" "$_pgxn_ver" -done +print(m)" "$_meta" "$1" 2>/dev/null +} +# TWO LITERAL CHECKS, not a loop: a name built from a loop variable is invisible to the +# twin grader, which reads only literal names. +check "META.json version is the version VERSION holds, in PGXN form" \ + "$(_meta_get version)" "$_pgxn_ver" +check "META.json provides.pgcolumnar.version is the version VERSION holds, in PGXN form" \ + "$(_meta_get provides.pgcolumnar.version)" "$_pgxn_ver" # The defect that actually shipped. `git archive` rather than a filesystem test, # because what matters is whether the DISTRIBUTION contains it -- an export-ignore @@ -724,18 +737,19 @@ fi _cl_seen=0 _cl_usedrows="" +_cl_nosec="" +_cl_bad="" +_cl_lost="" +# THREE CHECKS WITH FIXED NAMES, each naming the versions it found wrong in its +# reading. One check per version per property put the version in the NAME, which no +# twin grader can read and which grows by two names every release (#1341 port). while read -r _cl_v; do [ -n "$_cl_v" ] || continue _cl_seen=$((_cl_seen + 1)) - _cl_name="the $_cl_v section holds what v$_cl_v shipped, plus only what is recorded" - if [ -n "$_cl_why" ]; then - note "$_cl_name: not compared ($_cl_why)" - continue - fi + [ -z "$_cl_why" ] || continue _cl_tagged="$(_cl_git show "v$_cl_v:CHANGELOG.md" | _cl_entries "$_cl_v" | LC_ALL=C sort)" if [ -z "$_cl_tagged" ]; then - check "v$_cl_v carries a dated section of its own to compare against" \ - "no section for $_cl_v at v$_cl_v" "a section for $_cl_v at v$_cl_v" + _cl_nosec="$_cl_nosec $_cl_v" continue fi _cl_nowents="$(_cl_entries "$_cl_v" < "$SRCDIR/CHANGELOG.md" | LC_ALL=C sort)" @@ -748,11 +762,27 @@ while read -r _cl_v; do _cl_allowed="$(awk -F'\t' -v v="$_cl_v" '$1==v && $2 != "" {print $2}' "$_cl_allowfile" 2>/dev/null | LC_ALL=C sort)" [ -n "$_cl_allowed" ] && _cl_usedrows="$_cl_usedrows$_cl_allowed " - check "$_cl_name" "$_cl_extra" "$_cl_allowed" - check "the $_cl_v section still holds every entry v$_cl_v shipped" "$_cl_gone" "" + [ "$_cl_extra" = "$_cl_allowed" ] \ + || _cl_bad="$_cl_bad $_cl_v: added [$(printf '%s' "$_cl_extra" | tr '\n' '|')] recorded [$(printf '%s' "$_cl_allowed" | tr '\n' '|')]" + [ -z "$_cl_gone" ] || _cl_lost="$_cl_lost $_cl_v: [$(printf '%s' "$_cl_gone" | tr '\n' '|')]" done <> 16) % $1 )); } +pick() { local n=$#; rnd "$n"; shift "$_RND"; _PICK="$1"; } # Pick one element of the argument list. CODECS="none pglz lz4 zstd" @@ -54,6 +59,21 @@ CODECS="none pglz lz4 zstd" FUZZ_DEFS="id int, c_int int, c_num numeric(20,4), c_f8 double precision, c_bool boolean, c_date date, c_ts timestamp, c_vc varchar(40), c_text text" +# The check names are the property, not the iteration: each one is asserted once over +# the whole run and names the iterations that broke it. Names that carried the +# iteration's drawn parameters changed with the seed, so no ledger row or twin could +# name them (#1341). +bad_rows=""; bad_opts=""; its=0 +declare -A BAD +KINDS="whole-row count minmax sumavg range-int range-date eq isnull compound" +for k in $KINDS; do BAD[$k]=""; done +cmp_kind() { # kind tag query-template: record the iteration when columnar and heap differ + local hq hc + hq="$(pgc_set_hash "${3//%T/t_heap}")"; [ -z "$hq" ] && hq="HEAP_ERROR" + hc="$(pgc_set_hash "${3//%T/t_col}")"; [ -z "$hc" ] && hc="COL_ERROR" + [ "$hc" = "$hq" ] || BAD[$1]+="[$2: $hc vs $hq] " +} + for it in $(seq 1 "$ITERS"); do rnd 5000; N=$(( 1 + _RND )) # Occasionally snap N to a boundary of the chosen chunk-group limit. @@ -76,6 +96,11 @@ for it in $(seq 1 "$ITERS"); do q "SELECT pgcolumnar.set_options('t_col', chunk_group_row_limit => $CG, stripe_row_limit => $SR, compression => '$CODEC', compression_level => $LVL);" >/dev/null + # set_options' result was thrown away, so a call that failed or ignored an + # argument left the iteration on default storage and nothing said so. + got="$(q "SELECT chunk_group_row_limit || '/' || stripe_row_limit || '/' || compression + || '/' || compression_level FROM pgcolumnar.options WHERE regclass = 't_col'::regclass;")" + [ "$got" = "$CG/$SR/$CODEC/$LVL" ] || bad_opts+="[$tag: read back '$got'] " load_pair "SELECT g, @@ -89,17 +114,31 @@ for it in $(seq 1 "$ITERS"); do CASE WHEN g%$NULLMOD=0 THEN NULL WHEN g%997=0 THEN repeat('q',3000) ELSE ('t'||g) END FROM generate_series(1,$N) g" - check "$tag :: row count" "$(q 'SELECT count(*) FROM t_col;')" "$N" - - diff_query "$tag :: whole-row" "SELECT * FROM %T" - diff_query "$tag :: count" "SELECT count(*), count(c_int), count(c_text) FROM %T" - diff_query "$tag :: minmax" "SELECT min(c_int), max(c_int), min(c_vc), max(c_vc), min(c_date), max(c_date) FROM %T" - diff_query "$tag :: sumavg" "SELECT sum(c_int), avg(c_num), sum(c_f8) FROM %T" - diff_query "$tag :: range int" "SELECT id FROM %T WHERE c_int BETWEEN $LO AND $HI" - diff_query "$tag :: range date" "SELECT id FROM %T WHERE c_date >= DATE '2010-06-01'" - diff_query "$tag :: eq" "SELECT id, c_vc FROM %T WHERE id = $EQ" - diff_query "$tag :: isnull" "SELECT count(*) FROM %T WHERE c_text IS NULL" - diff_query "$tag :: compound" "SELECT id FROM %T WHERE c_int > 0 AND c_bool AND c_num < 1000" + n="$(q 'SELECT count(*) FROM t_col;')" + [ "$n" = "$N" ] || bad_rows+="[$tag: $n rows] " + cmp_kind whole-row "$tag" "SELECT * FROM %T" + cmp_kind count "$tag" "SELECT count(*), count(c_int), count(c_text) FROM %T" + cmp_kind minmax "$tag" "SELECT min(c_int), max(c_int), min(c_vc), max(c_vc), min(c_date), max(c_date) FROM %T" + cmp_kind sumavg "$tag" "SELECT sum(c_int), avg(c_num), sum(c_f8) FROM %T" + cmp_kind range-int "$tag" "SELECT id FROM %T WHERE c_int BETWEEN $LO AND $HI" + cmp_kind range-date "$tag" "SELECT id FROM %T WHERE c_date >= DATE '2010-06-01'" + cmp_kind eq "$tag" "SELECT id, c_vc FROM %T WHERE id = $EQ" + cmp_kind isnull "$tag" "SELECT count(*) FROM %T WHERE c_text IS NULL" + cmp_kind compound "$tag" "SELECT id FROM %T WHERE c_int > 0 AND c_bool AND c_num < 1000" + its=$((its + 1)) done +check "the run covered every iteration" "$its" "$ITERS" +check "every iteration's storage options read back as set" "${bad_opts:-none}" "none" +check "every iteration loaded all its rows" "${bad_rows:-none}" "none" +check "every iteration's whole rows agree with heap" "${BAD[whole-row]:-none}" "none" +check "every iteration's counts agree with heap" "${BAD[count]:-none}" "none" +check "every iteration's min and max agree with heap" "${BAD[minmax]:-none}" "none" +check "every iteration's sums and averages agree with heap" "${BAD[sumavg]:-none}" "none" +check "every iteration's integer range agrees with heap" "${BAD[range-int]:-none}" "none" +check "every iteration's date range agrees with heap" "${BAD[range-date]:-none}" "none" +check "every iteration's equality lookup agrees with heap" "${BAD[eq]:-none}" "none" +check "every iteration's IS NULL count agrees with heap" "${BAD[isnull]:-none}" "none" +check "every iteration's compound filter agrees with heap" "${BAD[compound]:-none}" "none" + pgc_summary diff --git a/test/fuzz_arrow.sh b/test/fuzz_arrow.sh index 86bc7a9f7..f484e3d8e 100755 --- a/test/fuzz_arrow.sh +++ b/test/fuzz_arrow.sh @@ -64,11 +64,8 @@ echo "-- corpus: $CORPUS" # since Arrow has no parquet_schema equivalent to derive it from the file. # --------------------------------------------------------------------------- mapfile -t SEEDLINES < <(python3 "$PGC_SRCDIR/test/arrow_corpus.py" "$CORPUS" 2>/dev/null) -if [ "${#SEEDLINES[@]}" -eq 0 ]; then - echo "FAIL the corpus generator produced no files" - PGC_FAIL=1 - pgc_summary -fi +check "the seed corpus generator produced files" \ + "$([ "${#SEEDLINES[@]}" -gt 0 ] && echo yes || echo no)" "yes" echo "-- ${#SEEDLINES[@]} seed files" # The composite type the struct seeds name in their column list. Created once here @@ -97,11 +94,10 @@ for line in "${SEEDLINES[@]}"; do done echo "-- ${#SEEDPATH[@]} seeds the importer accepts pristine" -if [ "${#SEEDPATH[@]}" -eq 0 ]; then - echo "FAIL no seed imported cleanly; the importer rejects its own corpus" - PGC_FAIL=1 - pgc_summary -fi +# A named check, not an unnamed FAIL: a reader that rejects its own corpus is a +# product failure, and the fuzz loop below then runs nothing at all. +check "the importer accepts some pristine seed" \ + "$([ "${#SEEDPATH[@]}" -gt 0 ] && echo yes || echo no)" "yes" # --------------------------------------------------------------------------- # Crash detection. Judged from the server log rather than psql's message: a @@ -112,6 +108,7 @@ fi # --------------------------------------------------------------------------- LOGPOS=0 crashes=0 +gen=0 hangs=0 sanitizer=0 errors=0 @@ -230,13 +227,15 @@ log_since echo "-- fuzzing" nseeds="${#SEEDPATH[@]}" -for ((i = 0; i < ITERS; i++)); do +[ "$nseeds" -gt 0 ] || ITERS_RUN=0 +for ((i = 0; i < ${ITERS_RUN-$ITERS}; i++)); do idx=$((i % nseeds)) sf="${SEEDPATH[$idx]}" tab="${SEEDTAB[$idx]}" ms=$((SEED + i)) python3 "$PGC_SRCDIR/test/arrow_mutate.py" "$sf" "$ms" "$MUT" 2>/dev/null || continue + gen=$((gen + 1)) chmod 644 "$MUT" 2>/dev/null # Empty the target first so an accepted mutant does not accumulate rows across @@ -258,6 +257,11 @@ echo "-- done: $ITERS mutants, ${errors} rejected with ERROR, ${clean} accepted" check "no mutant crashed the backend" "$crashes" "0" check "no mutant hung the decode" "$hangs" "0" check "no mutant tripped a sanitizer" "$sanitizer" "0" +# Every mutant generated, and every statement run with an outcome. A mutate that +# failed was skipped with `continue`, and a corpus run that generated almost nothing +# passed: the reach check below needs only one ERROR. +check "every mutant was generated and ran the import" \ + "$gen/$((errors + clean + crashes + hangs + sanitizer))" "$ITERS/$((ITERS * 1))" # A corpus rejected outright teaches nothing and would make the three checks above # pass for the wrong reason. Some mutants must have reached the decoder and been diff --git a/test/fuzz_avro.sh b/test/fuzz_avro.sh index dc81336f8..ecea0d397 100755 --- a/test/fuzz_avro.sh +++ b/test/fuzz_avro.sh @@ -55,7 +55,7 @@ check "control: the valid manifest-list reaches the decoder (1 entry)" \ "$(q "$RD_LIST")" "1" LOGPOS=0 -crashes=0; hangs=0; sanitizer=0; errors=0; clean=0 +crashes=0; hangs=0; sanitizer=0; errors=0; clean=0; fresh=0; fileerr=0 log_since() { local sz sz=$(stat -c %s "$PGC_LOGFILE" 2>/dev/null || echo 0) @@ -78,7 +78,12 @@ for ((i = 0; i < ITERS; i++)); do s=$((SEED + i)) if (( i % 2 == 0 )); then base="$BASE"; RD="$RD_MANIFEST" else base="$BASE_LIST"; RD="$RD_LIST"; fi - python3 "$CORPUS" "$base" "$s" "$MUT"; chmod 644 "$MUT" 2>/dev/null + # The mutant is removed first, so a generator that writes nothing leaves no file + # rather than the previous one: a stale file read as a clean accept or a decoder + # ERROR made every check below pass with no mutant ever generated. + rm -f "$MUT" + python3 "$CORPUS" "$base" "$s" "$MUT" 2>/dev/null; chmod 644 "$MUT" 2>/dev/null + [ -f "$MUT" ] && fresh=$((fresh + 1)) log_since out="$(timeout 30 env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" \ -U postgres -d "$PGC_DB" -v ON_ERROR_STOP=0 \ @@ -99,6 +104,8 @@ for ((i = 0; i < ITERS; i++)); do crashes=$((crashes + 1)); echo " FINDING [crash] seed=$s"; wait_for_cluster || true elif grep -q '^ERROR:' <<<"$out"; then errors=$((errors + 1)) + # an ERROR opening the file never reached the decoder + grep -qE 'could not open|No such file|Permission denied' <<<"$out" && fileerr=$((fileerr + 1)) else clean=$((clean + 1)) fi @@ -109,7 +116,9 @@ check "no mutant crashed the backend" "$crashes" "0" check "no mutant hung the decoder" "$hangs" "0" check "no mutant tripped a sanitizer" "$sanitizer" "0" check "every mutant ran (errors + clean == iters)" "$((errors + clean))" "$ITERS" +check "every iteration wrote a fresh mutant" "$fresh" "$ITERS" +# Reached means decoded: a file the backend could not open never reached it. check "the corpus reached the decoder at all" \ - "$([ "$((errors + clean))" -gt 0 ] && echo yes || echo no)" "yes" + "$([ "$((errors - fileerr + clean))" -gt 0 ] && echo yes || echo "no ($fileerr of $errors errors were opening the file)")" "yes" pgc_summary diff --git a/test/fuzz_listing.sh b/test/fuzz_listing.sh index 9636c49b7..81c7e0d74 100755 --- a/test/fuzz_listing.sh +++ b/test/fuzz_listing.sh @@ -84,7 +84,7 @@ check "control: a valid listing reaches the parser and reads the object" \ "$(q "$READ")" "3" LOGPOS=0 -crashes=0; hangs=0; sanitizer=0; errors=0; clean=0 +crashes=0; hangs=0; sanitizer=0; errors=0; clean=0; fresh=0 log_since() { local sz sz=$(stat -c %s "$PGC_LOGFILE" 2>/dev/null || echo 0) @@ -105,7 +105,12 @@ log_since echo "-- fuzzing $ITERS listing mutants (seed base $SEED)" for ((i = 0; i < ITERS; i++)); do s=$((SEED + i)) - python3 "$CORPUS" "$s" "$OVERRIDE" + # The override is removed first: with no override the fixture serves the real + # listing, so a generator that writes nothing read as a clean accept every time and + # every check below passed with no mutant ever generated. + rm -f "$OVERRIDE" + python3 "$CORPUS" "$s" "$OVERRIDE" 2>/dev/null + [ -f "$OVERRIDE" ] && fresh=$((fresh + 1)) log_since out="$(timeout 30 env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" \ -U postgres -d "$PGC_DB" -v ON_ERROR_STOP=0 \ @@ -140,6 +145,7 @@ check "no mutant tripped a sanitizer" "$sanitizer" "0" # (the control above proved the pipeline; here the count must add up). check "every mutant ran (errors + clean == iters)" \ "$((errors + clean))" "$ITERS" +check "every iteration wrote a fresh mutant" "$fresh" "$ITERS" check "the corpus reached the parser at all (some mutant errored or accepted)" \ "$([ "$((errors + clean))" -gt 0 ] && echo yes || echo no)" "yes" diff --git a/test/fuzz_parquet.sh b/test/fuzz_parquet.sh index 2979ab0a8..beaa9d3de 100755 --- a/test/fuzz_parquet.sh +++ b/test/fuzz_parquet.sh @@ -68,11 +68,8 @@ echo "-- corpus: $CORPUS" # Seeds. Every one is valid; see test/parquet_corpus.py for why that matters. # --------------------------------------------------------------------------- mapfile -t SEEDLINES < <(python3 "$PGC_SRCDIR/test/parquet_corpus.py" "$CORPUS" 2>/dev/null) -if [ "${#SEEDLINES[@]}" -eq 0 ]; then - echo "FAIL the corpus generator produced no files" - PGC_FAIL=1 - pgc_summary -fi +check "the seed corpus generator produced files" \ + "$([ "${#SEEDLINES[@]}" -gt 0 ] && echo yes || echo no)" "yes" echo "-- ${#SEEDLINES[@]} seed files" # The column definition list read_parquet requires, derived from each pristine @@ -90,11 +87,10 @@ for line in "${SEEDLINES[@]}"; do done echo "-- ${#SEEDPATH[@]} seeds with a usable column list" -if [ "${#SEEDPATH[@]}" -eq 0 ]; then - echo "FAIL no seed produced a readable schema; the reader rejects its own corpus" - PGC_FAIL=1 - pgc_summary -fi +# A named check, not an unnamed FAIL: a reader that rejects its own corpus is a +# product failure, and the fuzz loop below then runs nothing at all. +check "the reader renders some pristine seed's schema" \ + "$([ "${#SEEDPATH[@]}" -gt 0 ] && echo yes || echo no)" "yes" # --------------------------------------------------------------------------- # Crash detection. @@ -107,6 +103,7 @@ fi # --------------------------------------------------------------------------- LOGPOS=0 crashes=0 +gen=0 hangs=0 sanitizer=0 errors=0 @@ -253,13 +250,15 @@ psql_run "DROP TABLE IF EXISTS fz_target;" >/dev/null 2>&1 echo "-- fuzzing" nseeds="${#SEEDPATH[@]}" -for ((i = 0; i < ITERS; i++)); do +[ "$nseeds" -gt 0 ] || ITERS_RUN=0 +for ((i = 0; i < ${ITERS_RUN-$ITERS}; i++)); do idx=$((i % nseeds)) sf="${SEEDPATH[$idx]}" cols="${SEEDCOLS[$idx]}" ms=$((SEED + i)) python3 "$PGC_SRCDIR/test/parquet_mutate.py" "$sf" "$ms" "$MUT" 2>/dev/null || continue + gen=$((gen + 1)) chmod 644 "$MUT" 2>/dev/null run_stmt "SELECT * FROM pgcolumnar.parquet_schema('$MUT');" "$sf" "$ms" @@ -287,6 +286,11 @@ echo "-- done: $ITERS mutants, ${errors} rejected with ERROR, ${clean} accepted" check "no mutant crashed the backend" "$crashes" "0" check "no mutant hung the decode" "$hangs" "0" check "no mutant tripped a sanitizer" "$sanitizer" "0" +# Every mutant generated, and every statement run with an outcome. A mutate that +# failed was skipped with `continue`, and a corpus run that generated almost nothing +# passed: the reach check below needs only one ERROR. +check "every mutant was generated and ran its three statements" \ + "$gen/$((errors + clean + crashes + hangs + sanitizer))" "$ITERS/$((ITERS * 3))" # A corpus that is rejected outright teaches nothing, and would make the three # checks above pass for the wrong reason. At least some mutants must have got diff --git a/test/hilbert_curve.sh b/test/hilbert_curve.sh index 1e814886c..7dab0744e 100755 --- a/test/hilbert_curve.sh +++ b/test/hilbert_curve.sh @@ -335,20 +335,13 @@ battery_arm_names() { } BATTERY_ARMS="$(battery_arm_names)" -# Report a whole group of arms in one state, so a run that could not build -# still lists what it did not measure. -arms_unrunnable() { # arms_unrunnable "LIST" REASON DETAIL - local _a - while IFS= read -r _a; do - [ -n "$_a" ] && check_unrunnable "$_a" "$2" "$3" - done <<< "$1" -} -arms_failed() { # arms_failed "LIST" DETAIL - local _a - while IFS= read -r _a; do - [ -n "$_a" ] && pgc_fail "$_a" "$2" - done <<< "$1" -} +# A run that could not build reports a whole group of arms in one state, so it +# still lists what it did not measure. The loops that do it are written out at each +# site rather than in a helper taking the list: a helper that reaches a recorder +# through a loop variable is a name slot compare_to_bash cannot resolve, and it +# refused this whole suite for that (#1341). At the top level the same loop is read +# as one unnamed call, which the grader drops, and the literal arms below are what +# it grades. # A CC INHERITED FROM THE ENVIRONMENT IS NOT A COMPILER UNTIL IT COMPILES. # @@ -376,8 +369,9 @@ if [ "$cc_usable" != yes ]; then sed 's/^/ /' "$WORK/smoke.log" 2>/dev/null _why="CC=$CC cannot build C" fi - arms_unrunnable "$CONTROL_ARMS" MISSING_DEPENDENCY "$_why" - arms_unrunnable "$BATTERY_ARMS" MISSING_DEPENDENCY "$_why" + while IFS= read -r _a; do + [ -n "$_a" ] && check_unrunnable "$_a" MISSING_DEPENDENCY "$_why" + done <<< "$CONTROL_ARMS"$'\n'"$BATTERY_ARMS" pgc_summary fi echo "-- compiler: $CC ($("$CC" --version 2>/dev/null | head -1))" @@ -1178,7 +1172,9 @@ if "$CC" -O2 -Wall -I"$WORK" -o "$WORK/controls" "$WORK/controls.c" >"$CTLLOG" 2 else echo "---- the control program would not build or run ----" sed 's/^/ /' "$CTLLOG" - arms_failed "$CONTROL_ARMS" "the control program would not build or run" + while IFS= read -r _a; do + [ -n "$_a" ] && pgc_fail "$_a" "the control program would not build or run" + done <<< "$CONTROL_ARMS" fi echo "-- instrument arms: $((PGC_CHECKS - _pre_checks)) reported" \ "($((PGC_PASSED - _pre_passed)) passed, $((PGC_FAILED - _pre_failed)) failed);" \ @@ -1214,6 +1210,22 @@ cat > "$WORK/battery.c" <<'CEOF' static void (*const pin_transpose) (uint64 *, int) = cluster_hilbert_transpose; static void (*const pin_pack) (const uint64 *, int, unsigned char *) = cluster_pack_interleave; +/* + * Every key buffer is filled with 0xa5 before the pack writes it. A byte the pack + * never establishes then reads 0xa5 -- the same in every run and in the pytest + * twin -- rather than whatever the previous key left there, or, on the first use of + * a buffer, uninitialized memory. Measured: under a pack whose memset stops one + * byte short, which goldens went red depended on what the previous call had left + * in that byte. The C5 extent arms below fill their own buffers with 0x00 and 0xff + * on purpose and call the pack directly; nothing else does. + */ +static void +pk(const uint64 *ord, int ncols, unsigned char *out) +{ + memset(out, 0xa5, (size_t) ncols * 8); + cluster_pack_interleave(ord, ncols, out); +} + /* Deterministic, so two runs of this suite compare the same points. */ static unsigned rs = 987654321u; static unsigned @@ -1332,7 +1344,7 @@ enc_hilbert(const unsigned *c, int ncols, int b, unsigned char *key) for (j = 0; j < ncols; j++) X[j] = (uint64) c[j] << (64 - b); cluster_hilbert_transpose(X, ncols); - cluster_pack_interleave(X, ncols, key); + pk(X, ncols, key); } int @@ -1431,9 +1443,9 @@ main(void) Xf[j] = Xb[j] | (rnd64() & ((((uint64) 1) << (64 - b)) - 1)); } cluster_hilbert_transpose(Xb, ncols); - cluster_pack_interleave(Xb, ncols, kb); + pk(Xb, ncols, kb); cluster_hilbert_transpose(Xf, ncols); - cluster_pack_interleave(Xf, ncols, kf); + pk(Xf, ncols, kf); points += 1; ixf = pgc_key_index(kf, ncols, b); if (pgc_key_index(kb, ncols, b) != ixf) @@ -1494,7 +1506,7 @@ main(void) untranspose(A0, ncols, 0); memcpy(B, A0, sizeof(uint64) * ncols); cluster_hilbert_transpose(B, ncols); - cluster_pack_interleave(B, ncols, kk); + pk(B, ncols, kk); trips++; ok0 = (memcmp(kk, key, (size_t) ncols * 8) == 0); if (!ok0) @@ -1505,7 +1517,7 @@ main(void) untranspose(A1, ncols, 1); memcpy(B, A1, sizeof(uint64) * ncols); cluster_hilbert_transpose(B, ncols); - cluster_pack_interleave(B, ncols, kk); + pk(B, ncols, kk); if (memcmp(kk, key, (size_t) ncols * 8) != 0) mutbad++; @@ -1547,7 +1559,7 @@ main(void) */ memcpy(B, A1, sizeof(uint64) * ncols); cluster_hilbert_transpose(B, ncols); - cluster_pack_interleave(B, ncols, kk); + pk(B, ncols, kk); ok1 = (memcmp(kk, key2, (size_t) ncols * 8) == 0); adj++; @@ -1599,7 +1611,7 @@ main(void) pgc_makevec(k, ncols, X); memcpy(Y, X, sizeof(uint64) * ncols); cluster_hilbert_transpose(Y, ncols); - cluster_pack_interleave(Y, ncols, key); + pk(Y, ncols, key); pgc_hex(key, ncols * 8, h); printf("golden_n%d_%s=%s\n", ncols, pgc_vecname[k], h); } @@ -1609,12 +1621,12 @@ main(void) v[0] = (uint64) 1 << 63; v[1] = 0; - cluster_pack_interleave(v, 2, key); + pk(v, 2, key); pgc_hex(key, 16, h); printf("packpin_hi=%s\n", h); v[0] = 0; v[1] = (uint64) 1 << 63; - cluster_pack_interleave(v, 2, key); + pk(v, 2, key); pgc_hex(key, 16, h); printf("packpin_lo=%s\n", h); } @@ -1703,7 +1715,7 @@ main(void) { X[0] = vals[k]; cluster_hilbert_transpose(X, 1); - cluster_pack_interleave(X, 1, key); + pk(X, 1, key); pgc_hex(key, 8, h); printf("ident_n1_%s=%s\n", nm[k], h); } @@ -1728,9 +1740,9 @@ main(void) X[0] = x; cluster_hilbert_transpose(X, 1); - cluster_pack_interleave(X, 1, kh); + pk(X, 1, kh); Y[0] = x; - cluster_pack_interleave(Y, 1, kz); + pk(Y, 1, kz); for (q = 0; q < 8; q++) be[q] = (unsigned char) (x >> (56 - 8 * q)); n6++; @@ -1749,7 +1761,7 @@ main(void) for (k = 0; k < PGC_NVEC; k++) { pgc_makevec(k, ncols, X); - cluster_pack_interleave(X, ncols, key); + pk(X, ncols, key); pgc_hex(key, ncols * 8, h); printf("zorder_n%d_%s=%s\n", ncols, pgc_vecname[k], h); } @@ -1766,7 +1778,9 @@ if [ ! -f "$CURVE_C" ] || [ ! -f "$CURVE_H" ]; then [ -f "$CURVE_C" ] || _missing="$_missing src/columnar_curve.c" [ -f "$CURVE_H" ] || _missing="$_missing src/columnar_curve.h" echo "-- the code under test is absent:$_missing" - arms_failed "$BATTERY_ARMS" "the code under test is absent:$_missing" + while IFS= read -r _a; do + [ -n "$_a" ] && pgc_fail "$_a" "the code under test is absent:$_missing" + done <<< "$BATTERY_ARMS" else cp "$CURVE_C" "$CURVE_H" "$WORK/" if "$CC" -O2 -Wall -Werror=incompatible-pointer-types -I"$WORK" \ @@ -1777,7 +1791,9 @@ else else echo "---- the battery would not build or run against $CURVE_C ----" sed 's/^/ /' "$BATLOG" | head -40 - arms_failed "$BATTERY_ARMS" "the battery would not build or run; see the log above" + while IFS= read -r _a; do + [ -n "$_a" ] && pgc_fail "$_a" "the battery would not build or run; see the log above" + done <<< "$BATTERY_ARMS" fi fi diff --git a/test/iceberg_deletes.sh b/test/iceberg_deletes.sh index b9e88a875..413959ea7 100755 --- a/test/iceberg_deletes.sh +++ b/test/iceberg_deletes.sh @@ -15,22 +15,73 @@ set -uo pipefail . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" -sqlstate_of() { - env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ - -d "$PGC_DB" -qtA 2>&1 < the SQLSTATE when SQL is refused with PHRASE in its message; +# "SQLSTATE (another refusal: MESSAGE)" when a different guard refused it; "no +# error" when it ran; nothing when the server could not be asked. Each guard here +# has its own message, and several share a SQLSTATE (XX001 and 22023), so an arm +# counts only its own guard's refusal. +refusal() { + local out st msg + await_up + out="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ + -d "$PGC_DB" -qtA 2>&1 <&1 < wait, at most 60 s, until the server answers. A refusal arm that kills a +# backend sends the server through crash recovery; without the wait, the next arms' +# readings fail on recovery and red as collateral, more or fewer of them by timing +# (with the partition spec's fields guard removed, the backend segfaults and two +# later arms went red). The crash itself is still caught: survived() counts it. +await_up() { + local i + for i in $(seq 1 120); do + [ "$(q 'SELECT 1')" = 1 ] && return 0 + sleep 0.5 + done +} + +# Backends killed by a signal, from the server log: a number, or nothing when the +# log can't be read. A fresh connection answers again after crash recovery, so +# "SELECT 1" alone passed with a backend killed by a refusal arm: with abort() in +# place of the null-pos refusal, or the fields guard removed (a segfault), and the +# check run once recovery had finished. The survival checks below also require +# that this count has not moved since their section began. +crashes() { + local _n + _n="$(pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null)" + case "$_n" in '' | *[!0-9]*) ;; *) echo "$_n" ;; esac +} +survived() { # survived BEFORE -> "1" when the server answers and no backend died since + local _up _now + await_up + _up="$(q 'SELECT 1')"; _now="$(crashes)" + if ! [[ "$1" =~ ^[0-9]+$ && "$_now" =~ ^[0-9]+$ ]]; then echo "no (crash count unreadable)" + elif [ "$_up" = 1 ] && [ "$_now" = "$1" ]; then echo 1 + else echo "no ([$_up], $(( _now - $1 )) backend crashes)"; fi +} + FX="$(dirname "${BASH_SOURCE[0]}")/fixtures/iceberg" WHD="$FX/warehouse_del" [ -f "$WHD/db/t/metadata/apply.metadata.json" ] || pgc_skip fixture "delete fixture is missing" @@ -44,7 +95,7 @@ chmod -R u+rwX "$DEST" MDIR="$DEST/db/t/metadata" # the full data, ignoring deletes, is 5 rows -- the baseline the delete removes from -check "premise: the data file has 5 rows before deletes" \ +check "the data file has 5 rows before deletes" \ "$(q "SELECT count(*) FROM pgcolumnar.read_parquet('$DEST/db/t/data/data.parquet') AS t(id bigint, region text, amount int)")" "5" @@ -55,20 +106,24 @@ o = json.load(open(sys.argv[1])) print("\n".join("%d|%s|%d" % (r["id"], r["region"], r["amount"]) for r in o["surviving"])) PY )" -check "position deletes drop the listed rows (surviving == oracle)" \ +await_up +check_text "position deletes drop the listed rows (surviving == oracle)" \ "$(q "SELECT id || '|' || region || '|' || amount FROM pgcolumnar.iceberg_scan('$MDIR/apply.metadata.json') AS t(id bigint, region text, amount int) ORDER BY id")" \ "$ORACLE" +await_up check "the deleted ids (2, 4) are gone" \ "$(q "SELECT count(*) FROM pgcolumnar.iceberg_scan('$MDIR/apply.metadata.json') AS t(id bigint, region text, amount int) WHERE id IN (2,4)")" "0" +await_up check "exactly the two deleted rows were removed (5 - 2 = 3)" \ "$(q "SELECT count(*) FROM pgcolumnar.iceberg_scan('$MDIR/apply.metadata.json') AS t(id bigint, region text, amount int)")" "3" # deletes are by row ordinal, independent of column projection: reading only one # column must still drop the same rows (the ordinal is the row index, not a # column value) +await_up check "position deletes apply under column projection (id only -> 1,3,5)" \ "$(q "SELECT string_agg(id::text, ',' ORDER BY id) FROM pgcolumnar.iceberg_scan('$MDIR/apply.metadata.json') AS t(id bigint)")" \ @@ -79,9 +134,11 @@ check "position deletes apply under column projection (id only -> 1,3,5)" \ # which the spec says applies (data_seq <= delete_seq) -- the boundary that a # strict > would wrongly exclude. This arm is the other side: a delete OLDER than # the data (seq 4 < data seq 5) must NOT apply, so all five rows survive. +await_up check "a delete older than the data (lower seq) does not apply (5 rows)" \ "$(q "SELECT count(*) FROM pgcolumnar.iceberg_scan('$MDIR/noapply.metadata.json') AS t(id bigint, region text, amount int)")" "5" +await_up check "the would-be-deleted ids survive when the delete is older than the data" \ "$(q "SELECT count(*) FROM pgcolumnar.iceberg_scan('$MDIR/noapply.metadata.json') AS t(id bigint, region text, amount int) WHERE id IN (2,4)")" "2" @@ -90,6 +147,7 @@ check "the would-be-deleted ids survive when the delete is older than the data" # real writers record a null entry sequence number and inherit the manifest's; # this variant's delete entry is null, inheriting seq 5 from the manifest, so it # must still apply over the seq-5 data (3 rows survive), proving inheritance. +await_up check "a position delete with an inherited (null-entry) sequence number applies" \ "$(q "SELECT count(*) FROM pgcolumnar.iceberg_scan('$MDIR/inherit.metadata.json') AS t(id bigint, region text, amount int)")" "3" @@ -98,6 +156,7 @@ check "a position delete with an inherited (null-entry) sequence number applies" # the delete is applicable by sequence, but its rows target another data file, so # none of data.parquet's rows may be dropped (proves the path match excludes, not # just includes -- a basename-only or always-true match would wrongly delete). +await_up check "a position delete naming a different data file deletes nothing (5 rows)" \ "$(q "SELECT count(*) FROM pgcolumnar.iceberg_scan('$MDIR/wrongpath.metadata.json') AS t(id bigint, region text, amount int)")" "5" @@ -106,24 +165,27 @@ check "a position delete naming a different data file deletes nothing (5 rows)" # On current main a null pos is silently dropped (the `!n1 && !n2` filter) and a # negative pos is cast to a huge uint64 that matches nothing: the scan returns # the phantom row(s) with NO error. The reader must instead refuse (XX001). +CRASHES_BEFORE="$(crashes)" check "a position delete with a null pos is refused (XX001, not silently dropped)" \ - "$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/nullpos.metadata.json') - AS t(id bigint, region text, amount int)")" "XX001" + "$(refusal "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/nullpos.metadata.json') + AS t(id bigint, region text, amount int)" "has a null pos")" "XX001" check "...and the null-pos refusal names pos" \ "$(errmsg_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/nullpos.metadata.json') AS t(id bigint, region text, amount int)" | grep -c "null pos")" "1" -check "backend still up after the null-pos refusal" "$(q 'SELECT 1')" "1" +check "backend still up after the null-pos refusal" "$(survived "$CRASHES_BEFORE")" "1" +CRASHES_BEFORE="$(crashes)" check "a position delete with a negative pos is refused (XX001, not cast inert)" \ - "$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/negpos.metadata.json') - AS t(id bigint, region text, amount int)")" "XX001" + "$(refusal "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/negpos.metadata.json') + AS t(id bigint, region text, amount int)" "has a negative pos")" "XX001" check "...and the negative-pos refusal names the negative pos" \ "$(errmsg_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/negpos.metadata.json') AS t(id bigint, region text, amount int)" | grep -c "negative pos")" "1" -check "backend still up after the negative-pos refusal" "$(q 'SELECT 1')" "1" +check "backend still up after the negative-pos refusal" "$(survived "$CRASHES_BEFORE")" "1" # RISK control: pos == 0 is a LEGITIMATE delete of the first row (id 1). It must # apply, not be rejected -- a `pos <= 0` guard would wrongly turn this red. +await_up check "a position delete of ordinal 0 deletes the first row (id 1 gone, 4 remain)" \ "$(q "SELECT string_agg(id::text, ',' ORDER BY id) FROM pgcolumnar.iceberg_scan('$MDIR/zeropos.metadata.json') @@ -144,50 +206,62 @@ eq_ids_of() { AS t(id bigint, region text, amount int)" } -check "equality delete drops matching rows (survivors == oracle)" \ +CRASHES_BEFORE="$(crashes)" +await_up +check_text "equality delete drops matching rows (survivors == oracle)" \ "$(eq_ids_of eqapply)" "$(eq_oracle eqapply)" # THE boundary that distinguishes the equality rule (strict <) from the position # rule (<=): the same delete file at a sequence number EQUAL to the data's must # delete NOTHING (an equality delete never touches same-commit data) -check "an equality delete at an equal sequence number does not apply" \ +await_up +check_text "an equality delete at an equal sequence number does not apply" \ "$(eq_ids_of eqboundary)" "$(eq_oracle eqboundary)" # multi-column equality is an AND: (3,'eu') matches no row (3 is 'us'), so only # (4,'us') deletes -check "multi-column equality matches all columns (AND, not OR)" \ +await_up +check_text "multi-column equality matches all columns (AND, not OR)" \ "$(eq_ids_of eqmulti)" "$(eq_oracle eqmulti)" # only the equality_ids columns define the match; the delete file's extra amount # column carries a wrong value that must be ignored -check "columns beyond equality_ids are ignored for matching" \ +await_up +check_text "columns beyond equality_ids are ignored for matching" \ "$(eq_ids_of eqextra)" "$(eq_oracle eqextra)" # null matches null (IS NULL semantics), and ONLY null: id 6 (region NULL) goes, # id 7 ('eu') stays -check "a null delete value matches only null data values" \ +await_up +check_text "a null delete value matches only null data values" \ "$(eq_ids_of eqnull)" "$(eq_oracle eqnull)" +await_up check "the null-region row is gone, the non-null rows remain" \ "$(q "SELECT count(*) FROM pgcolumnar.iceberg_scan('$MDIR/eqnull.metadata.json') AS t(id bigint, region text, amount int) WHERE region IS NULL")" "0" # two delete files with different equality_ids in one manifest: both apply -check "multiple equality-delete files apply their union" \ +await_up +check_text "multiple equality-delete files apply their union" \ "$(eq_ids_of eqtwo)" "$(eq_oracle eqtwo)" # position and equality deletes in one snapshot: the skip sets merge -check "position and equality deletes combine" \ +await_up +check_text "position and equality deletes combine" \ "$(eq_ids_of eqmixed)" "$(eq_oracle eqmixed)" # the delete columns come from the table schema, not the output list: a scan # projecting only amount must still drop the id-matched rows +await_up check "equality deletes apply when the delete column is not projected" \ "$(q "SELECT string_agg(amount::text, ',' ORDER BY amount) FROM pgcolumnar.iceberg_scan('$MDIR/eqapply.metadata.json') AS t(amount int)")" \ "10,30,50" -check "backend still up after the equality applications" "$(q 'SELECT 1')" "1" +check "backend still up after the equality applications" "$(survived "$CRASHES_BEFORE")" "1" # ---- 4b: equality refusal and corruption arms ------------------------------- # content=2 with null equality_ids is corrupt metadata (the spec requires it) +CRASHES_BEFORE="$(crashes)" check "an equality delete without equality_ids is refused (XX001)" \ - "$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/eqnoids.metadata.json') - AS t(id bigint, region text, amount int)")" "XX001" + "$(refusal "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/eqnoids.metadata.json') + AS t(id bigint, region text, amount int)" "has no equality_ids")" "XX001" # the pre-4b 'equality' fixture also lacks equality_ids, but its delete sits at # the data's own sequence number, so under the strict-< rule it can never apply # and is skipped before validation: the data reads untouched +await_up check "a never-applicable equality delete without equality_ids is skipped (5 rows)" \ "$(q "SELECT count(*) FROM pgcolumnar.iceberg_scan('$MDIR/equality.metadata.json') AS t(id bigint, region text, amount int)")" "5" @@ -209,13 +283,16 @@ o = json.load(open(sys.argv[1])) print(",".join(str(i) for i in o["part_surviving"][sys.argv[2]])) PY } -check "a partition-scoped equality delete applies only to its partition" \ +await_up +check_text "a partition-scoped equality delete applies only to its partition" \ "$(part_ids eqpart_apply)" "$(part_oracle eqpart_apply)" +await_up check "the us-partition rows survive an eu-scoped delete" \ "$(q "SELECT count(*) FROM pgcolumnar.iceberg_scan('$MDIR/eqpart_apply.metadata.json') AS t(id bigint, region text, grp int) WHERE region = 'us'")" "3" # a delete scoped to a partition (region=zz) that no data file is in applies to # nothing -- a legitimate no-op, not an error and not a global delete +await_up check "a delete scoped to an empty partition deletes nothing (5 rows)" \ "$(q "SELECT count(*) FROM pgcolumnar.iceberg_scan('$MDIR/eqpart_nomatch.metadata.json') AS t(id bigint, region text, grp int)")" "5" @@ -223,23 +300,24 @@ check "a delete scoped to an empty partition deletes nothing (5 rows)" \ # partition-scoped delete, would silently under-delete; refuse it (the mirror # of the delete-side has_spec_id check) check "a data file lacking partition_spec_id is refused under a partitioned delete (XX001)" \ - "$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/eqpart_datanospec.metadata.json') - AS t(id bigint, region text, grp int)")" "XX001" + "$(refusal "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/eqpart_datanospec.metadata.json') + AS t(id bigint, region text, grp int)" "carries no partition_spec_id, which a partition-scoped")" "XX001" # a delete written under a partition spec no data file uses (partition # evolution): per the spec a partitioned delete never crosses spec ids, so it # applies to nothing -- a no-op, all rows survive, never a global over-delete +await_up check "a cross-spec partition-scoped delete applies to nothing (5 rows)" \ "$(q "SELECT count(*) FROM pgcolumnar.iceberg_scan('$MDIR/eqpart_crossspec.metadata.json') AS t(id bigint, region text, grp int)")" "5" # a partition tuple the reader cannot compare exactly (a double) is refused check "a partition-scoped delete with an uncomparable partition value is refused (0A000)" \ - "$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/eqpart_incomparable.metadata.json') - AS t(id bigint, region text, grp int)")" "0A000" + "$(refusal "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/eqpart_incomparable.metadata.json') + AS t(id bigint, region text, grp int)" "has a partition tuple this reader cannot compare")" "0A000" # an equality column of a type with no supported mapping (timestamp) is refused # before any file is opened check "an equality delete on an unsupported column type is refused (0A000)" \ - "$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/eqtype.metadata.json') - AS t(id bigint, region text, amount int)")" "0A000" + "$(refusal "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/eqtype.metadata.json') + AS t(id bigint, region text, amount int)" "is not supported")" "0A000" check "...and the refusal names the type" \ "$(errmsg_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/eqtype.metadata.json') AS t(id bigint, region text, amount int)" | grep -c "type")" "1" @@ -247,65 +325,69 @@ check "...and the refusal names the type" \ # data file errors loudly (the reader's missing-field-id error), never silently # skips the delete check "an equality column missing from the data file errors (22023)" \ - "$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/eqmissing.metadata.json') - AS t(id bigint, region text, amount int)")" "22023" + "$(refusal "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/eqmissing.metadata.json') + AS t(id bigint, region text, amount int)" "no Parquet column has field id")" "22023" # a delete-file path outside the table root is stopped by the path boundary # (22023 from the boundary, BEFORE the file is opened; if the boundary were # gone, /etc/hostname would be read and fail differently) check "an equality-delete path outside the table location is refused (22023)" \ - "$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/eqescape.metadata.json') - AS t(id bigint, region text, amount int)")" "22023" + "$(refusal "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/eqescape.metadata.json') + AS t(id bigint, region text, amount int)" "is not under the table location")" "22023" check "...and the boundary refusal names the escape" \ "$(errmsg_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/eqescape.metadata.json') AS t(id bigint, region text, amount int)" | grep -c "table location")" "1" # a path under the table root that climbs out with ".." is refused by the # containment (canonicalize collapses it and the re-check fails) check "a dotdot-escaping delete path is refused (22023)" \ - "$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/eqdotdot.metadata.json') - AS t(id bigint, region text, amount int)")" "22023" -check "backend still up after the equality refusals" "$(q 'SELECT 1')" "1" + "$(refusal "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/eqdotdot.metadata.json') + AS t(id bigint, region text, amount int)" "escapes the table location")" "22023" +check "backend still up after the equality refusals" "$(survived "$CRASHES_BEFORE")" "1" # ---- 4b audit arms (multi-agent adversarial audit findings) ----------------- # an equality_ids value beyond int32 must refuse the manifest (XX001); a silent # 32-bit truncation would alias field 2^32+2 onto the real field 2 and delete # rows the manifest never named +CRASHES_BEFORE="$(crashes)" check "an equality_ids value beyond int32 refuses the manifest (XX001)" \ - "$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/eqbigid.metadata.json') - AS t(id bigint, region text, amount int)")" "XX001" + "$(refusal "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/eqbigid.metadata.json') + AS t(id bigint, region text, amount int)" "corrupt Avro manifest entry")" "XX001" # a manifest list omitting partition_spec_id leaves an equality delete's scope # undecidable; defaulting it to 0 (unpartitioned) would defeat the # partition-scope guard, so the reader must refuse (XX001) check "a manifest list without partition_spec_id is refused for equality deletes (XX001)" \ - "$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/eqnospec.metadata.json') - AS t(id bigint, region text, amount int)")" "XX001" + "$(refusal "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/eqnospec.metadata.json') + AS t(id bigint, region text, amount int)" "the manifest list carries no partition_spec_id")" "XX001" # a partition-specs entry matching the delete's spec id but lacking the # required "fields" key: treating it as unpartitioned would silently globalize # the delete, so it is corrupt metadata (XX001) check "a partition spec without its fields key is refused (XX001)" \ - "$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/eqnofields.metadata.json') - AS t(id bigint, region text, amount int)")" "XX001" + "$(refusal "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/eqnofields.metadata.json') + AS t(id bigint, region text, amount int)" "has no fields array")" "XX001" # an equality delete that can never apply (its sequence number exceeds no data # file's) is skipped per spec, not validated: this variant's delete has an # unsupported column type (field 8, timestamp) at the data's own sequence # number, and must NOT refuse -- all five rows read +await_up check "a never-applicable equality delete is skipped, not validated (5 rows)" \ "$(q "SELECT count(*) FROM pgcolumnar.iceberg_scan('$MDIR/eqstaletype.metadata.json') AS t(id bigint, region text, amount int)")" "5" # a v1-shaped manifest (no sequence_number column in the entry schema) defaults # every file's sequence number to 0 per spec, even for EXISTING entries -- the # opposite of badseq, whose v2 schema HAS the column and carries a null +await_up check "a v1 manifest without a sequence_number column reads (default 0, 5 rows)" \ "$(q "SELECT count(*) FROM pgcolumnar.iceberg_scan('$MDIR/v1seq.metadata.json') AS t(id bigint, region text, amount int)")" "5" -check "backend still up after the audit arms" "$(q 'SELECT 1')" "1" +check "backend still up after the audit arms" "$(survived "$CRASHES_BEFORE")" "1" # ---- spec: inheritance is ADDED-only; a null seq on an EXISTING entry is bad -- # the spec inherits a null sequence number only for status-1 (ADDED) entries; an # EXISTING (status 0) entry with a null sequence number is corrupt, and silently # inheriting the too-new manifest number could keep rows a delete should remove. +CRASHES_BEFORE="$(crashes)" check "an EXISTING entry with a null sequence number is refused (XX001)" \ - "$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/badseq.metadata.json') - AS t(id bigint, region text, amount int)")" "XX001" -check "backend still up after the bad-sequence refusal" "$(q 'SELECT 1')" "1" + "$(refusal "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/badseq.metadata.json') + AS t(id bigint, region text, amount int)" "has no sequence number")" "XX001" +check "backend still up after the bad-sequence refusal" "$(survived "$CRASHES_BEFORE")" "1" pgc_summary diff --git a/test/iceberg_dv.sh b/test/iceberg_dv.sh index d9df48ca8..5f8d80677 100755 --- a/test/iceberg_dv.sh +++ b/test/iceberg_dv.sh @@ -16,22 +16,71 @@ set -uo pipefail . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" -sqlstate_of() { - env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ - -d "$PGC_DB" -qtA 2>&1 < the SQLSTATE when SQL is refused with PHRASE in its message; +# "SQLSTATE (another refusal: MESSAGE)" when a different guard refused it; "no +# error" when it ran; nothing when the server could not be asked. An arm used to +# read the SQLSTATE alone, and several guards here raise XX001: with the +# manifest/footer offset match removed, a later XX001 refused the same file and +# that arm still passed. +refusal() { + local out st msg + await_up + out="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ + -d "$PGC_DB" -qtA 2>&1 <&1 < wait, at most 60 s, until the server answers. A refusal arm that kills a +# backend sends the server through crash recovery; without the wait, the next arms' +# readings fail on recovery and red as collateral, more or fewer of them by timing. +# The crash itself is still caught: survived() counts it. +await_up() { + local i + for i in $(seq 1 120); do + [ "$(q 'SELECT 1')" = 1 ] && return 0 + sleep 0.5 + done +} + +# Backends killed by a signal, from the server log: a number, or nothing when the +# log can't be read. A fresh connection answers again after crash recovery, so +# "SELECT 1" alone passed with a backend killed by a refusal arm (no +# referenced_data_file check: a NULL path reached strcmp); the survival checks +# below also require that this count has not moved. +crashes() { + local _n + _n="$(pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null)" + case "$_n" in '' | *[!0-9]*) ;; *) echo "$_n" ;; esac +} +survived() { # survived BEFORE -> "1" when the server answers and no backend died since + local _up _now + _up="$(q 'SELECT 1')"; _now="$(crashes)" + if ! [[ "$1" =~ ^[0-9]+$ && "$_now" =~ ^[0-9]+$ ]]; then echo "no (crash count unreadable)" + elif [ "$_up" = 1 ] && [ "$_now" = "$1" ]; then echo 1 + else echo "no ([$_up], $(( _now - $1 )) backend crashes)"; fi +} + FX="$(dirname "${BASH_SOURCE[0]}")/fixtures/iceberg" WHD="$FX/warehouse_del" [ -f "$WHD/db/t/metadata/dvapply.metadata.json" ] || pgc_skip fixture "deletion-vector fixtures are missing" @@ -60,100 +109,103 @@ dv_ids_of() { # ---- deletion vectors are applied ------------------------------------------- # THE <= boundary: a DV at the data file's own sequence number applies (same # rule as 4a position-delete files; a strict > would wrongly exclude it) -check "a deletion vector at an equal sequence number applies (survivors == oracle)" \ +CRASHES_BEFORE="$(crashes)" +check_text "a deletion vector at an equal sequence number applies (survivors == oracle)" \ "$(dv_ids_of dvapply)" "$(dv_oracle dvapply)" # the other side: a DV strictly older than the data applies to nothing -check "a deletion vector older than the data does not apply" \ +check_text "a deletion vector older than the data does not apply" \ "$(dv_ids_of dvnoapply)" "$(dv_oracle dvnoapply)" # the spec's supersede rule: a position-delete FILE applies only when no DV # must be applied to the data file. The fixture's DV drops ordinal 1 and its # (also applicable) Parquet posdel names ordinals 1 and 3; a UNION would drop # both, supersede drops only ordinal 1. -check "an applicable DV supersedes position-delete files for its data file" \ +check_text "an applicable DV supersedes position-delete files for its data file" \ "$(dv_ids_of dvsupersede)" "$(dv_oracle dvsupersede)" # supersede is per data file: a posdel targeting a DIFFERENT data file still # applies alongside the DV -check "supersede is per data file, not global" \ +check_text "supersede is per data file, not global" \ "$(dv_ids_of dvother)" "$(dv_oracle dvother)" # roaring coverage: run-form containers plus a second 64-bit bucket -check "a DV with run containers and a high-bucket position decodes" \ +check_text "a DV with run containers and a high-bucket position decodes" \ "$(dv_ids_of dvwide)" "$(dv_oracle dvwide)" # cookie 12347 (run containers marked in the run bitset) -check "a run-optimized DV (cookie 12347) decodes" \ +check_text "a run-optimized DV (cookie 12347) decodes" \ "$(dv_ids_of dvrun)" "$(dv_oracle dvrun)" # a bitset container (cardinality above 4096) -check "a bitset-container DV decodes" \ +check_text "a bitset-container DV decodes" \ "$(dv_ids_of dvbitset)" "$(dv_oracle dvbitset)" # one Puffin file carrying two DV blobs for two data files -check "two DV blobs in one Puffin file each apply to their own data file" \ +check_text "two DV blobs in one Puffin file each apply to their own data file" \ "$(dv_ids_of dvtwo)" "$(dv_oracle dvtwo)" -check "backend still up after the DV applications" "$(q 'SELECT 1')" "1" +check "backend still up after the DV applications" "$(survived "$CRASHES_BEFORE")" "1" # ---- refusal and corruption arms -------------------------------------------- +CRASHES_BEFORE="$(crashes)" # two DV entries referencing one data file: the spec says scan results are # undefined and readers may raise; we raise rather than guess check "two DV entries for one data file are refused (XX001)" \ - "$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvdup.metadata.json') - AS t(id bigint, region text, amount int)")" "XX001" + "$(refusal "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvdup.metadata.json') + AS t(id bigint, region text, amount int)" "two deletion vectors reference data file")" "XX001" check "a corrupted DV CRC is refused (XX001)" \ - "$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvbadcrc.metadata.json') - AS t(id bigint, region text, amount int)")" "XX001" + "$(refusal "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvbadcrc.metadata.json') + AS t(id bigint, region text, amount int)" "fails its CRC-32 check")" "XX001" check "a wrong DV blob magic is refused (XX001)" \ - "$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvbadmagic.metadata.json') - AS t(id bigint, region text, amount int)")" "XX001" + "$(refusal "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvbadmagic.metadata.json') + AS t(id bigint, region text, amount int)" "has a bad magic sequence")" "XX001" # the manifest's content_offset must exactly match the footer's blob offset check "a manifest/footer offset mismatch is refused (XX001)" \ - "$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvoffmismatch.metadata.json') - AS t(id bigint, region text, amount int)")" "XX001" + "$(refusal "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvoffmismatch.metadata.json') + AS t(id bigint, region text, amount int)" "has no blob matching the manifest's offset")" "XX001" # referenced_data_file is required for DVs check "a DV entry without referenced_data_file is refused (XX001)" \ - "$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvnoref.metadata.json') - AS t(id bigint, region text, amount int)")" "XX001" + "$(refusal "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvnoref.metadata.json') + AS t(id bigint, region text, amount int)" "has no referenced_data_file")" "XX001" # record_count is defined as the DV's cardinality check "a record_count that disagrees with the DV cardinality is refused (XX001)" \ - "$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvbadcount.metadata.json') - AS t(id bigint, region text, amount int)")" "XX001" + "$(refusal "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvbadcount.metadata.json') + AS t(id bigint, region text, amount int)" "positions but its manifest entry records")" "XX001" # deletion-vector-v1 blobs must not declare a compression codec check "a DV blob declaring a compression codec is refused (XX001)" \ - "$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvcompressed.metadata.json') - AS t(id bigint, region text, amount int)")" "XX001" + "$(refusal "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvcompressed.metadata.json') + AS t(id bigint, region text, amount int)" "declares a compression codec")" "XX001" # a compressed Puffin footer is refused loudly, never misparsed check "a compressed Puffin footer is refused (0A000)" \ - "$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvflags.metadata.json') - AS t(id bigint, region text, amount int)")" "0A000" + "$(refusal "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvflags.metadata.json') + AS t(id bigint, region text, amount int)" "has a compressed Puffin footer")" "0A000" check "...and the refusal names the compressed footer" \ "$(errmsg_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvflags.metadata.json') AS t(id bigint, region text, amount int)" | grep -c "compressed")" "1" # deletion vectors are v3-only check "a deletion vector in a format-version 2 table is refused (0A000)" \ - "$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvv2.metadata.json') - AS t(id bigint, region text, amount int)")" "0A000" + "$(refusal "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvv2.metadata.json') + AS t(id bigint, region text, amount int)" "is a deletion vector, which format-version")" "0A000" check "...and the refusal names the format version" \ "$(errmsg_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvv2.metadata.json') AS t(id bigint, region text, amount int)" | grep -c "format-version")" "1" # a Puffin path outside the table root is stopped by the path boundary check "a Puffin path outside the table location is refused (22023)" \ - "$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvescape.metadata.json') - AS t(id bigint, region text, amount int)")" "22023" -check "backend still up after the DV refusals" "$(q 'SELECT 1')" "1" + "$(refusal "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvescape.metadata.json') + AS t(id bigint, region text, amount int)" "is not under the table location")" "22023" +check "backend still up after the DV refusals" "$(survived "$CRASHES_BEFORE")" "1" # ---- audit arms (multi-agent adversarial audit findings) -------------------- +CRASHES_BEFORE="$(crashes)" # content_offset + content_size chosen to overflow int64: a bounds check that # adds the operands would wrap negative and read a wild pointer; testing each # operand against the file size refuses it cleanly check "a DV whose offset+size overflows int64 is refused, not read (XX001)" \ - "$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvbigoff.metadata.json') - AS t(id bigint, region text, amount int)")" "XX001" + "$(refusal "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvbigoff.metadata.json') + AS t(id bigint, region text, amount int)" "lies outside the file's blob region")" "XX001" # a run container whose start+length exceeds the 16-bit container range would # fabricate positions in a neighboring container; refuse rather than corrupt check "a run container overflowing its 16-bit range is refused (XX001)" \ - "$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvrunoverflow.metadata.json') - AS t(id bigint, region text, amount int)")" "XX001" + "$(refusal "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvrunoverflow.metadata.json') + AS t(id bigint, region text, amount int)" "overflows its 16-bit range")" "XX001" # two DV entries for one data file recorded with and without the file:// scheme: # the one-DV-per-file check strips the scheme, so the aliased pair is caught check "an aliased duplicate DV (scheme vs no scheme) is refused (XX001)" \ - "$(sqlstate_of "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvdupscheme.metadata.json') - AS t(id bigint, region text, amount int)")" "XX001" -check "backend still up after the audit arms" "$(q 'SELECT 1')" "1" + "$(refusal "SELECT * FROM pgcolumnar.iceberg_scan('$MDIR/dvdupscheme.metadata.json') + AS t(id bigint, region text, amount int)" "two deletion vectors reference data file")" "XX001" +check "backend still up after the audit arms" "$(survived "$CRASHES_BEFORE")" "1" pgc_summary diff --git a/test/iceberg_rest.sh b/test/iceberg_rest.sh index 8383f5d34..a97ac65b2 100755 --- a/test/iceberg_rest.sh +++ b/test/iceberg_rest.sh @@ -41,12 +41,42 @@ rest_teardown() { } trap rest_teardown EXIT INT TERM -sqlstate_of() { - env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ - -d "$PGC_DB" -qtA 2>&1 < "