From b76cae624db97b532ac15b09c37c03cba682355f Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 01:11:24 +0000 Subject: [PATCH 1/4] test: the time-series recipes in how-to.md answer what they claim (#1296), red Gap fill, LOCF and linear interpolation are recipes, not features (phase 0 on #1296). This suite reads each recipe from docs/how-to.md, as the first sql block under its heading, runs it against the table the document names, and compares every bucket with a plpgsql reference that walks the raw rows and shares no window function or join with the recipes. The 24-hour fixture holds a leading gap, a run of gaps, a bucket with some NULLs, a bucket of only NULLs and a trailing gap; values grow as the square of the hour. The three 'the how-to gives the ... recipe' checks read the document, so they are arms, not premises (#1236; OffgridwithJD's review of #1311). Red on main: the how-to has no such headings, so those three read 0 and the comparisons get nothing. The reference and columnar premises pass. Co-Authored-By: Claude Opus 5.5 --- test/timeseries_recipes.sh | 116 +++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100755 test/timeseries_recipes.sh diff --git a/test/timeseries_recipes.sh b/test/timeseries_recipes.sh new file mode 100755 index 000000000..bbb2d7af7 --- /dev/null +++ b/test/timeseries_recipes.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# +# pgColumnar #1296: the time-series recipes in docs/how-to.md answer what they +# claim, on a columnar table. Gap filling, last observation carried forward +# (LOCF) and linear interpolation need no feature: core SQL expresses them, and +# phase 0 on #1296 measured the gap fill at 0.69 s on 8.75M columnar rows with +# the vectorized grouped node still under the LEFT JOIN. So the deliverable is +# the recipe, and this suite holds the recipe to a reference. +# +# THE RECIPES ARE READ FROM THE DOCUMENT, not copied here: each is the first +# ```sql block after its heading in docs/how-to.md, run as written against a +# table named as the document names it. A recipe edited in the docs is the +# recipe tested; one deleted from the docs fails its premise. +# +# THE REFERENCE IS COMPUTED ANOTHER WAY: a plpgsql loop over the 24 hours that +# reads the raw rows of each hour and walks back and forward for neighbours. It +# shares no window function and no LEFT JOIN with the recipes. +# +# The fixture's 24 hours are built to hold every case a recipe can get wrong: +# hour 0 empty, first nothing to carry; nothing to interpolate from +# hours 1, 2 data +# hours 3..5 empty, a run carried from 2; interpolated 2 -> 6 +# hour 6 data with some NULLs avg ignores the NULLs +# hour 7 rows, all v NULL a bucket with rows but no value: missing +# hours 8..21 data +# hour 22, 23 empty, last carried from 21; nothing to interpolate to +# +# Usage: test/timeseries_recipes.sh [PG_CONFIG] +# Written fresh for pgColumnar. + +set -uo pipefail +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" + +DOC="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/docs/how-to.md" +# These three are arms, not premises: they read the document, so editing it +# reddens them (#1236; OffgridwithJD's review of #1311). +# recipe : the first ```sql block after that heading, its ';' removed. +recipe() { + awk -v h="$1" '$0 == h {f=1; next} f && /^```sql/ {g=1; next} g && /^```/ {exit} g {print}' "$DOC" 2>/dev/null | + sed -e 's/;[[:space:]]*$//' +} +FILL="$(recipe '### Fill empty buckets')" +LOCF="$(recipe '### Carry the last value forward')" +INTERP="$(recipe '### Interpolate between known values')" +check "the how-to gives the gap-fill recipe" "$(printf '%s' "$FILL" | grep -c 'generate_series')" "1" +check "the how-to gives the LOCF recipe" "$(printf '%s' "$LOCF" | grep -c 'readings')" "1" +check "the how-to gives the interpolation recipe" "$(printf '%s' "$INTERP" | grep -c 'readings')" "1" + +# The table the document names. Values grow as the square of the hour, so a +# straight line through the wrong neighbours lands on the wrong value; they are +# multiples of 1/4, so every average and interpolation below is exact in float8. +psql_run "CREATE TABLE readings (ts timestamptz, v float8) USING pgcolumnar;" +psql_run "SET TimeZone = 'UTC'; + INSERT INTO readings + SELECT timestamptz '2026-10-01 00:00+00' + h * interval '1 hour' + m * interval '1 minute', + CASE WHEN h = 7 THEN NULL + WHEN h = 6 AND m % 3 = 0 THEN NULL + ELSE (h * h * 4 + m % 4) / 4.0 END + FROM generate_series(0, 23) h, generate_series(0, 59, 5) m + WHERE h NOT IN (0, 3, 4, 5, 22, 23);" +check "premise: the readings table is columnar" \ + "$(q "SELECT a.amname FROM pg_class c JOIN pg_am a ON a.oid = c.relam WHERE c.oid = 'readings'::regclass;")" "pgcolumnar" + +# The reference: bucket | value | carried | interpolated, one line per hour. +psql_run "CREATE FUNCTION ref_series() RETURNS TABLE (bucket timestamptz, val float8, locf float8, interp float8) +LANGUAGE plpgsql AS \$\$ +DECLARE + t0 timestamptz := timestamptz '2026-10-01 00:00+00'; + vals float8[] := array_fill(NULL::float8, ARRAY[24]); + i int; j int; p int; n int; a float8; +BEGIN + FOR i IN 0..23 LOOP + SELECT avg(r.v) INTO a FROM readings r + WHERE r.ts >= t0 + i * interval '1 hour' AND r.ts < t0 + (i + 1) * interval '1 hour'; + vals[i + 1] := a; + END LOOP; + FOR i IN 0..23 LOOP + bucket := t0 + i * interval '1 hour'; + val := vals[i + 1]; + p := NULL; n := NULL; + FOR j IN REVERSE i..0 LOOP IF vals[j + 1] IS NOT NULL THEN p := j; EXIT; END IF; END LOOP; + FOR j IN i..23 LOOP IF vals[j + 1] IS NOT NULL THEN n := j; EXIT; END IF; END LOOP; + locf := CASE WHEN p IS NULL THEN NULL ELSE vals[p + 1] END; + interp := CASE WHEN val IS NOT NULL THEN val + WHEN p IS NULL OR n IS NULL THEN NULL + ELSE vals[p + 1] + (vals[n + 1] - vals[p + 1]) * (i - p)::float8 / (n - p) END; + RETURN NEXT; + END LOOP; +END \$\$;" + +# Rows as text, rounded to 9 places, NULL spelled out, so a mismatch prints. +asrows() { q "SELECT string_agg(to_char(bucket AT TIME ZONE 'UTC', 'HH24') || '=' || COALESCE(round(x::numeric, 9)::text, 'NULL'), ' ' ORDER BY bucket) FROM ($1) s(bucket, x);"; } + +check "premise: the reference has 24 hours, 6 of them empty and one all-NULL" \ + "$(q "SELECT count(*) || '/' || count(*) FILTER (WHERE val IS NULL) FROM ref_series();")" "24/7" + +check "the gap-fill recipe returns every bucket, empty ones as NULL" \ + "$(asrows "SELECT * FROM ($FILL) r")" "$(asrows "SELECT bucket, val FROM ref_series()")" +check "the LOCF recipe carries the last value forward, and leaves a leading gap NULL" \ + "$(asrows "SELECT bucket, avg_v_locf FROM ($LOCF) r")" "$(asrows "SELECT bucket, locf FROM ref_series()")" +check "the interpolation recipe fills a gap linearly, and leaves both ends NULL" \ + "$(asrows "SELECT bucket, avg_v_interp FROM ($INTERP) r")" "$(asrows "SELECT bucket, interp FROM ref_series()")" + +# date_trunc truncates in the session's zone, so in a zone with a half-hour +# offset its buckets start at :30 UTC and match no bucket of the series. The +# recipe uses date_bin with the series start as origin, which has no zone in it. +check "the gap-fill recipe gives the same buckets in a session at a half-hour offset" \ + "$(PGTZ=Asia/Kolkata asrows "SELECT * FROM ($FILL) r")" "$(asrows "SELECT bucket, val FROM ref_series()")" + +# On a columnar table the recipes are not just correct but take the fast path: +# the aggregate under the LEFT JOIN runs as the vectorized grouped node. +check "the gap-fill recipe's aggregate runs as the vectorized grouped node when it is on" \ + "$(q "SET pgcolumnar.enable_group_vectorization = on; EXPLAIN (COSTS OFF) $FILL" | grep -c 'Columnar Vectorized Group Keys')" "1" + +pgc_summary From ed2a9f92515984acece57a26c8f4084112a34143 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 01:11:24 +0000 Subject: [PATCH 2/4] docs: recipes to fill gaps, carry values forward and interpolate (#1296) A how-to section with three recipes over readings (ts, v) in one-hour buckets: a series LEFT JOIN the bucketed aggregate; LOCF by a running count of non-null values and first_value; linear interpolation by running counts in both directions. The buckets come from date_bin with the series start as origin, not date_trunc, which truncates in the session's zone: with date_trunc in the recipe, an Asia/Kolkata session read all 24 buckets NULL. Co-Authored-By: Claude Opus 5.5 --- docs/how-to.md | 109 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/docs/how-to.md b/docs/how-to.md index 606a77eca..7d66f4272 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -342,6 +342,115 @@ the time zone your sessions use. For a cutoff in your session's time zone, run `pgcolumnar.reset_options` has no argument for the retention. To stop expiring, set `ttl_interval` to a span no row will reach. +## Fill gaps in a time series + +A bucketed query returns no row for a bucket with no data. These three recipes +return a row for every bucket, then fill the empty ones. They are plain SQL, so +they need nothing beyond pgColumnar. + +The examples use a table `readings (ts timestamptz, v float8)` and one-hour +buckets over one day. To adapt one, change the table, the column names, the +range and the width. Keep the origin of `date_bin` equal to the start of the +series, so every bucket the aggregate makes is a bucket the series has. + +`date_bin` with an explicit origin does not depend on the session's time zone. +`date_trunc('hour', ts)` does: in a zone with a half-hour offset, its buckets +start at half past the hour and match no bucket in the series. + +### Fill empty buckets + +Join the bucketed aggregate to a series of every bucket. An empty bucket +returns `NULL`. + +```sql +SELECT b.bucket, a.avg_v + FROM generate_series(timestamptz '2026-10-01 00:00+00', + timestamptz '2026-10-01 23:00+00', + interval '1 hour') AS b(bucket) + LEFT JOIN (SELECT date_bin(interval '1 hour', ts, timestamptz '2026-10-01 00:00+00') AS bucket, + avg(v) AS avg_v + FROM readings + GROUP BY 1) AS a USING (bucket) + ORDER BY b.bucket; +``` + +With `pgcolumnar.enable_group_vectorization` on, the aggregate under the join +runs as the vectorized grouped node. + +### Carry the last value forward + +Last observation carried forward (LOCF) fills an empty bucket with the value of +the latest bucket before it that has one. A running count of non-null values +puts each empty bucket in the same group as the last known value, and +`first_value` reads that value. + +```sql +WITH filled AS ( +SELECT b.bucket, a.avg_v + FROM generate_series(timestamptz '2026-10-01 00:00+00', + timestamptz '2026-10-01 23:00+00', + interval '1 hour') AS b(bucket) + LEFT JOIN (SELECT date_bin(interval '1 hour', ts, timestamptz '2026-10-01 00:00+00') AS bucket, + avg(v) AS avg_v + FROM readings + GROUP BY 1) AS a USING (bucket) +), grouped AS ( + SELECT bucket, avg_v, count(avg_v) OVER (ORDER BY bucket) AS grp + FROM filled +) +SELECT bucket, avg_v, + first_value(avg_v) OVER (PARTITION BY grp ORDER BY bucket) AS avg_v_locf + FROM grouped + ORDER BY bucket; +``` + +A bucket before the first value stays `NULL`, because there is nothing to +carry. A bucket whose rows all have `v` null has an average of `NULL`, so it +counts as empty and is filled. + +### Interpolate between known values + +Linear interpolation fills an empty bucket on the straight line between the +known buckets on either side. A running count in each direction finds both +neighbours. + +```sql +WITH filled AS ( +SELECT b.bucket, a.avg_v + FROM generate_series(timestamptz '2026-10-01 00:00+00', + timestamptz '2026-10-01 23:00+00', + interval '1 hour') AS b(bucket) + LEFT JOIN (SELECT date_bin(interval '1 hour', ts, timestamptz '2026-10-01 00:00+00') AS bucket, + avg(v) AS avg_v + FROM readings + GROUP BY 1) AS a USING (bucket) +), marks AS ( + SELECT bucket, avg_v, + count(avg_v) OVER (ORDER BY bucket) AS g_prev, + count(avg_v) OVER (ORDER BY bucket DESC) AS g_next + FROM filled +), ends AS ( + SELECT bucket, avg_v, + first_value(bucket) OVER (PARTITION BY g_prev ORDER BY bucket) AS prev_t, + first_value(avg_v) OVER (PARTITION BY g_prev ORDER BY bucket) AS prev_v, + first_value(bucket) OVER (PARTITION BY g_next ORDER BY bucket DESC) AS next_t, + first_value(avg_v) OVER (PARTITION BY g_next ORDER BY bucket DESC) AS next_v + FROM marks +) +SELECT bucket, + CASE WHEN avg_v IS NOT NULL THEN avg_v + WHEN prev_v IS NULL OR next_v IS NULL THEN NULL + ELSE prev_v + (next_v - prev_v) + * extract(epoch FROM bucket - prev_t) + / extract(epoch FROM next_t - prev_t) + END AS avg_v_interp + FROM ends + ORDER BY bucket; +``` + +A bucket before the first value or after the last one stays `NULL`, because it +has a neighbour on one side only. + ## Create time partitions ahead `pgcolumnar.partition_create_ahead` keeps a time-partitioned table ready for From 7eb0a174a811d8d77dfd993e20955090206b0189 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 01:11:24 +0000 Subject: [PATCH 3/4] test: the pytest twin of timeseries_recipes, and the suite registered (#1296) Independent of the shell suite: rows generated in Python and a reference computed in Python from them. compare_to_bash grades the pair 10 of 10. TESTS.md section 96; cluster_tests 503 -> 505 by collection. CHANGELOG. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 10 ++ test/pytest/TESTS.md | 20 ++++ test/pytest/expected_tests.txt | 5 +- test/pytest/test_compare_to_bash.py | 1 + test/pytest/test_timeseries_recipes.py | 149 +++++++++++++++++++++++++ test/run_all_versions.sh | 1 + 6 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 test/pytest/test_timeseries_recipes.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d69446de5..a309698b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,16 @@ true until the next version shipped. on, the aged rows go, the fresh rows stay, and a table with no retention is never touched or named; a failing `expire` does not undo a compaction; and the daemon yields `expire`'s lock to an `ACCESS EXCLUSIVE` request within 8 s of a 30 s hold. +- docs/how-to.md has recipes to fill gaps in a time series, carry the last value + forward (LOCF) and interpolate linearly (#1296). These need no feature: phase 0 + on #1296 measured the gap fill at 0.69 s on 8.75M columnar rows, with the + vectorized grouped node still under the LEFT JOIN, and found the `hll` and + `tdigest` extensions already read through pgColumnar's scan. The recipes bucket + with `date_bin` and the series start as origin, not `date_trunc`: in a session + at a half-hour offset (Asia/Kolkata), `date_trunc` made every bucket empty. + `test/timeseries_recipes.sh` and an independent `test_timeseries_recipes.py` + read each recipe from the document and hold it to a reference computed another + way, over a fixture with leading, inner and trailing gaps and an all-NULL hour. - `test/format_downgrade.sh` and `test_format_downgrade.py` test #1137's promise with both real libraries. They build `v1.0-alpha4`, write with it, upgrade to the diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 40e76eded..b8c389afd 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -143,6 +143,7 @@ behaviour, the source of that number is named. - [95. test_docs_how_to_sections.py: the pg_partman how-to keeps its own heading](#95-test_docs_how_to_sectionspy-the-pg_partman-how-to-keeps-its-own-heading) - [96. test_autovacuum_expire.py: the daemon expires retention tables only when told to](#96-test_autovacuum_expirepy-the-daemon-expires-retention-tables-only-when-told-to) - [97. test_rollup.py: incremental rollups fold appended row groups, and equal a full recompute](#97-test_rolluppy-incremental-rollups-fold-appended-row-groups-and-equal-a-full-recompute) +- [98. test_timeseries_recipes.py: the gap-fill, LOCF and interpolation recipes answer what they claim](#98-test_timeseries_recipespy-the-gap-fill-locf-and-interpolation-recipes-answer-what-they-claim) ## 1. How to read a test in here @@ -6619,3 +6620,22 @@ Removal proofs, each red on its arm: no `OFFSET 0` (the cost arm), no isolation | test | what it holds | | --- | --- | | `test_rollup_phase_one` | the rollup equals a full recompute after the first refresh, appends with late rows, a bulk append, and a rebase; a refresh with nothing new folds nothing; rows committed during a refresh are folded by the next, once; two concurrent refreshes fold each row once; zones agree; a refresh after a small append reads under a fifth of the source; a rewritten source is refused until `rebase_rollup`; a delete is refused (phase 1); a heap rollup behaves the same; REPEATABLE READ is refused; an owner who is not a superuser creates and refreshes a rollup, and another role sees none of it; bad definitions are refused with `22023` | + +## 98. test_timeseries_recipes.py: the gap-fill, LOCF and interpolation recipes answer what they claim + +Port of `timeseries_recipes.sh` (#1296). Phase 0 on #1296 found that core SQL expresses gap filling, last observation carried forward and linear interpolation fast enough on columnar, so the deliverable is three recipes in `docs/how-to.md`, and this file holds each one to a reference. + +THE RECIPES ARE READ FROM THE DOCUMENT, as the shell suite reads them: the first `sql` block under each heading, run as written against the table the document names. Everything else is this file's own. The rows are generated in Python, and the reference is computed in Python from those same rows, so it shares nothing with the shell suite's plpgsql reference. `compare_to_bash` grades the pair 10 of 10. + +THE FIXTURE holds an empty first hour, a run of empty hours, an hour with some NULL values, an hour of only NULL values and two empty last hours. Values grow as the square of the hour, so a line through the wrong neighbours lands on the wrong value. + +THE HALF-HOUR ZONE ARM is why the recipes bucket with `date_bin` and an explicit origin. `date_trunc` truncates in the session's zone, so in Asia/Kolkata its buckets start at :30 UTC. With `date_trunc` in the document, this arm read all 24 buckets as NULL. + +Red first: on main the how-to has no recipes, so the premise reads 0 and the comparisons get nothing. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_the_recipes_are_in_the_how_to` | each of the three headings has a `sql` block naming `readings` | +| `test_each_recipe_matches_the_reference` | the table is columnar; the gap fill, LOCF and interpolation match the Python reference for all 24 hours; the gap fill gives the same buckets in an Asia/Kolkata session; its aggregate runs as the vectorized grouped node when that is on | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index fde61ab8a..2c3009b81 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -574,4 +574,7 @@ guard_tests 415 # # cluster_tests 510 -> 511 for #1294's test_rollup.py, one test, collected over the # cluster files on this branch rebased onto b6656ac (#1307). -cluster_tests 511 +# +# cluster_tests 511 -> 513 for #1296's test_timeseries_recipes.py, two tests, by collection +# on this branch rebased onto 024e737. +cluster_tests 513 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index eeb2fa596..49c1cec9b 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -136,6 +136,7 @@ "storage_row_repair", "temporal", "tid_range_scan", + "timeseries_recipes", "ttl_expire", "validity_elision", "zonemap_boundaries"] diff --git a/test/pytest/test_timeseries_recipes.py b/test/pytest/test_timeseries_recipes.py new file mode 100644 index 000000000..b369e37a4 --- /dev/null +++ b/test/pytest/test_timeseries_recipes.py @@ -0,0 +1,149 @@ +"""The time-series recipes in docs/how-to.md answer what they claim (#1296). + +Gap filling, last observation carried forward (LOCF) and linear interpolation are +recipes rather than features: phase 0 on #1296 measured core SQL fast enough on +columnar, with the vectorized grouped node still under the LEFT JOIN. So the +deliverable is the recipe, and this file holds each one to a reference. + +Independent of `test/timeseries_recipes.sh`. The recipes are read from the +document, as the shell suite reads them, because the document is what is under +test. Everything else is this file's own: the rows are generated in Python and +inserted, and the reference is computed in Python from those same rows, so it +shares nothing with the shell suite's plpgsql reference. Check names match so the +pair can be graded by name. + +The fixture's 24 hours hold every case a recipe can get wrong: an empty first +hour, a run of empty hours, an hour with some NULL values, an hour of only NULL +values, and two empty last hours. Values grow as the square of the hour, so a +line through the wrong neighbours lands on the wrong value, and every value is a +multiple of 1/16, so every comparison is exact in float8. +""" + +import datetime +import pathlib +import re + +DOC = pathlib.Path(__file__).resolve().parents[2] / "docs" / "how-to.md" +T0 = datetime.datetime(2026, 10, 1, tzinfo=datetime.timezone.utc) +EMPTY = {0, 3, 4, 5, 22, 23} + + +def _recipe(heading): + """-> the first ```sql block after `heading` in the how-to, its ';' removed.""" + text = DOC.read_text() + parts = text.split("\n" + heading + "\n", 1) + if len(parts) != 2: + return None + m = re.search(r"```sql\n(.*?)```", parts[1], re.S) + return m.group(1).strip().rstrip(";") if m else None + + +def _rows(): + """-> (ts, v) for the fixture: every 10 minutes in each hour that has data.""" + out = [] + for h in range(24): + if h in EMPTY: + continue + for k in range(6): + ts = T0 + datetime.timedelta(hours=h, minutes=10 * k) + if h == 7 or (h == 6 and k % 2 == 0): + v = None + else: + v = (h * h * 4 + k % 4) / 4.0 + out.append((ts, v)) + return out + + +def _reference(rows): + """-> per hour: (average, carried, interpolated), computed from the raw rows.""" + by_hour = {h: [] for h in range(24)} + for ts, v in rows: + if v is not None: + by_hour[int((ts - T0).total_seconds() // 3600)].append(v) + avg = [sum(by_hour[h]) / len(by_hour[h]) if by_hour[h] else None for h in range(24)] + ref = [] + for h in range(24): + prev = next((p for p in range(h, -1, -1) if avg[p] is not None), None) + nxt = next((n for n in range(h, 24) if avg[n] is not None), None) + locf = avg[prev] if prev is not None else None + if avg[h] is not None: + interp = avg[h] + elif prev is None or nxt is None: + interp = None + else: + interp = avg[prev] + (avg[nxt] - avg[prev]) * (h - prev) / (nxt - prev) + ref.append((avg[h], locf, interp)) + return ref + + +def _fmt(pairs): + """-> 'HH=value ...' with values rounded to 9 places, so a mismatch prints.""" + return " ".join(f"{b.astimezone(datetime.timezone.utc):%H}=" + + ("NULL" if x is None else f"{round(x, 9):.9f}") for b, x in pairs) + + +def _query(conn, sql, cols): + with conn.cursor() as cur: + cur.execute(f"SELECT {cols} FROM ({sql}) r ORDER BY 1") + return [(row[0], row[1]) for row in cur.fetchall()] + + +def _expected(column): + ref = _reference(_rows()) + return _fmt([(T0 + datetime.timedelta(hours=h), ref[h][column]) for h in range(24)]) + + +def _setup(conn): + conn.execute("CREATE TABLE readings (ts timestamptz, v float8) USING pgcolumnar") + with conn.cursor() as cur: + cur.executemany("INSERT INTO readings VALUES (%s, %s)", _rows()) + + +def test_the_recipes_are_in_the_how_to(expect): + fill, locf, interp = (_recipe("### Fill empty buckets"), + _recipe("### Carry the last value forward"), + _recipe("### Interpolate between known values")) + expect.num(1 if fill and "generate_series" in fill else 0, 1, + "the how-to gives the gap-fill recipe") + expect.num(1 if locf and "readings" in locf else 0, 1, + "the how-to gives the LOCF recipe") + expect.num(1 if interp and "readings" in interp else 0, 1, + "the how-to gives the interpolation recipe") + + +def test_each_recipe_matches_the_reference(pgc_conn, expect): + _setup(pgc_conn) + am = pgc_conn.execute("SELECT a.amname FROM pg_class c JOIN pg_am a ON a.oid = c.relam" + " WHERE c.oid = 'readings'::regclass").fetchone()[0] + expect.text(am, "pgcolumnar", "premise: the readings table is columnar") + ref = _reference(_rows()) + expect.text(f"{len(ref)}/{sum(1 for a, _, _ in ref if a is None)}", "24/7", + "premise: the reference has 24 hours, 6 of them empty and one all-NULL") + + fill = _recipe("### Fill empty buckets") or "SELECT NULL::timestamptz, NULL::float8 WHERE false" + locf = _recipe("### Carry the last value forward") or "SELECT NULL::timestamptz AS bucket, NULL::float8 AS avg_v_locf WHERE false" + interp = _recipe("### Interpolate between known values") or "SELECT NULL::timestamptz AS bucket, NULL::float8 AS avg_v_interp WHERE false" + expect.text(_fmt(_query(pgc_conn, fill, "*")), _expected(0), + "the gap-fill recipe returns every bucket, empty ones as NULL") + expect.text(_fmt(_query(pgc_conn, locf, "bucket, avg_v_locf")), _expected(1), + "the LOCF recipe carries the last value forward, and leaves a leading gap NULL") + expect.text(_fmt(_query(pgc_conn, interp, "bucket, avg_v_interp")), _expected(2), + "the interpolation recipe fills a gap linearly, and leaves both ends NULL") + + # date_trunc truncates in the session's zone; at a half-hour offset its + # buckets start at :30 UTC and match none of the series. date_bin has no zone. + pgc_conn.execute("SET TimeZone = 'Asia/Kolkata'") + try: + kolkata = _fmt(_query(pgc_conn, fill, "*")) + finally: + pgc_conn.execute("RESET TimeZone") + expect.text(kolkata, _expected(0), + "the gap-fill recipe gives the same buckets in a session at a half-hour offset") + + pgc_conn.execute("SET pgcolumnar.enable_group_vectorization = on") + try: + plan = "\n".join(r[0] for r in pgc_conn.execute(f"EXPLAIN (COSTS OFF) {fill}").fetchall()) + finally: + pgc_conn.execute("RESET pgcolumnar.enable_group_vectorization") + expect.num(plan.count("Columnar Vectorized Group Keys"), 1, + "the gap-fill recipe's aggregate runs as the vectorized grouped node when it is on") diff --git a/test/run_all_versions.sh b/test/run_all_versions.sh index 9859dd5f5..3ed98890a 100755 --- a/test/run_all_versions.sh +++ b/test/run_all_versions.sh @@ -298,6 +298,7 @@ SUITES=( tablesample temporal tid_range_scan + timeseries_recipes truncate_cleanup ttl_expire ungrouped_vector_agg From 2fbc40c7f322b5c750eaac4acd458691bd112986 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 01:11:24 +0000 Subject: [PATCH 4/4] test: ledger rows for timeseries_recipes from five majors and eight mutations (#1296) 10 rows from a clean run (10/10 on each of 15-19) and eight mutations of the recipes, each on all five majors, merged one log at a time with --mutation and --target: date_trunc buckets, an inner join, LOCF by lag, interpolation toward the wrong neighbour, an ordered aggregate, and each of the three recipe headings renamed. Eight dated; the two still never are fixture premises. Census 1705 -> 1707, rows 1984 -> 1994. Co-Authored-By: Claude Opus 5.5 --- test/check_ledger.tsv | 10 ++++++++++ test/check_ledger_budget.txt | 17 ++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 7ad4c8dc7..345ee9186 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2032,6 +2032,16 @@ tid_range_scan tid_range_scan premise: the subquery cursor plans a TID range sca tid_range_scan tid_range_scan premise: the table has many row groups, so skipping can show 15;16;17;18;19 never - tid_range_scan tid_range_scan premise: the table holds every row 15;16;17;18;19 never - tid_range_scan tid_range_scan rows inserted earlier in the same transaction are in the range, as the filter sees them 15;16;17;18;19 never - +timeseries_recipes timeseries_recipes premise: the readings table is columnar 15;16;17;18;19 never - +timeseries_recipes timeseries_recipes premise: the reference has 24 hours, 6 of them empty and one all-NULL 15;16;17;18;19 never - +timeseries_recipes timeseries_recipes the LOCF recipe carries the last value forward, and leaves a leading gap NULL 15;16;17;18;19 2026-10-13 the how-to's 'Carry the last value forward' heading renamed, so the recipe is no longer found;the how-to's LOCF recipe carries with lag(), one bucket only, instead of first_value over the running-count group;the how-to's recipes join the series to the aggregate with JOIN instead of LEFT JOIN +timeseries_recipes timeseries_recipes the gap-fill recipe gives the same buckets in a session at a half-hour offset 15;16;17;18;19 2026-10-13 the how-to's 'Fill empty buckets' heading renamed, so the recipe is no longer found;the how-to's recipes bucket with date_trunc('hour', ts) instead of date_bin with the series start as origin;the how-to's recipes join the series to the aggregate with JOIN instead of LEFT JOIN +timeseries_recipes timeseries_recipes the gap-fill recipe returns every bucket, empty ones as NULL 15;16;17;18;19 2026-10-13 the how-to's 'Fill empty buckets' heading renamed, so the recipe is no longer found;the how-to's recipes join the series to the aggregate with JOIN instead of LEFT JOIN +timeseries_recipes timeseries_recipes the gap-fill recipe's aggregate runs as the vectorized grouped node when it is on 15;16;17;18;19 2026-10-13 the how-to's 'Fill empty buckets' heading renamed, so the recipe is no longer found;the how-to's recipes aggregate avg(v ORDER BY v), which the vectorized grouped node does not take +timeseries_recipes timeseries_recipes the how-to gives the LOCF recipe 15;16;17;18;19 2026-10-13 the how-to's 'Carry the last value forward' heading renamed, so the recipe is no longer found +timeseries_recipes timeseries_recipes the how-to gives the gap-fill recipe 15;16;17;18;19 2026-10-13 the how-to's 'Fill empty buckets' heading renamed, so the recipe is no longer found +timeseries_recipes timeseries_recipes the how-to gives the interpolation recipe 15;16;17;18;19 2026-10-13 the how-to's 'Interpolate between known values' heading renamed, so the recipe is no longer found +timeseries_recipes timeseries_recipes the interpolation recipe fills a gap linearly, and leaves both ends NULL 15;16;17;18;19 2026-10-13 the how-to's 'Interpolate between known values' heading renamed, so the recipe is no longer found;the how-to's interpolation recipe uses prev_v in place of next_v, so every gap is filled flat;the how-to's recipes join the series to the aggregate with JOIN instead of LEFT JOIN validity_elision validity_elision a chunk claiming no bitmap while it holds fewer values than rows is refused 15;16;17;18;19 never - validity_elision validity_elision a column with no nulls stores no validity bitmap 15;16;17;18;19 2026-09-18 writer: flush_one_column's presentCount == rowCount forced false, so the bitmap is written even when the chunk holds no null validity_elision validity_elision a null-free column elides its bitmap beside a null-bearing one in the same row group 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 242658166..663fb4c2c 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -1428,4 +1428,19 @@ suites_not_covered 247 # never 1686 + 12 = 1698 # dated 317 + 44 = 361 # 1698 + 361 == 2059 -checks_never_observed_red 1698 +# +# #1296 phase 1 (timeseries_recipes, PR #1311): 1698 -> 1700. 10 checks from +# five-major runs on this tree, so the suite is covered and suites_not_covered +# stays 247. Eight are dated by eight mutations of the recipes in docs/how-to.md, +# each merged one log at a time with --mutation and --target: date_trunc +# buckets, an inner join, LOCF by lag, interpolation toward the wrong +# neighbour, an ordered aggregate, and each of the three recipe headings +# renamed. The three 'the how-to gives the ... recipe' checks were premises in +# the first draft; editing the document reddens them, so by #1236 they are arms +# (OffgridwithJD's review of #1311). The two still never are fixture premises. +# +# ledger rows 2059 + 10 = 2069 +# never 1698 + 2 = 1700 +# dated 361 + 8 = 369 +# 1700 + 369 == 2069 +checks_never_observed_red 1700