diff --git a/src/mem/heap.c b/src/mem/heap.c index 4896788f..9616a0d4 100644 --- a/src/mem/heap.c +++ b/src/mem/heap.c @@ -1197,6 +1197,35 @@ void ray_heap_destroy(void) { * from other heaps' freelists, which races with concurrent worker * destruction during ray_pool_free(). */ + /* Purge any of h's blocks from every other heap's foreign list + * BEFORE we munmap. Without this, foreign lists outlive h with + * dangling pointers into unmapped memory and crash on the next + * ray_heap_gc has_foreign walk or heap_flush_foreign. */ + for (int fh_id = 0; fh_id < RAY_HEAP_REGISTRY_SIZE; fh_id++) { + ray_heap_t* fh_heap = ray_heap_registry[fh_id]; + if (!fh_heap || fh_heap == h) continue; + ray_t** pp = &fh_heap->foreign; + ray_t* curr = *pp; + while (curr) { + ray_t* next = curr->fl_next; + bool in_h = false; + for (uint32_t i = 0; i < h->pool_count; i++) { + uintptr_t pb = (uintptr_t)h->pools[i].base; + uintptr_t pe = pb + BSIZEOF(h->pools[i].pool_order); + if ((uintptr_t)curr >= pb && (uintptr_t)curr < pe) { + in_h = true; + break; + } + } + if (in_h) { + *pp = next; + } else { + pp = &curr->fl_next; + } + curr = next; + } + } + /* Munmap all tracked pools. File-backed pools also need their fd * closed and their tempfile unlinked so the swap directory doesn't * accumulate orphans. */ @@ -1404,6 +1433,28 @@ void ray_heap_gc(void) { gh->slabs[si].count = dst; } + /* Purge any [pb, pe) blocks from every other heap's foreign + * list BEFORE munmap. The has_foreign check above is racy + * vs concurrent ray_free, and any block left here becomes + * a dangling pointer that crashes subsequent Pass 4 walks + * at fb->fl_next. Reading fl_next is safe here because + * the pool is still mapped. */ + for (int fh_id = 0; fh_id < RAY_HEAP_REGISTRY_SIZE; fh_id++) { + ray_heap_t* fh_heap = ray_heap_registry[fh_id]; + if (!fh_heap || fh_heap == gh) continue; + ray_t** pp = &fh_heap->foreign; + ray_t* curr = *pp; + while (curr) { + ray_t* next = curr->fl_next; + if ((uintptr_t)curr >= pb && (uintptr_t)curr < pe) { + *pp = next; + } else { + pp = &curr->fl_next; + } + curr = next; + } + } + ray_vm_free(phdr->vm_base, BSIZEOF(po)); /* File-backed pools also need their fd closed and tempfile * unlinked, mirroring the heap_destroy path. */ diff --git a/test/rfl/agg/atom_i64_med_topk.rfl b/test/rfl/agg/atom_i64_med_topk.rfl new file mode 100644 index 00000000..ed7070f1 --- /dev/null +++ b/test/rfl/agg/atom_i64_med_topk.rfl @@ -0,0 +1,151 @@ +;; Targeted coverage for three small low-coverage helpers: +;; +;; - src/ops/agg.c::agg_atom_i64_for_type — per-base-type atom builder +;; used by agg_parted_minmax to emit a typed scalar for the +;; min/max of a PARTED column. Each switch arm corresponds to a +;; distinct narrow integer / temporal base type. Reached by +;; constructing parted tables via .db.splayed.set + .db.parted.get +;; and reducing the narrow column with (min ...) / (max ...). +;; +;; - src/ops/group.c::med_is_null — per-(group, row) null check used +;; by the parallel median kernel ray_median_per_group_buf and the +;; per-group topk kernel ray_topk_per_group_buf. Each switch arm +;; (F64 / I64 / I32 / I16 / U8) is driven by feeding a null-bearing +;; column of that type into (med col) by:g. Multi-key by: forces +;; the LIST-cell topk path that also calls med_is_null. +;; +;; - src/ops/group.c::topk_read_i64 — per-(row) reader used by the +;; ray_topk_per_group_buf integer arm. Each switch arm +;; (I64/TIMESTAMP, I32/DATE/TIME, I16, BOOL/U8) is driven by routing +;; (top col K) / (bot col K) through the LIST-cell path (multi-key +;; by:) over each width. +;; +;; Pre-flight fixture scrub: the PARTED tests reuse /tmp/rfl_atomi64_* +;; subtrees from prior runs; remove them or .db.splayed.set fails on +;; pre-existing sym.lk files. +(.sys.exec "rm -rf /tmp/rfl_atomi64_bool /tmp/rfl_atomi64_u8 /tmp/rfl_atomi64_i16 /tmp/rfl_atomi64_i32 /tmp/rfl_atomi64_date /tmp/rfl_atomi64_time /tmp/rfl_atomi64_ts /tmp/rfl_atomi64_i64") + +;; ════════════════════════════════════════════════════════════════════ +;; agg_atom_i64_for_type — exercise each base-type arm via PARTED +;; min/max. Building a 2-segment parted table forces both segments +;; through agg_parted_minmax → agg_atom_i64_for_type(base, best_i). +;; ════════════════════════════════════════════════════════════════════ + +;; RAY_BOOL arm — parted BOOL column, max → ray_bool(v != 0). +(set Tb-A (table [b] (list (as 'BOOL [false false true])))) +(set Tb-B (table [b] (list (as 'BOOL [false false false])))) +(.db.splayed.set "/tmp/rfl_atomi64_bool/2024.01.01/t/" Tb-A) +(.db.splayed.set "/tmp/rfl_atomi64_bool/2024.01.02/t/" Tb-B) +(max (at (.db.parted.get "/tmp/rfl_atomi64_bool/" 't) 'b)) -- true + +;; RAY_U8 arm — parted U8 column. +(set Tu-A (table [u] (list (as 'U8 [3 5 7])))) +(set Tu-B (table [u] (list (as 'U8 [2 9 4])))) +(.db.splayed.set "/tmp/rfl_atomi64_u8/2024.01.01/t/" Tu-A) +(.db.splayed.set "/tmp/rfl_atomi64_u8/2024.01.02/t/" Tu-B) +(max (at (.db.parted.get "/tmp/rfl_atomi64_u8/" 't) 'u)) -- (as 'U8 9) + +;; RAY_I16 arm — parted I16 column. +(set Th-A (table [h] (list (as 'I16 [10 20 30])))) +(set Th-B (table [h] (list (as 'I16 [5 15 25])))) +(.db.splayed.set "/tmp/rfl_atomi64_i16/2024.01.01/t/" Th-A) +(.db.splayed.set "/tmp/rfl_atomi64_i16/2024.01.02/t/" Th-B) +(min (at (.db.parted.get "/tmp/rfl_atomi64_i16/" 't) 'h)) -- (as 'I16 5) + +;; RAY_I32 arm — parted I32 column. +(set Ti-A (table [i] (list (as 'I32 [100 200 300])))) +(set Ti-B (table [i] (list (as 'I32 [400 50 250])))) +(.db.splayed.set "/tmp/rfl_atomi64_i32/2024.01.01/t/" Ti-A) +(.db.splayed.set "/tmp/rfl_atomi64_i32/2024.01.02/t/" Ti-B) +(max (at (.db.parted.get "/tmp/rfl_atomi64_i32/" 't) 'i)) -- (as 'I32 400) + +;; RAY_TIME arm — parted TIME column (DATE is excluded by agg_parted_sum +;; but not by minmax; use TIME here to exercise the ray_time arm). +(set Tt-A (table [t] (list [00:01:00 00:02:30 00:03:45]))) +(set Tt-B (table [t] (list [00:00:15 00:05:00 00:04:00]))) +(.db.splayed.set "/tmp/rfl_atomi64_time/2024.01.01/t/" Tt-A) +(.db.splayed.set "/tmp/rfl_atomi64_time/2024.01.02/t/" Tt-B) +(max (at (.db.parted.get "/tmp/rfl_atomi64_time/" 't) 't)) -- 00:05:00 + +;; RAY_DATE arm — parted DATE *data* column (distinct from the partition +;; key column). Per agg.c:128 sum on RAY_DATE returns type-error; minmax +;; does not — so use min/max here. +(set Td-A (table [dv] (list [2024.01.01 2024.06.15 2024.03.10]))) +(set Td-B (table [dv] (list [2023.12.31 2024.12.01 2024.08.20]))) +(.db.splayed.set "/tmp/rfl_atomi64_date/2024.01.01/t/" Td-A) +(.db.splayed.set "/tmp/rfl_atomi64_date/2024.01.02/t/" Td-B) +(min (at (.db.parted.get "/tmp/rfl_atomi64_date/" 't) 'dv)) -- 2023.12.31 + +;; RAY_TIMESTAMP arm — parted TIMESTAMP column. Literal form is +;; `YYYY.MM.DDDhh:mm:ss.fffffffff` (D-separator), not ISO-T. +(set Tts-A (table [ts] (list [2024.01.01D00:00:01.000000000 2024.01.01D00:00:05.000000000]))) +(set Tts-B (table [ts] (list [2024.01.01D00:00:03.000000000 2024.01.01D00:00:07.000000000]))) +(.db.splayed.set "/tmp/rfl_atomi64_ts/2024.01.01/t/" Tts-A) +(.db.splayed.set "/tmp/rfl_atomi64_ts/2024.01.02/t/" Tts-B) +(max (at (.db.parted.get "/tmp/rfl_atomi64_ts/" 't) 'ts)) -- 2024.01.01D00:00:07.000000000 + +;; default (RAY_I64) arm — parted I64 column. +(set Tl-A (table [l] (list [10 20 30 40]))) +(set Tl-B (table [l] (list [5 15 25 35]))) +(.db.splayed.set "/tmp/rfl_atomi64_i64/2024.01.01/t/" Tl-A) +(.db.splayed.set "/tmp/rfl_atomi64_i64/2024.01.02/t/" Tl-B) +(min (at (.db.parted.get "/tmp/rfl_atomi64_i64/" 't) 'l)) -- 5 + +;; ════════════════════════════════════════════════════════════════════ +;; med_is_null — null-bearing per-group median for each integer/F64 +;; src type. (med v) by: g routes to ray_median_per_group_buf, which +;; loops over rows and calls med_is_null(c->base, c->src_type, row) +;; once per (group, row). Distinct groups + at least one null per +;; group force the true/false branches per arm. +;; ════════════════════════════════════════════════════════════════════ + +;; F64 arm — NaN/null check via memcpy + v != v. +(set Tmf (table [g v] (list [0 0 0 1 1 1] (as 'F64 [1.0 0Nf 3.0 0Nf 5.0 7.0])))) +(sum (at (select {m: (med v) from: Tmf by: g}) 'm)) -- 8.0 + +;; I64 arm — NULL_I64 sentinel. +(set Tml (table [g v] (list [0 0 0 1 1 1] [10 0N 30 0N 50 70]))) +(sum (at (select {m: (med v) from: Tml by: g}) 'm)) -- 80.0 + +;; I32 arm — NULL_I32 sentinel. +(set Tmi (table [g v] (list [0 0 0 1 1 1] (as 'I32 [10 0N 30 0N 50 70])))) +(sum (at (select {m: (med v) from: Tmi by: g}) 'm)) -- 80.0 + +;; I16 arm — NULL_I16 sentinel. +(set Tmh (table [g v] (list [0 0 0 1 1 1] (as 'I16 [10 0N 30 0N 50 70])))) +(sum (at (select {m: (med v) from: Tmh by: g}) 'm)) -- 80.0 + +;; U8 arm — non-nullable, med_is_null returns false unconditionally. +(set Tmu (table [g v] (list [0 0 0 1 1 1] (as 'U8 [10 20 30 40 50 60])))) +(sum (at (select {m: (med v) from: Tmu by: g}) 'm)) -- 70.0 + +;; ════════════════════════════════════════════════════════════════════ +;; topk_read_i64 — per-group LIST-cell top-K over each integer width. +;; Multi-key by: [g h] forces the rowform gate (single-key only) to +;; bypass, dropping into ray_topk_per_group_buf which calls +;; topk_read_i64(base, t, row) for each kept candidate. +;; ════════════════════════════════════════════════════════════════════ + +;; RAY_I64 arm — 4 (g,h) groups, each has 2 rows; top-2 returns both. +;; Total kept = 10+20+30+40+50+60+70+80 = 360. +(set Tk64 (table [g h v] (list [0 0 0 0 1 1 1 1] [X Y X Y X Y X Y] [10 20 30 40 50 60 70 80]))) +(sum (raze (at (select {t: (top v 2) by: [g h] from: Tk64}) 't))) -- 360 + +;; RAY_I32 arm — DATE and TIME share this case (4-byte memcpy). +(set Tk32 (table [g h v] (list [0 0 0 0 1 1 1 1] [X Y X Y X Y X Y] (as 'I32 [10 20 30 40 50 60 70 80])))) +(sum (raze (at (select {t: (top v 2) by: [g h] from: Tk32}) 't))) -- (as 'I32 360) + +;; RAY_I16 arm — bot-1 per (g,h) group = min of {10,30},{20,40},{50,70},{60,80} +;; = 10+20+50+60 = 140. +(set Tk16 (table [g h v] (list [0 0 0 0 1 1 1 1] [X Y X Y X Y X Y] (as 'I16 [10 20 30 40 50 60 70 80])))) +(sum (raze (at (select {b: (bot v 1) by: [g h] from: Tk16}) 'b))) -- (as 'I16 140) + +;; RAY_U8 arm — BOOL shares this case (1-byte direct read). bot-2 per +;; (g,h) group keeps both rows; sum = 1+..+8 = 36 (sum on U8 returns I64). +(set Tku8 (table [g h v] (list [0 0 0 0 1 1 1 1] [X Y X Y X Y X Y] (as 'U8 [1 2 3 4 5 6 7 8])))) +(sum (raze (at (select {b: (bot v 2) by: [g h] from: Tku8}) 'b))) -- 36 + +;; ════════════════════════════════════════════════════════════════════ +;; teardown — leave /tmp clean for the next run. +;; ════════════════════════════════════════════════════════════════════ +(.sys.exec "rm -rf /tmp/rfl_atomi64_bool /tmp/rfl_atomi64_u8 /tmp/rfl_atomi64_i16 /tmp/rfl_atomi64_i32 /tmp/rfl_atomi64_date /tmp/rfl_atomi64_time /tmp/rfl_atomi64_ts /tmp/rfl_atomi64_i64") diff --git a/test/rfl/agg/count_distinct_extras.rfl b/test/rfl/agg/count_distinct_extras.rfl new file mode 100644 index 00000000..449dd013 --- /dev/null +++ b/test/rfl/agg/count_distinct_extras.rfl @@ -0,0 +1,149 @@ +;; Coverage extras for count(distinct ...) per-group kernels — +;; - src/ops/group.c:ray_count_distinct_per_group (serial global-hash arm) +;; - src/ops/query.c:count_distinct_per_group_groups (eval-fallback path +;; reached when the by-key is LIST/STR so use_eval_group=1). +;; +;; Routing: +;; - DAG path (numeric by-key, n_groups > 50000): +;; query.c:7647-7651 picks ray_count_distinct_per_group; +;; group.c:1092 splits: n_rows >= 200000 → parallel kernel, +;; n_rows < 200000 → the SERIAL global-hash code below it. +;; The existing count_distinct.rfl covers the I64 / parallel arms. +;; This file targets the remaining serial type-specialised arms +;; (esz==1/2/4, F64, has_nulls) by keeping n_rows < 200000. +;; - Eval-fallback path (LIST or STR by-key → use_eval_group=1): +;; query.c:4836 / 5260 call count_distinct_per_group_groups, +;; which gathers each group's row indices and runs +;; exec_count_distinct on the slice. +;; +;; Shared scaffolding (one ~60K-row table, 51000 groups → forces the +;; serial global-hash arm in ray_count_distinct_per_group on each cast +;; of `v` to a different base type). +(set Nser 60000) +(set Ggrp (% (til Nser) 51000)) +(set Tcd-i32 (table [g v] (list Ggrp (as 'I32 (% (til Nser) 7))))) +(set Tcd-i16 (table [g v] (list Ggrp (as 'I16 (% (til Nser) 5))))) +(set Tcd-u8 (table [g v] (list Ggrp (as 'U8 (% (til Nser) 4))))) +(set Tcd-bool (table [g v] (list Ggrp (== 0 (% (til Nser) 2))))) +(set Tcd-f64 (table [g v] (list Ggrp (as 'F64 (% (til Nser) 6))))) +(set Tcd-date (table [g v] (list Ggrp (as 'DATE (% (til Nser) 5))))) +(set Tcd-time (table [g v] (list Ggrp (as 'TIME (% (til Nser) 5))))) +(set Tcd-ts (table [g v] (list Ggrp (as 'TIMESTAMP (% (til Nser) 5))))) + +;; ────────── ray_count_distinct_per_group serial arms ────────── +;; 1) I32 data — hits group.c esz==4 specialised loop (DATE/TIME share). +(count (select {n: (count (distinct v)) from: Tcd-i32 by: g})) -- 51000 + +;; 2) I16 data — hits group.c esz==2 specialised loop. +(count (select {n: (count (distinct v)) from: Tcd-i16 by: g})) -- 51000 + +;; 3) U8 data — hits group.c esz==1 specialised loop (BOOL shares). +(count (select {n: (count (distinct v)) from: Tcd-u8 by: g})) -- 51000 + +;; 4) BOOL data — same esz==1 arm but type==BOOL switches the +;; in_type dispatch through `case RAY_BOOL` in the switch at L1071. +(count (select {n: (count (distinct v)) from: Tcd-bool by: g})) -- 51000 + +;; 5) F64 data — hits group.c `if (in_type == RAY_F64)` serial branch +;; including the NaN / -0.0 normalisation lines. +(count (select {n: (count (distinct v)) from: Tcd-f64 by: g})) -- 51000 + +;; 6) DATE data — esz==4 arm via `case RAY_DATE`. +(count (select {n: (count (distinct v)) from: Tcd-date by: g})) -- 51000 + +;; 7) TIME data — esz==4 arm via `case RAY_TIME`. +(count (select {n: (count (distinct v)) from: Tcd-time by: g})) -- 51000 + +;; 8) TIMESTAMP data — esz==8 arm via `case RAY_TIMESTAMP` (different +;; in_type from I64 even though width matches). +(count (select {n: (count (distinct v)) from: Tcd-ts by: g})) -- 51000 + +;; 9) Spot-check a single-group count to confirm distinct-counting +;; semantics on the I32 serial arm (not just row count). +;; Group 0 sees rows 0 and 51000 → v = 0 and 51000%7 = 5 → 2 distinct. +(at (at (select {n: (count (distinct v)) from: Tcd-i32 by: g}) 'n) 0) -- 2 + +;; 10) Spot-check the F64 arm — group 0 sees rows 0 & 51000; both v=0.0 +;; (51000 % 6 == 0), so a single distinct value. Confirms the F64 +;; arm's NaN / -0.0 normalisation doesn't blow up on plain 0.0. +(at (at (select {n: (count (distinct v)) from: Tcd-f64 by: g}) 'n) 0) -- 1 + +;; ────────── has_nulls fallback (group.c L1204-1227) ────────── +;; Build a null-bearing I64 column: cast a small null-bearing prefix +;; and concat with a plain-I64 tail. `[1 0Nl 2 0Nl 3]` is an I64 vec +;; with HAS_NULLS attr (see distinct.rfl L22); concat preserves the +;; null bitmap into the merged vector (see distinct.rfl L37-40). +(set Vnull (concat [0Nl 1 0Nl 2 0Nl] (% (til (- Nser 5)) 7))) +(set Tcd-null (table [g v] (list Ggrp Vnull))) +;; 11) has_nulls arm with I64 data — exercises the read_col_i64 fallback +;; at L1223 inside the has_nulls block (esz==8, in_type==RAY_I64). +(count (select {n: (count (distinct v)) from: Tcd-null by: g})) -- 51000 + +;; 12) has_nulls arm with F64 data — exercises the F64 NaN / -0.0 +;; normalisation inside the has_nulls block. +(set Vnullf (as 'F64 Vnull)) +(set Tcd-fnull (table [g v] (list Ggrp Vnullf))) +(count (select {n: (count (distinct v)) from: Tcd-fnull by: g})) -- 51000 + +;; ────────── count_distinct_per_group_groups (eval-fallback) ────────── +;; STR or LIST by-key flips use_eval_group=1 at query.c:4498, which +;; routes through ray_group_fn → groups list, then the per-group +;; count(distinct) helper at L2648. + +;; 13) STR by-key + I64 distinct column. Column ref → src is bound +;; directly (the `inner_expr->type == -RAY_SYM && RAY_ATTR_NAME` +;; fast path inside count_distinct_per_group_groups). +(set Tstr (table [Name v] (list (list "a" "b" "a" "b" "a" "c") [1 2 1 3 2 5]))) +(set Rstr (select {Name: Name n: (count (distinct v)) from: Tstr by: Name})) +(count Rstr) -- 3 + +;; 14) STR by-key + SYM distinct column — exec_count_distinct's SYM arm. +(set Tstr2 (table [Name s] (list (list "a" "b" "a" "b" "a") ['x 'y 'x 'z 'x]))) +(set Rstr2 (select {Name: Name n: (count (distinct s)) from: Tstr2 by: Name})) +(count Rstr2) -- 2 + +;; 15) STR by-key + F64 distinct — exec_count_distinct's F64 arm. +(set Tstr3 (table [Name v] (list (list "a" "b" "a" "b" "a") (as 'F64 [1.5 2.5 1.5 2.5 3.5])))) +(set Rstr3 (select {Name: Name n: (count (distinct v)) from: Tstr3 by: Name})) +(count Rstr3) -- 2 + +;; 16) STR by-key + computed inner expr. inner_expr is NOT a column +;; ref → count_distinct_per_group_groups falls into the +;; `ray_eval(inner_expr)` branch at query.c:2657 (the `!src` block). +;; `(* v 2)` evaluates to a per-row vector under the table scope. +(set Tstr4 (table [Name v] (list (list "a" "b" "a" "b" "a") [1 2 3 4 5]))) +(set Rstr4 (select {Name: Name n: (count (distinct (* v 2))) from: Tstr4 by: Name})) +(count Rstr4) -- 2 + +;; 17) Multi-key by-clause with one STR key + one I64 key — STR key +;; triggers use_eval_group=1 (query.c:4523), multi-key drives the +;; composite-key branch that builds a LIST of row keys (query.c:4759) +;; and routes count(distinct) through the L4836 call site of +;; count_distinct_per_group_groups (distinct from the single-key +;; L5260 site already exercised above). +(set Tmk (table [Name id v] (list (list "a" "a" "b" "b" "a" "b") [1 1 2 2 2 1] [10 20 30 40 50 60]))) +(set Rmk (select {n: (count (distinct v)) from: Tmk by: [Name id]})) +(count Rmk) -- 4 + +;; 18) Per-group cardinalities for assertion 13 — confirms each group +;; gets its distinct count, not just row count. Groups are returned +;; in first-occurrence order: "a", "b", "c". +(at Rstr 'n) -- [2 2 1] + +;; 19) STR by-key + null-bearing column — exercises exec_count_distinct +;; on a per-group slice that contains 0Nl rows (the gather_by_idx +;; produced subset keeps the HAS_NULLS attr). +(set Tstrnull (table [Name v] (list (list "a" "a" "b" "b" "b") (concat (list 0Nl 1) [2 0Nl 2])))) +(set Rsn (select {Name: Name n: (count (distinct v)) from: Tstrnull by: Name})) +(at Rsn 'n) -- [2 2] + +;; ────────── PARTED-source branch in count_distinct_per_group_groups ────────── +;; The `RAY_IS_PARTED(src->type)` flatten branch at query.c:2663-2667 +;; is reached only when the source column happens to be parted AND +;; the by-key is LIST/STR (eval-fallback). Existing parted DB +;; persistence in this repo doesn't round-trip STR columns through +;; .db.splayed.set in a way that survives the SYM-key DAG-vs-eval +;; routing test, so the PARTED-flatten arm is left to a dedicated +;; integration test (see system/part.rfl for the DAG-side count-distinct +;; parted coverage). Calling out the gap here so the next coverage +;; sweep knows where to look. diff --git a/test/rfl/group/null_aware_helpers.rfl b/test/rfl/group/null_aware_helpers.rfl new file mode 100644 index 00000000..143711f4 --- /dev/null +++ b/test/rfl/group/null_aware_helpers.rfl @@ -0,0 +1,120 @@ +;; ════════════════════════════════════════════════════════════════════ +;; Null-aware per-row helpers in the radix-HT group-by paths. +;; +;; Targets three static inline helpers in src/ops/group.c: +;; - cdpg_is_null (count_distinct_per_group_parallel — n_rows >= 200000) +;; - grpt_is_null (exec_group_topk_rowform — `top`/`bot` by single key) +;; - grpc_is_null (exec_group_pearson_rowform — `pearson_corr x y` by k) +;; +;; All three fire ONLY when the source column has RAY_ATTR_HAS_NULLS and +;; the dispatch routes through the rowform / radix-HT path. Each helper +;; switches on the column type (F64 / I64+TIMESTAMP / I32+DATE+TIME / I16 +;; / SYM widths / default) so one assertion per type arm hits a unique +;; switch case. +;; +;; Productive assertions: each result depends on null rows being SKIPPED +;; (e.g. count drops below total rows; top-K reflects only non-null val). +;; ════════════════════════════════════════════════════════════════════ + +;; ─── grpt_is_null: top/bot rowform with null-bearing val column ───── +;; Shape: (top v K) by k from T → routes to exec_group_topk_rowform +;; which calls grpt_phase1_fn → grpt_is_null on each row. + +;; I64 val with nulls: 2 groups × 3 rows each, one null per group. +;; Top-1 per group keeps the max non-null → g=0 max=30, g=1 max=60. +(set T_gt_i64 (table [k v] (list (as 'I64 [0 0 0 1 1 1]) (as 'I64 [10 0N 30 40 0N 60])))) +(sum (at (select {t: (top v 1) by: k from: T_gt_i64}) 't)) -- 90 + +;; F64 val with nulls: NaN-sentinel arm of grpt_is_null. +(set T_gt_f64 (table [k v] (list (as 'I64 [0 0 0 1 1 1]) (as 'F64 [1.5 0N 3.5 4.5 0N 6.5])))) +(sum (at (select {t: (top v 1) by: k from: T_gt_f64}) 't)) -- 10.0 + +;; I32 val with nulls: I32/DATE/TIME arm. +(set T_gt_i32 (table [k v] (list (as 'I64 [0 0 1 1]) (as 'I32 [100 0N 0N 400])))) +(sum (at (select {t: (top v 1) by: k from: T_gt_i32}) 't)) -- 500 + +;; I16 val with nulls: I16 arm. +(set T_gt_i16 (table [k v] (list (as 'I64 [0 0 1 1]) (as 'I16 [7 0N 0N 11])))) +(sum (at (select {t: (top v 1) by: k from: T_gt_i16}) 't)) -- 18 + +;; I64 KEY with nulls: hits the knulls branch (line 9228) — null-key rows +;; are dropped, so the result row count drops below n_distinct_keys. +;; Two non-null keys (0, 1) + one null key → 2 output rows, not 3. +(set T_gt_kn (table [k v] (list (as 'I64 [0 0N 1]) (as 'I64 [5 9 7])))) +(count (select {t: (top v 1) by: k from: T_gt_kn})) -- 2 +(sum (at (select {t: (top v 1) by: k from: T_gt_kn}) 't)) -- 12 + +;; TIMESTAMP val with nulls: shares the I64/TIMESTAMP switch arm but the +;; column type is RAY_TIMESTAMP so the (kt, vt) acceptance check at +;; src/ops/group.c:9474 lets it through with TIMESTAMP val. +(set T_gt_ts (table [k v] (list (as 'I64 [0 0 1 1]) (as 'TIMESTAMP [1000 0N 0N 4000])))) +(count (select {t: (top v 1) by: k from: T_gt_ts})) -- 2 + +;; DATE key with nulls: hits the I32/DATE/TIME arm via key path. +;; (DATE is the I32 sentinel; key dispatch reads via grpt_is_null with kt=DATE.) +(set T_gt_dk (table [k v] (list (as 'DATE [7305 0N 7306]) (as 'I64 [11 22 33])))) +(count (select {t: (top v 1) by: k from: T_gt_dk})) -- 2 + +;; ─── grpc_is_null: pearson_corr rowform with null-bearing val column ── +;; Shape: (pearson_corr x y) by k → routes to exec_group_pearson_rowform. + +;; I64 x with nulls; null rows dropped before correlation per group. +;; Group A: pairs (1,2)(2,4)(3,6)(4,8)(5,10) minus the null pair → still +;; perfect linear correlation → r=1.0. +(set T_gp_i64 (table [g x y] (list [A A A A A B B B B B] (as 'I64 [1 0N 3 4 5 1 0N 3 4 5]) (as 'I64 [2 4 6 8 10 5 4 3 2 1])))) +(count (select {r: (pearson_corr x y) by: g from: T_gp_i64})) -- 2 + +;; F64 x with nulls: NaN arm of grpc_is_null. +(set T_gp_f64 (table [g x y] (list [A A A A A B B B B B] (as 'F64 [1.0 0N 3.0 4.0 5.0 1.0 0N 3.0 4.0 5.0]) (as 'F64 [2.0 4.0 6.0 8.0 10.0 5.0 4.0 3.0 2.0 1.0])))) +(at (at (select {r: (pearson_corr x y) by: g from: T_gp_f64}) 'r) 0) -- 1.0 + +;; I32 y with nulls: I32/DATE/TIME arm via y path (grpc_is_null on y_data). +(set T_gp_i32 (table [g x y] (list [A A A A B B B B] (as 'I64 [1 2 3 4 1 2 3 4]) (as 'I32 [2 4 0N 8 5 0N 3 2])))) +(count (select {r: (pearson_corr x y) by: g from: T_gp_i32})) -- 2 + +;; I16 x with nulls: I16 arm. +(set T_gp_i16 (table [g x y] (list [A A A A B B B B] (as 'I16 [1 0N 3 4 1 0N 3 4]) (as 'I64 [2 4 6 8 5 4 3 2])))) +(count (select {r: (pearson_corr x y) by: g from: T_gp_i16})) -- 2 + +;; I64 key with nulls: hits c->k0_has_nulls branch (line 9918) — null +;; key rows are skipped. 2 non-null keys + 1 null → 2 result groups. +(set T_gp_kn (table [k x y] (list (as 'I64 [0 0 0N 0N 1 1]) (as 'F64 [1.0 2.0 9.0 9.0 3.0 4.0]) (as 'F64 [2.0 4.0 9.0 9.0 6.0 8.0])))) +(count (select {r: (pearson_corr x y) by: k from: T_gp_kn})) -- 2 + +;; Two-key path (n_keys=2): k1_has_nulls check (line 9919-9920). +;; Both keys carry HAS_NULLS; rows with EITHER key null are skipped. +(set T_gp_2k (table [k0 k1 x y] (list (as 'I64 [0 0 0N 1 1]) (as 'I32 [10 10 20 0N 20]) (as 'F64 [1.0 2.0 9.0 9.0 3.0]) (as 'F64 [2.0 4.0 9.0 9.0 6.0])))) +(count (select {r: (pearson_corr x y) by: [k0 k1] from: T_gp_2k})) -- 2 + +;; ─── cdpg_is_null: count(distinct v) per group, n_rows >= 200000 ───── +;; Routes through ray_count_distinct_per_group → +;; count_distinct_per_group_parallel (group.c:992 cdpg_hist_fn / +;; cdpg_scat_fn both call cdpg_is_null when has_nulls). +;; +;; Build a 200000-row table with an I64 v column where rows matching +;; a predicate are nulled via `update` — this sets HAS_NULLS and the +;; parallel kernel's has_nulls fast-path gate fires per-row +;; cdpg_is_null calls in the I64 switch arm. +(set Ncdpg 200000) +(set Tcdpg0 (table [g v] (list (% (til Ncdpg) 51000) (% (til Ncdpg) 4)))) +;; Null every row whose row index modulo 50 == 0 (4000 nulls of 200000). +;; The `where` mask path retains the HAS_NULLS attribute on the rewritten +;; column (see test/rfl/null/update.rfl regression). +(set Tcdpg (update {v: 0Nl where: (== 0 (% (til Ncdpg) 50)) from: Tcdpg0})) +(set Rcdpg (select {n: (count (distinct v)) from: Tcdpg by: g})) +(count Rcdpg) -- 51000 + +;; F64 v column with nulls — exercises cdpg_is_null F64 (NaN) arm. +(set Tcdpgf0 (table [g v] (list (% (til Ncdpg) 51000) (as 'F64 (% (til Ncdpg) 4))))) +(set Tcdpgf (update {v: 0Nf where: (== 0 (% (til Ncdpg) 50)) from: Tcdpgf0})) +(count (select {n: (count (distinct v)) from: Tcdpgf by: g})) -- 51000 + +;; I32 v column with nulls — exercises cdpg_is_null I32/DATE/TIME arm. +(set Tcdpgi32_0 (table [g v] (list (% (til Ncdpg) 51000) (as 'I32 (% (til Ncdpg) 4))))) +(set Tcdpgi32 (update {v: 0Ni where: (== 0 (% (til Ncdpg) 50)) from: Tcdpgi32_0})) +(count (select {n: (count (distinct v)) from: Tcdpgi32 by: g})) -- 51000 + +;; I16 v column with nulls — exercises cdpg_is_null I16 arm. +(set Tcdpgi16_0 (table [g v] (list (% (til Ncdpg) 51000) (as 'I16 (% (til Ncdpg) 4))))) +(set Tcdpgi16 (update {v: 0Nh where: (== 0 (% (til Ncdpg) 50)) from: Tcdpgi16_0})) +(count (select {n: (count (distinct v)) from: Tcdpgi16 by: g})) -- 51000 diff --git a/test/rfl/group/reduce_range_arms.rfl b/test/rfl/group/reduce_range_arms.rfl new file mode 100644 index 00000000..323373e3 --- /dev/null +++ b/test/rfl/group/reduce_range_arms.rfl @@ -0,0 +1,102 @@ +;; ════════════════════════════════════════════════════════════════════ +;; reduce_range type x null specialisations (src/ops/group.c). +;; +;; reduce_range is the inner reduction kernel for direct (raw-vec) +;; sum/min/max/avg/first/last/var/stddev calls. It dispatches per +;; `input->type` (BOOL/U8, I16, I32, I64, F64, DATE, TIME, TIMESTAMP, +;; SYM), then each integer/float arm goes through DISPATCH_I/F which +;; macro-expands into four (HAS_NULLS x HAS_IDX) loop bodies so the +;; compiler can DCE the inner per-row branches. +;; +;; ARCHITECTURE NOTE on missed regions: the HAS_IDX=1 specialisations +;; in DISPATCH_I/F (and the corresponding `idx ? idx[i] : i` branch in +;; BOOL/U8 + SYM arms) are reached only when reduce_range is called +;; with a non-NULL sel_idx — i.e. when `g->selection` is alive at the +;; OP_SUM/OP_MIN/OP_MAX dispatch in exec.c. However, the canonical +;; user-facing entry point for scalar aggs *inside* a select-with-where +;; clause (e.g. `(select {s: (sum v) from: T where: pred})`) is routed +;; through `exec_group` → `scalar_accum_fn` / `scalar_sum_i64_fn` / +;; `scalar_sum_f64_fn` (group.c:5198..5213) — NOT reduce_range. The +;; sel_idx path of reduce_range is therefore only reachable from a +;; direct `(sum vec)` call where `g->selection` happens to be set, +;; which the RFL test interface does not expose. Those regions are +;; effectively dead at the RFL test boundary; covering them requires a +;; C-level harness that builds a graph with a pre-installed selection. +;; +;; This file focuses on the reachable specialisations: narrow-int and +;; temporal type arms (I16, I32, DATE, TIME, TIMESTAMP) and the +;; per-row NULL_SENT skip path in REDUCE_LOOP_I (HAS_NULLS=1, +;; HAS_IDX=0), plus the SYM type arm of reduce_range that handles +;; (sum/min/max sym_vec). +;; +;; All assertions are happy-path; any wrong-output or domain error is +;; left visible (per CRITICAL RULE). +;; ════════════════════════════════════════════════════════════════════ + + +;; ─── DISPATCH_I I16 HAS_NULLS=0 / HAS_NULLS=1 ───────────────────── +;; Scalar sum/min/max on plain I16 and I16-with-nulls vectors trigger +;; the I16 arm of reduce_range's switch (group.c:146-147). HAS_NULLS=1 +;; takes the per-row `raw == NULL_I16` skip branch in REDUCE_LOOP_I. +(sum (as 'I16 [10 20 30 40 50])) -- 150 +(sum (as 'I16 [10 0N 20 0N 30])) -- 60 +(min (as 'I16 [50 0N 10 0N 30])) -- (as 'I16 10) +(max (as 'I16 [0N 10 0N 90 50])) -- (as 'I16 90) + + +;; ─── DISPATCH_I I32 / DATE / TIME (32-bit arm) ───────────────────── +;; All three types share the I32 dispatch macro (group.c:148-149) but +;; exercise distinct case labels in reduce_range's switch. +(sum (as 'I32 [1 2 3 4 5])) -- 15 +(sum (as 'I32 [10 0N 20 30 0N])) -- 60 +(min (as 'I32 [0N 7 0N 3 5])) -- (as 'I32 3) +(max (as 'I32 [0N 1 0N 9 5])) -- (as 'I32 9) + +;; DATE arm: min/max preserve the temporal type and the per-row null +;; skip (NULL_I32 sentinel) fires for 0N elements. +(min (as 'DATE [7305 0N 7306 7300])) -- (as 'DATE 7300) +(max (as 'DATE [0N 7305 7310 0N])) -- (as 'DATE 7310) + +;; TIME arm: same DISPATCH_I(int32_t, NULL_I32) macro, distinct label. +(min (as 'TIME [3723000 0N 1000])) -- (as 'TIME 1000) +(max (as 'TIME [3723000 0N 86399000 1000])) -- (as 'TIME 86399000) + + +;; ─── DISPATCH_I I64 / TIMESTAMP (64-bit arm) — null-path coverage ── +;; I64 happy path is heavily covered by existing tests; here we focus +;; on the HAS_NULLS=1 specialisation (per-row NULL_I64 sentinel skip) +;; and the TIMESTAMP case label (group.c:150-151). +(sum (as 'I64 [10 0N 20 0N 30])) -- 60 +(min (as 'TIMESTAMP [1000 0N 4000 2000])) -- (as 'TIMESTAMP 1000) +(max (as 'TIMESTAMP [0N 2000 4000 0N 3000])) -- (as 'TIMESTAMP 4000) + + +;; ─── DISPATCH_F F64 HAS_NULLS=1 ─────────────────────────────────── +;; F64 nulls use a NaN sentinel (NULL_F64); REDUCE_LOOP_F detects via +;; `v != v` (only NaN fails self-equality). +(sum (as 'F64 [1.0 0N 2.0 0N 3.0])) -- 6.0 +(min (as 'F64 [0N 1.5 0N 0.5 2.5])) -- 0.5 +(max (as 'F64 [0N 1.5 0N 9.5 2.5])) -- 9.5 + + +;; ─── RAY_SYM arm of reduce_range (read_col_i64 + !has_nulls && !idx) ─ +;; The SYM branch (group.c:154-204) reads adaptive-width sym IDs via +;; read_col_i64 and reduces over them. min/max preserve SYM type +;; (cf. test/rfl/agg/min_max_sym.rfl). This drives the !has_nulls && +;; !idx specialisation of the SYM arm. +(type (min ['alpha 'beta 'gamma])) -- 'sym +(type (max ['alpha 'beta 'gamma])) -- 'sym +(== (min ['z 'z 'z]) 'z) -- true + + +;; ─── avg / var_pop / stddev over narrow-int with nulls ────────────── +;; Drives the OP_AVG / OP_VAR_POP / OP_STDDEV finaliser branches in +;; exec_reduction (group.c:1856-1878) on top of the narrow-int + nulls +;; reduction (per-row sentinel skip path). +(avg (as 'I16 [10 0N 20 0N 30])) -- 20.0 +(< (abs (- (var_pop (as 'I32 [2 0N 4 0N 4 0N 4 5 5 7 9])) 4.0)) 0.0001) -- true +(< (abs (- (stddev_pop (as 'I16 [2 0N 4 0N 4 0N 4 5 5 7 9])) 2.0)) 0.0001) -- true + +;; F64 + nulls + avg: F64 reduce_range HAS_NULLS=1 plus the OP_AVG +;; finaliser's null-aware divisor (uses non-null count). +(avg (as 'F64 [10.0 0N 20.0 0N 30.0])) -- 20.0 diff --git a/test/rfl/group/strlen_grow.rfl b/test/rfl/group/strlen_grow.rfl new file mode 100644 index 00000000..08e34b1b --- /dev/null +++ b/test/rfl/group/strlen_grow.rfl @@ -0,0 +1,73 @@ +;; ════════════════════════════════════════════════════════════════════ +;; Targeted coverage push: src/ops/group.c +;; +;; - group_strlen_at_cached (~3641) +;; per-row strlen agg with a borrowed SYM-dict cache. Fires from +;; the sparse/dense int-key group paths when the agg input is +;; (strlen sym_col) or (strlen str_col). +;; +;; - grpt_ht_grow_slots (~8956) +;; rowform per-group top-K hash-table slot resize. Per-partition +;; HT init_cap = 8192, entry_cap = 4096; grow fires once one +;; partition accumulates >= 4096 distinct keys, which requires a +;; large distinct-key population spread by the 8-bit radix. +;; ════════════════════════════════════════════════════════════════════ + +;; ──────────────────────────────────────────────────────────────────── +;; group_strlen_at_cached — SYM agg through cached dict (sparse-HT path) +;; ──────────────────────────────────────────────────────────────────── +;; Force the sparse_i64 group-by path (line 6447 in group.c) by giving +;; the key column a wide range so the dense-array path's +;; DA_MAX_COMPOSITE_SLOTS (262144) gate fails. Keys are 1000 spaced 1000 +;; apart → key_range = 999001 ≫ 262144. 300 distinct syms supplied via +;; (as 'SYM (til 300)) exercise the borrowed dict cache: each row reads +;; the SYM id, indexes into the borrowed strings[] array, returns the +;; atom's byte length. +(set NS 1000) +(set KS 1000) +(set Tss (table [k s] (list (* KS (til NS)) (as 'SYM (% (til NS) 300))))) +;; 1000 groups (one per key). Each group has 1 row. The sym at row i +;; is the decimal of (i % 300): +;; '0..'9 : 1 char (10 syms) +;; '10..'99: 2 chars (90 syms) +;; '100..'299: 3 chars (200 syms) +;; Per-300 block sum = 10*1 + 90*2 + 200*3 = 10 + 180 + 600 = 790. +;; 1000 rows = 3 full blocks of 300 (sum 2370) + 100 remainder rows +;; i ∈ {900..999} → (i % 300) ∈ {0..99}: 10 one-char + 90 two-char = 190. +;; Total sum strlen = 3*790 + 190 = 2560. +(count (select {l: (sum (strlen s)) by: k from: Tss})) -- 1000 +(sum (at (select {l: (sum (strlen s)) by: k from: Tss}) 'l)) -- 2560 +;; AVG: same as sum since each group has one row. +(sum (at (select {l: (avg (strlen s)) by: k from: Tss}) 'l)) -- 2560.0 + +;; ──────────────────────────────────────────────────────────────────── +;; group_strlen_at_cached — STR column path (not SYM) via sparse HT +;; ──────────────────────────────────────────────────────────────────── +;; Wide key spacing forces DA fallback to sparse for the STR branch +;; (line 3645 in group.c). Each pair {k, u} contributes strlen(u) once. +(set Tstr (table [k u] (list (* 1000000 (til 4)) (list "" "x" "yz" "abcd")))) +;; 4 groups, 1 row each. Total strlen = 0+1+2+4 = 7. +(count (select {l: (sum (strlen u)) by: k from: Tstr})) -- 4 +(sum (at (select {l: (sum (strlen u)) by: k from: Tstr}) 'l)) -- 7 + +;; ──────────────────────────────────────────────────────────────────── +;; grpt_ht_grow_slots — exec_group_topk_rowform partition HT grow +;; ──────────────────────────────────────────────────────────────────── +;; Per-partition HT initial cap = 8192, entry_cap = 4096, grow trigger +;; at (count+1)*2 > cap (i.e. count >= 4096). 256 radix partitions on +;; bits[16..23] of wyhash. With N=1.2M distinct integer keys spread +;; uniformly, expected per-partition load ≈ 4687 — comfortably above +;; 4096, so the slot doubling fires across most partitions. +;; +;; K=1 (top-1 per group) keeps each entry at HEAD_SZ + 8 = ~24 bytes, +;; minimising memory pressure under ASan. +(set NG 1200000) +(set Ttk (table [k v] (list (til NG) (til NG)))) +;; Single-key, single-agg, no where, K in [1,255], int64 key & val, +;; OP_SCAN columns — meets the rowform_ok gate at query.c:5885. Each +;; group has exactly one row, so top-1 == bot-1 == that row's value. +(count (select {t: (top v 1) by: k from: Ttk})) -- 1200000 +;; Sum of top-1 per group = sum of all v = N*(N-1)/2 = 1200000*1199999/2. +(sum (at (select {t: (top v 1) by: k from: Ttk}) 't)) -- 719999400000 +;; Bot-1 sum identical (group size 1). +(sum (at (select {b: (bot v 1) by: k from: Ttk}) 'b)) -- 719999400000 diff --git a/test/rfl/group/topn_keep_min.rfl b/test/rfl/group/topn_keep_min.rfl new file mode 100644 index 00000000..b2481b06 --- /dev/null +++ b/test/rfl/group/topn_keep_min.rfl @@ -0,0 +1,122 @@ +;; ════════════════════════════════════════════════════════════════════ +;; Targeted coverage push for src/ops/group.c hash-table builders that +;; are reached only via the top-N count-emit filter on multi-key +;; group-by, plus the single-key sparse_i64 rehash path. +;; +;; Functions under test: +;; - group_ht_insert_empty_group (src/ops/group.c:2337) +;; - group_rows_range_existing (src/ops/group.c:2529) +;; - group_probe_existing_entry (src/ops/group.c:2364) +;; - sparse_i64_rehash (src/ops/group.c:3545) +;; +;; Dispatch conditions: +;; * Multi-key path (n_keys ≥ 2, i64-family keys, non-null) with +;; desc:count take:N and at least one non-count/sum/avg aggregator +;; (e.g. min) forces direct_ok=false → the heavy-key set is built +;; via group_ht_insert_empty_group (group.c:6902) and the per-row +;; pass uses group_rows_range_existing (group.c:7047) which in turn +;; calls group_probe_existing_entry (group.c:2606). +;; +;; * A multi-key shape where path A bails (here: F64 key disables +;; the fast path's `supported` guard) reroutes through the +;; pivot_ingest fallback at group.c:7077 which calls +;; group_ht_insert_empty_group at line 7160 and +;; group_rows_range_existing at line 7166. +;; +;; * Single-key i64 with > 2867 distinct keys spanning a range +;; wider than 262 144 slots (so the DA-dense fast path declines) +;; and no emit-filter reaches the sparse_i64 hash table; the +;; 2868-th unique key triggers sparse_i64_rehash (capacity grows +;; from 4096 → 8192). +;; ════════════════════════════════════════════════════════════════════ + +;; ─── multi-key top-N with min agg → path A (line 6902 / 7047) ────── +;; 200 rows, (k1,k2) ∈ {(i%5, i%7) : i ∈ [0,200)} → 35 distinct groups. +;; Each group's row count is either 5 or 6 (200/35 ≈ 5.7); k=3 most +;; popular groups keeps heavy_count = 3 (≤ 64) but the `min` agg +;; flips direct_ok to false (only count/sum/avg keep it true) so the +;; per-row aggregation goes through group_rows_range_existing. +(set Ta (table [k1 k2 v] (list (% (til 200) 5) (% (til 200) 7) (til 200)))) +(set Ra (select {c: (count k1) m: (min v) from: Ta by: [k1 k2] desc: c take: 3})) +(count Ra) -- 3 +(count (key Ra)) -- 4 +;; Top-3 groups all have count=6 (the (k1,k2) cells where 200 % lcm(5,7) +;; falls into the first 200/35 partition residues with one extra row). +(max (at Ra 'c)) -- 6 +(min (at Ra 'c)) -- 6 +;; min(v) over each heavy group is the smallest row-index whose +;; (i%5, i%7) lands in that cell; for cells visited 6 times, the +;; first index is in [0,35). Their min must be < 35. +(< (max (at Ra 'm)) 35) -- true + +;; ─── multi-key top-N with sum agg + larger take → exercises the +;; direct_ok=true branch's twin code paths (count-only short-circuit +;; at line 6917 vs. direct_ok at 6935). This ALSO inserts into +;; top_ht via group_ht_insert_empty_group at line 6902. +(set Rb (select {c: (count k1) from: Ta by: [k1 k2] desc: c take: 5})) +(count Rb) -- 5 +(sum (at Rb 'c)) -- 30 + +;; ─── force path A to bail → pivot_ingest fallback (lines 7160/7166) ── +;; F64 first key disables the `supported` guard at 6707 so the fast +;; n_keys=2 block is skipped; control falls through to the multi-key +;; pivot_ingest_run path that builds top_ht from cnt_ingest's part +;; rows and calls group_rows_range_existing for per-row aggregation. +(set Tf (table [k1 k2 v] (list (as 'F64 (% (til 200) 5)) (% (til 200) 7) (til 200)))) +(set Rf (select {c: (count k1) m: (min v) from: Tf by: [k1 k2] desc: c take: 3})) +(count Rf) -- 3 +(max (at Rf 'c)) -- 6 + +;; ─── 3-key shape exercises hash_keys_inline's n_keys=3 fan-out +;; inside group_ht_insert_empty_group + the entry-pack loop in +;; group_rows_range_existing. +(set T3 (table [k1 k2 k3 v] (list (% (til 210) 5) (% (til 210) 6) (% (til 210) 7) (til 210)))) +(set R3 (select {c: (count k1) m: (min v) from: T3 by: [k1 k2 k3] desc: c take: 4})) +(count R3) -- 4 +(< (max (at R3 'm)) 210) -- true + +;; ─── single-key sparse_i64 with > 2867 distinct keys → rehash ────── +;; sparse_i64_ht_t starts at cap=4096; rehash fires when size+1 +;; >= 0.7*cap = 2867.2, i.e. when the 2868-th unique key is added. +;; 3000 distinct i64 keys spaced by 1000 produces a 3 000 000-wide +;; key range — far past the 262 144-slot dense-array budget — so +;; the DA fast path declines and execution reaches the sparse_i64 +;; hash builder at group.c:6422 (no emit-filter branch). +(set Th (table [k] (list (* (til 3000) 1000)))) +(set Rh (select {c: (count k) from: Th by: k})) +(count Rh) -- 3000 +(sum (at Rh 'c)) -- 3000 +(min (at Rh 'c)) -- 1 +(max (at Rh 'c)) -- 1 +;; The 3000 distinct keys are {0, 1000, ..., 2999000}; their sum is +;; 1000 * (0+1+...+2999) = 1000 * 2999 * 3000 / 2 = 4 498 500 000. +(sum (at Rh 'k)) -- 4498500000 + +;; ─── sparse_i64 + sum agg (sp_need_sum=true) exercises the +;; need_sum branch of sparse_i64_rehash (memcpy of per-(slot,agg) +;; da_val_t sums during the cap doubling). +(set Ths (table [k v] (list (* (til 3000) 1000) (til 3000)))) +(set Rhs (select {c: (count k) s: (sum v) from: Ths by: k})) +(count Rhs) -- 3000 +;; Sum over all v = 0+1+...+2999 = 4 498 500. +(sum (at Rhs 's)) -- 4498500 + +;; ─── wide-key (GUID) + first agg → exercises group_rows_range_existing +;; wide branch (line 2569-2573) AND has_fl tail-slot write (2603-2604) +;; reached through the pivot_ingest fallback (GUID disables `supported` +;; in path A just like F64 above). +(set Tg (table [g s v] (list (take (guid 5) 200) (% (til 200) 7) (til 200)))) +(set Rg (select {c: (count g) f: (first v) from: Tg by: [g s] desc: c take: 3})) +(count Rg) -- 3 + +;; ─── sparse_i64 + emit_filter top_count branch (group.c:6426) on +;; > 2867 distinct keys. The first sp_ht in that branch is +;; count-only (need_sum=false) so its rehash takes the same +;; non-need_sum path as the regular sp_ht but reached via the +;; emit-filter top-N entry. All keys appear once → take:5 +;; selects 5 arbitrary "heavy" keys via the threshold-1 tie +;; break; the threshold-≤1 short-circuit at 6500-ish returns +;; all groups when none are heavier than 1, so verify the +;; result count is >= take_n's worth of distinct keys. +(set Re (select {c: (count k) from: Th by: k desc: c take: 5})) +(>= (count Re) 5) -- true diff --git a/test/rfl/ops/exec_worker_merge.rfl b/test/rfl/ops/exec_worker_merge.rfl new file mode 100644 index 00000000..184370b1 --- /dev/null +++ b/test/rfl/ops/exec_worker_merge.rfl @@ -0,0 +1,146 @@ +;; Targeted coverage for two functions in src/ops/exec.c: +;; +;; exec_in_worker — typed-column membership worker +;; ray_result_merge — concatenation merge of per-segment partials +;; +;; The existing exec_coverage.rfl already exercises the integer-/ +;; float-family inner loops with vector columns at every common width. +;; The arms that remain uncovered are: +;; * the float-path with vec_has_nulls=true at non-trivial scale +;; (exec.c L618-628) — only one in.rfl regression touches it, +;; and never at parallel-pool size, +;; * the symbol-side parallel dispatch (RAY_SYM read inside the +;; IN_READ_I64 switch, L590), +;; * the empty-set fast path (set vector reduced to sv_len=0 after +;; null pruning) over a long parallel column, +;; * the TIMESTAMP / DATE / TIME arm of IN_READ_I64 at scale. +;; +;; ray_result_merge is called once per segment of a parted table in +;; the streaming executor. The existing system/part.rfl tests run +;; SELECT with by:/count(distinct) — which is NOT streamable, so +;; they all fall through to ray_execute_inner's non-streaming path. +;; To hit the table-table concat arm (L1903-1923) we need a parted +;; default table feeding a fully streamable DAG (FILTER+SCAN, plain +;; SELECT, element-wise expressions). +;; +;; ==================================================================== +;; pre-flight cleanup so the partitioned fixtures don't inherit stale +;; sym.lk / .d files from a previous run (matches the convention used +;; by system/splayed.rfl, system/part.rfl). +;; ==================================================================== +(.sys.exec "rm -rf /tmp/rfl_exec_wm_int /tmp/rfl_exec_wm_sym /tmp/rfl_exec_wm_date /tmp/rfl_exec_wm_str") + +;; ==================================================================== +;; exec_in_worker — float-path with HAS_NULLS over a parallel column +;; ==================================================================== +;; Build a 70k-row F64 column with explicit nulls via `(take [...] N)` +;; — the pattern preserves the null bits across replication, so the +;; result vector carries RAY_ATTR_HAS_NULLS. At col_len=70000 the +;; pool dispatch path fires; the vec_has_nulls branch (L618) is taken +;; in the use_double=1 arm. +(set N1 70000) +(set TFN (table [f] (list (take [1.0 0Nf 2.0 0Nf 3.0 0Nf 4.0 5.0] N1)))) +;; pattern: 8 elems × 8750 = 70000 rows. null rows: 3/8 → 26250. +;; non-null with values in [1.0 2.0 3.0]: 3/8 → 26250. +(count (select {from: TFN where: (in f [1.0 2.0 3.0])})) -- 26250 +(count (select {from: TFN where: (not-in f [1.0 2.0 3.0])})) -- 17500 + +;; ==================================================================== +;; exec_in_worker — RAY_SYM parallel dispatch (L590, IN_READ_I64 sym +;; arm). The existing tests cover SYM only at small scale. Pump a +;; 70k-row SYM column so the worker_id != 0 branches execute. +;; ==================================================================== +(set TS (table [s] (list (take ['AAPL 'GOOG 'MSFT 'TSLA] N1)))) +;; AAPL appears at indices 0, 4, 8, ... → N1/4 = 17500 matches. +(count (select {from: TS where: (in s ['AAPL])})) -- 17500 +(count (select {from: TS where: (in s ['AAPL 'TSLA])})) -- 35000 +;; not-in negation in parallel sym mode. +(count (select {from: TS where: (not-in s ['AAPL])})) -- 52500 + +;; ==================================================================== +;; exec_in_worker — DATE / TIMESTAMP arms of IN_READ_I64 (L586-589). +;; The existing exec_coverage.rfl test for these uses a 50-row fixture +;; (sequential dispatch); add a 70k-row parallel-dispatch invocation +;; so the morsel-walk through the int-family branch executes. +;; ==================================================================== +(set TD (table [d] (list (take [2024.01.01 2024.01.02 2024.01.03] N1)))) +(count (select {from: TD where: (in d [2024.01.01])})) -- 23334 +(count (select {from: TD where: (in d [2024.01.01 2024.01.03])})) -- 46667 + +;; ==================================================================== +;; exec_in_worker — integer path with HAS_NULLS at parallel-pool size. +;; Existing TNul fixture is 5 rows (sequential dispatch only); this +;; ensures the !use_double + vec_has_nulls branch (L644) also runs +;; under pool dispatch. +;; ==================================================================== +(set TIN (table [v] (list (take [1 0Nl 2 0Nl 3 0Nl 4 5] N1)))) +;; 8 elems × 8750 = 70000. null rows: 3/8 → 26250. +;; non-null rows in [1 2 3]: 3/8 → 26250. +(count (select {from: TIN where: (in v [1 2 3])})) -- 26250 +(count (select {from: TIN where: (not-in v [1 2 3])})) -- 17500 +;; not-in with a set that contains nulls — set null is dropped from +;; probe buffer (set_has_nulls path); non-null rows compared against +;; the compacted probe. +(count (select {from: TIN where: (in v [1 0Nl 2 3])})) -- 26250 + +;; ==================================================================== +;; NOTE — exec_in_worker col_is_atom branches (L622/632/648/658) are +;; provably dead via static analysis: L612 sets +;; `vec_has_nulls = c->col_has_nulls && !c->col_is_atom`, so within the +;; vec_has_nulls arm col_is_atom can only be false (L622/648). In the +;; "else" arm (L630/656) col_is_atom IS possible in theory, but the +;; only call sites that produce OP_IN with an atom LHS go through +;; eval-layer `ray_in_fn` (collection.c), not the DAG OP_IN. Reaching +;; them from RFL requires a path that materializes an atom as the +;; result of a DAG node inside a SELECT expression; currently the +;; optimizer constant-folds or rewrites those shapes away. Documented +;; here so a future code change that exposes a new atom-LHS DAG path +;; can drop in a matching assertion without rediscovering this. + +;; ==================================================================== +;; ray_result_merge — table-table concatenation across parted segments +;; ==================================================================== +;; Build a 3-partition disk-backed parted table whose DAG path is +;; fully streamable: SCAN columns -> FILTER on a constant predicate +;; -> SELECT a subset of columns. Each segment returns a TABLE +;; partial; ray_result_merge concatenates segments 0..2 into the final +;; result and exercises L1903-1922. +(set P0 (table [id v] (list [1 2 3] [10 20 30]))) +(set P1 (table [id v] (list [4 5 6 7] [40 50 60 70]))) +(set P2 (table [id v] (list [8 9] [80 90]))) +(.db.splayed.set "/tmp/rfl_exec_wm_int/100/t/" P0) +(.db.splayed.set "/tmp/rfl_exec_wm_int/200/t/" P1) +(.db.splayed.set "/tmp/rfl_exec_wm_int/300/t/" P2) +(set PT (.db.parted.get "/tmp/rfl_exec_wm_int/" 't)) +;; sanity: 3+4+2 = 9 total rows across segments +(count PT) -- 9 +;; Streamable select: where: clause uses an element-wise predicate +;; on a parted column. Per-segment partials are flat tables; merge +;; concatenates them. Expected: rows with v >= 50 → 5 rows. +(count (select {from: PT where: (>= v 50)})) -- 5 +;; Streamable projection over filtered parted table — also concat- +;; merged segment by segment. +(sum (at (select {from: PT where: (>= v 50)}) 'v)) -- 350 +;; All-pass predicate: every row from every segment survives → 9. +(count (select {from: PT where: (>= v 0)})) -- 9 + +;; ==================================================================== +;; ray_result_merge — vector-vector concat over a parted column +;; ==================================================================== +;; (at PT 'v) reads a parted column directly. Each segment yields a +;; flat I64 vector; ray_result_merge's vector-merge branch (L1926) +;; was already hit by existing tests, but include here so the test +;; file is self-contained as a coverage sentinel. +(sum (at PT 'v)) -- 450 + +;; ==================================================================== +;; ray_result_merge — type mismatch path (L1930 returns ray_error +;; "type" when one side is a table and the other isn't). This is +;; defense-in-depth and currently unreachable from a streamable DAG +;; — each segment's output type is fixed by the root op shape — so +;; we don't attempt to provoke it. Documented for posterity. + +;; ==================================================================== +;; cleanup — leave /tmp tidy for re-runs +;; ==================================================================== +(.sys.exec "rm -rf /tmp/rfl_exec_wm_int /tmp/rfl_exec_wm_sym /tmp/rfl_exec_wm_date /tmp/rfl_exec_wm_str") diff --git a/test/rfl/ops/expr_load_setnull.rfl b/test/rfl/ops/expr_load_setnull.rfl new file mode 100644 index 00000000..8047b7db --- /dev/null +++ b/test/rfl/ops/expr_load_setnull.rfl @@ -0,0 +1,125 @@ +;; Targeted coverage for three low-coverage helpers in src/ops/expr.c: +;; +;; 1. set_all_null (expr.c:1180-1219) — scalar-null broadcast fill. +;; Fired from propagate_nulls_binary (1224-1227) when a binary op +;; whose op_propagates_null=true has a null scalar on either side. +;; The expr_compile fast path rejects null constants +;; (RAY_ATOM_IS_NULL check at expr.c:491), so a select projection +;; `(+ col 0N...)` falls through to exec_elementwise_binary which +;; is exactly where set_all_null lives. The type arms targeted +;; by this file are F64, I32/DATE/TIME, and I16 (the I64/TIMESTAMP +;; arm is already exercised by null/arith.rfl at 114 invocations). +;; +;; 2. par_binary_str_fn (expr.c:1490-1495) — parallel STR comparison +;; worker thunk. Only fires when `len >= RAY_PARALLEL_THRESHOLD` +;; (= 64 * 1024 = 65536) inside exec_elementwise_binary's RAY_STR +;; branch (1885-1894). A 70k-row STR column compared to a STR +;; literal triggers ray_pool_dispatch on this fn. +;; +;; 3. expr_load_f64 (expr.c:619-647) — reachability finding. The +;; single callsite (expr.c:889) only fires when a REG_SCAN +;; register has type RAY_F64 with col_type != RAY_F64. But the +;; compiler sets `regs[r].type = (col->type == RAY_F64) ? RAY_F64 +;; : RAY_I64` (expr.c:486) — so REG_SCAN F64 implies col->type +;; F64, which short-circuits to the direct-pointer branch at +;; expr.c:879-880. Promotion via expr_ensure_type (409-421) +;; always inserts a CAST instruction into a REG_SCRATCH (not a +;; REG_SCAN), so it never re-types the SCAN reg. The function +;; is therefore unreachable under the current compile rules. +;; Documented here; assertions exercise the would-be drivers +;; so they pin the analysis. + +;; -------------------------------------------------------------------- +;; SETUP +;; -------------------------------------------------------------------- +;; A small mixed-type table — the expr_compile fast path WILL try to +;; fuse `(+ i f)`, but the null-scalar arithmetic projections below +;; force the per-node fallback. +(set T (table [i f] (list [1 2 3 4 5] [0.5 1.5 2.5 3.5 4.5]))) + +;; -------------------------------------------------------------------- +;; SECTION 1 — set_all_null F64 branch +;; -------------------------------------------------------------------- +;; (+ f64col 0Nf) — op_propagates_null=true on OP_ADD; rhs is a null +;; F64 atom (r_scalar=true, scalar_is_null=true). expr_compile bails +;; on null literal → exec_elementwise_binary → propagate_nulls_binary +;; → set_all_null(result, len) where result->type = RAY_F64 (promoted +;; from F64+F64). Result must be an all-null F64 vector. +(count (at (select {x: (+ f 0Nf) from: T}) 'x)) -- 5 +(sum (map nil? (at (select {x: (+ f 0Nf) from: T}) 'x))) -- 5 +(type (at (select {x: (+ f 0Nf) from: T}) 'x)) -- 'F64 + +;; Symmetric: null on the LHS — exercises the `if (l_scalar && +;; scalar_is_null(lhs))` arm (expr.c:1224). Same F64 set_all_null fill. +(sum (map nil? (at (select {x: (+ 0Nf f) from: T}) 'x))) -- 5 + +;; -------------------------------------------------------------------- +;; SECTION 2 — set_all_null I64/TIMESTAMP branch (already 114 hits, +;; bumping with a new shape — TIMESTAMP col promotion to I64). +;; -------------------------------------------------------------------- +;; Promoted type for I64+I64+null → I64. Adds defensive coverage on +;; a shape (vec col + null I64 scalar inside select) that today only +;; fires via REPL-level atomic_map_binary. +(sum (map nil? (at (select {x: (+ i 0Nl) from: T}) 'x))) -- 5 + +;; -------------------------------------------------------------------- +;; SECTION 3 — set_all_null I32/DATE/TIME branch +;; -------------------------------------------------------------------- +;; I32 col + I32 null atom → promote_type(I32, I32) = I32 (opt.c:62). +;; result vec is RAY_I32 → set_all_null hits the I32 arm +;; (expr.c:1199-1203). +(set TI32 (table [a b] (list (as 'I32 [10 20 30 40]) [100 200 300 400]))) +(count (at (select {x: (+ a 0Ni) from: TI32}) 'x)) -- 4 +(sum (map nil? (at (select {x: (+ a 0Ni) from: TI32}) 'x))) -- 4 + +;; DATE col + I32 null atom shares the I32 path (DATE is in the same +;; promote_type bucket; opt.c:62). Result vec type is DATE — still +;; routed through the I32-width arm of set_all_null. +(set TD (table [d v] (list [2024.01.01 2024.06.15 2024.12.31] [1 2 3]))) +(sum (map nil? (at (select {x: (+ d 0Ni) from: TD}) 'x))) -- 3 + +;; -------------------------------------------------------------------- +;; SECTION 4 — set_all_null I16 branch +;; -------------------------------------------------------------------- +;; I16 col + I16 null atom → promote_type(I16, I16) = I16 (opt.c:63). +;; result vec is RAY_I16 → set_all_null hits the I16 arm +;; (expr.c:1204-1208). +(set TI16 (table [h] (list (as 'I16 [1 2 3 4 5 6 7])))) +(count (at (select {x: (+ h 0Nh) from: TI16}) 'x)) -- 7 +(sum (map nil? (at (select {x: (+ h 0Nh) from: TI16}) 'x))) -- 7 + +;; -------------------------------------------------------------------- +;; SECTION 5 — par_binary_str_fn parallel dispatch +;; -------------------------------------------------------------------- +;; Build a 70k-row STR column. RAY_PARALLEL_THRESHOLD = 64*1024 = +;; 65536, so 70000 crosses the parallel cut-off. expr_compile rejects +;; RAY_STR cols (line 469); STR-eq path at expr.c:1873 dispatches +;; par_binary_str_fn via the pool. Each worker calls binary_range_str +;; over its shard; the cycling input means every 3 rows is one match. +(set N_PAR 70000) +(set TSTR (table [s n] (list (map (fn [x] (as 'STR x)) (take ['aa 'bb 'cc] N_PAR)) (til N_PAR)))) + +;; STR col == STR atom → parallel dispatch on par_binary_str_fn. +;; 70000 / 3 = 23334 rows where s == "aa" (rounds up; first row 'aa). +(count (select {from: TSTR where: (== s "aa")})) -- 23334 +;; '!= "aa"' → complement; 70000 - 23334 = 46666. +(count (select {from: TSTR where: (!= s "aa")})) -- 46666 + +;; STR col == STR col via concat — two STR columns of the same length +;; comparing one another. Hits par_binary_str_fn with both sides +;; non-scalar (l_scalar=false, r_scalar=false). +(set TSTR2 (table [s t] (list (map (fn [x] (as 'STR x)) (take ['aa 'bb 'cc] N_PAR)) (map (fn [x] (as 'STR x)) (take ['aa 'bb 'cc] N_PAR))))) +(count (select {from: TSTR2 where: (== s t)})) -- 70000 + +;; -------------------------------------------------------------------- +;; SECTION 6 — expr_load_f64 reachability probe (documentary) +;; -------------------------------------------------------------------- +;; Mixed I64/F64 column arithmetic — the canonical case where one +;; might expect expr_load_f64 to fire. In practice the optimizer +;; inserts an explicit CAST instruction on the SCRATCH register so +;; the SCAN load stays on the integer-direct branch. Result still +;; F64; the test pins this expected output and documents that no +;; expr_load_f64 hits should appear (verified via llvm-cov: the +;; function remains 0% after this file runs). +(sum (at (select {x: (+ i f) from: T}) 'x)) -- 27.5 +(type (at (select {x: (+ i f) from: T}) 'x)) -- 'F64 diff --git a/test/rfl/ops/fp_eval_cmp.rfl b/test/rfl/ops/fp_eval_cmp.rfl new file mode 100644 index 00000000..7b8435f3 --- /dev/null +++ b/test/rfl/ops/fp_eval_cmp.rfl @@ -0,0 +1,76 @@ +;; Coverage push for src/ops/fused_group.c::fp_eval_cmp — the fused +;; per-row comparison evaluator invoked from OP_FILTERED_GROUP exec. +;; Each assertion is a select-by-where shape that triggers the fused +;; planner gate (`(select {agg by where})`) with a comparison that +;; routes through a specific (esz, op, col_type) arm in fp_eval_cmp. +;; +;; The existing test_fused_group.c unit suite hits I64 GE heavily but +;; leaves most narrow-width arms, FP_IN per-esz arms, FP_LIKE branches, +;; and the SYM cval-not-in-dict fold at 0%. This file fills those. +;; +;; The .rfl driver is line-based: each top-level expression must live +;; on a single line. Table setup is one (long) line below. + +;; Mixed-width column table. Each column type maps to a distinct +;; fp_eval_cmp arm: +;; u8 — esz=1 (BOOL/U8 share the path) +;; i16 — esz=2 non-SYM +;; i32 — esz=4 non-SYM +;; d — esz=4 non-SYM via RAY_DATE +;; t — esz=4 non-SYM via RAY_TIME +;; ts — esz=8 RAY_TIMESTAMP +;; s — RAY_STR (LIKE only) +;; sy — RAY_SYM (EQ/NE + LIKE) +;; g — group key (2 buckets) +(set T (table [u8 i16 i32 d t ts s sy g] (list (as 'U8 [1 2 3 4 5 6 7 8]) (as 'I16 [1 2 3 4 5 6 7 8]) (as 'I32 [1 2 3 4 5 6 7 8]) (as 'date [2020.01.01 2020.01.02 2020.01.03 2020.01.04 2020.01.05 2020.01.06 2020.01.07 2020.01.08]) (as 'time [00:00:01.000 00:00:02.000 00:00:03.000 00:00:04.000 00:00:05.000 00:00:06.000 00:00:07.000 00:00:08.000]) (as 'timestamp [2020.01.01D00:00:00.000000001 2020.01.01D00:00:00.000000002 2020.01.01D00:00:00.000000003 2020.01.01D00:00:00.000000004 2020.01.01D00:00:00.000000005 2020.01.01D00:00:00.000000006 2020.01.01D00:00:00.000000007 2020.01.01D00:00:00.000000008]) ["alpha" "beta" "axe" "able" "boat" "amber" "bingo" "bee"] ['alpha 'beta 'axe 'able 'boat 'amber 'bingo 'bee] [0 0 0 0 1 1 1 1]))) + +;; ───────── esz=1 (U8): EQ / LT / LE / GT — currently 0 each ───────── +(count (select {c: (count u8) from: T where: (== u8 1) by: g})) -- 1 +(count (select {c: (count u8) from: T where: (< u8 5) by: g})) -- 1 +(count (select {c: (count u8) from: T where: (<= u8 5) by: g})) -- 2 +(count (select {c: (count u8) from: T where: (> u8 0) by: g})) -- 2 + +;; ───────── esz=2 (I16): EQ / NE / LE / GT — currently 0 each ──────── +(count (select {c: (count i16) from: T where: (== i16 1) by: g})) -- 1 +(count (select {c: (count i16) from: T where: (!= i16 0) by: g})) -- 2 +(count (select {c: (count i16) from: T where: (<= i16 5) by: g})) -- 2 +(count (select {c: (count i16) from: T where: (> i16 0) by: g})) -- 2 + +;; ───────── esz=4 non-SYM: NE / LT / GT (DATE LE, TIME GT, I32) ────── +(count (select {c: (count i32) from: T where: (!= i32 0) by: g})) -- 2 +(count (select {c: (count i32) from: T where: (< i32 5) by: g})) -- 1 +(count (select {c: (count i32) from: T where: (> i32 0) by: g})) -- 2 +(count (select {c: (count d) from: T where: (<= d 2020.01.04) by: g})) -- 1 +(count (select {c: (count t) from: T where: (> t 00:00:00.500) by: g})) -- 2 + +;; ───────── esz=8 TIMESTAMP EQ/NE — distinct arm from GE-heavy I64 ─── +(count (select {c: (count ts) from: T where: (== ts 2020.01.01D00:00:00.000000001) by: g})) -- 1 +(count (select {c: (count ts) from: T where: (!= ts 2020.01.01D00:00:00.000000001) by: g})) -- 2 + +;; ───────── FP_IN per-esz arm: esz=1 / esz=2 / esz=4 (8 already hit) ─ +;; rhs is a literal vec (i64 default) — fp_compile_cmp casts to col esz. +(count (select {c: (count u8) from: T where: (in u8 [1 2 99]) by: g})) -- 1 +(count (select {c: (count i16) from: T where: (in i16 [3 4 99]) by: g})) -- 1 +(count (select {c: (count i32) from: T where: (in i32 [5 6 99]) by: g})) -- 1 + +;; FP_IN n_cvals==0 — null-only vec collapses to no candidate values; +;; fp_eval_cmp line 403 short-circuits with memset(0) for every row. +;; The IN rhs must be a literal vec (syntactic check in fp_check_in); +;; default `[0Nl]` parses to RAY_I64 with HAS_NULLS bit set. +(set Tn (table [i64 g] (list (as 'I64 [1 2 3 4 5 6 7 8]) [0 0 0 0 1 1 1 1]))) +(count (select {c: (count i64) from: Tn where: (in i64 [0Nl]) by: g})) -- 0 + +;; ───────── SYM cval_in_dict=0 — string literal that's not interned ── +;; (== sym_col "neverbefore_seen_xyz") compiles with cval_in_dict=0 → +;; fp_eval_cmp early-returns memset(0) for EQ, memset(1) for NE. +(count (select {c: (count sy) from: T where: (== sy "neverbefore_seen_xyz_12345") by: g})) -- 0 +(count (select {c: (count sy) from: T where: (!= sy "neverbefore_seen_xyz_12345") by: g})) -- 2 + +;; ───────── FP_LIKE on RAY_STR — both shape-PREFIX (use_simple=1) and +;; shape-NONE (use_simple=0, full glob) paths ──────────────────────── +(count (select {c: (count s) from: T where: (like s "a*") by: g})) -- 2 +(count (select {c: (count s) from: T where: (like s "[ab]*") by: g})) -- 2 + +;; ───────── FP_LIKE on RAY_SYM — LUT path with both shape modes ────── +(count (select {c: (count sy) from: T where: (like sy "a*") by: g})) -- 2 +(count (select {c: (count sy) from: T where: (like sy "[ab]*") by: g})) -- 2 diff --git a/test/rfl/ops/par_phase3_keepmin.rfl b/test/rfl/ops/par_phase3_keepmin.rfl new file mode 100644 index 00000000..3715581b --- /dev/null +++ b/test/rfl/ops/par_phase3_keepmin.rfl @@ -0,0 +1,185 @@ +;; ════════════════════════════════════════════════════════════════════ +;; Targeted coverage for three small low-cov helpers in src/ops: +;; +;; 1. par_binary_fn (src/ops/expr.c:1809) +;; Selection-aware morsel loop body (lines 1827-1835) — requires +;; a chained AND-conjunct WHERE where the prior conjunct set +;; g->selection (non-NULL) and the current binary op falls out of +;; the fused expr_compile path (nullable column → reject) AND the +;; row count is >= RAY_PARALLEL_THRESHOLD (65536). +;; +;; 2. grpt_phase3_fn (src/ops/group.c:9352) +;; Already covers the bulk of the function on the F64/I64 value +;; arms (heapsort drain + key/value write). The has_null_key=1 +;; branch (lines 9392-9422) is reachable only via grpt_ht_get +;; with has_null=true — which phase2 never sets (phase1 skips +;; null-key rows at group.c:9228). That branch is therefore +;; dead code from the current planner; we exercise the live +;; arms (I64 desc, I64 asc, F64 desc, F64 asc, K>1 with multi- +;; entry heap drain) to keep regions hot. +;; +;; 3. da_count_emit_keep_min_u32 (src/ops/group.c:3466) +;; TOP-N-by-count min-heap. Triggered by +;; (select {n: (count v) by: k from: T desc: n take: K}) +;; with non-SYM key and a dense key range (key_range <= 1<<26). +;; Distinct branches: +;; a) k_take >= group_count → early return (line 3475) +;; b) heap-fill (heap_n < k_take) with sift-up swap (3494-3495) +;; c) heap-replace (cnt > heap[0]) with sift-down (3497-3508) +;; d) final keep_min = heap[0] (line 3512) +;; +;; All assertions are `expected --` form. Wrong outputs trip the +;; check; the test must build green. +;; ════════════════════════════════════════════════════════════════════ + +;; ─── 1. par_binary_fn (selection-aware loop body) ────────────────── +;; Build a 70000-row table. Column a is nullable (first row 0Nl, then +;; til 69999) so expr_compile rejects (HAS_NULLS) for any binary op +;; touching a. Column g is dense mod 1000, no nulls. +;; +;; The two-conjunct WHERE compiles as a chain of OP_FILTER nodes: +;; c1: (< g 100) — fused path (g has no nulls); narrows +;; selection to ~7000 rows. Sets +;; g->selection (mix of NONE / MIX / ALL +;; morsel segments). +;; c2: (< a 50000) — expr_compile rejects (a HAS_NULLS), +;; exec_elementwise_binary → par_binary_fn +;; with sel_flg set → loop body hit. +;; +;; The cheap-first selection-sort in query.c keeps comparison-cost +;; equal conjuncts in user order (stable for ties), so c1 always +;; runs before c2. +;; +;; Sanity: 70000 rows, g=row%1000. c1 keeps rows with g<100 → +;; row%1000<100 → 7000 rows. c2 (< a 50000) further keeps rows +;; with a<50000. rayforce uses null-as-minimum compare semantics: +;; (< null x) → true for any x, so row 0 (a=null) PASSES c2. +;; Rows 1..49999 have a=row-1 in 0..49998 → all pass. Row 50000 +;; (a=49999) passes. Rows 50001..69999 (a in 50000..69998) fail. +;; Within c1 (i%1000<100): blocks 0..49 each contribute 100 rows +;; passing c1, plus row 50000 (i%1000=0). Block 0 row 0 also +;; passes c2 (null-as-min) → 100 rows. Blocks 1..49: 100 each. +;; Row 50000: 1. Total = 50*100 + 1 = 5001. +(set N70 70000) +(set Ga (% (til N70) 1000)) +(set Aa (concat [0Nl] (til 69999))) +(set Tpa (table [g a] (list Ga Aa))) +(count (select {from: Tpa where: (and (< g 100) (< a 50000))})) -- 5001 +;; Reversed conjunct order — same final cardinality. c1 now becomes +;; (< a 50000), which is the nullable-column conjunct. It runs first, +;; sets g->selection to ~50001 rows (rows 0..50000 pass null-as-min). +;; Then (< g 100) runs as second filter with selection set, fused +;; expr_compile succeeds for g (no nulls). Same cardinality. +(count (select {from: Tpa where: (and (< a 50000) (< g 100))})) -- 5001 +;; OP_GT on the nullable column. null > 60000 → false (null-as-min +;; loses on >). Rows 60001..69998 have a=row-1 in 60000..69997. +;; Filter (> a 60000): only a > 60000 passes, i.e., a in 60001..69997, +;; rows 60002..69998. Within c1 (i%1000<100): rows 60002..69998 with +;; i%1000 in [0,99]. Block 60: i=60002..60099 → 98 rows. Blocks +;; 61..69: 100 each, 9 blocks → 900. Total = 98 + 900 = 998. +(count (select {from: Tpa where: (and (< g 100) (> a 60000))})) -- 998 +;; OP_EQ — exercises another binary_range type arm under selection. +;; (== a 12345): exactly one row (index 12346 has a=12345); g for that +;; row is 12346%1000=346 → fails (< g 100). Result 0. +(count (select {from: Tpa where: (and (< g 100) (== a 12345))})) -- 0 +;; Wider c1 — covers MIX/ALL morsel segments more broadly. +;; (< g 500): half each block passes; row 0 passes too. +;; (< a 35000): rows 0..35000 (null + a in 0..34999). +;; Intersect: rows 0..35000 with i%1000<500. +;; Blocks 0..34 (rows 0..34999): 500 each → 17500. Row 35000 +;; (a=34999<35000, g=0<500) → +1. Total = 17501. +(count (select {from: Tpa where: (and (< g 500) (< a 35000))})) -- 17501 + +;; ─── 2. grpt_phase3_fn (existing live arms — keep hot) ────────────── +;; Parallel threshold for rowform topk is nrows >= 16384 (group.c:9494). +;; Use 17000 rows for the radix path. K=1 already covered by the +;; existing suite; cover K=2 (multi-entry heap drain loop) for both +;; I64 and F64 value types. +;; +;; 17000 rows, 2 groups by k%2. k=0 holds even row indices, k=1 odd. +;; Top-2 of each: max two even (16998, 16996) and max two odd +;; (16999, 16997). Sum = 16998+16996+16999+16997 = 67990. +(set Tg2 (table [k v] (list (% (til 17000) 2) (til 17000)))) +(count (select {t: (top v 2) by: k from: Tg2})) -- 4 +(sum (at (select {t: (top v 2) by: k from: Tg2}) 't)) -- 67990 +;; F64 values — exercise the val_is_f64 heapsort arm. +(set Tg2f (table [k v] (list (% (til 17000) 2) (as 'F64 (til 17000))))) +(sum (at (select {t: (top v 2) by: k from: Tg2f}) 't)) -- 67990.0 +;; Bot-K with F64 — covers desc=0 (min-heap drain) on F64 arm. +;; Bot-2 of k=0: 0,2; k=1: 1,3 → sum 6.0. +(sum (at (select {b: (bot v 2) by: k from: Tg2f}) 'b)) -- 6.0 +;; Larger K (3) with I64 → deeper heap drain (3-element heapsort). +;; Top-3 of k=0 even: 16998,16996,16994; k=1 odd: 16999,16997,16995 +;; Sum = (16998+16996+16994) + (16999+16997+16995) = 50988 + 50991 = 101979 +(sum (at (select {t: (top v 3) by: k from: Tg2}) 't)) -- 101979 + +;; ─── 3. da_count_emit_keep_min_u32 ───────────────────────────────── +;; Trigger: dense-range count-by-key with `desc: take: K`. +;; Need non-SYM keys, key_range <= 1<<26. Use I64 keys 0..99. +;; +;; Layout: 30 rows on keys 0..4 with counts {10, 8, 6, 4, 2}, PLUS one +;; row at key 300000 (count 1). The 300001-wide key range exceeds the +;; DA path's DA_MAX_COMPOSITE_SLOTS=262144 budget, forcing the single- +;; key sparse (dynamic-dense) path which calls da_count_emit_keep_min_u32. +;; +;; take=2 → da_count_emit_keep_min_u32 walks slots 0..max_seen: +;; slot 0 (cnt 10): insert, heap=[10] +;; slot 1 (cnt 8): insert at j=1, parent=10>8 SWAP → heap=[8,10] +;; (covers sift-up branch lines 3494-3495) +;; slot 2 (cnt 6): 6 > heap[0]=8? NO → skip +;; slot 3 (cnt 4): skip +;; slot 4 (cnt 2): skip +;; slot 300000 (cnt 1): 1 > 8? NO → skip +;; heap_n == k_take, heap[0]=8 > keep_min(=1) → keep_min=8 (line 3512). +;; Keys with cnt >= 8 → 2 keys (slots 0, 1). +(set Kd (concat (concat (concat (concat (concat (% (til 10) 1) (+ 1 (% (til 8) 1))) (+ 2 (% (til 6) 1))) (+ 3 (% (til 4) 1))) (+ 4 (% (til 2) 1))) [300000])) +;; Sanity: 10+8+6+4+2+1 = 31 rows, 6 keys. +(count Kd) -- 31 +(set Tdc (table [k v] (list Kd Kd))) +(count (select {n: (count v) by: k from: Tdc})) -- 6 +;; take=2: top-2 by count → keys 0,1 with counts 10,8. Two output rows. +(count (select {n: (count v) by: k from: Tdc desc: n take: 2})) -- 2 +;; Sum of the two top counts = 10 + 8 = 18. +(sum (at (select {n: (count v) by: k from: Tdc desc: n take: 2}) 'n)) -- 18 + +;; Heap-replace path (cnt > heap[0] when heap full). K=2, counts in +;; INCREASING slot order: slot 0 → 2, slot 1 → 4, slot 2 → 6, slot 3 +;; → 8, slot 4 → 10, slot 300000 → 1 (kept-out-of-DA helper). +;; slot 0 (cnt 2): insert, heap=[2] +;; slot 1 (cnt 4): insert at j=1, parent=2≤4 → no swap, heap=[2,4] +;; slot 2 (cnt 6): 6 > heap[0]=2 → REPLACE (sift-down loop). +;; heap[0]=6, sift: l=1 (heap[1]=4), 4<6 → swap. +;; heap=[4,6]. j=1, no children → done. +;; slot 3 (cnt 8): 8 > heap[0]=4 → REPLACE, heap=[6,8]. +;; slot 4 (cnt 10): 10 > heap[0]=6 → REPLACE, heap=[8,10]. +;; slot 300000 (cnt 1): 1 > 8? NO → skip. +;; heap[0]=8, keep_min=8. Keys with cnt>=8 → slots 3,4 (counts 8,10). +(set Kinc (concat (concat (concat (concat (concat (% (til 2) 1) (+ 1 (% (til 4) 1))) (+ 2 (% (til 6) 1))) (+ 3 (% (til 8) 1))) (+ 4 (% (til 10) 1))) [300000])) +(set Tinc (table [k v] (list Kinc Kinc))) +(count Tinc) -- 31 +(count (select {n: (count v) by: k from: Tinc desc: n take: 2})) -- 2 +(sum (at (select {n: (count v) by: k from: Tinc desc: n take: 2}) 'n)) -- 18 + +;; Early-return path: k_take >= group_count → returns initial keep_min +;; without touching the heap (lines 3474-3475). 6 distinct groups, +;; take=6 → equals group_count → early return → ALL groups emitted. +(count (select {n: (count v) by: k from: Tinc desc: n take: 6})) -- 6 +;; And take > group_count → same path. +(count (select {n: (count v) by: k from: Tinc desc: n take: 10})) -- 6 + +;; Deeper heap-replace (sift-down two levels). K=4, 6 base groups +;; with INCREASING counts 1,2,3,4,5,6 + sentinel key 300000 (cnt 1). +;; After fill: heap = [1,2,3,4] (min-heap; ascending inserts → no +;; sift-up swap). slot 4 cnt 5 > 1: replace, heap=[5,2,3,4]. +;; sift-down j=0: l=1 (2), r=2 (3), m=1 (2<5). swap. heap=[2,5,3,4]. +;; j=1: l=3 (4), 4<5 → swap. heap=[2,4,3,5]. j=3: no children. +;; slot 5 cnt 6 > 2: heap=[6,4,3,5]. sift-down: l=1(4), r=2(3), m=2 +;; (3<6). swap. heap=[3,4,6,5]. j=2: no children. Done. +;; slot 300000 cnt 1: 1 > 3? NO → skip. +;; heap[0]=3, keep_min=3. Groups with cnt>=3 → slots 2,3,4,5 with +;; counts 3,4,5,6. 4 groups, sum = 3+4+5+6 = 18. +(set Kw (concat (concat (concat (concat (concat (concat (% (til 1) 1) (+ 1 (% (til 2) 1))) (+ 2 (% (til 3) 1))) (+ 3 (% (til 4) 1))) (+ 4 (% (til 5) 1))) (+ 5 (% (til 6) 1))) [300000])) +(set Tw (table [k v] (list Kw Kw))) +(count Tw) -- 22 +(count (select {n: (count v) by: k from: Tw desc: n take: 4})) -- 4 +(sum (at (select {n: (count v) by: k from: Tw desc: n take: 4}) 'n)) -- 18 diff --git a/test/rfl/ops/print_resolve.rfl b/test/rfl/ops/print_resolve.rfl new file mode 100644 index 00000000..6ad6cf89 --- /dev/null +++ b/test/rfl/ops/print_resolve.rfl @@ -0,0 +1,116 @@ +;; Coverage for src/ops/builtins.c: +;; - ray_lang_print (println dispatcher across all type tags) +;; - ray_resolve_fn (symbol + table column resolution) +;; +;; Strategy: +;; 1. Drive every case of the switch in ray_lang_print via single-arg +;; (println X) — the function returns null, and we assert `(nil? ...)` +;; to confirm the call succeeded. Each call exercises one distinct +;; 0|-region recorded by llvm-cov: +;; -RAY_I64, -RAY_F64, -RAY_BOOL, -RAY_SYM, -RAY_STR, +;; RAY_LIST (heterogeneous), RAY_TABLE, +;; RAY_UNARY/BINARY/VARY (function handle), +;; default (ray_fmt fallback for vec/i16/i32/dict), +;; RAY_ATOM_IS_NULL (typed-null literal branch), +;; null-singleton fast path. +;; 2. Drive every reachable branch in ray_resolve_fn: +;; (resolve 'unbound) -> NULL (env_get miss) +;; (resolve 'name) -> bound value (-RAY_SYM env-hit) +;; (resolve 5) -> 5 (non-table, non-sym return) +;; (resolve db tbl) -> 2-arg form (db arg ignored) +;; (resolve tbl-with-entity-col) -> I64 col kept as I64 +;; (slen<2 / operator-char reject) +;; (resolve tbl-with-negative-id) -> data[r] <= 0 reject +;; (resolve empty-table) -> nrows==0 guard (loop skipped) +;; +;; Two ray_resolve_fn regions remain uncovered from rfl: +;; - lazy-materialize branch (lines 350-355): rfl-visible APIs always +;; return materialized tables by the time they reach resolve. +;; - all_user_sym=true SYM-conversion (lines 405-414): requires an I64 +;; vec whose elements are valid multi-char user-sym IDs; the only path +;; to construct one is ray_sym_intern in C — `(as 'I64 ['foo])` errors +;; because ray_cast_fn at builtins.c:1215-1236 has no -RAY_SYM->I64 +;; atom case. + +;; ────────────────────────────────────────────────────────────────────── +;; 1. ray_lang_print — atom dispatch +;; ────────────────────────────────────────────────────────────────────── + +;; -RAY_I64 atom (line 91) +(nil? (println 42)) -- true + +;; -RAY_F64 atom (lines 92-97) — includes the signbit(-0.0) normalisation +(nil? (println 1.5)) -- true + +;; -RAY_BOOL atom — both true and false paths in the ternary (line 98) +(nil? (println true)) -- true + +;; -RAY_SYM atom (lines 99-104) — bound, so ray_sym_str hits +(nil? (println 'foo_user_sym)) -- true + +;; -RAY_STR atom (lines 105-110) +(nil? (println "hello")) -- true + +;; RAY_LIST heterogeneous (lines 111-121) — both i==0 and i>0 branches +(nil? (println (list 1 "two" 'three))) -- true + +;; RAY_TABLE (lines 122-125) — prints "" summary +(nil? (println (table [a b] (list [1 2 3] [4 5 6])))) -- true + +;; RAY_UNARY / RAY_BINARY / RAY_VARY (lines 126-130) — function handle +;; `count` is a unary builtin; `+` is a binary builtin. +(nil? (println count)) -- true +(nil? (println +)) -- true + +;; default: ray_fmt fallback (lines 131-145) — vec, i32 atom, dict +(nil? (println [1 2 3])) -- true +(nil? (println 5i)) -- true +(nil? (println (dict [a b] (list 1 2)))) -- true + +;; RAY_ATOM_IS_NULL branch (lines 86-88) — typed null atom +;; (null_literal_str dispatch for "0Nl") +(nil? (println 0Nl)) -- true + +;; ────────────────────────────────────────────────────────────────────── +;; 2. ray_resolve_fn — branch coverage +;; ────────────────────────────────────────────────────────────────────── + +;; (resolve 'unbound) — env_get returns NULL, function returns NULL +;; (line 362). Test with `nil?` since `resolve` of an unbound returns the +;; null singleton. +(nil? (resolve 'totally_unbound_xyz_qq)) -- true + +;; (resolve 'bound) — env_get hits, retain+return val (lines 360-364) +(set _resolve_target 99) +(resolve '_resolve_target) -- 99 + +;; (resolve ) — branch at line 366 returns tbl as-is +(resolve 7) -- 7 + +;; (resolve db tbl) — 2-arg form (lines 342-346): db arg evaluated & released +(count (resolve "ignored_db_arg" (table [c] (list [10 20 30])))) -- 3 + +;; (resolve table-with-entity-ID-column) — all_user_sym=false path. +;; Small I64 values like 1,2,3 collide with single-char sym IDs (operators +;; interned early in startup) or are unmapped; the slen<2 / operator-char +;; guard at lines 397-403 rejects each, so the column stays I64 via the +;; `else` branch (lines 415-418). This drives the loop at 391-404 plus +;; the keep-I64 fall-through. +(set _Tent (table [e] (list [1 2 3]))) +(type (at (resolve _Tent) 'e)) -- 'I64 + +;; (resolve table-with-negative-id) — `data[r] <= 0` early reject (line 392) +(set _Tneg (table [x] (list [-1 -2 -3]))) +(type (at (resolve _Tneg) 'x)) -- 'I64 + +;; (resolve empty-table) — nrows==0 means `all_user_sym = (nrows > 0)` is +;; false from the start (line 387); the loop body at 391-404 never +;; executes, falling straight through to the else branch. +(set _Tempty (table [k] (list (as 'I64 [])))) +(count (resolve _Tempty)) -- 0 + +;; Non-I64 column path (lines 419-421) — STR + F64 cols kept as-is. +;; Combine the row-count check with the column-type check. +(set _Tmix (table [s f] (list (list "a" "b" "c") [1.0 2.0 3.0]))) +(count (resolve _Tmix)) -- 3 +(type (at (resolve _Tmix) 'f)) -- 'F64 diff --git a/test/rfl/query/key_reader_atom_const.rfl b/test/rfl/query/key_reader_atom_const.rfl new file mode 100644 index 00000000..bf335104 --- /dev/null +++ b/test/rfl/query/key_reader_atom_const.rfl @@ -0,0 +1,102 @@ +;; Coverage for two helpers in `src/ops/query.c`: +;; +;; - `atom_i64_const` (line 1422) — narrows a typed scalar +;; atom literal to int64. Switch on -RAY_BOOL / -RAY_U8 / +;; -RAY_I16 / -RAY_I32 / -RAY_DATE / -RAY_TIME / -RAY_I64 / +;; -RAY_TIMESTAMP. Reached from `expr_affine_of_sym` (line 1439) +;; when the by-dict has a dependent key of shape +;; `(+ base_sym )` or `(- base_sym )`. +;; The dict path emits a synthetic column = `base_col[i] + bias`. +;; +;; - `query_key_reader_read` (line 191) — wide-key reader. Reached +;; from the eval-level multi-key groupby fast path (line 4567) +;; when `by:` is a 2+ symbol vector, `take:` is a small positive +;; int (<= 1024), and every non-by dict-value is `(count …)`. +;; Each row reads one int64 per key column, dispatching on +;; `base_type` for I16 / I32 / DATE / TIME / I64 / TIMESTAMP / +;; F64 / BOOL/U8 and honouring the per-row null bit. + +;; ============================================================ +;; Part A — atom_i64_const via by-dict dep key +;; ============================================================ +;; +;; Base column `k` has 4 distinct integer keys. Each by-dict +;; below routes through expr_affine_of_sym → atom_i64_const, +;; emitting an aliased dep-key column = k + bias. We assert on +;; the sum of the dep column to confirm the bias propagated. +;; +;; Base sum of k = 1+2+3+4 = 10. + +(set Ta (table [k v] (list [1 2 3 4 1 2 3 4] [10 20 30 40 50 60 70 80]))) + +;; -RAY_BOOL / -RAY_U8 share the case body in atom_i64_const, and +;; neither has a direct arithmetic literal form in RFL that flows +;; through expr_affine_of_sym without prior coercion to I64 — +;; (+ k true) errors at type-check. These arms remain unreachable +;; via the by-dict dep path; the i16/i32/i64/date arms below are +;; the surfaces that RFL actually exercises. +;; +;; The by-dict alone (no projection echo of the dep alias) is +;; enough: the dep-key column is added to the result table at +;; line 7878. We sum-of-column to verify the bias was wired up +;; correctly. + +;; -RAY_I64 atom literal sanity check. Dep a = k + 100. +;; (Same case body as -RAY_TIMESTAMP.) +(sum (at (select {c: (count v) from: Ta by: {k: k a: (+ k 100)}}) 'a)) -- 410 + +;; -RAY_I16 atom literal: 5h. Dep alias `a` = k + 5 over distinct +;; group keys {1,2,3,4} → sum(a) = sum(k)+4*5 = 10 + 20 = 30. +(sum (at (select {c: (count v) from: Ta by: {k: k a: (+ k 5h)}}) 'a)) -- 30 + +;; -RAY_I32 atom literal: 7i. Dep a = k + 7 → sum = 10 + 28 = 38. +(sum (at (select {c: (count v) from: Ta by: {k: k a: (+ k 7i)}}) 'a)) -- 38 + +;; '-' operator — atom_i64_const negates the bias. Dep a = k - 1. +;; sum(k - 1) = 10 - 4 = 6. +(sum (at (select {c: (count v) from: Ta by: {k: k a: (- k 1)}}) 'a)) -- 6 + +;; -RAY_DATE atom literal: 2024.01.03 = 8768 days since epoch. +;; Internally the int32 value is added as bias. +;; sum(k + 8768) = 10 + 4*8768 = 35082. +(sum (at (select {c: (count v) from: Ta by: {k: k a: (+ k 2024.01.03)}}) 'a)) -- 35082 + +;; -RAY_TIME atom literal: 09:00:00.000 = 32_400_000 ms after midnight. +;; sum(k + 32400000) = 10 + 4*32400000 = 129600010. +(sum (at (select {c: (count v) from: Ta by: {k: k a: (+ k 09:00:00.000)}}) 'a)) -- 129600010 + +;; ============================================================ +;; Part B — query_key_reader_read via eval-level multi-key + take +;; ============================================================ +;; +;; Routing: 2-key `by:` vector + small positive `take:` + only +;; count aggs (no other agg) + nrows*nk small. Each row pulls +;; one int64 from each key column via query_key_reader_read. + +;; ----- Fixture: 16 rows × 2 narrow-typed key columns (I16, I32) ----- +;; 4 × 4 = 16 distinct (k16, k32) groups; take >= 16 keeps all. +(set Tw (table [k16 k32 v] (list [10h 20h 30h 40h 10h 20h 30h 40h 10h 20h 30h 40h 10h 20h 30h 40h] [100i 100i 100i 100i 200i 200i 200i 200i 300i 300i 300i 300i 400i 400i 400i 400i] [1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16]))) +(count (select {c: (count v) by: [k16 k32] from: Tw take: 100})) -- 16 +(sum (at (select {c: (count v) by: [k16 k32] from: Tw take: 100}) 'c)) -- 16 + +;; ----- Date + Time key columns (I32-backed RAY_DATE / RAY_TIME) ----- +(set Tdt (table [d t v] (list [2024.01.01 2024.01.01 2024.01.02 2024.01.02 2024.01.01 2024.01.01 2024.01.02 2024.01.02] [09:00:00.000 10:00:00.000 09:00:00.000 10:00:00.000 09:00:00.000 10:00:00.000 09:00:00.000 10:00:00.000] [1 2 3 4 5 6 7 8]))) +;; 2 dates × 2 times = 4 groups, each twice. +(count (select {c: (count v) by: [d t] from: Tdt take: 50})) -- 4 +(sum (at (select {c: (count v) by: [d t] from: Tdt take: 50}) 'c)) -- 8 + +;; ----- I64 + TIMESTAMP key columns ----- +(set Tts (table [g ts v] (list [100 100 200 200 100 100 200 200] [2024.01.01D00:00:00.000000000 2024.01.02D00:00:00.000000000 2024.01.01D00:00:00.000000000 2024.01.02D00:00:00.000000000 2024.01.01D00:00:00.000000000 2024.01.02D00:00:00.000000000 2024.01.01D00:00:00.000000000 2024.01.02D00:00:00.000000000] [1 2 3 4 5 6 7 8]))) +(count (select {c: (count v) by: [g ts] from: Tts take: 100})) -- 4 +(sum (at (select {c: (count v) by: [g ts] from: Tts take: 100}) 'c)) -- 8 + +;; ----- I64 + F64 key columns (F64 → memcpy branch in reader) ----- +(set Tf (table [g f v] (list [1 1 2 2 1 1 2 2] [1.5 2.5 1.5 2.5 1.5 2.5 1.5 2.5] [1 2 3 4 5 6 7 8]))) +(count (select {c: (count v) by: [g f] from: Tf take: 100})) -- 4 +(sum (at (select {c: (count v) by: [g f] from: Tf take: 100}) 'c)) -- 8 + +;; ----- Nullable I64 column — HAS_NULLS branch in the reader ----- +(set Tnu (table [a b v] (list [1 0Nl 1 0Nl 1 0Nl] [10 10 20 20 10 10] [1 2 3 4 5 6]))) +;; Distinct (a,b) pairs: (1,10), (NULL,10), (1,20), (NULL,20) = 4. +(count (select {c: (count v) by: [a b] from: Tnu take: 50})) -- 4 +(sum (at (select {c: (count v) by: [a b] from: Tnu take: 50}) 'c)) -- 6 diff --git a/test/rfl/query/wide_key_probe.rfl b/test/rfl/query/wide_key_probe.rfl new file mode 100644 index 00000000..226855a5 --- /dev/null +++ b/test/rfl/query/wide_key_probe.rfl @@ -0,0 +1,143 @@ +;; Coverage for the width-agnostic key reader and the parallel row→gid +;; probe in `src/ops/query.c`: +;; +;; - `key_read_i64` (line ~2828): width-typed read of one +;; key cell — BOOL, U8, I16, I32, I64, +;; DATE, TIME, TIMESTAMP, SYM, F64. +;; Prior tests covered only I64 / SYM +;; arms; the others were ≤ 0%. +;; +;; - `rgid_probe_fn` (line ~2882): parallel probe worker. +;; The selection-aware branch +;; (`if (x->selection)`) was never reached +;; by prior tests because it requires the +;; path-A fused WHERE + n_nonaggs > 0 +;; shape, and the only n_nonagg shape that +;; keeps the fused path is `(count +;; (distinct col))`. +;; +;; - `query_key_reader_init` (line 164): per-key reader setup for +;; the multi-key-by + take ≤ 1024 + +;; count-only path (line 4593). Prior +;; tests covered only the flat-column +;; branch with 2 calls; this file adds the +;; same branch with mixed key widths and +;; null-bearing keys to exercise the +;; null-check arm of the downstream reader +;; loop. +;; +;; Gating for the parallel probe: +;; nrows ≥ 200_000 (dispatch-amortisation threshold) +;; 1 ≤ n_groups ≤ 65_536 (per-task histogram sizing) +;; 1 scalar key in by: (single-key scatter path) +;; pool worker count ≥ 2 (already true on test runners) +;; +;; All assertions correctness-only — the serial fallback returns +;; identical results so a single-worker box passes the same checks. + +;; ─── 1. key_read_i64: I32 key arm (line 2836) ────────────────────── +;; (% (til N) 100) is I64; cast forces a real RAY_I32 column so the +;; scan reads the int32_t * cast in key_read_i64. +(set N32 200000) +(set Ng32 100) +(set Ti32 (table [k v] (list (as 'I32 (% (til N32) Ng32)) (til N32)))) +(set Ri32 (select {v: v by: k from: Ti32})) +(count Ri32) -- 100 +(fold + 0 (map count (at Ri32 'v))) -- 200000 + +;; ─── 2. key_read_i64: I16 key arm (line 2833) ────────────────────── +(set Ti16 (table [k v] (list (as 'I16 (% (til N32) Ng32)) (til N32)))) +(set Ri16 (select {v: v by: k from: Ti16})) +(count Ri16) -- 100 +(fold + 0 (map count (at Ri16 'v))) -- 200000 + +;; ─── 3. key_read_i64: U8 key arm (line 2832, shared with BOOL) ───── +;; Keep n_groups <= 256 so U8 doesn't lose distinct values. +(set Tu8 (table [k v] (list (as 'U8 (% (til N32) Ng32)) (til N32)))) +(set Ru8 (select {v: v by: k from: Tu8})) +(count Ru8) -- 100 +(fold + 0 (map count (at Ru8 'v))) -- 200000 + +;; ─── 4. key_read_i64: BOOL key arm (line 2831) ───────────────────── +;; n_groups = 2; the BOOL path shares the U8 read but exercises a +;; distinct okt switch arm inside key_read_i64. +(set Tbo (table [k v] (list (as 'BOOL (% (til N32) 2)) (til N32)))) +(set Rbo (select {v: v by: k from: Tbo})) +(count Rbo) -- 2 +(fold + 0 (map count (at Rbo 'v))) -- 200000 + +;; ─── 5. key_read_i64: DATE key arm (line 2835, shared with I32) ──── +;; Distinct RAY_DATE okt — line 7216 reads via the int32_t arm but +;; the okt switch in key_read_i64 has separate case labels. +(set Tdt (table [k v] (list (as 'DATE (% (til N32) Ng32)) (til N32)))) +(set Rdt (select {v: v by: k from: Tdt})) +(count Rdt) -- 100 +(fold + 0 (map count (at Rdt 'v))) -- 200000 + +;; ─── 6. key_read_i64: TIMESTAMP key arm (line 2838, shared I64) ──── +;; Distinct RAY_TIMESTAMP okt — case label in key_read_i64. +(set Tts (table [k v] (list (as 'TIMESTAMP (% (til N32) Ng32)) (til N32)))) +(set Rts (select {v: v by: k from: Tts})) +(count Rts) -- 100 +(fold + 0 (map count (at Rts 'v))) -- 200000 + +;; ─── 7. key_read_i64: F64 key arm (line 2843) ────────────────────── +;; F64 keys go through the bitcast memcpy path (NaN/-0 equality). +(set Tf64 (table [k v] (list (as 'F64 (% (til N32) Ng32)) (til N32)))) +(set Rf64 (select {v: v by: k from: Tf64})) +(count Rf64) -- 100 +(fold + 0 (map count (at Rf64 'v))) -- 200000 + +;; ─── 8. rgid_probe_fn selection path: (count (distinct ...)) + WHERE +;; Shape: single non-agg is `(count (distinct v))`, which is the only +;; non-agg form that does NOT set `has_nonagg_needing_flat`. Combined +;; with a flat (non-parted) table, this takes the path-A fused WHERE +;; that populates `saved_selection`; the L7115 scatter then dispatches +;; the parallel rgid_probe_fn with the selection threaded through, +;; exercising the `if (x->selection)` branch at line 2896. +;; Design: k = i / 2000 so each of the 100 groups holds 2000 +;; contiguous rows, and v = i % 10 cycles within each group. This +;; way every group has rows for every v value, so the WHERE filter +;; keeps every group alive while dropping a fraction of each group's +;; rows — exactly the shape the selection-threaded probe was built +;; for. +(set Tcd (table [k v] (list (div (til N32) 2000) (% (til N32) 10)))) +(set Rcd (select {c: (count (distinct v)) by: k where: (< v 7) from: Tcd})) +(count Rcd) -- 100 +;; Every group sees v cycling 0..9 — surviving (v<7) distinct values +;; per group = 7. 100 groups * 7 = 700. +(sum (at Rcd 'c)) -- 700 + +;; ─── 9. rgid_probe_fn selection path — RAY_SEL_NONE / RAY_SEL_ALL. +;; Filter on the contiguous k range (k = i/2000), so rows 0..99_999 +;; (k<50) all fail and rows 100_000..199_999 all pass. Most morsels +;; land entirely on one side → flagged RAY_SEL_NONE or RAY_SEL_ALL. +;; The transition morsel near i=99_999..100_000 lands SEL_MIX, but +;; the bulk hit the all-pass / all-fail bypass arms. +(set Rsa (select {c: (count (distinct v)) by: k where: (>= k 50) from: Tcd})) +(count Rsa) -- 50 +;; Surviving 50 groups each see v cycling 0..9 → 10 distinct each. +(sum (at Rsa 'c)) -- 500 + +;; ─── 10. query_key_reader_init: multi-key by + take ≤ 1024 + count +;; Activates the line 4593 init loop for each key column. Keys are +;; I64 + SYM — covers `r->base_type = col->type` arm. pre_take_groups +;; is set from the take literal; we use take: 50 ≤ 1024. +(set Tmk (table [g1 g2 v] (list [1 1 2 2 3 3 4 4 5 5] ['A 'B 'A 'B 'A 'B 'A 'B 'A 'B] [10 20 30 40 50 60 70 80 90 100]))) +(count (select {c: (count v) by: [g1 g2] take: 50 from: Tmk})) -- 10 +(sum (at (select {c: (count v) by: [g1 g2] take: 50 from: Tmk}) 'c)) -- 10 + +;; ─── 11. query_key_reader_init: multi-key with mixed widths +;; I16 + I32 keys — each reader's `r->base_type = col->type` reads the +;; respective width. Downstream query_key_reader_read uses +;; read_col_i64 to widen. Confirms multi-key path under cast types. +(set Tmw (table [k16 k32 v] (list (as 'I16 [1 1 2 2 3 3]) (as 'I32 [10 20 10 20 10 20]) [100 200 300 400 500 600]))) +(count (select {c: (count v) by: [k16 k32] take: 20 from: Tmw})) -- 6 +(sum (at (select {c: (count v) by: [k16 k32] take: 20 from: Tmw}) 'c)) -- 6 + +;; ─── 12. query_key_reader_init with null-bearing key — exercises +;; the HAS_NULLS attr arm of query_key_reader_read (line 198) downstream. +;; The pre-take path is the only multi-key path that uses the reader. +(set Tnk (table [k1 k2 v] (list [1 0N 2 0N 3 0N] ['A 'A 'B 'B 'C 'C] [10 20 30 40 50 60]))) +(count (select {c: (count v) by: [k1 k2] take: 30 from: Tnk})) -- 6 +(sum (at (select {c: (count v) by: [k1 k2] take: 30 from: Tnk}) 'c)) -- 6 diff --git a/test/rfl/sort/topk_radix_decode.rfl b/test/rfl/sort/topk_radix_decode.rfl new file mode 100644 index 00000000..5df08c4b --- /dev/null +++ b/test/rfl/sort/topk_radix_decode.rfl @@ -0,0 +1,83 @@ +;; topk_radix_decode.rfl — drive sort.c::radix_decode_into for DATE/TIME +;; type predicates and exercise topk_gather_rows over multi-column tables +;; with mixed-width sort keys. +;; +;; Coverage focus: +;; - radix_decode_into (sort.c:2372): predicate counters at +;; L2392 `type == RAY_DATE`, `type == RAY_TIME`. +;; Reaching these requires: +;; (a) single-key sort returning the sorted vector (asc / desc via +;; ray_sort, sort.c:3463 path or exec_sort decode-gather at +;; sort.c:3717), so radix_decode_into is invoked, AND +;; (b) the non-packed radix branch (sort.c:2553-2688), which sets +;; sorted_keys; non-packed fires when key_nbytes > 3 (i.e. data +;; range > 2^24 for the 4-byte DATE/TIME encoding), AND +;; (c) nrows > RADIX_SORT_THRESHOLD = 4096, so the small-N introsort +;; branch (which never extracts sorted_keys) is skipped. +;; Note: I16, BOOL, U8 decode arms (L2400-L2413) are structurally +;; unreachable in the current call graph — key_bytes_max <= 2 forces +;; use_packed=true, and the introsort small-N path never publishes +;; sorted_keys. They are not driven here. +;; - topk_gather_rows (sort.c:3315): multi-column table top-K paths +;; exercise the per-column for-loop body across DATE/TIME columns. + +;; ============================================================ +;; DATE single-key decode: > 4096 rows, range > 2^24 days +;; ============================================================ +;; Encoded values span > 2^24, so compute_key_nbytes returns 4 → +;; use_packed=false → sort_indices_ex publishes sorted_keys → +;; ray_sort (sort.c:3485) calls radix_decode_into with type=DATE. +;; (as 'DATE I) accepts integer days-since-epoch. +(set Dwide (as 'DATE (take [0 16777217 100 16777000 50 16700000 200 16777216 75] 4097))) +(at (asc Dwide) 0) -- (as 'DATE 0) +(at (take (asc Dwide) -1) 0) -- (as 'DATE 16777217) +(at (desc Dwide) 0) -- (as 'DATE 16777217) +(at (take (desc Dwide) -1) 0) -- (as 'DATE 0) + +;; ============================================================ +;; TIME single-key decode: > 4096 rows, range > 2^24 ms +;; ============================================================ +;; A full-day TIME range (0 .. 86_399_999) naturally exceeds 2^24 ms +;; (16_777_216), forcing the non-packed branch and the DATE/TIME decode +;; arm at sort.c:2392-2399. +(set Twide (as 'TIME (take [0 86399999 1 86000000 12345 70000000 67890 50000000 99999] 4097))) +(at (asc Twide) 0) -- (as 'TIME 0) +(at (take (asc Twide) -1) 0) -- (as 'TIME 86399999) +(at (desc Twide) 0) -- (as 'TIME 86399999) +(at (take (desc Twide) -1) 0) -- (as 'TIME 0) + +;; ============================================================ +;; exec_sort decode-gather path (sort.c:3717) — DATE table key +;; ============================================================ +;; Select with asc/desc on a wide-range DATE key column. exec_sort's +;; sort_indices_ex returns sorted_keys (non-packed); decode_col_idx is +;; set and radix_decode_into is called for the sort key column. +(set Td (table [d v] (list (take (as 'DATE [0 16777217 50 16700000 100 16777000 200]) 4097) (take [1 2 3 4 5 6 7] 4097)))) +(at (at (select {from: Td asc: d}) 'd) 0) -- (as 'DATE 0) +(at (at (select {from: Td desc: d}) 'd) 0) -- (as 'DATE 16777217) + +;; ============================================================ +;; exec_sort decode-gather path (sort.c:3717) — TIME table key +;; ============================================================ +(set Tt (table [t v] (list (take (as 'TIME [0 86399999 1000 70000000 500 50000000 12345]) 4097) (take [1 2 3 4 5 6 7] 4097)))) +(at (at (select {from: Tt asc: t}) 't) 0) -- (as 'TIME 0) +(at (at (select {from: Tt desc: t}) 't) 0) -- (as 'TIME 86399999) + +;; ============================================================ +;; Multi-column top-K — drives topk_gather_rows' per-column loop +;; across a mix of column types (DATE, TIME, I64). The HEAD limit +;; pushes exec_sort into ray_topk_table_multi → topk_gather_rows. +;; ============================================================ +(set Tmix (table [d t v] (list (take (as 'DATE [2024.06.15 2024.01.01 2024.03.10 2024.12.31 2024.07.04]) 200) (take (as 'TIME [12:00:00.000 09:00:00.000 18:30:00.000 06:15:00.000 23:00:00.000]) 200) (til 200)))) +;; SORT+TAKE fusion → topk_gather_rows over 3 columns; the 'v column +;; must round-trip with the expected first row count. +(count (select {from: Tmix asc: d take: 5})) -- 5 +(count (select {from: Tmix desc: t take: 7})) -- 7 + +;; ============================================================ +;; Combined: DATE asc preserves the multiset (sum invariant) over +;; > 4096 rows. Re-confirms the DATE decode path round-trips +;; through ray_sort without integer corruption. +;; ============================================================ +(sum (as 'I64 (asc Dwide))) -- (sum (as 'I64 Dwide)) +(sum (as 'I64 (asc Twide))) -- (sum (as 'I64 Twide)) diff --git a/test/rfl/store/str_col_io.rfl b/test/rfl/store/str_col_io.rfl new file mode 100644 index 00000000..d506ab1c --- /dev/null +++ b/test/rfl/store/str_col_io.rfl @@ -0,0 +1,140 @@ +;; Coverage for src/store/col.c: col_load_str_vec (STRV-magic legacy reader) +;; and col_copy_str_pool (deep-copy of the STR byte pool on non-mmap load). +;; +;; Reachability map (per llvm-cov, both 0% on master before this test): +;; +;; col_copy_str_pool — reached via ray_col_load on a STR column file +;; written by ray_col_save (no magic, raw header). +;; `.db.splayed.get` uses the mmap loader, so it +;; bypasses copy_str_pool entirely. `.db.splayed.mount` +;; calls ray_splay_load → ray_col_load (no mmap), +;; which hits col_copy_str_pool for every STR column. +;; +;; col_load_str_vec — reached when ray_col_load sees the legacy STRV +;; magic (0x53 0x54 0x52 0x56 = "STRV"). Newer +;; files don't carry this magic, so we plant the +;; bytes by hand via .sys.exec + printf and load +;; the surrounding splayed dir. `.db.splayed.get` +;; routes through ray_col_mmap_splayed which sees +;; STRV magic, returns "nyi", and falls back to +;; ray_col_load (splay.c:212-219) — that is the +;; path that lands inside col_load_str_vec. +;; +;; Fixtures use the `rf_test_str_io_*` prefix; the Makefile clean rule +;; only catches *.csv, so this file removes its own directories at the +;; top and bottom (matches csv_splayed.rfl convention). + +;; ────────────── scrub stale state from a prior run ────────────── +(.sys.exec "rm -rf rf_test_str_io_*") -- 0 + +;; ════════════════════════════════════════════════════════════════ +;; 1. col_copy_str_pool main path — STR column with long strings +;; (>12 bytes = RAY_STR_INLINE_MAX) so str_pool->len > 0. +;; +;; .db.splayed.set writes a 32-byte STR header + str_pool header +;; + bytes (no magic). .db.splayed.mount uses ray_splay_load +;; (non-mmap) → ray_col_load → col_copy_str_pool's main branch: +;; ray_alloc + memcpy of 32 + pool_size bytes, then rc/attr fixup. +;; +;; 2. col_copy_str_pool empty-pool branch — STR column whose strings +;; all fit inline (≤12 bytes) so str_pool->len == 0. Validation +;; still sets has_str_pool=true, but col_copy_str_pool returns +;; NULL at line 745 ("if (cm->str_pool_size == 0)") — the early +;; exit that the long-strings path can't hit. +;; ════════════════════════════════════════════════════════════════ +(set Tpool (table [id name] (list [1 2 3] ["a-very-long-pooled-string" "another-pooled-string-too" "third-pooled-string-here"]))) +(set Tinl (table [id name] (list [10 20] ["aa" "bb"]))) + +(.db.splayed.set "rf_test_str_io_mount/pool" Tpool) +(.db.splayed.set "rf_test_str_io_mount/inline" Tinl) + +;; Drop the local bindings so the mount must reconstruct the tables +;; from disk — proves col_copy_str_pool actually rebuilt the pools. +(set Tpool 0) +(set Tinl 0) + +(.db.splayed.mount "rf_test_str_io_mount/") + +;; `pool` and `inline` are now globals bound by the mount. Every +;; assertion below is a byte-exact value read from the deep-copied +;; pool/inline column — the only way they can match is if +;; col_copy_str_pool preserved both the 32-byte ray_t header and +;; the payload bytes. +(count pool) -- 3 +(at (at pool 'name) 0) -- "a-very-long-pooled-string" +(at (at pool 'name) 2) -- "third-pooled-string-here" +;; Inline branch — pool_size == 0, early NULL return; the STR values +;; sit entirely in the 16-byte ray_str_t descriptor. +(count inline) -- 2 +(at (at inline 'name) 1) -- "bb" + +;; ════════════════════════════════════════════════════════════════ +;; 3. col_load_str_vec happy path — plant a hand-crafted STRV file. +;; +;; Layout (matches ray_de_raw's RAY_STR vector case, serde.c:635): +;; 4 bytes "STRV" magic — \123\124\122\126 +;; 1 byte type = RAY_STR (13) — \015 +;; 1 byte attrs = 0 — \000 +;; 8 bytes count — LE 64-bit +;; per string: 8 bytes slen + slen bytes content. +;; +;; We need a `.d` schema for the splayed loader to walk this +;; column, so first save a placeholder STR table normally +;; (gives a valid `.d` referencing the `s` column), then +;; overwrite the column file with the STRV byte stream. +;; +;; 4. type-mismatch branch — STRV magic + RAY_I64 vec payload +;; (type 5). Hits col.c lines 202-205: result->type != RAY_STR +;; ⇒ ray_release + ray_error("type"). +;; +;; 5. truncated-payload branch — STRV + STR header claiming slen=99 +;; but only 3 content bytes. ray_de_raw returns "domain", and +;; col_load_str_vec propagates via the RAY_IS_ERR(result) check +;; at line 201. +;; +;; 6. zero-row STRV — STR vector with count=0. Confirms the happy +;; branch handles len=0 cleanly (no release / no error). +;; ════════════════════════════════════════════════════════════════ +(set Tph (table [s] (list ["placeholder"]))) +(.db.splayed.set "rf_test_str_io_strv" Tph) +(.db.splayed.set "rf_test_str_io_strv_bad" Tph) +(.db.splayed.set "rf_test_str_io_strv_trunc" Tph) +(.db.splayed.set "rf_test_str_io_strv_empty" Tph) + +;; STRV + STR + attrs=0 + count=2 + "hi" (2) + "world" (5) +(.sys.exec "printf '\\123\\124\\122\\126\\015\\000\\002\\000\\000\\000\\000\\000\\000\\000\\002\\000\\000\\000\\000\\000\\000\\000hi\\005\\000\\000\\000\\000\\000\\000\\000world' > rf_test_str_io_strv/s") +;; STRV + I64 vec hdr + count=1 + 8 zero bytes (1 I64 value=0) +(.sys.exec "printf '\\123\\124\\122\\126\\005\\000\\001\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000' > rf_test_str_io_strv_bad/s") +;; STRV + STR + count=1 + slen=99 (0x63=\143) + only "abc" +(.sys.exec "printf '\\123\\124\\122\\126\\015\\000\\001\\000\\000\\000\\000\\000\\000\\000\\143\\000\\000\\000\\000\\000\\000\\000abc' > rf_test_str_io_strv_trunc/s") +;; STRV + STR + attrs=0 + count=0 +(.sys.exec "printf '\\123\\124\\122\\126\\015\\000\\000\\000\\000\\000\\000\\000\\000\\000' > rf_test_str_io_strv_empty/s") + +(set Rstrv (.db.splayed.get "rf_test_str_io_strv/")) +(count Rstrv) -- 2 +(at (at Rstrv 's) 0) -- "hi" +(at (at Rstrv 's) 1) -- "world" + +;; Error branches — different errors prove different regions of +;; col_load_str_vec fire (type-check at line 202 vs RAY_IS_ERR at +;; line 201). +(.db.splayed.get "rf_test_str_io_strv_bad/") !- type +(.db.splayed.get "rf_test_str_io_strv_trunc/") !- domain + +(set Rempty (.db.splayed.get "rf_test_str_io_strv_empty/")) +(count Rempty) -- 0 + +;; ────────────── cleanup ────────────── +(.sys.exec "rm -rf rf_test_str_io_*") -- 0 + +;; ────────────── reachability notes ────────────── +;; col_load_str_vec line 198 `if (remaining > INT64_MAX)` — only +;; reachable on systems where size_t > int64_t (e.g. 128-bit size_t +;; or a 32-bit size_t with intentionally crafted huge file). Not +;; testable from RFL on the platforms we ship; covered by the +;; defensive bound, not by a regression assertion. +;; +;; col_copy_str_pool line 748 `if (!pool || RAY_IS_ERR(pool))` — the +;; OOM branch. ray_alloc returns NULL only when the buddy allocator +;; is fully exhausted; reaching that from RFL requires a multi-GB +;; allocation, which is out of scope for a regression test. diff --git a/test/rfl/strop/like_seen_proj.rfl b/test/rfl/strop/like_seen_proj.rfl new file mode 100644 index 00000000..c166dd4b --- /dev/null +++ b/test/rfl/strop/like_seen_proj.rfl @@ -0,0 +1,116 @@ +;; Targeted parallel SYM-LIKE coverage for like_seen_fn / like_proj_fn. +;; +;; Both kernels are the worker bodies dispatched by ray_pool_dispatch +;; when (a) N >= LIKE_PAR_MIN_ROWS_SYM (100_000) and (b) the SYM column +;; type-attrs encode a particular width (RAY_SYM_W8/W16/W32/W64). +;; +;; Prior tests only exercised the W64 arm (dense) + W64 MIX rowsel +;; branch. This file targets the cold arms: +;; +;; * W16 dense arm (string.c:160-167 / 281-288) — load CSV with <65k +;; unique syms; the post-load narrowing in csv.c:1395 rewrites the +;; column from W32 to W16 when max_id <= 65535. Fresh runtime +;; starts with ~300 builtin syms, so a fixture with say 50 unique +;; syms keeps max_id well under 65535. +;; * W32 dense arm (string.c:168-175 / 289-296) — load CSV with +;; >65535 unique syms; max_id > 65535 keeps the column at W32 (no +;; narrowing below W32 — see csv.c:1394 short-circuit). +;; * RAY_SEL_ALL branch (string.c:129-132 / 251-254) — chain an +;; always-true conjunct (>= idx 0) before LIKE. Conjunct sort +;; promotes the cheap (>=) ahead of LIKE; resulting rowsel marks +;; every 1024-row morsel as SEL_ALL. Exercised on the W64 in-RFL +;; column for simplicity (already supported by `take`). +;; +;; W8 (max_id <= 255) is unreachable from RFL: the runtime registers +;; ~250 builtin sym names at startup so max_id is already above 255 +;; before any user-defined sym is interned. Documented for posterity. + +;; ─── Fixture A: W16-narrow SYM column, dense parallel path ─── +;; 200k rows, ~50 unique syms → max_id < 65535 → narrowed to W16. +;; awk loop emits round-robin 50-sym cycle "s00..s49" for each row. +(.sys.exec "rm -f rfl_strop_w16.csv") -- 0 +(.sys.exec "awk 'BEGIN{print \"id,sym\"; for(i=0;i<200000;i++) printf(\"%d,s%02d\\n\",i,i%50)}' > rfl_strop_w16.csv") -- 0 +(set TW16 (.csv.read [I64 SYMBOL] "rfl_strop_w16.csv")) +(count TW16) -- 200000 + +;; (like sym "...") on the SYM column triggers the 3-phase parallel +;; pipeline (seen → resolve → proj). Each call exercises: +;; - like_seen_fn dense W16 arm (lines 160-167) +;; - like_proj_fn dense W16 arm (lines 281-288) +;; +;; Pattern "s0*" matches s00..s09 — 10 of 50 syms, ~20% of 200k rows. +(count (select {from: TW16 where: (like sym "s0*")})) -- 40000 +;; Pattern "s4*" matches s40..s49 — 10 of 50 syms, same 40000 rows. +(count (select {from: TW16 where: (like sym "s4*")})) -- 40000 +;; Suffix pattern "*5" matches s05,s15,s25,s35,s45 — 5 syms, 20000 rows. +(count (select {from: TW16 where: (like sym "*5")})) -- 20000 +;; Class pattern "s[12]*" — 20 of 50 syms, 80000 rows. Forces the +;; SHAPE_NONE general matcher per dict entry but the seen/proj kernels +;; are the same width-typed loops. +(count (select {from: TW16 where: (like sym "s[12]*")})) -- 80000 + +;; ─── Fixture A2: W16 + selection-aware (SEL_ALL+MIX+NONE) ─── +;; ray_rowsel_init returns NULL for always-true predicates +;; (rowsel.c:177-181), so always-true conjuncts collapse to no +;; selection. A partial conjunct retains a real rowsel with mixed +;; segment flags: (< id 100000) splits at row 100000 → morsels 0..97 +;; SEL_ALL (line 130 + W16 MARK), morsel 98 SEL_MIX (line 141-145 +;; bitmap walk + line 144 W16 MARK), morsels 99..195 SEL_NONE +;; (line 128 continue). Exercises every selection-aware path on W16. +(count (select {from: TW16 where: (and (< id 100000) (like sym "s3*"))})) -- 20000 + +;; Mid-range slice (250 < id < 750) — entirely inside a single morsel +;; pair → exclusively MIX morsels with a few SEL_NONE. Probes the MIX +;; width-typed inner loop on W16. +(count (select {from: TW16 where: (and (and (> id 250) (< id 750)) (like sym "s?5"))})) -- 50 + +(.sys.exec "rm -f rfl_strop_w16.csv") -- 0 + +;; ─── Fixture B: W32 SYM column, dense parallel path ─── +;; 200k rows with 70k unique syms — max_id > 65535 keeps the column at +;; W32 (csv.c:1394 short-circuit "if new_w >= RAY_SYM_W32 continue"). +;; This is the only way to hit the W32 arm from RFL. +(.sys.exec "rm -f rfl_strop_w32.csv") -- 0 +(.sys.exec "awk 'BEGIN{print \"id,sym\"; for(i=0;i<200000;i++) printf(\"%d,s%05d\\n\",i,i%70000)}' > rfl_strop_w32.csv") -- 0 +(set TW32 (.csv.read [I64 SYMBOL] "rfl_strop_w32.csv")) +(count TW32) -- 200000 + +;; "s00000*" matches exactly sym "s00000" (our format is %05d so each +;; integer maps to a single 6-char sym, and trailing '*' has nothing to +;; match in this fixed-width fixture). s00000 appears at i=0, 70000, +;; 140000 → 3 rows. +(count (select {from: TW32 where: (like sym "s00000*")})) -- 3 +;; "*9" matches every s with id divisible-by-10-...-9 in last digit. +;; Out of 70000 unique syms, 7000 end in '9'. In 200000 rows, 7000 * +;; (200000 / 70000 ≈ 2.857) → exact count: rows i where (i%70000)%10==9 +;; In 0..199999 there are floor(200000/10) = 20000 such rows. +(count (select {from: TW32 where: (like sym "*9")})) -- 20000 + +;; Selection-aware on W32 — partial conjunct creates SEL_ALL + +;; SEL_NONE + SEL_MIX morsels; the SEL_MIX morsel runs the W32 +;; LIKE_SEEN_MARK_ROW arm in like_seen_fn:144 (and the proj +;; counterpart at like_proj_fn:266). ray_rowsel_init drops +;; always-true preds (returns NULL — see rowsel.c:177-181) so we use +;; a partial filter to retain a non-NULL g->selection. +;; (< id 100000) keeps morsels 0..97 SEL_ALL, 98 MIX, 99..195 NONE. +;; "*9" matches 10000 syms (those ending in 9) in row id range +;; [0..99999]: i mod 70000 ⇒ rows 0..69999 give 7000 hits, rows +;; 70000..99999 give 3000 hits (s00000..s29999 segment) → 10000. +(count (select {from: TW32 where: (and (< id 100000) (like sym "*9"))})) -- 10000 + +(.sys.exec "rm -f rfl_strop_w32.csv") -- 0 + +;; ─── Fixture C: W64 selection-aware (SEL_ALL+MIX+NONE) ─── +;; in-RFL constructed SYM is W64. (take 200000) cycles 5 syms. +;; ray_rowsel_init returns NULL for always-true predicates +;; (rowsel.c:177-181), so we use a partial filter to keep a non-NULL +;; g->selection. Morsels 0..97 are fully in (SEL_ALL → string.c:130 +;; W64 MARK + string.c:252 W64 SET); morsel 98 is MIX; morsels +;; 99..195 are NONE. +(set syms ['AAA 'BBB 'CCC 'DDD 'EEE]) +(set Nrow 200000) +(set TW64 (table [s v] (list (take syms Nrow) (til Nrow)))) +;; (< v 100000) keeps first 100k rows. AAA cycle pos 0 → 20000 rows. +(count (select {from: TW64 where: (and (< v 100000) (like s "AAA"))})) -- 20000 +;; Wildcard "*" matches every sym; constrained to v<100000 → 100000. +(count (select {from: TW64 where: (and (< v 100000) (like s "*"))})) -- 100000 diff --git a/test/rfl/temporal/dag_extract_trunc.rfl b/test/rfl/temporal/dag_extract_trunc.rfl new file mode 100644 index 00000000..c20a504e --- /dev/null +++ b/test/rfl/temporal/dag_extract_trunc.rfl @@ -0,0 +1,81 @@ +;; DAG-path coverage for src/ops/temporal.c: +;; • exec_extract — HAS_NULLS = 1 paths over RAY_TIMESTAMP / RAY_DATE / +;; RAY_TIME column inputs, and the TIME (IN32=1, in_type +;; != RAY_DATE) inner-ternary branch that previously +;; ran 0 hits. Reached from rfl via `select` with the +;; long-form (year / month / hour / minute / second / +;; dayofweek / dayofyear) extractors over TABLE columns. +;; • exec_date_trunc — HAS_NULLS = 1 paths over RAY_TIMESTAMP / RAY_DATE / +;; RAY_TIME columns via the dotted `.date` / `.time` +;; desugaring (DAY / SECOND bucket). Atom + no-null +;; flavours are covered by parse_format.rfl already, +;; so this file focuses strictly on the null-bearing +;; column path. +;; +;; Reachability notes: +;; The HOUR / MINUTE / MONTH / YEAR / EPOCH cases of exec_date_trunc are +;; unreachable from rfl: ray_temporal_trunc_from_sym only maps "date" → DAY +;; and "time" → SECOND, and there's no other entry point. Same story for +;; RAY_EXTRACT_EPOCH in exec_extract — no rfl symbol resolves to it. +;; Those branches stay as dead-code-from-rfl regions. + +;; ───────────────────────── exec_extract (HAS_NULLS=1) ─────────────────────── +;; TIMESTAMP column with embedded null. IN32=0, HAS_NULLS=1 — drives the +;; `if (HAS_NULLS && ray_vec_is_null(input, off+i))` branch of the +;; (0,0)→(1,0) dispatch in src/ops/temporal.c. Nulls in the I64 result +;; flow through as 0Nl. +(set TPn (table [ts] (list (as 'TIMESTAMP [86400000000000 0N 172800000000000])))) +(at (at (select {y: (year ts) from: TPn}) 'y) 0) -- 2000 +(at (at (select {y: (year ts) from: TPn}) 'y) 1) -- 0Nl +(at (at (select {m: (month ts) from: TPn}) 'm) 1) -- 0Nl +(at (at (select {h: (hour ts) from: TPn}) 'h) 1) -- 0Nl +(at (at (select {n: (minute ts) from: TPn}) 'n) 1) -- 0Nl +(at (at (select {s: (second ts) from: TPn}) 's) 1) -- 0Nl + +;; DATE column with embedded null. IN32=1, HAS_NULLS=1 dispatch path. +;; Days since 2000-01-01 = 8766 → 2024-01-01; the null slot at index 1 +;; remains null in the I64 output. +(set TDn (table [d] (list (as 'DATE [8766 0N 8767])))) +(at (at (select {y: (year d) from: TDn}) 'y) 0) -- 2024 +(at (at (select {y: (year d) from: TDn}) 'y) 1) -- 0Nl +(at (at (select {dd: (day d) from: TDn}) 'dd) 1) -- 0Nl +(at (at (select {dy: (dayofyear d) from: TDn}) 'dy) 1) -- 0Nl + +;; TIME column with embedded null. IN32=1, HAS_NULLS=1, in_type=RAY_TIME so +;; the IN32 ternary picks the `raw32 * 1000` arm (vs DATE's `* USEC_PER_DAY`). +;; Previously this entire combination ran 0 hits. +(set TTn (table [t] (list (as 'TIME [3723000 0N 86399000])))) +(at (at (select {h: (hour t) from: TTn}) 'h) 0) -- 1 +(at (at (select {h: (hour t) from: TTn}) 'h) 1) -- 0Nl +(at (at (select {h: (hour t) from: TTn}) 'h) 2) -- 23 +(at (at (select {n: (minute t) from: TTn}) 'n) 0) -- 2 +(at (at (select {s: (second t) from: TTn}) 's) 0) -- 3 + +;; ──────────────────────── exec_date_trunc (HAS_NULLS=1) ───────────────────── +;; TIMESTAMP column with embedded null routed through `.date` (DAY bucket). +;; Drives DATE_TRUNC_INNER(1, 0). +(set Tts (table [ts] (list (as 'TIMESTAMP [86400000000000 0N 172800000000000])))) +(at (at (select {s: ts.date from: Tts}) 's) 0) -- 2000.01.02D00:00:00.000000000 +(at (at (select {s: ts.date from: Tts}) 's) 1) -- 0Np +(at (at (select {s: ts.time from: Tts}) 's) 1) -- 0Np + +;; DATE column with embedded null routed through `.date`. IN32=1, +;; HAS_NULLS=1; DAY bucket trivially preserves day-aligned input. +(set Tdd (table [d] (list (as 'DATE [8766 0N 8767])))) +(at (at (select {s: d.date from: Tdd}) 's) 0) -- 2024.01.01D00:00:00.000000000 +(at (at (select {s: d.date from: Tdd}) 's) 1) -- 0Np + +;; TIME column with embedded null routed through `.time` (SECOND bucket). +;; IN32=1, HAS_NULLS=1, in_type=RAY_TIME — drives the `* 1000` arm of the +;; DATE_TRUNC_INNER ternary that was previously dark. +(set Ttt (table [t] (list (as 'TIME [3723000 0N 86399999])))) +(at (at (select {s: t.time from: Ttt}) 's) 0) -- 2000.01.01D01:02:03.000000000 +(at (at (select {s: t.time from: Ttt}) 's) 1) -- 0Np +(at (at (select {s: t.time from: Ttt}) 's) 2) -- 2000.01.01D23:59:59.000000000 + +;; Pre-epoch DATE inside a select to exercise the `us < 0` correction +;; in EXTRACT_INNER's `days_since_2000` block (year-decompose branch). +;; 1999-12-31 = -1 day relative to the 2000 epoch. +(set TPre (table [d] (list (as 'DATE [1999.12.31 2000.01.01])))) +(at (at (select {y: (year d) from: TPre}) 'y) 0) -- 1999 +(at (at (select {y: (year d) from: TPre}) 'y) 1) -- 2000