From 5e164b1a5d728b1cb7ddb0d1e031eb914038a224 Mon Sep 17 00:00:00 2001 From: Churkin Aleksey Date: Tue, 15 Sep 2026 18:12:10 +0300 Subject: [PATCH 1/4] benchmarks: from_decs_count totals archetype sizes in the width they are `total` counted archetype sizes in an int while Archetype.size is an int64, so the file did not compile and the benchmark had never run. The count is a sum of sizes, so the accumulator takes the width of what it sums rather than the sum being narrowed to it. The entity id the fixture block never reads takes the underscore that says so. --- benchmarks/decs/bench_from_decs_count.das | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/decs/bench_from_decs_count.das b/benchmarks/decs/bench_from_decs_count.das index 978d96ea50..3157206b62 100644 --- a/benchmarks/decs/bench_from_decs_count.das +++ b/benchmarks/decs/bench_from_decs_count.das @@ -20,7 +20,7 @@ struct BenchCountRow { def fixture(n : int) { restart() - create_entities(n) $(eid : EntityId; i : int; var cmp : ComponentMap) { + create_entities(n) $(_eid : EntityId; i : int; var cmp : ComponentMap) { apply_decs_template(cmp, BenchCountRow(val = i)) } } @@ -32,7 +32,7 @@ def from_decs_count_m1_hand(b : B?) { b |> run("m1_hand_arch_size/{N}", N) { var erq : EcsRequest erq.req |> push("bench_val") - var total = 0 + var total = 0l for_each_archetype(erq) $(arch : Archetype) { total += arch.size } From e47bdd2abfe55dd3efe1a542408d0809e478fe85 Mon Sep 17 00:00:00 2001 From: Churkin Aleksey Date: Tue, 15 Sep 2026 10:56:00 +0300 Subject: [PATCH 2/4] daslib: clargs answers the host binary's path, so a program that spawns daslang stops reading argv[0] itself daslib/clargs wraps get_command_line_arguments behind three accessors - the post-`--` slice, argv[1..], and get_user_args picking between them by host - and every program in the tree is meant to reach for those rather than the builtin. The one thing it did not answer was the binary a program is running under, which is what a program spawning the same daslang it runs under needs. get_host_binary() is that accessor: argv[0] as an absolute path. The CLI reference names it beside the other three, and the compile-time benchmark group two commits along is its first caller. --- daslib/clargs.das | 6 ++++++ skills/daslang/references/cli-and-config.md | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/daslib/clargs.das b/daslib/clargs.das index 996ea09738..02d3af43dc 100644 --- a/daslib/clargs.das +++ b/daslib/clargs.das @@ -74,6 +74,12 @@ def public get_user_args() : array { } } +//! The running host binary as an absolute path - `argv[0]` resolved - for a program that spawns +//! the same daslang it runs under. +def public get_host_binary() : string { + return get_full_file_name(get_command_line_arguments()[0]) +} + def clargs_flag_name(field_name : string) : string { return "--" + replace_multiple(field_name, [(text="_", replacement="-")]) } diff --git a/skills/daslang/references/cli-and-config.md b/skills/daslang/references/cli-and-config.md index 5b4b9d2253..dacaaf6e0b 100644 --- a/skills/daslang/references/cli-and-config.md +++ b/skills/daslang/references/cli-and-config.md @@ -30,7 +30,9 @@ def main : int { **Don't pick an argv accessor.** `parse_args` pulls argv through `get_user_args()`: `argv[1..]` for a standalone `-exe` binary, the post-`--` slice under the interpreter or the JIT. -`get_program_args()` / `get_cli_arguments()` force one slice regardless of host. +`get_program_args()` / `get_cli_arguments()` force one slice regardless of host. A program that +spawns the daslang it runs under takes the path from `get_host_binary()` - `argv[0]` resolved to an +absolute path - never from `get_command_line_arguments()` itself. Field types: `string`, `int`, `float`, `bool`, an enum (clargs validates the value), `array` for a repeatable flag. The long name is the field name with underscores as hyphens; clargs adds the From 0a35099aa7c1af820e81a763757c253169252eb8 Mon Sep 17 00:00:00 2001 From: Churkin Aleksey Date: Tue, 15 Sep 2026 10:56:00 +0300 Subject: [PATCH 3/4] bench-stand: the viewer - one chart per arm over commits, static, no build step and no dependencies utils/internal/bench-stand/site is the page served at /bench/: index.html, app.js, style.css and nothing else - it fetches data.json and status.json from its own directory and renders everything client-side, so publishing a night is a file copy and the box runs no service behind it. caddy.snippet is the route that serves the directory, and the only place that route is written down: it goes into the daslang.io block of the box's Caddyfile, no-cache, so a visitor never reads yesterday's night out of a cache. suite.json rides along: the suite the nights measure - root, excludes, the two lanes and their per-file exclusions, repeats, the per-file timeout, the regression gate's three numbers, and the skips with the reason each one is listed on the page every night. It opens with the latest night: the build's log tail when the build is what failed, otherwise the failures with the message to act on, the lanes that did not run and the skips with their reasons, then the regressions and improvements against their baseline and the noise the gate was measured against. Below that is one chart per benchmark arm, grouped by directory - the minimum ns/op per night as the line, the night's spread as a band, a crosshair snapping to the nearest run with a tooltip listing every lane there, and a click through to the commit; a night the file failed is marked on the axis, and a gap in the line is a night the arm did not measure. The run history closes the page, each row linking its own record. Colors come from the lane through one `--lane` custom property, never from a series' position, so a chart added today is comparable with one read yesterday, and a chart drawing more than one lane carries its legend. --- utils/internal/bench-stand/caddy.snippet | 17 + utils/internal/bench-stand/site/app.js | 343 +++++++++++++++++++++ utils/internal/bench-stand/site/index.html | 39 +++ utils/internal/bench-stand/site/style.css | 157 ++++++++++ utils/internal/bench-stand/suite.json | 29 ++ 5 files changed, 585 insertions(+) create mode 100644 utils/internal/bench-stand/caddy.snippet create mode 100644 utils/internal/bench-stand/site/app.js create mode 100644 utils/internal/bench-stand/site/index.html create mode 100644 utils/internal/bench-stand/site/style.css create mode 100644 utils/internal/bench-stand/suite.json diff --git a/utils/internal/bench-stand/caddy.snippet b/utils/internal/bench-stand/caddy.snippet new file mode 100644 index 0000000000..3f500be9d9 --- /dev/null +++ b/utils/internal/bench-stand/caddy.snippet @@ -0,0 +1,17 @@ +# bench-stand routes for the daslang.io vhost in /etc/caddy/Caddyfile. +# This file is the authoritative copy of the stand's public boundary — the +# deployed Caddyfile is edited to match it, never the other way round. +# +# Paste inside the `daslang.io { ... }` block, ahead of `root`/`file_server`. +# `bench-stand-deploy.sh caddy` does the splice, validates, and reloads. + +# The nightly benchmark stand: a static tree the `bench` user rewrites every +# night (viewer, data.json, status.json, summary.md, runs/). No service behind +# it. no-cache: the viewer fetches data.json on every visit and a day-old +# cached copy would show yesterday's night as tonight's. +redir /bench /bench/ 308 +handle_path /bench/* { + root * /srv/bench-stand/site + header Cache-Control "no-cache" + file_server +} diff --git a/utils/internal/bench-stand/site/app.js b/utils/internal/bench-stand/site/app.js new file mode 100644 index 0000000000..7084ceac35 --- /dev/null +++ b/utils/internal/bench-stand/site/app.js @@ -0,0 +1,343 @@ +"use strict"; + +const LANES = ["interp", "jit"]; +const state = { data: null, status: null, runWindow: 90, lanes: new Set(LANES), group: "" }; + +const $ = (sel) => document.querySelector(sel); +const el = (tag, cls, text) => { + const e = document.createElement(tag); + if (cls) e.className = cls; + if (text !== undefined) e.textContent = text; + return e; +}; +const svgEl = (tag, attrs) => { + const e = document.createElementNS("http://www.w3.org/2000/svg", tag); + for (const k in attrs) e.setAttribute(k, attrs[k]); + return e; +}; +function fmtNs(v) { + if (v >= 1e9) return (v / 1e9).toFixed(2) + " s"; + if (v >= 1e6) return (v / 1e6).toFixed(2) + " ms"; + if (v >= 1e3) return (v / 1e3).toFixed(2) + " us"; + return v.toFixed(v >= 100 ? 0 : v >= 10 ? 1 : 2) + " ns"; +} +const fmtSec = (s) => (s >= 3600 ? (s / 3600).toFixed(1) + " h" : s >= 60 ? Math.round(s / 60) + " min" : Math.round(s) + " s"); +const shortSha = (sha) => (sha || "").slice(0, 8); +const dateOf = (iso) => (iso || "").slice(0, 10); +const commitUrl = (sha) => (state.data.repo_url && sha ? state.data.repo_url.replace(/\/$/, "") + "/commit/" + sha : null); +const recordUrl = (run) => "runs/" + encodeURIComponent(run.id) + ".json"; +const anchorId = (id) => "b-" + id.replace(/[^A-Za-z0-9_-]/g, "_"); + +async function load() { + const [dataRes, statusRes] = await Promise.allSettled([ + fetch("data.json", { cache: "no-store" }).then((r) => (r.ok ? r.json() : Promise.reject(new Error("data.json " + r.status)))), + fetch("status.json", { cache: "no-store" }).then((r) => (r.ok ? r.json() : null)), + ]); + state.status = statusRes.status === "fulfilled" ? statusRes.value : null; + if (dataRes.status === "fulfilled") state.data = dataRes.value; + else $("main").replaceChildren(el("p", "notice notice--error", "No data.json yet: the stand has not published a report (" + dataRes.reason.message + ").")); + renderStatus(); + if (state.data) renderAll(); +} + +function visibleRunIndices() { + const n = state.data.runs.length; + const from = state.runWindow > 0 ? Math.max(0, n - state.runWindow) : 0; + return Array.from({ length: n - from }, (_, i) => from + i); +} + +function renderStatus() { + const box = $("#status"); + box.replaceChildren(); + const st = state.status; + if (st && st.state === "running") box.append(el("span", "pill pill--running", "running " + (st.run_id || ""))); + if (!state.data || state.data.latest < 0) { if (!st) box.append(el("span", "pill", "no runs")); return; } + const run = state.data.runs[state.data.latest]; + box.append(el("span", "pill pill--" + run.status, run.status.replace("_", " "))); + const link = el("a", null, shortSha(run.commit.sha) + " " + run.commit.subject); + link.href = commitUrl(run.commit.sha) || "#"; + box.append(link, el("span", "muted", dateOf(run.started) + " on " + run.machine.host + ", build " + fmtSec(run.build.seconds) + ", suite " + fmtSec(run.seconds))); + const ageDays = (Date.now() - Date.parse(run.started)) / 86400000; + if (ageDays > 2 && !(st && st.state === "running")) box.append(el("span", "pill pill--stale", "last run " + Math.floor(ageDays) + " days ago")); + $("#generated").textContent = "report generated " + (state.data.generated || ""); +} + +function wireFilters() { + document.querySelectorAll(".chip[data-range]").forEach((b) => b.addEventListener("click", () => { + document.querySelectorAll(".chip[data-range]").forEach((x) => x.classList.remove("is-on")); + b.classList.add("is-on"); + state.runWindow = Number(b.dataset.range); + renderAll(); + })); + document.querySelectorAll("input[data-lane]").forEach((c) => c.addEventListener("change", () => { + if (c.checked) state.lanes.add(c.dataset.lane); else state.lanes.delete(c.dataset.lane); + renderAll(); + })); + $("#group").addEventListener("change", (e) => { state.group = e.target.value; renderSeries(); }); +} + +function fillGroups() { + const sel = $("#group"); + const keep = sel.value; + sel.replaceChildren(el("option", null, "all groups")); + sel.firstChild.value = ""; + for (const g of state.data.groups) { const o = el("option", null, g); o.value = g; sel.append(o); } + sel.value = state.data.groups.includes(keep) ? keep : ""; +} + +function header(title, count, sub) { + const h = el("h3"); + h.append(title); + if (count !== undefined) h.append(el("span", "count", String(count))); + if (sub) h.append(el("span", "muted", sub)); + return h; +} +function list(items, render, emptyText) { + const ul = el("ul"); + if (!items.length) { if (emptyText) ul.append(el("li", "empty", emptyText)); return ul; } + for (const it of items) ul.append(render(it)); + return ul; +} + +function renderNight() { + const box = $("#night"); + box.replaceChildren(); + const d = state.data; + if (d.latest < 0) return; + const run = d.runs[d.latest]; + const card = el("div", "card"); + const rec = el("a", "msg", "run record"); + rec.href = recordUrl(run); + if (run.build.status && run.build.status !== "ok") { + card.append(header("The build failed", undefined, "nothing was measured")); + const pre = el("pre", "logtail"); + pre.textContent = run.build.log_tail || "the driver recorded no build log"; + card.append(el("div", "msg", "after " + fmtSec(run.build.seconds) + "; last lines of the build log:"), pre, rec); + box.append(card); + return; + } + card.append(header("Failures", run.failures.length)); + card.append(list(run.failures, (f) => { + const li = el("li"); + li.append(el("span", "tag tag--failure", f.status.replace("_", " ")), el("span", "tag tag--" + f.lane, f.lane), el("code", null, f.path)); + const m = el("div", "msg"); + m.append(el("code", null, f.message)); + li.append(m); + return li; + }, "every file ran")); + const notes = el("div", "msg"); + for (const lane of Object.keys(run.lanes)) if (run.lanes[lane] !== "ok") notes.append(el("div", null, lane + " lane did not run: " + run.lanes[lane])); + for (const f of run.skipped) { const line = el("div"); line.append("skipped ", el("code", null, f.path), " (" + f.lane + "): " + f.message); notes.append(line); } + card.append(notes, rec); + + box.append(card); +} + +const W = 420, H = 150, PAD = { l: 44, r: 54, t: 10, b: 22 }; + +function niceTicks(lo, hi, count) { + if (!(hi > lo)) hi = lo + 1; + const step0 = (hi - lo) / Math.max(1, count); + const mag = Math.pow(10, Math.floor(Math.log10(step0))); + const step = [1, 2, 2.5, 5, 10].map((m) => m * mag).find((s) => s >= step0) || 10 * mag; + const out = []; + for (let v = Math.floor(lo / step) * step; v <= hi + step * 0.5; v += step) out.push(+v.toPrecision(12)); + return out; +} + +function drawChart(lines, runIdx, failedRuns, label) { + const svg = svgEl("svg", { viewBox: `0 0 ${W} ${H}`, class: "chart", role: "img", "aria-label": label }); + const n = runIdx.length; + const xOf = new Map(runIdx.map((ri, i) => [ri, PAD.l + (n === 1 ? (W - PAD.l - PAD.r) / 2 : (i * (W - PAD.l - PAD.r)) / (n - 1))])); + let lo = Infinity, hi = -Infinity; + for (const ln of lines) for (const p of ln.points) if (xOf.has(p.r)) { lo = Math.min(lo, p.v); hi = Math.max(hi, p.v * (1 + p.s)); } + if (!isFinite(lo)) { lo = 0; hi = 1; } + const pad = (hi - lo) * 0.12 || hi * 0.1 || 1; + lo = Math.max(0, Math.min(lo, 0) - pad); hi += pad; + const ticks = niceTicks(lo, hi, 4); + lo = Math.min(lo, ticks[0]); hi = Math.max(hi, ticks[ticks.length - 1]); + const yOf = (v) => PAD.t + (H - PAD.t - PAD.b) * (1 - (v - lo) / (hi - lo)); + for (const tv of ticks) { + if (tv < lo || tv > hi) continue; + svg.append(svgEl("line", { class: "grid-line", x1: PAD.l, x2: W - PAD.r, y1: yOf(tv), y2: yOf(tv) })); + const t = svgEl("text", { x: PAD.l - 6, y: yOf(tv) + 3.5, "text-anchor": "end" }); + t.textContent = fmtNs(tv); + svg.append(t); + } + const runs = state.data.runs; + const labelEvery = Math.max(1, Math.ceil(n / 5)); + runIdx.forEach((ri, i) => { + if (i % labelEvery !== 0 && i !== n - 1) return; + const t = svgEl("text", { x: xOf.get(ri), y: H - 6, "text-anchor": i === n - 1 ? "end" : i === 0 ? "start" : "middle" }); + t.textContent = dateOf(runs[ri].commit.date).slice(5); + svg.append(t); + }); + for (const ri of failedRuns) if (xOf.has(ri)) svg.append(svgEl("rect", { class: "fail-mark", x: xOf.get(ri) - 2, y: H - PAD.b - 5, width: 4, height: 5 })); + for (const ln of lines) { + const pts = ln.points.filter((p) => xOf.has(p.r)); + if (!pts.length) continue; + let band = ""; + for (const p of pts) band += (band ? "L" : "M") + xOf.get(p.r).toFixed(1) + "," + yOf(p.v * (1 + p.s)).toFixed(1); + for (let i = pts.length - 1; i >= 0; i--) band += "L" + xOf.get(pts[i].r).toFixed(1) + "," + yOf(pts[i].v).toFixed(1); + svg.append(svgEl("path", { class: "band band--" + ln.lane, d: band + "Z" })); + let d = "", prev = -2; + for (const p of pts) { + const at = runIdx.indexOf(p.r); + d += (at - prev > 1 ? "M" : "L") + xOf.get(p.r).toFixed(1) + "," + yOf(p.v).toFixed(1); + prev = at; + } + svg.append(svgEl("path", { class: "series series--" + ln.lane, d })); + if (pts.length === 1) svg.append(svgEl("circle", { class: "marker marker--" + ln.lane, cx: xOf.get(pts[0].r), cy: yOf(pts[0].v), r: 4 })); + const last = pts[pts.length - 1]; + const lbl = svgEl("text", { class: "end-label", x: W - PAD.r + 6, y: yOf(last.v) + 3.5 }); + lbl.textContent = fmtNs(last.v); + svg.append(lbl); + } + attachHover(svg, lines, runIdx, failedRuns, xOf, yOf); + return svg; +} + +function attachHover(svg, lines, runIdx, failedRuns, xOf, yOf) { + const runs = state.data.runs; + const cross = svgEl("line", { class: "crosshair", x1: 0, x2: 0, y1: PAD.t, y2: H - PAD.b, visibility: "hidden" }); + const markers = lines.map((ln) => svgEl("circle", { class: "marker marker--" + ln.lane, r: 4, visibility: "hidden" })); + const hit = svgEl("rect", { class: "hit", x: PAD.l - 10, y: 0, width: W - PAD.l - PAD.r + 20, height: H }); + svg.append(cross, ...markers, hit); + const tooltip = $("#tooltip"); + const nearestRun = (evt) => { + const rect = svg.getBoundingClientRect(); + const px = ((evt.clientX - rect.left) / rect.width) * W; + let best = -1, bestD = Infinity; + for (const ri of runIdx) { const dx = Math.abs(xOf.get(ri) - px); if (dx < bestD) { bestD = dx; best = ri; } } + return best; + }; + hit.addEventListener("pointermove", (evt) => { + const best = nearestRun(evt); + if (best < 0) return; + const x = xOf.get(best); + cross.setAttribute("x1", x); cross.setAttribute("x2", x); cross.setAttribute("visibility", "visible"); + const run = runs[best]; + tooltip.replaceChildren(el("div", "tt-head", dateOf(run.commit.date) + " " + shortSha(run.commit.sha) + " " + run.commit.subject)); + lines.forEach((ln, i) => { + const p = ln.points.find((q) => q.r === best); + if (!p) { markers[i].setAttribute("visibility", "hidden"); return; } + markers[i].setAttribute("cx", x); markers[i].setAttribute("cy", yOf(p.v)); markers[i].setAttribute("visibility", "visible"); + const row = el("div", "tt-row"); + row.append(el("span", "tt-key tt-key--" + ln.lane), el("span", "tt-val", fmtNs(p.v)), el("span", null, ln.lane), el("span", "tt-sub", "spread " + (p.s * 100).toFixed(1) + "%")); + tooltip.append(row); + }); + if (failedRuns.has(best)) tooltip.append(el("div", "tt-sub", "this file failed that night")); + tooltip.hidden = false; + const tw = tooltip.offsetWidth, th = tooltip.offsetHeight; + tooltip.style.left = (evt.clientX + 14 + tw > window.innerWidth - 8 ? evt.clientX - tw - 14 : evt.clientX + 14) + "px"; + tooltip.style.top = (evt.clientY + 14 + th > window.innerHeight - 8 ? evt.clientY - th - 14 : evt.clientY + 14) + "px"; + }); + hit.addEventListener("pointerleave", () => { + cross.setAttribute("visibility", "hidden"); + markers.forEach((m) => m.setAttribute("visibility", "hidden")); + tooltip.hidden = true; + }); + hit.addEventListener("click", (evt) => { + const best = nearestRun(evt); + const url = best >= 0 ? commitUrl(runs[best].commit.sha) : null; + if (url) window.open(url, "_blank", "noopener"); + }); +} + +function chartCard(id, file, lines, runIdx, failedRuns) { + const card = el("div", "card chart-card"); + card.id = anchorId(id); + const head = el("div", "chart-card__head"); + const ttl = el("div", "chart-card__title"); + ttl.append(el("span", "file", file + " "), id.slice(file.length + 1)); + head.append(ttl); + card.append(head, drawChart(lines, runIdx, failedRuns, id)); + if (lines.length > 1) { + const lg = el("div", "legend"); + for (const ln of lines) lg.append(el("span", "legend--" + ln.lane, ln.lane)); + card.append(lg); + } + return card; +} + +function renderSeries() { + const box = $("#series"); + box.replaceChildren(); + const d = state.data; + const runIdx = visibleRunIndices(); + const visible = new Set(runIdx); + const failedBy = new Map(); + d.runs.forEach((run, ri) => { + for (const f of run.failures) { + const key = f.lane + "\t" + f.path.replace(/\.das$/, ""); + if (!failedBy.has(key)) failedBy.set(key, new Set()); + failedBy.get(key).add(ri); + } + }); + const arms = new Map(); + for (const s of d.series) { + if (!state.lanes.has(s.lane) || (state.group && s.group !== state.group) || !s.runs.some((r) => visible.has(r))) continue; + if (!arms.has(s.id)) arms.set(s.id, { group: s.group, file: s.file, lines: [] }); + const arm = arms.get(s.id); + arm.lines.push({ lane: s.lane, points: s.runs.map((r, i) => ({ r, v: s.ns[i], s: s.spread[i] })) }); + } + $("#series-count").textContent = arms.size + " of " + new Set(d.series.map((s) => s.id)).size + " arms"; + if (!arms.size) { box.append(el("p", "notice", "Nothing matches the filters.")); return; } + const byGroup = new Map(); + for (const [id, arm] of arms) { if (!byGroup.has(arm.group)) byGroup.set(arm.group, []); byGroup.get(arm.group).push([id, arm]); } + for (const [group, entries] of [...byGroup.entries()].sort((a, b) => a[0].localeCompare(b[0]))) { + const section = el("div", "group"); + const h = el("h3"); + h.append(group, el("span", "muted", entries.length + " arms")); + const grid = el("div", "grid"); + for (const [id, arm] of entries.sort((a, b) => a[0].localeCompare(b[0]))) { + arm.lines.sort((a, b) => LANES.indexOf(a.lane) - LANES.indexOf(b.lane)); + const failedRuns = new Set(); + for (const ln of arm.lines) for (const ri of failedBy.get(ln.lane + "\t" + arm.file) || []) failedRuns.add(ri); + grid.append(chartCard(id, arm.file, arm.lines, runIdx, failedRuns)); + } + section.append(h, grid); + box.append(section); + } +} + +function renderRuns() { + const table = $("#runs"); + table.replaceChildren(); + const head = el("tr"); + for (const [txt, cls] of [["date", ""], ["commit", ""], ["status", ""], ["lanes", ""], ["build", "num"], ["suite", "num"], ["ok", "num"], ["failed", "num"], ["record", ""]]) head.append(el("th", cls, txt)); + table.append(head); + for (const ri of visibleRunIndices().reverse()) { + const run = state.data.runs[ri]; + const tr = el("tr"); + const c = el("td"); + const a = el("a", null, shortSha(run.commit.sha)); + a.href = commitUrl(run.commit.sha) || "#"; + c.append(a, " ", el("span", "muted", run.commit.subject.length > 60 ? run.commit.subject.slice(0, 60) + "..." : run.commit.subject)); + const s = el("td"); + s.append(el("span", "pill pill--" + run.status, run.status.replace("_", " "))); + const lanes = el("td"); + for (const lane of Object.keys(run.lanes)) { + const t = el("span", "tag tag--" + lane, lane); + if (run.lanes[lane] !== "ok") { t.textContent = lane + " off"; t.title = run.lanes[lane]; t.className = "tag"; } + lanes.append(t, " "); + } + const rec = el("td"); + const link = el("a", null, "json"); + link.href = recordUrl(run); + rec.append(link); + tr.append(el("td", null, dateOf(run.started)), c, s, lanes, el("td", "num", fmtSec(run.build.seconds)), el("td", "num", fmtSec(run.seconds)), + el("td", "num", String(run.files_ok)), el("td", "num", String(run.failures.length)), rec); + table.append(tr); + } +} + +function renderAll() { + fillGroups(); + renderNight(); + renderSeries(); + renderRuns(); +} + +wireFilters(); +load(); diff --git a/utils/internal/bench-stand/site/index.html b/utils/internal/bench-stand/site/index.html new file mode 100644 index 0000000000..54d602a1b1 --- /dev/null +++ b/utils/internal/bench-stand/site/index.html @@ -0,0 +1,39 @@ + + + + + +daslang benchmark stand + + + + +
+
daslang

benchmark stand

+
+
+
+
+
+ + + +
+
+ + +
+ +
+
+

Benchmarks

+

Run history

+
+
+ Every point is the minimum ns/op over the night's repeats of a dastest --bench run; the band is the spread. Source: utils/internal/bench-stand. + +
+ + + + diff --git a/utils/internal/bench-stand/site/style.css b/utils/internal/bench-stand/site/style.css new file mode 100644 index 0000000000..a5ee5e63e1 --- /dev/null +++ b/utils/internal/bench-stand/site/style.css @@ -0,0 +1,157 @@ +:root { + color-scheme: light; + --page: #f9f9f7; + --surface: #fcfcfb; + --ink: #0b0b0b; + --ink-2: #52514e; + --muted: #898781; + --grid: #e1e0d9; + --axis: #c3c2b7; + --border: rgba(11, 11, 11, 0.10); + --interp: #2a78d6; + --jit: #eb6834; + --good: #0ca30c; + --good-text: #006300; + --warning: #fab219; + --serious: #ec835a; + --critical: #d03b3b; + --chip-on: #e8eef9; +} +@media (prefers-color-scheme: dark) { + :root { + color-scheme: dark; + --page: #0d0d0d; + --surface: #1a1a19; + --ink: #ffffff; + --ink-2: #c3c2b7; + --muted: #898781; + --grid: #2c2c2a; + --axis: #383835; + --border: rgba(255, 255, 255, 0.10); + --interp: #3987e5; + --jit: #d95926; + --good-text: #0ca30c; + --chip-on: #22314a; + } +} + +/* One colour per lane, read by every lane-marked element: chips, tags, legend, lines, bands, markers, tooltip keys. */ +.lane--interp, .tag--interp, .legend--interp, .series--interp, .band--interp, .marker--interp, .tt-key--interp { --lane: var(--interp); } +.lane--jit, .tag--jit, .legend--jit, .series--jit, .band--jit, .marker--jit, .tt-key--jit { --lane: var(--jit); } +* { box-sizing: border-box; } +html, body { margin: 0; } +body { + background: var(--page); + color: var(--ink); + font: 14px/1.45 system-ui, -apple-system, "Segoe UI", sans-serif; +} +a { color: inherit; } +code { font: 12px/1.4 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; } +h1, h2, h3 { font-weight: 600; margin: 0; } +h2 { font-size: 16px; margin: 0 0 12px; } +.muted { color: var(--muted); font-weight: 400; font-size: 13px; } + +.top { + display: flex; flex-wrap: wrap; align-items: baseline; justify-content: space-between; gap: 12px; + padding: 18px 24px 8px; +} +.top__title { display: flex; align-items: baseline; gap: 10px; } +.top__brand { text-decoration: none; color: var(--ink-2); font-weight: 600; } +.top__title h1 { font-size: 20px; } +.top__status { display: flex; flex-wrap: wrap; gap: 10px; align-items: center; color: var(--ink-2); } + +main { padding: 0 24px 32px; max-width: 1500px; } +.section { margin-top: 28px; } + +.filters { + display: flex; flex-wrap: wrap; gap: 18px; align-items: center; + padding: 10px 0 14px; border-bottom: 1px solid var(--grid); +} +.filters__group { display: flex; flex-wrap: wrap; gap: 6px; align-items: center; } +.chip { + background: var(--surface); color: var(--ink-2); border: 1px solid var(--border); + border-radius: 999px; padding: 4px 10px; cursor: pointer; font: inherit; font-size: 13px; +} +.chip.is-on { background: var(--chip-on); color: var(--ink); border-color: transparent; font-weight: 600; } +.lane { display: inline-flex; align-items: center; gap: 6px; cursor: pointer; padding: 4px 8px; } +.lane input { accent-color: var(--lane, var(--ink-2)); } + +.pill { + display: inline-flex; align-items: center; gap: 6px; padding: 2px 10px; border-radius: 999px; + font-size: 13px; font-weight: 600; background: var(--surface); border: 1px solid var(--border); +} +.pill::before { content: ""; width: 8px; height: 8px; border-radius: 50%; background: var(--muted); } +.pill--ok::before { background: var(--good); } +.pill--bench_failed::before, .pill--build_failed::before { background: var(--critical); } +.pill--running::before { background: var(--warning); } +.pill--stale::before { background: var(--serious); } + +.night { margin-top: 18px; display: grid; gap: 14px; grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); } +.card { + background: var(--surface); border: 1px solid var(--border); border-radius: 10px; padding: 12px 14px; +} +.card h3 { font-size: 14px; margin-bottom: 8px; display: flex; gap: 8px; align-items: baseline; } +.card h3 .count { color: var(--muted); font-weight: 400; } +.card ul { margin: 0; padding: 0; list-style: none; } +.card li { padding: 4px 0; border-top: 1px solid var(--grid); display: flex; gap: 8px; flex-wrap: wrap; align-items: baseline; } +.card li:first-child { border-top: 0; } +.card .empty { color: var(--muted); } +.tag { font-size: 12px; padding: 0 6px; border-radius: 4px; border: 1px solid var(--border); color: var(--ink-2); } +.tag--interp, .tag--jit { color: var(--lane); border-color: var(--lane); } +.tag--failure { color: var(--critical); border-color: var(--critical); } +.msg { color: var(--ink-2); font-size: 13px; } +.msg code { white-space: pre-wrap; word-break: break-word; } +pre.logtail { + margin: 6px 0 0; padding: 8px 10px; max-height: 220px; overflow: auto; + background: var(--page); border: 1px solid var(--grid); border-radius: 6px; + font: 12px/1.45 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; color: var(--ink-2); + white-space: pre-wrap; word-break: break-word; +} + +.grid { display: grid; gap: 14px; grid-template-columns: repeat(auto-fill, minmax(360px, 1fr)); } +.group { margin-top: 18px; } +.group > h3 { font-size: 15px; margin: 0 0 8px; display: flex; gap: 10px; align-items: baseline; } +.group > h3 .muted { font-size: 13px; } + +.chart-card { padding: 10px 12px 6px; } +.chart-card__head { display: flex; gap: 8px; align-items: baseline; flex-wrap: wrap; } +.chart-card__title { font-size: 13px; font-weight: 600; word-break: break-all; } +.chart-card__title .file { color: var(--muted); font-weight: 400; } +.legend { display: flex; gap: 12px; font-size: 12px; color: var(--ink-2); margin: 4px 0 0; } +.legend span::before { content: ""; display: inline-block; width: 14px; height: 2px; vertical-align: middle; margin-right: 5px; border-radius: 1px; background: var(--lane, var(--muted)); } +svg.chart { display: block; width: 100%; height: auto; overflow: visible; touch-action: pan-y; } +svg.chart text { fill: var(--muted); font-size: 11px; font-variant-numeric: tabular-nums; } +svg.chart .grid-line { stroke: var(--grid); stroke-width: 1; } +svg.chart .axis-line { stroke: var(--axis); stroke-width: 1; } +svg.chart .series { fill: none; stroke: var(--lane); stroke-width: 2; stroke-linejoin: round; stroke-linecap: round; } +svg.chart .band { fill: var(--lane); opacity: 0.10; } +svg.chart .end-label { fill: var(--ink-2); font-weight: 600; } +svg.chart .crosshair { stroke: var(--axis); stroke-width: 1; } +svg.chart .marker { fill: var(--lane); stroke: var(--surface); stroke-width: 2; } +svg.chart .fail-mark { fill: var(--critical); } +svg.chart .hit { fill: transparent; cursor: crosshair; } + +.table-wrap { overflow-x: auto; } +table { border-collapse: collapse; width: 100%; font-size: 13px; } +th, td { text-align: left; padding: 6px 8px; border-top: 1px solid var(--grid); vertical-align: top; } +th { color: var(--muted); font-weight: 500; border-top: 0; } +td.num, th.num { text-align: right; font-variant-numeric: tabular-nums; } + +.tooltip { + position: fixed; z-index: 10; pointer-events: none; max-width: 360px; + background: var(--surface); color: var(--ink); border: 1px solid var(--border); + border-radius: 8px; padding: 8px 10px; box-shadow: 0 6px 24px rgba(0, 0, 0, 0.12); font-size: 12px; +} +.tooltip .tt-head { color: var(--ink-2); margin-bottom: 4px; } +.tooltip .tt-row { display: flex; gap: 8px; align-items: baseline; } +.tooltip .tt-key { display: inline-block; width: 14px; height: 2px; border-radius: 1px; background: var(--lane, var(--muted)); } +.tooltip .tt-val { font-weight: 600; font-variant-numeric: tabular-nums; } +.tooltip .tt-sub { color: var(--muted); } + +.foot { display: flex; flex-wrap: wrap; justify-content: space-between; gap: 12px; padding: 16px 24px 28px; color: var(--muted); font-size: 12px; border-top: 1px solid var(--grid); } +.notice { padding: 16px; color: var(--ink-2); } +.notice--error { color: var(--critical); } +.select select { + font: inherit; font-size: 13px; color: var(--ink); background: var(--surface); + border: 1px solid var(--border); border-radius: 6px; padding: 4px 8px; +} diff --git a/utils/internal/bench-stand/suite.json b/utils/internal/bench-stand/suite.json new file mode 100644 index 0000000000..ad5fb5167f --- /dev/null +++ b/utils/internal/bench-stand/suite.json @@ -0,0 +1,29 @@ +{ + "root": "benchmarks", + "exclude": [ + "**/tests/**", + "**/_*.das" + ], + "lanes": [ + "interp", + "jit" + ], + "repeat": 3, + "timeout_seconds": 900, + "regression_threshold": 0.1, + "noise_multiplier": 3.0, + "baseline_runs": 7, + "files": { + "core/array/test01.das": { + "skip": "allocates ~19 GB under persistent_heap and gets OOM-killed; unskip once the benchmark is fixed" + }, + "compile/utils.das": { + "timeout_seconds": 7200 + } + }, + "lane_excludes": { + "jit": { + "compile/**": "times a child daslang -compile-only; the lane the parent runs in changes nothing about the child, and the interp lane already carries the group" + } + } +} From 43b94257683a12f04a94cb7941a2810cf02fc181 Mon Sep 17 00:00:00 2001 From: Churkin Aleksey Date: Tue, 15 Sep 2026 10:56:00 +0300 Subject: [PATCH 4/4] bench-stand: one box runs every [benchmark] in the tree, and the tool is benchctl's two new verbs The stand runs every [benchmark] under benchmarks/ on one box, in an interp and a jit lane, and publishes the numbers over commits. suite.json says what runs, with which limits and which exclusions; a run record per night carries the commit, the machine, the build and one result per file per lane; data.json is what the viewer reads. run_stand.sh is the single pass cron calls, and caddy.snippet is the only place the public route is written down. The tool itself is benchctl. benchctl already stored benchmark output, queried it by commit and compared two sets with a Welch test, so a second tool beside it would have restated that: a second parse of dastest's benchmark lines, a second copy of dastest's stats struct under another name, and its own median. It could not lend any of that out, because benchstat reached its numbers only through the SQL row and so pulled sqlite into anything that wanted a median. benchstat now works on BenchmarkRunStats - the struct dastest itself emits - and bench_table maps a stored row into that shape, so the statistics compile with no database behind them. median and median_i64 live there beside the rest; BenchStatsLine is gone as the duplicate of BenchmarkRunStats that it was; the runner's whole-output classifier, never a parser of the same kind, is classify_run_output. bench_suite, bench_runner, bench_history and the run and report verbs sit under utils/benchctl, which dispatches those two before its own parser sees a flag it does not know, and utils/internal/bench-stand keeps only what the box has. Every module orders its top level types, then private helpers, then its public functions, so the tail of a file is its API. --- CMakeLists.txt | 1 + benchmarks/README.md | 6 + benchmarks/compile/utils.das | 149 ++++++++++++ skills/writing_benchmarks.md | 4 +- utils/CMakeLists.txt | 1 + utils/benchctl/REVIEW.md | 42 ++++ utils/benchctl/_jit_probe.das | 6 + utils/benchctl/bench_history.das | 189 +++++++++++++++ utils/benchctl/bench_runner.das | 223 ++++++++++++++++++ utils/benchctl/bench_stand.das | 137 +++++++++++ utils/benchctl/bench_suite.das | 115 +++++++++ utils/benchctl/bench_table.das | 10 + utils/benchctl/benchstat.das | 24 +- utils/benchctl/main.das | 9 +- .../bench-stand => benchctl}/suite.json | 3 - utils/benchctl/tests/_fake_dastest.das | 51 ++++ utils/benchctl/tests/_test_common.das | 32 +++ utils/benchctl/tests/test_bench_cli.das | 78 ++++++ utils/benchctl/tests/test_bench_history.das | 111 +++++++++ utils/benchctl/tests/test_bench_runner.das | 110 +++++++++ utils/benchctl/tests/test_bench_suite.das | 101 ++++++++ utils/internal/bench-stand/README.md | 104 ++++++++ utils/internal/bench-stand/REVIEW.md | 30 +++ utils/internal/bench-stand/caddy.snippet | 2 +- utils/internal/bench-stand/run_stand.sh | 45 ++++ 25 files changed, 1570 insertions(+), 13 deletions(-) create mode 100644 benchmarks/compile/utils.das create mode 100644 utils/benchctl/_jit_probe.das create mode 100644 utils/benchctl/bench_history.das create mode 100644 utils/benchctl/bench_runner.das create mode 100644 utils/benchctl/bench_stand.das create mode 100644 utils/benchctl/bench_suite.das rename utils/{internal/bench-stand => benchctl}/suite.json (88%) create mode 100644 utils/benchctl/tests/_fake_dastest.das create mode 100644 utils/benchctl/tests/_test_common.das create mode 100644 utils/benchctl/tests/test_bench_cli.das create mode 100644 utils/benchctl/tests/test_bench_history.das create mode 100644 utils/benchctl/tests/test_bench_runner.das create mode 100644 utils/benchctl/tests/test_bench_suite.das create mode 100644 utils/internal/bench-stand/README.md create mode 100644 utils/internal/bench-stand/REVIEW.md create mode 100755 utils/internal/bench-stand/run_stand.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index b78cecc45a..5f792a3cc2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2074,6 +2074,7 @@ install(FILES ${DAS_DASCOV_TEST_FILES} DESTINATION utils/dascov/tests) file(GLOB DAS_BENCHCTL_FILES ${PROJECT_SOURCE_DIR}/utils/benchctl/*.das) install(FILES ${DAS_BENCHCTL_FILES} DESTINATION utils/benchctl) install(FILES ${PROJECT_SOURCE_DIR}/utils/benchctl/README.md DESTINATION utils/benchctl) +install(FILES ${PROJECT_SOURCE_DIR}/utils/benchctl/suite.json DESTINATION utils/benchctl) # Install lint (unified linter) + its rule-coverage suite, so a shipped SDK can # self-check the linter the same way run_utils_tests does in-tree. diff --git a/benchmarks/README.md b/benchmarks/README.md index 1f8eb160ab..4fbd79ef77 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -4,6 +4,12 @@ Every `.das` benchmark file in this directory tree is listed below, grouped by subdirectory. +## compile/ + +| File | Description | +|---|---| +| `utils.das` | Compile-only wall time of every tool under `utils/` the nightly build can compile, one `[benchmark]` per tool, each a child `daslang -compile-only -no-module-cache ` - interp lane only, since the child is the same binary whatever lane the parent runs in | + ## terminal/ | File | Description | diff --git a/benchmarks/compile/utils.das b/benchmarks/compile/utils.das new file mode 100644 index 0000000000..94d882dbbe --- /dev/null +++ b/benchmarks/compile/utils.das @@ -0,0 +1,149 @@ +options gen2 + +require dastest/testing_boost +require daslib/clargs +require daslib/fio +require daslib/strings_boost +require strings + +def private compile_tool(b : B?; entry : string) { + let bin = get_host_binary() + let path = path_join(path_join(get_das_root(), "utils"), entry) + let cmd = "\"{bin}\" -compile-only -no-module-cache \"{path}\" 2>&1" + var rc = 0 + var first_error = "" + b |> run(entry) { + first_error = "" + rc = unsafe(popen_timeout(cmd, 600.0) $(f) { + if (f != null) { + while (!feof(f)) { + let ln = fgets(f) + if (empty(first_error) && find(ln, "error[") >= 0) { + first_error = strip(ln) + } + } + } + }) + } + b |> equal(rc, 0, "{entry}: daslang -compile-only exited with {rc}{empty(first_error) ? "" : " - " + first_error}") +} + +[benchmark] +def compile_aot(b : B?) { + compile_tool(b, "aot/main.das") +} + +[benchmark] +def compile_benchctl(b : B?) { + compile_tool(b, "benchctl/main.das") +} + +[benchmark] +def compile_dascov(b : B?) { + compile_tool(b, "dascov/main.das") +} + +[benchmark] +def compile_daspkg(b : B?) { + compile_tool(b, "daspkg/main.das") +} + +[benchmark] +def compile_detect_dupe(b : B?) { + compile_tool(b, "detect-dupe/main.das") +} + +[benchmark] +def compile_fix_lint_errors(b : B?) { + compile_tool(b, "fix-lint-errors/main.das") +} + +[benchmark] +def compile_gen1_to_gen2(b : B?) { + compile_tool(b, "gen1-to-gen2/main.das") +} + +[benchmark] +def compile_lint(b : B?) { + compile_tool(b, "lint/main.das") +} + +[benchmark] +def compile_watchdog(b : B?) { + compile_tool(b, "watchdog/main.das") +} + +[benchmark] +def compile_das_fmt(b : B?) { + compile_tool(b, "das-fmt/dasfmt.das") +} + +[benchmark] +def compile_arch_extract(b : B?) { + compile_tool(b, "internal/arch-extract/main.das") +} + +[benchmark] +def compile_ast_fuzz(b : B?) { + compile_tool(b, "internal/ast-fuzz/main.das") +} + +[benchmark] +def compile_dasweb_verify(b : B?) { + compile_tool(b, "internal/dasweb-verify/main.das") +} + +[benchmark] +def compile_doc_verify(b : B?) { + compile_tool(b, "internal/doc-verify/main.das") +} + +[benchmark] +def compile_flatten_fuzz(b : B?) { + compile_tool(b, "internal/flatten-fuzz/main.das") +} + +[benchmark] +def compile_hygiene(b : B?) { + compile_tool(b, "internal/hygiene/main.das") +} + +[benchmark] +def compile_jit(b : B?) { + compile_tool(b, "internal/jit/main.das") +} + +[benchmark] +def compile_lineinfo_audit(b : B?) { + compile_tool(b, "internal/lineinfo-audit/main.das") +} + +[benchmark] +def compile_make_pr(b : B?) { + compile_tool(b, "internal/make-pr/main.das") +} + +[benchmark] +def compile_pr_babysit(b : B?) { + compile_tool(b, "internal/pr-babysit/main.das") +} + +[benchmark] +def compile_preflight(b : B?) { + compile_tool(b, "internal/preflight/main.das") +} + +[benchmark] +def compile_requirefix(b : B?) { + compile_tool(b, "internal/requirefix/main.das") +} + +[benchmark] +def compile_review_md(b : B?) { + compile_tool(b, "internal/review-md/main.das") +} + +[benchmark] +def compile_test_release(b : B?) { + compile_tool(b, "internal/test-release/main.das") +} diff --git a/skills/writing_benchmarks.md b/skills/writing_benchmarks.md index 98a9c227a5..dd5cf4931e 100644 --- a/skills/writing_benchmarks.md +++ b/skills/writing_benchmarks.md @@ -14,7 +14,9 @@ bin/daslang -jit dastest/dastest.das -- --bench --test path/to/directory/ - `-jit` goes **before** `dastest.das`: it puts dastest itself in JIT mode, which is what gets the benchmark code JIT-compiled. Use it for any performance number you intend to believe -- `--bench-names name1,name2` - run only those benchmark functions +- `--bench-names prefix` - run only the benchmark functions whose name starts with `prefix`; + repeat the flag for several (`--bench-names a --bench-names b`) - a comma-joined list is one + prefix that matches nothing Dropping `--bench` turns the same command into a fast compile check: it reports 0 tests but surfaces every compile error. diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 19755f8562..a06b456bab 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -232,6 +232,7 @@ SET(DAS_UTILS_TO_TEST internal/hygiene internal/preflight/tests internal/ast-fuzz + benchctl/tests internal/requirefix internal/test-release daspkg/test_daspkg.das diff --git a/utils/benchctl/REVIEW.md b/utils/benchctl/REVIEW.md index d7bab13f5f..8c0bd15bed 100644 --- a/utils/benchctl/REVIEW.md +++ b/utils/benchctl/REVIEW.md @@ -2,3 +2,45 @@ **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: `README.md`. + +**Never put a `[test]` file outside `tests/`, and never let a test touch the filesystem outside a +`temp_directory`-rooted path or leave behind what it creates.** + +**A module file orders its top level types, then `private` helpers, then its public functions, so +the tail of the file is the module's whole API.** + +**Never add a run-record field without saying in `README.md` what reads it - the viewer, or a +person opening the record.** A field nobody named is one nobody notices going wrong. + +**A diff that changes a run-record field keeps the new reader parsing a record written by the old +code, a missing field keeping its declared default.** Records already on the box are never +rewritten. + +**Never let a child's own output overwrite a `timeout` status in `run_bench_file` - a killed child +that printed a passing report is still killed.** Loosen `timeout_seconds` in `suite.json` instead. + +**Never read a benchmark's identity from anywhere but its path under `benchmarks/`** - the group +is the directory, the id is the path without `.das`. + +**Statistics live in `benchstat.das` and nowhere else** - it depends on no storage, so a second +median or spread helper anywhere in this folder is a defect. + +**Placement - one file, one line: a diff keeps each file inside its line, and a new file adds its +line here, with its tests, in the same change.** + +- `main.das` - argv, subcommand dispatch, exit codes. +- `bench_args.das` - the database verbs' flags. +- `bench_table.das` - the stored row and its mapping to the shape the statistics read. +- `benchstat.das` - every statistic: median, spread, outlier filtering, geomean, Welch. No storage. +- `bench_suite.das` - `suite.json` and file discovery. No processes. +- `suite.json` - what is benchmarked, in which lanes, with which limits. +- `bench_runner.das` - one file in one lane: spawn, timeout, output to samples and verdict. +- `_jit_probe.das` - the program `probe_jit` runs to prove the jit lane works. +- `bench_history.das` - run records to dataset. No processes, no argv, no statistics. +- `bench_stand.das` - the `run` and `report` verbs the nightly box drives. +- `table_fmt.das`, `utils.das` - output formatting and small shared helpers. +- `tests/test_bench_suite.das`, `tests/test_bench_runner.das`, `tests/test_bench_history.das` - + the module suites; `tests/test_bench_cli.das` - the verbs, spawned; `tests/_test_common.das` - + the fixtures they share; `tests/_fake_dastest.das` - the stand-in dastest a spawn test measures + against. + diff --git a/utils/benchctl/_jit_probe.das b/utils/benchctl/_jit_probe.das new file mode 100644 index 0000000000..313558b4a2 --- /dev/null +++ b/utils/benchctl/_jit_probe.das @@ -0,0 +1,6 @@ +options gen2 + +[export] +def main() { + print("jit-probe-ok\n") +} diff --git a/utils/benchctl/bench_history.das b/utils/benchctl/bench_history.das new file mode 100644 index 0000000000..9694df8e0a --- /dev/null +++ b/utils/benchctl/bench_history.das @@ -0,0 +1,189 @@ +options gen2 +options indenting = 4 + +module bench_history public + +require bench_runner +require daslib/fio +require daslib/json_boost + +enum RunStatus { + ok + build_failed + bench_failed +} + +enum StandExit { + ok = 0 + failed = 1 +} + +// JSON: run record, written per night, read by `report` - see README.md section 2 +struct CommitInfo { + sha, date, subject, author : string +} + +// JSON: run record +struct MachineInfo { + host : string + cores : int +} + +// JSON: run record +struct BuildInfo { + status : string = "ok" + seconds : double + log_tail : string +} + +// JSON: run record +struct RunRecord { + schema : int = 1 + run_id, started, finished : string + seconds : double + commit : CommitInfo + machine : MachineInfo + build : BuildInfo = BuildInfo() + lanes : table + status : RunStatus + files : array +} + +// JSON: run record +struct Failure { + path, lane, status, message : string +} + +// JSON: data.json, read by site/app.js +struct RunSummary { + id, started : string + status : RunStatus + seconds : double + commit : CommitInfo + build : BuildInfo = BuildInfo() + machine : MachineInfo + lanes : table + failures, skipped : array + files_ok : int +} + +// JSON: data.json, read by site/app.js +struct Series { + id, group, file, lane : string + runs : array + ns, spread : array +} + +// JSON: data.json, read by site/app.js +struct Dataset { + schema : int = 1 + generated, repo_url : string + latest : int + runs : array + groups : array + series : array +} + +def derive_status(rec : RunRecord) : RunStatus { + return RunStatus.build_failed if (rec.build.status != "ok") + var measured = 0 + for (f in rec.files) { + return RunStatus.bench_failed if (is_failure(f.status)) + measured++ if (f.status == FileStatus.ok) + } + return measured == 0 ? RunStatus.bench_failed : RunStatus.ok +} + +def mark_failed_run(var rec : RunRecord&; finished, reason : string) { + rec.finished = finished + rec.files |> clear() + if (rec.build.status == "ok") { + rec.files |> emplace(FileResult(path = "run_stand.sh", id = "bench-stand", group = "stand", lane = "driver", + status = FileStatus.exit_nonzero, exit_code = 1, message = empty(reason) ? "the driver could not run the suite" : reason)) + } + rec.status = derive_status(rec) +} + +def load_runs(runs_dir : string; var errors : array) : array { + var runs : array + dir(runs_dir) $(name) { + return if (extension(name) != ".json") + var rec = RunRecord() + if (!sscan_json(fread(path_join(runs_dir, name)), rec) || empty(rec.run_id)) { + errors |> push("run record does not parse or has no run_id: {path_join(runs_dir, name)}") + return + } + rec.status = derive_status(rec) + runs |> emplace(rec) + } + sort(runs) $(a, b) { + return a.commit.date != b.commit.date ? a.commit.date < b.commit.date : a.started < b.started + } + return <- runs +} + +def latest_run_index(runs : array) : int { + var best = -1 + for (i in range(length(runs))) { + if (best < 0 || runs[i].started >= runs[best].started) { + best = i + } + } + return best +} + +def private summarize_run(rec : RunRecord) : RunSummary { + var s <- RunSummary(id = rec.run_id, started = rec.started, status = rec.status, seconds = rec.seconds, + commit = rec.commit, build = rec.build, machine = rec.machine) + s.lanes := rec.lanes + for (f in rec.files) { + if (f.status == FileStatus.ok) { + s.files_ok++ + } else { + let row = Failure(path = f.path, lane = f.lane, status = "{f.status}", message = f.message) + (f.status == FileStatus.skipped ? s.skipped : s.failures) |> push(row) + } + } + return <- s +} + +def private build_series(runs : array; var groups : table) : array { + var index_of : table + var out : array + for (ri, rec in range(length(runs)), runs) { + for (f in rec.files) { + groups |> insert(f.group) if (!empty(f.samples)) + for (smp in f.samples) { + let key = "{f.lane}\t{f.id}#{smp.id}" + var si = index_of?[key] ?? -1 + if (si < 0) { + si = length(out) + index_of[key] = si + out |> emplace(Series(id = "{f.id}#{smp.id}", group = f.group, file = f.id, lane = f.lane)) + } + continue if (!empty(out[si].runs) && out[si].runs[length(out[si].runs) - 1] == ri) + out[si].runs |> push(ri) + out[si].ns |> push(smp.ns) + out[si].spread |> push(smp.spread) + } + } + } + sort(out) $(a, b) { + return a.lane != b.lane ? a.lane < b.lane : a.id < b.id + } + return <- out +} + +def build_dataset(runs : array; generated, repo_url : string) : Dataset { + var ds <- Dataset(generated = generated, repo_url = repo_url, latest = latest_run_index(runs)) + var groups : table + ds.series <- build_series(runs, groups) + ds.groups <- [for (g in keys(groups)); g] + sort(ds.groups) + ds.runs <- [for (rec in runs); summarize_run(rec)] + return <- ds +} + +def report_verdict(ds : Dataset) : StandExit { + return ds.latest < 0 || ds.runs[ds.latest].status != RunStatus.ok ? StandExit.failed : StandExit.ok +} diff --git a/utils/benchctl/bench_runner.das b/utils/benchctl/bench_runner.das new file mode 100644 index 0000000000..9eda0bc9c9 --- /dev/null +++ b/utils/benchctl/bench_runner.das @@ -0,0 +1,223 @@ +options gen2 +options indenting = 4 + +module bench_runner public + +require bench_suite +require benchstat +require daslib/fio +require daslib/json_boost +require daslib/strings_boost +require strings + +enum FileStatus { + ok + skipped + spawn_failed + compile_error + failed + timeout + exit_nonzero +} + +// JSON: run record +struct Sample { + id : string + ns, ns_median, spread : double + runs : int + bytes, allocs, string_bytes, string_allocs : int64 +} + +// JSON: run record +struct FileResult { + path, id, group, lane : string + status : FileStatus + exit_code : int + seconds : double + message, log_tail : string + samples : array +} + +struct RunLimits { + timeout_seconds : int = 900 + repeat : int = 3 +} + +// JSON: dastest --json-file report +struct ReportTest { + name : string + passed : bool +} + +// JSON: dastest --json-file report +struct DastestReport { + success : bool + tests : array +} + +let TAIL_LINES = 40 + +struct private OutputScan { + by_arm : table> + arm_order, tail : array + compile_error, failed_to_compile, fatal : string + failed_fns : table +} + +def private note(var slot : string&; line : string; hit : bool) { + if (hit && empty(slot)) { + slot = strip(line) + } +} + +def private scan_line(var scan : OutputScan; line : string) { + var st = BenchmarkRunStats() + if (starts_with(line, "\{") && sscan_json(line, st) && st.n > 0) { + let key = "{st.name}/{st.sub_name}" + scan.arm_order |> push(key) if (!key_exists(scan.by_arm, key)) + scan.by_arm[key] |> push(st) + return + } + scan.tail |> erase(0) if (length(scan.tail) >= TAIL_LINES) + scan.tail |> push(length(line) > 400 ? slice(line, 0, 400) : line) + note(scan.compile_error, line, find(line, "error[") >= 0 && find(line, "]: ") >= 0) + note(scan.failed_to_compile, line, find(line, "Failed to compile") >= 0) + note(scan.fatal, line, starts_with(line, "FATAL")) + let at = find(line, "--- FAIL '") + return if (at < 0) + let name_end = find(line, "'", at + 10) + scan.failed_fns |> insert(slice(line, at + 10, name_end)) if (name_end > at + 10) +} + +def private aggregate_arm(key : string; stats : array) : Sample { + var ns <- [for (st in stats); double(st.time_ns) / double(st.n)] + var bytes <- [for (st in stats); int64(st.heap_bytes) / int64(st.n)] + var allocs <- [for (st in stats); int64(st.allocs) / int64(st.n)] + var sbytes <- [for (st in stats); int64(st.string_heap_bytes) / int64(st.n)] + var sallocs <- [for (st in stats); int64(st.string_allocs) / int64(st.n)] + sort(ns) + let lo = ns[0] + let hi = ns[length(ns) - 1] + return Sample(id = key, ns = lo, ns_median = median(ns), spread = lo > 0.0lf ? (hi - lo) / lo : 0.0lf, runs = length(ns), + bytes = median_i64(bytes), allocs = median_i64(allocs), string_bytes = median_i64(sbytes), string_allocs = median_i64(sallocs)) +} + +def is_failure(st : FileStatus) : bool { + return st != FileStatus.ok && st != FileStatus.skipped +} + +def classify_run_output(lines : array; report_text : string; exit_code : int; var res : FileResult&) { + var scan <- OutputScan() + for (line in lines) { + scan_line(scan, line) + } + var report : DastestReport + let report_ok = !empty(report_text) && sscan_json(report_text, report) + for (test in report.tests) { + scan.failed_fns |> insert(test.name) if (!test.passed) + } + res.log_tail = join(scan.tail, "\n") + res.exit_code = exit_code + for (key in scan.arm_order) { + res.samples |> emplace(aggregate_arm(key, scan.by_arm[key])) if (!key_exists(scan.failed_fns, slice(key, 0, find(key, "/")))) + } + return if (res.status == FileStatus.timeout) + if (!empty(scan.failed_to_compile)) { + res.status = FileStatus.compile_error + res.message = !empty(scan.compile_error) ? scan.compile_error : scan.failed_to_compile + } elif (!empty(scan.failed_fns)) { + var names <- [for (n in keys(scan.failed_fns)); n] + sort(names) + res.status = FileStatus.failed + res.message = "benchmark function(s) failed: {join(names, ", ")}" + } elif (exit_code != 0 && report_ok && !empty(res.samples)) { + res.status = FileStatus.ok + res.message = "measured, then exited with code {exit_code} at shutdown{empty(scan.fatal) ? "" : ": " + scan.fatal}" + } elif (exit_code != 0 || !report_ok) { + res.status = FileStatus.exit_nonzero + res.message = !empty(scan.fatal) ? scan.fatal : exit_code != 0 ? "process exited with code {exit_code} without a finished dastest report" : "dastest wrote no report file - the process ended before the suite finished" + } elif (empty(res.samples)) { + res.status = FileStatus.failed + res.message = "the file ran but produced no benchmark samples" + } else { + res.status = FileStatus.ok + } +} + +def run_bench_file(bin, repo_root : string; f : BenchFile; lane : string; limits : RunLimits) : FileResult { + var res <- FileResult(path = f.path, id = f.id, group = f.group, lane = lane) + res.message = skip_reason(f, lane) + if (!empty(res.message)) { + res.status = FileStatus.skipped + return <- res + } + var err = "" + let report_path = create_temp_file("bench_stand_", ".json", err) + if (empty(report_path)) { + res.status = FileStatus.spawn_failed + res.message = "cannot create the dastest report file: {err}" + return <- res + } + remove(report_path) + var argv <- [bin] + argv |> push("-jit") if (lane == "jit") + argv |> push_from([to_generic_path(path_join(repo_root, "dastest/dastest.das")), "--", "--bench", "--bench-format", "json", "--json-file", report_path, + "--count", "{limits.repeat}", "--test", to_generic_path(path_join(path_join(repo_root, "benchmarks"), f.path))]) + var lines : array + var exit_code = process_running + let t0 = ref_time_ticks() + let noenv : array + with_process(argv, repo_root, noenv) $(var p) { + unsafe { + if (process_pid(p) <= 0) { + res.status = FileStatus.spawn_failed + res.message = "cannot spawn {bin}" + return + } + while (exit_code == process_running) { + process_drain(p) $(line) { + lines |> push(line) + } + exit_code = process_poll(p) + if (exit_code == process_running && double(get_time_usec(t0)) / 1000000.0lf > double(limits.timeout_seconds)) { + res.status = FileStatus.timeout + res.message = "killed after {limits.timeout_seconds} s (timeout_seconds)" + process_kill(p) + exit_code = process_wait(p, 10.0) + } + sleep(100u) if (exit_code == process_running) + } + process_drain(p) $(line) { + lines |> push(line) + } + } + } + res.seconds = double(get_time_usec(t0)) / 1000000.0lf + return <- res if (res.status == FileStatus.spawn_failed) + let report_text = fread(report_path) + remove(report_path) + classify_run_output(lines, report_text, exit_code, res) + return <- res +} + +def probe_jit(daslang_bin : string) : string { + var err = "" + let probe = create_temp_file("bench_stand_jit_", ".das", err) + return "cannot create the probe file: {err}" if (empty(probe)) + let source = fread(path_join(get_das_root(), "utils/benchctl/_jit_probe.das")) + return "cannot read utils/benchctl/_jit_probe.das" if (empty(source)) + fwrite(probe, source) + var out = "" + let code = run_and_capture([daslang_bin, "-jit", probe], out, 120.0) + remove(probe) + return "" if (find(out, "jit-probe-ok") >= 0) + var said = "" + for (raw in split(out, "\n")) { + let line = strip(raw) + continue if (empty(line)) + said = line + break if (starts_with(line, "FATAL") || starts_with(line, "error") || find(line, "[E]") >= 0) + } + return "daslang -jit produced no jit-compiled run (exit {code}) and printed nothing - is {daslang_bin} runnable?" if (empty(said)) + return "daslang -jit produced no jit-compiled run (exit {code}): {said}" +} diff --git a/utils/benchctl/bench_stand.das b/utils/benchctl/bench_stand.das new file mode 100644 index 0000000000..7f09fb9354 --- /dev/null +++ b/utils/benchctl/bench_stand.das @@ -0,0 +1,137 @@ +options gen2 +options indenting = 4 + +module bench_stand public + +require bench_suite +require bench_runner +require bench_history +require daslib/clargs +require daslib/fio +require daslib/json_boost +require daslib/strings_boost +require strings + +[CommandLineArgs] +struct Config { + @clarg_positional + @clarg_doc = "run | report" + verb : string + @clarg_doc = "Repo root (default: .)" + root : string + @clarg_doc = "suite.json to use (default: the one beside this tool)" + suite : string + @clarg_doc = "run: daslang binary to measure with (default: bin/daslang under --root)" + bin : string + @clarg_doc = "run: JSON with run_id, started, commit, machine, build - written by run_stand.sh" + meta : string + @clarg_doc = "run: where to write the run record" + out : string + @clarg_doc = "run: only files whose path contains this substring" + filter : string + @clarg_doc = "run: comma-separated lanes to run (default: suite.json lanes)" + lanes : string + @clarg_doc = "run: benchmark nothing - write the record of a night that could not run the suite, with this reason, and exit 1" + failed : string + @clarg_doc = "run: repeats per file (default: suite.json repeat)" + repeat : int + @clarg_doc = "report: directory of run records" + runs : string + @clarg_doc = "report: where to write data.json" + out_data : string + @clarg_doc = "report: repository URL for commit links" + repo_url : string + @clarg_short = "?" + @clarg_name = "show-help" + @clarg_doc = "Show this help and exit" + help : bool +} + +def private fail(msg : string) : StandExit { + to_log(LOG_ERROR, "bench-stand: {msg}\n") + return StandExit.failed +} + +def private write_record(path : string; value) : bool { + return true if (fwrite(path, sprint_json(value, true))) + fail("cannot write {path}") + return false +} + +def private measure(var rec : RunRecord&; files : array; lanes : array; bin, root, filter : string; repeat : int) : int { + var limits = RunLimits(repeat = repeat) + var failed = 0 + for (f in files) { + continue if (!empty(filter) && find(f.path, filter) < 0) + for (lane in lanes) { + continue if (rec.lanes[lane] != "ok") + limits.timeout_seconds = f.timeout_seconds + var res <- run_bench_file(bin, root, f, lane, limits) + failed++ if (is_failure(res.status)) + to_log(is_failure(res.status) ? LOG_ERROR : LOG_INFO, "{f.path} {lane}: {res.status} {res.seconds:.1f}s {length(res.samples)} arms{empty(res.message) ? "" : " - " + res.message}\n") + rec.files |> emplace(res) + } + } + return failed +} + +def private verb_run(cfg : Config; suite : SuiteConfig) : StandExit { + return fail("run: --meta and --out are required") if (empty(cfg.meta) || empty(cfg.out)) + let root = empty(cfg.root) ? "." : cfg.root + let bin = empty(cfg.bin) ? path_join(root, "bin/daslang") : cfg.bin + var rec = RunRecord() + return fail("run: --meta {cfg.meta} does not parse or has no run_id") if (!sscan_json(fread(cfg.meta), rec) || empty(rec.run_id)) + if (!empty(cfg.failed)) { + mark_failed_run(rec, "{iso8601_now()}", cfg.failed) + write_record(cfg.out, rec) + return fail("run: {rec.status} - {cfg.failed}; record {cfg.out}") + } + var error = "" + let files <- discover_files(root, suite, error) + return fail("run: {error}") if (!empty(error)) + let lanes <- empty(cfg.lanes) ? clone_to_move(suite.lanes) : split(cfg.lanes, ",") + for (lane in lanes) { + let why = lane == "jit" ? probe_jit(bin) : "" + rec.lanes[lane] = empty(why) ? "ok" : why + to_log(LOG_INFO, "lane {lane}: {rec.lanes[lane]}\n") + } + let t0 = ref_time_ticks() + let failed = measure(rec, files, lanes, bin, root, cfg.filter, cfg.repeat > 0 ? cfg.repeat : suite.repeat) + rec.seconds = double(get_time_usec(t0)) / 1000000.0lf + rec.finished = "{iso8601_now()}" + rec.status = derive_status(rec) + return StandExit.failed if (!write_record(cfg.out, rec)) + to_log(rec.status == RunStatus.ok ? LOG_INFO : LOG_ERROR, "bench-stand run: {rec.status} - {length(rec.files)} file runs, {failed} failed, {rec.seconds:.0f} s; record {cfg.out}\n") + return rec.status == RunStatus.ok ? StandExit.ok : StandExit.failed +} + +def private verb_report(cfg : Config) : StandExit { + return fail("report: --runs and --out-data are required") if (empty(cfg.runs) || empty(cfg.out_data)) + var errors : array + let runs <- load_runs(cfg.runs, errors) + for (e in errors) { + to_log(LOG_WARNING, "bench-stand report: {e}\n") + } + let ds <- build_dataset(runs, "{iso8601_now()}", cfg.repo_url) + return fail("cannot write {cfg.out_data}") if (!fwrite(cfg.out_data, sprint_json(ds, false))) + to_log(LOG_INFO, "bench-stand report: {length(runs)} runs, {length(ds.series)} series -> {cfg.out_data}\n") + return report_verdict(ds) +} + +def stand_main() : int { + var r <- parse_args(type) + if (r |> is_err) { + to_log(LOG_ERROR, "error: {r |> unwrap_err}\n") + print_help(get_command_info(type), "bench-stand") + return 1 + } + let cfg <- r |> move_unwrap + if (cfg.help || (cfg.verb != "run" && cfg.verb != "report")) { + print_help(get_command_info(type), "bench-stand") + return int(cfg.help ? StandExit.ok : StandExit.failed) + } + var error = "" + let suite <- load_suite_config(empty(cfg.suite) ? path_join(get_das_root(), "utils/benchctl/suite.json") : cfg.suite, error) + return int(fail(error)) if (!empty(error)) + return int(cfg.verb == "run" ? verb_run(cfg, suite) : verb_report(cfg)) +} diff --git a/utils/benchctl/bench_suite.das b/utils/benchctl/bench_suite.das new file mode 100644 index 0000000000..4e80151336 --- /dev/null +++ b/utils/benchctl/bench_suite.das @@ -0,0 +1,115 @@ +options gen2 +options indenting = 4 + +module bench_suite public + +require daslib/fio +require daslib/json_boost + +// JSON: suite.json +struct FileOverride { + @optional skip : string + @optional skip_lanes : table + @optional timeout_seconds : int +} + +// JSON: suite.json +struct SuiteConfig { + root : string = "benchmarks" + exclude, lanes : array + repeat : int = 3 + timeout_seconds : int = 900 + files : table + lane_excludes : table> +} + +struct BenchFile { + path, id, group : string + timeout_seconds : int + skip : string + skip_lanes : table +} + +def skip_reason(f : BenchFile; lane : string) : string { + return !empty(f.skip) ? f.skip : f.skip_lanes?[lane] ?? "" +} + +def load_suite_config(path : string; var error : string&) : SuiteConfig { + error = "" + var cfg = SuiteConfig() + if (!stat(path).is_reg) { + error = "suite config not found: {path}" + } elif (!sscan_json(fread(path), cfg)) { + error = "suite config does not parse as JSON: {path}" + } elif (empty(cfg.lanes)) { + error = "suite config lists no lanes: {path}" + } elif (cfg.repeat <= 0 || cfg.timeout_seconds <= 0) { + error = "suite config needs positive repeat and timeout_seconds: {path}" + } + for (lane in keys(cfg.lane_excludes)) { + if (find_index(cfg.lanes, lane) < 0) { + error = "suite config excludes files from lane \"{lane}\", which it does not run: {path}" + } + } + for (lane in cfg.lanes) { + if (lane != "interp" && lane != "jit") { + error = "suite config names an unknown lane \"{lane}\" (interp or jit): {path}" + } + } + return <- cfg +} + +def discover_files(repo_root : string; cfg : SuiteConfig; var error : string&) : array { + error = "" + let root = path_join(repo_root, cfg.root) + var rels : array + dir_rec(root) $(name, is_dir) { + let rel = to_generic_path(name) + return if (is_dir || extension(rel) != ".das") + for (pattern in cfg.exclude) { + return if (match_glob(pattern, rel)) + } + rels |> push(rel) + } + sort(rels) + var out : array + out |> reserve(length(rels)) + for (rel in rels) { + var f = BenchFile(path = rel, id = stem_path(rel), group = group_of(rel), timeout_seconds = cfg.timeout_seconds) + for (lane in keys(cfg.lane_excludes)) { + cfg.lane_excludes |> get(lane) $(globs) { + for (pattern in keys(globs)) { + if (match_glob(pattern, rel)) { + f.skip_lanes[lane] = globs?[pattern] ?? "" + } + } + } + } + cfg.files |> get(rel) $(ov) { + f.skip = ov.skip + for (lane in keys(ov.skip_lanes)) { + f.skip_lanes[lane] = ov.skip_lanes?[lane] ?? "" + } + if (ov.timeout_seconds > 0) { + f.timeout_seconds = ov.timeout_seconds + } + } + out |> emplace(f) + } + for (k in keys(cfg.files)) { + if (!stat(path_join(root, k)).is_reg) { + error = "suite config overrides \"{k}\", which is not a file under {root}" + } + } + return <- out +} + +def group_of(rel : string) : string { + let d = to_generic_path(dir_name(rel)) + return (empty(d) || d == ".") ? "root" : d +} + +def stem_path(rel : string) : string { + let d = group_of(rel) + return d == "root" ? stem(rel) : "{d}/{stem(rel)}" +} diff --git a/utils/benchctl/bench_table.das b/utils/benchctl/bench_table.das index bbf41270b2..8d357accb0 100644 --- a/utils/benchctl/bench_table.das +++ b/utils/benchctl/bench_table.das @@ -3,6 +3,7 @@ options gen2 require daslib/sql require sqlite/sqlite_boost require daslib/sql_linq +require dastest/testing_boost public [sql_table(name = "benchmarks")] struct Benchmark { @@ -21,3 +22,12 @@ struct Benchmark { string_allocs : int64 string_heap_bytes : int64 } + +//! The stored row carries commit and tag metadata the statistics never read - benchstat works on +//! the shape dastest emits, so a query hands its rows over as those. +def to_run_stats(rows : array) : array { + return <- [for (r in rows); BenchmarkRunStats(name = r.name, sub_name = r.sub_name, + n = int(r.n), time_ns = r.time_ns, allocs = uint64(r.allocs), heap_bytes = uint64(r.heap_bytes), + string_allocs = uint64(r.string_allocs), string_heap_bytes = uint64(r.string_heap_bytes), + func_type = r.mode)] +} diff --git a/utils/benchctl/benchstat.das b/utils/benchctl/benchstat.das index 4c1878baa4..cd64e5be6b 100644 --- a/utils/benchctl/benchstat.das +++ b/utils/benchctl/benchstat.das @@ -1,15 +1,13 @@ options gen2 -require dastest/testing_boost +require dastest/testing_boost public require daslib/strings_boost require daslib/json_boost require math -require bench_table public - struct BenchmarkSampleSet { key : string - list : array + list : array // Assigned later, when stats are computed. stats : BenchmarkStats? = null @@ -63,7 +61,7 @@ def parse_bench_output(data : string) : array { return <- entries } -def make_sample_sets(entries : array) : table { +def make_sample_sets(entries : array) : table { var result : table for (e in entries) { let key = e.name + "/" + e.sub_name @@ -92,7 +90,7 @@ def fill_sample_stats(var samples : BenchmarkSampleSet?) : string { // An example: there can be N entries for "foo/bar" benchmark, // every such entry is considered to be a single sample for analysis. for (e in samples.list) { - if (e.n == 0l) continue + if (e.n == 0) continue let n = double(e.n) stats.time_ns.values |> push(double(e.time_ns) / n) stats.heap_bytes.values |> push(double(e.heap_bytes) / n) @@ -291,3 +289,17 @@ def private betacf(a : double; b : double; x : double) : double { } return h } + +def median(vals : array) : double { + return 0.0lf if (empty(vals)) + var sorted := vals + sort(sorted) + let n = length(sorted) + return (n % 2 == 1) ? sorted[n / 2] : (sorted[n / 2 - 1] + sorted[n / 2]) / 2.0lf +} + +def median_i64(var vals : array) : int64 { + return 0l if (empty(vals)) + sort(vals) + return vals[length(vals) / 2] +} diff --git a/utils/benchctl/main.das b/utils/benchctl/main.das index 6b6127dc37..6017308803 100644 --- a/utils/benchctl/main.das +++ b/utils/benchctl/main.das @@ -13,6 +13,7 @@ require bench_args require benchstat require utils require table_fmt +require bench_stand def print_usage() { print("benchctl - benchmark database management tool\n\n") @@ -23,6 +24,8 @@ def print_usage() { print(" insert Insert benchmark JSON output files into the database\n") print(" query Display benchmark records (filter by --commit / --tag)\n") print(" compare Compare two sets of results statistically\n") + print(" run Benchmark a tree into one run record (the nightly stand)\n") + print(" report Turn run records into the viewer's data.json, with a verdict exit code\n") print(" help Show this help message\n\n") print_benchctl_help() } @@ -41,6 +44,8 @@ def script_argv() : array { def main : int { let argv <- script_argv() + if (!empty(argv) && (argv[0] == "run" || argv[0] == "report")) return stand_main() + if (empty(argv) || find_index(argv, "--help") != -1 || find_index(argv, "-h") != -1) { print_usage() return 0 @@ -375,8 +380,8 @@ def run_compare_cmd(db : SqlRunner; parsed : ParsedArgs) : string { } } - var old_samples = make_sample_sets(old_entries) - var new_samples = make_sample_sets(new_entries) + var old_samples = make_sample_sets(to_run_stats(old_entries)) + var new_samples = make_sample_sets(to_run_stats(new_entries)) var bench_order <- [for (k in keys(old_samples)); k] sort(bench_order) $(x, y) { diff --git a/utils/internal/bench-stand/suite.json b/utils/benchctl/suite.json similarity index 88% rename from utils/internal/bench-stand/suite.json rename to utils/benchctl/suite.json index ad5fb5167f..6708d94e33 100644 --- a/utils/internal/bench-stand/suite.json +++ b/utils/benchctl/suite.json @@ -10,9 +10,6 @@ ], "repeat": 3, "timeout_seconds": 900, - "regression_threshold": 0.1, - "noise_multiplier": 3.0, - "baseline_runs": 7, "files": { "core/array/test01.das": { "skip": "allocates ~19 GB under persistent_heap and gets OOM-killed; unskip once the benchmark is fixed" diff --git a/utils/benchctl/tests/_fake_dastest.das b/utils/benchctl/tests/_fake_dastest.das new file mode 100644 index 0000000000..d7c080e175 --- /dev/null +++ b/utils/benchctl/tests/_fake_dastest.das @@ -0,0 +1,51 @@ +options gen2 + +require daslib/clargs +require daslib/fio +require daslib/json_boost +require strings + +[CommandLineArgs] +struct FakeArgs { + @clarg_doc = "where to write the dastest report" + json_file : string + @clarg_doc = "the benchmark file under test" + test : string + @clarg_doc = "repeats" + count : int + @clarg_doc = "measure" + bench : bool + @clarg_doc = "output format" + bench_format : string + @clarg_doc = "behaviour: ok | hang | no_report" + use_aot : bool +} + +def private mode(test_file : string) : string { + let m = strip(fread(path_join(dir_name(dir_name(test_file)), "fake_mode.txt"))) + return empty(m) ? "ok" : m +} + +def private stats(name, sub : string; n : int; time_ns : int64) : string { + return write_json_compact(JV((name = name, sub_name = sub, n = n, time_ns = time_ns, + allocs = 0, heap_bytes = 0, string_allocs = 0, string_heap_bytes = 0))) +} + +[export] +def main : int { + var r <- parse_args(type) + return 1 if (r |> is_err) + let args <- r |> move_unwrap + let how = mode(args.test) + print("lane={jit_enabled() ? "jit" : "interp"}\n") + print("{stats("b", "arm", 10, 1000l)}\n") + if (how == "hang") { + while (true) { + sleep(200u) + } + } + if (how != "no_report") { + fwrite(args.json_file, write_json_compact(JV((success = true, tests = JV([JV((name = "b", passed = true))]))))) + } + return 0 +} diff --git a/utils/benchctl/tests/_test_common.das b/utils/benchctl/tests/_test_common.das new file mode 100644 index 0000000000..318f731bdd --- /dev/null +++ b/utils/benchctl/tests/_test_common.das @@ -0,0 +1,32 @@ +options gen2 +module _test_common public + +require dastest/testing_boost +require daslib/fio +require daslib/json_boost + +def with_temp_dir(prefix : string; blk : block<(dir : string) : void>) { + var err = "" + let dir = create_temp_directory(prefix, err) + verify(!empty(dir), "temp dir created") + invoke(blk, dir) + rmdir_rec(dir) +} + +def fake_repo(root, mode : string) { + mkdir_rec(path_join(root, "dastest")) + mkdir_rec(path_join(root, "benchmarks/x")) + fwrite(path_join(root, "dastest/dastest.das"), fread(path_join(get_das_root(), "utils/benchctl/tests/_fake_dastest.das"))) + fwrite(path_join(root, "benchmarks/x/y.das"), "options gen2\n") + fwrite(path_join(root, "benchmarks/fake_mode.txt"), mode) +} + +def report_json(names, failed : array) : string { + var tests <- [for (n in names); JV((name = n, passed = find_index(failed, n) < 0))] + return write_json_compact(JV((success = empty(failed), tests = JV(tests)))) +} + +def stats_line(name, sub : string; n : int; time_ns : int64; allocs : int = 0) : string { + return write_json_compact(JV((name = name, sub_name = sub, n = n, time_ns = time_ns, + allocs = allocs, heap_bytes = allocs * 16, string_allocs = 0, string_heap_bytes = 0))) +} diff --git a/utils/benchctl/tests/test_bench_cli.das b/utils/benchctl/tests/test_bench_cli.das new file mode 100644 index 0000000000..66a668a7d9 --- /dev/null +++ b/utils/benchctl/tests/test_bench_cli.das @@ -0,0 +1,78 @@ +options gen2 + +require dastest/testing_boost public +require ../bench_history.das +require ../bench_runner.das +require _test_common +require daslib/fio +require daslib/clargs +require daslib/json_boost + +let private TOOL_DIR = "utils/benchctl" + +def private write_meta(dir : string; build_status : string) : string { + let path = path_join(dir, "meta.json") + fwrite(path, write_json(JV((run_id = "20260908T030000Z-abcdef12", started = "2026-09-08T03:00:00Z", + commit = JV((sha = "abcdef1234567890", date = "2026-09-08T01:00:00Z", subject = "a commit", author = "dev")), + machine = JV((host = "box", cores = 4)), + build = JV((status = build_status, seconds = 12, log_tail = "ninja: build stopped")))))) + return path +} + +def private run_tool(args : array; var output : string&) : int { + var argv <- ["bin/daslang", path_join(TOOL_DIR, "main.das"), "--"] + argv |> push_from(args) + return run_and_capture(argv, output, 240.0) +} + +def private write_run_record(runs_dir : string; n : int; ns : double) { + var rec <- RunRecord(run_id = "r{n}", started = "2026-09-0{n + 1}T03:00:00Z", commit = CommitInfo(sha = "sha{n}", date = "2026-09-0{n + 1}T01:00:00Z", subject = "c{n}"), + machine = MachineInfo(host = "box"), build = BuildInfo(status = "ok", seconds = 1.0lf)) + rec.lanes["interp"] = "ok" + var f <- FileResult(path = "core/one.das", id = "core/one", group = "core", lane = "interp", status = FileStatus.ok) + f.samples |> emplace(Sample(id = "b/arm", ns = ns, ns_median = ns, runs = 1)) + rec.files |> emplace(f) + fwrite(path_join(runs_dir, "r{n}.json"), sprint_json(rec, false)) +} + +[test] +def test_verbs(t : T?) { + with_temp_dir("bench_cli_") $(dir) { + let suite = path_join(dir, "suite.json") + fwrite(suite, write_json(JV((root = "benchmarks", exclude = JV(["**/_*.das"]), lanes = JV(["interp"]), repeat = 1, timeout_seconds = 120)))) + t |> run("run records one file per lane and exits on the night's verdict; --failed records a night that never ran") @(tt : T?) { + fake_repo(dir, "ok") + fwrite(path_join(dir, "benchmarks/x/z.das"), "options gen2\n") + let record = path_join(dir, "run.json") + var out = "" + let code = run_tool(["run", "--root", dir, "--bin", get_host_binary(), "--suite", suite, "--meta", write_meta(dir, "ok"), "--out", record, "--filter", "y.das"], out) + tt |> equal(code, 0, "exit 0 when every file ran: {out}") + var rec = RunRecord() + tt |> success(sscan_json(fread(record), rec), "the record parses") + tt |> equal(rec.status, RunStatus.ok) + tt |> equal(length(rec.files), 1, "--filter narrowed two files to one") + tt |> equal(rec.files[0].path, "x/y.das") + tt |> equal(length(rec.files[0].samples), 1) + let failed = path_join(dir, "failed.json") + tt |> equal(run_tool(["run", "--root", dir, "--bin", get_host_binary(), "--suite", suite, "--meta", write_meta(dir, "failed"), "--out", failed, "--failed", "the build failed"], out), 1) + tt |> success(sscan_json(fread(failed), rec) && rec.status == RunStatus.build_failed && empty(rec.files), "a failed build's record carries no file") + tt |> equal(run_tool(["run", "--root", dir, "--out", record], out), 1, "a missing --meta exits 1") + } + t |> run("report exits 0 on a clean night and 1 with no records") @(tt : T?) { + let runs_dir = path_join(dir, "runs") + mkdir_rec(runs_dir) + var out = "" + tt |> equal(run_tool(["report", "--suite", suite, "--runs", runs_dir, "--out-data", path_join(dir, "d0.json")], out), 1, "no records yet") + for (n in range(4)) { + write_run_record(runs_dir, n, 100.0lf) + } + let data = path_join(dir, "data.json") + tt |> equal(run_tool(["report", "--suite", suite, "--runs", runs_dir, "--out-data", data, "--repo-url", "https://example.invalid/repo"], out), 0, "a clean latest night: {out}") + var ds = Dataset() + tt |> success(sscan_json(fread(data), ds), "data.json parses") + tt |> equal(length(ds.runs), 4) + tt |> equal(length(ds.series), 1) + tt |> equal(ds.repo_url, "https://example.invalid/repo") + } + } +} diff --git a/utils/benchctl/tests/test_bench_history.das b/utils/benchctl/tests/test_bench_history.das new file mode 100644 index 0000000000..ae3b40c2d4 --- /dev/null +++ b/utils/benchctl/tests/test_bench_history.das @@ -0,0 +1,111 @@ +options gen2 + +require dastest/testing_boost public +require ../bench_history.das +require ../benchstat.das +require ../bench_runner.das +require _test_common +require daslib/fio +require daslib/json_boost +require daslib/strings_boost +require strings +require math + +def private sample(id : string; ns : double) : Sample { + return Sample(id = id, ns = ns, ns_median = ns, spread = 0.05lf, runs = 3) +} + +def private file_ok(id, lane : string; samples : array) : FileResult { + var f <- FileResult(path = "{id}.das", id = id, group = dir_name(id), lane = lane, status = FileStatus.ok) + f.samples := samples + return <- f +} + +def private file_bad(id, lane : string; status : FileStatus; message : string) : FileResult { + return <- FileResult(path = "{id}.das", id = id, group = dir_name(id), lane = lane, status = status, message = message) +} + +def private night(n : int; files : array) : RunRecord { + var rec <- RunRecord(run_id = "r{n:02}", started = "2026-09-{n + 1:02}T03:00:00Z", finished = "2026-09-{n + 1:02}T04:00:00Z", seconds = 3600.0lf, + commit = CommitInfo(sha = "sha{n:02}abcdef", date = "2026-09-{n + 1:02}T00:00:00Z", subject = "commit {n}", author = "dev"), + machine = MachineInfo(host = "box", cores = 4), build = BuildInfo(status = "ok", seconds = 600.0lf)) + rec.lanes["interp"] = "ok" + rec.lanes["jit"] = "ok" + rec.files := files + rec.status = derive_status(rec) + return <- rec +} + +def private eight_nights_with_a_jump() : array { + var runs <- [for (n in range(7)); night(n, [ + file_ok("core/hash/t", "interp", [sample("a/x", 100.0lf + double(n % 3) * 0.5lf), sample("b/y", 200.0lf - double(n % 3) * 0.5lf)]), + file_ok("core/hash/t", "jit", [sample("a/x", 50.0lf + double(n % 3) * 0.5lf)])])] + runs |> emplace(night(7, [ + file_ok("core/hash/t", "interp", [sample("a/x", 130.0lf), sample("c/z", 5.0lf)]), + file_ok("core/hash/t", "jit", [sample("a/x", 40.0lf)]), + file_bad("sort/s", "interp", FileStatus.compile_error, "error[30344]: mismatch")])) + return <- runs +} + +[test] +def test_load_runs(t : T?) { + with_temp_dir("bench_hist_") $(dir) { + t |> run("records load in commit order, a corrupt file is named and skipped, status derives") @(tt : T?) { + for (rec in [night(2, [file_ok("core/t", "interp", [sample("a/x", 1.0lf)])]), night(0, [file_ok("core/t", "interp", [sample("a/x", 1.0lf)])]), night(1, [file_bad("core/t", "interp", FileStatus.timeout, "killed")])]) { + fwrite(path_join(dir, "{rec.run_id}.json"), sprint_json(rec, false)) + } + fwrite(path_join(dir, "zz_corrupt.json"), "\{ not json") + fwrite(path_join(dir, "notes.txt"), "ignored") + var errors : array + let runs <- load_runs(dir, errors) + tt |> equal(join([for (r in runs); r.run_id], " "), "r00 r01 r02") + tt |> equal(runs[1].status, RunStatus.bench_failed) + tt |> equal(runs[0].status, RunStatus.ok) + tt |> equal(length(errors), 1) + tt |> success(find(errors[0], "zz_corrupt.json") >= 0, "corrupt file named") + } + t |> run("a night that measured nothing is not ok, a failed build is build_failed, a driver failure names the driver") @(tt : T?) { + let no_files : array + tt |> equal(derive_status(night(30, no_files)), RunStatus.bench_failed, "no file ran at all") + tt |> equal(derive_status(night(31, [file_bad("core/x", "interp", FileStatus.skipped, "listed")])), RunStatus.bench_failed, "nothing but skips is no data point") + tt |> equal(derive_status(night(32, [file_ok("core/x", "interp", [sample("a/x", 1.0lf)])])), RunStatus.ok, "one measured file is a night") + var rec <- night(5, no_files) + rec.build.status = "failed" + tt |> equal(derive_status(rec), RunStatus.build_failed) + var driver <- night(6, no_files) + mark_failed_run(driver, "now", "the tool crashed") + tt |> equal(driver.status, RunStatus.bench_failed) + tt |> equal(length(driver.files), 1) + } + } +} + +[test] +def test_build_dataset(t : T?) { + t |> run("series are keyed by lane and arm, points carry the run index; the dataset round-trips") @(tt : T?) { + let ds <- build_dataset(eight_nights_with_a_jump(), "2026-09-08T05:00:00Z", "https://github.com/x/y") + tt |> equal(length(ds.runs), 8) + tt |> equal(ds.latest, 7) + tt |> equal(join(ds.groups, " "), "core/hash") + tt |> equal(length(ds.series), 4) + tt |> equal(ds.series[0].lane, "interp") + tt |> equal(ds.series[0].id, "core/hash/t#a/x") + tt |> equal(length(ds.series[0].runs), 8) + tt |> success(abs(ds.series[0].ns[7] - 130.0lf) < 1e-9lf, "tonight's value") + tt |> equal(ds.series[3].lane, "jit") + var back = Dataset() + tt |> success(sscan_json(sprint_json(ds, false), back), "parses back") + tt |> equal(length(back.series), 4) + tt |> equal(back.runs[7].commit.sha, "sha07abcdef") + } + t |> run("the run summary carries failures with their message and the ok count; a failed night exits 1") @(tt : T?) { + let ds <- build_dataset(eight_nights_with_a_jump(), "", "") + tt |> equal(ds.runs[7].status, RunStatus.bench_failed) + tt |> equal(ds.runs[7].files_ok, 2) + tt |> equal(length(ds.runs[7].failures), 1) + tt |> equal(ds.runs[7].failures[0].path, "sort/s.das") + tt |> equal(ds.runs[7].failures[0].status, "compile_error") + tt |> equal(ds.runs[7].failures[0].message, "error[30344]: mismatch") + tt |> equal(report_verdict(ds), StandExit.failed) + } +} diff --git a/utils/benchctl/tests/test_bench_runner.das b/utils/benchctl/tests/test_bench_runner.das new file mode 100644 index 0000000000..040ce25d24 --- /dev/null +++ b/utils/benchctl/tests/test_bench_runner.das @@ -0,0 +1,110 @@ +options gen2 + +require dastest/testing_boost public +require ../bench_runner.das +require ../bench_suite.das +require _test_common +require daslib/clargs +require strings + +def private fresh_result(status : FileStatus = FileStatus.ok) : FileResult { + return <- FileResult(path = "core/x.das", id = "core/x", group = "core", lane = "interp", status = status) +} + +def private parsed(lines : array; report : string; exit_code : int; status : FileStatus = FileStatus.ok) : FileResult { + var res <- fresh_result(status) + classify_run_output(lines, report, exit_code, res) + return <- res +} + +[test] +def test_classify_run_output(t : T?) { + t |> run("samples aggregate over repeats: min, median, spread, per-op allocation medians") @(tt : T?) { + let res <- parsed([stats_line("b", "insert", 10, 1000l, 20), stats_line("b", "insert", 10, 1200l, 30), stats_line("b", "insert", 10, 1100l, 40), "noise"], report_json(["b"], []), 0) + tt |> equal(res.status, FileStatus.ok) + tt |> equal(length(res.samples), 1) + tt |> equal(res.samples[0].id, "b/insert") + tt |> equal(res.samples[0].ns, 100.0lf) + tt |> equal(res.samples[0].ns_median, 110.0lf) + tt |> success(res.samples[0].spread > 0.19lf && res.samples[0].spread < 0.21lf, "spread is (max - min) / min") + tt |> equal(res.samples[0].runs, 3) + tt |> equal(res.samples[0].allocs, 3l) + tt |> equal(res.log_tail, "noise") + } + t |> run("a compile error names the first error line") @(tt : T?) { + let res <- parsed(["x.das:3:1: error[30001]: bad", "Failed to compile x.das"], "", 1) + tt |> equal(res.status, FileStatus.compile_error) + tt |> success(find(res.message, "error[30001]") >= 0, "message carries the error line") + } + t |> run("a failed benchmark function drops its arms and names itself; the others survive") @(tt : T?) { + let res <- parsed([stats_line("good", "a", 10, 100l), stats_line("bad", "b", 10, 100l), "--- FAIL 'bad' in 'x.das'"], report_json(["good", "bad"], ["bad"]), 1) + tt |> equal(res.status, FileStatus.failed) + tt |> equal(length(res.samples), 1) + tt |> equal(res.samples[0].id, "good/a") + tt |> success(find(res.message, "bad") >= 0, "message names the failed function") + } + t |> run("a measured file that dies at shutdown keeps its numbers; one with no finished report is a failure") @(tt : T?) { + let shutdown <- parsed([stats_line("b", "a", 10, 100l)], report_json(["b"], []), 1) + tt |> equal(shutdown.status, FileStatus.ok) + tt |> success(find(shutdown.message, "shutdown") >= 0, "message says the shutdown failed") + let crashed <- parsed([stats_line("b", "a", 10, 100l), "FATAL: boom"], "", 139) + tt |> equal(crashed.status, FileStatus.exit_nonzero) + tt |> equal(crashed.message, "FATAL: boom") + } + t |> run("a timeout verdict survives whatever the output says; a clean exit with no samples is not ok") @(tt : T?) { + let killed <- parsed([stats_line("b", "a", 10, 100l)], report_json(["b"], []), 0, FileStatus.timeout) + tt |> equal(killed.status, FileStatus.timeout) + tt |> equal(length(killed.samples), 1) + tt |> equal(parsed(["nothing"], report_json([], []), 0).status, FileStatus.failed) + tt |> success(is_failure(FileStatus.exit_nonzero) && !is_failure(FileStatus.skipped), "is_failure separates ok and skipped from the rest") + } +} + +def private bench_file() : BenchFile { + return BenchFile(path = "x/y.das", id = "x/y", group = "x", timeout_seconds = 2) +} + +[test] +def test_run_bench_file(t : T?) { + with_temp_dir("bench_run_") $(root) { + t |> run("a skipped file never spawns and carries its reason") @(tt : T?) { + var f = bench_file() + f.skip = "too slow" + let res <- run_bench_file(get_host_binary(), root, f, "interp", RunLimits()) + tt |> equal(res.status, FileStatus.skipped) + tt |> equal(res.message, "too slow") + } + t |> run("a clean child is ok, and the jit lane's -jit reaches the host that runs it") @(tt : T?) { + fake_repo(root, "ok") + let jit <- run_bench_file(get_host_binary(), root, bench_file(), "jit", RunLimits(timeout_seconds = 120)) + tt |> equal(jit.status, FileStatus.ok, "{jit.message} {jit.log_tail}") + tt |> equal(length(jit.samples), 1) + tt |> equal(jit.samples[0].id, "b/arm") + tt |> equal(jit.samples[0].ns, 100.0lf) + tt |> success(find(jit.log_tail, "lane=jit") >= 0, "the child ran jitted: {jit.log_tail}") + let interp <- run_bench_file(get_host_binary(), root, bench_file(), "interp", RunLimits(timeout_seconds = 120)) + tt |> success(find(interp.log_tail, "lane=interp") >= 0, "the interp lane passes no -jit: {interp.log_tail}") + } + t |> run("a child that outlives timeout_seconds is killed; the arms it printed survive") @(tt : T?) { + fake_repo(root, "hang") + let res <- run_bench_file(get_host_binary(), root, bench_file(), "interp", RunLimits(timeout_seconds = 20)) + tt |> equal(res.status, FileStatus.timeout) + tt |> success(find(res.message, "timeout_seconds") >= 0, "message names the knob") + tt |> equal(length(res.samples), 1, "the arm it printed before the kill stands") + } + t |> run("an unrunnable binary reports that nothing was measured") @(tt : T?) { + let res <- run_bench_file("/nonexistent/daslang", root, bench_file(), "interp", RunLimits(timeout_seconds = 30)) + tt |> success(is_failure(res.status), "a missing binary is a failure: {res.status}") + tt |> equal(length(res.samples), 0) + } + } +} + +[test] +def test_probe_jit(t : T?) { + t |> run("a daslang that runs the probe reports the lane available; one that cannot run says so") @(tt : T?) { + tt |> equal(probe_jit(get_host_binary()), "", "the running host prints the marker") + let why = probe_jit("/nonexistent/daslang") + tt |> success(find(why, "/nonexistent/daslang") >= 0, "the reason names the binary it could not run: {why}") + } +} diff --git a/utils/benchctl/tests/test_bench_suite.das b/utils/benchctl/tests/test_bench_suite.das new file mode 100644 index 0000000000..874075a3a5 --- /dev/null +++ b/utils/benchctl/tests/test_bench_suite.das @@ -0,0 +1,101 @@ +options gen2 + +require dastest/testing_boost public +require ../bench_suite.das +require _test_common +require daslib/fio +require daslib/json_boost +require daslib/strings_boost +require strings + +def private touch(dir, rel : string) { + let p = path_join(dir, rel) + mkdir_rec(dir_name(p)) + fwrite(p, "options gen2\n") +} + +def private good_config() : string { + let files <- { "core/slow.das" => JV((skip = "too slow")), "core/big.das" => JV((timeout_seconds = 60)) } + let globs <- { "sort/**" => "not here" } + return write_json(JV((root = "benchmarks", exclude = JV(["**/tests/**", "**/_*.das"]), lanes = JV(["interp"]), + repeat = 2, timeout_seconds = 30, lane_excludes = JV((interp = JV(globs))), files = JV(files)))) +} + +def private load_cfg(dir : string) : SuiteConfig { + var error = "" + return <- load_suite_config(path_join(dir, "suite.json"), error) +} + +[test] +def test_load_suite_config(t : T?) { + with_temp_dir("bench_stand_") $(dir) { + t |> run("a well-formed config parses with its overrides") @(tt : T?) { + fwrite(path_join(dir, "suite.json"), good_config()) + var error = "" + let cfg <- load_suite_config(path_join(dir, "suite.json"), error) + tt |> equal(error, "") + tt |> equal(cfg.repeat, 2) + tt |> equal(cfg.timeout_seconds, 30) + tt |> equal(length(cfg.exclude), 2) + tt |> equal(length(cfg.files), 2) + cfg.files |> get("core/big.das") $(ov) { + tt |> equal(ov.timeout_seconds, 60) + } + } + t |> run("a missing, unparseable or out-of-range config fails closed, naming what is wrong") @(tt : T?) { + var error = "" + load_suite_config(path_join(dir, "missing.json"), error) + tt |> success(find(error, "missing.json") >= 0, "error names the path") + let path = path_join(dir, "bad.json") + var bodies <- [ + "not a config at all", + write_json(JV((lanes = JV(array())))), + write_json(JV((lanes = JV(["interp"]), repeat = 0))), + write_json(JV((lanes = JV(["vulkan"])))), + write_json(JV((lanes = JV(["interp"]), lane_excludes = JV((jit = JV(table())))))) + ] + for (body, word in bodies, ["parse", "no lanes", "positive", "vulkan", "does not run"]) { + fwrite(path, body) + load_suite_config(path, error) + tt |> success(find(error, word) >= 0, "{body} -> {error}") + } + fwrite(path, write_json(JV((lanes = JV(["interp", "jit"]))))) + load_suite_config(path, error) + tt |> equal(error, "", "the two real lanes are accepted") + } + } +} + +[test] +def test_discover_files(t : T?) { + with_temp_dir("bench_stand_") $(dir) { + fwrite(path_join(dir, "suite.json"), good_config()) + for (rel in ["core/hash/test02.das", "core/hash/_common.das", "core/slow.das", "core/big.das", "sql/tests/test_update.das", "sort/sort.das", "sort/README.md"]) { + touch(dir, "benchmarks/{rel}") + } + t |> run("helpers, test dirs and non-das files are excluded; the rest is sorted, with identity from the path and overrides applied") @(tt : T?) { + var err = "" + let files <- discover_files(dir, load_cfg(dir), err) + tt |> equal(err, "") + tt |> equal(join([for (f in files); f.path], " "), "core/big.das core/hash/test02.das core/slow.das sort/sort.das") + tt |> equal(files[1].group, "core/hash") + tt |> equal(files[1].id, "core/hash/test02") + tt |> equal(group_of("top.das"), "root") + tt |> equal(stem_path("top.das"), "top") + tt |> equal(files[2].skip, "too slow") + tt |> equal(files[2].timeout_seconds, 30) + tt |> equal(files[0].timeout_seconds, 60) + tt |> equal(files[1].skip, "") + tt |> equal(skip_reason(files[3], "interp"), "not here", "a lane exclude reaches the file it matches") + tt |> equal(skip_reason(files[1], "interp"), "") + } + t |> run("an override naming a missing file is reported") @(tt : T?) { + let cfg <- load_cfg(dir) + remove(path_join(dir, "benchmarks/core/slow.das")) + var err = "" + let files <- discover_files(dir, cfg, err) + tt |> equal(length(files), 3) + tt |> success(find(err, "core/slow.das") >= 0, "error names the stale override") + } + } +} diff --git a/utils/internal/bench-stand/README.md b/utils/internal/bench-stand/README.md new file mode 100644 index 0000000000..9adff33a89 --- /dev/null +++ b/utils/internal/bench-stand/README.md @@ -0,0 +1,104 @@ +# bench-stand - the nightly benchmark stand + +Every night one box checks out master, builds it, runs every `[benchmark]` file under +`benchmarks/` in the interpreter and JIT lanes, and publishes the result at +https://daslang.io/bench/ - a chart per benchmark arm over commits and a plain statement of what +failed and what moved. Review rules: `REVIEW.md`. + +## 1. Layout + +- The tool is `utils/benchctl` - `run` benchmarks the tree into one run record, `report` turns + every record into `data.json` with an exit code, and its database verbs (`insert`, `query`, + `compare`) answer ad-hoc questions about the same numbers. Every statistic it computes lives in + `benchctl/benchstat.das`, which depends on no storage. +- `site/` - the viewer: `index.html`, `app.js`, `style.css`. Static, no build step, no + dependencies; reads `data.json` and `status.json` beside it. +- `run_stand.sh` - one pass on a ref, the thing cron calls (section 3); `caddy.snippet` is the public route. + +## 2. Data model + +Benchmark identity: `#/` per lane, where the file id is the path under +`benchmarks/` without `.das` (`core/hash/test02#builtin_table/insert/600000`, lane `interp`). +The group is the file's directory; a new benchmark joins its group by living in the right folder. + +`runs/.json` - one per night, `RunRecord` in `bench_history.das`, never rewritten. The run +id is `-`. It carries the commit, the machine, the build (status, seconds, log +tail), the lane states, and one `FileResult` per file per lane: status, exit code, seconds, +message, the last 40 log lines, and the samples. A sample is one arm over the night's repeats: +`ns` = the minimum ns/op (the recorded value), `ns_median`, `spread` = (max - min) / min, and +per-op allocation medians. A record's `status` (`ok`, `build_failed`, `bench_failed`) is +re-derived from its parts on every read. + +File statuses: `ok`, `skipped` (listed in `suite.json` with a reason), `compile_error`, `failed` +(a `[benchmark]` function failed or panicked - its arms are dropped, the other functions' kept), +`timeout` (killed at `timeout_seconds`), `exit_nonzero` (the process died without a finished +dastest report), `spawn_failed`. A file whose report finished with its arms in hand stays `ok` on +a non-zero exit - only the shutdown went wrong - and the message says so; the JIT lane's probe +likewise believes the marker its program prints, not the exit code. + +`site/data.json` - `Dataset`: run summaries (with failures, skips, lane states), groups, series +(columnar: `runs` indexes `Dataset.runs`, `ns`, `spread`). +`site/status.json` is written by `run_stand.sh` at start and end (`running` / `finished`, run id, +exit), so a night whose build failed is still visible. + +### 2.1 Reading the numbers + +The stand publishes the series and leaves the judgement to a person. Nothing decides what counts +as a regression: `benchctl compare --old-commit --new-commit ` answers that on demand, with +a Welch test over the samples, for whichever two commits are actually in question. + +### 2.2 Exit codes + +`run`: 0 when every file ran ok, 1 otherwise. `report`: 0 when the latest night is ok, 1 when it +failed. `run --failed ""` benchmarks nothing: it writes the record of a night that could +not run the suite (the driver's way to record a failed build) and exits 1. + +## 3. The box + +`dasweb-1` (the daslang.io origin) runs `run_stand.sh master` from the `bench` user's cron under +`/srv/bench-stand`: `src/` (the clone), `runs/`, `site/` (what Caddy serves at `/bench/`), +`logs/`. The night builds Release (RelWithDebInfo arms the C++ allocation tracker, whose exit-time +report costs the run time) with the module set the benchmarks require, then `run`, then `report`; +`bin/` survives the checkout's clean, so a night whose build fails still renders a red night with +the previous binary. There is no CI runner and no ssh path into the box: the repository is public. + +One-time setup, as root: + +```sh +useradd -r -m -d /srv/bench-stand -s /bin/bash bench +# llvm-22-dev is required: the night builds with -DDAS_LLVM_DISABLED=OFF, and without it +# there is no jit lane. apt.llvm.org carries it for noble, as in extended_checks. +apt-get install -y git cmake ninja-build g++ ccache llvm-22-dev +su - bench -c 'git clone https://github.com/GaijinEntertainment/daScript src && mkdir -p runs site logs' +su - bench -c '/srv/bench-stand/src/utils/internal/bench-stand/run_stand.sh master' # first pass by hand +echo '0 5 * * * bench /srv/bench-stand/src/utils/internal/bench-stand/run_stand.sh master >> /srv/bench-stand/logs/cron.log 2>&1' > /etc/cron.d/bench-stand +``` + +then paste `caddy.snippet` into the `daslang.io` block of the Caddyfile and `systemctl reload caddy`. + +Arguments after the ref go to `benchctl run` - `run_stand.sh master --filter core/math/ --repeat 1` +is a slice of a night. `BENCH_STAND_HOME` moves the layout, `BENCH_STAND_BUILD=skip` reuses the +last build; both are for local dry runs. + +## 4. Configuration - `utils/benchctl/suite.json` + +`root`, `exclude` (globs over the path under root; helper modules `_*.das` and `**/tests/**` by +default), `lanes` (`interp`, `jit`), `lane_excludes` (per lane, glob -> the reason those files do +not run in it), `repeat` (dastest `--count`), `timeout_seconds` per file, and `files` - per-file +`timeout_seconds` or a `skip` with its reason. A skipped file is listed on the site every night, so a skip is visible debt; an override +naming a file that no longer exists is an error. + +## 5. Running locally + +```sh +bin/daslang utils/internal/bench-stand/main.das -- run --meta meta.json --out /tmp/stand/runs/n1.json --filter core/math/ --repeat 2 +bin/daslang utils/internal/bench-stand/main.das -- report --runs /tmp/stand/runs --out-data /tmp/stand/site/data.json +cp utils/internal/bench-stand/site/* /tmp/stand/site/ && ln -sfn ../runs /tmp/stand/site/runs +bin/daslang dastest/dastest.das -- --test utils/internal/bench-stand +``` + +`meta.json` is what `run_stand.sh` writes: `{"run_id", "started", "commit": {"sha", "date", +"subject", "author"}, "machine": {...}, "build": {"status", "seconds", "log_tail"}}`; any +subset parses. The whole pipeline runs locally too: `BENCH_STAND_HOME= +BENCH_STAND_BUILD=skip run_stand.sh --filter core/math/` with `/src` a clone +holding a built `bin/daslang`. diff --git a/utils/internal/bench-stand/REVIEW.md b/utils/internal/bench-stand/REVIEW.md new file mode 100644 index 0000000000..dfcb3e3de2 --- /dev/null +++ b/utils/internal/bench-stand/REVIEW.md @@ -0,0 +1,30 @@ +# bench-stand Code Review Checklist + +**Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: +`README.md`. + +**Never add a run-record field without saying in `README.md` section 2 what reads it - the +viewer, or a person opening the record.** A field nobody named is one nobody notices going wrong. + +**A diff that changes a run-record field keeps the new reader parsing a record written by the old +code, a missing field keeping its declared default.** Records already on the box are never +rewritten. + +**Never let a child's own output overwrite a `timeout` status in `run_bench_file` - a killed child +that printed a passing report is still killed.** Loosen `timeout_seconds` in `suite.json` instead. + +**A series a diff adds or recolors in `site/app.js` takes its color from its lane, never from its +position in the series list, and a chart drawing more than one lane shows a legend.** + +**Never read a benchmark's identity from anywhere but its path under `benchmarks/`** - the group +is the directory, the id is the path without `.das`. + +**Placement - one file, one line: a diff keeps each file inside its line, and a new file adds its +line here, with its tests, in the same change.** + +- `site/` - the viewer. Zero dependencies, zero build step. +- `run_stand.sh` - one pass on a ref: checkout, build, run, report, publish. Cron calls it. +- `caddy.snippet` - the public route, and the only place a route is written down. + +The tool this box runs is `utils/benchctl` - its modules, verbs and tests answer to +`utils/benchctl/REVIEW.md`. diff --git a/utils/internal/bench-stand/caddy.snippet b/utils/internal/bench-stand/caddy.snippet index 3f500be9d9..d30f764380 100644 --- a/utils/internal/bench-stand/caddy.snippet +++ b/utils/internal/bench-stand/caddy.snippet @@ -3,7 +3,7 @@ # deployed Caddyfile is edited to match it, never the other way round. # # Paste inside the `daslang.io { ... }` block, ahead of `root`/`file_server`. -# `bench-stand-deploy.sh caddy` does the splice, validates, and reloads. +# Splice it by hand, then `caddy validate --config /etc/caddy/Caddyfile` and reload. # The nightly benchmark stand: a static tree the `bench` user rewrites every # night (viewer, data.json, status.json, summary.md, runs/). No service behind diff --git a/utils/internal/bench-stand/run_stand.sh b/utils/internal/bench-stand/run_stand.sh new file mode 100755 index 0000000000..4f65159e54 --- /dev/null +++ b/utils/internal/bench-stand/run_stand.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# One pass of the stand on a ref - checkout, build, benchmark, report, publish. README.md. +set -euo pipefail +HOME_DIR=${BENCH_STAND_HOME:-/srv/bench-stand}; SRC=$HOME_DIR/src; SITE=$HOME_DIR/site; TOOL=utils/internal/bench-stand; BENCHCTL=utils/benchctl +REF=${1:-master}; shift || true +CMAKE_ARGS=(-G Ninja -DCMAKE_BUILD_TYPE=Release -DDAS_SQLITE_DISABLED=OFF -DDAS_PUGIXML_DISABLED=OFF -DDAS_LLVM_DISABLED=OFF -DDAS_GLFW_DISABLED=ON -DDAS_HV_DISABLED=ON) +TARGETS=(daslang dasModuleSQLITE dasModulePUGIXML dasModuleAudio dasModuleMinfft dasModuleTerminal dasModuleUnitTest dasModuleLLVM) + +now() { date -u +%Y-%m-%dT%H:%M:%SZ; } +jstr() { printf '%s' "$1" | tr -d '\r' | LC_ALL=C awk 'BEGIN{ORS=""} {gsub(/\\/,"\\\\"); gsub(/"/,"\\\""); gsub(/\t/,"\\t"); if (NR>1) printf "\\n"; printf "%s", $0}' | tr '\000-\037' ' '; } +status() { printf '{"state":"%s","run_id":"%s","started":"%s","finished":"%s","sha":"%s","exit":%s}\n' "$1" "$RUN_ID" "$STARTED" "$(now)" "$SHA" "${2:-null}" > "$SITE/status.json"; } + +mkdir -p "$HOME_DIR/runs" "$SITE" "$HOME_DIR/logs" +exec 9>"$HOME_DIR/lock"; flock -n 9 || { echo "another pass holds $HOME_DIR/lock"; exit 3; } +git -C "$SRC" fetch -q origin +SHA=$(git -C "$SRC" rev-parse --verify "origin/$REF^{commit}" 2>/dev/null || git -C "$SRC" rev-parse --verify "$REF^{commit}") +RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-${SHA:0:8}"; STARTED=$(now) +status running; trap 'status finished 1' ERR +git -C "$SRC" checkout -q --detach "$SHA" +git -C "$SRC" submodule update -q --init --recursive || true +git -C "$SRC" clean -fdxq --exclude=build --exclude=bin --exclude=lib --exclude=.jitted_scripts + +LOG=$HOME_DIR/logs/build-$RUN_ID.log; BUILD=ok; t0=$(date +%s) +if [ "${BENCH_STAND_BUILD:-}" = skip ]; then + : > "$LOG"; [ -x "$SRC/bin/daslang" ] || BUILD=failed +else + launcher=(); command -v ccache >/dev/null && launcher=(-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache) + ( cd "$SRC" && cmake --no-warn-unused-cli -B build "${CMAKE_ARGS[@]}" "${launcher[@]}" && cmake --build build --parallel "$(nproc)" --target "${TARGETS[@]}" ) > "$LOG" 2>&1 || BUILD=failed +fi +SECS=$(( $(date +%s) - t0 )) +META=$HOME_DIR/meta.json +printf '{"run_id":"%s","started":"%s","commit":{"sha":"%s","date":"%s","subject":"%s","author":"%s"},"machine":{"host":"%s","cores":%s},"build":{"status":"%s","seconds":%s,"log_tail":"%s"}}\n' \ + "$RUN_ID" "$STARTED" "$SHA" "$(TZ=UTC git -C "$SRC" log -1 --format=%cd --date=iso-strict-local | sed 's/+00:00$/Z/')" \ + "$(jstr "$(git -C "$SRC" log -1 --format=%s)")" "$(jstr "$(git -C "$SRC" log -1 --format=%an)")" "$(jstr "$(hostname)")" "$(nproc)" \ + "$BUILD" "$SECS" "$(jstr "$(tail -n 40 "$LOG" | cut -c1-400)")" > "$META" + +[ -x "$SRC/bin/daslang" ] || { echo "no binary: the build failed and none survives from an earlier pass"; trap - ERR; status finished 1; exit 1; } +[ "$BUILD" = ok ] || set -- "$@" --failed "the build failed after $SECS s - $LOG on the box has the whole log" +rc=0; report_rc=0 +( cd "$SRC" && bin/daslang "$BENCHCTL/main.das" -- run --root . --meta "$META" --out "$HOME_DIR/runs/$RUN_ID.json" "$@" ) || rc=$? +"$SRC/bin/daslang" "$SRC/$BENCHCTL/main.das" -- report --runs "$HOME_DIR/runs" --out-data "$SITE/data.json.tmp" --repo-url https://github.com/GaijinEntertainment/daScript || report_rc=$? +[ -s "$SITE/data.json.tmp" ] && mv "$SITE/data.json.tmp" "$SITE/data.json" +cp "$SRC/$TOOL"/site/{index.html,app.js,style.css} "$SITE/"; ln -sfn ../runs "$SITE/runs" +[ $rc -eq 0 ] && rc=$report_rc +trap - ERR; status finished "$rc"; exit "$rc"