diff --git a/CMakeLists.txt b/CMakeLists.txt index b78cecc45a..5f792a3cc2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2074,6 +2074,7 @@ install(FILES ${DAS_DASCOV_TEST_FILES} DESTINATION utils/dascov/tests) file(GLOB DAS_BENCHCTL_FILES ${PROJECT_SOURCE_DIR}/utils/benchctl/*.das) install(FILES ${DAS_BENCHCTL_FILES} DESTINATION utils/benchctl) install(FILES ${PROJECT_SOURCE_DIR}/utils/benchctl/README.md DESTINATION utils/benchctl) +install(FILES ${PROJECT_SOURCE_DIR}/utils/benchctl/suite.json DESTINATION utils/benchctl) # Install lint (unified linter) + its rule-coverage suite, so a shipped SDK can # self-check the linter the same way run_utils_tests does in-tree. diff --git a/benchmarks/README.md b/benchmarks/README.md index 1f8eb160ab..4fbd79ef77 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -4,6 +4,12 @@ Every `.das` benchmark file in this directory tree is listed below, grouped by subdirectory. +## compile/ + +| File | Description | +|---|---| +| `utils.das` | Compile-only wall time of every tool under `utils/` the nightly build can compile, one `[benchmark]` per tool, each a child `daslang -compile-only -no-module-cache ` - interp lane only, since the child is the same binary whatever lane the parent runs in | + ## terminal/ | File | Description | diff --git a/benchmarks/compile/utils.das b/benchmarks/compile/utils.das new file mode 100644 index 0000000000..94d882dbbe --- /dev/null +++ b/benchmarks/compile/utils.das @@ -0,0 +1,149 @@ +options gen2 + +require dastest/testing_boost +require daslib/clargs +require daslib/fio +require daslib/strings_boost +require strings + +def private compile_tool(b : B?; entry : string) { + let bin = get_host_binary() + let path = path_join(path_join(get_das_root(), "utils"), entry) + let cmd = "\"{bin}\" -compile-only -no-module-cache \"{path}\" 2>&1" + var rc = 0 + var first_error = "" + b |> run(entry) { + first_error = "" + rc = unsafe(popen_timeout(cmd, 600.0) $(f) { + if (f != null) { + while (!feof(f)) { + let ln = fgets(f) + if (empty(first_error) && find(ln, "error[") >= 0) { + first_error = strip(ln) + } + } + } + }) + } + b |> equal(rc, 0, "{entry}: daslang -compile-only exited with {rc}{empty(first_error) ? "" : " - " + first_error}") +} + +[benchmark] +def compile_aot(b : B?) { + compile_tool(b, "aot/main.das") +} + +[benchmark] +def compile_benchctl(b : B?) { + compile_tool(b, "benchctl/main.das") +} + +[benchmark] +def compile_dascov(b : B?) { + compile_tool(b, "dascov/main.das") +} + +[benchmark] +def compile_daspkg(b : B?) { + compile_tool(b, "daspkg/main.das") +} + +[benchmark] +def compile_detect_dupe(b : B?) { + compile_tool(b, "detect-dupe/main.das") +} + +[benchmark] +def compile_fix_lint_errors(b : B?) { + compile_tool(b, "fix-lint-errors/main.das") +} + +[benchmark] +def compile_gen1_to_gen2(b : B?) { + compile_tool(b, "gen1-to-gen2/main.das") +} + +[benchmark] +def compile_lint(b : B?) { + compile_tool(b, "lint/main.das") +} + +[benchmark] +def compile_watchdog(b : B?) { + compile_tool(b, "watchdog/main.das") +} + +[benchmark] +def compile_das_fmt(b : B?) { + compile_tool(b, "das-fmt/dasfmt.das") +} + +[benchmark] +def compile_arch_extract(b : B?) { + compile_tool(b, "internal/arch-extract/main.das") +} + +[benchmark] +def compile_ast_fuzz(b : B?) { + compile_tool(b, "internal/ast-fuzz/main.das") +} + +[benchmark] +def compile_dasweb_verify(b : B?) { + compile_tool(b, "internal/dasweb-verify/main.das") +} + +[benchmark] +def compile_doc_verify(b : B?) { + compile_tool(b, "internal/doc-verify/main.das") +} + +[benchmark] +def compile_flatten_fuzz(b : B?) { + compile_tool(b, "internal/flatten-fuzz/main.das") +} + +[benchmark] +def compile_hygiene(b : B?) { + compile_tool(b, "internal/hygiene/main.das") +} + +[benchmark] +def compile_jit(b : B?) { + compile_tool(b, "internal/jit/main.das") +} + +[benchmark] +def compile_lineinfo_audit(b : B?) { + compile_tool(b, "internal/lineinfo-audit/main.das") +} + +[benchmark] +def compile_make_pr(b : B?) { + compile_tool(b, "internal/make-pr/main.das") +} + +[benchmark] +def compile_pr_babysit(b : B?) { + compile_tool(b, "internal/pr-babysit/main.das") +} + +[benchmark] +def compile_preflight(b : B?) { + compile_tool(b, "internal/preflight/main.das") +} + +[benchmark] +def compile_requirefix(b : B?) { + compile_tool(b, "internal/requirefix/main.das") +} + +[benchmark] +def compile_review_md(b : B?) { + compile_tool(b, "internal/review-md/main.das") +} + +[benchmark] +def compile_test_release(b : B?) { + compile_tool(b, "internal/test-release/main.das") +} diff --git a/benchmarks/decs/bench_from_decs_count.das b/benchmarks/decs/bench_from_decs_count.das index 978d96ea50..3157206b62 100644 --- a/benchmarks/decs/bench_from_decs_count.das +++ b/benchmarks/decs/bench_from_decs_count.das @@ -20,7 +20,7 @@ struct BenchCountRow { def fixture(n : int) { restart() - create_entities(n) $(eid : EntityId; i : int; var cmp : ComponentMap) { + create_entities(n) $(_eid : EntityId; i : int; var cmp : ComponentMap) { apply_decs_template(cmp, BenchCountRow(val = i)) } } @@ -32,7 +32,7 @@ def from_decs_count_m1_hand(b : B?) { b |> run("m1_hand_arch_size/{N}", N) { var erq : EcsRequest erq.req |> push("bench_val") - var total = 0 + var total = 0l for_each_archetype(erq) $(arch : Archetype) { total += arch.size } diff --git a/daslib/clargs.das b/daslib/clargs.das index 996ea09738..02d3af43dc 100644 --- a/daslib/clargs.das +++ b/daslib/clargs.das @@ -74,6 +74,12 @@ def public get_user_args() : array { } } +//! The running host binary as an absolute path - `argv[0]` resolved - for a program that spawns +//! the same daslang it runs under. +def public get_host_binary() : string { + return get_full_file_name(get_command_line_arguments()[0]) +} + def clargs_flag_name(field_name : string) : string { return "--" + replace_multiple(field_name, [(text="_", replacement="-")]) } diff --git a/skills/daslang/references/cli-and-config.md b/skills/daslang/references/cli-and-config.md index 5b4b9d2253..dacaaf6e0b 100644 --- a/skills/daslang/references/cli-and-config.md +++ b/skills/daslang/references/cli-and-config.md @@ -30,7 +30,9 @@ def main : int { **Don't pick an argv accessor.** `parse_args` pulls argv through `get_user_args()`: `argv[1..]` for a standalone `-exe` binary, the post-`--` slice under the interpreter or the JIT. -`get_program_args()` / `get_cli_arguments()` force one slice regardless of host. +`get_program_args()` / `get_cli_arguments()` force one slice regardless of host. A program that +spawns the daslang it runs under takes the path from `get_host_binary()` - `argv[0]` resolved to an +absolute path - never from `get_command_line_arguments()` itself. Field types: `string`, `int`, `float`, `bool`, an enum (clargs validates the value), `array` for a repeatable flag. The long name is the field name with underscores as hyphens; clargs adds the diff --git a/skills/writing_benchmarks.md b/skills/writing_benchmarks.md index 98a9c227a5..dd5cf4931e 100644 --- a/skills/writing_benchmarks.md +++ b/skills/writing_benchmarks.md @@ -14,7 +14,9 @@ bin/daslang -jit dastest/dastest.das -- --bench --test path/to/directory/ - `-jit` goes **before** `dastest.das`: it puts dastest itself in JIT mode, which is what gets the benchmark code JIT-compiled. Use it for any performance number you intend to believe -- `--bench-names name1,name2` - run only those benchmark functions +- `--bench-names prefix` - run only the benchmark functions whose name starts with `prefix`; + repeat the flag for several (`--bench-names a --bench-names b`) - a comma-joined list is one + prefix that matches nothing Dropping `--bench` turns the same command into a fast compile check: it reports 0 tests but surfaces every compile error. diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 19755f8562..a06b456bab 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -232,6 +232,7 @@ SET(DAS_UTILS_TO_TEST internal/hygiene internal/preflight/tests internal/ast-fuzz + benchctl/tests internal/requirefix internal/test-release daspkg/test_daspkg.das diff --git a/utils/benchctl/REVIEW.md b/utils/benchctl/REVIEW.md index d7bab13f5f..8c0bd15bed 100644 --- a/utils/benchctl/REVIEW.md +++ b/utils/benchctl/REVIEW.md @@ -2,3 +2,45 @@ **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: `README.md`. + +**Never put a `[test]` file outside `tests/`, and never let a test touch the filesystem outside a +`temp_directory`-rooted path or leave behind what it creates.** + +**A module file orders its top level types, then `private` helpers, then its public functions, so +the tail of the file is the module's whole API.** + +**Never add a run-record field without saying in `README.md` what reads it - the viewer, or a +person opening the record.** A field nobody named is one nobody notices going wrong. + +**A diff that changes a run-record field keeps the new reader parsing a record written by the old +code, a missing field keeping its declared default.** Records already on the box are never +rewritten. + +**Never let a child's own output overwrite a `timeout` status in `run_bench_file` - a killed child +that printed a passing report is still killed.** Loosen `timeout_seconds` in `suite.json` instead. + +**Never read a benchmark's identity from anywhere but its path under `benchmarks/`** - the group +is the directory, the id is the path without `.das`. + +**Statistics live in `benchstat.das` and nowhere else** - it depends on no storage, so a second +median or spread helper anywhere in this folder is a defect. + +**Placement - one file, one line: a diff keeps each file inside its line, and a new file adds its +line here, with its tests, in the same change.** + +- `main.das` - argv, subcommand dispatch, exit codes. +- `bench_args.das` - the database verbs' flags. +- `bench_table.das` - the stored row and its mapping to the shape the statistics read. +- `benchstat.das` - every statistic: median, spread, outlier filtering, geomean, Welch. No storage. +- `bench_suite.das` - `suite.json` and file discovery. No processes. +- `suite.json` - what is benchmarked, in which lanes, with which limits. +- `bench_runner.das` - one file in one lane: spawn, timeout, output to samples and verdict. +- `_jit_probe.das` - the program `probe_jit` runs to prove the jit lane works. +- `bench_history.das` - run records to dataset. No processes, no argv, no statistics. +- `bench_stand.das` - the `run` and `report` verbs the nightly box drives. +- `table_fmt.das`, `utils.das` - output formatting and small shared helpers. +- `tests/test_bench_suite.das`, `tests/test_bench_runner.das`, `tests/test_bench_history.das` - + the module suites; `tests/test_bench_cli.das` - the verbs, spawned; `tests/_test_common.das` - + the fixtures they share; `tests/_fake_dastest.das` - the stand-in dastest a spawn test measures + against. + diff --git a/utils/benchctl/_jit_probe.das b/utils/benchctl/_jit_probe.das new file mode 100644 index 0000000000..313558b4a2 --- /dev/null +++ b/utils/benchctl/_jit_probe.das @@ -0,0 +1,6 @@ +options gen2 + +[export] +def main() { + print("jit-probe-ok\n") +} diff --git a/utils/benchctl/bench_history.das b/utils/benchctl/bench_history.das new file mode 100644 index 0000000000..9694df8e0a --- /dev/null +++ b/utils/benchctl/bench_history.das @@ -0,0 +1,189 @@ +options gen2 +options indenting = 4 + +module bench_history public + +require bench_runner +require daslib/fio +require daslib/json_boost + +enum RunStatus { + ok + build_failed + bench_failed +} + +enum StandExit { + ok = 0 + failed = 1 +} + +// JSON: run record, written per night, read by `report` - see README.md section 2 +struct CommitInfo { + sha, date, subject, author : string +} + +// JSON: run record +struct MachineInfo { + host : string + cores : int +} + +// JSON: run record +struct BuildInfo { + status : string = "ok" + seconds : double + log_tail : string +} + +// JSON: run record +struct RunRecord { + schema : int = 1 + run_id, started, finished : string + seconds : double + commit : CommitInfo + machine : MachineInfo + build : BuildInfo = BuildInfo() + lanes : table + status : RunStatus + files : array +} + +// JSON: run record +struct Failure { + path, lane, status, message : string +} + +// JSON: data.json, read by site/app.js +struct RunSummary { + id, started : string + status : RunStatus + seconds : double + commit : CommitInfo + build : BuildInfo = BuildInfo() + machine : MachineInfo + lanes : table + failures, skipped : array + files_ok : int +} + +// JSON: data.json, read by site/app.js +struct Series { + id, group, file, lane : string + runs : array + ns, spread : array +} + +// JSON: data.json, read by site/app.js +struct Dataset { + schema : int = 1 + generated, repo_url : string + latest : int + runs : array + groups : array + series : array +} + +def derive_status(rec : RunRecord) : RunStatus { + return RunStatus.build_failed if (rec.build.status != "ok") + var measured = 0 + for (f in rec.files) { + return RunStatus.bench_failed if (is_failure(f.status)) + measured++ if (f.status == FileStatus.ok) + } + return measured == 0 ? RunStatus.bench_failed : RunStatus.ok +} + +def mark_failed_run(var rec : RunRecord&; finished, reason : string) { + rec.finished = finished + rec.files |> clear() + if (rec.build.status == "ok") { + rec.files |> emplace(FileResult(path = "run_stand.sh", id = "bench-stand", group = "stand", lane = "driver", + status = FileStatus.exit_nonzero, exit_code = 1, message = empty(reason) ? "the driver could not run the suite" : reason)) + } + rec.status = derive_status(rec) +} + +def load_runs(runs_dir : string; var errors : array) : array { + var runs : array + dir(runs_dir) $(name) { + return if (extension(name) != ".json") + var rec = RunRecord() + if (!sscan_json(fread(path_join(runs_dir, name)), rec) || empty(rec.run_id)) { + errors |> push("run record does not parse or has no run_id: {path_join(runs_dir, name)}") + return + } + rec.status = derive_status(rec) + runs |> emplace(rec) + } + sort(runs) $(a, b) { + return a.commit.date != b.commit.date ? a.commit.date < b.commit.date : a.started < b.started + } + return <- runs +} + +def latest_run_index(runs : array) : int { + var best = -1 + for (i in range(length(runs))) { + if (best < 0 || runs[i].started >= runs[best].started) { + best = i + } + } + return best +} + +def private summarize_run(rec : RunRecord) : RunSummary { + var s <- RunSummary(id = rec.run_id, started = rec.started, status = rec.status, seconds = rec.seconds, + commit = rec.commit, build = rec.build, machine = rec.machine) + s.lanes := rec.lanes + for (f in rec.files) { + if (f.status == FileStatus.ok) { + s.files_ok++ + } else { + let row = Failure(path = f.path, lane = f.lane, status = "{f.status}", message = f.message) + (f.status == FileStatus.skipped ? s.skipped : s.failures) |> push(row) + } + } + return <- s +} + +def private build_series(runs : array; var groups : table) : array { + var index_of : table + var out : array + for (ri, rec in range(length(runs)), runs) { + for (f in rec.files) { + groups |> insert(f.group) if (!empty(f.samples)) + for (smp in f.samples) { + let key = "{f.lane}\t{f.id}#{smp.id}" + var si = index_of?[key] ?? -1 + if (si < 0) { + si = length(out) + index_of[key] = si + out |> emplace(Series(id = "{f.id}#{smp.id}", group = f.group, file = f.id, lane = f.lane)) + } + continue if (!empty(out[si].runs) && out[si].runs[length(out[si].runs) - 1] == ri) + out[si].runs |> push(ri) + out[si].ns |> push(smp.ns) + out[si].spread |> push(smp.spread) + } + } + } + sort(out) $(a, b) { + return a.lane != b.lane ? a.lane < b.lane : a.id < b.id + } + return <- out +} + +def build_dataset(runs : array; generated, repo_url : string) : Dataset { + var ds <- Dataset(generated = generated, repo_url = repo_url, latest = latest_run_index(runs)) + var groups : table + ds.series <- build_series(runs, groups) + ds.groups <- [for (g in keys(groups)); g] + sort(ds.groups) + ds.runs <- [for (rec in runs); summarize_run(rec)] + return <- ds +} + +def report_verdict(ds : Dataset) : StandExit { + return ds.latest < 0 || ds.runs[ds.latest].status != RunStatus.ok ? StandExit.failed : StandExit.ok +} diff --git a/utils/benchctl/bench_runner.das b/utils/benchctl/bench_runner.das new file mode 100644 index 0000000000..9eda0bc9c9 --- /dev/null +++ b/utils/benchctl/bench_runner.das @@ -0,0 +1,223 @@ +options gen2 +options indenting = 4 + +module bench_runner public + +require bench_suite +require benchstat +require daslib/fio +require daslib/json_boost +require daslib/strings_boost +require strings + +enum FileStatus { + ok + skipped + spawn_failed + compile_error + failed + timeout + exit_nonzero +} + +// JSON: run record +struct Sample { + id : string + ns, ns_median, spread : double + runs : int + bytes, allocs, string_bytes, string_allocs : int64 +} + +// JSON: run record +struct FileResult { + path, id, group, lane : string + status : FileStatus + exit_code : int + seconds : double + message, log_tail : string + samples : array +} + +struct RunLimits { + timeout_seconds : int = 900 + repeat : int = 3 +} + +// JSON: dastest --json-file report +struct ReportTest { + name : string + passed : bool +} + +// JSON: dastest --json-file report +struct DastestReport { + success : bool + tests : array +} + +let TAIL_LINES = 40 + +struct private OutputScan { + by_arm : table> + arm_order, tail : array + compile_error, failed_to_compile, fatal : string + failed_fns : table +} + +def private note(var slot : string&; line : string; hit : bool) { + if (hit && empty(slot)) { + slot = strip(line) + } +} + +def private scan_line(var scan : OutputScan; line : string) { + var st = BenchmarkRunStats() + if (starts_with(line, "\{") && sscan_json(line, st) && st.n > 0) { + let key = "{st.name}/{st.sub_name}" + scan.arm_order |> push(key) if (!key_exists(scan.by_arm, key)) + scan.by_arm[key] |> push(st) + return + } + scan.tail |> erase(0) if (length(scan.tail) >= TAIL_LINES) + scan.tail |> push(length(line) > 400 ? slice(line, 0, 400) : line) + note(scan.compile_error, line, find(line, "error[") >= 0 && find(line, "]: ") >= 0) + note(scan.failed_to_compile, line, find(line, "Failed to compile") >= 0) + note(scan.fatal, line, starts_with(line, "FATAL")) + let at = find(line, "--- FAIL '") + return if (at < 0) + let name_end = find(line, "'", at + 10) + scan.failed_fns |> insert(slice(line, at + 10, name_end)) if (name_end > at + 10) +} + +def private aggregate_arm(key : string; stats : array) : Sample { + var ns <- [for (st in stats); double(st.time_ns) / double(st.n)] + var bytes <- [for (st in stats); int64(st.heap_bytes) / int64(st.n)] + var allocs <- [for (st in stats); int64(st.allocs) / int64(st.n)] + var sbytes <- [for (st in stats); int64(st.string_heap_bytes) / int64(st.n)] + var sallocs <- [for (st in stats); int64(st.string_allocs) / int64(st.n)] + sort(ns) + let lo = ns[0] + let hi = ns[length(ns) - 1] + return Sample(id = key, ns = lo, ns_median = median(ns), spread = lo > 0.0lf ? (hi - lo) / lo : 0.0lf, runs = length(ns), + bytes = median_i64(bytes), allocs = median_i64(allocs), string_bytes = median_i64(sbytes), string_allocs = median_i64(sallocs)) +} + +def is_failure(st : FileStatus) : bool { + return st != FileStatus.ok && st != FileStatus.skipped +} + +def classify_run_output(lines : array; report_text : string; exit_code : int; var res : FileResult&) { + var scan <- OutputScan() + for (line in lines) { + scan_line(scan, line) + } + var report : DastestReport + let report_ok = !empty(report_text) && sscan_json(report_text, report) + for (test in report.tests) { + scan.failed_fns |> insert(test.name) if (!test.passed) + } + res.log_tail = join(scan.tail, "\n") + res.exit_code = exit_code + for (key in scan.arm_order) { + res.samples |> emplace(aggregate_arm(key, scan.by_arm[key])) if (!key_exists(scan.failed_fns, slice(key, 0, find(key, "/")))) + } + return if (res.status == FileStatus.timeout) + if (!empty(scan.failed_to_compile)) { + res.status = FileStatus.compile_error + res.message = !empty(scan.compile_error) ? scan.compile_error : scan.failed_to_compile + } elif (!empty(scan.failed_fns)) { + var names <- [for (n in keys(scan.failed_fns)); n] + sort(names) + res.status = FileStatus.failed + res.message = "benchmark function(s) failed: {join(names, ", ")}" + } elif (exit_code != 0 && report_ok && !empty(res.samples)) { + res.status = FileStatus.ok + res.message = "measured, then exited with code {exit_code} at shutdown{empty(scan.fatal) ? "" : ": " + scan.fatal}" + } elif (exit_code != 0 || !report_ok) { + res.status = FileStatus.exit_nonzero + res.message = !empty(scan.fatal) ? scan.fatal : exit_code != 0 ? "process exited with code {exit_code} without a finished dastest report" : "dastest wrote no report file - the process ended before the suite finished" + } elif (empty(res.samples)) { + res.status = FileStatus.failed + res.message = "the file ran but produced no benchmark samples" + } else { + res.status = FileStatus.ok + } +} + +def run_bench_file(bin, repo_root : string; f : BenchFile; lane : string; limits : RunLimits) : FileResult { + var res <- FileResult(path = f.path, id = f.id, group = f.group, lane = lane) + res.message = skip_reason(f, lane) + if (!empty(res.message)) { + res.status = FileStatus.skipped + return <- res + } + var err = "" + let report_path = create_temp_file("bench_stand_", ".json", err) + if (empty(report_path)) { + res.status = FileStatus.spawn_failed + res.message = "cannot create the dastest report file: {err}" + return <- res + } + remove(report_path) + var argv <- [bin] + argv |> push("-jit") if (lane == "jit") + argv |> push_from([to_generic_path(path_join(repo_root, "dastest/dastest.das")), "--", "--bench", "--bench-format", "json", "--json-file", report_path, + "--count", "{limits.repeat}", "--test", to_generic_path(path_join(path_join(repo_root, "benchmarks"), f.path))]) + var lines : array + var exit_code = process_running + let t0 = ref_time_ticks() + let noenv : array + with_process(argv, repo_root, noenv) $(var p) { + unsafe { + if (process_pid(p) <= 0) { + res.status = FileStatus.spawn_failed + res.message = "cannot spawn {bin}" + return + } + while (exit_code == process_running) { + process_drain(p) $(line) { + lines |> push(line) + } + exit_code = process_poll(p) + if (exit_code == process_running && double(get_time_usec(t0)) / 1000000.0lf > double(limits.timeout_seconds)) { + res.status = FileStatus.timeout + res.message = "killed after {limits.timeout_seconds} s (timeout_seconds)" + process_kill(p) + exit_code = process_wait(p, 10.0) + } + sleep(100u) if (exit_code == process_running) + } + process_drain(p) $(line) { + lines |> push(line) + } + } + } + res.seconds = double(get_time_usec(t0)) / 1000000.0lf + return <- res if (res.status == FileStatus.spawn_failed) + let report_text = fread(report_path) + remove(report_path) + classify_run_output(lines, report_text, exit_code, res) + return <- res +} + +def probe_jit(daslang_bin : string) : string { + var err = "" + let probe = create_temp_file("bench_stand_jit_", ".das", err) + return "cannot create the probe file: {err}" if (empty(probe)) + let source = fread(path_join(get_das_root(), "utils/benchctl/_jit_probe.das")) + return "cannot read utils/benchctl/_jit_probe.das" if (empty(source)) + fwrite(probe, source) + var out = "" + let code = run_and_capture([daslang_bin, "-jit", probe], out, 120.0) + remove(probe) + return "" if (find(out, "jit-probe-ok") >= 0) + var said = "" + for (raw in split(out, "\n")) { + let line = strip(raw) + continue if (empty(line)) + said = line + break if (starts_with(line, "FATAL") || starts_with(line, "error") || find(line, "[E]") >= 0) + } + return "daslang -jit produced no jit-compiled run (exit {code}) and printed nothing - is {daslang_bin} runnable?" if (empty(said)) + return "daslang -jit produced no jit-compiled run (exit {code}): {said}" +} diff --git a/utils/benchctl/bench_stand.das b/utils/benchctl/bench_stand.das new file mode 100644 index 0000000000..7f09fb9354 --- /dev/null +++ b/utils/benchctl/bench_stand.das @@ -0,0 +1,137 @@ +options gen2 +options indenting = 4 + +module bench_stand public + +require bench_suite +require bench_runner +require bench_history +require daslib/clargs +require daslib/fio +require daslib/json_boost +require daslib/strings_boost +require strings + +[CommandLineArgs] +struct Config { + @clarg_positional + @clarg_doc = "run | report" + verb : string + @clarg_doc = "Repo root (default: .)" + root : string + @clarg_doc = "suite.json to use (default: the one beside this tool)" + suite : string + @clarg_doc = "run: daslang binary to measure with (default: bin/daslang under --root)" + bin : string + @clarg_doc = "run: JSON with run_id, started, commit, machine, build - written by run_stand.sh" + meta : string + @clarg_doc = "run: where to write the run record" + out : string + @clarg_doc = "run: only files whose path contains this substring" + filter : string + @clarg_doc = "run: comma-separated lanes to run (default: suite.json lanes)" + lanes : string + @clarg_doc = "run: benchmark nothing - write the record of a night that could not run the suite, with this reason, and exit 1" + failed : string + @clarg_doc = "run: repeats per file (default: suite.json repeat)" + repeat : int + @clarg_doc = "report: directory of run records" + runs : string + @clarg_doc = "report: where to write data.json" + out_data : string + @clarg_doc = "report: repository URL for commit links" + repo_url : string + @clarg_short = "?" + @clarg_name = "show-help" + @clarg_doc = "Show this help and exit" + help : bool +} + +def private fail(msg : string) : StandExit { + to_log(LOG_ERROR, "bench-stand: {msg}\n") + return StandExit.failed +} + +def private write_record(path : string; value) : bool { + return true if (fwrite(path, sprint_json(value, true))) + fail("cannot write {path}") + return false +} + +def private measure(var rec : RunRecord&; files : array; lanes : array; bin, root, filter : string; repeat : int) : int { + var limits = RunLimits(repeat = repeat) + var failed = 0 + for (f in files) { + continue if (!empty(filter) && find(f.path, filter) < 0) + for (lane in lanes) { + continue if (rec.lanes[lane] != "ok") + limits.timeout_seconds = f.timeout_seconds + var res <- run_bench_file(bin, root, f, lane, limits) + failed++ if (is_failure(res.status)) + to_log(is_failure(res.status) ? LOG_ERROR : LOG_INFO, "{f.path} {lane}: {res.status} {res.seconds:.1f}s {length(res.samples)} arms{empty(res.message) ? "" : " - " + res.message}\n") + rec.files |> emplace(res) + } + } + return failed +} + +def private verb_run(cfg : Config; suite : SuiteConfig) : StandExit { + return fail("run: --meta and --out are required") if (empty(cfg.meta) || empty(cfg.out)) + let root = empty(cfg.root) ? "." : cfg.root + let bin = empty(cfg.bin) ? path_join(root, "bin/daslang") : cfg.bin + var rec = RunRecord() + return fail("run: --meta {cfg.meta} does not parse or has no run_id") if (!sscan_json(fread(cfg.meta), rec) || empty(rec.run_id)) + if (!empty(cfg.failed)) { + mark_failed_run(rec, "{iso8601_now()}", cfg.failed) + write_record(cfg.out, rec) + return fail("run: {rec.status} - {cfg.failed}; record {cfg.out}") + } + var error = "" + let files <- discover_files(root, suite, error) + return fail("run: {error}") if (!empty(error)) + let lanes <- empty(cfg.lanes) ? clone_to_move(suite.lanes) : split(cfg.lanes, ",") + for (lane in lanes) { + let why = lane == "jit" ? probe_jit(bin) : "" + rec.lanes[lane] = empty(why) ? "ok" : why + to_log(LOG_INFO, "lane {lane}: {rec.lanes[lane]}\n") + } + let t0 = ref_time_ticks() + let failed = measure(rec, files, lanes, bin, root, cfg.filter, cfg.repeat > 0 ? cfg.repeat : suite.repeat) + rec.seconds = double(get_time_usec(t0)) / 1000000.0lf + rec.finished = "{iso8601_now()}" + rec.status = derive_status(rec) + return StandExit.failed if (!write_record(cfg.out, rec)) + to_log(rec.status == RunStatus.ok ? LOG_INFO : LOG_ERROR, "bench-stand run: {rec.status} - {length(rec.files)} file runs, {failed} failed, {rec.seconds:.0f} s; record {cfg.out}\n") + return rec.status == RunStatus.ok ? StandExit.ok : StandExit.failed +} + +def private verb_report(cfg : Config) : StandExit { + return fail("report: --runs and --out-data are required") if (empty(cfg.runs) || empty(cfg.out_data)) + var errors : array + let runs <- load_runs(cfg.runs, errors) + for (e in errors) { + to_log(LOG_WARNING, "bench-stand report: {e}\n") + } + let ds <- build_dataset(runs, "{iso8601_now()}", cfg.repo_url) + return fail("cannot write {cfg.out_data}") if (!fwrite(cfg.out_data, sprint_json(ds, false))) + to_log(LOG_INFO, "bench-stand report: {length(runs)} runs, {length(ds.series)} series -> {cfg.out_data}\n") + return report_verdict(ds) +} + +def stand_main() : int { + var r <- parse_args(type) + if (r |> is_err) { + to_log(LOG_ERROR, "error: {r |> unwrap_err}\n") + print_help(get_command_info(type), "bench-stand") + return 1 + } + let cfg <- r |> move_unwrap + if (cfg.help || (cfg.verb != "run" && cfg.verb != "report")) { + print_help(get_command_info(type), "bench-stand") + return int(cfg.help ? StandExit.ok : StandExit.failed) + } + var error = "" + let suite <- load_suite_config(empty(cfg.suite) ? path_join(get_das_root(), "utils/benchctl/suite.json") : cfg.suite, error) + return int(fail(error)) if (!empty(error)) + return int(cfg.verb == "run" ? verb_run(cfg, suite) : verb_report(cfg)) +} diff --git a/utils/benchctl/bench_suite.das b/utils/benchctl/bench_suite.das new file mode 100644 index 0000000000..4e80151336 --- /dev/null +++ b/utils/benchctl/bench_suite.das @@ -0,0 +1,115 @@ +options gen2 +options indenting = 4 + +module bench_suite public + +require daslib/fio +require daslib/json_boost + +// JSON: suite.json +struct FileOverride { + @optional skip : string + @optional skip_lanes : table + @optional timeout_seconds : int +} + +// JSON: suite.json +struct SuiteConfig { + root : string = "benchmarks" + exclude, lanes : array + repeat : int = 3 + timeout_seconds : int = 900 + files : table + lane_excludes : table> +} + +struct BenchFile { + path, id, group : string + timeout_seconds : int + skip : string + skip_lanes : table +} + +def skip_reason(f : BenchFile; lane : string) : string { + return !empty(f.skip) ? f.skip : f.skip_lanes?[lane] ?? "" +} + +def load_suite_config(path : string; var error : string&) : SuiteConfig { + error = "" + var cfg = SuiteConfig() + if (!stat(path).is_reg) { + error = "suite config not found: {path}" + } elif (!sscan_json(fread(path), cfg)) { + error = "suite config does not parse as JSON: {path}" + } elif (empty(cfg.lanes)) { + error = "suite config lists no lanes: {path}" + } elif (cfg.repeat <= 0 || cfg.timeout_seconds <= 0) { + error = "suite config needs positive repeat and timeout_seconds: {path}" + } + for (lane in keys(cfg.lane_excludes)) { + if (find_index(cfg.lanes, lane) < 0) { + error = "suite config excludes files from lane \"{lane}\", which it does not run: {path}" + } + } + for (lane in cfg.lanes) { + if (lane != "interp" && lane != "jit") { + error = "suite config names an unknown lane \"{lane}\" (interp or jit): {path}" + } + } + return <- cfg +} + +def discover_files(repo_root : string; cfg : SuiteConfig; var error : string&) : array { + error = "" + let root = path_join(repo_root, cfg.root) + var rels : array + dir_rec(root) $(name, is_dir) { + let rel = to_generic_path(name) + return if (is_dir || extension(rel) != ".das") + for (pattern in cfg.exclude) { + return if (match_glob(pattern, rel)) + } + rels |> push(rel) + } + sort(rels) + var out : array + out |> reserve(length(rels)) + for (rel in rels) { + var f = BenchFile(path = rel, id = stem_path(rel), group = group_of(rel), timeout_seconds = cfg.timeout_seconds) + for (lane in keys(cfg.lane_excludes)) { + cfg.lane_excludes |> get(lane) $(globs) { + for (pattern in keys(globs)) { + if (match_glob(pattern, rel)) { + f.skip_lanes[lane] = globs?[pattern] ?? "" + } + } + } + } + cfg.files |> get(rel) $(ov) { + f.skip = ov.skip + for (lane in keys(ov.skip_lanes)) { + f.skip_lanes[lane] = ov.skip_lanes?[lane] ?? "" + } + if (ov.timeout_seconds > 0) { + f.timeout_seconds = ov.timeout_seconds + } + } + out |> emplace(f) + } + for (k in keys(cfg.files)) { + if (!stat(path_join(root, k)).is_reg) { + error = "suite config overrides \"{k}\", which is not a file under {root}" + } + } + return <- out +} + +def group_of(rel : string) : string { + let d = to_generic_path(dir_name(rel)) + return (empty(d) || d == ".") ? "root" : d +} + +def stem_path(rel : string) : string { + let d = group_of(rel) + return d == "root" ? stem(rel) : "{d}/{stem(rel)}" +} diff --git a/utils/benchctl/bench_table.das b/utils/benchctl/bench_table.das index bbf41270b2..8d357accb0 100644 --- a/utils/benchctl/bench_table.das +++ b/utils/benchctl/bench_table.das @@ -3,6 +3,7 @@ options gen2 require daslib/sql require sqlite/sqlite_boost require daslib/sql_linq +require dastest/testing_boost public [sql_table(name = "benchmarks")] struct Benchmark { @@ -21,3 +22,12 @@ struct Benchmark { string_allocs : int64 string_heap_bytes : int64 } + +//! The stored row carries commit and tag metadata the statistics never read - benchstat works on +//! the shape dastest emits, so a query hands its rows over as those. +def to_run_stats(rows : array) : array { + return <- [for (r in rows); BenchmarkRunStats(name = r.name, sub_name = r.sub_name, + n = int(r.n), time_ns = r.time_ns, allocs = uint64(r.allocs), heap_bytes = uint64(r.heap_bytes), + string_allocs = uint64(r.string_allocs), string_heap_bytes = uint64(r.string_heap_bytes), + func_type = r.mode)] +} diff --git a/utils/benchctl/benchstat.das b/utils/benchctl/benchstat.das index 4c1878baa4..cd64e5be6b 100644 --- a/utils/benchctl/benchstat.das +++ b/utils/benchctl/benchstat.das @@ -1,15 +1,13 @@ options gen2 -require dastest/testing_boost +require dastest/testing_boost public require daslib/strings_boost require daslib/json_boost require math -require bench_table public - struct BenchmarkSampleSet { key : string - list : array + list : array // Assigned later, when stats are computed. stats : BenchmarkStats? = null @@ -63,7 +61,7 @@ def parse_bench_output(data : string) : array { return <- entries } -def make_sample_sets(entries : array) : table { +def make_sample_sets(entries : array) : table { var result : table for (e in entries) { let key = e.name + "/" + e.sub_name @@ -92,7 +90,7 @@ def fill_sample_stats(var samples : BenchmarkSampleSet?) : string { // An example: there can be N entries for "foo/bar" benchmark, // every such entry is considered to be a single sample for analysis. for (e in samples.list) { - if (e.n == 0l) continue + if (e.n == 0) continue let n = double(e.n) stats.time_ns.values |> push(double(e.time_ns) / n) stats.heap_bytes.values |> push(double(e.heap_bytes) / n) @@ -291,3 +289,17 @@ def private betacf(a : double; b : double; x : double) : double { } return h } + +def median(vals : array) : double { + return 0.0lf if (empty(vals)) + var sorted := vals + sort(sorted) + let n = length(sorted) + return (n % 2 == 1) ? sorted[n / 2] : (sorted[n / 2 - 1] + sorted[n / 2]) / 2.0lf +} + +def median_i64(var vals : array) : int64 { + return 0l if (empty(vals)) + sort(vals) + return vals[length(vals) / 2] +} diff --git a/utils/benchctl/main.das b/utils/benchctl/main.das index 6b6127dc37..6017308803 100644 --- a/utils/benchctl/main.das +++ b/utils/benchctl/main.das @@ -13,6 +13,7 @@ require bench_args require benchstat require utils require table_fmt +require bench_stand def print_usage() { print("benchctl - benchmark database management tool\n\n") @@ -23,6 +24,8 @@ def print_usage() { print(" insert Insert benchmark JSON output files into the database\n") print(" query Display benchmark records (filter by --commit / --tag)\n") print(" compare Compare two sets of results statistically\n") + print(" run Benchmark a tree into one run record (the nightly stand)\n") + print(" report Turn run records into the viewer's data.json, with a verdict exit code\n") print(" help Show this help message\n\n") print_benchctl_help() } @@ -41,6 +44,8 @@ def script_argv() : array { def main : int { let argv <- script_argv() + if (!empty(argv) && (argv[0] == "run" || argv[0] == "report")) return stand_main() + if (empty(argv) || find_index(argv, "--help") != -1 || find_index(argv, "-h") != -1) { print_usage() return 0 @@ -375,8 +380,8 @@ def run_compare_cmd(db : SqlRunner; parsed : ParsedArgs) : string { } } - var old_samples = make_sample_sets(old_entries) - var new_samples = make_sample_sets(new_entries) + var old_samples = make_sample_sets(to_run_stats(old_entries)) + var new_samples = make_sample_sets(to_run_stats(new_entries)) var bench_order <- [for (k in keys(old_samples)); k] sort(bench_order) $(x, y) { diff --git a/utils/benchctl/suite.json b/utils/benchctl/suite.json new file mode 100644 index 0000000000..6708d94e33 --- /dev/null +++ b/utils/benchctl/suite.json @@ -0,0 +1,26 @@ +{ + "root": "benchmarks", + "exclude": [ + "**/tests/**", + "**/_*.das" + ], + "lanes": [ + "interp", + "jit" + ], + "repeat": 3, + "timeout_seconds": 900, + "files": { + "core/array/test01.das": { + "skip": "allocates ~19 GB under persistent_heap and gets OOM-killed; unskip once the benchmark is fixed" + }, + "compile/utils.das": { + "timeout_seconds": 7200 + } + }, + "lane_excludes": { + "jit": { + "compile/**": "times a child daslang -compile-only; the lane the parent runs in changes nothing about the child, and the interp lane already carries the group" + } + } +} diff --git a/utils/benchctl/tests/_fake_dastest.das b/utils/benchctl/tests/_fake_dastest.das new file mode 100644 index 0000000000..d7c080e175 --- /dev/null +++ b/utils/benchctl/tests/_fake_dastest.das @@ -0,0 +1,51 @@ +options gen2 + +require daslib/clargs +require daslib/fio +require daslib/json_boost +require strings + +[CommandLineArgs] +struct FakeArgs { + @clarg_doc = "where to write the dastest report" + json_file : string + @clarg_doc = "the benchmark file under test" + test : string + @clarg_doc = "repeats" + count : int + @clarg_doc = "measure" + bench : bool + @clarg_doc = "output format" + bench_format : string + @clarg_doc = "behaviour: ok | hang | no_report" + use_aot : bool +} + +def private mode(test_file : string) : string { + let m = strip(fread(path_join(dir_name(dir_name(test_file)), "fake_mode.txt"))) + return empty(m) ? "ok" : m +} + +def private stats(name, sub : string; n : int; time_ns : int64) : string { + return write_json_compact(JV((name = name, sub_name = sub, n = n, time_ns = time_ns, + allocs = 0, heap_bytes = 0, string_allocs = 0, string_heap_bytes = 0))) +} + +[export] +def main : int { + var r <- parse_args(type) + return 1 if (r |> is_err) + let args <- r |> move_unwrap + let how = mode(args.test) + print("lane={jit_enabled() ? "jit" : "interp"}\n") + print("{stats("b", "arm", 10, 1000l)}\n") + if (how == "hang") { + while (true) { + sleep(200u) + } + } + if (how != "no_report") { + fwrite(args.json_file, write_json_compact(JV((success = true, tests = JV([JV((name = "b", passed = true))]))))) + } + return 0 +} diff --git a/utils/benchctl/tests/_test_common.das b/utils/benchctl/tests/_test_common.das new file mode 100644 index 0000000000..318f731bdd --- /dev/null +++ b/utils/benchctl/tests/_test_common.das @@ -0,0 +1,32 @@ +options gen2 +module _test_common public + +require dastest/testing_boost +require daslib/fio +require daslib/json_boost + +def with_temp_dir(prefix : string; blk : block<(dir : string) : void>) { + var err = "" + let dir = create_temp_directory(prefix, err) + verify(!empty(dir), "temp dir created") + invoke(blk, dir) + rmdir_rec(dir) +} + +def fake_repo(root, mode : string) { + mkdir_rec(path_join(root, "dastest")) + mkdir_rec(path_join(root, "benchmarks/x")) + fwrite(path_join(root, "dastest/dastest.das"), fread(path_join(get_das_root(), "utils/benchctl/tests/_fake_dastest.das"))) + fwrite(path_join(root, "benchmarks/x/y.das"), "options gen2\n") + fwrite(path_join(root, "benchmarks/fake_mode.txt"), mode) +} + +def report_json(names, failed : array) : string { + var tests <- [for (n in names); JV((name = n, passed = find_index(failed, n) < 0))] + return write_json_compact(JV((success = empty(failed), tests = JV(tests)))) +} + +def stats_line(name, sub : string; n : int; time_ns : int64; allocs : int = 0) : string { + return write_json_compact(JV((name = name, sub_name = sub, n = n, time_ns = time_ns, + allocs = allocs, heap_bytes = allocs * 16, string_allocs = 0, string_heap_bytes = 0))) +} diff --git a/utils/benchctl/tests/test_bench_cli.das b/utils/benchctl/tests/test_bench_cli.das new file mode 100644 index 0000000000..66a668a7d9 --- /dev/null +++ b/utils/benchctl/tests/test_bench_cli.das @@ -0,0 +1,78 @@ +options gen2 + +require dastest/testing_boost public +require ../bench_history.das +require ../bench_runner.das +require _test_common +require daslib/fio +require daslib/clargs +require daslib/json_boost + +let private TOOL_DIR = "utils/benchctl" + +def private write_meta(dir : string; build_status : string) : string { + let path = path_join(dir, "meta.json") + fwrite(path, write_json(JV((run_id = "20260908T030000Z-abcdef12", started = "2026-09-08T03:00:00Z", + commit = JV((sha = "abcdef1234567890", date = "2026-09-08T01:00:00Z", subject = "a commit", author = "dev")), + machine = JV((host = "box", cores = 4)), + build = JV((status = build_status, seconds = 12, log_tail = "ninja: build stopped")))))) + return path +} + +def private run_tool(args : array; var output : string&) : int { + var argv <- ["bin/daslang", path_join(TOOL_DIR, "main.das"), "--"] + argv |> push_from(args) + return run_and_capture(argv, output, 240.0) +} + +def private write_run_record(runs_dir : string; n : int; ns : double) { + var rec <- RunRecord(run_id = "r{n}", started = "2026-09-0{n + 1}T03:00:00Z", commit = CommitInfo(sha = "sha{n}", date = "2026-09-0{n + 1}T01:00:00Z", subject = "c{n}"), + machine = MachineInfo(host = "box"), build = BuildInfo(status = "ok", seconds = 1.0lf)) + rec.lanes["interp"] = "ok" + var f <- FileResult(path = "core/one.das", id = "core/one", group = "core", lane = "interp", status = FileStatus.ok) + f.samples |> emplace(Sample(id = "b/arm", ns = ns, ns_median = ns, runs = 1)) + rec.files |> emplace(f) + fwrite(path_join(runs_dir, "r{n}.json"), sprint_json(rec, false)) +} + +[test] +def test_verbs(t : T?) { + with_temp_dir("bench_cli_") $(dir) { + let suite = path_join(dir, "suite.json") + fwrite(suite, write_json(JV((root = "benchmarks", exclude = JV(["**/_*.das"]), lanes = JV(["interp"]), repeat = 1, timeout_seconds = 120)))) + t |> run("run records one file per lane and exits on the night's verdict; --failed records a night that never ran") @(tt : T?) { + fake_repo(dir, "ok") + fwrite(path_join(dir, "benchmarks/x/z.das"), "options gen2\n") + let record = path_join(dir, "run.json") + var out = "" + let code = run_tool(["run", "--root", dir, "--bin", get_host_binary(), "--suite", suite, "--meta", write_meta(dir, "ok"), "--out", record, "--filter", "y.das"], out) + tt |> equal(code, 0, "exit 0 when every file ran: {out}") + var rec = RunRecord() + tt |> success(sscan_json(fread(record), rec), "the record parses") + tt |> equal(rec.status, RunStatus.ok) + tt |> equal(length(rec.files), 1, "--filter narrowed two files to one") + tt |> equal(rec.files[0].path, "x/y.das") + tt |> equal(length(rec.files[0].samples), 1) + let failed = path_join(dir, "failed.json") + tt |> equal(run_tool(["run", "--root", dir, "--bin", get_host_binary(), "--suite", suite, "--meta", write_meta(dir, "failed"), "--out", failed, "--failed", "the build failed"], out), 1) + tt |> success(sscan_json(fread(failed), rec) && rec.status == RunStatus.build_failed && empty(rec.files), "a failed build's record carries no file") + tt |> equal(run_tool(["run", "--root", dir, "--out", record], out), 1, "a missing --meta exits 1") + } + t |> run("report exits 0 on a clean night and 1 with no records") @(tt : T?) { + let runs_dir = path_join(dir, "runs") + mkdir_rec(runs_dir) + var out = "" + tt |> equal(run_tool(["report", "--suite", suite, "--runs", runs_dir, "--out-data", path_join(dir, "d0.json")], out), 1, "no records yet") + for (n in range(4)) { + write_run_record(runs_dir, n, 100.0lf) + } + let data = path_join(dir, "data.json") + tt |> equal(run_tool(["report", "--suite", suite, "--runs", runs_dir, "--out-data", data, "--repo-url", "https://example.invalid/repo"], out), 0, "a clean latest night: {out}") + var ds = Dataset() + tt |> success(sscan_json(fread(data), ds), "data.json parses") + tt |> equal(length(ds.runs), 4) + tt |> equal(length(ds.series), 1) + tt |> equal(ds.repo_url, "https://example.invalid/repo") + } + } +} diff --git a/utils/benchctl/tests/test_bench_history.das b/utils/benchctl/tests/test_bench_history.das new file mode 100644 index 0000000000..ae3b40c2d4 --- /dev/null +++ b/utils/benchctl/tests/test_bench_history.das @@ -0,0 +1,111 @@ +options gen2 + +require dastest/testing_boost public +require ../bench_history.das +require ../benchstat.das +require ../bench_runner.das +require _test_common +require daslib/fio +require daslib/json_boost +require daslib/strings_boost +require strings +require math + +def private sample(id : string; ns : double) : Sample { + return Sample(id = id, ns = ns, ns_median = ns, spread = 0.05lf, runs = 3) +} + +def private file_ok(id, lane : string; samples : array) : FileResult { + var f <- FileResult(path = "{id}.das", id = id, group = dir_name(id), lane = lane, status = FileStatus.ok) + f.samples := samples + return <- f +} + +def private file_bad(id, lane : string; status : FileStatus; message : string) : FileResult { + return <- FileResult(path = "{id}.das", id = id, group = dir_name(id), lane = lane, status = status, message = message) +} + +def private night(n : int; files : array) : RunRecord { + var rec <- RunRecord(run_id = "r{n:02}", started = "2026-09-{n + 1:02}T03:00:00Z", finished = "2026-09-{n + 1:02}T04:00:00Z", seconds = 3600.0lf, + commit = CommitInfo(sha = "sha{n:02}abcdef", date = "2026-09-{n + 1:02}T00:00:00Z", subject = "commit {n}", author = "dev"), + machine = MachineInfo(host = "box", cores = 4), build = BuildInfo(status = "ok", seconds = 600.0lf)) + rec.lanes["interp"] = "ok" + rec.lanes["jit"] = "ok" + rec.files := files + rec.status = derive_status(rec) + return <- rec +} + +def private eight_nights_with_a_jump() : array { + var runs <- [for (n in range(7)); night(n, [ + file_ok("core/hash/t", "interp", [sample("a/x", 100.0lf + double(n % 3) * 0.5lf), sample("b/y", 200.0lf - double(n % 3) * 0.5lf)]), + file_ok("core/hash/t", "jit", [sample("a/x", 50.0lf + double(n % 3) * 0.5lf)])])] + runs |> emplace(night(7, [ + file_ok("core/hash/t", "interp", [sample("a/x", 130.0lf), sample("c/z", 5.0lf)]), + file_ok("core/hash/t", "jit", [sample("a/x", 40.0lf)]), + file_bad("sort/s", "interp", FileStatus.compile_error, "error[30344]: mismatch")])) + return <- runs +} + +[test] +def test_load_runs(t : T?) { + with_temp_dir("bench_hist_") $(dir) { + t |> run("records load in commit order, a corrupt file is named and skipped, status derives") @(tt : T?) { + for (rec in [night(2, [file_ok("core/t", "interp", [sample("a/x", 1.0lf)])]), night(0, [file_ok("core/t", "interp", [sample("a/x", 1.0lf)])]), night(1, [file_bad("core/t", "interp", FileStatus.timeout, "killed")])]) { + fwrite(path_join(dir, "{rec.run_id}.json"), sprint_json(rec, false)) + } + fwrite(path_join(dir, "zz_corrupt.json"), "\{ not json") + fwrite(path_join(dir, "notes.txt"), "ignored") + var errors : array + let runs <- load_runs(dir, errors) + tt |> equal(join([for (r in runs); r.run_id], " "), "r00 r01 r02") + tt |> equal(runs[1].status, RunStatus.bench_failed) + tt |> equal(runs[0].status, RunStatus.ok) + tt |> equal(length(errors), 1) + tt |> success(find(errors[0], "zz_corrupt.json") >= 0, "corrupt file named") + } + t |> run("a night that measured nothing is not ok, a failed build is build_failed, a driver failure names the driver") @(tt : T?) { + let no_files : array + tt |> equal(derive_status(night(30, no_files)), RunStatus.bench_failed, "no file ran at all") + tt |> equal(derive_status(night(31, [file_bad("core/x", "interp", FileStatus.skipped, "listed")])), RunStatus.bench_failed, "nothing but skips is no data point") + tt |> equal(derive_status(night(32, [file_ok("core/x", "interp", [sample("a/x", 1.0lf)])])), RunStatus.ok, "one measured file is a night") + var rec <- night(5, no_files) + rec.build.status = "failed" + tt |> equal(derive_status(rec), RunStatus.build_failed) + var driver <- night(6, no_files) + mark_failed_run(driver, "now", "the tool crashed") + tt |> equal(driver.status, RunStatus.bench_failed) + tt |> equal(length(driver.files), 1) + } + } +} + +[test] +def test_build_dataset(t : T?) { + t |> run("series are keyed by lane and arm, points carry the run index; the dataset round-trips") @(tt : T?) { + let ds <- build_dataset(eight_nights_with_a_jump(), "2026-09-08T05:00:00Z", "https://github.com/x/y") + tt |> equal(length(ds.runs), 8) + tt |> equal(ds.latest, 7) + tt |> equal(join(ds.groups, " "), "core/hash") + tt |> equal(length(ds.series), 4) + tt |> equal(ds.series[0].lane, "interp") + tt |> equal(ds.series[0].id, "core/hash/t#a/x") + tt |> equal(length(ds.series[0].runs), 8) + tt |> success(abs(ds.series[0].ns[7] - 130.0lf) < 1e-9lf, "tonight's value") + tt |> equal(ds.series[3].lane, "jit") + var back = Dataset() + tt |> success(sscan_json(sprint_json(ds, false), back), "parses back") + tt |> equal(length(back.series), 4) + tt |> equal(back.runs[7].commit.sha, "sha07abcdef") + } + t |> run("the run summary carries failures with their message and the ok count; a failed night exits 1") @(tt : T?) { + let ds <- build_dataset(eight_nights_with_a_jump(), "", "") + tt |> equal(ds.runs[7].status, RunStatus.bench_failed) + tt |> equal(ds.runs[7].files_ok, 2) + tt |> equal(length(ds.runs[7].failures), 1) + tt |> equal(ds.runs[7].failures[0].path, "sort/s.das") + tt |> equal(ds.runs[7].failures[0].status, "compile_error") + tt |> equal(ds.runs[7].failures[0].message, "error[30344]: mismatch") + tt |> equal(report_verdict(ds), StandExit.failed) + } +} diff --git a/utils/benchctl/tests/test_bench_runner.das b/utils/benchctl/tests/test_bench_runner.das new file mode 100644 index 0000000000..040ce25d24 --- /dev/null +++ b/utils/benchctl/tests/test_bench_runner.das @@ -0,0 +1,110 @@ +options gen2 + +require dastest/testing_boost public +require ../bench_runner.das +require ../bench_suite.das +require _test_common +require daslib/clargs +require strings + +def private fresh_result(status : FileStatus = FileStatus.ok) : FileResult { + return <- FileResult(path = "core/x.das", id = "core/x", group = "core", lane = "interp", status = status) +} + +def private parsed(lines : array; report : string; exit_code : int; status : FileStatus = FileStatus.ok) : FileResult { + var res <- fresh_result(status) + classify_run_output(lines, report, exit_code, res) + return <- res +} + +[test] +def test_classify_run_output(t : T?) { + t |> run("samples aggregate over repeats: min, median, spread, per-op allocation medians") @(tt : T?) { + let res <- parsed([stats_line("b", "insert", 10, 1000l, 20), stats_line("b", "insert", 10, 1200l, 30), stats_line("b", "insert", 10, 1100l, 40), "noise"], report_json(["b"], []), 0) + tt |> equal(res.status, FileStatus.ok) + tt |> equal(length(res.samples), 1) + tt |> equal(res.samples[0].id, "b/insert") + tt |> equal(res.samples[0].ns, 100.0lf) + tt |> equal(res.samples[0].ns_median, 110.0lf) + tt |> success(res.samples[0].spread > 0.19lf && res.samples[0].spread < 0.21lf, "spread is (max - min) / min") + tt |> equal(res.samples[0].runs, 3) + tt |> equal(res.samples[0].allocs, 3l) + tt |> equal(res.log_tail, "noise") + } + t |> run("a compile error names the first error line") @(tt : T?) { + let res <- parsed(["x.das:3:1: error[30001]: bad", "Failed to compile x.das"], "", 1) + tt |> equal(res.status, FileStatus.compile_error) + tt |> success(find(res.message, "error[30001]") >= 0, "message carries the error line") + } + t |> run("a failed benchmark function drops its arms and names itself; the others survive") @(tt : T?) { + let res <- parsed([stats_line("good", "a", 10, 100l), stats_line("bad", "b", 10, 100l), "--- FAIL 'bad' in 'x.das'"], report_json(["good", "bad"], ["bad"]), 1) + tt |> equal(res.status, FileStatus.failed) + tt |> equal(length(res.samples), 1) + tt |> equal(res.samples[0].id, "good/a") + tt |> success(find(res.message, "bad") >= 0, "message names the failed function") + } + t |> run("a measured file that dies at shutdown keeps its numbers; one with no finished report is a failure") @(tt : T?) { + let shutdown <- parsed([stats_line("b", "a", 10, 100l)], report_json(["b"], []), 1) + tt |> equal(shutdown.status, FileStatus.ok) + tt |> success(find(shutdown.message, "shutdown") >= 0, "message says the shutdown failed") + let crashed <- parsed([stats_line("b", "a", 10, 100l), "FATAL: boom"], "", 139) + tt |> equal(crashed.status, FileStatus.exit_nonzero) + tt |> equal(crashed.message, "FATAL: boom") + } + t |> run("a timeout verdict survives whatever the output says; a clean exit with no samples is not ok") @(tt : T?) { + let killed <- parsed([stats_line("b", "a", 10, 100l)], report_json(["b"], []), 0, FileStatus.timeout) + tt |> equal(killed.status, FileStatus.timeout) + tt |> equal(length(killed.samples), 1) + tt |> equal(parsed(["nothing"], report_json([], []), 0).status, FileStatus.failed) + tt |> success(is_failure(FileStatus.exit_nonzero) && !is_failure(FileStatus.skipped), "is_failure separates ok and skipped from the rest") + } +} + +def private bench_file() : BenchFile { + return BenchFile(path = "x/y.das", id = "x/y", group = "x", timeout_seconds = 2) +} + +[test] +def test_run_bench_file(t : T?) { + with_temp_dir("bench_run_") $(root) { + t |> run("a skipped file never spawns and carries its reason") @(tt : T?) { + var f = bench_file() + f.skip = "too slow" + let res <- run_bench_file(get_host_binary(), root, f, "interp", RunLimits()) + tt |> equal(res.status, FileStatus.skipped) + tt |> equal(res.message, "too slow") + } + t |> run("a clean child is ok, and the jit lane's -jit reaches the host that runs it") @(tt : T?) { + fake_repo(root, "ok") + let jit <- run_bench_file(get_host_binary(), root, bench_file(), "jit", RunLimits(timeout_seconds = 120)) + tt |> equal(jit.status, FileStatus.ok, "{jit.message} {jit.log_tail}") + tt |> equal(length(jit.samples), 1) + tt |> equal(jit.samples[0].id, "b/arm") + tt |> equal(jit.samples[0].ns, 100.0lf) + tt |> success(find(jit.log_tail, "lane=jit") >= 0, "the child ran jitted: {jit.log_tail}") + let interp <- run_bench_file(get_host_binary(), root, bench_file(), "interp", RunLimits(timeout_seconds = 120)) + tt |> success(find(interp.log_tail, "lane=interp") >= 0, "the interp lane passes no -jit: {interp.log_tail}") + } + t |> run("a child that outlives timeout_seconds is killed; the arms it printed survive") @(tt : T?) { + fake_repo(root, "hang") + let res <- run_bench_file(get_host_binary(), root, bench_file(), "interp", RunLimits(timeout_seconds = 20)) + tt |> equal(res.status, FileStatus.timeout) + tt |> success(find(res.message, "timeout_seconds") >= 0, "message names the knob") + tt |> equal(length(res.samples), 1, "the arm it printed before the kill stands") + } + t |> run("an unrunnable binary reports that nothing was measured") @(tt : T?) { + let res <- run_bench_file("/nonexistent/daslang", root, bench_file(), "interp", RunLimits(timeout_seconds = 30)) + tt |> success(is_failure(res.status), "a missing binary is a failure: {res.status}") + tt |> equal(length(res.samples), 0) + } + } +} + +[test] +def test_probe_jit(t : T?) { + t |> run("a daslang that runs the probe reports the lane available; one that cannot run says so") @(tt : T?) { + tt |> equal(probe_jit(get_host_binary()), "", "the running host prints the marker") + let why = probe_jit("/nonexistent/daslang") + tt |> success(find(why, "/nonexistent/daslang") >= 0, "the reason names the binary it could not run: {why}") + } +} diff --git a/utils/benchctl/tests/test_bench_suite.das b/utils/benchctl/tests/test_bench_suite.das new file mode 100644 index 0000000000..874075a3a5 --- /dev/null +++ b/utils/benchctl/tests/test_bench_suite.das @@ -0,0 +1,101 @@ +options gen2 + +require dastest/testing_boost public +require ../bench_suite.das +require _test_common +require daslib/fio +require daslib/json_boost +require daslib/strings_boost +require strings + +def private touch(dir, rel : string) { + let p = path_join(dir, rel) + mkdir_rec(dir_name(p)) + fwrite(p, "options gen2\n") +} + +def private good_config() : string { + let files <- { "core/slow.das" => JV((skip = "too slow")), "core/big.das" => JV((timeout_seconds = 60)) } + let globs <- { "sort/**" => "not here" } + return write_json(JV((root = "benchmarks", exclude = JV(["**/tests/**", "**/_*.das"]), lanes = JV(["interp"]), + repeat = 2, timeout_seconds = 30, lane_excludes = JV((interp = JV(globs))), files = JV(files)))) +} + +def private load_cfg(dir : string) : SuiteConfig { + var error = "" + return <- load_suite_config(path_join(dir, "suite.json"), error) +} + +[test] +def test_load_suite_config(t : T?) { + with_temp_dir("bench_stand_") $(dir) { + t |> run("a well-formed config parses with its overrides") @(tt : T?) { + fwrite(path_join(dir, "suite.json"), good_config()) + var error = "" + let cfg <- load_suite_config(path_join(dir, "suite.json"), error) + tt |> equal(error, "") + tt |> equal(cfg.repeat, 2) + tt |> equal(cfg.timeout_seconds, 30) + tt |> equal(length(cfg.exclude), 2) + tt |> equal(length(cfg.files), 2) + cfg.files |> get("core/big.das") $(ov) { + tt |> equal(ov.timeout_seconds, 60) + } + } + t |> run("a missing, unparseable or out-of-range config fails closed, naming what is wrong") @(tt : T?) { + var error = "" + load_suite_config(path_join(dir, "missing.json"), error) + tt |> success(find(error, "missing.json") >= 0, "error names the path") + let path = path_join(dir, "bad.json") + var bodies <- [ + "not a config at all", + write_json(JV((lanes = JV(array())))), + write_json(JV((lanes = JV(["interp"]), repeat = 0))), + write_json(JV((lanes = JV(["vulkan"])))), + write_json(JV((lanes = JV(["interp"]), lane_excludes = JV((jit = JV(table())))))) + ] + for (body, word in bodies, ["parse", "no lanes", "positive", "vulkan", "does not run"]) { + fwrite(path, body) + load_suite_config(path, error) + tt |> success(find(error, word) >= 0, "{body} -> {error}") + } + fwrite(path, write_json(JV((lanes = JV(["interp", "jit"]))))) + load_suite_config(path, error) + tt |> equal(error, "", "the two real lanes are accepted") + } + } +} + +[test] +def test_discover_files(t : T?) { + with_temp_dir("bench_stand_") $(dir) { + fwrite(path_join(dir, "suite.json"), good_config()) + for (rel in ["core/hash/test02.das", "core/hash/_common.das", "core/slow.das", "core/big.das", "sql/tests/test_update.das", "sort/sort.das", "sort/README.md"]) { + touch(dir, "benchmarks/{rel}") + } + t |> run("helpers, test dirs and non-das files are excluded; the rest is sorted, with identity from the path and overrides applied") @(tt : T?) { + var err = "" + let files <- discover_files(dir, load_cfg(dir), err) + tt |> equal(err, "") + tt |> equal(join([for (f in files); f.path], " "), "core/big.das core/hash/test02.das core/slow.das sort/sort.das") + tt |> equal(files[1].group, "core/hash") + tt |> equal(files[1].id, "core/hash/test02") + tt |> equal(group_of("top.das"), "root") + tt |> equal(stem_path("top.das"), "top") + tt |> equal(files[2].skip, "too slow") + tt |> equal(files[2].timeout_seconds, 30) + tt |> equal(files[0].timeout_seconds, 60) + tt |> equal(files[1].skip, "") + tt |> equal(skip_reason(files[3], "interp"), "not here", "a lane exclude reaches the file it matches") + tt |> equal(skip_reason(files[1], "interp"), "") + } + t |> run("an override naming a missing file is reported") @(tt : T?) { + let cfg <- load_cfg(dir) + remove(path_join(dir, "benchmarks/core/slow.das")) + var err = "" + let files <- discover_files(dir, cfg, err) + tt |> equal(length(files), 3) + tt |> success(find(err, "core/slow.das") >= 0, "error names the stale override") + } + } +} diff --git a/utils/internal/bench-stand/README.md b/utils/internal/bench-stand/README.md new file mode 100644 index 0000000000..9adff33a89 --- /dev/null +++ b/utils/internal/bench-stand/README.md @@ -0,0 +1,104 @@ +# bench-stand - the nightly benchmark stand + +Every night one box checks out master, builds it, runs every `[benchmark]` file under +`benchmarks/` in the interpreter and JIT lanes, and publishes the result at +https://daslang.io/bench/ - a chart per benchmark arm over commits and a plain statement of what +failed and what moved. Review rules: `REVIEW.md`. + +## 1. Layout + +- The tool is `utils/benchctl` - `run` benchmarks the tree into one run record, `report` turns + every record into `data.json` with an exit code, and its database verbs (`insert`, `query`, + `compare`) answer ad-hoc questions about the same numbers. Every statistic it computes lives in + `benchctl/benchstat.das`, which depends on no storage. +- `site/` - the viewer: `index.html`, `app.js`, `style.css`. Static, no build step, no + dependencies; reads `data.json` and `status.json` beside it. +- `run_stand.sh` - one pass on a ref, the thing cron calls (section 3); `caddy.snippet` is the public route. + +## 2. Data model + +Benchmark identity: `#/` per lane, where the file id is the path under +`benchmarks/` without `.das` (`core/hash/test02#builtin_table/insert/600000`, lane `interp`). +The group is the file's directory; a new benchmark joins its group by living in the right folder. + +`runs/.json` - one per night, `RunRecord` in `bench_history.das`, never rewritten. The run +id is `-`. It carries the commit, the machine, the build (status, seconds, log +tail), the lane states, and one `FileResult` per file per lane: status, exit code, seconds, +message, the last 40 log lines, and the samples. A sample is one arm over the night's repeats: +`ns` = the minimum ns/op (the recorded value), `ns_median`, `spread` = (max - min) / min, and +per-op allocation medians. A record's `status` (`ok`, `build_failed`, `bench_failed`) is +re-derived from its parts on every read. + +File statuses: `ok`, `skipped` (listed in `suite.json` with a reason), `compile_error`, `failed` +(a `[benchmark]` function failed or panicked - its arms are dropped, the other functions' kept), +`timeout` (killed at `timeout_seconds`), `exit_nonzero` (the process died without a finished +dastest report), `spawn_failed`. A file whose report finished with its arms in hand stays `ok` on +a non-zero exit - only the shutdown went wrong - and the message says so; the JIT lane's probe +likewise believes the marker its program prints, not the exit code. + +`site/data.json` - `Dataset`: run summaries (with failures, skips, lane states), groups, series +(columnar: `runs` indexes `Dataset.runs`, `ns`, `spread`). +`site/status.json` is written by `run_stand.sh` at start and end (`running` / `finished`, run id, +exit), so a night whose build failed is still visible. + +### 2.1 Reading the numbers + +The stand publishes the series and leaves the judgement to a person. Nothing decides what counts +as a regression: `benchctl compare --old-commit --new-commit ` answers that on demand, with +a Welch test over the samples, for whichever two commits are actually in question. + +### 2.2 Exit codes + +`run`: 0 when every file ran ok, 1 otherwise. `report`: 0 when the latest night is ok, 1 when it +failed. `run --failed ""` benchmarks nothing: it writes the record of a night that could +not run the suite (the driver's way to record a failed build) and exits 1. + +## 3. The box + +`dasweb-1` (the daslang.io origin) runs `run_stand.sh master` from the `bench` user's cron under +`/srv/bench-stand`: `src/` (the clone), `runs/`, `site/` (what Caddy serves at `/bench/`), +`logs/`. The night builds Release (RelWithDebInfo arms the C++ allocation tracker, whose exit-time +report costs the run time) with the module set the benchmarks require, then `run`, then `report`; +`bin/` survives the checkout's clean, so a night whose build fails still renders a red night with +the previous binary. There is no CI runner and no ssh path into the box: the repository is public. + +One-time setup, as root: + +```sh +useradd -r -m -d /srv/bench-stand -s /bin/bash bench +# llvm-22-dev is required: the night builds with -DDAS_LLVM_DISABLED=OFF, and without it +# there is no jit lane. apt.llvm.org carries it for noble, as in extended_checks. +apt-get install -y git cmake ninja-build g++ ccache llvm-22-dev +su - bench -c 'git clone https://github.com/GaijinEntertainment/daScript src && mkdir -p runs site logs' +su - bench -c '/srv/bench-stand/src/utils/internal/bench-stand/run_stand.sh master' # first pass by hand +echo '0 5 * * * bench /srv/bench-stand/src/utils/internal/bench-stand/run_stand.sh master >> /srv/bench-stand/logs/cron.log 2>&1' > /etc/cron.d/bench-stand +``` + +then paste `caddy.snippet` into the `daslang.io` block of the Caddyfile and `systemctl reload caddy`. + +Arguments after the ref go to `benchctl run` - `run_stand.sh master --filter core/math/ --repeat 1` +is a slice of a night. `BENCH_STAND_HOME` moves the layout, `BENCH_STAND_BUILD=skip` reuses the +last build; both are for local dry runs. + +## 4. Configuration - `utils/benchctl/suite.json` + +`root`, `exclude` (globs over the path under root; helper modules `_*.das` and `**/tests/**` by +default), `lanes` (`interp`, `jit`), `lane_excludes` (per lane, glob -> the reason those files do +not run in it), `repeat` (dastest `--count`), `timeout_seconds` per file, and `files` - per-file +`timeout_seconds` or a `skip` with its reason. A skipped file is listed on the site every night, so a skip is visible debt; an override +naming a file that no longer exists is an error. + +## 5. Running locally + +```sh +bin/daslang utils/internal/bench-stand/main.das -- run --meta meta.json --out /tmp/stand/runs/n1.json --filter core/math/ --repeat 2 +bin/daslang utils/internal/bench-stand/main.das -- report --runs /tmp/stand/runs --out-data /tmp/stand/site/data.json +cp utils/internal/bench-stand/site/* /tmp/stand/site/ && ln -sfn ../runs /tmp/stand/site/runs +bin/daslang dastest/dastest.das -- --test utils/internal/bench-stand +``` + +`meta.json` is what `run_stand.sh` writes: `{"run_id", "started", "commit": {"sha", "date", +"subject", "author"}, "machine": {...}, "build": {"status", "seconds", "log_tail"}}`; any +subset parses. The whole pipeline runs locally too: `BENCH_STAND_HOME= +BENCH_STAND_BUILD=skip run_stand.sh --filter core/math/` with `/src` a clone +holding a built `bin/daslang`. diff --git a/utils/internal/bench-stand/REVIEW.md b/utils/internal/bench-stand/REVIEW.md new file mode 100644 index 0000000000..dfcb3e3de2 --- /dev/null +++ b/utils/internal/bench-stand/REVIEW.md @@ -0,0 +1,30 @@ +# bench-stand Code Review Checklist + +**Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: +`README.md`. + +**Never add a run-record field without saying in `README.md` section 2 what reads it - the +viewer, or a person opening the record.** A field nobody named is one nobody notices going wrong. + +**A diff that changes a run-record field keeps the new reader parsing a record written by the old +code, a missing field keeping its declared default.** Records already on the box are never +rewritten. + +**Never let a child's own output overwrite a `timeout` status in `run_bench_file` - a killed child +that printed a passing report is still killed.** Loosen `timeout_seconds` in `suite.json` instead. + +**A series a diff adds or recolors in `site/app.js` takes its color from its lane, never from its +position in the series list, and a chart drawing more than one lane shows a legend.** + +**Never read a benchmark's identity from anywhere but its path under `benchmarks/`** - the group +is the directory, the id is the path without `.das`. + +**Placement - one file, one line: a diff keeps each file inside its line, and a new file adds its +line here, with its tests, in the same change.** + +- `site/` - the viewer. Zero dependencies, zero build step. +- `run_stand.sh` - one pass on a ref: checkout, build, run, report, publish. Cron calls it. +- `caddy.snippet` - the public route, and the only place a route is written down. + +The tool this box runs is `utils/benchctl` - its modules, verbs and tests answer to +`utils/benchctl/REVIEW.md`. diff --git a/utils/internal/bench-stand/caddy.snippet b/utils/internal/bench-stand/caddy.snippet new file mode 100644 index 0000000000..d30f764380 --- /dev/null +++ b/utils/internal/bench-stand/caddy.snippet @@ -0,0 +1,17 @@ +# bench-stand routes for the daslang.io vhost in /etc/caddy/Caddyfile. +# This file is the authoritative copy of the stand's public boundary — the +# deployed Caddyfile is edited to match it, never the other way round. +# +# Paste inside the `daslang.io { ... }` block, ahead of `root`/`file_server`. +# Splice it by hand, then `caddy validate --config /etc/caddy/Caddyfile` and reload. + +# The nightly benchmark stand: a static tree the `bench` user rewrites every +# night (viewer, data.json, status.json, summary.md, runs/). No service behind +# it. no-cache: the viewer fetches data.json on every visit and a day-old +# cached copy would show yesterday's night as tonight's. +redir /bench /bench/ 308 +handle_path /bench/* { + root * /srv/bench-stand/site + header Cache-Control "no-cache" + file_server +} diff --git a/utils/internal/bench-stand/run_stand.sh b/utils/internal/bench-stand/run_stand.sh new file mode 100755 index 0000000000..4f65159e54 --- /dev/null +++ b/utils/internal/bench-stand/run_stand.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# One pass of the stand on a ref - checkout, build, benchmark, report, publish. README.md. +set -euo pipefail +HOME_DIR=${BENCH_STAND_HOME:-/srv/bench-stand}; SRC=$HOME_DIR/src; SITE=$HOME_DIR/site; TOOL=utils/internal/bench-stand; BENCHCTL=utils/benchctl +REF=${1:-master}; shift || true +CMAKE_ARGS=(-G Ninja -DCMAKE_BUILD_TYPE=Release -DDAS_SQLITE_DISABLED=OFF -DDAS_PUGIXML_DISABLED=OFF -DDAS_LLVM_DISABLED=OFF -DDAS_GLFW_DISABLED=ON -DDAS_HV_DISABLED=ON) +TARGETS=(daslang dasModuleSQLITE dasModulePUGIXML dasModuleAudio dasModuleMinfft dasModuleTerminal dasModuleUnitTest dasModuleLLVM) + +now() { date -u +%Y-%m-%dT%H:%M:%SZ; } +jstr() { printf '%s' "$1" | tr -d '\r' | LC_ALL=C awk 'BEGIN{ORS=""} {gsub(/\\/,"\\\\"); gsub(/"/,"\\\""); gsub(/\t/,"\\t"); if (NR>1) printf "\\n"; printf "%s", $0}' | tr '\000-\037' ' '; } +status() { printf '{"state":"%s","run_id":"%s","started":"%s","finished":"%s","sha":"%s","exit":%s}\n' "$1" "$RUN_ID" "$STARTED" "$(now)" "$SHA" "${2:-null}" > "$SITE/status.json"; } + +mkdir -p "$HOME_DIR/runs" "$SITE" "$HOME_DIR/logs" +exec 9>"$HOME_DIR/lock"; flock -n 9 || { echo "another pass holds $HOME_DIR/lock"; exit 3; } +git -C "$SRC" fetch -q origin +SHA=$(git -C "$SRC" rev-parse --verify "origin/$REF^{commit}" 2>/dev/null || git -C "$SRC" rev-parse --verify "$REF^{commit}") +RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-${SHA:0:8}"; STARTED=$(now) +status running; trap 'status finished 1' ERR +git -C "$SRC" checkout -q --detach "$SHA" +git -C "$SRC" submodule update -q --init --recursive || true +git -C "$SRC" clean -fdxq --exclude=build --exclude=bin --exclude=lib --exclude=.jitted_scripts + +LOG=$HOME_DIR/logs/build-$RUN_ID.log; BUILD=ok; t0=$(date +%s) +if [ "${BENCH_STAND_BUILD:-}" = skip ]; then + : > "$LOG"; [ -x "$SRC/bin/daslang" ] || BUILD=failed +else + launcher=(); command -v ccache >/dev/null && launcher=(-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache) + ( cd "$SRC" && cmake --no-warn-unused-cli -B build "${CMAKE_ARGS[@]}" "${launcher[@]}" && cmake --build build --parallel "$(nproc)" --target "${TARGETS[@]}" ) > "$LOG" 2>&1 || BUILD=failed +fi +SECS=$(( $(date +%s) - t0 )) +META=$HOME_DIR/meta.json +printf '{"run_id":"%s","started":"%s","commit":{"sha":"%s","date":"%s","subject":"%s","author":"%s"},"machine":{"host":"%s","cores":%s},"build":{"status":"%s","seconds":%s,"log_tail":"%s"}}\n' \ + "$RUN_ID" "$STARTED" "$SHA" "$(TZ=UTC git -C "$SRC" log -1 --format=%cd --date=iso-strict-local | sed 's/+00:00$/Z/')" \ + "$(jstr "$(git -C "$SRC" log -1 --format=%s)")" "$(jstr "$(git -C "$SRC" log -1 --format=%an)")" "$(jstr "$(hostname)")" "$(nproc)" \ + "$BUILD" "$SECS" "$(jstr "$(tail -n 40 "$LOG" | cut -c1-400)")" > "$META" + +[ -x "$SRC/bin/daslang" ] || { echo "no binary: the build failed and none survives from an earlier pass"; trap - ERR; status finished 1; exit 1; } +[ "$BUILD" = ok ] || set -- "$@" --failed "the build failed after $SECS s - $LOG on the box has the whole log" +rc=0; report_rc=0 +( cd "$SRC" && bin/daslang "$BENCHCTL/main.das" -- run --root . --meta "$META" --out "$HOME_DIR/runs/$RUN_ID.json" "$@" ) || rc=$? +"$SRC/bin/daslang" "$SRC/$BENCHCTL/main.das" -- report --runs "$HOME_DIR/runs" --out-data "$SITE/data.json.tmp" --repo-url https://github.com/GaijinEntertainment/daScript || report_rc=$? +[ -s "$SITE/data.json.tmp" ] && mv "$SITE/data.json.tmp" "$SITE/data.json" +cp "$SRC/$TOOL"/site/{index.html,app.js,style.css} "$SITE/"; ln -sfn ../runs "$SITE/runs" +[ $rc -eq 0 ] && rc=$report_rc +trap - ERR; status finished "$rc"; exit "$rc" diff --git a/utils/internal/bench-stand/site/app.js b/utils/internal/bench-stand/site/app.js new file mode 100644 index 0000000000..7084ceac35 --- /dev/null +++ b/utils/internal/bench-stand/site/app.js @@ -0,0 +1,343 @@ +"use strict"; + +const LANES = ["interp", "jit"]; +const state = { data: null, status: null, runWindow: 90, lanes: new Set(LANES), group: "" }; + +const $ = (sel) => document.querySelector(sel); +const el = (tag, cls, text) => { + const e = document.createElement(tag); + if (cls) e.className = cls; + if (text !== undefined) e.textContent = text; + return e; +}; +const svgEl = (tag, attrs) => { + const e = document.createElementNS("http://www.w3.org/2000/svg", tag); + for (const k in attrs) e.setAttribute(k, attrs[k]); + return e; +}; +function fmtNs(v) { + if (v >= 1e9) return (v / 1e9).toFixed(2) + " s"; + if (v >= 1e6) return (v / 1e6).toFixed(2) + " ms"; + if (v >= 1e3) return (v / 1e3).toFixed(2) + " us"; + return v.toFixed(v >= 100 ? 0 : v >= 10 ? 1 : 2) + " ns"; +} +const fmtSec = (s) => (s >= 3600 ? (s / 3600).toFixed(1) + " h" : s >= 60 ? Math.round(s / 60) + " min" : Math.round(s) + " s"); +const shortSha = (sha) => (sha || "").slice(0, 8); +const dateOf = (iso) => (iso || "").slice(0, 10); +const commitUrl = (sha) => (state.data.repo_url && sha ? state.data.repo_url.replace(/\/$/, "") + "/commit/" + sha : null); +const recordUrl = (run) => "runs/" + encodeURIComponent(run.id) + ".json"; +const anchorId = (id) => "b-" + id.replace(/[^A-Za-z0-9_-]/g, "_"); + +async function load() { + const [dataRes, statusRes] = await Promise.allSettled([ + fetch("data.json", { cache: "no-store" }).then((r) => (r.ok ? r.json() : Promise.reject(new Error("data.json " + r.status)))), + fetch("status.json", { cache: "no-store" }).then((r) => (r.ok ? r.json() : null)), + ]); + state.status = statusRes.status === "fulfilled" ? statusRes.value : null; + if (dataRes.status === "fulfilled") state.data = dataRes.value; + else $("main").replaceChildren(el("p", "notice notice--error", "No data.json yet: the stand has not published a report (" + dataRes.reason.message + ").")); + renderStatus(); + if (state.data) renderAll(); +} + +function visibleRunIndices() { + const n = state.data.runs.length; + const from = state.runWindow > 0 ? Math.max(0, n - state.runWindow) : 0; + return Array.from({ length: n - from }, (_, i) => from + i); +} + +function renderStatus() { + const box = $("#status"); + box.replaceChildren(); + const st = state.status; + if (st && st.state === "running") box.append(el("span", "pill pill--running", "running " + (st.run_id || ""))); + if (!state.data || state.data.latest < 0) { if (!st) box.append(el("span", "pill", "no runs")); return; } + const run = state.data.runs[state.data.latest]; + box.append(el("span", "pill pill--" + run.status, run.status.replace("_", " "))); + const link = el("a", null, shortSha(run.commit.sha) + " " + run.commit.subject); + link.href = commitUrl(run.commit.sha) || "#"; + box.append(link, el("span", "muted", dateOf(run.started) + " on " + run.machine.host + ", build " + fmtSec(run.build.seconds) + ", suite " + fmtSec(run.seconds))); + const ageDays = (Date.now() - Date.parse(run.started)) / 86400000; + if (ageDays > 2 && !(st && st.state === "running")) box.append(el("span", "pill pill--stale", "last run " + Math.floor(ageDays) + " days ago")); + $("#generated").textContent = "report generated " + (state.data.generated || ""); +} + +function wireFilters() { + document.querySelectorAll(".chip[data-range]").forEach((b) => b.addEventListener("click", () => { + document.querySelectorAll(".chip[data-range]").forEach((x) => x.classList.remove("is-on")); + b.classList.add("is-on"); + state.runWindow = Number(b.dataset.range); + renderAll(); + })); + document.querySelectorAll("input[data-lane]").forEach((c) => c.addEventListener("change", () => { + if (c.checked) state.lanes.add(c.dataset.lane); else state.lanes.delete(c.dataset.lane); + renderAll(); + })); + $("#group").addEventListener("change", (e) => { state.group = e.target.value; renderSeries(); }); +} + +function fillGroups() { + const sel = $("#group"); + const keep = sel.value; + sel.replaceChildren(el("option", null, "all groups")); + sel.firstChild.value = ""; + for (const g of state.data.groups) { const o = el("option", null, g); o.value = g; sel.append(o); } + sel.value = state.data.groups.includes(keep) ? keep : ""; +} + +function header(title, count, sub) { + const h = el("h3"); + h.append(title); + if (count !== undefined) h.append(el("span", "count", String(count))); + if (sub) h.append(el("span", "muted", sub)); + return h; +} +function list(items, render, emptyText) { + const ul = el("ul"); + if (!items.length) { if (emptyText) ul.append(el("li", "empty", emptyText)); return ul; } + for (const it of items) ul.append(render(it)); + return ul; +} + +function renderNight() { + const box = $("#night"); + box.replaceChildren(); + const d = state.data; + if (d.latest < 0) return; + const run = d.runs[d.latest]; + const card = el("div", "card"); + const rec = el("a", "msg", "run record"); + rec.href = recordUrl(run); + if (run.build.status && run.build.status !== "ok") { + card.append(header("The build failed", undefined, "nothing was measured")); + const pre = el("pre", "logtail"); + pre.textContent = run.build.log_tail || "the driver recorded no build log"; + card.append(el("div", "msg", "after " + fmtSec(run.build.seconds) + "; last lines of the build log:"), pre, rec); + box.append(card); + return; + } + card.append(header("Failures", run.failures.length)); + card.append(list(run.failures, (f) => { + const li = el("li"); + li.append(el("span", "tag tag--failure", f.status.replace("_", " ")), el("span", "tag tag--" + f.lane, f.lane), el("code", null, f.path)); + const m = el("div", "msg"); + m.append(el("code", null, f.message)); + li.append(m); + return li; + }, "every file ran")); + const notes = el("div", "msg"); + for (const lane of Object.keys(run.lanes)) if (run.lanes[lane] !== "ok") notes.append(el("div", null, lane + " lane did not run: " + run.lanes[lane])); + for (const f of run.skipped) { const line = el("div"); line.append("skipped ", el("code", null, f.path), " (" + f.lane + "): " + f.message); notes.append(line); } + card.append(notes, rec); + + box.append(card); +} + +const W = 420, H = 150, PAD = { l: 44, r: 54, t: 10, b: 22 }; + +function niceTicks(lo, hi, count) { + if (!(hi > lo)) hi = lo + 1; + const step0 = (hi - lo) / Math.max(1, count); + const mag = Math.pow(10, Math.floor(Math.log10(step0))); + const step = [1, 2, 2.5, 5, 10].map((m) => m * mag).find((s) => s >= step0) || 10 * mag; + const out = []; + for (let v = Math.floor(lo / step) * step; v <= hi + step * 0.5; v += step) out.push(+v.toPrecision(12)); + return out; +} + +function drawChart(lines, runIdx, failedRuns, label) { + const svg = svgEl("svg", { viewBox: `0 0 ${W} ${H}`, class: "chart", role: "img", "aria-label": label }); + const n = runIdx.length; + const xOf = new Map(runIdx.map((ri, i) => [ri, PAD.l + (n === 1 ? (W - PAD.l - PAD.r) / 2 : (i * (W - PAD.l - PAD.r)) / (n - 1))])); + let lo = Infinity, hi = -Infinity; + for (const ln of lines) for (const p of ln.points) if (xOf.has(p.r)) { lo = Math.min(lo, p.v); hi = Math.max(hi, p.v * (1 + p.s)); } + if (!isFinite(lo)) { lo = 0; hi = 1; } + const pad = (hi - lo) * 0.12 || hi * 0.1 || 1; + lo = Math.max(0, Math.min(lo, 0) - pad); hi += pad; + const ticks = niceTicks(lo, hi, 4); + lo = Math.min(lo, ticks[0]); hi = Math.max(hi, ticks[ticks.length - 1]); + const yOf = (v) => PAD.t + (H - PAD.t - PAD.b) * (1 - (v - lo) / (hi - lo)); + for (const tv of ticks) { + if (tv < lo || tv > hi) continue; + svg.append(svgEl("line", { class: "grid-line", x1: PAD.l, x2: W - PAD.r, y1: yOf(tv), y2: yOf(tv) })); + const t = svgEl("text", { x: PAD.l - 6, y: yOf(tv) + 3.5, "text-anchor": "end" }); + t.textContent = fmtNs(tv); + svg.append(t); + } + const runs = state.data.runs; + const labelEvery = Math.max(1, Math.ceil(n / 5)); + runIdx.forEach((ri, i) => { + if (i % labelEvery !== 0 && i !== n - 1) return; + const t = svgEl("text", { x: xOf.get(ri), y: H - 6, "text-anchor": i === n - 1 ? "end" : i === 0 ? "start" : "middle" }); + t.textContent = dateOf(runs[ri].commit.date).slice(5); + svg.append(t); + }); + for (const ri of failedRuns) if (xOf.has(ri)) svg.append(svgEl("rect", { class: "fail-mark", x: xOf.get(ri) - 2, y: H - PAD.b - 5, width: 4, height: 5 })); + for (const ln of lines) { + const pts = ln.points.filter((p) => xOf.has(p.r)); + if (!pts.length) continue; + let band = ""; + for (const p of pts) band += (band ? "L" : "M") + xOf.get(p.r).toFixed(1) + "," + yOf(p.v * (1 + p.s)).toFixed(1); + for (let i = pts.length - 1; i >= 0; i--) band += "L" + xOf.get(pts[i].r).toFixed(1) + "," + yOf(pts[i].v).toFixed(1); + svg.append(svgEl("path", { class: "band band--" + ln.lane, d: band + "Z" })); + let d = "", prev = -2; + for (const p of pts) { + const at = runIdx.indexOf(p.r); + d += (at - prev > 1 ? "M" : "L") + xOf.get(p.r).toFixed(1) + "," + yOf(p.v).toFixed(1); + prev = at; + } + svg.append(svgEl("path", { class: "series series--" + ln.lane, d })); + if (pts.length === 1) svg.append(svgEl("circle", { class: "marker marker--" + ln.lane, cx: xOf.get(pts[0].r), cy: yOf(pts[0].v), r: 4 })); + const last = pts[pts.length - 1]; + const lbl = svgEl("text", { class: "end-label", x: W - PAD.r + 6, y: yOf(last.v) + 3.5 }); + lbl.textContent = fmtNs(last.v); + svg.append(lbl); + } + attachHover(svg, lines, runIdx, failedRuns, xOf, yOf); + return svg; +} + +function attachHover(svg, lines, runIdx, failedRuns, xOf, yOf) { + const runs = state.data.runs; + const cross = svgEl("line", { class: "crosshair", x1: 0, x2: 0, y1: PAD.t, y2: H - PAD.b, visibility: "hidden" }); + const markers = lines.map((ln) => svgEl("circle", { class: "marker marker--" + ln.lane, r: 4, visibility: "hidden" })); + const hit = svgEl("rect", { class: "hit", x: PAD.l - 10, y: 0, width: W - PAD.l - PAD.r + 20, height: H }); + svg.append(cross, ...markers, hit); + const tooltip = $("#tooltip"); + const nearestRun = (evt) => { + const rect = svg.getBoundingClientRect(); + const px = ((evt.clientX - rect.left) / rect.width) * W; + let best = -1, bestD = Infinity; + for (const ri of runIdx) { const dx = Math.abs(xOf.get(ri) - px); if (dx < bestD) { bestD = dx; best = ri; } } + return best; + }; + hit.addEventListener("pointermove", (evt) => { + const best = nearestRun(evt); + if (best < 0) return; + const x = xOf.get(best); + cross.setAttribute("x1", x); cross.setAttribute("x2", x); cross.setAttribute("visibility", "visible"); + const run = runs[best]; + tooltip.replaceChildren(el("div", "tt-head", dateOf(run.commit.date) + " " + shortSha(run.commit.sha) + " " + run.commit.subject)); + lines.forEach((ln, i) => { + const p = ln.points.find((q) => q.r === best); + if (!p) { markers[i].setAttribute("visibility", "hidden"); return; } + markers[i].setAttribute("cx", x); markers[i].setAttribute("cy", yOf(p.v)); markers[i].setAttribute("visibility", "visible"); + const row = el("div", "tt-row"); + row.append(el("span", "tt-key tt-key--" + ln.lane), el("span", "tt-val", fmtNs(p.v)), el("span", null, ln.lane), el("span", "tt-sub", "spread " + (p.s * 100).toFixed(1) + "%")); + tooltip.append(row); + }); + if (failedRuns.has(best)) tooltip.append(el("div", "tt-sub", "this file failed that night")); + tooltip.hidden = false; + const tw = tooltip.offsetWidth, th = tooltip.offsetHeight; + tooltip.style.left = (evt.clientX + 14 + tw > window.innerWidth - 8 ? evt.clientX - tw - 14 : evt.clientX + 14) + "px"; + tooltip.style.top = (evt.clientY + 14 + th > window.innerHeight - 8 ? evt.clientY - th - 14 : evt.clientY + 14) + "px"; + }); + hit.addEventListener("pointerleave", () => { + cross.setAttribute("visibility", "hidden"); + markers.forEach((m) => m.setAttribute("visibility", "hidden")); + tooltip.hidden = true; + }); + hit.addEventListener("click", (evt) => { + const best = nearestRun(evt); + const url = best >= 0 ? commitUrl(runs[best].commit.sha) : null; + if (url) window.open(url, "_blank", "noopener"); + }); +} + +function chartCard(id, file, lines, runIdx, failedRuns) { + const card = el("div", "card chart-card"); + card.id = anchorId(id); + const head = el("div", "chart-card__head"); + const ttl = el("div", "chart-card__title"); + ttl.append(el("span", "file", file + " "), id.slice(file.length + 1)); + head.append(ttl); + card.append(head, drawChart(lines, runIdx, failedRuns, id)); + if (lines.length > 1) { + const lg = el("div", "legend"); + for (const ln of lines) lg.append(el("span", "legend--" + ln.lane, ln.lane)); + card.append(lg); + } + return card; +} + +function renderSeries() { + const box = $("#series"); + box.replaceChildren(); + const d = state.data; + const runIdx = visibleRunIndices(); + const visible = new Set(runIdx); + const failedBy = new Map(); + d.runs.forEach((run, ri) => { + for (const f of run.failures) { + const key = f.lane + "\t" + f.path.replace(/\.das$/, ""); + if (!failedBy.has(key)) failedBy.set(key, new Set()); + failedBy.get(key).add(ri); + } + }); + const arms = new Map(); + for (const s of d.series) { + if (!state.lanes.has(s.lane) || (state.group && s.group !== state.group) || !s.runs.some((r) => visible.has(r))) continue; + if (!arms.has(s.id)) arms.set(s.id, { group: s.group, file: s.file, lines: [] }); + const arm = arms.get(s.id); + arm.lines.push({ lane: s.lane, points: s.runs.map((r, i) => ({ r, v: s.ns[i], s: s.spread[i] })) }); + } + $("#series-count").textContent = arms.size + " of " + new Set(d.series.map((s) => s.id)).size + " arms"; + if (!arms.size) { box.append(el("p", "notice", "Nothing matches the filters.")); return; } + const byGroup = new Map(); + for (const [id, arm] of arms) { if (!byGroup.has(arm.group)) byGroup.set(arm.group, []); byGroup.get(arm.group).push([id, arm]); } + for (const [group, entries] of [...byGroup.entries()].sort((a, b) => a[0].localeCompare(b[0]))) { + const section = el("div", "group"); + const h = el("h3"); + h.append(group, el("span", "muted", entries.length + " arms")); + const grid = el("div", "grid"); + for (const [id, arm] of entries.sort((a, b) => a[0].localeCompare(b[0]))) { + arm.lines.sort((a, b) => LANES.indexOf(a.lane) - LANES.indexOf(b.lane)); + const failedRuns = new Set(); + for (const ln of arm.lines) for (const ri of failedBy.get(ln.lane + "\t" + arm.file) || []) failedRuns.add(ri); + grid.append(chartCard(id, arm.file, arm.lines, runIdx, failedRuns)); + } + section.append(h, grid); + box.append(section); + } +} + +function renderRuns() { + const table = $("#runs"); + table.replaceChildren(); + const head = el("tr"); + for (const [txt, cls] of [["date", ""], ["commit", ""], ["status", ""], ["lanes", ""], ["build", "num"], ["suite", "num"], ["ok", "num"], ["failed", "num"], ["record", ""]]) head.append(el("th", cls, txt)); + table.append(head); + for (const ri of visibleRunIndices().reverse()) { + const run = state.data.runs[ri]; + const tr = el("tr"); + const c = el("td"); + const a = el("a", null, shortSha(run.commit.sha)); + a.href = commitUrl(run.commit.sha) || "#"; + c.append(a, " ", el("span", "muted", run.commit.subject.length > 60 ? run.commit.subject.slice(0, 60) + "..." : run.commit.subject)); + const s = el("td"); + s.append(el("span", "pill pill--" + run.status, run.status.replace("_", " "))); + const lanes = el("td"); + for (const lane of Object.keys(run.lanes)) { + const t = el("span", "tag tag--" + lane, lane); + if (run.lanes[lane] !== "ok") { t.textContent = lane + " off"; t.title = run.lanes[lane]; t.className = "tag"; } + lanes.append(t, " "); + } + const rec = el("td"); + const link = el("a", null, "json"); + link.href = recordUrl(run); + rec.append(link); + tr.append(el("td", null, dateOf(run.started)), c, s, lanes, el("td", "num", fmtSec(run.build.seconds)), el("td", "num", fmtSec(run.seconds)), + el("td", "num", String(run.files_ok)), el("td", "num", String(run.failures.length)), rec); + table.append(tr); + } +} + +function renderAll() { + fillGroups(); + renderNight(); + renderSeries(); + renderRuns(); +} + +wireFilters(); +load(); diff --git a/utils/internal/bench-stand/site/index.html b/utils/internal/bench-stand/site/index.html new file mode 100644 index 0000000000..54d602a1b1 --- /dev/null +++ b/utils/internal/bench-stand/site/index.html @@ -0,0 +1,39 @@ + + + + + +daslang benchmark stand + + + + + +
+
+
+ + + +
+
+ + +
+ +
+
+

Benchmarks

+

Run history

+
+
+ Every point is the minimum ns/op over the night's repeats of a dastest --bench run; the band is the spread. Source: utils/internal/bench-stand. + +
+ + + + diff --git a/utils/internal/bench-stand/site/style.css b/utils/internal/bench-stand/site/style.css new file mode 100644 index 0000000000..a5ee5e63e1 --- /dev/null +++ b/utils/internal/bench-stand/site/style.css @@ -0,0 +1,157 @@ +:root { + color-scheme: light; + --page: #f9f9f7; + --surface: #fcfcfb; + --ink: #0b0b0b; + --ink-2: #52514e; + --muted: #898781; + --grid: #e1e0d9; + --axis: #c3c2b7; + --border: rgba(11, 11, 11, 0.10); + --interp: #2a78d6; + --jit: #eb6834; + --good: #0ca30c; + --good-text: #006300; + --warning: #fab219; + --serious: #ec835a; + --critical: #d03b3b; + --chip-on: #e8eef9; +} +@media (prefers-color-scheme: dark) { + :root { + color-scheme: dark; + --page: #0d0d0d; + --surface: #1a1a19; + --ink: #ffffff; + --ink-2: #c3c2b7; + --muted: #898781; + --grid: #2c2c2a; + --axis: #383835; + --border: rgba(255, 255, 255, 0.10); + --interp: #3987e5; + --jit: #d95926; + --good-text: #0ca30c; + --chip-on: #22314a; + } +} + +/* One colour per lane, read by every lane-marked element: chips, tags, legend, lines, bands, markers, tooltip keys. */ +.lane--interp, .tag--interp, .legend--interp, .series--interp, .band--interp, .marker--interp, .tt-key--interp { --lane: var(--interp); } +.lane--jit, .tag--jit, .legend--jit, .series--jit, .band--jit, .marker--jit, .tt-key--jit { --lane: var(--jit); } +* { box-sizing: border-box; } +html, body { margin: 0; } +body { + background: var(--page); + color: var(--ink); + font: 14px/1.45 system-ui, -apple-system, "Segoe UI", sans-serif; +} +a { color: inherit; } +code { font: 12px/1.4 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; } +h1, h2, h3 { font-weight: 600; margin: 0; } +h2 { font-size: 16px; margin: 0 0 12px; } +.muted { color: var(--muted); font-weight: 400; font-size: 13px; } + +.top { + display: flex; flex-wrap: wrap; align-items: baseline; justify-content: space-between; gap: 12px; + padding: 18px 24px 8px; +} +.top__title { display: flex; align-items: baseline; gap: 10px; } +.top__brand { text-decoration: none; color: var(--ink-2); font-weight: 600; } +.top__title h1 { font-size: 20px; } +.top__status { display: flex; flex-wrap: wrap; gap: 10px; align-items: center; color: var(--ink-2); } + +main { padding: 0 24px 32px; max-width: 1500px; } +.section { margin-top: 28px; } + +.filters { + display: flex; flex-wrap: wrap; gap: 18px; align-items: center; + padding: 10px 0 14px; border-bottom: 1px solid var(--grid); +} +.filters__group { display: flex; flex-wrap: wrap; gap: 6px; align-items: center; } +.chip { + background: var(--surface); color: var(--ink-2); border: 1px solid var(--border); + border-radius: 999px; padding: 4px 10px; cursor: pointer; font: inherit; font-size: 13px; +} +.chip.is-on { background: var(--chip-on); color: var(--ink); border-color: transparent; font-weight: 600; } +.lane { display: inline-flex; align-items: center; gap: 6px; cursor: pointer; padding: 4px 8px; } +.lane input { accent-color: var(--lane, var(--ink-2)); } + +.pill { + display: inline-flex; align-items: center; gap: 6px; padding: 2px 10px; border-radius: 999px; + font-size: 13px; font-weight: 600; background: var(--surface); border: 1px solid var(--border); +} +.pill::before { content: ""; width: 8px; height: 8px; border-radius: 50%; background: var(--muted); } +.pill--ok::before { background: var(--good); } +.pill--bench_failed::before, .pill--build_failed::before { background: var(--critical); } +.pill--running::before { background: var(--warning); } +.pill--stale::before { background: var(--serious); } + +.night { margin-top: 18px; display: grid; gap: 14px; grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); } +.card { + background: var(--surface); border: 1px solid var(--border); border-radius: 10px; padding: 12px 14px; +} +.card h3 { font-size: 14px; margin-bottom: 8px; display: flex; gap: 8px; align-items: baseline; } +.card h3 .count { color: var(--muted); font-weight: 400; } +.card ul { margin: 0; padding: 0; list-style: none; } +.card li { padding: 4px 0; border-top: 1px solid var(--grid); display: flex; gap: 8px; flex-wrap: wrap; align-items: baseline; } +.card li:first-child { border-top: 0; } +.card .empty { color: var(--muted); } +.tag { font-size: 12px; padding: 0 6px; border-radius: 4px; border: 1px solid var(--border); color: var(--ink-2); } +.tag--interp, .tag--jit { color: var(--lane); border-color: var(--lane); } +.tag--failure { color: var(--critical); border-color: var(--critical); } +.msg { color: var(--ink-2); font-size: 13px; } +.msg code { white-space: pre-wrap; word-break: break-word; } +pre.logtail { + margin: 6px 0 0; padding: 8px 10px; max-height: 220px; overflow: auto; + background: var(--page); border: 1px solid var(--grid); border-radius: 6px; + font: 12px/1.45 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; color: var(--ink-2); + white-space: pre-wrap; word-break: break-word; +} + +.grid { display: grid; gap: 14px; grid-template-columns: repeat(auto-fill, minmax(360px, 1fr)); } +.group { margin-top: 18px; } +.group > h3 { font-size: 15px; margin: 0 0 8px; display: flex; gap: 10px; align-items: baseline; } +.group > h3 .muted { font-size: 13px; } + +.chart-card { padding: 10px 12px 6px; } +.chart-card__head { display: flex; gap: 8px; align-items: baseline; flex-wrap: wrap; } +.chart-card__title { font-size: 13px; font-weight: 600; word-break: break-all; } +.chart-card__title .file { color: var(--muted); font-weight: 400; } +.legend { display: flex; gap: 12px; font-size: 12px; color: var(--ink-2); margin: 4px 0 0; } +.legend span::before { content: ""; display: inline-block; width: 14px; height: 2px; vertical-align: middle; margin-right: 5px; border-radius: 1px; background: var(--lane, var(--muted)); } +svg.chart { display: block; width: 100%; height: auto; overflow: visible; touch-action: pan-y; } +svg.chart text { fill: var(--muted); font-size: 11px; font-variant-numeric: tabular-nums; } +svg.chart .grid-line { stroke: var(--grid); stroke-width: 1; } +svg.chart .axis-line { stroke: var(--axis); stroke-width: 1; } +svg.chart .series { fill: none; stroke: var(--lane); stroke-width: 2; stroke-linejoin: round; stroke-linecap: round; } +svg.chart .band { fill: var(--lane); opacity: 0.10; } +svg.chart .end-label { fill: var(--ink-2); font-weight: 600; } +svg.chart .crosshair { stroke: var(--axis); stroke-width: 1; } +svg.chart .marker { fill: var(--lane); stroke: var(--surface); stroke-width: 2; } +svg.chart .fail-mark { fill: var(--critical); } +svg.chart .hit { fill: transparent; cursor: crosshair; } + +.table-wrap { overflow-x: auto; } +table { border-collapse: collapse; width: 100%; font-size: 13px; } +th, td { text-align: left; padding: 6px 8px; border-top: 1px solid var(--grid); vertical-align: top; } +th { color: var(--muted); font-weight: 500; border-top: 0; } +td.num, th.num { text-align: right; font-variant-numeric: tabular-nums; } + +.tooltip { + position: fixed; z-index: 10; pointer-events: none; max-width: 360px; + background: var(--surface); color: var(--ink); border: 1px solid var(--border); + border-radius: 8px; padding: 8px 10px; box-shadow: 0 6px 24px rgba(0, 0, 0, 0.12); font-size: 12px; +} +.tooltip .tt-head { color: var(--ink-2); margin-bottom: 4px; } +.tooltip .tt-row { display: flex; gap: 8px; align-items: baseline; } +.tooltip .tt-key { display: inline-block; width: 14px; height: 2px; border-radius: 1px; background: var(--lane, var(--muted)); } +.tooltip .tt-val { font-weight: 600; font-variant-numeric: tabular-nums; } +.tooltip .tt-sub { color: var(--muted); } + +.foot { display: flex; flex-wrap: wrap; justify-content: space-between; gap: 12px; padding: 16px 24px 28px; color: var(--muted); font-size: 12px; border-top: 1px solid var(--grid); } +.notice { padding: 16px; color: var(--ink-2); } +.notice--error { color: var(--critical); } +.select select { + font: inherit; font-size: 13px; color: var(--ink); background: var(--surface); + border: 1px solid var(--border); border-radius: 6px; padding: 4px 8px; +}
+
daslang

benchmark stand

+
+