the nightly benchmark stand: one box, every [benchmark] in the tree, plotted over commits

CMakeLists.txt

@@ -2074,6 +2074,7 @@ install(FILES ${DAS_DASCOV_TEST_FILES} DESTINATION utils/dascov/tests)
 file(GLOB DAS_BENCHCTL_FILES ${PROJECT_SOURCE_DIR}/utils/benchctl/*.das)
 install(FILES ${DAS_BENCHCTL_FILES} DESTINATION utils/benchctl)
 install(FILES ${PROJECT_SOURCE_DIR}/utils/benchctl/README.md DESTINATION utils/benchctl)
+install(FILES ${PROJECT_SOURCE_DIR}/utils/benchctl/suite.json DESTINATION utils/benchctl)
 
 # Install lint (unified linter) + its rule-coverage suite, so a shipped SDK can
 # self-check the linter the same way run_utils_tests does in-tree.

benchmarks/README.md

@@ -4,6 +4,12 @@
 
 Every `.das` benchmark file in this directory tree is listed below, grouped by subdirectory.
 
+## compile/
+
+| File | Description |
+|---|---|
+| `utils.das` | Compile-only wall time of every tool under `utils/` the nightly build can compile, one `[benchmark]` per tool, each a child `daslang -compile-only -no-module-cache <entry>` - interp lane only, since the child is the same binary whatever lane the parent runs in |
+
 ## terminal/
 
 | File | Description |

benchmarks/compile/utils.das

@@ -0,0 +1,149 @@
+options gen2
+
+require dastest/testing_boost
+require daslib/clargs
+require daslib/fio
+require daslib/strings_boost
+require strings
+
+def private compile_tool(b : B?; entry : string) {
+    let bin = get_host_binary()
+    let path = path_join(path_join(get_das_root(), "utils"), entry)
+    let cmd = "\"{bin}\" -compile-only -no-module-cache \"{path}\" 2>&1"
+    var rc = 0
+    var first_error = ""
+    b |> run(entry) {
+        first_error = ""
+        rc = unsafe(popen_timeout(cmd, 600.0) $(f) {
+            if (f != null) {
+                while (!feof(f)) {
+                    let ln = fgets(f)
+                    if (empty(first_error) && find(ln, "error[") >= 0) {
+                        first_error = strip(ln)
+                    }
+                }
+            }
+        })
+    }
+    b |> equal(rc, 0, "{entry}: daslang -compile-only exited with {rc}{empty(first_error) ? "" : " - " + first_error}")
+}
+
+[benchmark]
+def compile_aot(b : B?) {
+    compile_tool(b, "aot/main.das")
+}
+
+[benchmark]
+def compile_benchctl(b : B?) {
+    compile_tool(b, "benchctl/main.das")
+}
+
+[benchmark]
+def compile_dascov(b : B?) {
+    compile_tool(b, "dascov/main.das")
+}
+
+[benchmark]
+def compile_daspkg(b : B?) {
+    compile_tool(b, "daspkg/main.das")
+}
+
+[benchmark]
+def compile_detect_dupe(b : B?) {
+    compile_tool(b, "detect-dupe/main.das")
+}
+
+[benchmark]
+def compile_fix_lint_errors(b : B?) {
+    compile_tool(b, "fix-lint-errors/main.das")
+}
+
+[benchmark]
+def compile_gen1_to_gen2(b : B?) {
+    compile_tool(b, "gen1-to-gen2/main.das")
+}
+
+[benchmark]
+def compile_lint(b : B?) {
+    compile_tool(b, "lint/main.das")
+}
+
+[benchmark]
+def compile_watchdog(b : B?) {
+    compile_tool(b, "watchdog/main.das")
+}
+
+[benchmark]
+def compile_das_fmt(b : B?) {
+    compile_tool(b, "das-fmt/dasfmt.das")
+}
+
+[benchmark]
+def compile_arch_extract(b : B?) {
+    compile_tool(b, "internal/arch-extract/main.das")
+}
+
+[benchmark]
+def compile_ast_fuzz(b : B?) {
+    compile_tool(b, "internal/ast-fuzz/main.das")
+}
+
+[benchmark]
+def compile_dasweb_verify(b : B?) {
+    compile_tool(b, "internal/dasweb-verify/main.das")
+}
+
+[benchmark]
+def compile_doc_verify(b : B?) {
+    compile_tool(b, "internal/doc-verify/main.das")
+}
+
+[benchmark]
+def compile_flatten_fuzz(b : B?) {
+    compile_tool(b, "internal/flatten-fuzz/main.das")
+}
+
+[benchmark]
+def compile_hygiene(b : B?) {
+    compile_tool(b, "internal/hygiene/main.das")
+}
+
+[benchmark]
+def compile_jit(b : B?) {
+    compile_tool(b, "internal/jit/main.das")
+}
+
+[benchmark]
+def compile_lineinfo_audit(b : B?) {
+    compile_tool(b, "internal/lineinfo-audit/main.das")
+}
+
+[benchmark]
+def compile_make_pr(b : B?) {
+    compile_tool(b, "internal/make-pr/main.das")
+}
+
+[benchmark]
+def compile_pr_babysit(b : B?) {
+    compile_tool(b, "internal/pr-babysit/main.das")
+}
+
+[benchmark]
+def compile_preflight(b : B?) {
+    compile_tool(b, "internal/preflight/main.das")
+}
+
+[benchmark]
+def compile_requirefix(b : B?) {
+    compile_tool(b, "internal/requirefix/main.das")
+}
+
+[benchmark]
+def compile_review_md(b : B?) {
+    compile_tool(b, "internal/review-md/main.das")
+}
+
+[benchmark]
+def compile_test_release(b : B?) {
+    compile_tool(b, "internal/test-release/main.das")
+}

benchmarks/decs/bench_from_decs_count.das

@@ -20,7 +20,7 @@ struct BenchCountRow {
 
 def fixture(n : int) {
     restart()
-    create_entities(n) $(eid : EntityId; i : int; var cmp : ComponentMap) {
+    create_entities(n) $(_eid : EntityId; i : int; var cmp : ComponentMap) {
         apply_decs_template(cmp, BenchCountRow(val = i))
     }
 }
@@ -32,7 +32,7 @@ def from_decs_count_m1_hand(b : B?) {
     b |> run("m1_hand_arch_size/{N}", N) {
         var erq : EcsRequest
         erq.req |> push("bench_val")
-        var total = 0
+        var total = 0l
         for_each_archetype(erq) $(arch : Archetype) {
             total += arch.size
         }

daslib/clargs.das

@@ -74,6 +74,12 @@ def public get_user_args() : array<string> {
     }
 }
 
+//! The running host binary as an absolute path - `argv[0]` resolved - for a program that spawns
+//! the same daslang it runs under.
+def public get_host_binary() : string {
+    return get_full_file_name(get_command_line_arguments()[0])
+}
+
 def clargs_flag_name(field_name : string) : string {
     return "--" + replace_multiple(field_name, [(text="_", replacement="-")])
 }

skills/daslang/references/cli-and-config.md

@@ -30,7 +30,9 @@ def main : int {
 
 **Don't pick an argv accessor.** `parse_args` pulls argv through `get_user_args()`: `argv[1..]` for
 a standalone `-exe` binary, the post-`--` slice under the interpreter or the JIT.
-`get_program_args()` / `get_cli_arguments()` force one slice regardless of host.
+`get_program_args()` / `get_cli_arguments()` force one slice regardless of host. A program that
+spawns the daslang it runs under takes the path from `get_host_binary()` - `argv[0]` resolved to an
+absolute path - never from `get_command_line_arguments()` itself.
 
 Field types: `string`, `int`, `float`, `bool`, an enum (clargs validates the value), `array<T>` for
 a repeatable flag. The long name is the field name with underscores as hyphens; clargs adds the

skills/writing_benchmarks.md

@@ -14,7 +14,9 @@ bin/daslang -jit dastest/dastest.das -- --bench --test path/to/directory/
 - `-jit` goes **before** `dastest.das`: it puts dastest itself in JIT mode, which is what
   gets the benchmark code JIT-compiled. Use it for any performance number you intend to
   believe
-- `--bench-names name1,name2` - run only those benchmark functions
+- `--bench-names prefix` - run only the benchmark functions whose name starts with `prefix`;
+  repeat the flag for several (`--bench-names a --bench-names b`) - a comma-joined list is one
+  prefix that matches nothing
 
 Dropping `--bench` turns the same command into a fast compile check: it reports 0 tests but
 surfaces every compile error.

utils/CMakeLists.txt

@@ -232,6 +232,7 @@ SET(DAS_UTILS_TO_TEST
     internal/hygiene
     internal/preflight/tests
     internal/ast-fuzz
+    benchctl/tests
     internal/requirefix
     internal/test-release
     daspkg/test_daspkg.das

utils/benchctl/REVIEW.md

@@ -2,3 +2,45 @@
 
 **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc:
 `README.md`.
+
+**Never put a `[test]` file outside `tests/`, and never let a test touch the filesystem outside a
+`temp_directory`-rooted path or leave behind what it creates.**
+
+**A module file orders its top level types, then `private` helpers, then its public functions, so
+the tail of the file is the module's whole API.**
+
+**Never add a run-record field without saying in `README.md` what reads it - the viewer, or a
+person opening the record.** A field nobody named is one nobody notices going wrong.
+
+**A diff that changes a run-record field keeps the new reader parsing a record written by the old
+code, a missing field keeping its declared default.** Records already on the box are never
+rewritten.
+
+**Never let a child's own output overwrite a `timeout` status in `run_bench_file` - a killed child
+that printed a passing report is still killed.** Loosen `timeout_seconds` in `suite.json` instead.
+
+**Never read a benchmark's identity from anywhere but its path under `benchmarks/`** - the group
+is the directory, the id is the path without `.das`.
+
+**Statistics live in `benchstat.das` and nowhere else** - it depends on no storage, so a second
+median or spread helper anywhere in this folder is a defect.
+
+**Placement - one file, one line: a diff keeps each file inside its line, and a new file adds its
+line here, with its tests, in the same change.**
+
+- `main.das` - argv, subcommand dispatch, exit codes.
+- `bench_args.das` - the database verbs' flags.
+- `bench_table.das` - the stored row and its mapping to the shape the statistics read.
+- `benchstat.das` - every statistic: median, spread, outlier filtering, geomean, Welch. No storage.
+- `bench_suite.das` - `suite.json` and file discovery. No processes.
+- `suite.json` - what is benchmarked, in which lanes, with which limits.
+- `bench_runner.das` - one file in one lane: spawn, timeout, output to samples and verdict.
+- `_jit_probe.das` - the program `probe_jit` runs to prove the jit lane works.
+- `bench_history.das` - run records to dataset. No processes, no argv, no statistics.
+- `bench_stand.das` - the `run` and `report` verbs the nightly box drives.
+- `table_fmt.das`, `utils.das` - output formatting and small shared helpers.
+- `tests/test_bench_suite.das`, `tests/test_bench_runner.das`, `tests/test_bench_history.das` -
+  the module suites; `tests/test_bench_cli.das` - the verbs, spawned; `tests/_test_common.das` -
+  the fixtures they share; `tests/_fake_dastest.das` - the stand-in dastest a spawn test measures
+  against.
+

utils/benchctl/_jit_probe.das

@@ -0,0 +1,6 @@
+options gen2
+
+[export]
+def main() {
+    print("jit-probe-ok\n")
+}