From e434a6d47f1f87f3d260742385656585992572bc Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 15:19:30 -0400 Subject: [PATCH 01/10] SQLite tests: run all-mode child jobs on Kandelo --- apps/browser-demos/pages/sqlite-test/main.ts | 100 ++++++- scripts/run-browser-sqlite-official-tests.sh | 80 ++++++ scripts/run-sqlite-official-tests.sh | 259 ++++++++++++++++++- 3 files changed, 430 insertions(+), 9 deletions(-) diff --git a/apps/browser-demos/pages/sqlite-test/main.ts b/apps/browser-demos/pages/sqlite-test/main.ts index 85bdab7f88..64d9858140 100644 --- a/apps/browser-demos/pages/sqlite-test/main.ts +++ b/apps/browser-demos/pages/sqlite-test/main.ts @@ -91,6 +91,98 @@ async function collectArtifactsFromKernel( return artifacts.length > 0 ? artifacts : undefined; } +const testrunnerPlatformShim = [ + "# Kandelo platform shim for child testrunner jobs.", + "# SQLite all-mode reruns config variants by invoking test/testrunner.tcl", + "# directly, so the platform override has to live in that file too.", + "set ::tcl_platform(os) OpenBSD", + "set ::tcl_platform(platform) unix", +].join("\n"); + +const testrunnerGuestPathShim = [ + "# Kandelo guest path shim for all-mode child jobs.", + "# testrunner.tcl builds child run.sh files from host-normalized paths;", + "# convert workdir-local paths back to paths relative to each testdirN", + "# directory, because SQLite runs the script after cd-ing into it.", + "proc kandelo_guest_path {path} {", + " if {[file pathtype $path] != \"absolute\" && [string equal $path [info nameofexec]]} {", + " return $path", + " }", + " set normalized [file normalize $path]", + " set topdir [file normalize [file dirname $::testdir]]", + " set script [file normalize [info script]]", + " if {[string equal $normalized $script]} { return \"../test/testrunner.tcl\" }", + " if {[string equal $normalized $topdir]} { return \"..\" }", + " set prefix \"${topdir}/\"", + " if {[string first $prefix $normalized] == 0} {", + " return \"../[string range $normalized [string length $prefix] end]\"", + " }", + " return $path", + "}", + "set ::kandelo_inline_run_sh 1", +].join("\n"); + +function patchTestrunnerForKandelo(runner: string): string { + let patched = runner; + + if (!patched.includes("Kandelo platform shim for child testrunner jobs")) { + const lines = patched.split("\n"); + lines.splice(3, 0, "", testrunnerPlatformShim); + patched = lines.join("\n"); + } + + if (!patched.includes("Kandelo guest path shim for all-mode child jobs")) { + patched = patched.replace("cd $dir\n", `cd $dir\n\n${testrunnerGuestPathShim}\n`); + patched = patched.replace( + " set displayname [string map [list $topdir/ {}] $f]\n", + [ + " set displayname [string map [list $topdir/ {}] $f]", + " set testfixture_guest [kandelo_guest_path $testfixture]", + " set testrunner_tcl_guest [kandelo_guest_path $testrunner_tcl]", + " set f_guest [kandelo_guest_path $f]", + "", + ].join("\n"), + ); + patched = patched + .replace(" set cmd \"$testfixture $f\"", " set cmd \"$testfixture_guest $f_guest\"") + .replace( + " set cmd \"$testfixture $testrunner_tcl $config $f\"", + " set cmd \"$testfixture_guest $testrunner_tcl_guest $config $f_guest\"", + ) + .replace( + " set set_tmp_dir \"export SQLITE_TMPDIR=\\\"[file normalize $dir]\\\"\"", + " set set_tmp_dir \"export SQLITE_TMPDIR=.\"", + ) + .replace( + " set fd [open \"|$TRG(runcmd) 2>@1\" r]", + [ + " if {[info exists ::kandelo_inline_run_sh] && $::kandelo_inline_run_sh} {", + " set inline_cmd \"$set_tmp_dir\\n$job(cmd)\"", + " set fd [open \"|sh -c [list $inline_cmd] 2>@1\" r]", + " } else {", + " set fd [open \"|$TRG(runcmd) 2>@1\" r]", + " }", + ].join("\n"), + ); + } + + return patched; +} + +function installTestrunnerPatches(fs: MemoryFileSystem): void { + const runnerPath = "/sqlite/test/testrunner.tcl"; + const decoder = new TextDecoder(); + const runner = decoder.decode(readVfsFile(fs, runnerPath)); + writeVfsFile(fs, runnerPath, patchTestrunnerForKandelo(runner), 0o644); + + writeVfsFile(fs, "/sqlite/kandelo-testrunner.tcl", [ + testrunnerPlatformShim, + "set argv0 test/testrunner.tcl", + "source $argv0", + "", + ].join("\n"), 0o644); +} + function createFs(): MemoryFileSystem { if (!vfsImageBytes) throw new Error("SQLite test VFS image not loaded"); const fs = MemoryFileSystem.fromImage(vfsImageBytes, { @@ -153,13 +245,7 @@ async function init() { // SharedArrayBuffer across the per-test loop (Safari OOM fix). const buildFs = createFs(); if (argv[1] === "kandelo-testrunner.tcl") { - writeVfsFile(buildFs, "/sqlite/kandelo-testrunner.tcl", [ - "set ::tcl_platform(os) OpenBSD", - "set ::tcl_platform(platform) unix", - "set argv0 test/testrunner.tcl", - "source $argv0", - "", - ].join("\n"), 0o644); + installTestrunnerPatches(buildFs); } const vfsImage = await finalizeKernelOwnedImage(buildFs); const kernel = new BrowserKernel({ diff --git a/scripts/run-browser-sqlite-official-tests.sh b/scripts/run-browser-sqlite-official-tests.sh index 1103e76d70..9e35b17317 100755 --- a/scripts/run-browser-sqlite-official-tests.sh +++ b/scripts/run-browser-sqlite-official-tests.sh @@ -69,12 +69,89 @@ if [ -z "$RESULTS_DIR" ]; then fi mkdir -p "$RESULTS_DIR" +write_unavailable_outcome_lists() { + local reason="$1" + local out="$RESULTS_DIR/outcome-lists" + + mkdir -p "$out" + printf 'jobid\tstate\tdisplaytype\tdisplayname\tcases\terrors\tms\tsource\n' > "$out/passed-jobs.tsv" + printf 'jobid\tstate\tdisplaytype\tdisplayname\tcases\terrors\tms\tsource\n' > "$out/failed-jobs.tsv" + printf 'jobid\tstate\tdisplaytype\tdisplayname\tcases\terrors\tms\treason\tsource\n' > "$out/skipped-jobs.tsv" + printf 'jobid\tstate\tdisplaytype\tdisplayname\tcases\terrors\tms\treason\tsource\n' > "$out/incomplete-jobs.tsv" + printf '\tunavailable\t\t\t0\t0\t0\t%s\trunner\n' "$reason" >> "$out/incomplete-jobs.tsv" + { + printf 'passed_jobs\tfailed_jobs\tskipped_jobs\tincomplete_jobs\tnote\n' + printf '0\t0\t0\t1\t%s\n' "$reason" + } > "$out/counts.tsv" +} + +write_outcome_lists() { + local db="$1" + local out="$RESULTS_DIR/outcome-lists" + + mkdir -p "$out" + sqlite3 -header -separator $'\t' "$db" \ + "SELECT jobid, state, displaytype, displayname, + coalesce(ntest, 0) AS cases, + coalesce(nerr, 0) AS errors, + coalesce(span, 0) AS ms, + 'testrunner.db' AS source + FROM jobs + WHERE state = 'done' + ORDER BY jobid;" > "$out/passed-jobs.tsv" + + sqlite3 -header -separator $'\t' "$db" \ + "SELECT jobid, state, displaytype, displayname, + coalesce(ntest, 0) AS cases, + coalesce(nerr, 0) AS errors, + coalesce(span, 0) AS ms, + 'testrunner.db' AS source + FROM jobs + WHERE state = 'failed' + ORDER BY jobid;" > "$out/failed-jobs.tsv" + + sqlite3 -header -separator $'\t' "$db" \ + "SELECT jobid, state, displaytype, displayname, + coalesce(ntest, 0) AS cases, + coalesce(nerr, 0) AS errors, + coalesce(span, 0) AS ms, + 'runner omitted' AS reason, + 'testrunner.db' AS source + FROM jobs + WHERE state = 'omit' + ORDER BY jobid;" > "$out/skipped-jobs.tsv" + + sqlite3 -header -separator $'\t' "$db" \ + "SELECT jobid, state, displaytype, displayname, + coalesce(ntest, 0) AS cases, + coalesce(nerr, 0) AS errors, + coalesce(span, 0) AS ms, + CASE state + WHEN 'running' THEN 'runner exited before job completed' + WHEN 'ready' THEN 'not started before runner exit' + ELSE 'not completed before runner exit' + END AS reason, + 'testrunner.db' AS source + FROM jobs + WHERE state IN ('running', 'ready') + ORDER BY state, jobid;" > "$out/incomplete-jobs.tsv" + + sqlite3 -header -separator $'\t' "$db" \ + "SELECT sum(state='done') AS passed_jobs, + sum(state='failed') AS failed_jobs, + sum(state='omit') AS skipped_jobs, + sum(state IN ('running','ready')) AS incomplete_jobs, + 'testrunner.db' AS source + FROM jobs;" > "$out/counts.tsv" +} + write_sqlite_report() { local db="$RESULTS_DIR/testrunner.db" local report="$RESULTS_DIR/summary.txt" local failures="$RESULTS_DIR/failures.tsv" if [ ! -f "$db" ]; then echo "No testrunner.db was created at $db" > "$report" + write_unavailable_outcome_lists "No testrunner.db was created at $db." return fi @@ -93,11 +170,14 @@ write_sqlite_report() { find "$RESULTS_DIR" -maxdepth 1 -type f -name 'testrunner.*' -print | sort } > "$report" : > "$failures" + write_unavailable_outcome_lists "No usable jobs table was found in $db." echo "===== SQLite official testrunner database summary =====" cat "$report" return fi + write_outcome_lists "$db" + { echo "SQLite official testrunner summary" echo "host=browser" diff --git a/scripts/run-sqlite-official-tests.sh b/scripts/run-sqlite-official-tests.sh index a27544ad4d..b59ea58d45 100755 --- a/scripts/run-sqlite-official-tests.sh +++ b/scripts/run-sqlite-official-tests.sh @@ -12,6 +12,7 @@ SQLITE_FULL="$REPO_ROOT/packages/registry/sqlite/sqlite-full-src" TCL_INSTALL="$REPO_ROOT/packages/registry/tcl/tcl-install" TESTFIXTURE="$REPO_ROOT/packages/registry/sqlite/bin/testfixture.wasm" SQLITE3="$REPO_ROOT/packages/registry/sqlite/sqlite-install/bin/sqlite3.wasm" +GUEST_SHELL="${SQLITE_TEST_SHELL:-}" HOST="node" PERMUTATION="full" @@ -131,6 +132,31 @@ if [ ! -f "$TESTFIXTURE" ] || [ ! -f "$SQLITE3" ] || [ ! -d "$SQLITE_FULL/test" exit 1 fi +if [ -z "$GUEST_SHELL" ]; then + for candidate in \ + "$REPO_ROOT/local-binaries/programs/wasm32/sh.wasm" \ + "$REPO_ROOT/local-binaries/programs/sh.wasm" \ + "$REPO_ROOT/local-binaries/programs/wasm32/dash.wasm" \ + "$REPO_ROOT/local-binaries/programs/dash.wasm" \ + "$REPO_ROOT/binaries/programs/wasm32/sh.wasm" \ + "$REPO_ROOT/binaries/programs/sh.wasm" \ + "$REPO_ROOT/binaries/programs/wasm32/dash.wasm" \ + "$REPO_ROOT/binaries/programs/dash.wasm" \ + "$REPO_ROOT/packages/registry/dash/bin/dash.wasm" + do + if [ -f "$candidate" ]; then + GUEST_SHELL="$candidate" + break + fi + done +fi + +if [ "$PERMUTATION" = "all" ] && [ -z "$GUEST_SHELL" ]; then + echo "ERROR: SQLite all-mode config jobs require a guest /bin/sh-compatible shell." >&2 + echo "Build or fetch dash/sh, or set SQLITE_TEST_SHELL=/path/to/sh.wasm." >&2 + exit 1 +fi + if [ -z "$WORKDIR" ]; then WORKDIR="$(mktemp -d "${SQLITE_OFFICIAL_TMPDIR:-/tmp}/kandelo-sqlite-official.XXXXXX")" else @@ -143,17 +169,99 @@ if [ -z "$RESULTS_DIR" ]; then fi mkdir -p "$RESULTS_DIR" +write_unavailable_outcome_lists() { + local reason="$1" + local out="$RESULTS_DIR/outcome-lists" + + mkdir -p "$out" + printf 'jobid\tstate\tdisplaytype\tdisplayname\tcases\terrors\tms\tsource\n' > "$out/passed-jobs.tsv" + printf 'jobid\tstate\tdisplaytype\tdisplayname\tcases\terrors\tms\tsource\n' > "$out/failed-jobs.tsv" + printf 'jobid\tstate\tdisplaytype\tdisplayname\tcases\terrors\tms\treason\tsource\n' > "$out/skipped-jobs.tsv" + printf 'jobid\tstate\tdisplaytype\tdisplayname\tcases\terrors\tms\treason\tsource\n' > "$out/incomplete-jobs.tsv" + printf '\tunavailable\t\t\t0\t0\t0\t%s\trunner\n' "$reason" >> "$out/incomplete-jobs.tsv" + { + printf 'passed_jobs\tfailed_jobs\tskipped_jobs\tincomplete_jobs\tnote\n' + printf '0\t0\t0\t1\t%s\n' "$reason" + } > "$out/counts.tsv" +} + +write_outcome_lists() { + local db="$1" + local out="$RESULTS_DIR/outcome-lists" + + mkdir -p "$out" + sqlite3 -header -separator $'\t' "$db" \ + "SELECT jobid, state, displaytype, displayname, + coalesce(ntest, 0) AS cases, + coalesce(nerr, 0) AS errors, + coalesce(span, 0) AS ms, + 'testrunner.db' AS source + FROM jobs + WHERE state = 'done' AND coalesce(nerr, 0) = 0 + ORDER BY jobid;" > "$out/passed-jobs.tsv" + + sqlite3 -header -separator $'\t' "$db" \ + "SELECT jobid, state, displaytype, displayname, + coalesce(ntest, 0) AS cases, + coalesce(nerr, 0) AS errors, + coalesce(span, 0) AS ms, + 'testrunner.db' AS source + FROM jobs + WHERE state = 'failed' OR (state = 'done' AND coalesce(nerr, 0) > 0) + ORDER BY jobid;" > "$out/failed-jobs.tsv" + + sqlite3 -header -separator $'\t' "$db" \ + "SELECT jobid, state, displaytype, displayname, + coalesce(ntest, 0) AS cases, + coalesce(nerr, 0) AS errors, + coalesce(span, 0) AS ms, + 'runner omitted' AS reason, + 'testrunner.db' AS source + FROM jobs + WHERE state = 'omit' + ORDER BY jobid;" > "$out/skipped-jobs.tsv" + + sqlite3 -header -separator $'\t' "$db" \ + "SELECT jobid, state, displaytype, displayname, + coalesce(ntest, 0) AS cases, + coalesce(nerr, 0) AS errors, + coalesce(span, 0) AS ms, + CASE state + WHEN 'running' THEN 'runner exited before job completed' + WHEN 'ready' THEN 'not started before runner exit' + ELSE 'not completed before runner exit' + END AS reason, + 'testrunner.db' AS source + FROM jobs + WHERE state IN ('running', 'ready') + ORDER BY state, jobid;" > "$out/incomplete-jobs.tsv" + + sqlite3 -header -separator $'\t' "$db" \ + "SELECT coalesce(sum(state='done' AND coalesce(nerr, 0)=0), 0) AS passed_jobs, + coalesce(sum(state='failed' OR (state='done' AND coalesce(nerr, 0)>0)), 0) AS failed_jobs, + coalesce(sum(state='omit'), 0) AS skipped_jobs, + coalesce(sum(state IN ('running','ready')), 0) AS incomplete_jobs, + 'testrunner.db' AS source + FROM jobs;" > "$out/counts.tsv" +} + write_sqlite_report() { local db="$WORKDIR/testrunner.db" local report="$RESULTS_DIR/summary.txt" local failures="$RESULTS_DIR/failures.tsv" if [ ! -f "$db" ]; then echo "No testrunner.db was created at $db" > "$report" + write_unavailable_outcome_lists "No testrunner.db was created at $db." return fi mkdir -p "$RESULTS_DIR" - for artifact in testrunner.db testrunner.log testrunner_build.log; do + + # SQLite's testrunner keeps its control database in WAL mode. Checkpoint + # before copying so timeout artifacts remain self-contained after cleanup. + sqlite3 "$db" "PRAGMA wal_checkpoint(TRUNCATE);" >/dev/null 2>&1 || true + + for artifact in testrunner.db testrunner.db-wal testrunner.db-shm testrunner.log testrunner_build.log; do if [ -f "$WORKDIR/$artifact" ]; then cp "$WORKDIR/$artifact" "$RESULTS_DIR/$artifact" fi @@ -175,11 +283,14 @@ write_sqlite_report() { find "$RESULTS_DIR" -maxdepth 1 -type f -name 'testrunner.*' -print | sort } > "$report" : > "$failures" + write_unavailable_outcome_lists "No usable jobs table was found in $db." echo "===== SQLite official testrunner database summary =====" cat "$report" return fi + write_outcome_lists "$db" + { echo "SQLite official testrunner summary" echo "host=$HOST" @@ -234,6 +345,7 @@ write_sqlite_report() { coalesce(nerr, 0) AS errors, coalesce(span, 0) AS ms FROM jobs WHERE state IN ('failed', 'running', 'omit') + OR (state='done' AND coalesce(nerr, 0)>0) ORDER BY state, jobid;" } > "$report" @@ -242,12 +354,147 @@ write_sqlite_report() { coalesce(nerr, 0) AS errors, coalesce(span, 0) AS ms FROM jobs WHERE state IN ('failed', 'running', 'omit') + OR (state='done' AND coalesce(nerr, 0)>0) ORDER BY state, jobid;" > "$failures" echo "===== SQLite official testrunner database summary =====" cat "$report" } +patch_sqlite_testrunner_platform() { + local runner="$1" + local tmp + + if grep -q "Kandelo platform shim for child testrunner jobs" "$runner"; then + return + fi + + tmp="${runner}.kandelo-platform.$$" + awk ' + NR == 4 { + print "" + print "# Kandelo platform shim for child testrunner jobs." + print "# SQLite all-mode reruns config variants by invoking this file directly," + print "# so the platform override has to live in the copied runner as well as" + print "# the initial kandelo-testrunner.tcl wrapper." + print "set ::tcl_platform(os) OpenBSD" + print "set ::tcl_platform(platform) unix" + } + { print } + ' "$runner" > "$tmp" + mv "$tmp" "$runner" + chmod a+r "$runner" +} + +patch_sqlite_testrunner_guest_paths() { + local runner="$1" + local tmp + + if grep -q "Kandelo guest path shim for all-mode child jobs" "$runner"; then + return + fi + + tmp="${runner}.kandelo-paths.$$" + awk ' + { + print + if (!inserted && $0 == "cd $dir") { + print "" + print "# Kandelo guest path shim for all-mode child jobs." + print "# testrunner.tcl builds child run.sh files from host-normalized paths;" + print "# convert workdir-local paths back to paths relative to each testdirN" + print "# directory, because SQLite runs the script after cd-ing into it." + print "proc kandelo_guest_path {path} {" + print " set normalized [file normalize $path]" + print " set topdir [file normalize [file dirname $::testdir]]" + print " set exe [file normalize [info nameofexec]]" + print " set script [file normalize [info script]]" + print " if {[string equal $normalized $exe]} { return \"../testfixture.wasm\" }" + print " if {[string equal $normalized $script]} { return \"../test/testrunner.tcl\" }" + print " if {[string equal $normalized $topdir]} { return \"..\" }" + print " set prefix \"${topdir}/\"" + print " if {[string first $prefix $normalized] == 0} {" + print " return \"../[string range $normalized [string length $prefix] end]\"" + print " }" + print " return $path" + print "}" + print "set ::kandelo_inline_run_sh 1" + print "set ::kandelo_chunk_pipe_output 1" + inserted = 1 + } else if ($0 == " set displayname [string map [list $topdir/ {}] $f]") { + print " set testfixture_guest [kandelo_guest_path $testfixture]" + print " set testrunner_tcl_guest [kandelo_guest_path $testrunner_tcl]" + print " set f_guest [kandelo_guest_path $f]" + } + } + ' "$runner" > "$tmp" + mv "$tmp" "$runner" + + tmp="${runner}.kandelo-paths-subst.$$" + awk ' + $0 == " set cmd \"$testfixture $f\"" { + print " set cmd \"$testfixture_guest $f_guest\"" + next + } + $0 == " set cmd \"$testfixture $testrunner_tcl $config $f\"" { + print " set cmd \"$testfixture_guest $testrunner_tcl_guest $config $f_guest\"" + next + } + $0 == " set set_tmp_dir \"export SQLITE_TMPDIR=\\\"[file normalize $dir]\\\"\"" { + print " set set_tmp_dir \"export SQLITE_TMPDIR=.\"" + next + } + $0 == " set fd [open \"|$TRG(runcmd) 2>@1\" r]" { + print " if {[info exists ::kandelo_inline_run_sh] && $::kandelo_inline_run_sh} {" + print " set inline_cmd \"$set_tmp_dir\\n$job(cmd)\"" + print " set fd [open \"|sh -c [list $inline_cmd] 2>@1\" r]" + print " } else {" + print " set fd [open \"|$TRG(runcmd) 2>@1\" r]" + print " }" + next + } + $0 == " set rc [catch { gets $fd line } res]" { + print " if {[info exists ::kandelo_chunk_pipe_output] && $::kandelo_chunk_pipe_output} {" + print " set rc [catch { read $fd 4096 } res]" + print " if {$rc} {" + print " puts \"ERROR $res\"" + print " }" + print " if {!$rc && [string length $res] > 0} {" + print " append O($iJob) $res" + print " }" + print " } else {" + print " set rc [catch { gets $fd line } res]" + next + } + $0 == " if {$res>=0} {" { + print " if {![info exists ::kandelo_chunk_pipe_output] || !$::kandelo_chunk_pipe_output} {" + print " if {$res>=0} {" + next + } + $0 == " append O($iJob) \"$line\\n\"" { + print + print " }" + print " }" + next + } + { print } + ' "$runner" > "$tmp" + mv "$tmp" "$runner" + chmod a+r "$runner" + + for required in \ + 'set ::kandelo_inline_run_sh 1' \ + 'set ::kandelo_chunk_pipe_output 1' \ + 'set fd [open "|sh -c [list $inline_cmd] 2>@1" r]' \ + 'set rc [catch { read $fd 4096 } res]' + do + if ! grep -Fq "$required" "$runner"; then + echo "ERROR: failed to patch SQLite testrunner.tcl for Kandelo all-mode jobs: missing $required" >&2 + exit 1 + fi + done +} + cleanup() { if [ "$KEEP_WORKDIR" = "1" ]; then echo "Keeping SQLite official workdir: $WORKDIR" @@ -272,6 +519,13 @@ cp "$TESTFIXTURE" "$WORKDIR/testfixture.wasm" cp "$SQLITE3" "$WORKDIR/sqlite3" cp "$SQLITE3" "$WORKDIR/sqlite3.wasm" chmod a+rx "$WORKDIR/testfixture" "$WORKDIR/testfixture.wasm" "$WORKDIR/sqlite3" "$WORKDIR/sqlite3.wasm" +if [ -n "$GUEST_SHELL" ]; then + cp "$GUEST_SHELL" "$WORKDIR/sh" + cp "$GUEST_SHELL" "$WORKDIR/sh.wasm" + chmod a+rx "$WORKDIR/sh" "$WORKDIR/sh.wasm" +fi +patch_sqlite_testrunner_platform "$WORKDIR/test/testrunner.tcl" +patch_sqlite_testrunner_guest_paths "$WORKDIR/test/testrunner.tcl" RUNNER_TCL="$WORKDIR/kandelo-testrunner.tcl" cat > "$RUNNER_TCL" <<'TCL' @@ -302,12 +556,13 @@ echo "Results dir: $RESULTS_DIR" set +e TCL_LIBRARY="$TCL_INSTALL/lib/tcl8.6" \ KERNEL_CWD="$WORKDIR" \ +KERNEL_PATH="$WORKDIR:${KERNEL_PATH:-/usr/local/bin:/usr/bin:/bin}" \ KERNEL_UID="${SQLITE_TEST_UID:-1000}" \ KERNEL_GID="${SQLITE_TEST_GID:-1000}" \ TIMEOUT="$TIMEOUT_MS" \ node --experimental-wasm-exnref --import tsx/esm \ "$REPO_ROOT/examples/run-example.ts" \ - "$TESTFIXTURE" \ + "$WORKDIR/testfixture.wasm" \ "${ARGS[@]}" status=$? set -e From e22b70f526ad3eff812f5f0e7715cfab7c52c6c5 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 15:20:29 -0400 Subject: [PATCH 02/10] SQLite tests: emit durable case outcome lists --- scripts/browser-sqlite-official-runner.ts | 35 +- scripts/run-browser-sqlite-official-tests.sh | 6 + scripts/run-sqlite-official-tests.sh | 6 + scripts/sqlite-case-outcomes.py | 469 +++++++++++++++++++ tests/scripts/sqlite-case-outcomes.sh | 124 +++++ 5 files changed, 639 insertions(+), 1 deletion(-) create mode 100755 scripts/sqlite-case-outcomes.py create mode 100755 tests/scripts/sqlite-case-outcomes.sh diff --git a/scripts/browser-sqlite-official-runner.ts b/scripts/browser-sqlite-official-runner.ts index 2715c11f99..cdbbf85880 100755 --- a/scripts/browser-sqlite-official-runner.ts +++ b/scripts/browser-sqlite-official-runner.ts @@ -1,5 +1,5 @@ #!/usr/bin/env tsx -import { spawn, type ChildProcess } from "node:child_process"; +import { spawn, spawnSync, type ChildProcess } from "node:child_process"; import { existsSync, mkdirSync, writeFileSync } from "node:fs"; import { createServer } from "node:net"; import { resolve } from "node:path"; @@ -108,6 +108,38 @@ function writeArtifacts(resultsDir: string, artifacts: BrowserArtifact[] | undef } } +function inferDisplayName(command: string[]): string { + const tests = command.filter((arg) => arg.endsWith(".test")); + if (tests.length > 0) return tests[tests.length - 1]; + return command.join(" "); +} + +function writeDirectOutcomeArtifacts(resultsDir: string, command: string[], stdout: string, stderr: string): void { + if (!resultsDir) return; + mkdirSync(resultsDir, { recursive: true }); + const stdoutPath = resolve(resultsDir, "runner.stdout"); + const stderrPath = resolve(resultsDir, "runner.stderr"); + const dbPath = resolve(resultsDir, "testrunner.db"); + writeFileSync(stdoutPath, stdout); + writeFileSync(stderrPath, stderr); + const sourceArgs = existsSync(dbPath) + ? ["--db", dbPath] + : ["--stdout-file", stdoutPath, "--display-name", inferDisplayName(command)]; + const outcome = spawnSync( + "python3", + [ + resolve(REPO_ROOT, "scripts/sqlite-case-outcomes.py"), + ...sourceArgs, + "--results-dir", resultsDir, + "--host", "browser", + ], + { cwd: REPO_ROOT, stdio: "inherit" }, + ); + if (outcome.status !== 0) { + console.error(`[sqlite-outcomes] case outcome extraction failed with status ${outcome.status}`); + } +} + async function main() { const argv = process.argv.slice(2); let timeoutMs = 600_000; @@ -189,6 +221,7 @@ async function main() { } if (!result.artifacts && latestArtifacts) result.artifacts = latestArtifacts; writeArtifacts(resultsDir, result.artifacts); + writeDirectOutcomeArtifacts(resultsDir, command, result.stdout, result.stderr); if (result.stdout) process.stdout.write(result.stdout); if (result.stderr) process.stderr.write(result.stderr); if (result.error) process.stderr.write(`${result.error}\n`); diff --git a/scripts/run-browser-sqlite-official-tests.sh b/scripts/run-browser-sqlite-official-tests.sh index 9e35b17317..ef95d402da 100755 --- a/scripts/run-browser-sqlite-official-tests.sh +++ b/scripts/run-browser-sqlite-official-tests.sh @@ -241,6 +241,12 @@ write_sqlite_report() { WHERE state IN ('failed', 'running', 'omit') ORDER BY state, jobid;" > "$failures" + python3 "$REPO_ROOT/scripts/sqlite-case-outcomes.py" \ + --db "$db" \ + --results-dir "$RESULTS_DIR" \ + --host browser \ + --permutation "$PERMUTATION" || true + echo "===== SQLite official testrunner database summary =====" cat "$report" } diff --git a/scripts/run-sqlite-official-tests.sh b/scripts/run-sqlite-official-tests.sh index b59ea58d45..9954199d2b 100755 --- a/scripts/run-sqlite-official-tests.sh +++ b/scripts/run-sqlite-official-tests.sh @@ -357,6 +357,12 @@ write_sqlite_report() { OR (state='done' AND coalesce(nerr, 0)>0) ORDER BY state, jobid;" > "$failures" + python3 "$REPO_ROOT/scripts/sqlite-case-outcomes.py" \ + --db "$db" \ + --results-dir "$RESULTS_DIR" \ + --host "$HOST" \ + --permutation "$PERMUTATION" || true + echo "===== SQLite official testrunner database summary =====" cat "$report" } diff --git a/scripts/sqlite-case-outcomes.py b/scripts/sqlite-case-outcomes.py new file mode 100755 index 0000000000..bfe38b788c --- /dev/null +++ b/scripts/sqlite-case-outcomes.py @@ -0,0 +1,469 @@ +#!/usr/bin/env python3 +"""Emit durable SQLite testrunner case and job outcome lists.""" + +from __future__ import annotations + +import argparse +import json +import re +import sqlite3 +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + + +OK_RE = re.compile(r"^(?P.+?)\.\.\. Ok$") +OMITTED_RE = re.compile(r"^(?P.+?)\.\.\. Omitted$") +FAIL_RE = re.compile(r"^! (?P\S+) expected:") +FAILURES_RE = re.compile(r"^!Failures on these tests:\s*(?P.*)$") +SUMMARY_RE = re.compile(r"\b(?P\d+) errors out of (?P\d+) tests\b") +OMITTED_DETAIL_RE = re.compile(r"^\.\s+(?P\S+)\s+(?P.+)$") + +FINALIZE_SUFFIX = "::__sqlite_finalize_testing__" + + +@dataclass +class ParsedOutput: + passed: list[str] = field(default_factory=list) + failed: list[str] = field(default_factory=list) + skipped_counted: list[tuple[str, str]] = field(default_factory=list) + skipped_detail: list[tuple[str, str]] = field(default_factory=list) + summary_tests: int | None = None + summary_errors: int | None = None + + +@dataclass +class Job: + jobid: int | None + state: str + displaytype: str + displayname: str + ntest: int | None + nerr: int | None + span: int | None + output: str + + +def normalize_output(text: str) -> str: + return text.replace("\r\n", "\n").replace("\r", "\n") + + +def append_unique(items: list[str], value: str) -> None: + if value not in items: + items.append(value) + + +def append_unique_pair(items: list[tuple[str, str]], value: tuple[str, str]) -> None: + if value not in items: + items.append(value) + + +def parse_output(text: str) -> ParsedOutput: + parsed = ParsedOutput() + in_omitted_detail = False + failure_summary: list[str] = [] + + for line in normalize_output(text).splitlines(): + line = line.strip() + if not line: + continue + + summary = SUMMARY_RE.search(line) + if summary: + parsed.summary_errors = int(summary.group("errors")) + parsed.summary_tests = int(summary.group("tests")) + + if line == "Omitted test cases:": + in_omitted_detail = True + continue + + if in_omitted_detail: + omitted_detail = OMITTED_DETAIL_RE.match(line) + if omitted_detail: + append_unique_pair( + parsed.skipped_detail, + (omitted_detail.group("name"), omitted_detail.group("reason")), + ) + continue + in_omitted_detail = False + + ok = OK_RE.match(line) + if ok: + parsed.passed.append(ok.group("name")) + continue + + omitted = OMITTED_RE.match(line) + if omitted: + append_unique_pair(parsed.skipped_counted, (omitted.group("name"), "omitted by SQLite test harness")) + continue + + failures = FAILURES_RE.match(line) + if failures: + failure_summary.extend(name for name in failures.group("names").split() if name) + continue + + fail = FAIL_RE.match(line) + if fail: + append_unique(parsed.failed, fail.group("name")) + + if failure_summary: + parsed.failed = failure_summary + + return parsed + + +def infer_case_prefix(displayname: str, parsed: ParsedOutput) -> str: + if parsed.passed: + return parsed.passed[0].split("-", 1)[0] + + basename = displayname.rsplit("/", 1)[-1] + if basename.endswith(".test"): + basename = basename[:-5] + return basename or "sqlite" + + +def read_jobs_from_db(db_path: Path) -> tuple[list[Job], str | None]: + if not db_path.exists(): + return [], f"testrunner database is missing: {db_path}" + + con = sqlite3.connect(str(db_path)) + try: + has_jobs = con.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='jobs' LIMIT 1" + ).fetchone() + if not has_jobs: + return [], f"testrunner database has no jobs table: {db_path}" + + rows = con.execute( + """ + SELECT jobid, state, displaytype, displayname, + ntest, nerr, span, coalesce(output, '') + FROM jobs + ORDER BY jobid + """ + ).fetchall() + finally: + con.close() + + jobs = [ + Job( + jobid=row[0], + state=row[1] or "", + displaytype=row[2] or "", + displayname=row[3] or "", + ntest=row[4], + nerr=row[5], + span=row[6], + output=row[7] or "", + ) + for row in rows + ] + return jobs, None + + +def read_stdout_job(stdout_path: Path, display_name: str) -> tuple[list[Job], str | None]: + if not stdout_path.exists(): + return [], f"stdout file is missing: {stdout_path}" + output = stdout_path.read_text(encoding="utf-8", errors="replace") + parsed = parse_output(output) + ntest = parsed.summary_tests + nerr = parsed.summary_errors + state = "failed" if nerr and nerr > 0 else "done" + return [ + Job( + jobid=1, + state=state, + displaytype="direct", + displayname=display_name or "direct stdout", + ntest=ntest, + nerr=nerr, + span=None, + output=output, + ) + ], None + + +def job_row(job: Job) -> list[Any]: + return [ + "" if job.jobid is None else job.jobid, + job.state, + job.displaytype, + job.displayname, + "" if job.ntest is None else job.ntest, + "" if job.nerr is None else job.nerr, + "" if job.span is None else job.span, + ] + + +def add_unavailable(unavailable: list[dict[str, Any]], category: str, reason: str, job: Job | None = None) -> None: + entry: dict[str, Any] = {"category": category, "reason": reason} + if job is not None: + entry["job"] = { + "jobid": job.jobid, + "state": job.state, + "displaytype": job.displaytype, + "displayname": job.displayname, + "ntest": job.ntest, + "nerr": job.nerr, + } + unavailable.append(entry) + + +def build_outcomes(jobs: list[Job], source_reason: str | None) -> dict[str, Any]: + passed_cases: list[str] = [] + failed_cases: list[str] = [] + skipped_cases: list[tuple[str, str]] = [] + passed_jobs: list[list[Any]] = [] + failed_jobs: list[list[Any]] = [] + skipped_jobs: list[list[Any]] = [] + incomplete_jobs: list[list[Any]] = [] + unavailable: list[dict[str, Any]] = [] + synthetic_cases: list[str] = [] + + if source_reason is not None: + add_unavailable(unavailable, "all", source_reason) + + reported_executed_cases = 0 + reported_case_errors = 0 + + for job in jobs: + parsed = parse_output(job.output) if job.output else None + effective_errors = job.nerr + if effective_errors is None and parsed is not None: + effective_errors = parsed.summary_errors + + if job.state == "done" and not effective_errors: + passed_jobs.append(job_row(job)) + elif job.state == "failed" or (job.state == "done" and effective_errors): + failed_jobs.append(job_row(job)) + elif job.state == "omit": + skipped_jobs.append(job_row(job)) + add_unavailable( + unavailable, + "skipped_cases", + "SQLite testrunner marked this job omitted; no case-level skip names are available.", + job, + ) + continue + elif job.state in {"ready", "running", "halt", ""}: + incomplete_jobs.append(job_row(job)) + add_unavailable( + unavailable, + "all_cases", + f"SQLite testrunner job is incomplete with state {job.state or ''}.", + job, + ) + continue + + if not job.output: + add_unavailable(unavailable, "all_cases", "SQLite job output is empty.", job) + continue + + assert parsed is not None + ntest = job.ntest if job.ntest is not None else parsed.summary_tests + nerr = effective_errors + if ntest is None or nerr is None: + add_unavailable(unavailable, "all_cases", "Could not find SQLite 'errors out of tests' summary.", job) + continue + + reported_executed_cases += ntest + reported_case_errors += nerr + + for skipped in parsed.skipped_counted + parsed.skipped_detail: + append_unique_pair(skipped_cases, skipped) + + if len(parsed.failed) == nerr: + failed_cases.extend(parsed.failed) + elif nerr == 0 and not parsed.failed: + pass + else: + failed_cases.extend(parsed.failed) + add_unavailable( + unavailable, + "failed_cases", + f"Parsed {len(parsed.failed)} failed case names, but SQLite reported {nerr} case errors.", + job, + ) + + expected_passed = ntest - nerr - len(parsed.skipped_counted) + missing_passed = expected_passed - len(parsed.passed) + job_passed = list(parsed.passed) + if missing_passed == 1: + synthetic = f"{infer_case_prefix(job.displayname, parsed)}{FINALIZE_SUFFIX}" + job_passed.append(synthetic) + synthetic_cases.append(synthetic) + elif missing_passed != 0: + add_unavailable( + unavailable, + "passed_cases", + f"Parsed {len(parsed.passed)} passed case names, but SQLite reported {expected_passed} passed executed cases.", + job, + ) + passed_cases.extend(job_passed) + + selected_cases = reported_executed_cases + len(skipped_cases) + categories = { + "passed_cases": { + "count": len(passed_cases), + "synthetic_count": len(synthetic_cases), + "synthetic_suffix": FINALIZE_SUFFIX, + "status": "available" + if not any(entry["category"] in {"passed_cases", "all_cases", "all"} for entry in unavailable) + else "partial", + }, + "failed_cases": { + "count": len(failed_cases), + "status": "available" + if not any(entry["category"] in {"failed_cases", "all_cases", "all"} for entry in unavailable) + else "partial", + }, + "skipped_cases": { + "count": len(skipped_cases), + "status": "available" + if not any(entry["category"] in {"skipped_cases", "all_cases", "all"} for entry in unavailable) + else "partial", + }, + } + + return { + "schema_version": 1, + "summary": { + "reported_executed_cases": reported_executed_cases, + "reported_case_errors": reported_case_errors, + "selected_cases": selected_cases, + "passed_cases": len(passed_cases), + "failed_cases": len(failed_cases), + "skipped_cases": len(skipped_cases), + "synthetic_passed_cases": synthetic_cases, + "jobs": { + "passed": len(passed_jobs), + "failed": len(failed_jobs), + "skipped": len(skipped_jobs), + "incomplete": len(incomplete_jobs), + }, + }, + "categories": categories, + "unavailable": unavailable, + "lists": { + "passed_cases": passed_cases, + "failed_cases": failed_cases, + "skipped_cases": skipped_cases, + "passed_jobs": passed_jobs, + "failed_jobs": failed_jobs, + "skipped_jobs": skipped_jobs, + "incomplete_jobs": incomplete_jobs, + }, + } + + +def write_lines(path: Path, lines: list[str]) -> None: + path.write_text("".join(f"{line}\n" for line in lines), encoding="utf-8") + + +def write_tsv(path: Path, header: list[str], rows: list[list[Any] | tuple[Any, ...]]) -> None: + out = ["\t".join(header)] + for row in rows: + out.append("\t".join(str(value) for value in row)) + path.write_text("\n".join(out) + "\n", encoding="utf-8") + + +def write_outputs(results_dir: Path, outcomes: dict[str, Any], source: dict[str, Any]) -> None: + out_dir = results_dir / "outcome-lists" + out_dir.mkdir(parents=True, exist_ok=True) + lists = outcomes["lists"] + + write_lines(out_dir / "passed-cases.txt", lists["passed_cases"]) + write_lines(out_dir / "failed-cases.txt", lists["failed_cases"]) + write_tsv(out_dir / "skipped-cases.tsv", ["case", "reason"], lists["skipped_cases"]) + job_header = ["jobid", "state", "displaytype", "displayname", "cases", "errors", "ms"] + write_tsv(out_dir / "passed-jobs.tsv", job_header, lists["passed_jobs"]) + write_tsv(out_dir / "failed-jobs.tsv", job_header, lists["failed_jobs"]) + write_tsv(out_dir / "skipped-jobs.tsv", job_header, lists["skipped_jobs"]) + write_tsv(out_dir / "incomplete-jobs.tsv", job_header, lists["incomplete_jobs"]) + + serializable = { + "schema_version": outcomes["schema_version"], + "source": source, + "summary": outcomes["summary"], + "categories": outcomes["categories"], + "unavailable": outcomes["unavailable"], + "artifacts": { + "passed_cases": str(out_dir / "passed-cases.txt"), + "failed_cases": str(out_dir / "failed-cases.txt"), + "skipped_cases": str(out_dir / "skipped-cases.tsv"), + "passed_jobs": str(out_dir / "passed-jobs.tsv"), + "failed_jobs": str(out_dir / "failed-jobs.tsv"), + "skipped_jobs": str(out_dir / "skipped-jobs.tsv"), + "incomplete_jobs": str(out_dir / "incomplete-jobs.tsv"), + }, + } + (out_dir / "case-outcomes.json").write_text( + json.dumps(serializable, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + (out_dir / "unavailable-categories.json").write_text( + json.dumps(outcomes["unavailable"], indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + summary = outcomes["summary"] + lines = [ + "# SQLite Case Outcomes", + "", + f"Executed cases reported by SQLite: {summary['reported_executed_cases']}", + f"Case errors reported by SQLite: {summary['reported_case_errors']}", + f"Passed case list entries: {summary['passed_cases']}", + f"Failed case list entries: {summary['failed_cases']}", + f"Skipped case list entries: {summary['skipped_cases']}", + f"Synthetic finalization entries: {len(summary['synthetic_passed_cases'])}", + "", + ] + if outcomes["unavailable"]: + lines.append("Unavailable or partial categories:") + for entry in outcomes["unavailable"]: + lines.append(f"- {entry['category']}: {entry['reason']}") + else: + lines.append("All emitted case categories are complete for the reported harness totals.") + lines.append("") + (results_dir / "summary-case-outcomes.md").write_text("\n".join(lines), encoding="utf-8") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--db", type=Path, help="SQLite testrunner.db to inspect") + parser.add_argument("--stdout-file", type=Path, help="Direct-run stdout file to inspect") + parser.add_argument("--results-dir", type=Path, required=True, help="Directory for outcome-list artifacts") + parser.add_argument("--host", default="", help="Host label for metadata") + parser.add_argument("--permutation", default="", help="SQLite permutation label for metadata") + parser.add_argument("--display-name", default="", help="Display name for stdout-only direct runs") + args = parser.parse_args() + + jobs: list[Job] + reason: str | None + source: dict[str, Any] = { + "host": args.host, + "permutation": args.permutation, + } + + if args.db is not None and args.db.exists(): + jobs, reason = read_jobs_from_db(args.db) + source.update({"kind": "testrunner-db", "path": str(args.db)}) + elif args.stdout_file is not None: + jobs, reason = read_stdout_job(args.stdout_file, args.display_name) + source.update({"kind": "direct-stdout", "path": str(args.stdout_file)}) + elif args.db is not None: + jobs, reason = read_jobs_from_db(args.db) + source.update({"kind": "testrunner-db", "path": str(args.db)}) + else: + jobs, reason = [], "no --db or --stdout-file source was provided" + source.update({"kind": "missing"}) + + outcomes = build_outcomes(jobs, reason) + write_outputs(args.results_dir, outcomes, source) + print(f"===== SQLite case outcome lists: {args.results_dir / 'outcome-lists'} =====") + print((args.results_dir / "summary-case-outcomes.md").read_text(encoding="utf-8")) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/scripts/sqlite-case-outcomes.sh b/tests/scripts/sqlite-case-outcomes.sh new file mode 100755 index 0000000000..3d64d1d8c7 --- /dev/null +++ b/tests/scripts/sqlite-case-outcomes.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +TMP_ROOT="$(mktemp -d)" +trap 'rm -rf "$TMP_ROOT"' EXIT + +STDOUT_FILE="$TMP_ROOT/direct.stdout" +RESULTS_DIR="$TMP_ROOT/results" + +cat > "$STDOUT_FILE" <<'EOF' +utf16.misc1-1.1... Ok +utf16.misc1-1.2... Ok +SQLite 2025-02-18 13:38:58 abcdef +0 errors out of 3 tests on OpenBSD 32-bit +Omitted test cases: +. misc1-10.1 skipped by focused test +EOF + +python3 "$REPO_ROOT/scripts/sqlite-case-outcomes.py" \ + --stdout-file "$STDOUT_FILE" \ + --results-dir "$RESULTS_DIR" \ + --host browser \ + --display-name "test/misc1.test" >/dev/null + +if [ "$(wc -l < "$RESULTS_DIR/outcome-lists/passed-cases.txt" | tr -d ' ')" != "3" ]; then + echo "expected three passed case entries" >&2 + exit 1 +fi + +if ! grep -qx 'utf16.misc1::__sqlite_finalize_testing__' "$RESULTS_DIR/outcome-lists/passed-cases.txt"; then + echo "missing synthetic finalize_testing case" >&2 + exit 1 +fi + +if [ "$(tail -n +2 "$RESULTS_DIR/outcome-lists/skipped-cases.tsv" | wc -l | tr -d ' ')" != "1" ]; then + echo "expected one skipped case entry" >&2 + exit 1 +fi + +python3 - "$RESULTS_DIR/outcome-lists/case-outcomes.json" <<'PY' +import json +import sys +from pathlib import Path + +data = json.loads(Path(sys.argv[1]).read_text()) +summary = data["summary"] +assert summary["reported_executed_cases"] == 3, summary +assert summary["passed_cases"] == 3, summary +assert summary["skipped_cases"] == 1, summary +assert summary["synthetic_passed_cases"] == ["utf16.misc1::__sqlite_finalize_testing__"], summary +assert data["unavailable"] == [], data["unavailable"] +PY + +DB="$TMP_ROOT/testrunner.db" +python3 - "$DB" "$STDOUT_FILE" <<'PY' +import sqlite3 +import sys +from pathlib import Path + +db_path = Path(sys.argv[1]) +stdout = Path(sys.argv[2]).read_text() +con = sqlite3.connect(db_path) +con.executescript(""" +CREATE TABLE jobs( + jobid INTEGER PRIMARY KEY, + displaytype TEXT NOT NULL, + displayname TEXT NOT NULL, + build TEXT NOT NULL DEFAULT '', + dirname TEXT NOT NULL DEFAULT '', + cmd TEXT NOT NULL, + depid INTEGER, + priority INTEGER NOT NULL, + starttime INTEGER, + endtime INTEGER, + span INTEGER, + estwork INTEGER, + state TEXT, + ntest INT, + nerr INT, + svers TEXT, + pltfm TEXT, + output TEXT +); +""") +con.execute( + "INSERT INTO jobs(jobid, displaytype, displayname, cmd, priority, state, ntest, nerr, output) VALUES(1, 'tcl', 'test/misc1.test', '', 1, 'done', 3, 0, ?)", + (stdout,), +) +con.execute( + "INSERT INTO jobs(jobid, displaytype, displayname, cmd, priority, state, ntest, nerr, output) VALUES(2, 'tcl', 'test/failing.test', '', 1, 'done', 1, 1, ?)", + ("! failing-1.1 expected: value\n1 errors out of 1 tests on OpenBSD 32-bit\n",), +) +con.commit() +con.close() +PY + +DB_RESULTS="$TMP_ROOT/db-results" +python3 "$REPO_ROOT/scripts/sqlite-case-outcomes.py" \ + --db "$DB" \ + --results-dir "$DB_RESULTS" \ + --host node >/dev/null + +if ! grep -qx 'utf16.misc1::__sqlite_finalize_testing__' "$DB_RESULTS/outcome-lists/passed-cases.txt"; then + echo "db mode missing synthetic finalize_testing case" >&2 + exit 1 +fi + +python3 - "$DB_RESULTS/outcome-lists/case-outcomes.json" <<'PY' +import json +import sys +from pathlib import Path + +data = json.loads(Path(sys.argv[1]).read_text()) +assert data["summary"]["jobs"] == { + "passed": 1, + "failed": 1, + "skipped": 0, + "incomplete": 0, +}, data["summary"]["jobs"] +assert data["summary"]["failed_cases"] == 1, data["summary"] +PY + +echo "sqlite-case-outcomes ok" From 49470ebe7692406de0de7b0ae30a7dad9804d5f4 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 15:22:40 -0400 Subject: [PATCH 03/10] SQLite tests: fail loudly when runner patches drift --- apps/browser-demos/pages/sqlite-test/main.ts | 82 +----------- .../pages/sqlite-test/testrunner-patch.ts | 118 ++++++++++++++++++ host/test/sqlite-testrunner-patch.test.ts | 36 ++++++ 3 files changed, 158 insertions(+), 78 deletions(-) create mode 100644 apps/browser-demos/pages/sqlite-test/testrunner-patch.ts create mode 100644 host/test/sqlite-testrunner-patch.test.ts diff --git a/apps/browser-demos/pages/sqlite-test/main.ts b/apps/browser-demos/pages/sqlite-test/main.ts index 64d9858140..3105cc4a87 100644 --- a/apps/browser-demos/pages/sqlite-test/main.ts +++ b/apps/browser-demos/pages/sqlite-test/main.ts @@ -7,6 +7,10 @@ import { BrowserKernel } from "@host/browser-kernel-host"; import { MemoryFileSystem } from "@host/vfs/memory-fs"; import { writeVfsFile } from "@host/vfs/image-helpers"; import { finalizeKernelOwnedImage, settleWebKitReclaim } from "../../lib/kernel-owned-boot"; +import { + patchTestrunnerForKandelo, + testrunnerPlatformShim, +} from "./testrunner-patch"; import kernelWasmUrl from "@kernel-wasm?url"; declare global { @@ -91,84 +95,6 @@ async function collectArtifactsFromKernel( return artifacts.length > 0 ? artifacts : undefined; } -const testrunnerPlatformShim = [ - "# Kandelo platform shim for child testrunner jobs.", - "# SQLite all-mode reruns config variants by invoking test/testrunner.tcl", - "# directly, so the platform override has to live in that file too.", - "set ::tcl_platform(os) OpenBSD", - "set ::tcl_platform(platform) unix", -].join("\n"); - -const testrunnerGuestPathShim = [ - "# Kandelo guest path shim for all-mode child jobs.", - "# testrunner.tcl builds child run.sh files from host-normalized paths;", - "# convert workdir-local paths back to paths relative to each testdirN", - "# directory, because SQLite runs the script after cd-ing into it.", - "proc kandelo_guest_path {path} {", - " if {[file pathtype $path] != \"absolute\" && [string equal $path [info nameofexec]]} {", - " return $path", - " }", - " set normalized [file normalize $path]", - " set topdir [file normalize [file dirname $::testdir]]", - " set script [file normalize [info script]]", - " if {[string equal $normalized $script]} { return \"../test/testrunner.tcl\" }", - " if {[string equal $normalized $topdir]} { return \"..\" }", - " set prefix \"${topdir}/\"", - " if {[string first $prefix $normalized] == 0} {", - " return \"../[string range $normalized [string length $prefix] end]\"", - " }", - " return $path", - "}", - "set ::kandelo_inline_run_sh 1", -].join("\n"); - -function patchTestrunnerForKandelo(runner: string): string { - let patched = runner; - - if (!patched.includes("Kandelo platform shim for child testrunner jobs")) { - const lines = patched.split("\n"); - lines.splice(3, 0, "", testrunnerPlatformShim); - patched = lines.join("\n"); - } - - if (!patched.includes("Kandelo guest path shim for all-mode child jobs")) { - patched = patched.replace("cd $dir\n", `cd $dir\n\n${testrunnerGuestPathShim}\n`); - patched = patched.replace( - " set displayname [string map [list $topdir/ {}] $f]\n", - [ - " set displayname [string map [list $topdir/ {}] $f]", - " set testfixture_guest [kandelo_guest_path $testfixture]", - " set testrunner_tcl_guest [kandelo_guest_path $testrunner_tcl]", - " set f_guest [kandelo_guest_path $f]", - "", - ].join("\n"), - ); - patched = patched - .replace(" set cmd \"$testfixture $f\"", " set cmd \"$testfixture_guest $f_guest\"") - .replace( - " set cmd \"$testfixture $testrunner_tcl $config $f\"", - " set cmd \"$testfixture_guest $testrunner_tcl_guest $config $f_guest\"", - ) - .replace( - " set set_tmp_dir \"export SQLITE_TMPDIR=\\\"[file normalize $dir]\\\"\"", - " set set_tmp_dir \"export SQLITE_TMPDIR=.\"", - ) - .replace( - " set fd [open \"|$TRG(runcmd) 2>@1\" r]", - [ - " if {[info exists ::kandelo_inline_run_sh] && $::kandelo_inline_run_sh} {", - " set inline_cmd \"$set_tmp_dir\\n$job(cmd)\"", - " set fd [open \"|sh -c [list $inline_cmd] 2>@1\" r]", - " } else {", - " set fd [open \"|$TRG(runcmd) 2>@1\" r]", - " }", - ].join("\n"), - ); - } - - return patched; -} - function installTestrunnerPatches(fs: MemoryFileSystem): void { const runnerPath = "/sqlite/test/testrunner.tcl"; const decoder = new TextDecoder(); diff --git a/apps/browser-demos/pages/sqlite-test/testrunner-patch.ts b/apps/browser-demos/pages/sqlite-test/testrunner-patch.ts new file mode 100644 index 0000000000..d7ef68b71d --- /dev/null +++ b/apps/browser-demos/pages/sqlite-test/testrunner-patch.ts @@ -0,0 +1,118 @@ +export const testrunnerPlatformShim = [ + "# Kandelo platform shim for child testrunner jobs.", + "# SQLite all-mode reruns config variants by invoking test/testrunner.tcl", + "# directly, so the platform override has to live in that file too.", + "set ::tcl_platform(os) OpenBSD", + "set ::tcl_platform(platform) unix", +].join("\n"); + +const testrunnerGuestPathShim = [ + "# Kandelo guest path shim for all-mode child jobs.", + "# testrunner.tcl builds child run.sh files from host-normalized paths;", + "# convert workdir-local paths back to paths relative to each testdirN", + "# directory, because SQLite runs the script after cd-ing into it.", + "proc kandelo_guest_path {path} {", + " if {[file pathtype $path] != \"absolute\" && [string equal $path [info nameofexec]]} {", + " return $path", + " }", + " set normalized [file normalize $path]", + " set topdir [file normalize [file dirname $::testdir]]", + " set script [file normalize [info script]]", + " if {[string equal $normalized $script]} { return \"../test/testrunner.tcl\" }", + " if {[string equal $normalized $topdir]} { return \"..\" }", + " set prefix \"${topdir}/\"", + " if {[string first $prefix $normalized] == 0} {", + " return \"../[string range $normalized [string length $prefix] end]\"", + " }", + " return $path", + "}", + "set ::kandelo_inline_run_sh 1", +].join("\n"); + +function replaceRequired(source: string, search: string, replacement: string, label: string): string { + if (!source.includes(search)) { + throw new Error(`SQLite testrunner patch is incompatible: missing ${label}`); + } + return source.replace(search, replacement); +} + +export function patchTestrunnerForKandelo(runner: string): string { + let patched = runner; + + if (!patched.includes("Kandelo platform shim for child testrunner jobs")) { + const lines = patched.split("\n"); + if (lines.length < 4) { + throw new Error("SQLite testrunner patch is incompatible: file is shorter than four lines"); + } + lines.splice(3, 0, "", testrunnerPlatformShim); + patched = lines.join("\n"); + } + + if (!patched.includes("Kandelo guest path shim for all-mode child jobs")) { + patched = replaceRequired( + patched, + "cd $dir\n", + `cd $dir\n\n${testrunnerGuestPathShim}\n`, + "child work-directory anchor", + ); + patched = replaceRequired( + patched, + " set displayname [string map [list $topdir/ {}] $f]\n", + [ + " set displayname [string map [list $topdir/ {}] $f]", + " set testfixture_guest [kandelo_guest_path $testfixture]", + " set testrunner_tcl_guest [kandelo_guest_path $testrunner_tcl]", + " set f_guest [kandelo_guest_path $f]", + "", + ].join("\n"), + "job display-name anchor", + ); + patched = replaceRequired( + patched, + " set cmd \"$testfixture $f\"", + " set cmd \"$testfixture_guest $f_guest\"", + "direct test command anchor", + ); + patched = replaceRequired( + patched, + " set cmd \"$testfixture $testrunner_tcl $config $f\"", + " set cmd \"$testfixture_guest $testrunner_tcl_guest $config $f_guest\"", + "configured test command anchor", + ); + patched = replaceRequired( + patched, + " set set_tmp_dir \"export SQLITE_TMPDIR=\\\"[file normalize $dir]\\\"\"", + " set set_tmp_dir \"export SQLITE_TMPDIR=.\"", + "temporary-directory anchor", + ); + patched = replaceRequired( + patched, + " set fd [open \"|$TRG(runcmd) 2>@1\" r]", + [ + " if {[info exists ::kandelo_inline_run_sh] && $::kandelo_inline_run_sh} {", + " set inline_cmd \"$set_tmp_dir\\n$job(cmd)\"", + " set fd [open \"|sh -c [list $inline_cmd] 2>@1\" r]", + " } else {", + " set fd [open \"|$TRG(runcmd) 2>@1\" r]", + " }", + ].join("\n"), + "child shell command anchor", + ); + } + + for (const required of [ + "Kandelo platform shim for child testrunner jobs", + "Kandelo guest path shim for all-mode child jobs", + "set testfixture_guest [kandelo_guest_path $testfixture]", + "set cmd \"$testfixture_guest $f_guest\"", + "set cmd \"$testfixture_guest $testrunner_tcl_guest $config $f_guest\"", + "set set_tmp_dir \"export SQLITE_TMPDIR=.\"", + "set fd [open \"|sh -c [list $inline_cmd] 2>@1\" r]", + ]) { + if (!patched.includes(required)) { + throw new Error(`SQLite testrunner patch is incomplete: missing ${required}`); + } + } + + return patched; +} diff --git a/host/test/sqlite-testrunner-patch.test.ts b/host/test/sqlite-testrunner-patch.test.ts new file mode 100644 index 0000000000..47c1ca5941 --- /dev/null +++ b/host/test/sqlite-testrunner-patch.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vitest"; +import { patchTestrunnerForKandelo } from "../../apps/browser-demos/pages/sqlite-test/testrunner-patch"; + +const upstreamFixture = [ + "#!/usr/bin/env tclsh", + "# SQLite test runner fixture", + "set TRG(nJob) 1", + "set dir testdir1", + "cd $dir", + " set displayname [string map [list $topdir/ {}] $f]", + " set cmd \"$testfixture $f\"", + " set cmd \"$testfixture $testrunner_tcl $config $f\"", + " set set_tmp_dir \"export SQLITE_TMPDIR=\\\"[file normalize $dir]\\\"\"", + " set fd [open \"|$TRG(runcmd) 2>@1\" r]", + "", +].join("\n"); + +describe("SQLite browser testrunner patch", () => { + it("rewrites all-mode child commands and is idempotent", () => { + const patched = patchTestrunnerForKandelo(upstreamFixture); + + expect(patched).toContain("set ::tcl_platform(os) OpenBSD"); + expect(patched).toContain("set testfixture_guest [kandelo_guest_path $testfixture]"); + expect(patched).toContain("set cmd \"$testfixture_guest $f_guest\""); + expect(patched).toContain("set cmd \"$testfixture_guest $testrunner_tcl_guest $config $f_guest\""); + expect(patched).toContain("set set_tmp_dir \"export SQLITE_TMPDIR=.\""); + expect(patched).toContain("set fd [open \"|sh -c [list $inline_cmd] 2>@1\" r]"); + expect(patchTestrunnerForKandelo(patched)).toBe(patched); + }); + + it("fails loudly when an upstream anchor changes", () => { + expect(() => patchTestrunnerForKandelo("one\ntwo\nthree\nfour\n")).toThrow( + "missing child work-directory anchor", + ); + }); +}); From 7d0c6d58393fcccfb8d2690ab0c16b1964649363 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 15:24:45 -0400 Subject: [PATCH 04/10] SQLite tests: report unnamed outcomes without fabrication --- scripts/sqlite-case-outcomes.py | 68 ++++++++++++++++----------- tests/scripts/sqlite-case-outcomes.sh | 31 +++++++----- 2 files changed, 58 insertions(+), 41 deletions(-) diff --git a/scripts/sqlite-case-outcomes.py b/scripts/sqlite-case-outcomes.py index bfe38b788c..a9282d5fbe 100755 --- a/scripts/sqlite-case-outcomes.py +++ b/scripts/sqlite-case-outcomes.py @@ -19,9 +19,6 @@ SUMMARY_RE = re.compile(r"\b(?P\d+) errors out of (?P\d+) tests\b") OMITTED_DETAIL_RE = re.compile(r"^\.\s+(?P\S+)\s+(?P.+)$") -FINALIZE_SUFFIX = "::__sqlite_finalize_testing__" - - @dataclass class ParsedOutput: passed: list[str] = field(default_factory=list) @@ -112,14 +109,12 @@ def parse_output(text: str) -> ParsedOutput: return parsed -def infer_case_prefix(displayname: str, parsed: ParsedOutput) -> str: - if parsed.passed: - return parsed.passed[0].split("-", 1)[0] - - basename = displayname.rsplit("/", 1)[-1] - if basename.endswith(".test"): - basename = basename[:-5] - return basename or "sqlite" +def decode_db_text(value: Any) -> str: + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + if value is None: + return "" + return str(value) def read_jobs_from_db(db_path: Path) -> tuple[list[Job], str | None]: @@ -127,6 +122,9 @@ def read_jobs_from_db(db_path: Path) -> tuple[list[Job], str | None]: return [], f"testrunner database is missing: {db_path}" con = sqlite3.connect(str(db_path)) + # Some archived browser runs contain non-UTF-8 bytes in Tcl output. Read + # TEXT as bytes so one diagnostic byte cannot make the whole report fail. + con.text_factory = bytes try: has_jobs = con.execute( "SELECT 1 FROM sqlite_master WHERE type='table' AND name='jobs' LIMIT 1" @@ -148,13 +146,13 @@ def read_jobs_from_db(db_path: Path) -> tuple[list[Job], str | None]: jobs = [ Job( jobid=row[0], - state=row[1] or "", - displaytype=row[2] or "", - displayname=row[3] or "", + state=decode_db_text(row[1]), + displaytype=decode_db_text(row[2]), + displayname=decode_db_text(row[3]), ntest=row[4], nerr=row[5], span=row[6], - output=row[7] or "", + output=decode_db_text(row[7]), ) for row in rows ] @@ -217,8 +215,8 @@ def build_outcomes(jobs: list[Job], source_reason: str | None) -> dict[str, Any] failed_jobs: list[list[Any]] = [] skipped_jobs: list[list[Any]] = [] incomplete_jobs: list[list[Any]] = [] + unattributed_passed_cases: list[list[Any]] = [] unavailable: list[dict[str, Any]] = [] - synthetic_cases: list[str] = [] if source_reason is not None: add_unavailable(unavailable, "all", source_reason) @@ -287,26 +285,33 @@ def build_outcomes(jobs: list[Job], source_reason: str | None) -> dict[str, Any] expected_passed = ntest - nerr - len(parsed.skipped_counted) missing_passed = expected_passed - len(parsed.passed) - job_passed = list(parsed.passed) - if missing_passed == 1: - synthetic = f"{infer_case_prefix(job.displayname, parsed)}{FINALIZE_SUFFIX}" - job_passed.append(synthetic) - synthetic_cases.append(synthetic) - elif missing_passed != 0: + if missing_passed > 0: + unattributed_passed_cases.append([ + "" if job.jobid is None else job.jobid, + job.displayname, + missing_passed, + "SQLite reported passed cases without corresponding named Ok lines", + ]) add_unavailable( unavailable, "passed_cases", - f"Parsed {len(parsed.passed)} passed case names, but SQLite reported {expected_passed} passed executed cases.", + f"SQLite reported {expected_passed} passed cases, but only {len(parsed.passed)} names were present in output.", job, ) - passed_cases.extend(job_passed) + elif missing_passed < 0: + add_unavailable( + unavailable, + "passed_cases", + f"Parsed {len(parsed.passed)} passed case names, but SQLite reported only {expected_passed} passed cases.", + job, + ) + passed_cases.extend(parsed.passed) selected_cases = reported_executed_cases + len(skipped_cases) categories = { "passed_cases": { "count": len(passed_cases), - "synthetic_count": len(synthetic_cases), - "synthetic_suffix": FINALIZE_SUFFIX, + "unattributed_count": sum(row[2] for row in unattributed_passed_cases), "status": "available" if not any(entry["category"] in {"passed_cases", "all_cases", "all"} for entry in unavailable) else "partial", @@ -334,7 +339,7 @@ def build_outcomes(jobs: list[Job], source_reason: str | None) -> dict[str, Any] "passed_cases": len(passed_cases), "failed_cases": len(failed_cases), "skipped_cases": len(skipped_cases), - "synthetic_passed_cases": synthetic_cases, + "unattributed_passed_cases": sum(row[2] for row in unattributed_passed_cases), "jobs": { "passed": len(passed_jobs), "failed": len(failed_jobs), @@ -348,6 +353,7 @@ def build_outcomes(jobs: list[Job], source_reason: str | None) -> dict[str, Any] "passed_cases": passed_cases, "failed_cases": failed_cases, "skipped_cases": skipped_cases, + "unattributed_passed_cases": unattributed_passed_cases, "passed_jobs": passed_jobs, "failed_jobs": failed_jobs, "skipped_jobs": skipped_jobs, @@ -375,6 +381,11 @@ def write_outputs(results_dir: Path, outcomes: dict[str, Any], source: dict[str, write_lines(out_dir / "passed-cases.txt", lists["passed_cases"]) write_lines(out_dir / "failed-cases.txt", lists["failed_cases"]) write_tsv(out_dir / "skipped-cases.tsv", ["case", "reason"], lists["skipped_cases"]) + write_tsv( + out_dir / "unattributed-passed-cases.tsv", + ["jobid", "displayname", "count", "reason"], + lists["unattributed_passed_cases"], + ) job_header = ["jobid", "state", "displaytype", "displayname", "cases", "errors", "ms"] write_tsv(out_dir / "passed-jobs.tsv", job_header, lists["passed_jobs"]) write_tsv(out_dir / "failed-jobs.tsv", job_header, lists["failed_jobs"]) @@ -391,6 +402,7 @@ def write_outputs(results_dir: Path, outcomes: dict[str, Any], source: dict[str, "passed_cases": str(out_dir / "passed-cases.txt"), "failed_cases": str(out_dir / "failed-cases.txt"), "skipped_cases": str(out_dir / "skipped-cases.tsv"), + "unattributed_passed_cases": str(out_dir / "unattributed-passed-cases.tsv"), "passed_jobs": str(out_dir / "passed-jobs.tsv"), "failed_jobs": str(out_dir / "failed-jobs.tsv"), "skipped_jobs": str(out_dir / "skipped-jobs.tsv"), @@ -415,7 +427,7 @@ def write_outputs(results_dir: Path, outcomes: dict[str, Any], source: dict[str, f"Passed case list entries: {summary['passed_cases']}", f"Failed case list entries: {summary['failed_cases']}", f"Skipped case list entries: {summary['skipped_cases']}", - f"Synthetic finalization entries: {len(summary['synthetic_passed_cases'])}", + f"Passed cases without names in SQLite output: {summary['unattributed_passed_cases']}", "", ] if outcomes["unavailable"]: diff --git a/tests/scripts/sqlite-case-outcomes.sh b/tests/scripts/sqlite-case-outcomes.sh index 3d64d1d8c7..2a279bf729 100755 --- a/tests/scripts/sqlite-case-outcomes.sh +++ b/tests/scripts/sqlite-case-outcomes.sh @@ -23,13 +23,13 @@ python3 "$REPO_ROOT/scripts/sqlite-case-outcomes.py" \ --host browser \ --display-name "test/misc1.test" >/dev/null -if [ "$(wc -l < "$RESULTS_DIR/outcome-lists/passed-cases.txt" | tr -d ' ')" != "3" ]; then - echo "expected three passed case entries" >&2 +if [ "$(wc -l < "$RESULTS_DIR/outcome-lists/passed-cases.txt" | tr -d ' ')" != "2" ]; then + echo "expected two named passed case entries" >&2 exit 1 fi -if ! grep -qx 'utf16.misc1::__sqlite_finalize_testing__' "$RESULTS_DIR/outcome-lists/passed-cases.txt"; then - echo "missing synthetic finalize_testing case" >&2 +if [ "$(tail -n +2 "$RESULTS_DIR/outcome-lists/unattributed-passed-cases.tsv" | wc -l | tr -d ' ')" != "1" ]; then + echo "expected one unattributed passed-case row" >&2 exit 1 fi @@ -46,10 +46,11 @@ from pathlib import Path data = json.loads(Path(sys.argv[1]).read_text()) summary = data["summary"] assert summary["reported_executed_cases"] == 3, summary -assert summary["passed_cases"] == 3, summary +assert summary["passed_cases"] == 2, summary assert summary["skipped_cases"] == 1, summary -assert summary["synthetic_passed_cases"] == ["utf16.misc1::__sqlite_finalize_testing__"], summary -assert data["unavailable"] == [], data["unavailable"] +assert summary["unattributed_passed_cases"] == 1, summary +assert data["categories"]["passed_cases"]["status"] == "partial", data["categories"] +assert data["unavailable"][0]["category"] == "passed_cases", data["unavailable"] PY DB="$TMP_ROOT/testrunner.db" @@ -91,6 +92,11 @@ con.execute( "INSERT INTO jobs(jobid, displaytype, displayname, cmd, priority, state, ntest, nerr, output) VALUES(2, 'tcl', 'test/failing.test', '', 1, 'done', 1, 1, ?)", ("! failing-1.1 expected: value\n1 errors out of 1 tests on OpenBSD 32-bit\n",), ) +invalid_output = b"invalid-\x80-1.1... Ok\n0 errors out of 1 tests on OpenBSD 32-bit\n" +con.execute( + "INSERT INTO jobs(jobid, displaytype, displayname, cmd, priority, state, ntest, nerr, output) VALUES(3, 'tcl', 'test/nonutf8.test', '', 1, 'done', 1, 0, CAST(? AS TEXT))", + (sqlite3.Binary(invalid_output),), +) con.commit() con.close() PY @@ -101,11 +107,6 @@ python3 "$REPO_ROOT/scripts/sqlite-case-outcomes.py" \ --results-dir "$DB_RESULTS" \ --host node >/dev/null -if ! grep -qx 'utf16.misc1::__sqlite_finalize_testing__' "$DB_RESULTS/outcome-lists/passed-cases.txt"; then - echo "db mode missing synthetic finalize_testing case" >&2 - exit 1 -fi - python3 - "$DB_RESULTS/outcome-lists/case-outcomes.json" <<'PY' import json import sys @@ -113,12 +114,16 @@ from pathlib import Path data = json.loads(Path(sys.argv[1]).read_text()) assert data["summary"]["jobs"] == { - "passed": 1, + "passed": 2, "failed": 1, "skipped": 0, "incomplete": 0, }, data["summary"]["jobs"] assert data["summary"]["failed_cases"] == 1, data["summary"] +assert data["summary"]["unattributed_passed_cases"] == 1, data["summary"] +assert "invalid-\ufffd-1.1" in Path( + data["artifacts"]["passed_cases"] +).read_text().splitlines() PY echo "sqlite-case-outcomes ok" From 413a446a55fd2a4cea7c674ae03ef8832adf0f20 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 15:26:41 -0400 Subject: [PATCH 05/10] SQLite tests: keep outcome evidence immutable --- scripts/run-sqlite-official-tests.sh | 6 +---- scripts/sqlite-case-outcomes.py | 23 +++++++++++++++--- tests/scripts/sqlite-case-outcomes.sh | 35 +++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 8 deletions(-) diff --git a/scripts/run-sqlite-official-tests.sh b/scripts/run-sqlite-official-tests.sh index 9954199d2b..e7ad14c965 100755 --- a/scripts/run-sqlite-official-tests.sh +++ b/scripts/run-sqlite-official-tests.sh @@ -257,10 +257,6 @@ write_sqlite_report() { mkdir -p "$RESULTS_DIR" - # SQLite's testrunner keeps its control database in WAL mode. Checkpoint - # before copying so timeout artifacts remain self-contained after cleanup. - sqlite3 "$db" "PRAGMA wal_checkpoint(TRUNCATE);" >/dev/null 2>&1 || true - for artifact in testrunner.db testrunner.db-wal testrunner.db-shm testrunner.log testrunner_build.log; do if [ -f "$WORKDIR/$artifact" ]; then cp "$WORKDIR/$artifact" "$RESULTS_DIR/$artifact" @@ -358,7 +354,7 @@ write_sqlite_report() { ORDER BY state, jobid;" > "$failures" python3 "$REPO_ROOT/scripts/sqlite-case-outcomes.py" \ - --db "$db" \ + --db "$RESULTS_DIR/testrunner.db" \ --results-dir "$RESULTS_DIR" \ --host "$HOST" \ --permutation "$PERMUTATION" || true diff --git a/scripts/sqlite-case-outcomes.py b/scripts/sqlite-case-outcomes.py index a9282d5fbe..1ba1ca2cbf 100755 --- a/scripts/sqlite-case-outcomes.py +++ b/scripts/sqlite-case-outcomes.py @@ -55,6 +55,14 @@ def append_unique_pair(items: list[tuple[str, str]], value: tuple[str, str]) -> items.append(value) +def upsert_named_pair(items: list[tuple[str, str]], value: tuple[str, str]) -> None: + for index, (name, _) in enumerate(items): + if name == value[0]: + items[index] = value + return + items.append(value) + + def parse_output(text: str) -> ParsedOutput: parsed = ParsedOutput() in_omitted_detail = False @@ -121,7 +129,8 @@ def read_jobs_from_db(db_path: Path) -> tuple[list[Job], str | None]: if not db_path.exists(): return [], f"testrunner database is missing: {db_path}" - con = sqlite3.connect(str(db_path)) + db_uri = f"{db_path.resolve().as_uri()}?mode=ro" + con = sqlite3.connect(db_uri, uri=True) # Some archived browser runs contain non-UTF-8 bytes in Tcl output. Read # TEXT as bytes so one diagnostic byte cannot make the whole report fail. con.text_factory = bytes @@ -267,8 +276,16 @@ def build_outcomes(jobs: list[Job], source_reason: str | None) -> dict[str, Any] reported_executed_cases += ntest reported_case_errors += nerr - for skipped in parsed.skipped_counted + parsed.skipped_detail: - append_unique_pair(skipped_cases, skipped) + job_skipped: list[tuple[str, str]] = [] + for skipped in parsed.skipped_counted: + upsert_named_pair(job_skipped, skipped) + for skipped in parsed.skipped_detail: + # SQLite prints some omissions twice: once as `... Omitted` and + # again in its detailed reason list. Keep one row and prefer the + # specific reason. + upsert_named_pair(job_skipped, skipped) + for skipped in job_skipped: + upsert_named_pair(skipped_cases, skipped) if len(parsed.failed) == nerr: failed_cases.extend(parsed.failed) diff --git a/tests/scripts/sqlite-case-outcomes.sh b/tests/scripts/sqlite-case-outcomes.sh index 2a279bf729..32053ee9de 100755 --- a/tests/scripts/sqlite-case-outcomes.sh +++ b/tests/scripts/sqlite-case-outcomes.sh @@ -53,6 +53,30 @@ assert data["categories"]["passed_cases"]["status"] == "partial", data["categori assert data["unavailable"][0]["category"] == "passed_cases", data["unavailable"] PY +OMITTED_STDOUT="$TMP_ROOT/omitted.stdout" +OMITTED_RESULTS="$TMP_ROOT/omitted-results" +cat > "$OMITTED_STDOUT" <<'EOF' +omitted-1.1... Omitted +0 errors out of 1 tests on OpenBSD 32-bit +Omitted test cases: +. omitted-1.1 requires an unavailable optional feature +EOF + +python3 "$REPO_ROOT/scripts/sqlite-case-outcomes.py" \ + --stdout-file "$OMITTED_STDOUT" \ + --results-dir "$OMITTED_RESULTS" \ + --host browser \ + --display-name "test/omitted.test" >/dev/null + +if [ "$(tail -n +2 "$OMITTED_RESULTS/outcome-lists/skipped-cases.tsv" | wc -l | tr -d ' ')" != "1" ]; then + echo "expected duplicate omitted output to produce one skipped row" >&2 + exit 1 +fi +if ! grep -q 'requires an unavailable optional feature' "$OMITTED_RESULTS/outcome-lists/skipped-cases.tsv"; then + echo "expected the detailed omission reason" >&2 + exit 1 +fi + DB="$TMP_ROOT/testrunner.db" python3 - "$DB" "$STDOUT_FILE" <<'PY' import sqlite3 @@ -101,12 +125,23 @@ con.commit() con.close() PY +# Reporting must not mutate or clean up the evidence it reads. +: > "$DB-wal" +: > "$DB-shm" +SIDECAR_HASHES_BEFORE="$(shasum -a 256 "$DB" "$DB-wal" "$DB-shm")" + DB_RESULTS="$TMP_ROOT/db-results" python3 "$REPO_ROOT/scripts/sqlite-case-outcomes.py" \ --db "$DB" \ --results-dir "$DB_RESULTS" \ --host node >/dev/null +SIDECAR_HASHES_AFTER="$(shasum -a 256 "$DB" "$DB-wal" "$DB-shm")" +if [ "$SIDECAR_HASHES_BEFORE" != "$SIDECAR_HASHES_AFTER" ]; then + echo "outcome extraction changed its source database or sidecars" >&2 + exit 1 +fi + python3 - "$DB_RESULTS/outcome-lists/case-outcomes.json" <<'PY' import json import sys From 2412212d109ea8a2b50602a3518484cbd6d576f7 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 15:31:30 -0400 Subject: [PATCH 06/10] SQLite tests: keep runner host metadata truthful Select Kandelo's helper launcher explicitly instead of presenting Tcl as OpenBSD, and keep browser pipe draining aligned with the Node harness. --- .../pages/sqlite-test/testrunner-patch.ts | 85 +++++++++++++++++-- host/test/sqlite-testrunner-patch.test.ts | 16 +++- scripts/run-sqlite-official-tests.sh | 51 ++++++++--- 3 files changed, 130 insertions(+), 22 deletions(-) diff --git a/apps/browser-demos/pages/sqlite-test/testrunner-patch.ts b/apps/browser-demos/pages/sqlite-test/testrunner-patch.ts index d7ef68b71d..6a7ff853ba 100644 --- a/apps/browser-demos/pages/sqlite-test/testrunner-patch.ts +++ b/apps/browser-demos/pages/sqlite-test/testrunner-patch.ts @@ -1,9 +1,8 @@ export const testrunnerPlatformShim = [ - "# Kandelo platform shim for child testrunner jobs.", - "# SQLite all-mode reruns config variants by invoking test/testrunner.tcl", - "# directly, so the platform override has to live in that file too.", - "set ::tcl_platform(os) OpenBSD", - "set ::tcl_platform(platform) unix", + "# Kandelo testrunner host selection for child jobs.", + "# This controls only SQLite's helper-script launcher. Keep tcl_platform", + "# unchanged so the tests continue to observe the real Tcl target.", + "set ::kandelo_testrunner_host Kandelo", ].join("\n"); const testrunnerGuestPathShim = [ @@ -27,6 +26,7 @@ const testrunnerGuestPathShim = [ " return $path", "}", "set ::kandelo_inline_run_sh 1", + "set ::kandelo_chunk_pipe_output 1", ].join("\n"); function replaceRequired(source: string, search: string, replacement: string, label: string): string { @@ -39,13 +39,43 @@ function replaceRequired(source: string, search: string, replacement: string, la export function patchTestrunnerForKandelo(runner: string): string { let patched = runner; - if (!patched.includes("Kandelo platform shim for child testrunner jobs")) { + if (!patched.includes("Kandelo testrunner host selection for child jobs")) { const lines = patched.split("\n"); if (lines.length < 4) { throw new Error("SQLite testrunner patch is incompatible: file is shorter than four lines"); } lines.splice(3, 0, "", testrunnerPlatformShim); patched = lines.join("\n"); + patched = replaceRequired( + patched, + "switch -nocase -glob -- $tcl_platform(os) {", + [ + "set testrunner_host $tcl_platform(os)", + "if {[info exists ::kandelo_testrunner_host]} {", + " set testrunner_host $::kandelo_testrunner_host", + "}", + "switch -nocase -glob -- $testrunner_host {", + ].join("\n"), + "host-selection switch", + ); + patched = replaceRequired( + patched, + " *openbsd* {\n", + [ + " *kandelo* {", + " set TRG(platform) linux", + " set TRG(make) make.sh", + " set TRG(makecmd) \"sh make.sh\"", + " set TRG(testfixture) testfixture", + " set TRG(shell) sqlite3", + " set TRG(run) run.sh", + " set TRG(runcmd) \"sh run.sh\"", + " }", + " *openbsd* {", + "", + ].join("\n"), + "OpenBSD host branch", + ); } if (!patched.includes("Kandelo guest path shim for all-mode child jobs")) { @@ -98,16 +128,57 @@ export function patchTestrunnerForKandelo(runner: string): string { ].join("\n"), "child shell command anchor", ); + patched = replaceRequired( + patched, + " set rc [catch { gets $fd line } res]", + [ + " if {[info exists ::kandelo_chunk_pipe_output] && $::kandelo_chunk_pipe_output} {", + " set rc [catch { read $fd 4096 } res]", + " if {$rc} {", + " puts \"ERROR $res\"", + " }", + " if {!$rc && [string length $res] > 0} {", + " append O($iJob) $res", + " }", + " } else {", + " set rc [catch { gets $fd line } res]", + ].join("\n"), + "pipe read anchor", + ); + patched = replaceRequired( + patched, + " if {$res>=0} {", + [ + " if {![info exists ::kandelo_chunk_pipe_output] || !$::kandelo_chunk_pipe_output} {", + " if {$res>=0} {", + ].join("\n"), + "line-result anchor", + ); + patched = replaceRequired( + patched, + " append O($iJob) \"$line\\n\"", + [ + " append O($iJob) \"$line\\n\"", + " }", + " }", + ].join("\n"), + "line append anchor", + ); } for (const required of [ - "Kandelo platform shim for child testrunner jobs", + "Kandelo testrunner host selection for child jobs", + "set ::kandelo_testrunner_host Kandelo", + "switch -nocase -glob -- $testrunner_host", + "*kandelo* {", "Kandelo guest path shim for all-mode child jobs", "set testfixture_guest [kandelo_guest_path $testfixture]", "set cmd \"$testfixture_guest $f_guest\"", "set cmd \"$testfixture_guest $testrunner_tcl_guest $config $f_guest\"", "set set_tmp_dir \"export SQLITE_TMPDIR=.\"", "set fd [open \"|sh -c [list $inline_cmd] 2>@1\" r]", + "set ::kandelo_chunk_pipe_output 1", + "set rc [catch { read $fd 4096 } res]", ]) { if (!patched.includes(required)) { throw new Error(`SQLite testrunner patch is incomplete: missing ${required}`); diff --git a/host/test/sqlite-testrunner-patch.test.ts b/host/test/sqlite-testrunner-patch.test.ts index 47c1ca5941..6bd2eeb28d 100644 --- a/host/test/sqlite-testrunner-patch.test.ts +++ b/host/test/sqlite-testrunner-patch.test.ts @@ -6,12 +6,20 @@ const upstreamFixture = [ "# SQLite test runner fixture", "set TRG(nJob) 1", "set dir testdir1", + "switch -nocase -glob -- $tcl_platform(os) {", + " *openbsd* {", + " }", + "}", "cd $dir", " set displayname [string map [list $topdir/ {}] $f]", " set cmd \"$testfixture $f\"", " set cmd \"$testfixture $testrunner_tcl $config $f\"", " set set_tmp_dir \"export SQLITE_TMPDIR=\\\"[file normalize $dir]\\\"\"", " set fd [open \"|$TRG(runcmd) 2>@1\" r]", + " set rc [catch { gets $fd line } res]", + " if {$res>=0} {", + " append O($iJob) \"$line\\n\"", + " }", "", ].join("\n"); @@ -19,18 +27,22 @@ describe("SQLite browser testrunner patch", () => { it("rewrites all-mode child commands and is idempotent", () => { const patched = patchTestrunnerForKandelo(upstreamFixture); - expect(patched).toContain("set ::tcl_platform(os) OpenBSD"); + expect(patched).toContain("set ::kandelo_testrunner_host Kandelo"); + expect(patched).not.toContain("set ::tcl_platform(os)"); + expect(patched).toContain("switch -nocase -glob -- $testrunner_host"); + expect(patched).toContain("*kandelo* {"); expect(patched).toContain("set testfixture_guest [kandelo_guest_path $testfixture]"); expect(patched).toContain("set cmd \"$testfixture_guest $f_guest\""); expect(patched).toContain("set cmd \"$testfixture_guest $testrunner_tcl_guest $config $f_guest\""); expect(patched).toContain("set set_tmp_dir \"export SQLITE_TMPDIR=.\""); expect(patched).toContain("set fd [open \"|sh -c [list $inline_cmd] 2>@1\" r]"); + expect(patched).toContain("set rc [catch { read $fd 4096 } res]"); expect(patchTestrunnerForKandelo(patched)).toBe(patched); }); it("fails loudly when an upstream anchor changes", () => { expect(() => patchTestrunnerForKandelo("one\ntwo\nthree\nfour\n")).toThrow( - "missing child work-directory anchor", + "missing host-selection switch", ); }); }); diff --git a/scripts/run-sqlite-official-tests.sh b/scripts/run-sqlite-official-tests.sh index e7ad14c965..de602c7f9c 100755 --- a/scripts/run-sqlite-official-tests.sh +++ b/scripts/run-sqlite-official-tests.sh @@ -367,7 +367,7 @@ patch_sqlite_testrunner_platform() { local runner="$1" local tmp - if grep -q "Kandelo platform shim for child testrunner jobs" "$runner"; then + if grep -q "Kandelo testrunner host selection for child jobs" "$runner"; then return fi @@ -375,17 +375,45 @@ patch_sqlite_testrunner_platform() { awk ' NR == 4 { print "" - print "# Kandelo platform shim for child testrunner jobs." - print "# SQLite all-mode reruns config variants by invoking this file directly," - print "# so the platform override has to live in the copied runner as well as" - print "# the initial kandelo-testrunner.tcl wrapper." - print "set ::tcl_platform(os) OpenBSD" - print "set ::tcl_platform(platform) unix" + print "# Kandelo testrunner host selection for child jobs." + print "# This controls only the SQLite helper-script launcher. Keep tcl_platform" + print "# unchanged so the tests continue to observe the real Tcl target." + print "set ::kandelo_testrunner_host Kandelo" + } + $0 == "switch -nocase -glob -- $tcl_platform(os) {" { + print "set testrunner_host $tcl_platform(os)" + print "if {[info exists ::kandelo_testrunner_host]} {" + print " set testrunner_host $::kandelo_testrunner_host" + print "}" + print "switch -nocase -glob -- $testrunner_host {" + next + } + $0 == " *openbsd* {" { + print " *kandelo* {" + print " set TRG(platform) linux" + print " set TRG(make) make.sh" + print " set TRG(makecmd) \"sh make.sh\"" + print " set TRG(testfixture) testfixture" + print " set TRG(shell) sqlite3" + print " set TRG(run) run.sh" + print " set TRG(runcmd) \"sh run.sh\"" + print " }" } { print } ' "$runner" > "$tmp" mv "$tmp" "$runner" chmod a+r "$runner" + + for required in \ + 'set ::kandelo_testrunner_host Kandelo' \ + 'switch -nocase -glob -- $testrunner_host {' \ + '*kandelo* {' + do + if ! grep -Fq "$required" "$runner"; then + echo "ERROR: failed to patch SQLite testrunner.tcl host selection: missing $required" >&2 + exit 1 + fi + done } patch_sqlite_testrunner_guest_paths() { @@ -531,12 +559,9 @@ patch_sqlite_testrunner_guest_paths "$WORKDIR/test/testrunner.tcl" RUNNER_TCL="$WORKDIR/kandelo-testrunner.tcl" cat > "$RUNNER_TCL" <<'TCL' -# Kandelo's Tcl build reports a target OS name that SQLite's testrunner.tcl -# does not classify. Present a Unix-like platform to the upstream runner and -# use its OpenBSD branch so generated helper scripts run with sh instead of -# bash. -set ::tcl_platform(os) OpenBSD -set ::tcl_platform(platform) unix +# Select Kandelo's helper-script launcher without changing Tcl target metadata +# observed by SQLite's tests. +set ::kandelo_testrunner_host Kandelo set argv0 test/testrunner.tcl source $argv0 TCL From e3cd0d0600078f4b4ae5e0a3764dc6dd380636f4 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 15:31:58 -0400 Subject: [PATCH 07/10] SQLite tests: do not double-count omitted cases SQLite includes ordinary Omitted lines in ntest. Add only detail-only skip entries when computing selected-case totals. --- scripts/sqlite-case-outcomes.py | 12 +++++++++++- tests/scripts/sqlite-case-outcomes.sh | 12 ++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/scripts/sqlite-case-outcomes.py b/scripts/sqlite-case-outcomes.py index 1ba1ca2cbf..cc5fbe2873 100755 --- a/scripts/sqlite-case-outcomes.py +++ b/scripts/sqlite-case-outcomes.py @@ -232,6 +232,8 @@ def build_outcomes(jobs: list[Job], source_reason: str | None) -> dict[str, Any] reported_executed_cases = 0 reported_case_errors = 0 + counted_skipped_cases = 0 + detail_only_skipped_cases = 0 for job in jobs: parsed = parse_output(job.output) if job.output else None @@ -277,6 +279,7 @@ def build_outcomes(jobs: list[Job], source_reason: str | None) -> dict[str, Any] reported_case_errors += nerr job_skipped: list[tuple[str, str]] = [] + counted_skipped_names = {name for name, _reason in parsed.skipped_counted} for skipped in parsed.skipped_counted: upsert_named_pair(job_skipped, skipped) for skipped in parsed.skipped_detail: @@ -286,6 +289,8 @@ def build_outcomes(jobs: list[Job], source_reason: str | None) -> dict[str, Any] upsert_named_pair(job_skipped, skipped) for skipped in job_skipped: upsert_named_pair(skipped_cases, skipped) + counted_skipped_cases += len(counted_skipped_names) + detail_only_skipped_cases += sum(1 for name, _reason in job_skipped if name not in counted_skipped_names) if len(parsed.failed) == nerr: failed_cases.extend(parsed.failed) @@ -324,7 +329,10 @@ def build_outcomes(jobs: list[Job], source_reason: str | None) -> dict[str, Any] ) passed_cases.extend(parsed.passed) - selected_cases = reported_executed_cases + len(skipped_cases) + # SQLite includes `... Omitted` cases in ntest, but some harness skips are + # only named in the detailed omission list. Add only those detail-only + # cases so selected_cases does not count ordinary omissions twice. + selected_cases = reported_executed_cases + detail_only_skipped_cases categories = { "passed_cases": { "count": len(passed_cases), @@ -356,6 +364,8 @@ def build_outcomes(jobs: list[Job], source_reason: str | None) -> dict[str, Any] "passed_cases": len(passed_cases), "failed_cases": len(failed_cases), "skipped_cases": len(skipped_cases), + "counted_skipped_cases": counted_skipped_cases, + "detail_only_skipped_cases": detail_only_skipped_cases, "unattributed_passed_cases": sum(row[2] for row in unattributed_passed_cases), "jobs": { "passed": len(passed_jobs), diff --git a/tests/scripts/sqlite-case-outcomes.sh b/tests/scripts/sqlite-case-outcomes.sh index 32053ee9de..88c766917c 100755 --- a/tests/scripts/sqlite-case-outcomes.sh +++ b/tests/scripts/sqlite-case-outcomes.sh @@ -77,6 +77,18 @@ if ! grep -q 'requires an unavailable optional feature' "$OMITTED_RESULTS/outcom exit 1 fi +python3 - "$OMITTED_RESULTS/outcome-lists/case-outcomes.json" <<'PY' +import json +import sys +from pathlib import Path + +summary = json.loads(Path(sys.argv[1]).read_text())["summary"] +assert summary["reported_executed_cases"] == 1, summary +assert summary["selected_cases"] == 1, summary +assert summary["counted_skipped_cases"] == 1, summary +assert summary["detail_only_skipped_cases"] == 0, summary +PY + DB="$TMP_ROOT/testrunner.db" python3 - "$DB" "$STDOUT_FILE" <<'PY' import sqlite3 From 17220efd9477e700809b1cc5693f50c056827fa5 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 15:32:55 -0400 Subject: [PATCH 08/10] SQLite tests: resolve the all-mode guest shell Use the repository binary resolver for dash so ABI and fork-instrumentation guards apply. Keep SQLITE_TEST_SHELL as an explicit override. --- scripts/run-sqlite-official-tests.sh | 27 +++++---------------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/scripts/run-sqlite-official-tests.sh b/scripts/run-sqlite-official-tests.sh index de602c7f9c..9cd14ba2a1 100755 --- a/scripts/run-sqlite-official-tests.sh +++ b/scripts/run-sqlite-official-tests.sh @@ -133,28 +133,11 @@ if [ ! -f "$TESTFIXTURE" ] || [ ! -f "$SQLITE3" ] || [ ! -d "$SQLITE_FULL/test" fi if [ -z "$GUEST_SHELL" ]; then - for candidate in \ - "$REPO_ROOT/local-binaries/programs/wasm32/sh.wasm" \ - "$REPO_ROOT/local-binaries/programs/sh.wasm" \ - "$REPO_ROOT/local-binaries/programs/wasm32/dash.wasm" \ - "$REPO_ROOT/local-binaries/programs/dash.wasm" \ - "$REPO_ROOT/binaries/programs/wasm32/sh.wasm" \ - "$REPO_ROOT/binaries/programs/sh.wasm" \ - "$REPO_ROOT/binaries/programs/wasm32/dash.wasm" \ - "$REPO_ROOT/binaries/programs/dash.wasm" \ - "$REPO_ROOT/packages/registry/dash/bin/dash.wasm" - do - if [ -f "$candidate" ]; then - GUEST_SHELL="$candidate" - break - fi - done -fi - -if [ "$PERMUTATION" = "all" ] && [ -z "$GUEST_SHELL" ]; then - echo "ERROR: SQLite all-mode config jobs require a guest /bin/sh-compatible shell." >&2 - echo "Build or fetch dash/sh, or set SQLITE_TEST_SHELL=/path/to/sh.wasm." >&2 - exit 1 + if ! GUEST_SHELL="$("$REPO_ROOT/scripts/resolve-binary.sh" programs/dash.wasm)"; then + echo "ERROR: SQLite testrunner child jobs require a current guest /bin/sh-compatible shell." >&2 + echo "Fetch/build dash, or set SQLITE_TEST_SHELL=/path/to/sh.wasm." >&2 + exit 1 + fi fi if [ -z "$WORKDIR" ]; then From b08bf6f31c625bc6b463b581bf2fa6d565fc065b Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 15:34:01 -0400 Subject: [PATCH 09/10] SQLite tests: fail when outcome evidence is incomplete Treat report generation as part of the run result, preserve an earlier runner failure, and classify done jobs with case errors consistently on Node and browser. --- scripts/browser-sqlite-official-runner.ts | 9 +-- scripts/run-browser-sqlite-official-tests.sh | 66 +++++++++++++++----- scripts/run-sqlite-official-tests.sh | 61 ++++++++++++++---- 3 files changed, 105 insertions(+), 31 deletions(-) diff --git a/scripts/browser-sqlite-official-runner.ts b/scripts/browser-sqlite-official-runner.ts index cdbbf85880..06d4498246 100755 --- a/scripts/browser-sqlite-official-runner.ts +++ b/scripts/browser-sqlite-official-runner.ts @@ -114,8 +114,8 @@ function inferDisplayName(command: string[]): string { return command.join(" "); } -function writeDirectOutcomeArtifacts(resultsDir: string, command: string[], stdout: string, stderr: string): void { - if (!resultsDir) return; +function writeDirectOutcomeArtifacts(resultsDir: string, command: string[], stdout: string, stderr: string): boolean { + if (!resultsDir) return true; mkdirSync(resultsDir, { recursive: true }); const stdoutPath = resolve(resultsDir, "runner.stdout"); const stderrPath = resolve(resultsDir, "runner.stderr"); @@ -138,6 +138,7 @@ function writeDirectOutcomeArtifacts(resultsDir: string, command: string[], stdo if (outcome.status !== 0) { console.error(`[sqlite-outcomes] case outcome extraction failed with status ${outcome.status}`); } + return outcome.status === 0; } async function main() { @@ -221,11 +222,11 @@ async function main() { } if (!result.artifacts && latestArtifacts) result.artifacts = latestArtifacts; writeArtifacts(resultsDir, result.artifacts); - writeDirectOutcomeArtifacts(resultsDir, command, result.stdout, result.stderr); + const outcomesOk = writeDirectOutcomeArtifacts(resultsDir, command, result.stdout, result.stderr); if (result.stdout) process.stdout.write(result.stdout); if (result.stderr) process.stderr.write(result.stderr); if (result.error) process.stderr.write(`${result.error}\n`); - process.exit(result.exitCode === 0 ? 0 : 1); + process.exit(result.exitCode === 0 && outcomesOk ? 0 : 1); } finally { await browser?.close().catch(() => {}); if (vite) { diff --git a/scripts/run-browser-sqlite-official-tests.sh b/scripts/run-browser-sqlite-official-tests.sh index ef95d402da..daa1e9d6d0 100755 --- a/scripts/run-browser-sqlite-official-tests.sh +++ b/scripts/run-browser-sqlite-official-tests.sh @@ -97,7 +97,7 @@ write_outcome_lists() { coalesce(span, 0) AS ms, 'testrunner.db' AS source FROM jobs - WHERE state = 'done' + WHERE state = 'done' AND coalesce(nerr, 0) = 0 ORDER BY jobid;" > "$out/passed-jobs.tsv" sqlite3 -header -separator $'\t' "$db" \ @@ -108,6 +108,7 @@ write_outcome_lists() { 'testrunner.db' AS source FROM jobs WHERE state = 'failed' + OR (state = 'done' AND coalesce(nerr, 0) > 0) ORDER BY jobid;" > "$out/failed-jobs.tsv" sqlite3 -header -separator $'\t' "$db" \ @@ -133,14 +134,14 @@ write_outcome_lists() { END AS reason, 'testrunner.db' AS source FROM jobs - WHERE state IN ('running', 'ready') + WHERE coalesce(state, '') NOT IN ('done', 'failed', 'omit') ORDER BY state, jobid;" > "$out/incomplete-jobs.tsv" sqlite3 -header -separator $'\t' "$db" \ - "SELECT sum(state='done') AS passed_jobs, - sum(state='failed') AS failed_jobs, + "SELECT sum(state='done' AND coalesce(nerr, 0)=0) AS passed_jobs, + sum(state='failed' OR (state='done' AND coalesce(nerr, 0)>0)) AS failed_jobs, sum(state='omit') AS skipped_jobs, - sum(state IN ('running','ready')) AS incomplete_jobs, + coalesce(sum(coalesce(state,'') NOT IN ('done','failed','omit')), 0) AS incomplete_jobs, 'testrunner.db' AS source FROM jobs;" > "$out/counts.tsv" } @@ -152,7 +153,7 @@ write_sqlite_report() { if [ ! -f "$db" ]; then echo "No testrunner.db was created at $db" > "$report" write_unavailable_outcome_lists "No testrunner.db was created at $db." - return + return 1 fi if ! sqlite3 "$db" "SELECT 1 FROM sqlite_master WHERE type='table' AND name='jobs' LIMIT 1;" | grep -qx 1; then @@ -173,7 +174,7 @@ write_sqlite_report() { write_unavailable_outcome_lists "No usable jobs table was found in $db." echo "===== SQLite official testrunner database summary =====" cat "$report" - return + return 1 fi write_outcome_lists "$db" @@ -225,12 +226,13 @@ write_sqlite_report() { GROUP BY config ORDER BY CASE WHEN config='full' THEN 0 ELSE 1 END, config;" echo - echo "Failed, running, and omitted jobs:" + echo "Unsuccessful, incomplete, and omitted jobs:" sqlite3 -header -column "$db" \ "SELECT jobid, state, displaytype, displayname, coalesce(ntest, 0) AS cases, coalesce(nerr, 0) AS errors, coalesce(span, 0) AS ms FROM jobs - WHERE state IN ('failed', 'running', 'omit') + WHERE coalesce(state, '')!='done' + OR coalesce(nerr, 0)>0 ORDER BY state, jobid;" } > "$report" @@ -238,17 +240,46 @@ write_sqlite_report() { "SELECT jobid, state, displaytype, displayname, coalesce(ntest, 0) AS cases, coalesce(nerr, 0) AS errors, coalesce(span, 0) AS ms FROM jobs - WHERE state IN ('failed', 'running', 'omit') + WHERE coalesce(state, '')!='done' + OR coalesce(nerr, 0)>0 ORDER BY state, jobid;" > "$failures" - python3 "$REPO_ROOT/scripts/sqlite-case-outcomes.py" \ + if ! python3 "$REPO_ROOT/scripts/sqlite-case-outcomes.py" \ --db "$db" \ --results-dir "$RESULTS_DIR" \ --host browser \ - --permutation "$PERMUTATION" || true + --permutation "$PERMUTATION" + then + echo "ERROR: failed to write SQLite case outcome artifacts" >&2 + return 1 + fi echo "===== SQLite official testrunner database summary =====" cat "$report" + + local total_jobs unsuccessful_jobs + total_jobs="$(sqlite3 "$db" "SELECT count(*) FROM jobs;")" + if [ "$total_jobs" -eq 0 ]; then + echo "ERROR: SQLite testrunner selected no jobs" >&2 + return 1 + fi + if $EXPLAIN; then + unsuccessful_jobs="$(sqlite3 "$db" \ + "SELECT count(*) FROM jobs + WHERE state NOT IN ('', 'ready') + OR coalesce(nerr, 0)>0;")" + else + unsuccessful_jobs="$(sqlite3 "$db" \ + "SELECT count(*) FROM jobs + WHERE state!='done' + OR ntest IS NULL + OR nerr IS NULL + OR nerr>0;")" + fi + if [ "$unsuccessful_jobs" -ne 0 ]; then + echo "ERROR: SQLite testrunner recorded $unsuccessful_jobs unsuccessful or incomplete job(s)" >&2 + return 1 + fi } ARGS=(testfixture kandelo-testrunner.tcl --jobs "$JOBS") @@ -272,5 +303,12 @@ node --import tsx/esm "$REPO_ROOT/scripts/browser-sqlite-official-runner.ts" \ status=$? set -e -write_sqlite_report || true -exit "$status" +set +e +(set -e; write_sqlite_report) +report_status=$? +set -e + +if [ "$status" -ne 0 ]; then + exit "$status" +fi +exit "$report_status" diff --git a/scripts/run-sqlite-official-tests.sh b/scripts/run-sqlite-official-tests.sh index 9cd14ba2a1..0e5adc84e3 100755 --- a/scripts/run-sqlite-official-tests.sh +++ b/scripts/run-sqlite-official-tests.sh @@ -216,14 +216,14 @@ write_outcome_lists() { END AS reason, 'testrunner.db' AS source FROM jobs - WHERE state IN ('running', 'ready') + WHERE coalesce(state, '') NOT IN ('done', 'failed', 'omit') ORDER BY state, jobid;" > "$out/incomplete-jobs.tsv" sqlite3 -header -separator $'\t' "$db" \ "SELECT coalesce(sum(state='done' AND coalesce(nerr, 0)=0), 0) AS passed_jobs, coalesce(sum(state='failed' OR (state='done' AND coalesce(nerr, 0)>0)), 0) AS failed_jobs, coalesce(sum(state='omit'), 0) AS skipped_jobs, - coalesce(sum(state IN ('running','ready')), 0) AS incomplete_jobs, + coalesce(sum(coalesce(state,'') NOT IN ('done','failed','omit')), 0) AS incomplete_jobs, 'testrunner.db' AS source FROM jobs;" > "$out/counts.tsv" } @@ -235,7 +235,7 @@ write_sqlite_report() { if [ ! -f "$db" ]; then echo "No testrunner.db was created at $db" > "$report" write_unavailable_outcome_lists "No testrunner.db was created at $db." - return + return 1 fi mkdir -p "$RESULTS_DIR" @@ -265,7 +265,7 @@ write_sqlite_report() { write_unavailable_outcome_lists "No usable jobs table was found in $db." echo "===== SQLite official testrunner database summary =====" cat "$report" - return + return 1 fi write_outcome_lists "$db" @@ -318,13 +318,13 @@ write_sqlite_report() { GROUP BY config ORDER BY CASE WHEN config='full' THEN 0 ELSE 1 END, config;" echo - echo "Failed, running, and omitted jobs:" + echo "Unsuccessful, incomplete, and omitted jobs:" sqlite3 -header -column "$db" \ "SELECT jobid, state, displaytype, displayname, coalesce(ntest, 0) AS cases, coalesce(nerr, 0) AS errors, coalesce(span, 0) AS ms FROM jobs - WHERE state IN ('failed', 'running', 'omit') - OR (state='done' AND coalesce(nerr, 0)>0) + WHERE coalesce(state, '')!='done' + OR coalesce(nerr, 0)>0 ORDER BY state, jobid;" } > "$report" @@ -332,18 +332,46 @@ write_sqlite_report() { "SELECT jobid, state, displaytype, displayname, coalesce(ntest, 0) AS cases, coalesce(nerr, 0) AS errors, coalesce(span, 0) AS ms FROM jobs - WHERE state IN ('failed', 'running', 'omit') - OR (state='done' AND coalesce(nerr, 0)>0) + WHERE coalesce(state, '')!='done' + OR coalesce(nerr, 0)>0 ORDER BY state, jobid;" > "$failures" - python3 "$REPO_ROOT/scripts/sqlite-case-outcomes.py" \ + if ! python3 "$REPO_ROOT/scripts/sqlite-case-outcomes.py" \ --db "$RESULTS_DIR/testrunner.db" \ --results-dir "$RESULTS_DIR" \ --host "$HOST" \ - --permutation "$PERMUTATION" || true + --permutation "$PERMUTATION" + then + echo "ERROR: failed to write SQLite case outcome artifacts" >&2 + return 1 + fi echo "===== SQLite official testrunner database summary =====" cat "$report" + + local total_jobs unsuccessful_jobs + total_jobs="$(sqlite3 "$db" "SELECT count(*) FROM jobs;")" + if [ "$total_jobs" -eq 0 ]; then + echo "ERROR: SQLite testrunner selected no jobs" >&2 + return 1 + fi + if $EXPLAIN; then + unsuccessful_jobs="$(sqlite3 "$db" \ + "SELECT count(*) FROM jobs + WHERE state NOT IN ('', 'ready') + OR coalesce(nerr, 0)>0;")" + else + unsuccessful_jobs="$(sqlite3 "$db" \ + "SELECT count(*) FROM jobs + WHERE state!='done' + OR ntest IS NULL + OR nerr IS NULL + OR nerr>0;")" + fi + if [ "$unsuccessful_jobs" -ne 0 ]; then + echo "ERROR: SQLite testrunner recorded $unsuccessful_jobs unsuccessful or incomplete job(s)" >&2 + return 1 + fi } patch_sqlite_testrunner_platform() { @@ -577,5 +605,12 @@ node --experimental-wasm-exnref --import tsx/esm \ status=$? set -e -write_sqlite_report || true -exit "$status" +set +e +(set -e; write_sqlite_report) +report_status=$? +set -e + +if [ "$status" -ne 0 ]; then + exit "$status" +fi +exit "$report_status" From f758ecb30b92b9c914e950094706f05169bb205e Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Mon, 13 Jul 2026 15:36:30 -0400 Subject: [PATCH 10/10] SQLite tests: clean up the browser runner Set the eventual process status without bypassing finally, so normal and failing runs both close Chromium and stop Vite. --- scripts/browser-sqlite-official-runner.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/browser-sqlite-official-runner.ts b/scripts/browser-sqlite-official-runner.ts index 06d4498246..f3ebac0129 100755 --- a/scripts/browser-sqlite-official-runner.ts +++ b/scripts/browser-sqlite-official-runner.ts @@ -226,7 +226,8 @@ async function main() { if (result.stdout) process.stdout.write(result.stdout); if (result.stderr) process.stderr.write(result.stderr); if (result.error) process.stderr.write(`${result.error}\n`); - process.exit(result.exitCode === 0 && outcomesOk ? 0 : 1); + process.exitCode = result.exitCode === 0 && outcomesOk ? 0 : 1; + return; } finally { await browser?.close().catch(() => {}); if (vite) {