diff --git a/apps/browser-demos/pages/sqlite-test/main.ts b/apps/browser-demos/pages/sqlite-test/main.ts index 85bdab7f8..3105cc4a8 100644 --- a/apps/browser-demos/pages/sqlite-test/main.ts +++ b/apps/browser-demos/pages/sqlite-test/main.ts @@ -7,6 +7,10 @@ import { BrowserKernel } from "@host/browser-kernel-host"; import { MemoryFileSystem } from "@host/vfs/memory-fs"; import { writeVfsFile } from "@host/vfs/image-helpers"; import { finalizeKernelOwnedImage, settleWebKitReclaim } from "../../lib/kernel-owned-boot"; +import { + patchTestrunnerForKandelo, + testrunnerPlatformShim, +} from "./testrunner-patch"; import kernelWasmUrl from "@kernel-wasm?url"; declare global { @@ -91,6 +95,20 @@ async function collectArtifactsFromKernel( return artifacts.length > 0 ? artifacts : undefined; } +function installTestrunnerPatches(fs: MemoryFileSystem): void { + const runnerPath = "/sqlite/test/testrunner.tcl"; + const decoder = new TextDecoder(); + const runner = decoder.decode(readVfsFile(fs, runnerPath)); + writeVfsFile(fs, runnerPath, patchTestrunnerForKandelo(runner), 0o644); + + writeVfsFile(fs, "/sqlite/kandelo-testrunner.tcl", [ + testrunnerPlatformShim, + "set argv0 test/testrunner.tcl", + "source $argv0", + "", + ].join("\n"), 0o644); +} + function createFs(): MemoryFileSystem { if (!vfsImageBytes) throw new Error("SQLite test VFS image not loaded"); const fs = MemoryFileSystem.fromImage(vfsImageBytes, { @@ -153,13 +171,7 @@ async function init() { // SharedArrayBuffer across the per-test loop (Safari OOM fix). const buildFs = createFs(); if (argv[1] === "kandelo-testrunner.tcl") { - writeVfsFile(buildFs, "/sqlite/kandelo-testrunner.tcl", [ - "set ::tcl_platform(os) OpenBSD", - "set ::tcl_platform(platform) unix", - "set argv0 test/testrunner.tcl", - "source $argv0", - "", - ].join("\n"), 0o644); + installTestrunnerPatches(buildFs); } const vfsImage = await finalizeKernelOwnedImage(buildFs); const kernel = new BrowserKernel({ diff --git a/apps/browser-demos/pages/sqlite-test/testrunner-patch.ts b/apps/browser-demos/pages/sqlite-test/testrunner-patch.ts new file mode 100644 index 000000000..6a7ff853b --- /dev/null +++ b/apps/browser-demos/pages/sqlite-test/testrunner-patch.ts @@ -0,0 +1,189 @@ +export const testrunnerPlatformShim = [ + "# Kandelo testrunner host selection for child jobs.", + "# This controls only SQLite's helper-script launcher. Keep tcl_platform", + "# unchanged so the tests continue to observe the real Tcl target.", + "set ::kandelo_testrunner_host Kandelo", +].join("\n"); + +const testrunnerGuestPathShim = [ + "# Kandelo guest path shim for all-mode child jobs.", + "# testrunner.tcl builds child run.sh files from host-normalized paths;", + "# convert workdir-local paths back to paths relative to each testdirN", + "# directory, because SQLite runs the script after cd-ing into it.", + "proc kandelo_guest_path {path} {", + " if {[file pathtype $path] != \"absolute\" && [string equal $path [info nameofexec]]} {", + " return $path", + " }", + " set normalized [file normalize $path]", + " set topdir [file normalize [file dirname $::testdir]]", + " set script [file normalize [info script]]", + " if {[string equal $normalized $script]} { return \"../test/testrunner.tcl\" }", + " if {[string equal $normalized $topdir]} { return \"..\" }", + " set prefix \"${topdir}/\"", + " if {[string first $prefix $normalized] == 0} {", + " return \"../[string range $normalized [string length $prefix] end]\"", + " }", + " return $path", + "}", + "set ::kandelo_inline_run_sh 1", + "set ::kandelo_chunk_pipe_output 1", +].join("\n"); + +function replaceRequired(source: string, search: string, replacement: string, label: string): string { + if (!source.includes(search)) { + throw new Error(`SQLite testrunner patch is incompatible: missing ${label}`); + } + return source.replace(search, replacement); +} + +export function patchTestrunnerForKandelo(runner: string): string { + let patched = runner; + + if (!patched.includes("Kandelo testrunner host selection for child jobs")) { + const lines = patched.split("\n"); + if (lines.length < 4) { + throw new Error("SQLite testrunner patch is incompatible: file is shorter than four lines"); + } + lines.splice(3, 0, "", testrunnerPlatformShim); + patched = lines.join("\n"); + patched = replaceRequired( + patched, + "switch -nocase -glob -- $tcl_platform(os) {", + [ + "set testrunner_host $tcl_platform(os)", + "if {[info exists ::kandelo_testrunner_host]} {", + " set testrunner_host $::kandelo_testrunner_host", + "}", + "switch -nocase -glob -- $testrunner_host {", + ].join("\n"), + "host-selection switch", + ); + patched = replaceRequired( + patched, + " *openbsd* {\n", + [ + " *kandelo* {", + " set TRG(platform) linux", + " set TRG(make) make.sh", + " set TRG(makecmd) \"sh make.sh\"", + " set TRG(testfixture) testfixture", + " set TRG(shell) sqlite3", + " set TRG(run) run.sh", + " set TRG(runcmd) \"sh run.sh\"", + " }", + " *openbsd* {", + "", + ].join("\n"), + "OpenBSD host branch", + ); + } + + if (!patched.includes("Kandelo guest path shim for all-mode child jobs")) { + patched = replaceRequired( + patched, + "cd $dir\n", + `cd $dir\n\n${testrunnerGuestPathShim}\n`, + "child work-directory anchor", + ); + patched = replaceRequired( + patched, + " set displayname [string map [list $topdir/ {}] $f]\n", + [ + " set displayname [string map [list $topdir/ {}] $f]", + " set testfixture_guest [kandelo_guest_path $testfixture]", + " set testrunner_tcl_guest [kandelo_guest_path $testrunner_tcl]", + " set f_guest [kandelo_guest_path $f]", + "", + ].join("\n"), + "job display-name anchor", + ); + patched = replaceRequired( + patched, + " set cmd \"$testfixture $f\"", + " set cmd \"$testfixture_guest $f_guest\"", + "direct test command anchor", + ); + patched = replaceRequired( + patched, + " set cmd \"$testfixture $testrunner_tcl $config $f\"", + " set cmd \"$testfixture_guest $testrunner_tcl_guest $config $f_guest\"", + "configured test command anchor", + ); + patched = replaceRequired( + patched, + " set set_tmp_dir \"export SQLITE_TMPDIR=\\\"[file normalize $dir]\\\"\"", + " set set_tmp_dir \"export SQLITE_TMPDIR=.\"", + "temporary-directory anchor", + ); + patched = replaceRequired( + patched, + " set fd [open \"|$TRG(runcmd) 2>@1\" r]", + [ + " if {[info exists ::kandelo_inline_run_sh] && $::kandelo_inline_run_sh} {", + " set inline_cmd \"$set_tmp_dir\\n$job(cmd)\"", + " set fd [open \"|sh -c [list $inline_cmd] 2>@1\" r]", + " } else {", + " set fd [open \"|$TRG(runcmd) 2>@1\" r]", + " }", + ].join("\n"), + "child shell command anchor", + ); + patched = replaceRequired( + patched, + " set rc [catch { gets $fd line } res]", + [ + " if {[info exists ::kandelo_chunk_pipe_output] && $::kandelo_chunk_pipe_output} {", + " set rc [catch { read $fd 4096 } res]", + " if {$rc} {", + " puts \"ERROR $res\"", + " }", + " if {!$rc && [string length $res] > 0} {", + " append O($iJob) $res", + " }", + " } else {", + " set rc [catch { gets $fd line } res]", + ].join("\n"), + "pipe read anchor", + ); + patched = replaceRequired( + patched, + " if {$res>=0} {", + [ + " if {![info exists ::kandelo_chunk_pipe_output] || !$::kandelo_chunk_pipe_output} {", + " if {$res>=0} {", + ].join("\n"), + "line-result anchor", + ); + patched = replaceRequired( + patched, + " append O($iJob) \"$line\\n\"", + [ + " append O($iJob) \"$line\\n\"", + " }", + " }", + ].join("\n"), + "line append anchor", + ); + } + + for (const required of [ + "Kandelo testrunner host selection for child jobs", + "set ::kandelo_testrunner_host Kandelo", + "switch -nocase -glob -- $testrunner_host", + "*kandelo* {", + "Kandelo guest path shim for all-mode child jobs", + "set testfixture_guest [kandelo_guest_path $testfixture]", + "set cmd \"$testfixture_guest $f_guest\"", + "set cmd \"$testfixture_guest $testrunner_tcl_guest $config $f_guest\"", + "set set_tmp_dir \"export SQLITE_TMPDIR=.\"", + "set fd [open \"|sh -c [list $inline_cmd] 2>@1\" r]", + "set ::kandelo_chunk_pipe_output 1", + "set rc [catch { read $fd 4096 } res]", + ]) { + if (!patched.includes(required)) { + throw new Error(`SQLite testrunner patch is incomplete: missing ${required}`); + } + } + + return patched; +} diff --git a/host/test/sqlite-testrunner-patch.test.ts b/host/test/sqlite-testrunner-patch.test.ts new file mode 100644 index 000000000..6bd2eeb28 --- /dev/null +++ b/host/test/sqlite-testrunner-patch.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; +import { patchTestrunnerForKandelo } from "../../apps/browser-demos/pages/sqlite-test/testrunner-patch"; + +const upstreamFixture = [ + "#!/usr/bin/env tclsh", + "# SQLite test runner fixture", + "set TRG(nJob) 1", + "set dir testdir1", + "switch -nocase -glob -- $tcl_platform(os) {", + " *openbsd* {", + " }", + "}", + "cd $dir", + " set displayname [string map [list $topdir/ {}] $f]", + " set cmd \"$testfixture $f\"", + " set cmd \"$testfixture $testrunner_tcl $config $f\"", + " set set_tmp_dir \"export SQLITE_TMPDIR=\\\"[file normalize $dir]\\\"\"", + " set fd [open \"|$TRG(runcmd) 2>@1\" r]", + " set rc [catch { gets $fd line } res]", + " if {$res>=0} {", + " append O($iJob) \"$line\\n\"", + " }", + "", +].join("\n"); + +describe("SQLite browser testrunner patch", () => { + it("rewrites all-mode child commands and is idempotent", () => { + const patched = patchTestrunnerForKandelo(upstreamFixture); + + expect(patched).toContain("set ::kandelo_testrunner_host Kandelo"); + expect(patched).not.toContain("set ::tcl_platform(os)"); + expect(patched).toContain("switch -nocase -glob -- $testrunner_host"); + expect(patched).toContain("*kandelo* {"); + expect(patched).toContain("set testfixture_guest [kandelo_guest_path $testfixture]"); + expect(patched).toContain("set cmd \"$testfixture_guest $f_guest\""); + expect(patched).toContain("set cmd \"$testfixture_guest $testrunner_tcl_guest $config $f_guest\""); + expect(patched).toContain("set set_tmp_dir \"export SQLITE_TMPDIR=.\""); + expect(patched).toContain("set fd [open \"|sh -c [list $inline_cmd] 2>@1\" r]"); + expect(patched).toContain("set rc [catch { read $fd 4096 } res]"); + expect(patchTestrunnerForKandelo(patched)).toBe(patched); + }); + + it("fails loudly when an upstream anchor changes", () => { + expect(() => patchTestrunnerForKandelo("one\ntwo\nthree\nfour\n")).toThrow( + "missing host-selection switch", + ); + }); +}); diff --git a/scripts/browser-sqlite-official-runner.ts b/scripts/browser-sqlite-official-runner.ts index 2715c11f9..f3ebac012 100755 --- a/scripts/browser-sqlite-official-runner.ts +++ b/scripts/browser-sqlite-official-runner.ts @@ -1,5 +1,5 @@ #!/usr/bin/env tsx -import { spawn, type ChildProcess } from "node:child_process"; +import { spawn, spawnSync, type ChildProcess } from "node:child_process"; import { existsSync, mkdirSync, writeFileSync } from "node:fs"; import { createServer } from "node:net"; import { resolve } from "node:path"; @@ -108,6 +108,39 @@ function writeArtifacts(resultsDir: string, artifacts: BrowserArtifact[] | undef } } +function inferDisplayName(command: string[]): string { + const tests = command.filter((arg) => arg.endsWith(".test")); + if (tests.length > 0) return tests[tests.length - 1]; + return command.join(" "); +} + +function writeDirectOutcomeArtifacts(resultsDir: string, command: string[], stdout: string, stderr: string): boolean { + if (!resultsDir) return true; + mkdirSync(resultsDir, { recursive: true }); + const stdoutPath = resolve(resultsDir, "runner.stdout"); + const stderrPath = resolve(resultsDir, "runner.stderr"); + const dbPath = resolve(resultsDir, "testrunner.db"); + writeFileSync(stdoutPath, stdout); + writeFileSync(stderrPath, stderr); + const sourceArgs = existsSync(dbPath) + ? ["--db", dbPath] + : ["--stdout-file", stdoutPath, "--display-name", inferDisplayName(command)]; + const outcome = spawnSync( + "python3", + [ + resolve(REPO_ROOT, "scripts/sqlite-case-outcomes.py"), + ...sourceArgs, + "--results-dir", resultsDir, + "--host", "browser", + ], + { cwd: REPO_ROOT, stdio: "inherit" }, + ); + if (outcome.status !== 0) { + console.error(`[sqlite-outcomes] case outcome extraction failed with status ${outcome.status}`); + } + return outcome.status === 0; +} + async function main() { const argv = process.argv.slice(2); let timeoutMs = 600_000; @@ -189,10 +222,12 @@ async function main() { } if (!result.artifacts && latestArtifacts) result.artifacts = latestArtifacts; writeArtifacts(resultsDir, result.artifacts); + const outcomesOk = writeDirectOutcomeArtifacts(resultsDir, command, result.stdout, result.stderr); if (result.stdout) process.stdout.write(result.stdout); if (result.stderr) process.stderr.write(result.stderr); if (result.error) process.stderr.write(`${result.error}\n`); - process.exit(result.exitCode === 0 ? 0 : 1); + process.exitCode = result.exitCode === 0 && outcomesOk ? 0 : 1; + return; } finally { await browser?.close().catch(() => {}); if (vite) { diff --git a/scripts/run-browser-sqlite-official-tests.sh b/scripts/run-browser-sqlite-official-tests.sh index 1103e76d7..daa1e9d6d 100755 --- a/scripts/run-browser-sqlite-official-tests.sh +++ b/scripts/run-browser-sqlite-official-tests.sh @@ -69,13 +69,91 @@ if [ -z "$RESULTS_DIR" ]; then fi mkdir -p "$RESULTS_DIR" +write_unavailable_outcome_lists() { + local reason="$1" + local out="$RESULTS_DIR/outcome-lists" + + mkdir -p "$out" + printf 'jobid\tstate\tdisplaytype\tdisplayname\tcases\terrors\tms\tsource\n' > "$out/passed-jobs.tsv" + printf 'jobid\tstate\tdisplaytype\tdisplayname\tcases\terrors\tms\tsource\n' > "$out/failed-jobs.tsv" + printf 'jobid\tstate\tdisplaytype\tdisplayname\tcases\terrors\tms\treason\tsource\n' > "$out/skipped-jobs.tsv" + printf 'jobid\tstate\tdisplaytype\tdisplayname\tcases\terrors\tms\treason\tsource\n' > "$out/incomplete-jobs.tsv" + printf '\tunavailable\t\t\t0\t0\t0\t%s\trunner\n' "$reason" >> "$out/incomplete-jobs.tsv" + { + printf 'passed_jobs\tfailed_jobs\tskipped_jobs\tincomplete_jobs\tnote\n' + printf '0\t0\t0\t1\t%s\n' "$reason" + } > "$out/counts.tsv" +} + +write_outcome_lists() { + local db="$1" + local out="$RESULTS_DIR/outcome-lists" + + mkdir -p "$out" + sqlite3 -header -separator $'\t' "$db" \ + "SELECT jobid, state, displaytype, displayname, + coalesce(ntest, 0) AS cases, + coalesce(nerr, 0) AS errors, + coalesce(span, 0) AS ms, + 'testrunner.db' AS source + FROM jobs + WHERE state = 'done' AND coalesce(nerr, 0) = 0 + ORDER BY jobid;" > "$out/passed-jobs.tsv" + + sqlite3 -header -separator $'\t' "$db" \ + "SELECT jobid, state, displaytype, displayname, + coalesce(ntest, 0) AS cases, + coalesce(nerr, 0) AS errors, + coalesce(span, 0) AS ms, + 'testrunner.db' AS source + FROM jobs + WHERE state = 'failed' + OR (state = 'done' AND coalesce(nerr, 0) > 0) + ORDER BY jobid;" > "$out/failed-jobs.tsv" + + sqlite3 -header -separator $'\t' "$db" \ + "SELECT jobid, state, displaytype, displayname, + coalesce(ntest, 0) AS cases, + coalesce(nerr, 0) AS errors, + coalesce(span, 0) AS ms, + 'runner omitted' AS reason, + 'testrunner.db' AS source + FROM jobs + WHERE state = 'omit' + ORDER BY jobid;" > "$out/skipped-jobs.tsv" + + sqlite3 -header -separator $'\t' "$db" \ + "SELECT jobid, state, displaytype, displayname, + coalesce(ntest, 0) AS cases, + coalesce(nerr, 0) AS errors, + coalesce(span, 0) AS ms, + CASE state + WHEN 'running' THEN 'runner exited before job completed' + WHEN 'ready' THEN 'not started before runner exit' + ELSE 'not completed before runner exit' + END AS reason, + 'testrunner.db' AS source + FROM jobs + WHERE coalesce(state, '') NOT IN ('done', 'failed', 'omit') + ORDER BY state, jobid;" > "$out/incomplete-jobs.tsv" + + sqlite3 -header -separator $'\t' "$db" \ + "SELECT sum(state='done' AND coalesce(nerr, 0)=0) AS passed_jobs, + sum(state='failed' OR (state='done' AND coalesce(nerr, 0)>0)) AS failed_jobs, + sum(state='omit') AS skipped_jobs, + coalesce(sum(coalesce(state,'') NOT IN ('done','failed','omit')), 0) AS incomplete_jobs, + 'testrunner.db' AS source + FROM jobs;" > "$out/counts.tsv" +} + write_sqlite_report() { local db="$RESULTS_DIR/testrunner.db" local report="$RESULTS_DIR/summary.txt" local failures="$RESULTS_DIR/failures.tsv" if [ ! -f "$db" ]; then echo "No testrunner.db was created at $db" > "$report" - return + write_unavailable_outcome_lists "No testrunner.db was created at $db." + return 1 fi if ! sqlite3 "$db" "SELECT 1 FROM sqlite_master WHERE type='table' AND name='jobs' LIMIT 1;" | grep -qx 1; then @@ -93,11 +171,14 @@ write_sqlite_report() { find "$RESULTS_DIR" -maxdepth 1 -type f -name 'testrunner.*' -print | sort } > "$report" : > "$failures" + write_unavailable_outcome_lists "No usable jobs table was found in $db." echo "===== SQLite official testrunner database summary =====" cat "$report" - return + return 1 fi + write_outcome_lists "$db" + { echo "SQLite official testrunner summary" echo "host=browser" @@ -145,12 +226,13 @@ write_sqlite_report() { GROUP BY config ORDER BY CASE WHEN config='full' THEN 0 ELSE 1 END, config;" echo - echo "Failed, running, and omitted jobs:" + echo "Unsuccessful, incomplete, and omitted jobs:" sqlite3 -header -column "$db" \ "SELECT jobid, state, displaytype, displayname, coalesce(ntest, 0) AS cases, coalesce(nerr, 0) AS errors, coalesce(span, 0) AS ms FROM jobs - WHERE state IN ('failed', 'running', 'omit') + WHERE coalesce(state, '')!='done' + OR coalesce(nerr, 0)>0 ORDER BY state, jobid;" } > "$report" @@ -158,11 +240,46 @@ write_sqlite_report() { "SELECT jobid, state, displaytype, displayname, coalesce(ntest, 0) AS cases, coalesce(nerr, 0) AS errors, coalesce(span, 0) AS ms FROM jobs - WHERE state IN ('failed', 'running', 'omit') + WHERE coalesce(state, '')!='done' + OR coalesce(nerr, 0)>0 ORDER BY state, jobid;" > "$failures" + if ! python3 "$REPO_ROOT/scripts/sqlite-case-outcomes.py" \ + --db "$db" \ + --results-dir "$RESULTS_DIR" \ + --host browser \ + --permutation "$PERMUTATION" + then + echo "ERROR: failed to write SQLite case outcome artifacts" >&2 + return 1 + fi + echo "===== SQLite official testrunner database summary =====" cat "$report" + + local total_jobs unsuccessful_jobs + total_jobs="$(sqlite3 "$db" "SELECT count(*) FROM jobs;")" + if [ "$total_jobs" -eq 0 ]; then + echo "ERROR: SQLite testrunner selected no jobs" >&2 + return 1 + fi + if $EXPLAIN; then + unsuccessful_jobs="$(sqlite3 "$db" \ + "SELECT count(*) FROM jobs + WHERE state NOT IN ('', 'ready') + OR coalesce(nerr, 0)>0;")" + else + unsuccessful_jobs="$(sqlite3 "$db" \ + "SELECT count(*) FROM jobs + WHERE state!='done' + OR ntest IS NULL + OR nerr IS NULL + OR nerr>0;")" + fi + if [ "$unsuccessful_jobs" -ne 0 ]; then + echo "ERROR: SQLite testrunner recorded $unsuccessful_jobs unsuccessful or incomplete job(s)" >&2 + return 1 + fi } ARGS=(testfixture kandelo-testrunner.tcl --jobs "$JOBS") @@ -186,5 +303,12 @@ node --import tsx/esm "$REPO_ROOT/scripts/browser-sqlite-official-runner.ts" \ status=$? set -e -write_sqlite_report || true -exit "$status" +set +e +(set -e; write_sqlite_report) +report_status=$? +set -e + +if [ "$status" -ne 0 ]; then + exit "$status" +fi +exit "$report_status" diff --git a/scripts/run-sqlite-official-tests.sh b/scripts/run-sqlite-official-tests.sh index a27544ad4..0e5adc84e 100755 --- a/scripts/run-sqlite-official-tests.sh +++ b/scripts/run-sqlite-official-tests.sh @@ -12,6 +12,7 @@ SQLITE_FULL="$REPO_ROOT/packages/registry/sqlite/sqlite-full-src" TCL_INSTALL="$REPO_ROOT/packages/registry/tcl/tcl-install" TESTFIXTURE="$REPO_ROOT/packages/registry/sqlite/bin/testfixture.wasm" SQLITE3="$REPO_ROOT/packages/registry/sqlite/sqlite-install/bin/sqlite3.wasm" +GUEST_SHELL="${SQLITE_TEST_SHELL:-}" HOST="node" PERMUTATION="full" @@ -131,6 +132,14 @@ if [ ! -f "$TESTFIXTURE" ] || [ ! -f "$SQLITE3" ] || [ ! -d "$SQLITE_FULL/test" exit 1 fi +if [ -z "$GUEST_SHELL" ]; then + if ! GUEST_SHELL="$("$REPO_ROOT/scripts/resolve-binary.sh" programs/dash.wasm)"; then + echo "ERROR: SQLite testrunner child jobs require a current guest /bin/sh-compatible shell." >&2 + echo "Fetch/build dash, or set SQLITE_TEST_SHELL=/path/to/sh.wasm." >&2 + exit 1 + fi +fi + if [ -z "$WORKDIR" ]; then WORKDIR="$(mktemp -d "${SQLITE_OFFICIAL_TMPDIR:-/tmp}/kandelo-sqlite-official.XXXXXX")" else @@ -143,17 +152,95 @@ if [ -z "$RESULTS_DIR" ]; then fi mkdir -p "$RESULTS_DIR" +write_unavailable_outcome_lists() { + local reason="$1" + local out="$RESULTS_DIR/outcome-lists" + + mkdir -p "$out" + printf 'jobid\tstate\tdisplaytype\tdisplayname\tcases\terrors\tms\tsource\n' > "$out/passed-jobs.tsv" + printf 'jobid\tstate\tdisplaytype\tdisplayname\tcases\terrors\tms\tsource\n' > "$out/failed-jobs.tsv" + printf 'jobid\tstate\tdisplaytype\tdisplayname\tcases\terrors\tms\treason\tsource\n' > "$out/skipped-jobs.tsv" + printf 'jobid\tstate\tdisplaytype\tdisplayname\tcases\terrors\tms\treason\tsource\n' > "$out/incomplete-jobs.tsv" + printf '\tunavailable\t\t\t0\t0\t0\t%s\trunner\n' "$reason" >> "$out/incomplete-jobs.tsv" + { + printf 'passed_jobs\tfailed_jobs\tskipped_jobs\tincomplete_jobs\tnote\n' + printf '0\t0\t0\t1\t%s\n' "$reason" + } > "$out/counts.tsv" +} + +write_outcome_lists() { + local db="$1" + local out="$RESULTS_DIR/outcome-lists" + + mkdir -p "$out" + sqlite3 -header -separator $'\t' "$db" \ + "SELECT jobid, state, displaytype, displayname, + coalesce(ntest, 0) AS cases, + coalesce(nerr, 0) AS errors, + coalesce(span, 0) AS ms, + 'testrunner.db' AS source + FROM jobs + WHERE state = 'done' AND coalesce(nerr, 0) = 0 + ORDER BY jobid;" > "$out/passed-jobs.tsv" + + sqlite3 -header -separator $'\t' "$db" \ + "SELECT jobid, state, displaytype, displayname, + coalesce(ntest, 0) AS cases, + coalesce(nerr, 0) AS errors, + coalesce(span, 0) AS ms, + 'testrunner.db' AS source + FROM jobs + WHERE state = 'failed' OR (state = 'done' AND coalesce(nerr, 0) > 0) + ORDER BY jobid;" > "$out/failed-jobs.tsv" + + sqlite3 -header -separator $'\t' "$db" \ + "SELECT jobid, state, displaytype, displayname, + coalesce(ntest, 0) AS cases, + coalesce(nerr, 0) AS errors, + coalesce(span, 0) AS ms, + 'runner omitted' AS reason, + 'testrunner.db' AS source + FROM jobs + WHERE state = 'omit' + ORDER BY jobid;" > "$out/skipped-jobs.tsv" + + sqlite3 -header -separator $'\t' "$db" \ + "SELECT jobid, state, displaytype, displayname, + coalesce(ntest, 0) AS cases, + coalesce(nerr, 0) AS errors, + coalesce(span, 0) AS ms, + CASE state + WHEN 'running' THEN 'runner exited before job completed' + WHEN 'ready' THEN 'not started before runner exit' + ELSE 'not completed before runner exit' + END AS reason, + 'testrunner.db' AS source + FROM jobs + WHERE coalesce(state, '') NOT IN ('done', 'failed', 'omit') + ORDER BY state, jobid;" > "$out/incomplete-jobs.tsv" + + sqlite3 -header -separator $'\t' "$db" \ + "SELECT coalesce(sum(state='done' AND coalesce(nerr, 0)=0), 0) AS passed_jobs, + coalesce(sum(state='failed' OR (state='done' AND coalesce(nerr, 0)>0)), 0) AS failed_jobs, + coalesce(sum(state='omit'), 0) AS skipped_jobs, + coalesce(sum(coalesce(state,'') NOT IN ('done','failed','omit')), 0) AS incomplete_jobs, + 'testrunner.db' AS source + FROM jobs;" > "$out/counts.tsv" +} + write_sqlite_report() { local db="$WORKDIR/testrunner.db" local report="$RESULTS_DIR/summary.txt" local failures="$RESULTS_DIR/failures.tsv" if [ ! -f "$db" ]; then echo "No testrunner.db was created at $db" > "$report" - return + write_unavailable_outcome_lists "No testrunner.db was created at $db." + return 1 fi mkdir -p "$RESULTS_DIR" - for artifact in testrunner.db testrunner.log testrunner_build.log; do + + for artifact in testrunner.db testrunner.db-wal testrunner.db-shm testrunner.log testrunner_build.log; do if [ -f "$WORKDIR/$artifact" ]; then cp "$WORKDIR/$artifact" "$RESULTS_DIR/$artifact" fi @@ -175,11 +262,14 @@ write_sqlite_report() { find "$RESULTS_DIR" -maxdepth 1 -type f -name 'testrunner.*' -print | sort } > "$report" : > "$failures" + write_unavailable_outcome_lists "No usable jobs table was found in $db." echo "===== SQLite official testrunner database summary =====" cat "$report" - return + return 1 fi + write_outcome_lists "$db" + { echo "SQLite official testrunner summary" echo "host=$HOST" @@ -228,12 +318,13 @@ write_sqlite_report() { GROUP BY config ORDER BY CASE WHEN config='full' THEN 0 ELSE 1 END, config;" echo - echo "Failed, running, and omitted jobs:" + echo "Unsuccessful, incomplete, and omitted jobs:" sqlite3 -header -column "$db" \ "SELECT jobid, state, displaytype, displayname, coalesce(ntest, 0) AS cases, coalesce(nerr, 0) AS errors, coalesce(span, 0) AS ms FROM jobs - WHERE state IN ('failed', 'running', 'omit') + WHERE coalesce(state, '')!='done' + OR coalesce(nerr, 0)>0 ORDER BY state, jobid;" } > "$report" @@ -241,11 +332,208 @@ write_sqlite_report() { "SELECT jobid, state, displaytype, displayname, coalesce(ntest, 0) AS cases, coalesce(nerr, 0) AS errors, coalesce(span, 0) AS ms FROM jobs - WHERE state IN ('failed', 'running', 'omit') + WHERE coalesce(state, '')!='done' + OR coalesce(nerr, 0)>0 ORDER BY state, jobid;" > "$failures" + if ! python3 "$REPO_ROOT/scripts/sqlite-case-outcomes.py" \ + --db "$RESULTS_DIR/testrunner.db" \ + --results-dir "$RESULTS_DIR" \ + --host "$HOST" \ + --permutation "$PERMUTATION" + then + echo "ERROR: failed to write SQLite case outcome artifacts" >&2 + return 1 + fi + echo "===== SQLite official testrunner database summary =====" cat "$report" + + local total_jobs unsuccessful_jobs + total_jobs="$(sqlite3 "$db" "SELECT count(*) FROM jobs;")" + if [ "$total_jobs" -eq 0 ]; then + echo "ERROR: SQLite testrunner selected no jobs" >&2 + return 1 + fi + if $EXPLAIN; then + unsuccessful_jobs="$(sqlite3 "$db" \ + "SELECT count(*) FROM jobs + WHERE state NOT IN ('', 'ready') + OR coalesce(nerr, 0)>0;")" + else + unsuccessful_jobs="$(sqlite3 "$db" \ + "SELECT count(*) FROM jobs + WHERE state!='done' + OR ntest IS NULL + OR nerr IS NULL + OR nerr>0;")" + fi + if [ "$unsuccessful_jobs" -ne 0 ]; then + echo "ERROR: SQLite testrunner recorded $unsuccessful_jobs unsuccessful or incomplete job(s)" >&2 + return 1 + fi +} + +patch_sqlite_testrunner_platform() { + local runner="$1" + local tmp + + if grep -q "Kandelo testrunner host selection for child jobs" "$runner"; then + return + fi + + tmp="${runner}.kandelo-platform.$$" + awk ' + NR == 4 { + print "" + print "# Kandelo testrunner host selection for child jobs." + print "# This controls only the SQLite helper-script launcher. Keep tcl_platform" + print "# unchanged so the tests continue to observe the real Tcl target." + print "set ::kandelo_testrunner_host Kandelo" + } + $0 == "switch -nocase -glob -- $tcl_platform(os) {" { + print "set testrunner_host $tcl_platform(os)" + print "if {[info exists ::kandelo_testrunner_host]} {" + print " set testrunner_host $::kandelo_testrunner_host" + print "}" + print "switch -nocase -glob -- $testrunner_host {" + next + } + $0 == " *openbsd* {" { + print " *kandelo* {" + print " set TRG(platform) linux" + print " set TRG(make) make.sh" + print " set TRG(makecmd) \"sh make.sh\"" + print " set TRG(testfixture) testfixture" + print " set TRG(shell) sqlite3" + print " set TRG(run) run.sh" + print " set TRG(runcmd) \"sh run.sh\"" + print " }" + } + { print } + ' "$runner" > "$tmp" + mv "$tmp" "$runner" + chmod a+r "$runner" + + for required in \ + 'set ::kandelo_testrunner_host Kandelo' \ + 'switch -nocase -glob -- $testrunner_host {' \ + '*kandelo* {' + do + if ! grep -Fq "$required" "$runner"; then + echo "ERROR: failed to patch SQLite testrunner.tcl host selection: missing $required" >&2 + exit 1 + fi + done +} + +patch_sqlite_testrunner_guest_paths() { + local runner="$1" + local tmp + + if grep -q "Kandelo guest path shim for all-mode child jobs" "$runner"; then + return + fi + + tmp="${runner}.kandelo-paths.$$" + awk ' + { + print + if (!inserted && $0 == "cd $dir") { + print "" + print "# Kandelo guest path shim for all-mode child jobs." + print "# testrunner.tcl builds child run.sh files from host-normalized paths;" + print "# convert workdir-local paths back to paths relative to each testdirN" + print "# directory, because SQLite runs the script after cd-ing into it." + print "proc kandelo_guest_path {path} {" + print " set normalized [file normalize $path]" + print " set topdir [file normalize [file dirname $::testdir]]" + print " set exe [file normalize [info nameofexec]]" + print " set script [file normalize [info script]]" + print " if {[string equal $normalized $exe]} { return \"../testfixture.wasm\" }" + print " if {[string equal $normalized $script]} { return \"../test/testrunner.tcl\" }" + print " if {[string equal $normalized $topdir]} { return \"..\" }" + print " set prefix \"${topdir}/\"" + print " if {[string first $prefix $normalized] == 0} {" + print " return \"../[string range $normalized [string length $prefix] end]\"" + print " }" + print " return $path" + print "}" + print "set ::kandelo_inline_run_sh 1" + print "set ::kandelo_chunk_pipe_output 1" + inserted = 1 + } else if ($0 == " set displayname [string map [list $topdir/ {}] $f]") { + print " set testfixture_guest [kandelo_guest_path $testfixture]" + print " set testrunner_tcl_guest [kandelo_guest_path $testrunner_tcl]" + print " set f_guest [kandelo_guest_path $f]" + } + } + ' "$runner" > "$tmp" + mv "$tmp" "$runner" + + tmp="${runner}.kandelo-paths-subst.$$" + awk ' + $0 == " set cmd \"$testfixture $f\"" { + print " set cmd \"$testfixture_guest $f_guest\"" + next + } + $0 == " set cmd \"$testfixture $testrunner_tcl $config $f\"" { + print " set cmd \"$testfixture_guest $testrunner_tcl_guest $config $f_guest\"" + next + } + $0 == " set set_tmp_dir \"export SQLITE_TMPDIR=\\\"[file normalize $dir]\\\"\"" { + print " set set_tmp_dir \"export SQLITE_TMPDIR=.\"" + next + } + $0 == " set fd [open \"|$TRG(runcmd) 2>@1\" r]" { + print " if {[info exists ::kandelo_inline_run_sh] && $::kandelo_inline_run_sh} {" + print " set inline_cmd \"$set_tmp_dir\\n$job(cmd)\"" + print " set fd [open \"|sh -c [list $inline_cmd] 2>@1\" r]" + print " } else {" + print " set fd [open \"|$TRG(runcmd) 2>@1\" r]" + print " }" + next + } + $0 == " set rc [catch { gets $fd line } res]" { + print " if {[info exists ::kandelo_chunk_pipe_output] && $::kandelo_chunk_pipe_output} {" + print " set rc [catch { read $fd 4096 } res]" + print " if {$rc} {" + print " puts \"ERROR $res\"" + print " }" + print " if {!$rc && [string length $res] > 0} {" + print " append O($iJob) $res" + print " }" + print " } else {" + print " set rc [catch { gets $fd line } res]" + next + } + $0 == " if {$res>=0} {" { + print " if {![info exists ::kandelo_chunk_pipe_output] || !$::kandelo_chunk_pipe_output} {" + print " if {$res>=0} {" + next + } + $0 == " append O($iJob) \"$line\\n\"" { + print + print " }" + print " }" + next + } + { print } + ' "$runner" > "$tmp" + mv "$tmp" "$runner" + chmod a+r "$runner" + + for required in \ + 'set ::kandelo_inline_run_sh 1' \ + 'set ::kandelo_chunk_pipe_output 1' \ + 'set fd [open "|sh -c [list $inline_cmd] 2>@1" r]' \ + 'set rc [catch { read $fd 4096 } res]' + do + if ! grep -Fq "$required" "$runner"; then + echo "ERROR: failed to patch SQLite testrunner.tcl for Kandelo all-mode jobs: missing $required" >&2 + exit 1 + fi + done } cleanup() { @@ -272,15 +560,19 @@ cp "$TESTFIXTURE" "$WORKDIR/testfixture.wasm" cp "$SQLITE3" "$WORKDIR/sqlite3" cp "$SQLITE3" "$WORKDIR/sqlite3.wasm" chmod a+rx "$WORKDIR/testfixture" "$WORKDIR/testfixture.wasm" "$WORKDIR/sqlite3" "$WORKDIR/sqlite3.wasm" +if [ -n "$GUEST_SHELL" ]; then + cp "$GUEST_SHELL" "$WORKDIR/sh" + cp "$GUEST_SHELL" "$WORKDIR/sh.wasm" + chmod a+rx "$WORKDIR/sh" "$WORKDIR/sh.wasm" +fi +patch_sqlite_testrunner_platform "$WORKDIR/test/testrunner.tcl" +patch_sqlite_testrunner_guest_paths "$WORKDIR/test/testrunner.tcl" RUNNER_TCL="$WORKDIR/kandelo-testrunner.tcl" cat > "$RUNNER_TCL" <<'TCL' -# Kandelo's Tcl build reports a target OS name that SQLite's testrunner.tcl -# does not classify. Present a Unix-like platform to the upstream runner and -# use its OpenBSD branch so generated helper scripts run with sh instead of -# bash. -set ::tcl_platform(os) OpenBSD -set ::tcl_platform(platform) unix +# Select Kandelo's helper-script launcher without changing Tcl target metadata +# observed by SQLite's tests. +set ::kandelo_testrunner_host Kandelo set argv0 test/testrunner.tcl source $argv0 TCL @@ -302,15 +594,23 @@ echo "Results dir: $RESULTS_DIR" set +e TCL_LIBRARY="$TCL_INSTALL/lib/tcl8.6" \ KERNEL_CWD="$WORKDIR" \ +KERNEL_PATH="$WORKDIR:${KERNEL_PATH:-/usr/local/bin:/usr/bin:/bin}" \ KERNEL_UID="${SQLITE_TEST_UID:-1000}" \ KERNEL_GID="${SQLITE_TEST_GID:-1000}" \ TIMEOUT="$TIMEOUT_MS" \ node --experimental-wasm-exnref --import tsx/esm \ "$REPO_ROOT/examples/run-example.ts" \ - "$TESTFIXTURE" \ + "$WORKDIR/testfixture.wasm" \ "${ARGS[@]}" status=$? set -e -write_sqlite_report || true -exit "$status" +set +e +(set -e; write_sqlite_report) +report_status=$? +set -e + +if [ "$status" -ne 0 ]; then + exit "$status" +fi +exit "$report_status" diff --git a/scripts/sqlite-case-outcomes.py b/scripts/sqlite-case-outcomes.py new file mode 100755 index 000000000..cc5fbe287 --- /dev/null +++ b/scripts/sqlite-case-outcomes.py @@ -0,0 +1,508 @@ +#!/usr/bin/env python3 +"""Emit durable SQLite testrunner case and job outcome lists.""" + +from __future__ import annotations + +import argparse +import json +import re +import sqlite3 +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + + +OK_RE = re.compile(r"^(?P.+?)\.\.\. Ok$") +OMITTED_RE = re.compile(r"^(?P.+?)\.\.\. Omitted$") +FAIL_RE = re.compile(r"^! (?P\S+) expected:") +FAILURES_RE = re.compile(r"^!Failures on these tests:\s*(?P.*)$") +SUMMARY_RE = re.compile(r"\b(?P\d+) errors out of (?P\d+) tests\b") +OMITTED_DETAIL_RE = re.compile(r"^\.\s+(?P\S+)\s+(?P.+)$") + +@dataclass +class ParsedOutput: + passed: list[str] = field(default_factory=list) + failed: list[str] = field(default_factory=list) + skipped_counted: list[tuple[str, str]] = field(default_factory=list) + skipped_detail: list[tuple[str, str]] = field(default_factory=list) + summary_tests: int | None = None + summary_errors: int | None = None + + +@dataclass +class Job: + jobid: int | None + state: str + displaytype: str + displayname: str + ntest: int | None + nerr: int | None + span: int | None + output: str + + +def normalize_output(text: str) -> str: + return text.replace("\r\n", "\n").replace("\r", "\n") + + +def append_unique(items: list[str], value: str) -> None: + if value not in items: + items.append(value) + + +def append_unique_pair(items: list[tuple[str, str]], value: tuple[str, str]) -> None: + if value not in items: + items.append(value) + + +def upsert_named_pair(items: list[tuple[str, str]], value: tuple[str, str]) -> None: + for index, (name, _) in enumerate(items): + if name == value[0]: + items[index] = value + return + items.append(value) + + +def parse_output(text: str) -> ParsedOutput: + parsed = ParsedOutput() + in_omitted_detail = False + failure_summary: list[str] = [] + + for line in normalize_output(text).splitlines(): + line = line.strip() + if not line: + continue + + summary = SUMMARY_RE.search(line) + if summary: + parsed.summary_errors = int(summary.group("errors")) + parsed.summary_tests = int(summary.group("tests")) + + if line == "Omitted test cases:": + in_omitted_detail = True + continue + + if in_omitted_detail: + omitted_detail = OMITTED_DETAIL_RE.match(line) + if omitted_detail: + append_unique_pair( + parsed.skipped_detail, + (omitted_detail.group("name"), omitted_detail.group("reason")), + ) + continue + in_omitted_detail = False + + ok = OK_RE.match(line) + if ok: + parsed.passed.append(ok.group("name")) + continue + + omitted = OMITTED_RE.match(line) + if omitted: + append_unique_pair(parsed.skipped_counted, (omitted.group("name"), "omitted by SQLite test harness")) + continue + + failures = FAILURES_RE.match(line) + if failures: + failure_summary.extend(name for name in failures.group("names").split() if name) + continue + + fail = FAIL_RE.match(line) + if fail: + append_unique(parsed.failed, fail.group("name")) + + if failure_summary: + parsed.failed = failure_summary + + return parsed + + +def decode_db_text(value: Any) -> str: + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + if value is None: + return "" + return str(value) + + +def read_jobs_from_db(db_path: Path) -> tuple[list[Job], str | None]: + if not db_path.exists(): + return [], f"testrunner database is missing: {db_path}" + + db_uri = f"{db_path.resolve().as_uri()}?mode=ro" + con = sqlite3.connect(db_uri, uri=True) + # Some archived browser runs contain non-UTF-8 bytes in Tcl output. Read + # TEXT as bytes so one diagnostic byte cannot make the whole report fail. + con.text_factory = bytes + try: + has_jobs = con.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='jobs' LIMIT 1" + ).fetchone() + if not has_jobs: + return [], f"testrunner database has no jobs table: {db_path}" + + rows = con.execute( + """ + SELECT jobid, state, displaytype, displayname, + ntest, nerr, span, coalesce(output, '') + FROM jobs + ORDER BY jobid + """ + ).fetchall() + finally: + con.close() + + jobs = [ + Job( + jobid=row[0], + state=decode_db_text(row[1]), + displaytype=decode_db_text(row[2]), + displayname=decode_db_text(row[3]), + ntest=row[4], + nerr=row[5], + span=row[6], + output=decode_db_text(row[7]), + ) + for row in rows + ] + return jobs, None + + +def read_stdout_job(stdout_path: Path, display_name: str) -> tuple[list[Job], str | None]: + if not stdout_path.exists(): + return [], f"stdout file is missing: {stdout_path}" + output = stdout_path.read_text(encoding="utf-8", errors="replace") + parsed = parse_output(output) + ntest = parsed.summary_tests + nerr = parsed.summary_errors + state = "failed" if nerr and nerr > 0 else "done" + return [ + Job( + jobid=1, + state=state, + displaytype="direct", + displayname=display_name or "direct stdout", + ntest=ntest, + nerr=nerr, + span=None, + output=output, + ) + ], None + + +def job_row(job: Job) -> list[Any]: + return [ + "" if job.jobid is None else job.jobid, + job.state, + job.displaytype, + job.displayname, + "" if job.ntest is None else job.ntest, + "" if job.nerr is None else job.nerr, + "" if job.span is None else job.span, + ] + + +def add_unavailable(unavailable: list[dict[str, Any]], category: str, reason: str, job: Job | None = None) -> None: + entry: dict[str, Any] = {"category": category, "reason": reason} + if job is not None: + entry["job"] = { + "jobid": job.jobid, + "state": job.state, + "displaytype": job.displaytype, + "displayname": job.displayname, + "ntest": job.ntest, + "nerr": job.nerr, + } + unavailable.append(entry) + + +def build_outcomes(jobs: list[Job], source_reason: str | None) -> dict[str, Any]: + passed_cases: list[str] = [] + failed_cases: list[str] = [] + skipped_cases: list[tuple[str, str]] = [] + passed_jobs: list[list[Any]] = [] + failed_jobs: list[list[Any]] = [] + skipped_jobs: list[list[Any]] = [] + incomplete_jobs: list[list[Any]] = [] + unattributed_passed_cases: list[list[Any]] = [] + unavailable: list[dict[str, Any]] = [] + + if source_reason is not None: + add_unavailable(unavailable, "all", source_reason) + + reported_executed_cases = 0 + reported_case_errors = 0 + counted_skipped_cases = 0 + detail_only_skipped_cases = 0 + + for job in jobs: + parsed = parse_output(job.output) if job.output else None + effective_errors = job.nerr + if effective_errors is None and parsed is not None: + effective_errors = parsed.summary_errors + + if job.state == "done" and not effective_errors: + passed_jobs.append(job_row(job)) + elif job.state == "failed" or (job.state == "done" and effective_errors): + failed_jobs.append(job_row(job)) + elif job.state == "omit": + skipped_jobs.append(job_row(job)) + add_unavailable( + unavailable, + "skipped_cases", + "SQLite testrunner marked this job omitted; no case-level skip names are available.", + job, + ) + continue + elif job.state in {"ready", "running", "halt", ""}: + incomplete_jobs.append(job_row(job)) + add_unavailable( + unavailable, + "all_cases", + f"SQLite testrunner job is incomplete with state {job.state or ''}.", + job, + ) + continue + + if not job.output: + add_unavailable(unavailable, "all_cases", "SQLite job output is empty.", job) + continue + + assert parsed is not None + ntest = job.ntest if job.ntest is not None else parsed.summary_tests + nerr = effective_errors + if ntest is None or nerr is None: + add_unavailable(unavailable, "all_cases", "Could not find SQLite 'errors out of tests' summary.", job) + continue + + reported_executed_cases += ntest + reported_case_errors += nerr + + job_skipped: list[tuple[str, str]] = [] + counted_skipped_names = {name for name, _reason in parsed.skipped_counted} + for skipped in parsed.skipped_counted: + upsert_named_pair(job_skipped, skipped) + for skipped in parsed.skipped_detail: + # SQLite prints some omissions twice: once as `... Omitted` and + # again in its detailed reason list. Keep one row and prefer the + # specific reason. + upsert_named_pair(job_skipped, skipped) + for skipped in job_skipped: + upsert_named_pair(skipped_cases, skipped) + counted_skipped_cases += len(counted_skipped_names) + detail_only_skipped_cases += sum(1 for name, _reason in job_skipped if name not in counted_skipped_names) + + if len(parsed.failed) == nerr: + failed_cases.extend(parsed.failed) + elif nerr == 0 and not parsed.failed: + pass + else: + failed_cases.extend(parsed.failed) + add_unavailable( + unavailable, + "failed_cases", + f"Parsed {len(parsed.failed)} failed case names, but SQLite reported {nerr} case errors.", + job, + ) + + expected_passed = ntest - nerr - len(parsed.skipped_counted) + missing_passed = expected_passed - len(parsed.passed) + if missing_passed > 0: + unattributed_passed_cases.append([ + "" if job.jobid is None else job.jobid, + job.displayname, + missing_passed, + "SQLite reported passed cases without corresponding named Ok lines", + ]) + add_unavailable( + unavailable, + "passed_cases", + f"SQLite reported {expected_passed} passed cases, but only {len(parsed.passed)} names were present in output.", + job, + ) + elif missing_passed < 0: + add_unavailable( + unavailable, + "passed_cases", + f"Parsed {len(parsed.passed)} passed case names, but SQLite reported only {expected_passed} passed cases.", + job, + ) + passed_cases.extend(parsed.passed) + + # SQLite includes `... Omitted` cases in ntest, but some harness skips are + # only named in the detailed omission list. Add only those detail-only + # cases so selected_cases does not count ordinary omissions twice. + selected_cases = reported_executed_cases + detail_only_skipped_cases + categories = { + "passed_cases": { + "count": len(passed_cases), + "unattributed_count": sum(row[2] for row in unattributed_passed_cases), + "status": "available" + if not any(entry["category"] in {"passed_cases", "all_cases", "all"} for entry in unavailable) + else "partial", + }, + "failed_cases": { + "count": len(failed_cases), + "status": "available" + if not any(entry["category"] in {"failed_cases", "all_cases", "all"} for entry in unavailable) + else "partial", + }, + "skipped_cases": { + "count": len(skipped_cases), + "status": "available" + if not any(entry["category"] in {"skipped_cases", "all_cases", "all"} for entry in unavailable) + else "partial", + }, + } + + return { + "schema_version": 1, + "summary": { + "reported_executed_cases": reported_executed_cases, + "reported_case_errors": reported_case_errors, + "selected_cases": selected_cases, + "passed_cases": len(passed_cases), + "failed_cases": len(failed_cases), + "skipped_cases": len(skipped_cases), + "counted_skipped_cases": counted_skipped_cases, + "detail_only_skipped_cases": detail_only_skipped_cases, + "unattributed_passed_cases": sum(row[2] for row in unattributed_passed_cases), + "jobs": { + "passed": len(passed_jobs), + "failed": len(failed_jobs), + "skipped": len(skipped_jobs), + "incomplete": len(incomplete_jobs), + }, + }, + "categories": categories, + "unavailable": unavailable, + "lists": { + "passed_cases": passed_cases, + "failed_cases": failed_cases, + "skipped_cases": skipped_cases, + "unattributed_passed_cases": unattributed_passed_cases, + "passed_jobs": passed_jobs, + "failed_jobs": failed_jobs, + "skipped_jobs": skipped_jobs, + "incomplete_jobs": incomplete_jobs, + }, + } + + +def write_lines(path: Path, lines: list[str]) -> None: + path.write_text("".join(f"{line}\n" for line in lines), encoding="utf-8") + + +def write_tsv(path: Path, header: list[str], rows: list[list[Any] | tuple[Any, ...]]) -> None: + out = ["\t".join(header)] + for row in rows: + out.append("\t".join(str(value) for value in row)) + path.write_text("\n".join(out) + "\n", encoding="utf-8") + + +def write_outputs(results_dir: Path, outcomes: dict[str, Any], source: dict[str, Any]) -> None: + out_dir = results_dir / "outcome-lists" + out_dir.mkdir(parents=True, exist_ok=True) + lists = outcomes["lists"] + + write_lines(out_dir / "passed-cases.txt", lists["passed_cases"]) + write_lines(out_dir / "failed-cases.txt", lists["failed_cases"]) + write_tsv(out_dir / "skipped-cases.tsv", ["case", "reason"], lists["skipped_cases"]) + write_tsv( + out_dir / "unattributed-passed-cases.tsv", + ["jobid", "displayname", "count", "reason"], + lists["unattributed_passed_cases"], + ) + job_header = ["jobid", "state", "displaytype", "displayname", "cases", "errors", "ms"] + write_tsv(out_dir / "passed-jobs.tsv", job_header, lists["passed_jobs"]) + write_tsv(out_dir / "failed-jobs.tsv", job_header, lists["failed_jobs"]) + write_tsv(out_dir / "skipped-jobs.tsv", job_header, lists["skipped_jobs"]) + write_tsv(out_dir / "incomplete-jobs.tsv", job_header, lists["incomplete_jobs"]) + + serializable = { + "schema_version": outcomes["schema_version"], + "source": source, + "summary": outcomes["summary"], + "categories": outcomes["categories"], + "unavailable": outcomes["unavailable"], + "artifacts": { + "passed_cases": str(out_dir / "passed-cases.txt"), + "failed_cases": str(out_dir / "failed-cases.txt"), + "skipped_cases": str(out_dir / "skipped-cases.tsv"), + "unattributed_passed_cases": str(out_dir / "unattributed-passed-cases.tsv"), + "passed_jobs": str(out_dir / "passed-jobs.tsv"), + "failed_jobs": str(out_dir / "failed-jobs.tsv"), + "skipped_jobs": str(out_dir / "skipped-jobs.tsv"), + "incomplete_jobs": str(out_dir / "incomplete-jobs.tsv"), + }, + } + (out_dir / "case-outcomes.json").write_text( + json.dumps(serializable, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + (out_dir / "unavailable-categories.json").write_text( + json.dumps(outcomes["unavailable"], indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + summary = outcomes["summary"] + lines = [ + "# SQLite Case Outcomes", + "", + f"Executed cases reported by SQLite: {summary['reported_executed_cases']}", + f"Case errors reported by SQLite: {summary['reported_case_errors']}", + f"Passed case list entries: {summary['passed_cases']}", + f"Failed case list entries: {summary['failed_cases']}", + f"Skipped case list entries: {summary['skipped_cases']}", + f"Passed cases without names in SQLite output: {summary['unattributed_passed_cases']}", + "", + ] + if outcomes["unavailable"]: + lines.append("Unavailable or partial categories:") + for entry in outcomes["unavailable"]: + lines.append(f"- {entry['category']}: {entry['reason']}") + else: + lines.append("All emitted case categories are complete for the reported harness totals.") + lines.append("") + (results_dir / "summary-case-outcomes.md").write_text("\n".join(lines), encoding="utf-8") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--db", type=Path, help="SQLite testrunner.db to inspect") + parser.add_argument("--stdout-file", type=Path, help="Direct-run stdout file to inspect") + parser.add_argument("--results-dir", type=Path, required=True, help="Directory for outcome-list artifacts") + parser.add_argument("--host", default="", help="Host label for metadata") + parser.add_argument("--permutation", default="", help="SQLite permutation label for metadata") + parser.add_argument("--display-name", default="", help="Display name for stdout-only direct runs") + args = parser.parse_args() + + jobs: list[Job] + reason: str | None + source: dict[str, Any] = { + "host": args.host, + "permutation": args.permutation, + } + + if args.db is not None and args.db.exists(): + jobs, reason = read_jobs_from_db(args.db) + source.update({"kind": "testrunner-db", "path": str(args.db)}) + elif args.stdout_file is not None: + jobs, reason = read_stdout_job(args.stdout_file, args.display_name) + source.update({"kind": "direct-stdout", "path": str(args.stdout_file)}) + elif args.db is not None: + jobs, reason = read_jobs_from_db(args.db) + source.update({"kind": "testrunner-db", "path": str(args.db)}) + else: + jobs, reason = [], "no --db or --stdout-file source was provided" + source.update({"kind": "missing"}) + + outcomes = build_outcomes(jobs, reason) + write_outputs(args.results_dir, outcomes, source) + print(f"===== SQLite case outcome lists: {args.results_dir / 'outcome-lists'} =====") + print((args.results_dir / "summary-case-outcomes.md").read_text(encoding="utf-8")) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/scripts/sqlite-case-outcomes.sh b/tests/scripts/sqlite-case-outcomes.sh new file mode 100755 index 000000000..88c766917 --- /dev/null +++ b/tests/scripts/sqlite-case-outcomes.sh @@ -0,0 +1,176 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +TMP_ROOT="$(mktemp -d)" +trap 'rm -rf "$TMP_ROOT"' EXIT + +STDOUT_FILE="$TMP_ROOT/direct.stdout" +RESULTS_DIR="$TMP_ROOT/results" + +cat > "$STDOUT_FILE" <<'EOF' +utf16.misc1-1.1... Ok +utf16.misc1-1.2... Ok +SQLite 2025-02-18 13:38:58 abcdef +0 errors out of 3 tests on OpenBSD 32-bit +Omitted test cases: +. misc1-10.1 skipped by focused test +EOF + +python3 "$REPO_ROOT/scripts/sqlite-case-outcomes.py" \ + --stdout-file "$STDOUT_FILE" \ + --results-dir "$RESULTS_DIR" \ + --host browser \ + --display-name "test/misc1.test" >/dev/null + +if [ "$(wc -l < "$RESULTS_DIR/outcome-lists/passed-cases.txt" | tr -d ' ')" != "2" ]; then + echo "expected two named passed case entries" >&2 + exit 1 +fi + +if [ "$(tail -n +2 "$RESULTS_DIR/outcome-lists/unattributed-passed-cases.tsv" | wc -l | tr -d ' ')" != "1" ]; then + echo "expected one unattributed passed-case row" >&2 + exit 1 +fi + +if [ "$(tail -n +2 "$RESULTS_DIR/outcome-lists/skipped-cases.tsv" | wc -l | tr -d ' ')" != "1" ]; then + echo "expected one skipped case entry" >&2 + exit 1 +fi + +python3 - "$RESULTS_DIR/outcome-lists/case-outcomes.json" <<'PY' +import json +import sys +from pathlib import Path + +data = json.loads(Path(sys.argv[1]).read_text()) +summary = data["summary"] +assert summary["reported_executed_cases"] == 3, summary +assert summary["passed_cases"] == 2, summary +assert summary["skipped_cases"] == 1, summary +assert summary["unattributed_passed_cases"] == 1, summary +assert data["categories"]["passed_cases"]["status"] == "partial", data["categories"] +assert data["unavailable"][0]["category"] == "passed_cases", data["unavailable"] +PY + +OMITTED_STDOUT="$TMP_ROOT/omitted.stdout" +OMITTED_RESULTS="$TMP_ROOT/omitted-results" +cat > "$OMITTED_STDOUT" <<'EOF' +omitted-1.1... Omitted +0 errors out of 1 tests on OpenBSD 32-bit +Omitted test cases: +. omitted-1.1 requires an unavailable optional feature +EOF + +python3 "$REPO_ROOT/scripts/sqlite-case-outcomes.py" \ + --stdout-file "$OMITTED_STDOUT" \ + --results-dir "$OMITTED_RESULTS" \ + --host browser \ + --display-name "test/omitted.test" >/dev/null + +if [ "$(tail -n +2 "$OMITTED_RESULTS/outcome-lists/skipped-cases.tsv" | wc -l | tr -d ' ')" != "1" ]; then + echo "expected duplicate omitted output to produce one skipped row" >&2 + exit 1 +fi +if ! grep -q 'requires an unavailable optional feature' "$OMITTED_RESULTS/outcome-lists/skipped-cases.tsv"; then + echo "expected the detailed omission reason" >&2 + exit 1 +fi + +python3 - "$OMITTED_RESULTS/outcome-lists/case-outcomes.json" <<'PY' +import json +import sys +from pathlib import Path + +summary = json.loads(Path(sys.argv[1]).read_text())["summary"] +assert summary["reported_executed_cases"] == 1, summary +assert summary["selected_cases"] == 1, summary +assert summary["counted_skipped_cases"] == 1, summary +assert summary["detail_only_skipped_cases"] == 0, summary +PY + +DB="$TMP_ROOT/testrunner.db" +python3 - "$DB" "$STDOUT_FILE" <<'PY' +import sqlite3 +import sys +from pathlib import Path + +db_path = Path(sys.argv[1]) +stdout = Path(sys.argv[2]).read_text() +con = sqlite3.connect(db_path) +con.executescript(""" +CREATE TABLE jobs( + jobid INTEGER PRIMARY KEY, + displaytype TEXT NOT NULL, + displayname TEXT NOT NULL, + build TEXT NOT NULL DEFAULT '', + dirname TEXT NOT NULL DEFAULT '', + cmd TEXT NOT NULL, + depid INTEGER, + priority INTEGER NOT NULL, + starttime INTEGER, + endtime INTEGER, + span INTEGER, + estwork INTEGER, + state TEXT, + ntest INT, + nerr INT, + svers TEXT, + pltfm TEXT, + output TEXT +); +""") +con.execute( + "INSERT INTO jobs(jobid, displaytype, displayname, cmd, priority, state, ntest, nerr, output) VALUES(1, 'tcl', 'test/misc1.test', '', 1, 'done', 3, 0, ?)", + (stdout,), +) +con.execute( + "INSERT INTO jobs(jobid, displaytype, displayname, cmd, priority, state, ntest, nerr, output) VALUES(2, 'tcl', 'test/failing.test', '', 1, 'done', 1, 1, ?)", + ("! failing-1.1 expected: value\n1 errors out of 1 tests on OpenBSD 32-bit\n",), +) +invalid_output = b"invalid-\x80-1.1... Ok\n0 errors out of 1 tests on OpenBSD 32-bit\n" +con.execute( + "INSERT INTO jobs(jobid, displaytype, displayname, cmd, priority, state, ntest, nerr, output) VALUES(3, 'tcl', 'test/nonutf8.test', '', 1, 'done', 1, 0, CAST(? AS TEXT))", + (sqlite3.Binary(invalid_output),), +) +con.commit() +con.close() +PY + +# Reporting must not mutate or clean up the evidence it reads. +: > "$DB-wal" +: > "$DB-shm" +SIDECAR_HASHES_BEFORE="$(shasum -a 256 "$DB" "$DB-wal" "$DB-shm")" + +DB_RESULTS="$TMP_ROOT/db-results" +python3 "$REPO_ROOT/scripts/sqlite-case-outcomes.py" \ + --db "$DB" \ + --results-dir "$DB_RESULTS" \ + --host node >/dev/null + +SIDECAR_HASHES_AFTER="$(shasum -a 256 "$DB" "$DB-wal" "$DB-shm")" +if [ "$SIDECAR_HASHES_BEFORE" != "$SIDECAR_HASHES_AFTER" ]; then + echo "outcome extraction changed its source database or sidecars" >&2 + exit 1 +fi + +python3 - "$DB_RESULTS/outcome-lists/case-outcomes.json" <<'PY' +import json +import sys +from pathlib import Path + +data = json.loads(Path(sys.argv[1]).read_text()) +assert data["summary"]["jobs"] == { + "passed": 2, + "failed": 1, + "skipped": 0, + "incomplete": 0, +}, data["summary"]["jobs"] +assert data["summary"]["failed_cases"] == 1, data["summary"] +assert data["summary"]["unattributed_passed_cases"] == 1, data["summary"] +assert "invalid-\ufffd-1.1" in Path( + data["artifacts"]["passed_cases"] +).read_text().splitlines() +PY + +echo "sqlite-case-outcomes ok"