SQLite tests: run all-mode jobs and preserve truthful outcomes

apps/browser-demos/pages/sqlite-test/main.ts

@@ -7,6 +7,10 @@ import { BrowserKernel } from "@host/browser-kernel-host";
 import { MemoryFileSystem } from "@host/vfs/memory-fs";
 import { writeVfsFile } from "@host/vfs/image-helpers";
 import { finalizeKernelOwnedImage, settleWebKitReclaim } from "../../lib/kernel-owned-boot";
+import {
+  patchTestrunnerForKandelo,
+  testrunnerPlatformShim,
+} from "./testrunner-patch";
 import kernelWasmUrl from "@kernel-wasm?url";
 
 declare global {
@@ -91,6 +95,20 @@ async function collectArtifactsFromKernel(
   return artifacts.length > 0 ? artifacts : undefined;
 }
 
+function installTestrunnerPatches(fs: MemoryFileSystem): void {
+  const runnerPath = "/sqlite/test/testrunner.tcl";
+  const decoder = new TextDecoder();
+  const runner = decoder.decode(readVfsFile(fs, runnerPath));
+  writeVfsFile(fs, runnerPath, patchTestrunnerForKandelo(runner), 0o644);
+
+  writeVfsFile(fs, "/sqlite/kandelo-testrunner.tcl", [
+    testrunnerPlatformShim,
+    "set argv0 test/testrunner.tcl",
+    "source $argv0",
+    "",
+  ].join("\n"), 0o644);
+}
+
 function createFs(): MemoryFileSystem {
   if (!vfsImageBytes) throw new Error("SQLite test VFS image not loaded");
   const fs = MemoryFileSystem.fromImage(vfsImageBytes, {
@@ -153,13 +171,7 @@ async function init() {
     // SharedArrayBuffer across the per-test loop (Safari OOM fix).
     const buildFs = createFs();
     if (argv[1] === "kandelo-testrunner.tcl") {
-      writeVfsFile(buildFs, "/sqlite/kandelo-testrunner.tcl", [
-        "set ::tcl_platform(os) OpenBSD",
-        "set ::tcl_platform(platform) unix",
-        "set argv0 test/testrunner.tcl",
-        "source $argv0",
-        "",
-      ].join("\n"), 0o644);
+      installTestrunnerPatches(buildFs);
     }
     const vfsImage = await finalizeKernelOwnedImage(buildFs);
     const kernel = new BrowserKernel({

apps/browser-demos/pages/sqlite-test/testrunner-patch.ts

@@ -0,0 +1,189 @@
+export const testrunnerPlatformShim = [
+  "# Kandelo testrunner host selection for child jobs.",
+  "# This controls only SQLite's helper-script launcher. Keep tcl_platform",
+  "# unchanged so the tests continue to observe the real Tcl target.",
+  "set ::kandelo_testrunner_host Kandelo",
+].join("\n");
+
+const testrunnerGuestPathShim = [
+  "# Kandelo guest path shim for all-mode child jobs.",
+  "# testrunner.tcl builds child run.sh files from host-normalized paths;",
+  "# convert workdir-local paths back to paths relative to each testdirN",
+  "# directory, because SQLite runs the script after cd-ing into it.",
+  "proc kandelo_guest_path {path} {",
+  "  if {[file pathtype $path] != \"absolute\" && [string equal $path [info nameofexec]]} {",
+  "    return $path",
+  "  }",
+  "  set normalized [file normalize $path]",
+  "  set topdir [file normalize [file dirname $::testdir]]",
+  "  set script [file normalize [info script]]",
+  "  if {[string equal $normalized $script]} { return \"../test/testrunner.tcl\" }",
+  "  if {[string equal $normalized $topdir]} { return \"..\" }",
+  "  set prefix \"${topdir}/\"",
+  "  if {[string first $prefix $normalized] == 0} {",
+  "    return \"../[string range $normalized [string length $prefix] end]\"",
+  "  }",
+  "  return $path",
+  "}",
+  "set ::kandelo_inline_run_sh 1",
+  "set ::kandelo_chunk_pipe_output 1",
+].join("\n");
+
+function replaceRequired(source: string, search: string, replacement: string, label: string): string {
+  if (!source.includes(search)) {
+    throw new Error(`SQLite testrunner patch is incompatible: missing ${label}`);
+  }
+  return source.replace(search, replacement);
+}
+
+export function patchTestrunnerForKandelo(runner: string): string {
+  let patched = runner;
+
+  if (!patched.includes("Kandelo testrunner host selection for child jobs")) {
+    const lines = patched.split("\n");
+    if (lines.length < 4) {
+      throw new Error("SQLite testrunner patch is incompatible: file is shorter than four lines");
+    }
+    lines.splice(3, 0, "", testrunnerPlatformShim);
+    patched = lines.join("\n");
+    patched = replaceRequired(
+      patched,
+      "switch -nocase -glob -- $tcl_platform(os) {",
+      [
+        "set testrunner_host $tcl_platform(os)",
+        "if {[info exists ::kandelo_testrunner_host]} {",
+        "  set testrunner_host $::kandelo_testrunner_host",
+        "}",
+        "switch -nocase -glob -- $testrunner_host {",
+      ].join("\n"),
+      "host-selection switch",
+    );
+    patched = replaceRequired(
+      patched,
+      "  *openbsd* {\n",
+      [
+        "  *kandelo* {",
+        "    set TRG(platform)    linux",
+        "    set TRG(make)        make.sh",
+        "    set TRG(makecmd)     \"sh make.sh\"",
+        "    set TRG(testfixture) testfixture",
+        "    set TRG(shell)       sqlite3",
+        "    set TRG(run)         run.sh",
+        "    set TRG(runcmd)      \"sh run.sh\"",
+        "  }",
+        "  *openbsd* {",
+        "",
+      ].join("\n"),
+      "OpenBSD host branch",
+    );
+  }
+
+  if (!patched.includes("Kandelo guest path shim for all-mode child jobs")) {
+    patched = replaceRequired(
+      patched,
+      "cd $dir\n",
+      `cd $dir\n\n${testrunnerGuestPathShim}\n`,
+      "child work-directory anchor",
+    );
+    patched = replaceRequired(
+      patched,
+      "    set displayname [string map [list $topdir/ {}] $f]\n",
+      [
+        "    set displayname [string map [list $topdir/ {}] $f]",
+        "    set testfixture_guest [kandelo_guest_path $testfixture]",
+        "    set testrunner_tcl_guest [kandelo_guest_path $testrunner_tcl]",
+        "    set f_guest [kandelo_guest_path $f]",
+        "",
+      ].join("\n"),
+      "job display-name anchor",
+    );
+    patched = replaceRequired(
+      patched,
+      "      set cmd \"$testfixture $f\"",
+      "      set cmd \"$testfixture_guest $f_guest\"",
+      "direct test command anchor",
+    );
+    patched = replaceRequired(
+      patched,
+      "      set cmd \"$testfixture $testrunner_tcl $config $f\"",
+      "      set cmd \"$testfixture_guest $testrunner_tcl_guest $config $f_guest\"",
+      "configured test command anchor",
+    );
+    patched = replaceRequired(
+      patched,
+      "    set set_tmp_dir \"export SQLITE_TMPDIR=\\\"[file normalize $dir]\\\"\"",
+      "    set set_tmp_dir \"export SQLITE_TMPDIR=.\"",
+      "temporary-directory anchor",
+    );
+    patched = replaceRequired(
+      patched,
+      "    set fd [open \"|$TRG(runcmd) 2>@1\" r]",
+      [
+        "    if {[info exists ::kandelo_inline_run_sh] && $::kandelo_inline_run_sh} {",
+        "      set inline_cmd \"$set_tmp_dir\\n$job(cmd)\"",
+        "      set fd [open \"|sh -c [list $inline_cmd] 2>@1\" r]",
+        "    } else {",
+        "      set fd [open \"|$TRG(runcmd) 2>@1\" r]",
+        "    }",
+      ].join("\n"),
+      "child shell command anchor",
+    );
+    patched = replaceRequired(
+      patched,
+      "    set rc [catch { gets $fd line } res]",
+      [
+        "    if {[info exists ::kandelo_chunk_pipe_output] && $::kandelo_chunk_pipe_output} {",
+        "      set rc [catch { read $fd 4096 } res]",
+        "      if {$rc} {",
+        "        puts \"ERROR $res\"",
+        "      }",
+        "      if {!$rc && [string length $res] > 0} {",
+        "        append O($iJob) $res",
+        "      }",
+        "    } else {",
+        "      set rc [catch { gets $fd line } res]",
+      ].join("\n"),
+      "pipe read anchor",
+    );
+    patched = replaceRequired(
+      patched,
+      "    if {$res>=0} {",
+      [
+        "    if {![info exists ::kandelo_chunk_pipe_output] || !$::kandelo_chunk_pipe_output} {",
+        "      if {$res>=0} {",
+      ].join("\n"),
+      "line-result anchor",
+    );
+    patched = replaceRequired(
+      patched,
+      "      append O($iJob) \"$line\\n\"",
+      [
+        "      append O($iJob) \"$line\\n\"",
+        "      }",
+        "    }",
+      ].join("\n"),
+      "line append anchor",
+    );
+  }
+
+  for (const required of [
+    "Kandelo testrunner host selection for child jobs",
+    "set ::kandelo_testrunner_host Kandelo",
+    "switch -nocase -glob -- $testrunner_host",
+    "*kandelo* {",
+    "Kandelo guest path shim for all-mode child jobs",
+    "set testfixture_guest [kandelo_guest_path $testfixture]",
+    "set cmd \"$testfixture_guest $f_guest\"",
+    "set cmd \"$testfixture_guest $testrunner_tcl_guest $config $f_guest\"",
+    "set set_tmp_dir \"export SQLITE_TMPDIR=.\"",
+    "set fd [open \"|sh -c [list $inline_cmd] 2>@1\" r]",
+    "set ::kandelo_chunk_pipe_output 1",
+    "set rc [catch { read $fd 4096 } res]",
+  ]) {
+    if (!patched.includes(required)) {
+      throw new Error(`SQLite testrunner patch is incomplete: missing ${required}`);
+    }
+  }
+
+  return patched;
+}

host/test/sqlite-testrunner-patch.test.ts

@@ -0,0 +1,48 @@
+import { describe, expect, it } from "vitest";
+import { patchTestrunnerForKandelo } from "../../apps/browser-demos/pages/sqlite-test/testrunner-patch";
+
+const upstreamFixture = [
+  "#!/usr/bin/env tclsh",
+  "# SQLite test runner fixture",
+  "set TRG(nJob) 1",
+  "set dir testdir1",
+  "switch -nocase -glob -- $tcl_platform(os) {",
+  "  *openbsd* {",
+  "  }",
+  "}",
+  "cd $dir",
+  "    set displayname [string map [list $topdir/ {}] $f]",
+  "      set cmd \"$testfixture $f\"",
+  "      set cmd \"$testfixture $testrunner_tcl $config $f\"",
+  "    set set_tmp_dir \"export SQLITE_TMPDIR=\\\"[file normalize $dir]\\\"\"",
+  "    set fd [open \"|$TRG(runcmd) 2>@1\" r]",
+  "    set rc [catch { gets $fd line } res]",
+  "    if {$res>=0} {",
+  "      append O($iJob) \"$line\\n\"",
+  "    }",
+  "",
+].join("\n");
+
+describe("SQLite browser testrunner patch", () => {
+  it("rewrites all-mode child commands and is idempotent", () => {
+    const patched = patchTestrunnerForKandelo(upstreamFixture);
+
+    expect(patched).toContain("set ::kandelo_testrunner_host Kandelo");
+    expect(patched).not.toContain("set ::tcl_platform(os)");
+    expect(patched).toContain("switch -nocase -glob -- $testrunner_host");
+    expect(patched).toContain("*kandelo* {");
+    expect(patched).toContain("set testfixture_guest [kandelo_guest_path $testfixture]");
+    expect(patched).toContain("set cmd \"$testfixture_guest $f_guest\"");
+    expect(patched).toContain("set cmd \"$testfixture_guest $testrunner_tcl_guest $config $f_guest\"");
+    expect(patched).toContain("set set_tmp_dir \"export SQLITE_TMPDIR=.\"");
+    expect(patched).toContain("set fd [open \"|sh -c [list $inline_cmd] 2>@1\" r]");
+    expect(patched).toContain("set rc [catch { read $fd 4096 } res]");
+    expect(patchTestrunnerForKandelo(patched)).toBe(patched);
+  });
+
+  it("fails loudly when an upstream anchor changes", () => {
+    expect(() => patchTestrunnerForKandelo("one\ntwo\nthree\nfour\n")).toThrow(
+      "missing host-selection switch",
+    );
+  });
+});

scripts/browser-sqlite-official-runner.ts

@@ -1,5 +1,5 @@
 #!/usr/bin/env tsx
-import { spawn, type ChildProcess } from "node:child_process";
+import { spawn, spawnSync, type ChildProcess } from "node:child_process";
 import { existsSync, mkdirSync, writeFileSync } from "node:fs";
 import { createServer } from "node:net";
 import { resolve } from "node:path";
@@ -108,6 +108,39 @@ function writeArtifacts(resultsDir: string, artifacts: BrowserArtifact[] | undef
   }
 }
 
+function inferDisplayName(command: string[]): string {
+  const tests = command.filter((arg) => arg.endsWith(".test"));
+  if (tests.length > 0) return tests[tests.length - 1];
+  return command.join(" ");
+}
+
+function writeDirectOutcomeArtifacts(resultsDir: string, command: string[], stdout: string, stderr: string): boolean {
+  if (!resultsDir) return true;
+  mkdirSync(resultsDir, { recursive: true });
+  const stdoutPath = resolve(resultsDir, "runner.stdout");
+  const stderrPath = resolve(resultsDir, "runner.stderr");
+  const dbPath = resolve(resultsDir, "testrunner.db");
+  writeFileSync(stdoutPath, stdout);
+  writeFileSync(stderrPath, stderr);
+  const sourceArgs = existsSync(dbPath)
+    ? ["--db", dbPath]
+    : ["--stdout-file", stdoutPath, "--display-name", inferDisplayName(command)];
+  const outcome = spawnSync(
+    "python3",
+    [
+      resolve(REPO_ROOT, "scripts/sqlite-case-outcomes.py"),
+      ...sourceArgs,
+      "--results-dir", resultsDir,
+      "--host", "browser",
+    ],
+    { cwd: REPO_ROOT, stdio: "inherit" },
+  );
+  if (outcome.status !== 0) {
+    console.error(`[sqlite-outcomes] case outcome extraction failed with status ${outcome.status}`);
+  }
+  return outcome.status === 0;
+}
+
 async function main() {
   const argv = process.argv.slice(2);
   let timeoutMs = 600_000;
@@ -189,10 +222,12 @@ async function main() {
     }
     if (!result.artifacts && latestArtifacts) result.artifacts = latestArtifacts;
     writeArtifacts(resultsDir, result.artifacts);
+    const outcomesOk = writeDirectOutcomeArtifacts(resultsDir, command, result.stdout, result.stderr);
     if (result.stdout) process.stdout.write(result.stdout);
     if (result.stderr) process.stderr.write(result.stderr);
     if (result.error) process.stderr.write(`${result.error}\n`);
-    process.exit(result.exitCode === 0 ? 0 : 1);
+    process.exitCode = result.exitCode === 0 && outcomesOk ? 0 : 1;
+    return;
   } finally {
     await browser?.close().catch(() => {});
     if (vite) {