From 26175372cffa14ec477083e0b4fcbe4e6483db59 Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Wed, 16 Sep 2026 18:39:39 -0400 Subject: [PATCH] feat: cover automatic runtime selection in package QA Package QA only ran plain client mode, and its test explicitly forbade client --auto, so the automatic runtime-selection path upstream tests had no downstream coverage at all. Add MESH_LLM_SMOKE_MODE to the shared readiness smoke. Auto mode runs --auto --disable-iroh-relays --nostr-relay ws://127.0.0.1:1/. That relay parses, so the client registers it without dialing, but it cannot connect, so discovery fails closed after its own bounded timeout and auto-selection reaches its local-mesh fallback. Readiness is the same structured passive_mode/status=ready/role=client event the direct client path already asserts. Nothing touches the public mesh. native-package-qa.sh now runs the direct client smoke and then the auto smoke, so every row proves both paths without waiting on a live mesh. Auto mode gets a 90-second default readiness budget rather than 45 to cover the discovery timeout. MESH_LLM_SMOKE_AUTO_RELAY is the seam for a local relay fixture that would extend this into real join coverage. Asserting a genuine public-mesh join stays upstream's ci-client-auto-test.sh, which is not something package rows should depend on. --- TODO.md | 16 ++++++++++ docs/native-packages.md | 11 +++++++ scripts/client-readiness-smoke.sh | 27 +++++++++++++++- scripts/native-package-qa.sh | 5 ++- tests/client-readiness-smoke.test.ts | 47 ++++++++++++++++++++++++++-- 5 files changed, 102 insertions(+), 4 deletions(-) diff --git a/TODO.md b/TODO.md index efb3ed5..02ebbc3 100644 --- a/TODO.md +++ b/TODO.md @@ -1,5 +1,21 @@ # Production Readiness TODO +- [x] Cover the automatic runtime-selection path in package QA without + depending on public Nostr discovery. + Final result: the shared readiness smoke takes `MESH_LLM_SMOKE_MODE`, and + `auto` runs `--auto --disable-iroh-relays --nostr-relay ws://127.0.0.1:1/`. + The pinned relay parses but cannot connect, so discovery fails closed after + its own bounded timeout and auto-selection reaches the same structured + `passive_mode`/`status=ready`/`role=client` event. `native-package-qa.sh` runs + the direct client smoke and then the auto smoke, so every row proves both + paths and no row waits on a live mesh. `MESH_LLM_SMOKE_AUTO_RELAY` is the seam + for a future local relay fixture that would extend this to join coverage. + QA: focused smoke tests prove auto mode passes the auto-selection and offline + relay flags, rejects an unknown mode, and that package QA runs both modes; + both modes were run end to end against a real `mesh-llm` 0.76.1 binary, exiting + 0 in 5s (client) and 10s (auto), the difference being the bounded discovery + timeout. + - [ ] Keep no-driver package QA reliable under hosted-runner load while still enforcing bounded SIGINT shutdown. QA: the shared Linux smoke and Homebrew formula use the same 30-second bound, focused regression tests pass, the full diff --git a/docs/native-packages.md b/docs/native-packages.md index d2566a1..dfb2042 100644 --- a/docs/native-packages.md +++ b/docs/native-packages.md @@ -44,6 +44,17 @@ unique API/console ports and cache/runtime roots to start `passive_mode`/`status=ready`/`role=client` event while the process is alive, and require bounded SIGINT shutdown. +It then repeats that readiness proof with `MESH_LLM_SMOKE_MODE=auto`, which adds +`--auto --disable-iroh-relays --nostr-relay ws://127.0.0.1:1/`. That covers the +automatic runtime-selection path upstream tests, without making any package row +depend on the public mesh: the pinned relay parses but cannot connect, so +discovery fails closed after its own bounded timeout and auto-selection lands on +its local-mesh fallback with the same structured readiness event. Auto mode gets +a 90-second default readiness budget instead of 45 to cover that discovery +timeout. Point `MESH_LLM_SMOKE_AUTO_RELAY` at a real local relay fixture to +extend this into join coverage; upstream's `scripts/ci-client-auto-test.sh` +remains the place where a genuine public-mesh join is asserted. + The per-row workflow builds the Dockerfile's final `runtime` target once, then runs external QA against that exact image. It verifies package ownership, rejects backend imports or unresolved libraries from the host executable, and diff --git a/scripts/client-readiness-smoke.sh b/scripts/client-readiness-smoke.sh index c366d65..20393c7 100644 --- a/scripts/client-readiness-smoke.sh +++ b/scripts/client-readiness-smoke.sh @@ -5,7 +5,22 @@ set -eu mesh_llm_bin="${MESH_LLM_SMOKE_BIN:-/usr/local/bin/mesh-llm}" -ready_timeout="${MESH_LLM_SMOKE_READY_TIMEOUT_SECONDS:-45}" +smoke_mode="${MESH_LLM_SMOKE_MODE:-client}" +# A relay URL that parses but cannot connect. The client registers it without +# dialing, the discovery fetch fails closed after its own bounded timeout, and +# auto-selection lands on its local-mesh fallback. Nothing reaches the public +# Nostr network. Point this at a real local relay fixture to cover joining. +auto_relay="${MESH_LLM_SMOKE_AUTO_RELAY:-ws://127.0.0.1:1/}" + +case "$smoke_mode" in + client) default_ready_timeout=45 ;; + # Auto-selection pays a bounded discovery timeout before the client API comes + # up, so it gets more room than the direct client path. + auto) default_ready_timeout=90 ;; + *) echo "unsupported smoke mode: $smoke_mode (expected client or auto)" >&2; exit 2 ;; +esac + +ready_timeout="${MESH_LLM_SMOKE_READY_TIMEOUT_SECONDS:-$default_ready_timeout}" shutdown_timeout="${MESH_LLM_SMOKE_SHUTDOWN_TIMEOUT_SECONDS:-30}" case "$ready_timeout:$shutdown_timeout" in @@ -111,6 +126,16 @@ chmod 700 \ # Package/image QA must be self-contained. Plain client mode exercises the # local API/passive runtime path without making readiness depend on public # Nostr discovery or the availability of a remote mesh. + # + # Auto mode covers the automatic runtime-selection path the same way: the + # pinned unreachable relay and disabled iroh relays keep discovery bounded and + # offline, so readiness proves auto-selection works rather than proving the + # public mesh happened to be up. + if [ "$smoke_mode" = "auto" ]; then + exec "$mesh_llm_bin" --log-format json --auto --disable-iroh-relays \ + --nostr-relay "$auto_relay" \ + --port "$api_port" --console "$console_port" --no-console client + fi exec "$mesh_llm_bin" --log-format json --port "$api_port" --console "$console_port" --no-console client ) >"$log" 2>&1 & pid=$! diff --git a/scripts/native-package-qa.sh b/scripts/native-package-qa.sh index 72dba95..b7d6963 100755 --- a/scripts/native-package-qa.sh +++ b/scripts/native-package-qa.sh @@ -128,7 +128,10 @@ run_package_container() { } # shellcheck disable=SC2016 -runtime_smoke='test "$(find "/usr/local/lib/mesh-llm/$EXPECTED_VERSION/native-runtimes" -name manifest.json -type f | wc -l)" -eq 1 && test -f "/usr/local/lib/mesh-llm/$EXPECTED_VERSION/product-manifest.json" && mesh-llm --version | grep -F "$EXPECTED_VERSION" && mesh-llm runtime list && sh /usr/local/bin/client-readiness-smoke' +# The second smoke run covers automatic runtime selection. It stays hermetic: +# the smoke pins an unreachable relay and disables iroh relays, so no package row +# depends on public Nostr discovery being up. +runtime_smoke='test "$(find "/usr/local/lib/mesh-llm/$EXPECTED_VERSION/native-runtimes" -name manifest.json -type f | wc -l)" -eq 1 && test -f "/usr/local/lib/mesh-llm/$EXPECTED_VERSION/product-manifest.json" && mesh-llm --version | grep -F "$EXPECTED_VERSION" && mesh-llm runtime list && sh /usr/local/bin/client-readiness-smoke && MESH_LLM_SMOKE_MODE=auto sh /usr/local/bin/client-readiness-smoke' case "$distro" in ubuntu) diff --git a/tests/client-readiness-smoke.test.ts b/tests/client-readiness-smoke.test.ts index 77d79b4..ae505e2 100644 --- a/tests/client-readiness-smoke.test.ts +++ b/tests/client-readiness-smoke.test.ts @@ -33,7 +33,7 @@ function fakeNodeExecutable( return { executable, marker }; } -function runSmoke(executable: string, marker: string, readyTimeout = "3", shutdownTimeout = "3") { +function runSmoke(executable: string, marker: string, readyTimeout = "3", shutdownTimeout = "3", mode?: string) { const env = { ...process.env }; delete env.NODE_TEST_CONTEXT; return spawnSync("sh", [smoke], { @@ -45,6 +45,7 @@ function runSmoke(executable: string, marker: string, readyTimeout = "3", shutdo MESH_LLM_SMOKE_READY_TIMEOUT_SECONDS: readyTimeout, MESH_LLM_SMOKE_SHUTDOWN_TIMEOUT_SECONDS: shutdownTimeout, SMOKE_MARKER: marker, + ...(mode === undefined ? {} : { MESH_LLM_SMOKE_MODE: mode }), }, }); } @@ -127,11 +128,53 @@ setInterval(() => {}, 1000) assertProcessAbsent(markerPids(fixture.marker)[0]); }); +test("auto mode selects a runtime without reaching public discovery", { concurrency: false }, (t) => { + const fixture = fakeNodeExecutable(t, ` +const fs = require('node:fs') +fs.writeFileSync(process.env.SMOKE_MARKER, \`start:\${process.pid}\\n\`) +const argv = process.argv.slice(2) +const relay = argv[argv.indexOf('--nostr-relay') + 1] +// Auto mode must ask for auto-selection and pin discovery offline. A relay +// pointing anywhere public would make readiness depend on the live mesh. +if (!argv.includes('client') || !argv.includes('--auto') || !argv.includes('--disable-iroh-relays') + || !/^ws:\\/\\/127\\.0\\.0\\.1:/.test(relay ?? '')) { + process.exit(64) +} +process.on('SIGINT', () => { + fs.appendFileSync(process.env.SMOKE_MARKER, \`int:\${process.pid}\\n\`) + process.exit(0) +}) +console.log('{"role":"client","status":"ready","event":"passive_mode"}') +setInterval(() => {}, 1000) +`); + const result = runSmoke(fixture.executable, fixture.marker, "3", "3", "auto"); + assert.equal(result.status, 0, result.stderr); + const [start, interrupted] = markerPids(fixture.marker); + assert.equal(start, interrupted); + assertProcessAbsent(start); +}); + +test("client readiness smoke rejects an unknown mode", { concurrency: false }, (t) => { + const fixture = fakeExecutable(t, "exit 0\n"); + const result = runSmoke(fixture.executable, fixture.marker, "3", "3", "public"); + assert.equal(result.status, 2); + assert.match(result.stderr, /unsupported smoke mode: public/); +}); + +test("package QA proves both the direct client and auto-selection paths", { concurrency: false }, () => { + const qa = readFileSync(resolve("scripts/native-package-qa.sh"), "utf8"); + assert.match(qa, /sh \/usr\/local\/bin\/client-readiness-smoke && MESH_LLM_SMOKE_MODE=auto sh \/usr\/local\/bin\/client-readiness-smoke/); +}); + test("client readiness smoke polls readiness without shell-signal wakeups", { concurrency: false }, () => { const source = readFileSync(smoke, "utf8"); assert.match(source, /--no-console client/); assert.match(source, /MESH_LLM_SMOKE_SHUTDOWN_TIMEOUT_SECONDS:-30/); - assert.doesNotMatch(source, /client --auto/); + // Auto-selection is opt-in per run. The default path stays the direct client + // so no package row depends on discovery, and auto stays pinned to loopback. + assert.match(source, /smoke_mode="\$\{MESH_LLM_SMOKE_MODE:-client\}"/); + assert.match(source, /auto_relay="\$\{MESH_LLM_SMOKE_AUTO_RELAY:-ws:\/\/127\.0\.0\.1:1\/\}"/); + assert.match(source, /--auto --disable-iroh-relays/); assert.match(source, /readiness_reached=false/); assert.match(source, /readiness_in_log/); assert.match(source, /if ! kill -0 "\$pid" 2>\/dev\/null; then/);