From 07c09c349f6b5bc34e3cdad42ffdebb2b7e5ad52 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 6 Aug 2026 22:01:22 -0700 Subject: [PATCH 01/12] fix(shields): wait for Hermes inference convergence Signed-off-by: Prekshi Vyas --- src/lib/shields/index.ts | 14 ++++ src/lib/shields/inference-convergence.test.ts | 76 +++++++++++++++++++ src/lib/shields/inference-convergence.ts | 60 +++++++++++++++ src/lib/shields/legacy-hermes-compat.test.ts | 6 ++ 4 files changed, 156 insertions(+) create mode 100644 src/lib/shields/inference-convergence.test.ts create mode 100644 src/lib/shields/inference-convergence.ts diff --git a/src/lib/shields/index.ts b/src/lib/shields/index.ts index fc239035411..ecfec7456c0 100644 --- a/src/lib/shields/index.ts +++ b/src/lib/shields/index.ts @@ -71,6 +71,9 @@ const { const { cleanupTempDir } = require("../onboard/temp-files"); const { verifyShieldsLockState }: typeof import("./verify-lock") = require("./verify-lock"); const { relockAndReconfirm }: typeof import("./relock-reconfirm") = require("./relock-reconfirm"); +const { + waitForHermesInferenceRouteConvergence, +}: typeof import("./inference-convergence") = require("./inference-convergence"); const { inspectAnyShieldsTransitionLockOwner, isShieldsTransitionLockUnavailable, @@ -3848,6 +3851,17 @@ function shieldsDownWithoutHostLock(sandboxName: string, opts: ShieldsDownOpts = opts.allowLegacyHermesProtocol === true, protocol, ); + if (target.agentName === "hermes") { + console.log(" Confirming Hermes inference route after policy transition..."); + const convergence = waitForHermesInferenceRouteConvergence(sandboxName); + if (!convergence.ok) { + const status = + convergence.httpStatus > 0 ? `HTTP ${convergence.httpStatus}` : "unavailable"; + throw new Error( + `Hermes inference route did not converge after policy transition (${status}; ${convergence.attempts} attempts)`, + ); + } + } } catch (err) { const message = err instanceof Error ? err.message : String(err); const rollback = rollbackShieldsDown( diff --git a/src/lib/shields/inference-convergence.test.ts b/src/lib/shields/inference-convergence.test.ts new file mode 100644 index 00000000000..6ae5f715326 --- /dev/null +++ b/src/lib/shields/inference-convergence.test.ts @@ -0,0 +1,76 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { waitForHermesInferenceRouteConvergence } from "./inference-convergence"; + +function probe(status: number, output: string) { + return { status, output, stderr: "" }; +} + +describe("Hermes inference convergence after a Shields policy transition", () => { + it("returns on the first healthy route probe", () => { + const capture = vi.fn(() => probe(0, "OK 200")); + const sleep = vi.fn(); + + const result = waitForHermesInferenceRouteConvergence("hermes-box", { capture, sleep }); + + expect(result).toEqual({ ok: true, attempts: 1, httpStatus: 200 }); + expect(sleep).not.toHaveBeenCalled(); + expect(capture.mock.calls[0]?.[0]).toEqual([ + "sandbox", + "exec", + "--name", + "hermes-box", + "--", + "sh", + "-c", + expect.stringContaining("https://inference.local/v1/models"), + ]); + }); + + it("waits for a transient HTTP 503 to converge", () => { + const capture = vi + .fn() + .mockReturnValueOnce(probe(0, "BROKEN 503")) + .mockReturnValueOnce(probe(0, "OK 200")); + const sleep = vi.fn(); + + const result = waitForHermesInferenceRouteConvergence("hermes-box", { + capture, + retryDelayMs: 750, + sleep, + }); + + expect(result).toEqual({ ok: true, attempts: 2, httpStatus: 200 }); + expect(sleep).toHaveBeenCalledOnce(); + expect(sleep).toHaveBeenCalledWith(750); + }); + + it("fails after the bounded probe budget instead of reporting Shields down ready", () => { + const capture = vi.fn(() => probe(0, "BROKEN 503")); + const sleep = vi.fn(); + + const result = waitForHermesInferenceRouteConvergence("hermes-box", { + capture, + maxAttempts: 3, + sleep, + }); + + expect(result).toEqual({ ok: false, attempts: 3, httpStatus: 503 }); + expect(capture).toHaveBeenCalledTimes(3); + expect(sleep).toHaveBeenCalledTimes(2); + }); + + it("does not accept untrusted or unavailable probe output as convergence", () => { + const capture = vi.fn(() => ({ status: 1, output: "attacker preamble\nOK 200", stderr: "" })); + + const result = waitForHermesInferenceRouteConvergence("hermes-box", { + capture, + maxAttempts: 1, + }); + + expect(result).toEqual({ ok: false, attempts: 1, httpStatus: 0 }); + }); +}); diff --git a/src/lib/shields/inference-convergence.ts b/src/lib/shields/inference-convergence.ts new file mode 100644 index 00000000000..49d2ac823b8 --- /dev/null +++ b/src/lib/shields/inference-convergence.ts @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + buildSandboxInferenceRouteProbeArgs, + parseSandboxInferenceRouteProbeResult, +} from "../actions/sandbox/connect-inference-route-probe"; +import { captureOpenshell } from "../adapters/openshell/runtime"; +import { OPENSHELL_INFERENCE_ROUTE_PROBE_TIMEOUT_MS } from "../adapters/openshell/timeouts"; +import { sleepMs } from "../core/wait"; + +const DEFAULT_MAX_ATTEMPTS = 4; +const DEFAULT_RETRY_DELAY_MS = 500; + +type CaptureInferenceRoute = typeof captureOpenshell; + +export interface InferenceRouteConvergenceResult { + ok: boolean; + attempts: number; + httpStatus: number; +} + +export interface InferenceRouteConvergenceOptions { + capture?: CaptureInferenceRoute; + maxAttempts?: number; + retryDelayMs?: number; + sleep?: (milliseconds: number) => void; +} + +/** + * Require the Hermes inference route to be usable after a live policy + * replacement. OpenShell's `policy set --wait` confirms the policy version is + * active, but the inference proxy can briefly continue returning HTTP 503 + * after that acknowledgement. Shields down must not report completion during + * that gap because callers immediately resume agent work. + */ +export function waitForHermesInferenceRouteConvergence( + sandboxName: string, + options: InferenceRouteConvergenceOptions = {}, +): InferenceRouteConvergenceResult { + const capture = options.capture ?? captureOpenshell; + const maxAttempts = Math.max(1, Math.trunc(options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS)); + const retryDelayMs = Math.max(0, Math.trunc(options.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS)); + const sleep = options.sleep ?? sleepMs; + let httpStatus = 0; + + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + const probe = capture(buildSandboxInferenceRouteProbeArgs(sandboxName, { name: "hermes" }), { + ignoreError: true, + includeStreams: true, + timeout: OPENSHELL_INFERENCE_ROUTE_PROBE_TIMEOUT_MS, + }); + const parsed = parseSandboxInferenceRouteProbeResult(probe); + httpStatus = parsed.httpStatus; + if (parsed.healthy) return { ok: true, attempts: attempt, httpStatus }; + if (attempt < maxAttempts) sleep(retryDelayMs); + } + + return { ok: false, attempts: maxAttempts, httpStatus }; +} diff --git a/src/lib/shields/legacy-hermes-compat.test.ts b/src/lib/shields/legacy-hermes-compat.test.ts index 16b3353f57a..d478e6bf534 100644 --- a/src/lib/shields/legacy-hermes-compat.test.ts +++ b/src/lib/shields/legacy-hermes-compat.test.ts @@ -100,6 +100,7 @@ describe("legacy Hermes shields compatibility", () => { const privilegedExec = requireSource("../sandbox/privileged-exec.js"); const dockerExec = requireSource("../adapters/docker/exec.js"); const stateDirLock = requireSource("./state-dir-lock.js"); + const inferenceConvergence = requireSource("./inference-convergence.js"); const audit = requireSource("./audit.js"); const permissiveRuntime = requireSource("./permissive-runtime.js"); const tempFiles = requireSource("../onboard/temp-files.js"); @@ -133,6 +134,11 @@ describe("legacy Hermes shields compatibility", () => { vi.spyOn(stateDirLock, "preflightStateDirLock").mockReturnValue([]), vi.spyOn(stateDirLock, "restoreStateDirLockPosture").mockReturnValue([]), vi.spyOn(stateDirLock, "stateLockPlanCompatibilityIssues").mockReturnValue([]), + vi.spyOn(inferenceConvergence, "waitForHermesInferenceRouteConvergence").mockReturnValue({ + ok: true, + attempts: 1, + httpStatus: 200, + }), vi.spyOn(audit, "appendAuditEntry").mockImplementation(() => undefined), vi .spyOn(permissiveRuntime, "buildRuntimePermissivePolicy") From 432052245a378cdcc518374dbd4fb391734b8a4e Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 6 Aug 2026 22:06:06 -0700 Subject: [PATCH 02/12] refactor(shields): preserve source dependency budgets Signed-off-by: Prekshi Vyas --- src/lib/shields/index.ts | 6 +-- src/lib/shields/inference-convergence.test.ts | 22 +++++----- src/lib/shields/inference-convergence.ts | 42 +++++++++++++------ src/lib/shields/legacy-hermes-compat.test.ts | 4 +- src/lib/shields/relock-reconfirm.ts | 2 + 5 files changed, 48 insertions(+), 28 deletions(-) diff --git a/src/lib/shields/index.ts b/src/lib/shields/index.ts index ecfec7456c0..a480d7ce570 100644 --- a/src/lib/shields/index.ts +++ b/src/lib/shields/index.ts @@ -70,10 +70,10 @@ const { }: typeof import("./permissive-runtime") = require("./permissive-runtime"); const { cleanupTempDir } = require("../onboard/temp-files"); const { verifyShieldsLockState }: typeof import("./verify-lock") = require("./verify-lock"); -const { relockAndReconfirm }: typeof import("./relock-reconfirm") = require("./relock-reconfirm"); const { + relockAndReconfirm, waitForHermesInferenceRouteConvergence, -}: typeof import("./inference-convergence") = require("./inference-convergence"); +}: typeof import("./relock-reconfirm") = require("./relock-reconfirm"); const { inspectAnyShieldsTransitionLockOwner, isShieldsTransitionLockUnavailable, @@ -3853,7 +3853,7 @@ function shieldsDownWithoutHostLock(sandboxName: string, opts: ShieldsDownOpts = ); if (target.agentName === "hermes") { console.log(" Confirming Hermes inference route after policy transition..."); - const convergence = waitForHermesInferenceRouteConvergence(sandboxName); + const convergence = waitForHermesInferenceRouteConvergence(sandboxName, { run }); if (!convergence.ok) { const status = convergence.httpStatus > 0 ? `HTTP ${convergence.httpStatus}` : "unavailable"; diff --git a/src/lib/shields/inference-convergence.test.ts b/src/lib/shields/inference-convergence.test.ts index 6ae5f715326..5692f9b03f0 100644 --- a/src/lib/shields/inference-convergence.test.ts +++ b/src/lib/shields/inference-convergence.test.ts @@ -6,19 +6,19 @@ import { describe, expect, it, vi } from "vitest"; import { waitForHermesInferenceRouteConvergence } from "./inference-convergence"; function probe(status: number, output: string) { - return { status, output, stderr: "" }; + return { status, stdout: output, stderr: "" }; } describe("Hermes inference convergence after a Shields policy transition", () => { it("returns on the first healthy route probe", () => { - const capture = vi.fn(() => probe(0, "OK 200")); + const run = vi.fn(() => probe(0, "OK 200")); const sleep = vi.fn(); - const result = waitForHermesInferenceRouteConvergence("hermes-box", { capture, sleep }); + const result = waitForHermesInferenceRouteConvergence("hermes-box", { run, sleep }); expect(result).toEqual({ ok: true, attempts: 1, httpStatus: 200 }); expect(sleep).not.toHaveBeenCalled(); - expect(capture.mock.calls[0]?.[0]).toEqual([ + expect(run.mock.calls[0]?.[0]).toEqual([ "sandbox", "exec", "--name", @@ -31,15 +31,15 @@ describe("Hermes inference convergence after a Shields policy transition", () => }); it("waits for a transient HTTP 503 to converge", () => { - const capture = vi + const run = vi .fn() .mockReturnValueOnce(probe(0, "BROKEN 503")) .mockReturnValueOnce(probe(0, "OK 200")); const sleep = vi.fn(); const result = waitForHermesInferenceRouteConvergence("hermes-box", { - capture, retryDelayMs: 750, + run, sleep, }); @@ -49,26 +49,26 @@ describe("Hermes inference convergence after a Shields policy transition", () => }); it("fails after the bounded probe budget instead of reporting Shields down ready", () => { - const capture = vi.fn(() => probe(0, "BROKEN 503")); + const run = vi.fn(() => probe(0, "BROKEN 503")); const sleep = vi.fn(); const result = waitForHermesInferenceRouteConvergence("hermes-box", { - capture, maxAttempts: 3, + run, sleep, }); expect(result).toEqual({ ok: false, attempts: 3, httpStatus: 503 }); - expect(capture).toHaveBeenCalledTimes(3); + expect(run).toHaveBeenCalledTimes(3); expect(sleep).toHaveBeenCalledTimes(2); }); it("does not accept untrusted or unavailable probe output as convergence", () => { - const capture = vi.fn(() => ({ status: 1, output: "attacker preamble\nOK 200", stderr: "" })); + const run = vi.fn(() => ({ status: 1, stdout: "attacker preamble\nOK 200", stderr: "" })); const result = waitForHermesInferenceRouteConvergence("hermes-box", { - capture, maxAttempts: 1, + run, }); expect(result).toEqual({ ok: false, attempts: 1, httpStatus: 0 }); diff --git a/src/lib/shields/inference-convergence.ts b/src/lib/shields/inference-convergence.ts index 49d2ac823b8..7312b515823 100644 --- a/src/lib/shields/inference-convergence.ts +++ b/src/lib/shields/inference-convergence.ts @@ -5,14 +5,26 @@ import { buildSandboxInferenceRouteProbeArgs, parseSandboxInferenceRouteProbeResult, } from "../actions/sandbox/connect-inference-route-probe"; -import { captureOpenshell } from "../adapters/openshell/runtime"; -import { OPENSHELL_INFERENCE_ROUTE_PROBE_TIMEOUT_MS } from "../adapters/openshell/timeouts"; -import { sleepMs } from "../core/wait"; const DEFAULT_MAX_ATTEMPTS = 4; const DEFAULT_RETRY_DELAY_MS = 500; +const INFERENCE_ROUTE_PROBE_TIMEOUT_MS = 10_000; -type CaptureInferenceRoute = typeof captureOpenshell; +interface InferenceRouteCommandResult { + status: number | null; + stdout?: string | Buffer | null; + stderr?: string | Buffer | null; +} + +type RunInferenceRoute = ( + command: readonly string[], + options: { ignoreError: true; suppressOutput: true; timeout: number }, +) => InferenceRouteCommandResult; + +function sleepMs(milliseconds: number): void { + if (milliseconds <= 0 || !Number.isFinite(milliseconds)) return; + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds); +} export interface InferenceRouteConvergenceResult { ok: boolean; @@ -21,9 +33,9 @@ export interface InferenceRouteConvergenceResult { } export interface InferenceRouteConvergenceOptions { - capture?: CaptureInferenceRoute; maxAttempts?: number; retryDelayMs?: number; + run: RunInferenceRoute; sleep?: (milliseconds: number) => void; } @@ -36,21 +48,27 @@ export interface InferenceRouteConvergenceOptions { */ export function waitForHermesInferenceRouteConvergence( sandboxName: string, - options: InferenceRouteConvergenceOptions = {}, + options: InferenceRouteConvergenceOptions, ): InferenceRouteConvergenceResult { - const capture = options.capture ?? captureOpenshell; const maxAttempts = Math.max(1, Math.trunc(options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS)); const retryDelayMs = Math.max(0, Math.trunc(options.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS)); const sleep = options.sleep ?? sleepMs; let httpStatus = 0; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { - const probe = capture(buildSandboxInferenceRouteProbeArgs(sandboxName, { name: "hermes" }), { - ignoreError: true, - includeStreams: true, - timeout: OPENSHELL_INFERENCE_ROUTE_PROBE_TIMEOUT_MS, + const probe = options.run( + buildSandboxInferenceRouteProbeArgs(sandboxName, { name: "hermes" }), + { + ignoreError: true, + suppressOutput: true, + timeout: INFERENCE_ROUTE_PROBE_TIMEOUT_MS, + }, + ); + const parsed = parseSandboxInferenceRouteProbeResult({ + status: probe.status, + output: String(probe.stdout ?? ""), + stderr: String(probe.stderr ?? ""), }); - const parsed = parseSandboxInferenceRouteProbeResult(probe); httpStatus = parsed.httpStatus; if (parsed.healthy) return { ok: true, attempts: attempt, httpStatus }; if (attempt < maxAttempts) sleep(retryDelayMs); diff --git a/src/lib/shields/legacy-hermes-compat.test.ts b/src/lib/shields/legacy-hermes-compat.test.ts index d478e6bf534..2ed5392d8dd 100644 --- a/src/lib/shields/legacy-hermes-compat.test.ts +++ b/src/lib/shields/legacy-hermes-compat.test.ts @@ -100,7 +100,7 @@ describe("legacy Hermes shields compatibility", () => { const privilegedExec = requireSource("../sandbox/privileged-exec.js"); const dockerExec = requireSource("../adapters/docker/exec.js"); const stateDirLock = requireSource("./state-dir-lock.js"); - const inferenceConvergence = requireSource("./inference-convergence.js"); + const relockReconfirm = requireSource("./relock-reconfirm.js"); const audit = requireSource("./audit.js"); const permissiveRuntime = requireSource("./permissive-runtime.js"); const tempFiles = requireSource("../onboard/temp-files.js"); @@ -134,7 +134,7 @@ describe("legacy Hermes shields compatibility", () => { vi.spyOn(stateDirLock, "preflightStateDirLock").mockReturnValue([]), vi.spyOn(stateDirLock, "restoreStateDirLockPosture").mockReturnValue([]), vi.spyOn(stateDirLock, "stateLockPlanCompatibilityIssues").mockReturnValue([]), - vi.spyOn(inferenceConvergence, "waitForHermesInferenceRouteConvergence").mockReturnValue({ + vi.spyOn(relockReconfirm, "waitForHermesInferenceRouteConvergence").mockReturnValue({ ok: true, attempts: 1, httpStatus: 200, diff --git a/src/lib/shields/relock-reconfirm.ts b/src/lib/shields/relock-reconfirm.ts index 9cbb266c828..d616cedee32 100644 --- a/src/lib/shields/relock-reconfirm.ts +++ b/src/lib/shields/relock-reconfirm.ts @@ -27,6 +27,8 @@ import { sleepMs } from "../core/wait"; +export { waitForHermesInferenceRouteConvergence } from "./inference-convergence"; + const DEFAULT_MAX_ATTEMPTS = 3; const DEFAULT_SETTLE_MS = 750; const MIN_SETTLE_MS = 0; From ac025038aff73ff36c2cc08880aea2da65e7cb66 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 6 Aug 2026 22:08:41 -0700 Subject: [PATCH 03/12] test(shields): type the convergence runner seam Signed-off-by: Prekshi Vyas --- src/lib/shields/inference-convergence.test.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/lib/shields/inference-convergence.test.ts b/src/lib/shields/inference-convergence.test.ts index 5692f9b03f0..5314df1814a 100644 --- a/src/lib/shields/inference-convergence.test.ts +++ b/src/lib/shields/inference-convergence.test.ts @@ -11,7 +11,7 @@ function probe(status: number, output: string) { describe("Hermes inference convergence after a Shields policy transition", () => { it("returns on the first healthy route probe", () => { - const run = vi.fn(() => probe(0, "OK 200")); + const run = vi.fn((_command: readonly string[], _options: object) => probe(0, "OK 200")); const sleep = vi.fn(); const result = waitForHermesInferenceRouteConvergence("hermes-box", { run, sleep }); @@ -32,7 +32,7 @@ describe("Hermes inference convergence after a Shields policy transition", () => it("waits for a transient HTTP 503 to converge", () => { const run = vi - .fn() + .fn((_command: readonly string[], _options: object) => probe(0, "OK 200")) .mockReturnValueOnce(probe(0, "BROKEN 503")) .mockReturnValueOnce(probe(0, "OK 200")); const sleep = vi.fn(); @@ -49,7 +49,7 @@ describe("Hermes inference convergence after a Shields policy transition", () => }); it("fails after the bounded probe budget instead of reporting Shields down ready", () => { - const run = vi.fn(() => probe(0, "BROKEN 503")); + const run = vi.fn((_command: readonly string[], _options: object) => probe(0, "BROKEN 503")); const sleep = vi.fn(); const result = waitForHermesInferenceRouteConvergence("hermes-box", { @@ -64,7 +64,11 @@ describe("Hermes inference convergence after a Shields policy transition", () => }); it("does not accept untrusted or unavailable probe output as convergence", () => { - const run = vi.fn(() => ({ status: 1, stdout: "attacker preamble\nOK 200", stderr: "" })); + const run = vi.fn((_command: readonly string[], _options: object) => ({ + status: 1, + stdout: "attacker preamble\nOK 200", + stderr: "", + })); const result = waitForHermesInferenceRouteConvergence("hermes-box", { maxAttempts: 1, From 34c454c8e193cf95c8230d9e8bd81250f64e4703 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 6 Aug 2026 22:19:20 -0700 Subject: [PATCH 04/12] fix(shields): harden inference convergence recovery Signed-off-by: Prekshi Vyas --- src/lib/shields/index.ts | 14 +++++-- src/lib/shields/inference-convergence.test.ts | 42 ++++++++++++++++++- src/lib/shields/inference-convergence.ts | 10 ++++- src/lib/shields/legacy-hermes-compat.test.ts | 40 +++++++++++++++--- 4 files changed, 93 insertions(+), 13 deletions(-) diff --git a/src/lib/shields/index.ts b/src/lib/shields/index.ts index a480d7ce570..0a34b2f7a5e 100644 --- a/src/lib/shields/index.ts +++ b/src/lib/shields/index.ts @@ -3843,6 +3843,7 @@ function shieldsDownWithoutHostLock(sandboxName: string, opts: ShieldsDownOpts = // OpenClaw uses sandbox:sandbox 0660/2770 here so the gateway UID, which // is a member of the sandbox group, can mutate runtime config. console.log(` Unlocking ${target.agentName} config (${target.configPath})...`); + let inferenceRouteConvergenceFailed = false; try { unlockAgentConfig( sandboxName, @@ -3855,6 +3856,7 @@ function shieldsDownWithoutHostLock(sandboxName: string, opts: ShieldsDownOpts = console.log(" Confirming Hermes inference route after policy transition..."); const convergence = waitForHermesInferenceRouteConvergence(sandboxName, { run }); if (!convergence.ok) { + inferenceRouteConvergenceFailed = true; const status = convergence.httpStatus > 0 ? `HTTP ${convergence.httpStatus}` : "unavailable"; throw new Error( @@ -3892,9 +3894,15 @@ function shieldsDownWithoutHostLock(sandboxName: string, opts: ShieldsDownOpts = ` Config rollback is incomplete.${timerAuthority} Manual intervention is required.`, ); } - console.error( - ` Re-run \`nemoclaw ${sandboxName} shields down\` after correcting file ownership.`, - ); + if (inferenceRouteConvergenceFailed) { + console.error( + ` Recover the Hermes inference route, then re-run \`nemoclaw ${sandboxName} shields down\`.`, + ); + } else { + console.error( + ` Re-run \`nemoclaw ${sandboxName} shields down\` after correcting file ownership.`, + ); + } return failShieldsCommand(message, opts.throwOnError); } diff --git a/src/lib/shields/inference-convergence.test.ts b/src/lib/shields/inference-convergence.test.ts index 5314df1814a..f05b4b8b56e 100644 --- a/src/lib/shields/inference-convergence.test.ts +++ b/src/lib/shields/inference-convergence.test.ts @@ -63,9 +63,9 @@ describe("Hermes inference convergence after a Shields policy transition", () => expect(sleep).toHaveBeenCalledTimes(2); }); - it("does not accept untrusted or unavailable probe output as convergence", () => { + it("does not accept preambled probe output as convergence even when the command succeeds", () => { const run = vi.fn((_command: readonly string[], _options: object) => ({ - status: 1, + status: 0, stdout: "attacker preamble\nOK 200", stderr: "", })); @@ -77,4 +77,42 @@ describe("Hermes inference convergence after a Shields policy transition", () => expect(result).toEqual({ ok: false, attempts: 1, httpStatus: 0 }); }); + + it.each([ + Number.NaN, + Number.POSITIVE_INFINITY, + ])("uses the bounded default attempt budget for non-finite maxAttempts (%s)", (maxAttempts) => { + const run = vi.fn((_command: readonly string[], _options: object) => probe(0, "BROKEN 503")); + const sleep = vi.fn(); + + const result = waitForHermesInferenceRouteConvergence("hermes-box", { + maxAttempts, + run, + sleep, + }); + + expect(result).toEqual({ ok: false, attempts: 4, httpStatus: 503 }); + expect(run).toHaveBeenCalledTimes(4); + expect(sleep).toHaveBeenCalledTimes(3); + }); + + it.each([ + Number.NaN, + Number.POSITIVE_INFINITY, + ])("uses the default delay for non-finite retryDelayMs (%s)", (retryDelayMs) => { + const run = vi + .fn((_command: readonly string[], _options: object) => probe(0, "OK 200")) + .mockReturnValueOnce(probe(0, "BROKEN 503")); + const sleep = vi.fn(); + + const result = waitForHermesInferenceRouteConvergence("hermes-box", { + retryDelayMs, + run, + sleep, + }); + + expect(result).toEqual({ ok: true, attempts: 2, httpStatus: 200 }); + expect(sleep).toHaveBeenCalledOnce(); + expect(sleep).toHaveBeenCalledWith(500); + }); }); diff --git a/src/lib/shields/inference-convergence.ts b/src/lib/shields/inference-convergence.ts index 7312b515823..914d9e6e64b 100644 --- a/src/lib/shields/inference-convergence.ts +++ b/src/lib/shields/inference-convergence.ts @@ -50,8 +50,14 @@ export function waitForHermesInferenceRouteConvergence( sandboxName: string, options: InferenceRouteConvergenceOptions, ): InferenceRouteConvergenceResult { - const maxAttempts = Math.max(1, Math.trunc(options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS)); - const retryDelayMs = Math.max(0, Math.trunc(options.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS)); + const configuredMaxAttempts = options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS; + const configuredRetryDelayMs = options.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS; + const maxAttempts = Number.isFinite(configuredMaxAttempts) + ? Math.max(1, Math.trunc(configuredMaxAttempts)) + : DEFAULT_MAX_ATTEMPTS; + const retryDelayMs = Number.isFinite(configuredRetryDelayMs) + ? Math.max(0, Math.trunc(configuredRetryDelayMs)) + : DEFAULT_RETRY_DELAY_MS; const sleep = options.sleep ?? sleepMs; let httpStatus = 0; diff --git a/src/lib/shields/legacy-hermes-compat.test.ts b/src/lib/shields/legacy-hermes-compat.test.ts index 2ed5392d8dd..779494da574 100644 --- a/src/lib/shields/legacy-hermes-compat.test.ts +++ b/src/lib/shields/legacy-hermes-compat.test.ts @@ -87,6 +87,8 @@ describe("legacy Hermes shields compatibility", () => { let dockerExecSpy: MockInstance; let privilegedExecArgvSpy: MockInstance; let applyStateDirLockModeSpy: MockInstance; + let inferenceConvergenceSpy: MockInstance; + let errorSpy: MockInstance; beforeEach(() => { homeDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-legacy-hermes-")); @@ -111,6 +113,14 @@ describe("legacy Hermes shields compatibility", () => { privilegedExecArgvSpy = vi .spyOn(privilegedExec, "privilegedSandboxExecArgv") .mockImplementation((_sandboxName: unknown, cmd: unknown) => cmd as string[]); + inferenceConvergenceSpy = vi + .spyOn(relockReconfirm, "waitForHermesInferenceRouteConvergence") + .mockReturnValue({ + ok: true, + attempts: 1, + httpStatus: 200, + }); + errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); spies.push( runSpy, vi.spyOn(runner, "runCapture").mockReturnValue("version: 1\nnetwork_policies:\n test: {}\n"), @@ -134,11 +144,7 @@ describe("legacy Hermes shields compatibility", () => { vi.spyOn(stateDirLock, "preflightStateDirLock").mockReturnValue([]), vi.spyOn(stateDirLock, "restoreStateDirLockPosture").mockReturnValue([]), vi.spyOn(stateDirLock, "stateLockPlanCompatibilityIssues").mockReturnValue([]), - vi.spyOn(relockReconfirm, "waitForHermesInferenceRouteConvergence").mockReturnValue({ - ok: true, - attempts: 1, - httpStatus: 200, - }), + inferenceConvergenceSpy, vi.spyOn(audit, "appendAuditEntry").mockImplementation(() => undefined), vi .spyOn(permissiveRuntime, "buildRuntimePermissivePolicy") @@ -146,7 +152,7 @@ describe("legacy Hermes shields compatibility", () => { vi.spyOn(tempFiles, "cleanupTempDir").mockImplementation(() => undefined), vi.spyOn(console, "log").mockImplementation(() => undefined), vi.spyOn(console, "warn").mockImplementation(() => undefined), - vi.spyOn(console, "error").mockImplementation(() => undefined), + errorSpy, ); shields = requireSource(INDEX_MODULE); @@ -406,6 +412,28 @@ describe("legacy Hermes shields compatibility", () => { expect(commands.some((cmd) => isGuardAction(cmd, "begin-shields-transition"))).toBe(true); }); + it("gives route-specific recovery guidance when inference convergence fails", () => { + installExecResponses(CURRENT_GUARD_HELP); + inferenceConvergenceSpy.mockReturnValue({ + ok: false, + attempts: 4, + httpStatus: 503, + }); + + expect(() => + shields.shieldsDown("current-hermes", { + skipTimer: true, + throwOnError: true, + }), + ).toThrow(/inference route did not converge.*HTTP 503.*4 attempts/i); + + const errors = errorSpy.mock.calls.map((call) => String(call[0])).join("\n"); + expect(errors).toContain( + "Recover the Hermes inference route, then re-run `nemoclaw current-hermes shields down`.", + ); + expect(errors).not.toContain("after correcting file ownership"); + }); + it("descriptor-safely protects and verifies the sandbox parent when a failed rebuild relocks an old image", () => { dockerExecSpy.mockImplementation((cmd: string[]) => { switch (true) { From f6c63f2f7a8b65b9e23b0d99132c48f5063f46d2 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 6 Aug 2026 22:23:34 -0700 Subject: [PATCH 05/12] test(shields): verify convergence rollback posture Signed-off-by: Prekshi Vyas --- src/lib/shields/legacy-hermes-compat.test.ts | 40 +++++++++++++++++--- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/src/lib/shields/legacy-hermes-compat.test.ts b/src/lib/shields/legacy-hermes-compat.test.ts index 779494da574..ca04fba3ec9 100644 --- a/src/lib/shields/legacy-hermes-compat.test.ts +++ b/src/lib/shields/legacy-hermes-compat.test.ts @@ -88,6 +88,7 @@ describe("legacy Hermes shields compatibility", () => { let privilegedExecArgvSpy: MockInstance; let applyStateDirLockModeSpy: MockInstance; let inferenceConvergenceSpy: MockInstance; + let auditSpy: MockInstance; let errorSpy: MockInstance; beforeEach(() => { @@ -120,6 +121,7 @@ describe("legacy Hermes shields compatibility", () => { attempts: 1, httpStatus: 200, }); + auditSpy = vi.spyOn(audit, "appendAuditEntry").mockImplementation(() => undefined); errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); spies.push( runSpy, @@ -145,7 +147,7 @@ describe("legacy Hermes shields compatibility", () => { vi.spyOn(stateDirLock, "restoreStateDirLockPosture").mockReturnValue([]), vi.spyOn(stateDirLock, "stateLockPlanCompatibilityIssues").mockReturnValue([]), inferenceConvergenceSpy, - vi.spyOn(audit, "appendAuditEntry").mockImplementation(() => undefined), + auditSpy, vi .spyOn(permissiveRuntime, "buildRuntimePermissivePolicy") .mockImplementation((basePath: unknown) => String(basePath)), @@ -165,21 +167,40 @@ describe("legacy Hermes shields compatibility", () => { fs.rmSync(homeDir, { recursive: true, force: true }); }); - function installExecResponses(help: string, hermesDirMode = "3770", finishError?: Error): void { + function installExecResponses( + help: string, + hermesDirMode = "3770", + finishError?: Error, + simulateLockTransition = false, + ): void { + let pendingMode: "locked" | "mutable" = "mutable"; + let appliedMode: "locked" | "mutable" = "mutable"; dockerExecSpy.mockImplementation((cmd: string[]) => { switch (true) { case cmd.includes(HERMES_GUARD) && cmd.includes("--help"): return help; - case isGuardAction(cmd, "begin-shields-transition"): + case isGuardAction(cmd, "begin-shields-transition"): { + const mode = cmd[cmd.indexOf("--shields-mode") + 1]; + pendingMode = mode === "locked" ? "locked" : "mutable"; return `lock_token=${LOCK_TOKEN} original_locked=1`; + } case isGuardAction(cmd, "apply-shields-transition"): - return "shields_mode=mutable chattr_applied=0"; + appliedMode = pendingMode; + return `shields_mode=${appliedMode} chattr_applied=0`; case isGuardAction(cmd, "finish-shields-transition") && finishError !== undefined: throw finishError; + case cmd[0] === "stat" && simulateLockTransition && appliedMode === "locked": + return cmd.at(-1) === "/sandbox/.hermes" + ? "3770 root:sandbox" + : cmd.at(-1) === "/sandbox" + ? "1775 root:sandbox" + : "444 root:root"; case cmd[0] === "stat": return cmd.at(-1) === "/sandbox/.hermes" ? `${hermesDirMode} sandbox:sandbox` : "640 sandbox:sandbox"; + case cmd[0] === "sha256sum": + return `${"b".repeat(64)} ${cmd.at(-1)}`; case cmd[0] === "lsattr": return `---------------- ${cmd.at(-1)}`; default: @@ -413,7 +434,7 @@ describe("legacy Hermes shields compatibility", () => { }); it("gives route-specific recovery guidance when inference convergence fails", () => { - installExecResponses(CURRENT_GUARD_HELP); + installExecResponses(CURRENT_GUARD_HELP, "3770", undefined, true); inferenceConvergenceSpy.mockReturnValue({ ok: false, attempts: 4, @@ -432,6 +453,15 @@ describe("legacy Hermes shields compatibility", () => { "Recover the Hermes inference route, then re-run `nemoclaw current-hermes shields down`.", ); expect(errors).not.toContain("after correcting file ownership"); + const stateDir = path.join(homeDir, ".nemoclaw", "state"); + const state = JSON.parse( + fs.readFileSync(path.join(stateDir, "shields-current-hermes.json"), "utf-8"), + ); + expect(state).toMatchObject({ shieldsDown: false }); + expect( + fs.readdirSync(stateDir).filter((entry) => entry.startsWith("shields-transition-")), + ).toEqual([]); + expect(auditSpy).not.toHaveBeenCalled(); }); it("descriptor-safely protects and verifies the sandbox parent when a failed rebuild relocks an old image", () => { From a838a26884ebd6e71273d2c0bb740b0994e0ae09 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 6 Aug 2026 22:35:16 -0700 Subject: [PATCH 06/12] test(shields): reject malformed transition modes Signed-off-by: Prekshi Vyas --- src/lib/shields/legacy-hermes-compat.test.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/lib/shields/legacy-hermes-compat.test.ts b/src/lib/shields/legacy-hermes-compat.test.ts index ca04fba3ec9..f303f4bc1d4 100644 --- a/src/lib/shields/legacy-hermes-compat.test.ts +++ b/src/lib/shields/legacy-hermes-compat.test.ts @@ -180,8 +180,12 @@ describe("legacy Hermes shields compatibility", () => { case cmd.includes(HERMES_GUARD) && cmd.includes("--help"): return help; case isGuardAction(cmd, "begin-shields-transition"): { - const mode = cmd[cmd.indexOf("--shields-mode") + 1]; - pendingMode = mode === "locked" ? "locked" : "mutable"; + const modeIndex = cmd.indexOf("--shields-mode"); + const mode = modeIndex >= 0 ? cmd[modeIndex + 1] : undefined; + if (mode !== "locked" && mode !== "mutable") { + throw new Error("Invalid --shields-mode in test fixture"); + } + pendingMode = mode; return `lock_token=${LOCK_TOKEN} original_locked=1`; } case isGuardAction(cmd, "apply-shields-transition"): From 4219a63b78ccea6690f1ddcbbe4d16c557cb96a0 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 6 Aug 2026 22:38:48 -0700 Subject: [PATCH 07/12] test(shields): keep transition fixture linear Signed-off-by: Prekshi Vyas --- src/lib/shields/legacy-hermes-compat.test.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/lib/shields/legacy-hermes-compat.test.ts b/src/lib/shields/legacy-hermes-compat.test.ts index f303f4bc1d4..9500ad1d801 100644 --- a/src/lib/shields/legacy-hermes-compat.test.ts +++ b/src/lib/shields/legacy-hermes-compat.test.ts @@ -182,10 +182,14 @@ describe("legacy Hermes shields compatibility", () => { case isGuardAction(cmd, "begin-shields-transition"): { const modeIndex = cmd.indexOf("--shields-mode"); const mode = modeIndex >= 0 ? cmd[modeIndex + 1] : undefined; - if (mode !== "locked" && mode !== "mutable") { - throw new Error("Invalid --shields-mode in test fixture"); + switch (mode) { + case "locked": + case "mutable": + pendingMode = mode; + break; + default: + throw new Error("Invalid --shields-mode in test fixture"); } - pendingMode = mode; return `lock_token=${LOCK_TOKEN} original_locked=1`; } case isGuardAction(cmd, "apply-shields-transition"): From 3a8984481237b6b7279600c986920f543459be4f Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 7 Aug 2026 00:14:54 -0700 Subject: [PATCH 08/12] fix(shields): execute convergence through OpenShell Signed-off-by: Prekshi Vyas --- src/lib/adapters/openshell/command-argv.ts | 13 +++ src/lib/policy/commands.ts | 22 ++--- src/lib/shields/inference-convergence.test.ts | 91 +++++++++++++------ src/lib/shields/inference-convergence.ts | 8 +- src/lib/shields/legacy-hermes-compat.test.ts | 10 +- 5 files changed, 102 insertions(+), 42 deletions(-) create mode 100644 src/lib/adapters/openshell/command-argv.ts diff --git a/src/lib/adapters/openshell/command-argv.ts b/src/lib/adapters/openshell/command-argv.ts new file mode 100644 index 00000000000..60251b5321e --- /dev/null +++ b/src/lib/adapters/openshell/command-argv.ts @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Namespace access keeps the resolver replaceable in focused command tests. +import * as openshellResolveModule from "./resolve"; + +export function resolveOpenshellBinary(): string { + return openshellResolveModule.resolveOpenshell() ?? "openshell"; +} + +export function buildOpenshellCommand(args: readonly string[]): string[] { + return [resolveOpenshellBinary(), ...args]; +} diff --git a/src/lib/policy/commands.ts b/src/lib/policy/commands.ts index 6016139d642..fdf4017593f 100644 --- a/src/lib/policy/commands.ts +++ b/src/lib/policy/commands.ts @@ -1,25 +1,25 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -// Late binding keeps tests able to replace the resolver without rewiring -// command builders that are shared by policy and Shields flows. -const openshellResolveModule = - require("../adapters/openshell/resolve") as typeof import("../adapters/openshell/resolve"); - -function resolveOpenshellBinary(): string { - return openshellResolveModule.resolveOpenshell() ?? "openshell"; -} +import { buildOpenshellCommand } from "../adapters/openshell/command-argv"; export function buildPolicySetCommand(policyFile: string, sandboxName: string): string[] { - return [resolveOpenshellBinary(), "policy", "set", "--policy", policyFile, "--wait", sandboxName]; + return buildOpenshellCommand([ + "policy", + "set", + "--policy", + policyFile, + "--wait", + sandboxName, + ]); } /** Read the round-trippable base policy before a mutation. */ export function buildPolicyGetCommand(sandboxName: string): string[] { - return [resolveOpenshellBinary(), "policy", "get", "--base", sandboxName]; + return buildOpenshellCommand(["policy", "get", "--base", sandboxName]); } /** Read the effective policy for status and other diagnostics. */ export function buildPolicyGetFullCommand(sandboxName: string): string[] { - return [resolveOpenshellBinary(), "policy", "get", "--full", sandboxName]; + return buildOpenshellCommand(["policy", "get", "--full", sandboxName]); } diff --git a/src/lib/shields/inference-convergence.test.ts b/src/lib/shields/inference-convergence.test.ts index f05b4b8b56e..d2abd4f2baf 100644 --- a/src/lib/shields/inference-convergence.test.ts +++ b/src/lib/shields/inference-convergence.test.ts @@ -1,15 +1,38 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; -import { waitForHermesInferenceRouteConvergence } from "./inference-convergence"; +import { resolveOpenshell } from "../adapters/openshell/resolve"; +import { + type InferenceRouteConvergenceOptions, + waitForHermesInferenceRouteConvergence, +} from "./inference-convergence"; + +vi.mock("../adapters/openshell/resolve", () => ({ + resolveOpenshell: vi.fn(() => "/opt/nvidia/bin/openshell"), +})); function probe(status: number, output: string) { return { status, stdout: output, stderr: "" }; } +const OPENSHELL_BINARY = "/opt/nvidia/bin/openshell"; + +function buildOpenshellCommand(args: readonly string[]): string[] { + return [OPENSHELL_BINARY, ...args]; +} + +function convergenceOptions( + run: InferenceRouteConvergenceOptions["run"], + options: Omit = {}, +): InferenceRouteConvergenceOptions { + return { ...options, buildOpenshellCommand, run }; +} + describe("Hermes inference convergence after a Shields policy transition", () => { + afterEach(() => vi.clearAllMocks()); + it("returns on the first healthy route probe", () => { const run = vi.fn((_command: readonly string[], _options: object) => probe(0, "OK 200")); const sleep = vi.fn(); @@ -17,8 +40,10 @@ describe("Hermes inference convergence after a Shields policy transition", () => const result = waitForHermesInferenceRouteConvergence("hermes-box", { run, sleep }); expect(result).toEqual({ ok: true, attempts: 1, httpStatus: 200 }); + expect(resolveOpenshell).toHaveBeenCalledOnce(); expect(sleep).not.toHaveBeenCalled(); expect(run.mock.calls[0]?.[0]).toEqual([ + OPENSHELL_BINARY, "sandbox", "exec", "--name", @@ -37,11 +62,10 @@ describe("Hermes inference convergence after a Shields policy transition", () => .mockReturnValueOnce(probe(0, "OK 200")); const sleep = vi.fn(); - const result = waitForHermesInferenceRouteConvergence("hermes-box", { - retryDelayMs: 750, - run, - sleep, - }); + const result = waitForHermesInferenceRouteConvergence( + "hermes-box", + convergenceOptions(run, { retryDelayMs: 750, sleep }), + ); expect(result).toEqual({ ok: true, attempts: 2, httpStatus: 200 }); expect(sleep).toHaveBeenCalledOnce(); @@ -52,11 +76,10 @@ describe("Hermes inference convergence after a Shields policy transition", () => const run = vi.fn((_command: readonly string[], _options: object) => probe(0, "BROKEN 503")); const sleep = vi.fn(); - const result = waitForHermesInferenceRouteConvergence("hermes-box", { - maxAttempts: 3, - run, - sleep, - }); + const result = waitForHermesInferenceRouteConvergence( + "hermes-box", + convergenceOptions(run, { maxAttempts: 3, sleep }), + ); expect(result).toEqual({ ok: false, attempts: 3, httpStatus: 503 }); expect(run).toHaveBeenCalledTimes(3); @@ -70,14 +93,32 @@ describe("Hermes inference convergence after a Shields policy transition", () => stderr: "", })); - const result = waitForHermesInferenceRouteConvergence("hermes-box", { - maxAttempts: 1, - run, - }); + const result = waitForHermesInferenceRouteConvergence( + "hermes-box", + convergenceOptions(run, { maxAttempts: 1 }), + ); expect(result).toEqual({ ok: false, attempts: 1, httpStatus: 0 }); }); + it.each([ + 401, 403, 404, + ])("does not accept HTTP %i as a usable Hermes inference response", (httpStatus) => { + const run = vi.fn((_command: readonly string[], _options: object) => + probe(0, `OK ${String(httpStatus)}`), + ); + const sleep = vi.fn(); + + const result = waitForHermesInferenceRouteConvergence( + "hermes-box", + convergenceOptions(run, { maxAttempts: 2, sleep }), + ); + + expect(result).toEqual({ ok: false, attempts: 2, httpStatus }); + expect(run).toHaveBeenCalledTimes(2); + expect(sleep).toHaveBeenCalledOnce(); + }); + it.each([ Number.NaN, Number.POSITIVE_INFINITY, @@ -85,11 +126,10 @@ describe("Hermes inference convergence after a Shields policy transition", () => const run = vi.fn((_command: readonly string[], _options: object) => probe(0, "BROKEN 503")); const sleep = vi.fn(); - const result = waitForHermesInferenceRouteConvergence("hermes-box", { - maxAttempts, - run, - sleep, - }); + const result = waitForHermesInferenceRouteConvergence( + "hermes-box", + convergenceOptions(run, { maxAttempts, sleep }), + ); expect(result).toEqual({ ok: false, attempts: 4, httpStatus: 503 }); expect(run).toHaveBeenCalledTimes(4); @@ -105,11 +145,10 @@ describe("Hermes inference convergence after a Shields policy transition", () => .mockReturnValueOnce(probe(0, "BROKEN 503")); const sleep = vi.fn(); - const result = waitForHermesInferenceRouteConvergence("hermes-box", { - retryDelayMs, - run, - sleep, - }); + const result = waitForHermesInferenceRouteConvergence( + "hermes-box", + convergenceOptions(run, { retryDelayMs, sleep }), + ); expect(result).toEqual({ ok: true, attempts: 2, httpStatus: 200 }); expect(sleep).toHaveBeenCalledOnce(); diff --git a/src/lib/shields/inference-convergence.ts b/src/lib/shields/inference-convergence.ts index 914d9e6e64b..415484662fa 100644 --- a/src/lib/shields/inference-convergence.ts +++ b/src/lib/shields/inference-convergence.ts @@ -5,6 +5,7 @@ import { buildSandboxInferenceRouteProbeArgs, parseSandboxInferenceRouteProbeResult, } from "../actions/sandbox/connect-inference-route-probe"; +import { buildOpenshellCommand } from "../adapters/openshell/command-argv"; const DEFAULT_MAX_ATTEMPTS = 4; const DEFAULT_RETRY_DELAY_MS = 500; @@ -35,6 +36,7 @@ export interface InferenceRouteConvergenceResult { export interface InferenceRouteConvergenceOptions { maxAttempts?: number; retryDelayMs?: number; + buildOpenshellCommand?: (args: readonly string[]) => string[]; run: RunInferenceRoute; sleep?: (milliseconds: number) => void; } @@ -58,12 +60,13 @@ export function waitForHermesInferenceRouteConvergence( const retryDelayMs = Number.isFinite(configuredRetryDelayMs) ? Math.max(0, Math.trunc(configuredRetryDelayMs)) : DEFAULT_RETRY_DELAY_MS; + const buildCommand = options.buildOpenshellCommand ?? buildOpenshellCommand; const sleep = options.sleep ?? sleepMs; let httpStatus = 0; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { const probe = options.run( - buildSandboxInferenceRouteProbeArgs(sandboxName, { name: "hermes" }), + buildCommand(buildSandboxInferenceRouteProbeArgs(sandboxName, { name: "hermes" })), { ignoreError: true, suppressOutput: true, @@ -76,7 +79,8 @@ export function waitForHermesInferenceRouteConvergence( stderr: String(probe.stderr ?? ""), }); httpStatus = parsed.httpStatus; - if (parsed.healthy) return { ok: true, attempts: attempt, httpStatus }; + const usable = parsed.healthy && httpStatus >= 200 && httpStatus < 300; + if (usable) return { ok: true, attempts: attempt, httpStatus }; if (attempt < maxAttempts) sleep(retryDelayMs); } diff --git a/src/lib/shields/legacy-hermes-compat.test.ts b/src/lib/shields/legacy-hermes-compat.test.ts index 9500ad1d801..b1ce25a56a8 100644 --- a/src/lib/shields/legacy-hermes-compat.test.ts +++ b/src/lib/shields/legacy-hermes-compat.test.ts @@ -441,12 +441,14 @@ describe("legacy Hermes shields compatibility", () => { expect(commands.some((cmd) => isGuardAction(cmd, "begin-shields-transition"))).toBe(true); }); - it("gives route-specific recovery guidance when inference convergence fails", () => { + it.each([ + 401, 403, 404, + ])("rolls back Shields down when Hermes returns unusable HTTP %i", (httpStatus) => { installExecResponses(CURRENT_GUARD_HELP, "3770", undefined, true); inferenceConvergenceSpy.mockReturnValue({ ok: false, attempts: 4, - httpStatus: 503, + httpStatus, }); expect(() => @@ -454,7 +456,9 @@ describe("legacy Hermes shields compatibility", () => { skipTimer: true, throwOnError: true, }), - ).toThrow(/inference route did not converge.*HTTP 503.*4 attempts/i); + ).toThrow( + new RegExp(`inference route did not converge.*HTTP ${String(httpStatus)}.*4 attempts`, "i"), + ); const errors = errorSpy.mock.calls.map((call) => String(call[0])).join("\n"); expect(errors).toContain( From 3e15114e6352d5cf024dc02d14a1f80faf70eacd Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 7 Aug 2026 00:16:54 -0700 Subject: [PATCH 09/12] chore(policy): format shared OpenShell argv Signed-off-by: Prekshi Vyas --- src/lib/policy/commands.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/lib/policy/commands.ts b/src/lib/policy/commands.ts index fdf4017593f..bcad4ef7eee 100644 --- a/src/lib/policy/commands.ts +++ b/src/lib/policy/commands.ts @@ -4,14 +4,7 @@ import { buildOpenshellCommand } from "../adapters/openshell/command-argv"; export function buildPolicySetCommand(policyFile: string, sandboxName: string): string[] { - return buildOpenshellCommand([ - "policy", - "set", - "--policy", - policyFile, - "--wait", - sandboxName, - ]); + return buildOpenshellCommand(["policy", "set", "--policy", policyFile, "--wait", sandboxName]); } /** Read the round-trippable base policy before a mutation. */ From ec2b6f936d40fc38c08756db189a0bd47f4ecf85 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 7 Aug 2026 00:29:59 -0700 Subject: [PATCH 10/12] docs(shields): explain convergence boundary Signed-off-by: Prekshi Vyas --- src/lib/shields/inference-convergence.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/lib/shields/inference-convergence.ts b/src/lib/shields/inference-convergence.ts index 415484662fa..dabc017b48c 100644 --- a/src/lib/shields/inference-convergence.ts +++ b/src/lib/shields/inference-convergence.ts @@ -47,6 +47,10 @@ export interface InferenceRouteConvergenceOptions { * active, but the inference proxy can briefly continue returning HTTP 503 * after that acknowledgement. Shields down must not report completion during * that gap because callers immediately resume agent work. + * OpenShell owns both the activation acknowledgement and proxy convergence; + * NemoClaw can only verify the postcondition here. Remove this wait once every + * supported OpenShell release makes `policy set --wait` guarantee that the + * sandbox inference route is usable before it returns. */ export function waitForHermesInferenceRouteConvergence( sandboxName: string, From 229814c6ca62bb34e788bfaa859473504611550f Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 7 Aug 2026 01:42:21 -0700 Subject: [PATCH 11/12] test(policy): cover resolved OpenShell argv Signed-off-by: Prekshi Vyas --- test/policies.test.ts | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/test/policies.test.ts b/test/policies.test.ts index 06dd8ff41fd..cea7bd9b7d3 100644 --- a/test/policies.test.ts +++ b/test/policies.test.ts @@ -468,16 +468,14 @@ exit 1 expect(waitIdx < nameIdx).toBeTruthy(); }); - it("uses the resolved openshell binary when provided by the installer path", () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openshell-bin-")); - const override = path.join(tmpDir, "openshell"); - fs.writeFileSync(override, "#!/bin/sh\nexit 0\n", { mode: 0o755 }); - const prev = process.env.NEMOCLAW_OPENSHELL_BIN; - process.env.NEMOCLAW_OPENSHELL_BIN = override; + it("uses the resolved openshell binary for every policy command", () => { + const resolved = "/opt/nvidia/bin/openshell"; + const resolveSpy = vi + .spyOn(resolveOpenshellModule, "resolveOpenshell") + .mockReturnValue(resolved); try { - const cmd = policies.buildPolicySetCommand("/tmp/policy.yaml", "my-assistant"); - expect(cmd).toEqual([ - override, + expect(policies.buildPolicySetCommand("/tmp/policy.yaml", "my-assistant")).toEqual([ + resolved, "policy", "set", "--policy", @@ -485,10 +483,22 @@ exit 1 "--wait", "my-assistant", ]); + expect(policies.buildPolicyGetCommand("my-assistant")).toEqual([ + resolved, + "policy", + "get", + "--base", + "my-assistant", + ]); + expect(policies.buildPolicyGetFullCommand("my-assistant")).toEqual([ + resolved, + "policy", + "get", + "--full", + "my-assistant", + ]); } finally { - if (prev === undefined) delete process.env.NEMOCLAW_OPENSHELL_BIN; - else process.env.NEMOCLAW_OPENSHELL_BIN = prev; - fs.rmSync(tmpDir, { recursive: true, force: true }); + resolveSpy.mockRestore(); } }); }); From 3a31307ad053bb932f4eb28dfacf310f3947c56f Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 7 Aug 2026 02:07:32 -0700 Subject: [PATCH 12/12] test(e2e): assert Hermes route after Shields down Signed-off-by: Prekshi Vyas --- test/e2e/live/hermes-shields-config.test.ts | 28 +++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test/e2e/live/hermes-shields-config.test.ts b/test/e2e/live/hermes-shields-config.test.ts index 344acd39767..4c09ae0016d 100644 --- a/test/e2e/live/hermes-shields-config.test.ts +++ b/test/e2e/live/hermes-shields-config.test.ts @@ -165,6 +165,30 @@ async function expectLockedPosture(sandbox: SandboxClient, cycle: number): Promi expect(result.stdout).toContain(`444 root:root ${HERMES_DIR}/.config-hash`); } +async function expectImmediateInferenceRoute(sandbox: SandboxClient, cycle: number): Promise { + const result = await sandboxShell( + sandbox, + [ + "set -eu", + 'response="$(mktemp)"', + "trap 'rm -f \"$response\"' EXIT", + 'curl -fsS --connect-timeout 3 --max-time 10 https://inference.local/v1/models -o "$response"', + "python3 - \"$response\" <<'PY'", + "import json, pathlib, sys", + "payload = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding='utf-8'))", + `assert ${JSON.stringify(COMPATIBLE_MODEL)} in {entry.get("id") for entry in payload["data"]}`, + "print('HERMES_SHIELDS_INFERENCE_ROUTE_READY')", + "PY", + ].join("\n"), + `cycle-${cycle}-immediate-inference-route`, + ); + assertExitZero( + result, + `probe Hermes inference route immediately after Shields down cycle ${cycle}`, + ); + expect(result.stdout).toContain("HERMES_SHIELDS_INFERENCE_ROUTE_READY"); +} + async function completeShieldsCycle( host: HostCliClient, sandbox: SandboxClient, @@ -176,6 +200,7 @@ async function completeShieldsCycle( `cycle-${cycle}-shields-down`, ); assertExitZero(down, `unlock fresh Hermes config in cycle ${cycle}`); + await expectImmediateInferenceRoute(sandbox, cycle); await expectShieldsStatus(host, "DOWN", `cycle-${cycle}-status-down`); await expectMutablePosture(sandbox, cycle); @@ -211,6 +236,7 @@ test("hermes-shields-config: stopped Hermes restores under both Shields postures "shields-up establishes the root-owned locked posture", "start restores a stopped Hermes sandbox while shields are up", "start restores a stopped Hermes sandbox while shields are down", + "each successful shields-down returns only after inference.local serves the configured model", "a second down/up cycle completes without corrupting config state", ], issue: "#6381", @@ -342,6 +368,7 @@ test("hermes-shields-config: stopped Hermes restores under both Shields postures "cycle-2-shields-down", ); assertExitZero(down, "unlock fresh Hermes config in cycle 2"); + await expectImmediateInferenceRoute(sandbox, 2); await expectShieldsStatus(host, "DOWN", "cycle-2-status-down"); await expectMutablePosture(sandbox, 2); await expectStopStartRecovery(host, "DOWN", "cycle-2-shields-down-start-recovery"); @@ -379,6 +406,7 @@ test("hermes-shields-config: stopped Hermes restores under both Shields postures freshNonrootTrigger: true, stateLockPlanModes: true, firstCycle: true, + postTransitionInferenceRoute: true, shieldsDownStartRecovery: true, shieldsUpStartRecovery: true, secondCycle: true,