fix(shields): wait for Hermes inference convergence

src/lib/adapters/openshell/command-argv.ts

@@ -0,0 +1,13 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+// Namespace access keeps the resolver replaceable in focused command tests.
+import * as openshellResolveModule from "./resolve";
+
+export function resolveOpenshellBinary(): string {
+  return openshellResolveModule.resolveOpenshell() ?? "openshell";
+}
+
+export function buildOpenshellCommand(args: readonly string[]): string[] {
+  return [resolveOpenshellBinary(), ...args];
+}

src/lib/policy/commands.ts

@@ -1,25 +1,18 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-// Late binding keeps tests able to replace the resolver without rewiring
-// command builders that are shared by policy and Shields flows.
-const openshellResolveModule =
-  require("../adapters/openshell/resolve") as typeof import("../adapters/openshell/resolve");
-
-function resolveOpenshellBinary(): string {
-  return openshellResolveModule.resolveOpenshell() ?? "openshell";
-}
+import { buildOpenshellCommand } from "../adapters/openshell/command-argv";
 
 export function buildPolicySetCommand(policyFile: string, sandboxName: string): string[] {
-  return [resolveOpenshellBinary(), "policy", "set", "--policy", policyFile, "--wait", sandboxName];
+  return buildOpenshellCommand(["policy", "set", "--policy", policyFile, "--wait", sandboxName]);
 }
 
 /** Read the round-trippable base policy before a mutation. */
 export function buildPolicyGetCommand(sandboxName: string): string[] {
-  return [resolveOpenshellBinary(), "policy", "get", "--base", sandboxName];
+  return buildOpenshellCommand(["policy", "get", "--base", sandboxName]);
 }
 
 /** Read the effective policy for status and other diagnostics. */
 export function buildPolicyGetFullCommand(sandboxName: string): string[] {
-  return [resolveOpenshellBinary(), "policy", "get", "--full", sandboxName];
+  return buildOpenshellCommand(["policy", "get", "--full", sandboxName]);
 }

src/lib/shields/index.ts

@@ -70,7 +70,10 @@ const {
 }: typeof import("./permissive-runtime") = require("./permissive-runtime");
 const { cleanupTempDir } = require("../onboard/temp-files");
 const { verifyShieldsLockState }: typeof import("./verify-lock") = require("./verify-lock");
-const { relockAndReconfirm }: typeof import("./relock-reconfirm") = require("./relock-reconfirm");
+const {
+  relockAndReconfirm,
+  waitForHermesInferenceRouteConvergence,
+}: typeof import("./relock-reconfirm") = require("./relock-reconfirm");
 const {
   inspectAnyShieldsTransitionLockOwner,
   isShieldsTransitionLockUnavailable,
@@ -3975,6 +3978,7 @@ function shieldsDownWithoutHostLock(sandboxName: string, opts: ShieldsDownOpts =
   //     OpenClaw uses sandbox:sandbox 0660/2770 here so the gateway UID, which
   //     is a member of the sandbox group, can mutate runtime config.
   console.log(`  Unlocking ${target.agentName} config (${target.configPath})...`);
+  let inferenceRouteConvergenceFailed = false;
   try {
     unlockAgentConfig(
       sandboxName,
@@ -3983,6 +3987,18 @@ function shieldsDownWithoutHostLock(sandboxName: string, opts: ShieldsDownOpts =
       opts.allowLegacyHermesProtocol === true,
       protocol,
     );
+    if (target.agentName === "hermes") {
+      console.log("  Confirming Hermes inference route after policy transition...");
+      const convergence = waitForHermesInferenceRouteConvergence(sandboxName, { run });
+      if (!convergence.ok) {
+        inferenceRouteConvergenceFailed = true;
+        const status =
+          convergence.httpStatus > 0 ? `HTTP ${convergence.httpStatus}` : "unavailable";
+        throw new Error(
+          `Hermes inference route did not converge after policy transition (${status}; ${convergence.attempts} attempts)`,
+        );
+      }
+    }
   } catch (err) {
     const message = err instanceof Error ? err.message : String(err);
     const rollback = rollbackShieldsDown(
@@ -4013,9 +4029,15 @@ function shieldsDownWithoutHostLock(sandboxName: string, opts: ShieldsDownOpts =
         `  Config rollback is incomplete.${timerAuthority} Manual intervention is required.`,
       );
     }
-    console.error(
-      `  Re-run \`nemoclaw ${sandboxName} shields down\` after correcting file ownership.`,
-    );
+    if (inferenceRouteConvergenceFailed) {
+      console.error(
+        `  Recover the Hermes inference route, then re-run \`nemoclaw ${sandboxName} shields down\`.`,
+      );
+    } else {
+      console.error(
+        `  Re-run \`nemoclaw ${sandboxName} shields down\` after correcting file ownership.`,
+      );
+    }
     return failShieldsCommand(message, opts.throwOnError);
   }
 

src/lib/shields/inference-convergence.test.ts

@@ -0,0 +1,157 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import { resolveOpenshell } from "../adapters/openshell/resolve";
+import {
+  type InferenceRouteConvergenceOptions,
+  waitForHermesInferenceRouteConvergence,
+} from "./inference-convergence";
+
+vi.mock("../adapters/openshell/resolve", () => ({
+  resolveOpenshell: vi.fn(() => "/opt/nvidia/bin/openshell"),
+}));
+
+function probe(status: number, output: string) {
+  return { status, stdout: output, stderr: "" };
+}
+
+const OPENSHELL_BINARY = "/opt/nvidia/bin/openshell";
+
+function buildOpenshellCommand(args: readonly string[]): string[] {
+  return [OPENSHELL_BINARY, ...args];
+}
+
+function convergenceOptions(
+  run: InferenceRouteConvergenceOptions["run"],
+  options: Omit<InferenceRouteConvergenceOptions, "buildOpenshellCommand" | "run"> = {},
+): InferenceRouteConvergenceOptions {
+  return { ...options, buildOpenshellCommand, run };
+}
+
+describe("Hermes inference convergence after a Shields policy transition", () => {
+  afterEach(() => vi.clearAllMocks());
+
+  it("returns on the first healthy route probe", () => {
+    const run = vi.fn((_command: readonly string[], _options: object) => probe(0, "OK 200"));
+    const sleep = vi.fn();
+
+    const result = waitForHermesInferenceRouteConvergence("hermes-box", { run, sleep });
+
+    expect(result).toEqual({ ok: true, attempts: 1, httpStatus: 200 });
+    expect(resolveOpenshell).toHaveBeenCalledOnce();
+    expect(sleep).not.toHaveBeenCalled();
+    expect(run.mock.calls[0]?.[0]).toEqual([
+      OPENSHELL_BINARY,
+      "sandbox",
+      "exec",
+      "--name",
+      "hermes-box",
+      "--",
+      "sh",
+      "-c",
+      expect.stringContaining("https://inference.local/v1/models"),
+    ]);
+  });
+
+  it("waits for a transient HTTP 503 to converge", () => {
+    const run = vi
+      .fn((_command: readonly string[], _options: object) => probe(0, "OK 200"))
+      .mockReturnValueOnce(probe(0, "BROKEN 503"))
+      .mockReturnValueOnce(probe(0, "OK 200"));
+    const sleep = vi.fn();
+
+    const result = waitForHermesInferenceRouteConvergence(
+      "hermes-box",
+      convergenceOptions(run, { retryDelayMs: 750, sleep }),
+    );
+
+    expect(result).toEqual({ ok: true, attempts: 2, httpStatus: 200 });
+    expect(sleep).toHaveBeenCalledOnce();
+    expect(sleep).toHaveBeenCalledWith(750);
+  });
+
+  it("fails after the bounded probe budget instead of reporting Shields down ready", () => {
+    const run = vi.fn((_command: readonly string[], _options: object) => probe(0, "BROKEN 503"));
+    const sleep = vi.fn();
+
+    const result = waitForHermesInferenceRouteConvergence(
+      "hermes-box",
+      convergenceOptions(run, { maxAttempts: 3, sleep }),
+    );
+
+    expect(result).toEqual({ ok: false, attempts: 3, httpStatus: 503 });
+    expect(run).toHaveBeenCalledTimes(3);
+    expect(sleep).toHaveBeenCalledTimes(2);
+  });
+
+  it("does not accept preambled probe output as convergence even when the command succeeds", () => {
+    const run = vi.fn((_command: readonly string[], _options: object) => ({
+      status: 0,
+      stdout: "attacker preamble\nOK 200",
+      stderr: "",
+    }));
+
+    const result = waitForHermesInferenceRouteConvergence(
+      "hermes-box",
+      convergenceOptions(run, { maxAttempts: 1 }),
+    );
+
+    expect(result).toEqual({ ok: false, attempts: 1, httpStatus: 0 });
+  });
+
+  it.each([
+    401, 403, 404,
+  ])("does not accept HTTP %i as a usable Hermes inference response", (httpStatus) => {
+    const run = vi.fn((_command: readonly string[], _options: object) =>
+      probe(0, `OK ${String(httpStatus)}`),
+    );
+    const sleep = vi.fn();
+
+    const result = waitForHermesInferenceRouteConvergence(
+      "hermes-box",
+      convergenceOptions(run, { maxAttempts: 2, sleep }),
+    );
+
+    expect(result).toEqual({ ok: false, attempts: 2, httpStatus });
+    expect(run).toHaveBeenCalledTimes(2);
+    expect(sleep).toHaveBeenCalledOnce();
+  });
+
+  it.each([
+    Number.NaN,
+    Number.POSITIVE_INFINITY,
+  ])("uses the bounded default attempt budget for non-finite maxAttempts (%s)", (maxAttempts) => {
+    const run = vi.fn((_command: readonly string[], _options: object) => probe(0, "BROKEN 503"));
+    const sleep = vi.fn();
+
+    const result = waitForHermesInferenceRouteConvergence(
+      "hermes-box",
+      convergenceOptions(run, { maxAttempts, sleep }),
+    );
+
+    expect(result).toEqual({ ok: false, attempts: 4, httpStatus: 503 });
+    expect(run).toHaveBeenCalledTimes(4);
+    expect(sleep).toHaveBeenCalledTimes(3);
+  });
+
+  it.each([
+    Number.NaN,
+    Number.POSITIVE_INFINITY,
+  ])("uses the default delay for non-finite retryDelayMs (%s)", (retryDelayMs) => {
+    const run = vi
+      .fn((_command: readonly string[], _options: object) => probe(0, "OK 200"))
+      .mockReturnValueOnce(probe(0, "BROKEN 503"));
+    const sleep = vi.fn();
+
+    const result = waitForHermesInferenceRouteConvergence(
+      "hermes-box",
+      convergenceOptions(run, { retryDelayMs, sleep }),
+    );
+
+    expect(result).toEqual({ ok: true, attempts: 2, httpStatus: 200 });
+    expect(sleep).toHaveBeenCalledOnce();
+    expect(sleep).toHaveBeenCalledWith(500);
+  });
+});

src/lib/shields/inference-convergence.ts

@@ -0,0 +1,92 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import {
+  buildSandboxInferenceRouteProbeArgs,
+  parseSandboxInferenceRouteProbeResult,
+} from "../actions/sandbox/connect-inference-route-probe";
+import { buildOpenshellCommand } from "../adapters/openshell/command-argv";
+
+const DEFAULT_MAX_ATTEMPTS = 4;
+const DEFAULT_RETRY_DELAY_MS = 500;
+const INFERENCE_ROUTE_PROBE_TIMEOUT_MS = 10_000;
+
+interface InferenceRouteCommandResult {
+  status: number | null;
+  stdout?: string | Buffer | null;
+  stderr?: string | Buffer | null;
+}
+
+type RunInferenceRoute = (
+  command: readonly string[],
+  options: { ignoreError: true; suppressOutput: true; timeout: number },
+) => InferenceRouteCommandResult;
+
+function sleepMs(milliseconds: number): void {
+  if (milliseconds <= 0 || !Number.isFinite(milliseconds)) return;
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds);
+}
+
+export interface InferenceRouteConvergenceResult {
+  ok: boolean;
+  attempts: number;
+  httpStatus: number;
+}
+
+export interface InferenceRouteConvergenceOptions {
+  maxAttempts?: number;
+  retryDelayMs?: number;
+  buildOpenshellCommand?: (args: readonly string[]) => string[];
+  run: RunInferenceRoute;
+  sleep?: (milliseconds: number) => void;
+}
+
+/**
+ * Require the Hermes inference route to be usable after a live policy
+ * replacement. OpenShell's `policy set --wait` confirms the policy version is
+ * active, but the inference proxy can briefly continue returning HTTP 503
+ * after that acknowledgement. Shields down must not report completion during
+ * that gap because callers immediately resume agent work.
+ * OpenShell owns both the activation acknowledgement and proxy convergence;
+ * NemoClaw can only verify the postcondition here. Remove this wait once every
+ * supported OpenShell release makes `policy set --wait` guarantee that the
+ * sandbox inference route is usable before it returns.
+ */
+export function waitForHermesInferenceRouteConvergence(
+  sandboxName: string,
+  options: InferenceRouteConvergenceOptions,
+): InferenceRouteConvergenceResult {
+  const configuredMaxAttempts = options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
+  const configuredRetryDelayMs = options.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS;
+  const maxAttempts = Number.isFinite(configuredMaxAttempts)
+    ? Math.max(1, Math.trunc(configuredMaxAttempts))
+    : DEFAULT_MAX_ATTEMPTS;
+  const retryDelayMs = Number.isFinite(configuredRetryDelayMs)
+    ? Math.max(0, Math.trunc(configuredRetryDelayMs))
+    : DEFAULT_RETRY_DELAY_MS;
+  const buildCommand = options.buildOpenshellCommand ?? buildOpenshellCommand;
+  const sleep = options.sleep ?? sleepMs;
+  let httpStatus = 0;
+
+  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+    const probe = options.run(
+      buildCommand(buildSandboxInferenceRouteProbeArgs(sandboxName, { name: "hermes" })),
+      {
+        ignoreError: true,
+        suppressOutput: true,
+        timeout: INFERENCE_ROUTE_PROBE_TIMEOUT_MS,
+      },
+    );
+    const parsed = parseSandboxInferenceRouteProbeResult({
+      status: probe.status,
+      output: String(probe.stdout ?? ""),
+      stderr: String(probe.stderr ?? ""),
+    });
+    httpStatus = parsed.httpStatus;
+    const usable = parsed.healthy && httpStatus >= 200 && httpStatus < 300;
+    if (usable) return { ok: true, attempts: attempt, httpStatus };
+    if (attempt < maxAttempts) sleep(retryDelayMs);
+  }
+
+  return { ok: false, attempts: maxAttempts, httpStatus };
+}