From a9978f027f6d8bac73f110883271b346910e0ddd Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Fri, 6 Nov 2026 15:06:13 -0500 Subject: [PATCH 1/4] Record why bounded recall fell back to free hot memory The fallback action now carries a classified failure (timeout, aborted, HTTP status and error code, or an error name) so local logs distinguish a client timeout from a server error or a recall deadline. Messages, URLs and bodies are never copied. --- src/runtime/hosted-client.ts | 12 ++++++++++++ src/runtime/lifecycle-engine.ts | 4 ++-- tests/runtime/hosted-client-security.test.ts | 13 ++++++++++++- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/runtime/hosted-client.ts b/src/runtime/hosted-client.ts index eab9638..32ca18c 100644 --- a/src/runtime/hosted-client.ts +++ b/src/runtime/hosted-client.ts @@ -246,6 +246,18 @@ export class HostedOnlyBotsRequestError extends Error { } } +/** + * Classify a failed hosted request for local diagnostics without copying its + * message, URL, headers or body: a timeout, an HTTP status/code, or an error name. + */ +export const describeHostedFailure = (error: unknown): Record => { + if (error instanceof HostedOnlyBotsRequestError) return { errorClass: "http", status: error.status, code: error.code }; + const name = error instanceof Error ? error.name : typeof error; + if (name === "TimeoutError") return { errorClass: "timeout" }; + if (name === "AbortError") return { errorClass: "aborted" }; + return { errorClass: /^[A-Za-z][A-Za-z0-9_]{0,63}$/.test(name) ? name : "unknown" }; +}; + const extractHostedError = (data: unknown): { message: string | null; code: string | null; details: unknown } => { if (!data || typeof data !== "object" || Array.isArray(data)) { return { message: null, code: null, details: undefined }; diff --git a/src/runtime/lifecycle-engine.ts b/src/runtime/lifecycle-engine.ts index 84001c9..8c282cf 100644 --- a/src/runtime/lifecycle-engine.ts +++ b/src/runtime/lifecycle-engine.ts @@ -19,7 +19,7 @@ import { pickHotContextValidationCommand, } from "./hot-context-cues"; import { buildRuntimeBillingRemediation, formatRuntimeBillingRemediation, isBillingBlockedError } from "./billing-remediation"; -import { HostedOnlyBotsClient, type HostedSession } from "./hosted-client"; +import { describeHostedFailure, HostedOnlyBotsClient, type HostedSession } from "./hosted-client"; import { isLocalOnlyRuntimeProfile, readLocalHotMemory, writeLocalHotMemory } from "./local-hot-memory"; import { workspaceHotScope, selectWorkspaceHotState } from "./workspace-hot-memory"; import { buildRuntimeContinuityOutcome, parseRuntimeContinuityOutcomeContent } from "./continuity-outcomes"; @@ -1785,7 +1785,7 @@ export class RuntimeLifecycleManager { warnings.push("Saved-context retrieval unavailable; continuing with free hot memory. A timed-out paid request may still settle; inspect billing before retrying."); actions.push(this.recordAction({ eventId: event.eventId, profileName, type: "context_pack", status: "failed", mode: profile.memoryMode, reason: "Bounded recall unavailable; preserved free hot-memory delivery", queueItemId: null, - details: { type: event.type, freeFallback: true } })); + details: { type: event.type, freeFallback: true, failure: describeHostedFailure(error) } })); } else { if (!isBillingBlockedError(error)) throw error; contextPack = buildLocalOnlyContextPack(event, packRequest); diff --git a/tests/runtime/hosted-client-security.test.ts b/tests/runtime/hosted-client-security.test.ts index 732b604..9fad693 100644 --- a/tests/runtime/hosted-client-security.test.ts +++ b/tests/runtime/hosted-client-security.test.ts @@ -4,7 +4,7 @@ import { schnorr } from "@noble/curves/secp256k1"; import { generateSecretKey, getPublicKey, nip19 } from "nostr-tools"; import { afterEach, describe, expect, it } from "vitest"; -import { HostedOnlyBotsClient } from "../../src/runtime/hosted-client"; +import { describeHostedFailure, HostedOnlyBotsClient, HostedOnlyBotsRequestError } from "../../src/runtime/hosted-client"; const servers: Server[] = []; afterEach(async () => { @@ -183,3 +183,14 @@ describe("hosted client security boundaries", () => { expect(receivedBody).toEqual({ code: "obj_test", agent_id: "agent-test" }); }); }); + +describe("describeHostedFailure", () => { + it("distinguishes a recall timeout from an HTTP failure without copying messages", () => { + const timeout = new DOMException("The operation was aborted due to timeout https://secret.example/?token=x", "TimeoutError"); + expect(describeHostedFailure(timeout)).toEqual({ errorClass: "timeout" }); + const http = new HostedOnlyBotsRequestError({ status: 503, code: "recall_deadline_exceeded", message: "secret detail" }); + expect(describeHostedFailure(http)).toEqual({ errorClass: "http", status: 503, code: "recall_deadline_exceeded" }); + expect(describeHostedFailure(new TypeError("fetch failed: secret.example"))).toEqual({ errorClass: "TypeError" }); + expect(JSON.stringify(describeHostedFailure(Object.assign(new Error("x"), { name: "bad name with token=1" })))).not.toContain("token"); + }); +}); From 782f02bd65dc0b503c203e442053201fe9326967 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Fri, 6 Nov 2026 16:32:27 -0500 Subject: [PATCH 2/4] Validate hosted failure codes and say when a bounded fallback was not charged The recorded failure code passes the same identifier check as the error name. The fallback warning now follows the failure class: a server deadline that reports nothing charged says so, a 4xx names the rejection, and only timeouts, network failures and unconfirmed server errors keep the may-still-settle caution. A lifecycle test asserts that the fallback action records details.failure. --- src/runtime/hosted-client.ts | 24 ++++++++- src/runtime/lifecycle-engine.ts | 4 +- tests/runtime/hosted-client-security.test.ts | 24 ++++++++- .../runtime/hosted-lifecycle-security.test.ts | 53 ++++++++++++++++++- 4 files changed, 99 insertions(+), 6 deletions(-) diff --git a/src/runtime/hosted-client.ts b/src/runtime/hosted-client.ts index 32ca18c..10c4bb9 100644 --- a/src/runtime/hosted-client.ts +++ b/src/runtime/hosted-client.ts @@ -250,12 +250,32 @@ export class HostedOnlyBotsRequestError extends Error { * Classify a failed hosted request for local diagnostics without copying its * message, URL, headers or body: a timeout, an HTTP status/code, or an error name. */ +const SAFE_FAILURE_LABEL = /^[A-Za-z][A-Za-z0-9_]{0,63}$/; +const safeFailureLabel = (value: string | null): string | null => (value !== null && SAFE_FAILURE_LABEL.test(value) ? value : null); + export const describeHostedFailure = (error: unknown): Record => { - if (error instanceof HostedOnlyBotsRequestError) return { errorClass: "http", status: error.status, code: error.code }; + // The server chooses `code`; validate it like an error name so it cannot carry message text. + if (error instanceof HostedOnlyBotsRequestError) return { errorClass: "http", status: error.status, code: safeFailureLabel(error.code) }; const name = error instanceof Error ? error.name : typeof error; if (name === "TimeoutError") return { errorClass: "timeout" }; if (name === "AbortError") return { errorClass: "aborted" }; - return { errorClass: /^[A-Za-z][A-Za-z0-9_]{0,63}$/.test(name) ? name : "unknown" }; + return { errorClass: safeFailureLabel(name) ?? "unknown" }; +}; + +/** + * Warning for a bounded recall that fell back to free hot memory. Only a timeout, abort, + * network failure or unconfirmed server error leaves open whether the paid request settled. + */ +export const describeBoundedRecallFallback = (error: unknown): string => { + const prefix = "Saved-context retrieval unavailable; continuing with free hot memory."; + if (error instanceof HostedOnlyBotsRequestError) { + const details = error.details && typeof error.details === "object" ? (error.details as Record) : null; + if (error.code === "recall_deadline_exceeded" || details?.charged_sats === 0) { + return `${prefix} The server missed its response budget (HTTP ${error.status}) and charged nothing; a retry is safe.`; + } + if (error.status < 500) return `${prefix} The server rejected the request (HTTP ${error.status}).`; + } + return `${prefix} A timed-out or failed paid request may still settle; inspect billing before retrying.`; }; const extractHostedError = (data: unknown): { message: string | null; code: string | null; details: unknown } => { diff --git a/src/runtime/lifecycle-engine.ts b/src/runtime/lifecycle-engine.ts index 8c282cf..da1f2af 100644 --- a/src/runtime/lifecycle-engine.ts +++ b/src/runtime/lifecycle-engine.ts @@ -19,7 +19,7 @@ import { pickHotContextValidationCommand, } from "./hot-context-cues"; import { buildRuntimeBillingRemediation, formatRuntimeBillingRemediation, isBillingBlockedError } from "./billing-remediation"; -import { describeHostedFailure, HostedOnlyBotsClient, type HostedSession } from "./hosted-client"; +import { describeBoundedRecallFallback, describeHostedFailure, HostedOnlyBotsClient, type HostedSession } from "./hosted-client"; import { isLocalOnlyRuntimeProfile, readLocalHotMemory, writeLocalHotMemory } from "./local-hot-memory"; import { workspaceHotScope, selectWorkspaceHotState } from "./workspace-hot-memory"; import { buildRuntimeContinuityOutcome, parseRuntimeContinuityOutcomeContent } from "./continuity-outcomes"; @@ -1782,7 +1782,7 @@ export class RuntimeLifecycleManager { } catch (error) { if (!isBillingBlockedError(error) && packRequest.recallPolicy === "bounded") { contextPack = buildLocalOnlyContextPack(event, packRequest); - warnings.push("Saved-context retrieval unavailable; continuing with free hot memory. A timed-out paid request may still settle; inspect billing before retrying."); + warnings.push(describeBoundedRecallFallback(error)); actions.push(this.recordAction({ eventId: event.eventId, profileName, type: "context_pack", status: "failed", mode: profile.memoryMode, reason: "Bounded recall unavailable; preserved free hot-memory delivery", queueItemId: null, details: { type: event.type, freeFallback: true, failure: describeHostedFailure(error) } })); diff --git a/tests/runtime/hosted-client-security.test.ts b/tests/runtime/hosted-client-security.test.ts index 9fad693..c86d563 100644 --- a/tests/runtime/hosted-client-security.test.ts +++ b/tests/runtime/hosted-client-security.test.ts @@ -4,7 +4,7 @@ import { schnorr } from "@noble/curves/secp256k1"; import { generateSecretKey, getPublicKey, nip19 } from "nostr-tools"; import { afterEach, describe, expect, it } from "vitest"; -import { describeHostedFailure, HostedOnlyBotsClient, HostedOnlyBotsRequestError } from "../../src/runtime/hosted-client"; +import { describeBoundedRecallFallback, describeHostedFailure, HostedOnlyBotsClient, HostedOnlyBotsRequestError } from "../../src/runtime/hosted-client"; const servers: Server[] = []; afterEach(async () => { @@ -192,5 +192,27 @@ describe("describeHostedFailure", () => { expect(describeHostedFailure(http)).toEqual({ errorClass: "http", status: 503, code: "recall_deadline_exceeded" }); expect(describeHostedFailure(new TypeError("fetch failed: secret.example"))).toEqual({ errorClass: "TypeError" }); expect(JSON.stringify(describeHostedFailure(Object.assign(new Error("x"), { name: "bad name with token=1" })))).not.toContain("token"); + // A server-chosen code that is not a plain identifier is dropped, not copied. + const unsafeCode = new HostedOnlyBotsRequestError({ status: 500, code: "oops token=secret https://x", message: "m" }); + expect(describeHostedFailure(unsafeCode)).toEqual({ errorClass: "http", status: 500, code: null }); + expect(describeHostedFailure(new HostedOnlyBotsRequestError({ status: 502, code: "a".repeat(65), message: "m" })).code).toBeNull(); + }); +}); + +describe("describeBoundedRecallFallback", () => { + it("says nothing was charged only when the server confirms it, and keeps the settle caution otherwise", () => { + const deadline = new HostedOnlyBotsRequestError({ status: 503, code: "recall_deadline_exceeded", message: "late", + details: { charged_sats: 0, retryable: true } }); + expect(describeBoundedRecallFallback(deadline)).toContain("charged nothing"); + expect(describeBoundedRecallFallback(deadline)).not.toContain("may still settle"); + const timeout = new DOMException("timed out", "TimeoutError"); + expect(describeBoundedRecallFallback(timeout)).toContain("may still settle"); + expect(describeBoundedRecallFallback(new TypeError("fetch failed"))).toContain("may still settle"); + const gateway = new HostedOnlyBotsRequestError({ status: 504, code: null, message: "gateway timeout" }); + expect(describeBoundedRecallFallback(gateway)).toContain("may still settle"); + const rejected = describeBoundedRecallFallback(new HostedOnlyBotsRequestError({ status: 400, code: "bad_request", message: "secret detail" })); + expect(rejected).toContain("HTTP 400"); + expect(rejected).not.toContain("may still settle"); + expect(rejected).not.toContain("secret detail"); }); }); diff --git a/tests/runtime/hosted-lifecycle-security.test.ts b/tests/runtime/hosted-lifecycle-security.test.ts index 27b281e..119564d 100644 --- a/tests/runtime/hosted-lifecycle-security.test.ts +++ b/tests/runtime/hosted-lifecycle-security.test.ts @@ -26,7 +26,7 @@ const jsonResponse = (body: unknown, status = 200): Response => headers: { "content-type": "application/json" }, }); -const installFetchMock = (npub: string, contextStatus: number) => { +const installFetchMock = (npub: string, contextStatus: number, contextPack?: () => Promise) => { vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => { const url = new URL(typeof input === "string" ? input : input instanceof URL ? input.href : input.url); if (url.pathname === "/v1/auth/challenge") { @@ -51,6 +51,7 @@ const installFetchMock = (npub: string, contextStatus: number) => { }); } if (url.pathname === "/v1/context/pack") { + if (contextPack) return contextPack(); if (contextStatus !== 200) { return jsonResponse({ error: { code: "payment_required", message: "funding required" } }, contextStatus); } @@ -170,4 +171,54 @@ describe("hosted lifecycle prompt boundary", () => { expect(promptSurface).not.toContain("SERVER_AUTHORED_FRAGMENT_MUST_NOT_REACH_THE_PROMPT"); if (contextStatus === 200) expect(result.contextPack?.packId).toBe("hosted-pack-safe-id"); }); + + it.each([ + { + label: "the server's recall deadline", + contextPack: async () => jsonResponse({ error: { code: "recall_deadline_exceeded", message: "late", + details: { deadline_ms: 2500, charged_sats: 0, retryable: true } } }, 503), + failure: { errorClass: "http", status: 503, code: "recall_deadline_exceeded" }, + warning: "charged nothing", + }, + { + label: "the client's recall timeout", + contextPack: async () => { throw new DOMException("The operation was aborted due to timeout", "TimeoutError"); }, + failure: { errorClass: "timeout" }, + warning: "may still settle", + }, + ])("records why bounded recall fell back to free hot memory after $label", async ({ contextPack, failure, warning }) => { + const signer = identity(); + const home = mkdtempSync(path.join(os.tmpdir(), "crosshead-bounded-fallback-")); + homes.push(home); + installFetchMock(signer.npub, 200, contextPack); + await setupRuntimeProfile({ + home, + profileName: "default", + apiBase: "https://crosshead.sh", + npub: signer.npub, + nsec: signer.nsec, + memoryMode: "assist", + }); + + const daemon = new OnlyBotsLocalRuntimeDaemon(home); + const result = await daemon.submitLifecycleEvent({ + runtime: "claude-code", + type: "turn.started", + workspaceId: "repo:security-test", + sessionId: "current-session", + turnId: "current-turn", + payload: { + goal: "Continue the verified local work", + recallPolicy: "bounded", + hotMemorySnapshot: verifiedHotSnapshot, + }, + }); + + const fallback = result.actions.find((action) => action.type === "context_pack" && action.status === "failed"); + expect(fallback?.details).toMatchObject({ freeFallback: true, failure }); + expect(result.warnings.filter((entry) => entry.startsWith("Saved-context retrieval unavailable"))).toEqual([ + expect.stringContaining(warning), + ]); + expect(JSON.stringify(result.contextPack)).toContain("CLIENT_VERIFIED_HOT_MEMORY_MUST_REACH_THE_PROMPT"); + }); }); From fa19c1807c034f350f84e6ba4a8ad092816d7754 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Fri, 6 Nov 2026 22:13:30 -0500 Subject: [PATCH 3/4] Distinguish unknown paid commits from confirmed free recall failures --- src/runtime/hosted-client.ts | 6 +++++- tests/runtime/hosted-client-security.test.ts | 10 ++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/runtime/hosted-client.ts b/src/runtime/hosted-client.ts index 2edcd5f..a5fe35d 100644 --- a/src/runtime/hosted-client.ts +++ b/src/runtime/hosted-client.ts @@ -270,9 +270,13 @@ export const describeBoundedRecallFallback = (error: unknown): string => { const prefix = "Saved-context retrieval unavailable; continuing with free hot memory."; if (error instanceof HostedOnlyBotsRequestError) { const details = error.details && typeof error.details === "object" ? (error.details as Record) : null; - if (error.code === "recall_deadline_exceeded" || details?.charged_sats === 0) { + if (error.code === "paid_operation_commit_unknown") { + return `${prefix} The payment outcome is unknown; retry only with the same idempotency key to resolve it without a duplicate charge.`; + } + if (error.code === "recall_deadline_exceeded" && details?.charged_sats === 0) { return `${prefix} The server missed its response budget (HTTP ${error.status}) and charged nothing; a retry is safe.`; } + if (details?.charged_sats === 0) return `${prefix} The server could not deliver the request (HTTP ${error.status}) and charged nothing; a retry is safe.`; if (error.status < 500) return `${prefix} The server rejected the request (HTTP ${error.status}).`; } return `${prefix} A timed-out or failed paid request may still settle; inspect billing before retrying.`; diff --git a/tests/runtime/hosted-client-security.test.ts b/tests/runtime/hosted-client-security.test.ts index 2c24d0f..e575404 100644 --- a/tests/runtime/hosted-client-security.test.ts +++ b/tests/runtime/hosted-client-security.test.ts @@ -225,6 +225,16 @@ describe("describeHostedFailure", () => { }); describe("describeBoundedRecallFallback", () => { + it("keeps unknown commits distinct from confirmed free failures and contradicting charge claims", () => { + const uncertain = describeBoundedRecallFallback(new HostedOnlyBotsRequestError({ status: 503, code: "paid_operation_commit_unknown", message: "private", details: { charged_sats: 0 } })); + expect(uncertain).toContain("same idempotency key"); + expect(uncertain).not.toContain("charged nothing"); + const contradictory = describeBoundedRecallFallback(new HostedOnlyBotsRequestError({ status: 503, code: "recall_deadline_exceeded", message: "private", details: { charged_sats: 2 } })); + expect(contradictory).toContain("may still settle"); + const rejected = describeBoundedRecallFallback(new HostedOnlyBotsRequestError({ status: 400, code: "bad_request", message: "private", details: { charged_sats: 0 } })); + expect(rejected).toContain("charged nothing"); + expect(rejected).not.toContain("missed its response budget"); + }); it("says nothing was charged only when the server confirms it, and keeps the settle caution otherwise", () => { const deadline = new HostedOnlyBotsRequestError({ status: 503, code: "recall_deadline_exceeded", message: "late", details: { charged_sats: 0, retryable: true } }); From 3c6a523e695c625f51db008e51a189e6e0a9fb79 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Fri, 6 Nov 2026 22:17:16 -0500 Subject: [PATCH 4/4] Exercise hosted fallback diagnostics independently of local-first defaults --- tests/runtime/hosted-lifecycle-security.test.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/runtime/hosted-lifecycle-security.test.ts b/tests/runtime/hosted-lifecycle-security.test.ts index 8b7bd09..bf21938 100644 --- a/tests/runtime/hosted-lifecycle-security.test.ts +++ b/tests/runtime/hosted-lifecycle-security.test.ts @@ -189,7 +189,16 @@ describe("hosted lifecycle prompt boundary", () => { failure: { errorClass: "timeout" }, warning: "may still settle", }, + { + label: "an unacknowledged paid commit", + contextPack: async () => jsonResponse({ error: { code: "paid_operation_commit_unknown", message: "uncertain", details: { retry_with_same_key: true } } }, 503), + failure: { errorClass: "http", status: 503, code: "paid_operation_commit_unknown" }, + warning: "same idempotency key", + }, ])("records why bounded recall fell back to free hot memory after $label", async ({ contextPack, failure, warning }) => { + // Local-first sessions deliberately skip paid automatic recall. Exercise the + // still-supported hosted path explicitly rather than depending on defaults. + vi.stubEnv("CROSSHEAD_HOT_SYNC_ENABLED", "false"); const signer = identity(); const home = mkdtempSync(path.join(os.tmpdir(), "crosshead-bounded-fallback-")); homes.push(home);