diff --git a/.agents/skills/contribute-adapter/SKILL.md b/.agents/skills/contribute-adapter/SKILL.md index f1c2845f3..6026e50b5 100644 --- a/.agents/skills/contribute-adapter/SKILL.md +++ b/.agents/skills/contribute-adapter/SKILL.md @@ -121,3 +121,6 @@ release. Public existence alone cannot verify trusted publishing. If the PR check reports HTTP 404, ask an authorized npm maintainer to bootstrap the exact manifest name, then rerun the check. Do not add CI write tokens or manually publish supported versions. + + +For execution deadline failures, normalize target-native errors to canonical `timeout` at the adapter boundary. Retain the owning layer, enforced limit or absolute deadline, and stable native cause in lifecycle error metadata or normalized error extensions. Keep startup, shutdown, transport, and configuration failures distinct; consumer bridges must not contain harness-specific timeout code tables. diff --git a/.agents/skills/validate-change/SKILL.md b/.agents/skills/validate-change/SKILL.md index f00a51a53..c21479de1 100644 --- a/.agents/skills/validate-change/SKILL.md +++ b/.agents/skills/validate-change/SKILL.md @@ -59,6 +59,7 @@ surfaces touched by a change. - **Adapter behavior changed** Run the focused adapter tests under `tests/adapters`, then `just test-python`. - **Harbor integration changed** + For deadline changes, test runtime-owned and adapter-normalized invocation deadlines, genuine outer expiration, and setup/transport failures with cleanup and verifier invocation counts. Verify Harbor phase ownership separately when its upstream fix is an external dependency; do not claim an unpatched Harbor summary preserves nested deadlines. Use `qualify-harbor` for real inference and container checks across the adapter catalog or a developer subset. Retain its complete catalog report; deterministic tests alone do not establish an all-harness integration pass. Run `tests/integrations/test_harbor_runner.py`, `tests/integrations/test_harbor_model_config.py`, and diff --git a/adapters/typescript/kilo/README.md b/adapters/typescript/kilo/README.md index 473d01971..6d790f328 100644 --- a/adapters/typescript/kilo/README.md +++ b/adapters/typescript/kilo/README.md @@ -76,7 +76,7 @@ authentication and per-server tool filters are unsupported. The adapter accepts plain-text input and returns the terminal assistant text in `output.response`. Ordered invocations reuse one warm Kilo Code session and its conversation history. A prompt has a 30-minute deadline; on timeout the adapter -asks Kilo Code to abort the prompt and reports `kilo_prompt_timeout`. +asks Kilo Code to abort the prompt and reports `timeout`, retaining `kilo_prompt_timeout` as the cause. The adapter disables project Kilo Code configuration and uses temporary Kilo and XDG directories. The child process receives required operating-system @@ -155,3 +155,8 @@ credentials, or MCP connectivity. Checks never send prompts or abort a turn. At most one native health request remains outstanding after a caller timeout. Refer to the [SDK health-check overview](../../../docs/sdk/health-checks.mdx) for shared behavior and a comparison of native evidence across adapters. + + +## Invocation deadlines + +Kilo invocation deadlines use canonical error code `timeout`. Error metadata retains `owner: "adapter"`, the native cause, and the enforced timeout in seconds or absolute `deadline_millis`. Setup and transport failures retain their existing classifications. diff --git a/adapters/typescript/kilo/kilo.fabric-adapter.json b/adapters/typescript/kilo/kilo.fabric-adapter.json index 4ee63c7ef..815b58500 100644 --- a/adapters/typescript/kilo/kilo.fabric-adapter.json +++ b/adapters/typescript/kilo/kilo.fabric-adapter.json @@ -45,6 +45,28 @@ "additionalProperties": false }, "extension_schemas": { + "run_error": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "owner": { + "const": "adapter" + }, + "timeout_seconds": { + "type": "number", + "minimum": 0 + }, + "cause_code": { + "const": "kilo_prompt_timeout" + } + }, + "required": [ + "owner", + "timeout_seconds", + "cause_code" + ], + "additionalProperties": false + }, "usage": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", diff --git a/adapters/typescript/kilo/src/kilo-sdk.ts b/adapters/typescript/kilo/src/kilo-sdk.ts index d4342ab44..ee36aec6e 100644 --- a/adapters/typescript/kilo/src/kilo-sdk.ts +++ b/adapters/typescript/kilo/src/kilo-sdk.ts @@ -611,7 +611,15 @@ class KiloSdkSessionHandle implements KiloSessionHandle { } catch { if (timedOut) { await abortPromise; - throw new LifecycleError("kilo_prompt_timeout", "Timed out while waiting for Kilo Code to complete the prompt", {retryable: false}); + throw new LifecycleError( + "timeout", + `Kilo adapter invocation timed out after ${this.promptTimeoutMs / 1000} seconds`, + {metadata: { + owner: "adapter", + timeout_seconds: this.promptTimeoutMs / 1000, + cause_code: "kilo_prompt_timeout", + }}, + ); } throw new LifecycleError("kilo_session_failed", "Kilo Code session communication failed", {retryable: false}); } finally { diff --git a/adapters/typescript/kilo/src/runtime.ts b/adapters/typescript/kilo/src/runtime.ts index 24eb3f509..4328cf68b 100644 --- a/adapters/typescript/kilo/src/runtime.ts +++ b/adapters/typescript/kilo/src/runtime.ts @@ -7,6 +7,7 @@ import type { AgentRunRequest, AgentRunResult, AgentUsage, + JsonObject, RuntimeContext, } from "nemo-fabric-adapter-contract"; import { @@ -19,7 +20,7 @@ import { /** Failed communication retains its usage snapshot while the session is retired. */ export class KiloPromptError extends LifecycleError { constructor(error: LifecycleError, readonly usage: AgentUsage) { - super(error.code, error.message, { retryable: error.retryable }); + super(error.code, error.message, { retryable: error.retryable, metadata: error.metadata }); } } @@ -27,8 +28,8 @@ export interface KiloPromptOutcome { text?: string; error?: boolean; usage?: Age export interface KiloSessionHandle { id: string; check(level: CheckLevel): Promise; prompt(text: string, deadlineMillis?: number): Promise; stop(): Promise } export interface KiloSessionFactory { create(input: AdapterStartInput): Promise } -function failed(code: string, message: string, usage?: AgentUsage): AgentRunResult { - return {status: "failed", output: null, error: {code, message, retryable: false}, ...(usage === undefined ? {} : {usage})}; +function failed(code: string, message: string, usage?: AgentUsage, metadata?: JsonObject): AgentRunResult { + return {status: "failed", output: null, error: {code, message, retryable: false, ...(metadata === undefined ? {} : {extensions: metadata})}, ...(usage === undefined ? {} : {usage})}; } export class KiloAdapterRuntime implements AdapterRuntime { @@ -57,7 +58,7 @@ export class KiloAdapterRuntime implements AdapterRuntime { this.generation += 1; this.session = undefined; try { await session.stop(); } catch { /* Preserve the prompt failure. */ } - if (error instanceof KiloPromptError) return failed(error.code, error.message, error.usage); + if (error instanceof KiloPromptError) return failed(error.code, error.message, error.usage, error.metadata); if (error instanceof LifecycleError) throw error; throw new LifecycleError("kilo_session_failed", "Kilo Code session communication failed", {retryable: true}); } finally { diff --git a/adapters/typescript/kilo/test/kilo-sdk.test.mjs b/adapters/typescript/kilo/test/kilo-sdk.test.mjs index 141743f2a..2d2f7bcfb 100644 --- a/adapters/typescript/kilo/test/kilo-sdk.test.mjs +++ b/adapters/typescript/kilo/test/kilo-sdk.test.mjs @@ -252,7 +252,10 @@ test("aborts the Kilo session when a prompt reaches its deadline", async () => { }, }, }); - await assert.rejects(handle.prompt("wait forever"), (error) => error.code === "kilo_prompt_timeout"); + await assert.rejects(handle.prompt("wait forever"), { + code: "timeout", + metadata: {owner: "adapter", timeout_seconds: 0, cause_code: "kilo_prompt_timeout"}, + }); assert.equal(promptSignal.aborted, true); assert.deepEqual(abortParameters, {sessionID: "session-timeout", directory: workspace}); assert.ok(abortSignal instanceof AbortSignal); @@ -338,7 +341,10 @@ for (const phase of ["headers", "body"]) { }); const prompting = handle.prompt("wait for the result"); if (expire) { - const rejected = assert.rejects(prompting, (error) => error.code === "kilo_prompt_timeout"); + const rejected = assert.rejects(prompting, { + code: "timeout", + metadata: {owner: "adapter", timeout_seconds: 0.5, cause_code: "kilo_prompt_timeout"}, + }); await requestReady; await rejected; await Promise.all([sessionAborted, requestClosed]); diff --git a/adapters/typescript/kilo/test/runtime.test.mjs b/adapters/typescript/kilo/test/runtime.test.mjs index c33a50237..685f12273 100644 --- a/adapters/typescript/kilo/test/runtime.test.mjs +++ b/adapters/typescript/kilo/test/runtime.test.mjs @@ -4,7 +4,8 @@ import assert from "node:assert/strict"; import test from "node:test"; -import {KiloAdapterRuntime} from "../dist/runtime.js"; +import {KiloAdapterRuntime, KiloPromptError} from "../dist/runtime.js"; +import {LifecycleError} from "nemo-fabric-adapters-common"; const context = {runtime_id: "r", invocation_id: "i", request_id: "q", artifacts: {}, environment: {environment_id: "e", provider: "local", ownership: "caller_owned", control_location: "external_control", env: {}}}; @@ -36,3 +37,23 @@ test("invalidates the runtime after transport failure", async () => { await assert.rejects(runtime.invoke({input: "two"}, context), (error) => error.code === "kilo_not_started"); assert.equal(stopped, 1); }); + + +test("preserves deadline ownership and usage through normalized failure and cleanup", async () => { + const metadata = {owner: "adapter", timeout_seconds: 300, cause_code: "kilo_prompt_timeout"}; + const usage = {input_tokens: 1, output_tokens: 2}; + let stopped = 0; + const runtime = new KiloAdapterRuntime({create: async () => ({ + id: "s", + prompt: async () => { throw new KiloPromptError(new LifecycleError("timeout", "Kilo deadline after 300 seconds", {metadata}), usage); }, + stop: async () => { stopped += 1; }, + })}); + await runtime.start({agentName: "agent", baseDir: ".", config: {}, runtimeContext: context}); + const result = await runtime.invoke({input: "test"}, context); + assert.equal(result.error.code, "timeout"); + assert.deepEqual(result.error.extensions, metadata); + assert.deepEqual(result.usage, usage); + assert.equal(stopped, 1); + await runtime.stop(); + assert.equal(stopped, 1); +}); diff --git a/adapters/typescript/kiro/README.md b/adapters/typescript/kiro/README.md index 3c2c00696..2ad246890 100644 --- a/adapters/typescript/kiro/README.md +++ b/adapters/typescript/kiro/README.md @@ -73,3 +73,8 @@ live, and a stopped or exited process fails the liveness observation. `active` and `ready` are unsupported. Checks do not send prompts, change session settings, or cancel invocations. Refer to the [SDK health-check overview](../../../docs/sdk/health-checks.mdx) for shared behavior and a comparison of native evidence across adapters. + + +## Invocation deadlines + +Kiro invocation deadlines use canonical error code `timeout`. Error metadata retains `owner: "adapter"`, the native cause, and the enforced timeout in seconds or absolute `deadline_millis`. Setup and transport failures retain their existing classifications. diff --git a/adapters/typescript/kiro/kiro.fabric-adapter.json b/adapters/typescript/kiro/kiro.fabric-adapter.json index ac16a5c5c..837946eac 100644 --- a/adapters/typescript/kiro/kiro.fabric-adapter.json +++ b/adapters/typescript/kiro/kiro.fabric-adapter.json @@ -32,6 +32,28 @@ "additionalProperties": false }, "extension_schemas": { + "run_error": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "owner": { + "const": "adapter" + }, + "deadline_millis": { + "type": "integer", + "minimum": 0 + }, + "cause_code": { + "const": "kiro_deadline_exceeded" + } + }, + "required": [ + "owner", + "deadline_millis", + "cause_code" + ], + "additionalProperties": false + }, "run_result": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", diff --git a/adapters/typescript/kiro/src/kiro-acp.ts b/adapters/typescript/kiro/src/kiro-acp.ts index 2aeae706e..09acbbd0d 100644 --- a/adapters/typescript/kiro/src/kiro-acp.ts +++ b/adapters/typescript/kiro/src/kiro-acp.ts @@ -55,7 +55,7 @@ function childEnvironment(input: AdapterStartInput): NodeJS.ProcessEnv { return environment; } -function translateRunnerError(error: AcpRunnerError, stage: "start" | "prompt"): LifecycleError { +function translateRunnerError(error: AcpRunnerError, stage: "start" | "prompt", timeoutMs = 300_000): LifecycleError { if (error.code === "acp_process_unavailable") { return new LifecycleError("kiro_harness_unavailable", "The installed Kiro CLI could not be started"); } @@ -78,7 +78,16 @@ function translateRunnerError(error: AcpRunnerError, stage: "start" | "prompt"): ); } if (error.code === "acp_prompt_timeout") { - return new LifecycleError("kiro_prompt_timeout", "Kiro did not finish the invocation before the deadline"); + return new LifecycleError( + "timeout", + `Kiro adapter invocation timed out after ${timeoutMs / 1000} seconds`, + { metadata: { + owner: "adapter", + timeout_seconds: timeoutMs / 1000, + cause_code: "kiro_prompt_timeout", + native_code: error.code, + } }, + ); } return new LifecycleError( stage === "start" ? "kiro_start_failed" : "kiro_session_failed", @@ -185,12 +194,12 @@ class KiroAcpSession implements KiroSessionHandle { }; } - async prompt(text: string, timeoutMs?: number): Promise { + async prompt(text: string, timeoutMs = 300_000): Promise { let result: AcpPromptResult; try { result = await this.session.prompt(text, timeoutMs); } catch (error) { - if (error instanceof AcpRunnerError) throw translateRunnerError(error, "prompt"); + if (error instanceof AcpRunnerError) throw translateRunnerError(error, "prompt", timeoutMs); throw new LifecycleError("kiro_session_failed", "Kiro ACP session communication failed"); } let usage: AgentUsage | undefined; diff --git a/adapters/typescript/kiro/src/runtime.ts b/adapters/typescript/kiro/src/runtime.ts index 61d88875d..f7f750544 100644 --- a/adapters/typescript/kiro/src/runtime.ts +++ b/adapters/typescript/kiro/src/runtime.ts @@ -89,9 +89,14 @@ export class KiroAdapterRuntime implements AdapterRuntime { status: "failed", output: null, error: { - code: "kiro_deadline_exceeded", - message: "The Kiro invocation deadline has already elapsed", + code: "timeout", + message: `Kiro adapter invocation deadline ${context.deadline_millis} (Unix milliseconds) has already elapsed`, retryable: false, + extensions: { + owner: "adapter", + deadline_millis: context.deadline_millis, + cause_code: "kiro_deadline_exceeded", + }, }, }; } diff --git a/adapters/typescript/kiro/test/kiro-acp.test.mjs b/adapters/typescript/kiro/test/kiro-acp.test.mjs index 417b3a1e0..39d96cff3 100644 --- a/adapters/typescript/kiro/test/kiro-acp.test.mjs +++ b/adapters/typescript/kiro/test/kiro-acp.test.mjs @@ -11,120 +11,134 @@ import { fileURLToPath } from "node:url"; import { KiroAcpSessionFactory } from "../dist/kiro-acp.js"; for (const fixtureType of process.platform === "win32" ? ["executable"] : ["shell", "executable"]) { - test(`runs a persistent Kiro-compatible ACP session with a ${fixtureType} fixture`, async () => { - const bin = await mkdtemp(join(tmpdir(), "fabric-kiro-bin-")); - const executable = join(bin, process.platform === "win32" ? "kiro-cli.exe" : "kiro-cli"); - if (fixtureType === "executable") { - await copyFile(process.execPath, executable); - await writeFile(join(bin, "acp"), 'import(require("node:url").pathToFileURL(process.env.TEST_ACP_FIXTURE).href);\n'); - } else { - await writeFile( - executable, - "#!/bin/sh\nexec \"$TEST_NODE_EXECUTABLE\" \"$TEST_ACP_FIXTURE\"\n", - "utf8", + for (const terminalState of ["timeout", "native exit"]) { + test(`runs a persistent Kiro-compatible ACP session with a ${fixtureType} fixture ending in ${terminalState}`, async () => { + const bin = await mkdtemp(join(tmpdir(), "fabric-kiro-bin-")); + const executable = join(bin, process.platform === "win32" ? "kiro-cli.exe" : "kiro-cli"); + if (fixtureType === "executable") { + await copyFile(process.execPath, executable); + await writeFile(join(bin, "acp"), 'import(require("node:url").pathToFileURL(process.env.TEST_ACP_FIXTURE).href);\n'); + } else { + await writeFile( + executable, + "#!/bin/sh\nexec \"$TEST_NODE_EXECUTABLE\" \"$TEST_ACP_FIXTURE\"\n", + "utf8", + ); + await chmod(executable, 0o755); + } + const fixture = resolve( + fileURLToPath(new URL("../../common/test/fixtures/acp-agent.mjs", import.meta.url)), ); - await chmod(executable, 0o755); - } - const fixture = resolve( - fileURLToPath(new URL("../../common/test/fixtures/acp-agent.mjs", import.meta.url)), - ); - const environment = { - PATH: `${bin}${delimiter}${process.env.PATH ?? ""}`, - TEST_NODE_EXECUTABLE: process.execPath, - TEST_ACP_FIXTURE: fixture, - TEST_PROMPT_DELAY_MS: "100", - TEST_PROMPT_STARTED: join(bin, "prompt-started"), - TEST_PROCESS_MARKER: join(bin, "native-pid"), - }; - const session = await new KiroAcpSessionFactory().create({ - agentName: "kiro-test", - baseDir: process.cwd(), - config: { - models: { default: { provider: "kiro", model: "model-b" } }, - harness: { settings: { mode_id: "plan", effort: "high", permission_mode: "allow_once" } }, - }, - runtimeContext: { - runtime_id: "runtime", - invocation_id: "start", - request_id: "request", - artifacts: {}, - environment: { - environment_id: "environment", - provider: "local", - ownership: "caller_owned", - control_location: "external_control", - workspace: bin, - env: environment, + const environment = { + PATH: `${bin}${delimiter}${process.env.PATH ?? ""}`, + TEST_NODE_EXECUTABLE: process.execPath, + TEST_ACP_FIXTURE: fixture, + TEST_PROMPT_DELAY_MS: "100", + TEST_PROMPT_STARTED: join(bin, "prompt-started"), + TEST_PROCESS_MARKER: join(bin, "native-pid"), + }; + const session = await new KiroAcpSessionFactory().create({ + agentName: "kiro-test", + baseDir: process.cwd(), + config: { + models: { default: { provider: "kiro", model: "model-b" } }, + harness: { settings: { mode_id: "plan", effort: "high", permission_mode: "allow_once" } }, + }, + runtimeContext: { + runtime_id: "runtime", + invocation_id: "start", + request_id: "request", + artifacts: {}, + environment: { + environment_id: "environment", + provider: "local", + ownership: "caller_owned", + control_location: "external_control", + workspace: bin, + env: environment, + }, }, - }, - }); - try { - assert.equal((await session.check("live")).outcome, "passed"); - assert.equal((await session.check("active")).outcome, "unsupported"); - assert.equal((await session.check("ready")).outcome, "unsupported"); - const firstPending = session.prompt("one"); - const deadline = Date.now() + 2000; - while (true) { - try { await readFile(environment.TEST_PROMPT_STARTED); break; } catch (error) { if (error.code !== "ENOENT") throw error; } - assert.ok(Date.now() < deadline); - await new Promise((resolveDelay) => setTimeout(resolveDelay, 5)); - } - assert.equal((await session.check("live")).outcome, "passed"); - const first = await firstPending; - assert.equal((await session.check("live")).outcome, "passed"); - const second = await session.prompt("permission"); - assert.equal(first.text, "one:1:plan:model-b:high:mcp=0:permission=none:ready=false"); - assert.equal(second.text, "permission:2:plan:model-b:high:mcp=0:permission=allow:ready=false"); - assert.deepEqual(first.usage, { input_tokens: 2, output_tokens: 3, total_tokens: 5 }); - assert.deepEqual(second.usage, { input_tokens: 2, output_tokens: 3, total_tokens: 5 }); - - assert.equal((await session.prompt("usage-missing")).usage, undefined); - assert.equal((await session.prompt("after-gap")).usage, undefined); - assert.deepEqual((await session.prompt("after-baseline")).usage, { - input_tokens: 2, - output_tokens: 3, - total_tokens: 5, - }); - assert.deepEqual((await session.prompt("cache-first")).usage, { - input_tokens: 2, - output_tokens: 3, - total_tokens: 5, - }); - assert.deepEqual((await session.prompt("cache-second")).usage, { - input_tokens: 2, - output_tokens: 3, - total_tokens: 5, - cached_input_tokens: 7, }); + try { + assert.equal((await session.check("live")).outcome, "passed"); + assert.equal((await session.check("active")).outcome, "unsupported"); + assert.equal((await session.check("ready")).outcome, "unsupported"); + const firstPending = session.prompt("one"); + const deadline = Date.now() + 2000; + while (true) { + try { await readFile(environment.TEST_PROMPT_STARTED); break; } catch (error) { if (error.code !== "ENOENT") throw error; } + assert.ok(Date.now() < deadline); + await new Promise((resolveDelay) => setTimeout(resolveDelay, 5)); + } + assert.equal((await session.check("live")).outcome, "passed"); + const first = await firstPending; + assert.equal((await session.check("live")).outcome, "passed"); + const second = await session.prompt("permission"); + assert.equal(first.text, "one:1:plan:model-b:high:mcp=0:permission=none:ready=false"); + assert.equal(second.text, "permission:2:plan:model-b:high:mcp=0:permission=allow:ready=false"); + assert.deepEqual(first.usage, { input_tokens: 2, output_tokens: 3, total_tokens: 5 }); + assert.deepEqual(second.usage, { input_tokens: 2, output_tokens: 3, total_tokens: 5 }); - for (const input of ["usage-reset", "usage-malformed"]) { - const result = await session.prompt(input); - assert.equal(result.stopReason, "end_turn"); - assert.ok(result.text.startsWith(`${input}:`)); - assert.equal(result.usage, undefined); - assert.equal((await session.prompt("after-invalid-usage")).usage, undefined); - assert.deepEqual((await session.prompt("after-new-baseline")).usage, { + assert.equal((await session.prompt("usage-missing")).usage, undefined); + assert.equal((await session.prompt("after-gap")).usage, undefined); + assert.deepEqual((await session.prompt("after-baseline")).usage, { input_tokens: 2, output_tokens: 3, total_tokens: 5, }); + assert.deepEqual((await session.prompt("cache-first")).usage, { + input_tokens: 2, + output_tokens: 3, + total_tokens: 5, + }); + assert.deepEqual((await session.prompt("cache-second")).usage, { + input_tokens: 2, + output_tokens: 3, + total_tokens: 5, + cached_input_tokens: 7, + }); + + for (const input of ["usage-reset", "usage-malformed"]) { + const result = await session.prompt(input); + assert.equal(result.stopReason, "end_turn"); + assert.ok(result.text.startsWith(`${input}:`)); + assert.equal(result.usage, undefined); + assert.equal((await session.prompt("after-invalid-usage")).usage, undefined); + assert.deepEqual((await session.prompt("after-new-baseline")).usage, { + input_tokens: 2, + output_tokens: 3, + total_tokens: 5, + }); + } + if (terminalState === "timeout") { + await assert.rejects(session.prompt("hang", 0), { + code: "timeout", + metadata: { + owner: "adapter", + timeout_seconds: 0, + cause_code: "kiro_prompt_timeout", + native_code: "acp_prompt_timeout", + }, + }); + } else { + const pid = Number(await readFile(environment.TEST_PROCESS_MARKER, "utf8")); + process.kill(pid, "SIGKILL"); + const exitDeadline = Date.now() + 2000; + let exited; + do { + exited = await session.check("live"); + if (exited.outcome === "failed") break; + assert.ok(Date.now() < exitDeadline, "native process exit was not observed"); + await new Promise((resolve) => setTimeout(resolve, 5)); + } while (true); + assert.equal(exited.reason_code, "agent_exited"); + assert.equal(exited.activity, "stopped"); + assert.ok(Number.isFinite(Date.parse(exited.observed_at))); + } + } finally { + await session.stop(); + assert.equal((await session.check("live")).reason_code, "session_closed"); } - const pid = Number(await readFile(environment.TEST_PROCESS_MARKER, "utf8")); - process.kill(pid, "SIGKILL"); - const exitDeadline = Date.now() + 2000; - let exited; - do { - exited = await session.check("live"); - if (exited.outcome === "failed") break; - assert.ok(Date.now() < exitDeadline, "native process exit was not observed"); - await new Promise((resolve) => setTimeout(resolve, 5)); - } while (true); - assert.equal(exited.reason_code, "agent_exited"); - assert.equal(exited.activity, "stopped"); - assert.ok(Number.isFinite(Date.parse(exited.observed_at))); - } finally { - await session.stop(); - assert.equal((await session.check("live")).reason_code, "session_closed"); - } - }); + }); + } } diff --git a/adapters/typescript/kiro/test/runtime.test.mjs b/adapters/typescript/kiro/test/runtime.test.mjs index 3f919069d..d9a779e27 100644 --- a/adapters/typescript/kiro/test/runtime.test.mjs +++ b/adapters/typescript/kiro/test/runtime.test.mjs @@ -98,17 +98,23 @@ test("rejects an expired deadline before dispatch and keeps the session usable", }); await runtime.start({ agentName: "agent", baseDir: ".", config: {}, runtimeContext: context }); + const deadline = Date.now() - 1; const expired = await runtime.invoke( { input: "expired" }, - { ...context, deadline_millis: Date.now() - 1 }, + { ...context, deadline_millis: deadline }, ); assert.deepEqual(expired, { status: "failed", output: null, error: { - code: "kiro_deadline_exceeded", - message: "The Kiro invocation deadline has already elapsed", + code: "timeout", + message: `Kiro adapter invocation deadline ${deadline} (Unix milliseconds) has already elapsed`, retryable: false, + extensions: { + owner: "adapter", + deadline_millis: deadline, + cause_code: "kiro_deadline_exceeded", + }, }, }); assert.deepEqual(prompts, []); diff --git a/adapters/typescript/strands/README.md b/adapters/typescript/strands/README.md index 0a66cd06d..bb3cfa9dd 100644 --- a/adapters/typescript/strands/README.md +++ b/adapters/typescript/strands/README.md @@ -152,3 +152,8 @@ Strands supports `live` through `Fabric.check()`. `live` reads the owned ACP chi Checks remain independent of an invocation. A busy invocation is healthy. Checks use the caller deadline and never cancel an invocation, stop the runtime, or repair a failure. Health does not establish valid model credentials, provider access, MCP connectivity, or successful model output. Reports contain only the generic health contract fields. Refer to the [SDK health-check overview](../../../docs/sdk/health-checks.mdx) for shared behavior and a comparison of native evidence across adapters. + + +## Invocation deadlines + +Strands prompt deadlines and already-expired invocation deadlines use canonical error code `timeout`. Error metadata retains `owner: "adapter"` and the native cause (`strands_prompt_timeout` or `strands_deadline_exceeded`), plus the enforced timeout in seconds or the absolute `deadline_millis`. Prompt expiration invalidates the session and attempts process cleanup. Startup and transport failures keep their existing error classifications. diff --git a/adapters/typescript/strands/src/runtime.ts b/adapters/typescript/strands/src/runtime.ts index 1e643dedd..238ef37b5 100644 --- a/adapters/typescript/strands/src/runtime.ts +++ b/adapters/typescript/strands/src/runtime.ts @@ -130,9 +130,14 @@ export class StrandsAdapterRuntime implements AdapterRuntime { status: "failed", output: null, error: { - code: "strands_deadline_exceeded", - message: "The Strands invocation deadline has already elapsed", + code: "timeout", + message: `Strands adapter invocation deadline ${context.deadline_millis} (Unix milliseconds) has already elapsed`, retryable: false, + extensions: { + owner: "adapter", + deadline_millis: context.deadline_millis, + cause_code: "strands_deadline_exceeded", + }, }, }; } diff --git a/adapters/typescript/strands/src/strands-acp.ts b/adapters/typescript/strands/src/strands-acp.ts index 7a490ce93..be9fc66e2 100644 --- a/adapters/typescript/strands/src/strands-acp.ts +++ b/adapters/typescript/strands/src/strands-acp.ts @@ -109,6 +109,7 @@ function childEnvironment(input: AdapterStartInput): NodeJS.ProcessEnv { function translateRunnerError( error: unknown, stage: "start" | "prompt", + timeoutMs = 300_000, ): LifecycleError { if (error instanceof LifecycleError) return error; if ( @@ -122,8 +123,16 @@ function translateRunnerError( } if (error instanceof AcpRunnerError && error.code === "acp_prompt_timeout") { return new LifecycleError( - "strands_prompt_timeout", - "Strands did not finish the invocation before the deadline", + "timeout", + `Strands adapter invocation timed out after ${timeoutMs / 1000} seconds`, + { + metadata: { + owner: "adapter", + timeout_seconds: timeoutMs / 1000, + cause_code: "strands_prompt_timeout", + native_code: error.code, + }, + }, ); } // Do not expose target error text: it may contain credentials or prompt content. @@ -205,7 +214,7 @@ class StrandsAcpSession implements StrandsSessionHandle { async prompt( text: string, - timeoutMs?: number, + timeoutMs = 300_000, ): Promise { try { const result = await this.session.prompt(text, timeoutMs); @@ -216,7 +225,7 @@ class StrandsAcpSession implements StrandsSessionHandle { ...(usage === undefined ? {} : { usage }), }; } catch (error) { - throw translateRunnerError(error, "prompt"); + throw translateRunnerError(error, "prompt", timeoutMs); } } diff --git a/adapters/typescript/strands/strands.fabric-adapter.json b/adapters/typescript/strands/strands.fabric-adapter.json index 550648af6..ab7e4f09f 100644 --- a/adapters/typescript/strands/strands.fabric-adapter.json +++ b/adapters/typescript/strands/strands.fabric-adapter.json @@ -106,6 +106,28 @@ ] }, "extension_schemas": { + "run_error": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "owner": { + "const": "adapter" + }, + "deadline_millis": { + "type": "integer", + "minimum": 0 + }, + "cause_code": { + "const": "strands_deadline_exceeded" + } + }, + "required": [ + "owner", + "deadline_millis", + "cause_code" + ], + "additionalProperties": false + }, "run_result": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", diff --git a/adapters/typescript/strands/test/runtime.test.mjs b/adapters/typescript/strands/test/runtime.test.mjs index d37d70cd7..14752663c 100644 --- a/adapters/typescript/strands/test/runtime.test.mjs +++ b/adapters/typescript/strands/test/runtime.test.mjs @@ -141,17 +141,23 @@ test("rejects an expired deadline before dispatch and keeps the session usable", runtimeContext: context, }); + const deadline = Date.now() - 1; const expired = await runtime.invoke( { input: "expired" }, - { ...context, deadline_millis: Date.now() - 1 }, + { ...context, deadline_millis: deadline }, ); assert.deepEqual(expired, { status: "failed", output: null, error: { - code: "strands_deadline_exceeded", - message: "The Strands invocation deadline has already elapsed", + code: "timeout", + message: `Strands adapter invocation deadline ${deadline} (Unix milliseconds) has already elapsed`, retryable: false, + extensions: { + owner: "adapter", + deadline_millis: deadline, + cause_code: "strands_deadline_exceeded", + }, }, }); assert.deepEqual(prompts, []); diff --git a/adapters/typescript/strands/test/strands-acp.test.mjs b/adapters/typescript/strands/test/strands-acp.test.mjs index ec3f6d8be..79e433725 100644 --- a/adapters/typescript/strands/test/strands-acp.test.mjs +++ b/adapters/typescript/strands/test/strands-acp.test.mjs @@ -133,7 +133,13 @@ test("passes MCP servers into the first ACP session and cleans up on timeout", a const result = await session.prompt("mcp"); assert.equal(JSON.parse(result.text)[0].name, "probe"); await assert.rejects(session.prompt("hang", 25), { - code: "strands_prompt_timeout", + code: "timeout", + metadata: { + owner: "adapter", + timeout_seconds: 0.025, + cause_code: "strands_prompt_timeout", + native_code: "acp_prompt_timeout", + }, }); await session.stop(); }); @@ -239,7 +245,11 @@ test("retains a timed-out runtime session until process cleanup succeeds", { ski }); await assert.rejects( runtime.invoke({ input: "hang" }, { ...input.runtimeContext, deadline_millis: Date.now() + 50 }), - { code: "strands_prompt_timeout" }, + (error) => + error.code === "timeout" && + error.metadata?.owner === "adapter" && + error.metadata?.cause_code === "strands_prompt_timeout" && + error.metadata?.native_code === "acp_prompt_timeout", ); originalKill(child, 0); await assert.rejects(runtime.invoke({ input: "next" }, input.runtimeContext), { code: "strands_not_started" }); diff --git a/crates/fabric-core/src/error.rs b/crates/fabric-core/src/error.rs index 424dc5cab..226740276 100644 --- a/crates/fabric-core/src/error.rs +++ b/crates/fabric-core/src/error.rs @@ -234,6 +234,8 @@ pub enum FabricError { message: String, /// Bounded adapter-host diagnostics. diagnostics: String, + /// Structured adapter or runtime failure evidence. + metadata: std::collections::BTreeMap, }, /// Owned lifecycle cleanup is unconfirmed; the original runtime remains owned. #[error("{0}")] diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index eaa6d4134..54eeefd27 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -1035,7 +1035,10 @@ pub fn run_plan(plan: &RunPlan, request: RunRequest) -> Result { code: "runtime_stop_failed".to_string(), message: error.to_string(), retryable: false, - metadata: BTreeMap::new(), + metadata: match &error { + FabricError::AdapterLifecycleOperation { metadata, .. } => metadata.clone(), + _ => BTreeMap::new(), + }, }); } Err(_) => {} @@ -3863,17 +3866,25 @@ fn exchange_lifecycle_message( let line = match host.responses.recv_timeout(timeout) { Ok(line) => line, Err(RecvTimeoutError::Timeout) => { - return Err(lifecycle_error( - operation, - runtime_id, - "host_timeout", - format!( - "persistent local adapter host did not complete {} within {} ms", + return Err(FabricError::AdapterLifecycleOperation { + operation: operation.as_str(), + runtime_id: runtime_id.to_string(), + code: "host_timeout".to_string(), + message: format!( + "Fabric runtime did not complete {} within {} seconds", operation.as_str(), - timeout.as_millis() + timeout.as_secs_f64() ), - local_host_diagnostics(host), - )); + diagnostics: local_host_diagnostics(host), + metadata: BTreeMap::from([ + ("owner".to_string(), serde_json::json!("fabric_runtime")), + ( + "timeout_seconds".to_string(), + serde_json::json!(timeout.as_secs_f64()), + ), + ("cause_code".to_string(), serde_json::json!("host_timeout")), + ]), + }); } Err(RecvTimeoutError::Disconnected) => { return Err(lifecycle_error( @@ -3962,13 +3973,14 @@ fn exchange_lifecycle_message( diagnostics.push_str("adapter metadata: "); diagnostics.push_str(&metadata); } - Err(lifecycle_error( - operation, - runtime_id, - error.code, - error.message, + Err(FabricError::AdapterLifecycleOperation { + operation: operation.as_str(), + runtime_id: runtime_id.to_string(), + code: error.code, + message: error.message, diagnostics, - )) + metadata: error.metadata, + }) } } } @@ -3986,6 +3998,7 @@ fn lifecycle_error( code: code.into(), message: message.into(), diagnostics: diagnostics.into(), + metadata: BTreeMap::new(), } } @@ -5810,6 +5823,13 @@ for line in sys.stdin: with open(os.environ["FABRIC_FAKE_HOST_STOP_MARKER"], "w", encoding="utf-8") as stream: stream.write("started") time.sleep(0.25) + if MODE == "stop_deadline": + error = failure("stop", "timeout", "adapter stop deadline expired") + error["metadata"] = { + "owner": "adapter", "timeout_seconds": 300, "cause_code": "adapter_stop_timeout", + } + response("stop", error=error) + sys.exit(18) if MODE == "stop_failure": response("stop", error=failure("stop", "fake_stop", "stop rejected")) sys.exit(18) @@ -8400,6 +8420,12 @@ else: .. } if code == "host_timeout" && diagnostics.contains("invoke accepted without response") )); + let FabricError::AdapterLifecycleOperation { metadata, .. } = &error else { + panic!("expected lifecycle deadline error"); + }; + assert_eq!(metadata["owner"], "fabric_runtime"); + assert_eq!(metadata["timeout_seconds"], 0.1); + assert_eq!(metadata["cause_code"], "host_timeout"); assert!(!local_hosts().contains_key(&runtime.runtime_id)); assert!( host.lock() @@ -9420,19 +9446,28 @@ else: #[test] fn run_plan_preserves_completed_result_when_stop_fails() { - let (root, plan) = local_host_plan("stop_failure"); - - let result = run_plan(&plan, RunRequest::text("completed")) - .expect("completed result with stop failure"); + for (mode, cause) in [("stop_failure", "fake_stop"), ("stop_deadline", "timeout")] { + let (root, plan) = local_host_plan(mode); - assert_eq!(result.status, RunStatus::Failed); - assert_eq!(result.output["input"], "completed"); - let error = result.error.expect("stop error"); - assert_eq!(error.stage, ErrorStage::Stop); - assert_eq!(error.code, "runtime_stop_failed"); - assert!(error.message.contains("fake_stop"), "{}", error.message); + let result = run_plan(&plan, RunRequest::text("completed")) + .expect("completed result with stop failure"); + + assert_eq!(result.status, RunStatus::Failed); + assert_eq!(result.output["input"], "completed"); + let error = result.error.expect("stop error"); + assert_eq!(error.stage, ErrorStage::Stop); + assert_eq!(error.code, "runtime_stop_failed"); + assert!(error.message.contains(cause), "{}", error.message); + if mode == "stop_deadline" { + assert_eq!(error.metadata["owner"], "adapter"); + assert_eq!(error.metadata["timeout_seconds"], 300); + assert_eq!(error.metadata["cause_code"], "adapter_stop_timeout"); + } else { + assert!(error.metadata.is_empty()); + } - let _ = fs::remove_dir_all(root); + let _ = fs::remove_dir_all(root); + } } #[test] fn default_environment_prepares_through_local_provider() { diff --git a/crates/fabric-python/src/lib.rs b/crates/fabric-python/src/lib.rs index 44e9d8f64..975ba7825 100644 --- a/crates/fabric-python/src/lib.rs +++ b/crates/fabric-python/src/lib.rs @@ -16,7 +16,7 @@ use nemo_fabric_core::{ resolve_diagnostic_plan_from_config_with_adapter_directories, resolve_run_plan_from_config_with_adapter_directories, run_plan, }; -use pyo3::exceptions::{PyRuntimeError, PyTimeoutError}; +use pyo3::exceptions::{PyException, PyRuntimeError, PyTimeoutError}; use pyo3::prelude::*; pyo3::create_exception!( @@ -447,8 +447,25 @@ fn to_py_error(error: nemo_fabric_core::FabricError) -> PyErr { }); } match &error { - FabricError::AdapterLifecycleOperation { code, .. } if code == "host_timeout" => { - PyTimeoutError::new_err(error.to_string()) + FabricError::AdapterLifecycleOperation { code, metadata, .. } + if code == "host_timeout" || code == "timeout" => + { + let timeout = PyTimeoutError::new_err(error.to_string()); + Python::attach(|py| { + let attachment = (|| -> PyResult<()> { + let details = py.import("json")?.call_method1( + "loads", + (serde_json::to_string(metadata) + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?,), + )?; + timeout.value(py).setattr("details", details) + })(); + // Metadata enrichment must not change the timeout classification. + match attachment { + Err(error) if !error.is_instance_of::(py) => error, + _ => timeout, + } + }) } FabricError::ResourceObservationUnavailable { .. } => { ResourceObservationUnavailableError::new_err(error.to_string()) diff --git a/docs/integrations/consumer/harbor.mdx b/docs/integrations/consumer/harbor.mdx index d182c85f8..e625b5660 100644 --- a/docs/integrations/consumer/harbor.mdx +++ b/docs/integrations/consumer/harbor.mdx @@ -85,3 +85,10 @@ The existing `FabricAgent` bridge follows this flow for `fabric_environment_env` `environment_env_names` is Harbor's existing transport field. `environment.env_names` is the core NeMo Fabric forwarding field described in [Set Environment Values Or Forward Them By Name](../../sdk/python.mdx#set-environment-values-or-forward-them-by-name). This addition does not migrate the Harbor bridge to the core field: Harbor's retained run specification contains names, but its runner still puts resolved values in `environment.env`, where values are serialized into the NeMo Fabric run plan. Keep using Harbor's managed environment and redaction rules for credentials. Core `env_names` does not replace Harbor's secret delivery or redaction. + + +## Execution Deadlines + +NeMo Fabric runtime and normalized adapter invocation deadlines use code `timeout` at stage `invoke`. The bridge raises Harbor's `AgentTimeoutError`, so both remain eligible for the original verifier. Startup and shutdown deadlines retain runtime failure handling and skip verification. The collected `fabric-result-*.json` retains the owning layer, actual limit, and native cause in `runner_error.details` or normalized error metadata. + +Harbor also needs its [phase timeout ownership fix](https://github.com/harbor-framework/harbor/pull/3376) to preserve the inner exception in the trial summary. Older Harbor versions can relabel an inner 300-second deadline as the outer 1200-second task timeout even when the outer context has not expired. NeMo Fabric evidence keeps the original deadline on those versions; upgrading NeMo Fabric alone does not fix that Harbor summary. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index 926a51ed3..8614346e8 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n        message: String,\n    },\n    ResolveBaseDirectory {\n        path: PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeCleanupPending(Box<RuntimeCleanupPending>),\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    StateInUse {\n        path: PathBuf,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentInUse {\n        environment_id: String,\n        runtime_id: String,\n    },\n    EnvironmentReleaseInProgress {\n        environment_id: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    ResourceObservationUnavailable {\n        runtime_id: String,\n        reason: String,\n    },\n    ResourceObservationUnsupported {\n        runtime_id: String,\n        reason: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
+
String,\n        message: String,\n    },\n    ResolveBaseDirectory {\n        path: PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n        metadata: BTreeMap<String, Value>,\n    },\n    RuntimeCleanupPending(Box<RuntimeCleanupPending>),\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    StateInUse {\n        path: PathBuf,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentInUse {\n        environment_id: String,\n        runtime_id: String,\n    },\n    EnvironmentReleaseInProgress {\n        environment_id: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    ResourceObservationUnavailable {\n        runtime_id: String,\n        reason: String,\n    },\n    ResourceObservationUnsupported {\n        runtime_id: String,\n        reason: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
Errors raised by NeMo Fabric config loading and validation. @@ -421,6 +421,10 @@ Human-readable failure message. Bounded adapter-host diagnostics. +##### `metadata: BTreeMap` + +Structured adapter or runtime failure evidence. + ### `RuntimeCleanupPending(Box)`
Box<RuntimeCleanupPending>)"}} />
diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 7c9aa27f8..859355eb4 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1581,6 +1581,8 @@ Core host-operation deadlines (`host_timeout`) become Python `TimeoutError`, the If invocation succeeds in `Fabric.run()` but stop reaches this deadline, it returns a failed `RunResult` with `error.stage == "stop"` and `error.code == "timeout"`. Check the returned result as well as raised errors. +Adapter lifecycle failures with canonical `code == "timeout"` also preserve deadline identity. Raised `FabricRuntimeError.details` retains structured source metadata: runtime host deadlines report `owner: "fabric_runtime"`, `timeout_seconds`, and `cause_code: "host_timeout"`; adapter deadlines retain their adapter-owned limit and cause. Native diagnostic messages remain available through exception chaining. + Adapters can report execution deadlines in normalized results with `result.error.code == "timeout"`. Other adapter-specific timeout codes are not automatically reclassified. Diagnostic text is not a timeout classifier. diff --git a/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json b/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json index 2bdefc795..aacc693e4 100644 --- a/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json +++ b/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json @@ -1178,6 +1178,28 @@ }, "contract_version": "fabric.adapter/v1alpha2", "extension_schemas": { + "run_error": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "cause_code": { + "const": "kilo_prompt_timeout" + }, + "owner": { + "const": "adapter" + }, + "timeout_seconds": { + "minimum": 0, + "type": "number" + } + }, + "required": [ + "owner", + "timeout_seconds", + "cause_code" + ], + "type": "object" + }, "usage": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -1350,7 +1372,7 @@ "name": "nemo-fabric-adapters-kilo", "version": "0.7.0" }, - "sha256": "6bf1bf8c3df7c62c7942c23c5b1639df54764773e2f4d945c19daf04c701a7a1", + "sha256": "bb236376f76d986c5b087952dd815e9c100dfa9ccd235964e25e8bc9f610cedd", "source": "adapters/typescript/kilo/kilo.fabric-adapter.json" }, "nvidia.fabric.kiro": { @@ -1376,6 +1398,28 @@ }, "contract_version": "fabric.adapter/v1alpha2", "extension_schemas": { + "run_error": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "cause_code": { + "const": "kiro_deadline_exceeded" + }, + "deadline_millis": { + "minimum": 0, + "type": "integer" + }, + "owner": { + "const": "adapter" + } + }, + "required": [ + "owner", + "deadline_millis", + "cause_code" + ], + "type": "object" + }, "run_result": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -1465,7 +1509,7 @@ "name": "nemo-fabric-adapters-kiro", "version": "0.7.0" }, - "sha256": "e7393a07550505f23a3396128a29f5b078492ce7f1580f6f3b79d5872e4587a2", + "sha256": "7273f357f8f8f3d429f4127d06f29653c015d20fb4d82d3c8c0f4dee9f84ff2c", "source": "adapters/typescript/kiro/kiro.fabric-adapter.json" }, "nvidia.fabric.langchain.deepagents": { @@ -3422,6 +3466,28 @@ }, "contract_version": "fabric.adapter/v1alpha2", "extension_schemas": { + "run_error": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "cause_code": { + "const": "strands_deadline_exceeded" + }, + "deadline_millis": { + "minimum": 0, + "type": "integer" + }, + "owner": { + "const": "adapter" + } + }, + "required": [ + "owner", + "deadline_millis", + "cause_code" + ], + "type": "object" + }, "run_result": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -3585,7 +3651,7 @@ "name": "nemo-fabric-adapters-strands", "version": "0.7.0" }, - "sha256": "37982ca315b241b03fce3edb1d49d2f1663e3224ebcf89824d3a976055ccd949", + "sha256": "3b6e7f1dec2fc416c30c00e24225b21a161160252b6aba3846028319478ca3c9", "source": "adapters/typescript/strands/strands.fabric-adapter.json" } }, diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py index 8464c1edd..c836a6b1c 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py @@ -83,6 +83,9 @@ def _runtime_error(error: Exception, *, stage: str) -> FabricRuntimeError: stage=stage, code=code if isinstance(code, str) else None, retryable=retryable if isinstance(retryable, bool) else False, + details=getattr(error, "details", None) + if isinstance(error, TimeoutError) + else None, ) cleanup = getattr(error, "_fabric_cleanup_runtime_handle", None) if cleanup is not None: diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/README.md b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/README.md index 90f7f4f20..9b02bf0eb 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/README.md +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/README.md @@ -77,17 +77,13 @@ Deploy matching bridge and runner versions: a runner predating name-only transpo The task runner writes normalized evidence before exiting nonzero for failed or cancelled runs. The bridge downloads that evidence before reporting a Harbor execution failure or cancellation, including when the runner exits nonzero. A completed answer can still receive verifier reward zero; verifier scoring is not an execution failure. A missing or malformed result does not hide an available process failure. -Lifecycle failures before a normalized result is available produce a separate -`runner_error` record. Core host-operation deadlines (`host_timeout`) use code -`timeout`. The bridge classifies only this canonical code as a deadline, whether -it appears in a runner error or normalized result, and raises `TimeoutError`, -which Harbor translates into `AgentTimeoutError`. Cancellation takes precedence. -An invocation with status `cancelled` becomes a trial execution error and retains -its normalized status; it does not interrupt the Harbor job. Actual orchestration -cancellation still propagates as `asyncio.CancelledError`. -The bridge does not -inspect harness IDs, native error-code lists, or diagnostic text. Other -adapter-specific timeout codes are not automatically reclassified. +Lifecycle failures before a normalized result is available produce a separate `runner_error` record. Core host-operation deadlines (`host_timeout`) and adapter lifecycle deadlines reported with canonical code `timeout` retain the lifecycle stage and structured `details`. Runtime deadlines include `owner: "fabric_runtime"`, `timeout_seconds`, and `cause_code: "host_timeout"`. Adapters normalize native execution deadlines at their boundary, with their own owner, limit, and native cause; the bridge does not inspect harness IDs, native error-code lists, or diagnostic text. + +Only canonical `timeout` failures at stage `invoke` raise Harbor's `AgentTimeoutError` and remain eligible for verification. Startup and shutdown deadlines, ordinary runtime failures, and configuration errors raise `RuntimeError` and skip verification. Equivalent runtime and adapter invocation deadlines use the same policy. The verifier still owns the reward. + +Harbor versions with [phase timeout ownership](https://github.com/harbor-framework/harbor/pull/3376) preserve the inner exception and its actual deadline. Older Harbor versions catch all nested `TimeoutError` subclasses and can still overwrite the trial exception message with their outer budget, even when that budget did not expire. The downloaded NeMo Fabric evidence retains the original deadline and cause in either case; this NeMo Fabric change alone does not fix Harbor's phase ownership. + +Cancellation takes precedence. An invocation with status `cancelled` becomes a trial execution error and retains its normalized status; it does not interrupt the Harbor job. Actual orchestration cancellation still propagates as `asyncio.CancelledError`. Raised lifecycle failures retain an optional versioned `evidence` object beside `runner_error`. It records only obtained identities, cleanup events, selected software evidence, validated persisted stats and artifact references. Each dimension has explicit coverage; missing usage and cost remain unknown, and absent samples are not reconstructed. A result obtained before a later publication failure is retained as `partial_result`, without changing the failure into a successful run. Partial usage remains evidence and does not populate Harbor's final accounting totals. diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/fabric_agent.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/fabric_agent.py index df17cfefc..bd7c8be34 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/fabric_agent.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/fabric_agent.py @@ -1224,8 +1224,14 @@ def raise_run_failure(status: str, error: dict[str, Any] | None) -> NoReturn: # An invocation outcome must not cancel Harbor's job orchestration. # Actual task cancellation still propagates from the awaited operations. raise RuntimeError(message) - if error is not None and error.get("code") == "timeout": - raise TimeoutError(message) + if ( + error is not None + and error.get("code") == "timeout" + and error.get("stage") == "invoke" + ): + from harbor.trial.errors import AgentTimeoutError + + raise AgentTimeoutError(message) raise RuntimeError(message) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/models.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/models.py index 0231000bd..09b055ba7 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/models.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/models.py @@ -8,6 +8,7 @@ from pathlib import PurePosixPath import re from typing import Annotated +from typing import Any from typing import Literal from typing import Self @@ -78,6 +79,7 @@ class FabricRunnerError(BaseModel): code: str | None = None message: str retryable: bool = False + details: dict[str, Any] = Field(default_factory=dict) _FAILURE_DIMENSIONS = ( diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/runner.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/runner.py index 52c0a4e37..899d1681a 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/runner.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/runner.py @@ -113,6 +113,16 @@ def main() -> None: else f"Fabric runner failed ({type(error).__name__})" ), retryable=error.retryable if isinstance(error, FabricError) else False, + # Runtime observations belong in the bounded evidence envelope. + details=( + { + key: value + for key, value in error.details.items() + if key != "runtime_evidence" + } + if isinstance(error, FabricError) + else {} + ), ) try: document = FabricRunnerFailure( diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index 1d2e6e634..d70f53615 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -1079,13 +1079,14 @@ def run() -> dict[str, Any]: if result is None: raise if result.get("status") == "succeeded": + stop_error = _runtime_error(error, stage="stop") result["status"] = "failed" result["error"] = { "stage": "stop", - "code": _runtime_error(error, stage="stop").code - or "runtime_stop_failed", + "code": stop_error.code or "runtime_stop_failed", "message": str(error) or "runtime shutdown failed", "retryable": False, + "metadata": stop_error.details, } stop_events = [] if result is not None and isinstance(stop_events, list): @@ -1153,14 +1154,25 @@ def _resource_snapshot_error(native: Any, error: Exception) -> FabricRuntimeErro async def _call_blocking( func: Any, *, executor: ThreadPoolExecutor | None = None ) -> Any: + def call() -> tuple[Any, TimeoutError | None]: + try: + return func(), None + except TimeoutError as error: + # Python 3.11/3.12's executor bridge recreates TimeoutError, losing + # its details. Carry the original as a result across that boundary. + return None, error + if executor is None: - worker = asyncio.create_task(asyncio.to_thread(func)) + worker = asyncio.create_task(asyncio.to_thread(call)) else: worker = asyncio.ensure_future( - asyncio.get_running_loop().run_in_executor(executor, func) + asyncio.get_running_loop().run_in_executor(executor, call) ) try: - return await asyncio.shield(worker) + result, timeout_error = await asyncio.shield(worker) + if timeout_error is not None: + raise timeout_error + return result except asyncio.CancelledError as cancelled: # Re-awaiting shield in a cancelled task can prevent the worker result # from becoming observable, so let the event loop finish it first. diff --git a/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py b/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py index 728ed1f4c..cb35973b5 100644 --- a/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py +++ b/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py @@ -45,6 +45,26 @@ async def invoke( "hermes_runtime_not_started", "shim runtime is not started", ) + if "normalized_deadline_result" in request.context: + return AgentRunResult( + status=AgentRunStatus.FAILED, + output=None, + error=AgentRunError( + code="timeout", + message="Adapter invocation deadline expired", + extensions=request.context["normalized_deadline_result"], + ), + ) + if request.context.get("normalized_deadline"): + raise lifecycle.LifecycleError( + "timeout", + "Adapter invocation exceeded 300 seconds", + metadata={ + "owner": "adapter", + "timeout_seconds": 300, + "cause_code": "strands_prompt_timeout", + }, + ) pid_file = request.context.get("child_pid_file") if pid_file is not None: # Start a direct child in its own session (POSIX) and a grandchild diff --git a/tests/python/test_harbor_outcomes.py b/tests/python/test_harbor_outcomes.py index d4ed4749a..e3bba2a8c 100644 --- a/tests/python/test_harbor_outcomes.py +++ b/tests/python/test_harbor_outcomes.py @@ -371,12 +371,14 @@ async def test_completed_wrong_answer_is_not_an_execution_failure(bridge): @pytest.mark.parametrize( - ("code", "exception"), - [("timeout", TimeoutError), ("connection_failed", RuntimeError)], + ("code", "is_timeout"), + [("timeout", True), ("connection_failed", False)], ) -async def test_bridge_classifies_only_canonical_deadline(bridge, code, exception): +async def test_bridge_classifies_only_canonical_deadline(bridge, code, is_timeout): from harbor.models.agent.context import AgentContext + from harbor.trial.errors import AgentTimeoutError + exception = AgentTimeoutError if is_timeout else RuntimeError agent, environment, document = bridge document["status"] = "failed" document["error"] = { @@ -392,7 +394,7 @@ async def test_bridge_classifies_only_canonical_deadline(bridge, code, exception @pytest.mark.parametrize( ("code", "exception"), - [("timeout", TimeoutError), ("configuration_failed", RuntimeError)], + [("timeout", RuntimeError), ("configuration_failed", RuntimeError)], ) async def test_bridge_retains_runner_failure_record(bridge, code, exception): from harbor.models.agent.context import AgentContext @@ -410,7 +412,10 @@ async def test_bridge_retains_runner_failure_record(bridge, code, exception): with pytest.raises(exception): await agent.run("test", environment, context) agent.populate_context_post_run(context) - assert context.metadata["fabric"]["runner_error"] == document["runner_error"] + assert context.metadata["fabric"]["runner_error"] == { + "details": {}, + **document["runner_error"], + } @pytest.mark.parametrize("runner_failure", [False, True]) @@ -455,8 +460,10 @@ async def test_inner_deadline_enters_harbor_agent_timeout_path( user=None, ) - assert type(caught.value.__cause__) is TimeoutError - assert "inner invocation deadline" in str(caught.value.__cause__) + # Older Harbor releases wrap this error; ownership-aware versions retain it. + inner = caught.value.__cause__ or caught.value + assert isinstance(inner, AgentTimeoutError) + assert "inner invocation deadline" in str(inner) assert target.agent_execution.finished_at is not None agent.populate_context_post_run(target.agent_result) failure = target.agent_result.metadata["fabric"] @@ -484,6 +491,7 @@ def test_runner_serializes_structured_deadline(runner_cli, monkeypatch, stage): "code": "timeout", "message": "deadline", "retryable": False, + "details": {}, } @@ -512,6 +520,103 @@ def test_runner_invalid_spec_preserves_safe_evidence(runner_cli): assert "ValidationError" in runner_cli.read_text() +@pytest.mark.parametrize("stage", ["start", "invoke", "stop"]) +async def test_bridge_deadline_policy_preserves_owner_and_cause(bridge, stage): + from harbor.models.agent.context import AgentContext + from harbor.trial.errors import AgentTimeoutError + + agent, environment, document = bridge + document.clear() + document["runner_error"] = { + "stage": stage, + "code": "timeout", + "message": "Fabric runtime invoke deadline expired after 300 seconds", + "retryable": False, + "details": { + "owner": "fabric_runtime", + "timeout_seconds": 300, + "cause_code": "host_timeout", + }, + } + context = AgentContext() + expected = AgentTimeoutError if stage == "invoke" else RuntimeError + with pytest.raises(expected) as caught: + async with asyncio.timeout(1200): + await agent.run("test", environment, context) + assert type(caught.value) is expected + assert "300 seconds" in str(caught.value) + assert "1200" not in str(caught.value) + agent.populate_context_post_run(context) + assert context.metadata["fabric"]["runner_error"] == { + "details": {}, + **document["runner_error"], + } + + +@pytest.mark.parametrize("runner_failure", [False, True]) +@pytest.mark.parametrize( + "code,stage,verifier_calls", + [ + ("timeout", "invoke", 1), + ("connection_failed", "invoke", 0), + ("timeout", "start", 0), + ("timeout", "stop", 0), + ], +) +async def test_deadline_verifier_policy_and_cleanup( + bridge, runner_failure, code, stage, verifier_calls +): + from functools import partial + from harbor.models.agent.context import AgentContext + from harbor.models.task.config import TaskConfig + from harbor.models.task.task import Task + from harbor.models.trial.result import TrialResult + from harbor.trial.single_step import SingleStepTrial + + agent, environment, document = bridge + error = { + "stage": stage, + "code": code, + "message": "adapter invocation exceeded 300 seconds", + "retryable": False, + } + if runner_failure: + document.clear() + document["runner_error"] = error + else: + document.update(status="failed", error=error) + trial = MagicMock(spec=SingleStepTrial) + trial.result = MagicMock(spec=TrialResult) + trial.result.agent_result = AgentContext() + trial.task = MagicMock(spec=Task) + trial.task.config = TaskConfig() + trial.task.instruction = "test" + trial._load_trajectory = None + trial._agent_timeout_sec = 1200 + trial._run_agent.side_effect = partial(SingleStepTrial._run_agent, trial) + + # Model Harbor's ownership-preserving phase contract. Its upstream fix + # is tracked separately in harbor-framework/harbor#3376. + async def phase(**kwargs): + async with asyncio.timeout(kwargs["timeout_sec"]): + await agent.run("test", environment, kwargs["target"].agent_result) + + trial._run_agent_phase.side_effect = phase + if verifier_calls: + await SingleStepTrial._run(trial) + else: + with pytest.raises(RuntimeError): + await SingleStepTrial._run(trial) + assert trial._run_verifier.await_count == verifier_calls + trial._sync_agent_output.assert_awaited_once() + if verifier_calls: + failure = trial._record_exception.call_args.args[0] + assert "300 seconds" in str(failure) + assert "1200" not in str(failure) + else: + trial._record_exception.assert_not_called() + + @pytest.mark.parametrize("stage", ["start", "invoke"]) def test_runner_retains_obtained_failure_evidence(runner_cli, monkeypatch, stage): from nemo_fabric import FabricRuntimeError @@ -522,10 +627,15 @@ def test_runner_retains_obtained_failure_evidence(runner_cli, monkeypatch, stage stage=stage, code="timeout", details={ + "owner": "fabric_runtime", + "timeout_seconds": 300, + "cause_code": "host_timeout", "runtime_evidence": { "request_id": "request-test", "runtime_id": "runtime-test" if stage == "invoke" else None, - } + "execution": "x" * 262145, + "private": "EVIDENCE_PRIVATE_SENTINEL", + }, }, ) monkeypatch.setattr(runner, "run", AsyncMock(side_effect=error)) @@ -533,6 +643,16 @@ def test_runner_retains_obtained_failure_evidence(runner_cli, monkeypatch, stage runner.main() document = json.loads(runner_cli.read_text()) assert document["runner_error"]["message"] == "original" + assert document["runner_error"]["details"] == { + "owner": "fabric_runtime", + "timeout_seconds": 300, + "cause_code": "host_timeout", + } + assert document["evidence"]["execution"] is None + assert document["evidence"]["coverage"]["execution"] in {"invalid", "truncated"} + serialized = json.dumps(document) + assert "EVIDENCE_PRIVATE_SENTINEL" not in serialized + assert len(serialized.encode()) < 262144 assert document["evidence"]["request_id"] == "request-test" assert document["evidence"]["runtime_id"] == ( "runtime-test" if stage == "invoke" else None @@ -767,7 +887,7 @@ async def test_failure_consumer_keeps_partial_usage_out_of_totals(bridge): }, ) environment.exec.return_value.return_code = 1 - with pytest.raises(TimeoutError, match="original"): + with pytest.raises(RuntimeError, match="original"): await agent.run("test", environment, context) agent.populate_context_post_run(context) assert context.metadata["fabric"]["evidence"]["usage"]["output_tokens"] == 7 @@ -942,5 +1062,6 @@ def test_optional_evidence_collection_failure_preserves_primary_diagnostic( "code": "timeout", "message": "original", "retryable": False, + "details": {}, } assert "evidence" not in document diff --git a/tests/python/test_native_sdk.py b/tests/python/test_native_sdk.py index 716be5aa6..89de856c3 100644 --- a/tests/python/test_native_sdk.py +++ b/tests/python/test_native_sdk.py @@ -137,6 +137,11 @@ async def test_native_deadline_keeps_type_and_evicts_runtime( assert caught.value.code == "timeout" assert caught.value.stage == "invoke" assert isinstance(caught.value.__cause__, TimeoutError) + assert caught.value.details == { + "owner": "fabric_runtime", + "timeout_seconds": 5.0, + "cause_code": "host_timeout", + } assert runtime.status == RuntimeStatus.FAILED with pytest.raises(FabricStateError): await runtime.invoke(input="retry") @@ -192,6 +197,9 @@ def test_runner_real_inner_deadline_preserves_failure_evidence( failure = FabricRunnerFailure.model_validate_json(result.read_text()) assert failure.runner_error.code == "timeout" assert failure.runner_error.stage == "invoke" + assert failure.runner_error.details["owner"] == "fabric_runtime" + assert failure.runner_error.details["timeout_seconds"] == 1.0 + assert failure.runner_error.details["cause_code"] == "host_timeout" assert not failure.runner_error.retryable assert failure.evidence.runtime_id.startswith("runtime-") @@ -463,6 +471,108 @@ async def smoke(client: Fabric, fixture_agent: Path) -> None: assert empty_result.output["openai_stream_invocation_count"] == 2 +async def test_adapter_deadline_keeps_normalized_code_and_native_cause( + hermes_shim_agent_dir, +): + with pytest.raises(FabricRuntimeError) as caught: + await Fabric().run( + hermes_shim_config(), + base_dir=hermes_shim_agent_dir, + request=RunRequest(input="test", context={"normalized_deadline": True}), + ) + assert caught.value.code == "timeout" + assert caught.value.stage == "invoke" + assert caught.value.details["owner"] == "adapter" + assert caught.value.details["timeout_seconds"] == 300 + assert caught.value.details["cause_code"] == "strands_prompt_timeout" + assert caught.value.details["runtime_evidence"]["cleanup"] == "completed" + assert isinstance(caught.value.__cause__, TimeoutError) + + +@pytest.mark.parametrize( + "attachment_error_type", [RecursionError, ValueError, KeyboardInterrupt, SystemExit] +) +def test_native_deadline_attachment_failure_preserves_exception_kind( + hermes_shim_agent_dir, monkeypatch, attachment_error_type +): + plan = native.plan_config( + json.dumps(hermes_shim_config().to_mapping()), str(hermes_shim_agent_dir) + ) + runtime = native.start_runtime(plan) + request = json.dumps( + RunRequest(input="test", context={"normalized_deadline": True}).to_mapping() + ) + attachment_error = attachment_error_type("metadata attachment failed") + ordinary_error = isinstance(attachment_error, Exception) + expected = TimeoutError if ordinary_error else attachment_error_type + calls = [] + + def fail_attachment(*args, **kwargs): + calls.append(args) + raise attachment_error + + try: + with monkeypatch.context() as patch: + patch.setattr(json, "loads", fail_attachment) + with pytest.raises(expected) as caught: + native.invoke_runtime(plan, runtime, request) + assert len(calls) == 1 + if ordinary_error: + assert "300 seconds" in str(caught.value) + else: + assert caught.value is attachment_error + finally: + native.stop_runtime(plan, runtime) + + +@pytest.mark.parametrize("adapter", ["strands", "kiro", "kilo"]) +@pytest.mark.parametrize("invalid_metadata", [False, True]) +async def test_committed_deadline_schema_survives_core_result_validation( + hermes_shim_agent_dir, repo_root, adapter, invalid_metadata +): + source = json.loads( + ( + repo_root / f"adapters/typescript/{adapter}/{adapter}.fabric-adapter.json" + ).read_text() + ) + descriptor_path = ( + hermes_shim_agent_dir / "adapters/hermes-shim/hermes-shim.fabric-adapter.json" + ) + descriptor = json.loads(descriptor_path.read_text()) + descriptor["extension_schemas"].update(source["extension_schemas"]) + descriptor_path.write_text(json.dumps(descriptor)) + metadata = ( + { + "owner": "adapter", + "timeout_seconds": 300, + "cause_code": "kilo_prompt_timeout", + } + if adapter == "kilo" + else { + "owner": "adapter", + "deadline_millis": 300_000, + "cause_code": f"{adapter}_deadline_exceeded", + } + ) + if invalid_metadata: + metadata["undeclared"] = "must not cross the runtime boundary" + operation = Fabric().run( + hermes_shim_config(), + base_dir=hermes_shim_agent_dir, + request=RunRequest( + input="deadline", context={"normalized_deadline_result": metadata} + ), + ) + if invalid_metadata: + with pytest.raises(FabricRuntimeError, match="extensions"): + await operation + else: + result = await operation + assert result.status == "failed" + assert result.error.code == "timeout" + assert result.error.metadata == metadata + + @pytest.mark.skipif( sys.platform == "win32", reason="Unix host directory repair fixture" ) diff --git a/tests/python/test_runtime.py b/tests/python/test_runtime.py index 8499a2ce0..5071280c9 100644 --- a/tests/python/test_runtime.py +++ b/tests/python/test_runtime.py @@ -310,28 +310,43 @@ def occupy_worker(): @pytest.mark.parametrize("stage", ["start", "invoke", "stop"]) @pytest.mark.parametrize( - ("error_type", "expected_code"), - [(TimeoutError, "timeout"), (RuntimeError, None)], + ("error_type", "native_code", "expected_code"), + [ + (TimeoutError, "native_timeout", "timeout"), + (RuntimeError, None, None), + (RuntimeError, "adapter_failed", "adapter_failed"), + ], ) async def test_deadline_identity_survives_stateful_sdk_boundaries( - native_client, mock_native, stage, error_type, expected_code + native_client, mock_native, stage, error_type, native_code, expected_code ): + error = error_type("deadline-looking text") + if native_code is not None: + error.code = native_code + details = { + "owner": "fabric_runtime", + "timeout_seconds": 300, + "cause_code": "host_timeout", + } + if expected_code == "timeout": + error.details = details if stage == "start": - mock_native.start_runtime.side_effect = error_type("deadline-looking text") + mock_native.start_runtime.side_effect = error operation = native_client.start_runtime(_config()) else: runtime = await native_client.start_runtime(_config()) if stage == "invoke": - mock_native.invoke_runtime.side_effect = error_type("deadline-looking text") + mock_native.invoke_runtime.side_effect = error operation = runtime.invoke(input="test") else: - mock_native.stop_runtime.side_effect = error_type("deadline-looking text") + mock_native.stop_runtime.side_effect = error operation = runtime.stop() with pytest.raises(FabricRuntimeError) as caught: await operation assert caught.value.stage == stage assert caught.value.code == expected_code - assert isinstance(caught.value.__cause__, error_type) + assert caught.value.__cause__ is error + assert caught.value.details == (details if expected_code == "timeout" else {}) if stage == "invoke": await runtime.stop() @@ -355,16 +370,25 @@ async def test_one_shot_deadline_preserves_stage_and_cleanup( async def test_one_shot_stop_deadline_has_canonical_result_error( native_client, mock_native, structured ): - mock_native.stop_runtime.side_effect = ( - FabricRuntimeError("deadline", stage="stop", code="timeout") + details = { + "owner": "fabric_runtime", + "timeout_seconds": 300, + "cause_code": "host_timeout", + } + error = ( + FabricRuntimeError("deadline", stage="stop", code="timeout", details=details) if structured else TimeoutError("deadline") ) + error.details = details + mock_native.stop_runtime.side_effect = error result = await native_client.run(_config(), input="test") assert result.status == "failed" assert result.error.stage == "stop" assert result.error.code == "timeout" + assert result.error.metadata == details + @pytest.mark.parametrize("stage", ["start", "invoke"]) async def test_one_shot_preserves_existing_structured_error( @@ -1315,6 +1339,26 @@ async def test_native_unavailable_uses_typed_error(monkeypatch: pytest.MonkeyPat Fabric().plan(_config()) +async def test_native_deadline_details_survive_sdk_and_cleanup( + native_client, mock_native +): + native_error = TimeoutError("host invocation exceeded 300 seconds") + native_error.details = { + "owner": "fabric_runtime", + "timeout_seconds": 300, + "cause_code": "host_timeout", + } + mock_native.invoke_runtime.side_effect = native_error + with pytest.raises(FabricRuntimeError) as caught: + await native_client.run(_config(), input="test") + for name, value in native_error.details.items(): + assert caught.value.details[name] == value + assert caught.value.details["runtime_evidence"]["runtime_id"] == "runtime-1" + assert caught.value.details["runtime_evidence"]["cleanup"] == "completed" + assert caught.value.__cause__ is native_error + mock_native.stop_runtime.assert_called_once() + + @pytest.mark.parametrize("state", [RuntimeStatus.ACTIVE, RuntimeStatus.FAILED]) async def test_runtime_status_reads_owned_state_without_native_work( native_client, mock_native, state