From a296ce6e48113946ed9e2f4455073f93217270d9 Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Tue, 3 Nov 2026 21:52:06 +0000 Subject: [PATCH 01/10] fix: preserve execution deadline ownership through Harbor Signed-off-by: Ajay Thorve --- .agents/skills/contribute-adapter/SKILL.md | 3 + .agents/skills/validate-change/SKILL.md | 1 + adapters/typescript/strands/README.md | 5 + adapters/typescript/strands/src/runtime.ts | 7 +- .../typescript/strands/src/strands-acp.ts | 17 ++- .../typescript/strands/test/runtime.test.mjs | 10 +- .../strands/test/strands-acp.test.mjs | 10 +- crates/fabric-core/src/error.rs | 2 + crates/fabric-core/src/runtime.rs | 46 ++++--- crates/fabric-python/src/lib.rs | 16 ++- docs/integrations/consumer/harbor.mdx | 7 ++ docs/sdk/python.mdx | 2 + .../src/nemo_fabric/errors.py | 3 + .../nemo_fabric/integrations/harbor/README.md | 18 ++- .../integrations/harbor/fabric_agent.py | 10 +- .../nemo_fabric/integrations/harbor/models.py | 2 + .../nemo_fabric/integrations/harbor/runner.py | 1 + .../src/nemo_fabric/runtime.py | 1 + skills/nemo-fabric-build-adapter/SKILL.md | 1 + .../references/results-and-errors.md | 2 + .../hermes_shim/adapter.py | 10 ++ tests/python/test_harbor_outcomes.py | 119 ++++++++++++++++-- tests/python/test_native_sdk.py | 29 +++++ tests/python/test_runtime.py | 30 ++++- 24 files changed, 304 insertions(+), 48 deletions(-) diff --git a/.agents/skills/contribute-adapter/SKILL.md b/.agents/skills/contribute-adapter/SKILL.md index f1c2845f3..6026e50b5 100644 --- a/.agents/skills/contribute-adapter/SKILL.md +++ b/.agents/skills/contribute-adapter/SKILL.md @@ -121,3 +121,6 @@ release. Public existence alone cannot verify trusted publishing. If the PR check reports HTTP 404, ask an authorized npm maintainer to bootstrap the exact manifest name, then rerun the check. Do not add CI write tokens or manually publish supported versions. + + +For execution deadline failures, normalize target-native errors to canonical `timeout` at the adapter boundary. Retain the owning layer, enforced limit or absolute deadline, and stable native cause in lifecycle error metadata or normalized error extensions. Keep startup, shutdown, transport, and configuration failures distinct; consumer bridges must not contain harness-specific timeout code tables. diff --git a/.agents/skills/validate-change/SKILL.md b/.agents/skills/validate-change/SKILL.md index 8ae866ff7..26bc5f26f 100644 --- a/.agents/skills/validate-change/SKILL.md +++ b/.agents/skills/validate-change/SKILL.md @@ -59,6 +59,7 @@ surfaces touched by a change. - **Adapter behavior changed** Run the focused adapter tests under `tests/adapters`, then `just test-python`. - **Harbor integration changed** + For deadline changes, test runtime-owned and adapter-normalized invocation deadlines, genuine outer expiration, and setup/transport failures with cleanup and verifier invocation counts. Verify Harbor phase ownership separately when its upstream fix is an external dependency; do not claim an unpatched Harbor summary preserves nested deadlines. Use `qualify-harbor` for real inference and container checks across the adapter catalog or a developer subset. Retain its complete catalog report; deterministic tests alone do not establish an all-harness integration pass. Run `tests/integrations/test_harbor_runner.py` and `tests/python/test_adapter_capabilities.py`, then `just test-python`. diff --git a/adapters/typescript/strands/README.md b/adapters/typescript/strands/README.md index 0a66cd06d..bb3cfa9dd 100644 --- a/adapters/typescript/strands/README.md +++ b/adapters/typescript/strands/README.md @@ -152,3 +152,8 @@ Strands supports `live` through `Fabric.check()`. `live` reads the owned ACP chi Checks remain independent of an invocation. A busy invocation is healthy. Checks use the caller deadline and never cancel an invocation, stop the runtime, or repair a failure. Health does not establish valid model credentials, provider access, MCP connectivity, or successful model output. Reports contain only the generic health contract fields. Refer to the [SDK health-check overview](../../../docs/sdk/health-checks.mdx) for shared behavior and a comparison of native evidence across adapters. + + +## Invocation deadlines + +Strands prompt deadlines and already-expired invocation deadlines use canonical error code `timeout`. Error metadata retains `owner: "adapter"` and the native cause (`strands_prompt_timeout` or `strands_deadline_exceeded`), plus the enforced timeout in seconds or the absolute `deadline_millis`. Prompt expiration invalidates the session and attempts process cleanup. Startup and transport failures keep their existing error classifications. diff --git a/adapters/typescript/strands/src/runtime.ts b/adapters/typescript/strands/src/runtime.ts index 1e643dedd..8c2ebc2be 100644 --- a/adapters/typescript/strands/src/runtime.ts +++ b/adapters/typescript/strands/src/runtime.ts @@ -130,9 +130,14 @@ export class StrandsAdapterRuntime implements AdapterRuntime { status: "failed", output: null, error: { - code: "strands_deadline_exceeded", + code: "timeout", message: "The Strands invocation deadline has already elapsed", retryable: false, + extensions: { + owner: "adapter", + deadline_millis: context.deadline_millis, + cause_code: "strands_deadline_exceeded", + }, }, }; } diff --git a/adapters/typescript/strands/src/strands-acp.ts b/adapters/typescript/strands/src/strands-acp.ts index 7a490ce93..be9fc66e2 100644 --- a/adapters/typescript/strands/src/strands-acp.ts +++ b/adapters/typescript/strands/src/strands-acp.ts @@ -109,6 +109,7 @@ function childEnvironment(input: AdapterStartInput): NodeJS.ProcessEnv { function translateRunnerError( error: unknown, stage: "start" | "prompt", + timeoutMs = 300_000, ): LifecycleError { if (error instanceof LifecycleError) return error; if ( @@ -122,8 +123,16 @@ function translateRunnerError( } if (error instanceof AcpRunnerError && error.code === "acp_prompt_timeout") { return new LifecycleError( - "strands_prompt_timeout", - "Strands did not finish the invocation before the deadline", + "timeout", + `Strands adapter invocation timed out after ${timeoutMs / 1000} seconds`, + { + metadata: { + owner: "adapter", + timeout_seconds: timeoutMs / 1000, + cause_code: "strands_prompt_timeout", + native_code: error.code, + }, + }, ); } // Do not expose target error text: it may contain credentials or prompt content. @@ -205,7 +214,7 @@ class StrandsAcpSession implements StrandsSessionHandle { async prompt( text: string, - timeoutMs?: number, + timeoutMs = 300_000, ): Promise { try { const result = await this.session.prompt(text, timeoutMs); @@ -216,7 +225,7 @@ class StrandsAcpSession implements StrandsSessionHandle { ...(usage === undefined ? {} : { usage }), }; } catch (error) { - throw translateRunnerError(error, "prompt"); + throw translateRunnerError(error, "prompt", timeoutMs); } } diff --git a/adapters/typescript/strands/test/runtime.test.mjs b/adapters/typescript/strands/test/runtime.test.mjs index d37d70cd7..fa6ca73d7 100644 --- a/adapters/typescript/strands/test/runtime.test.mjs +++ b/adapters/typescript/strands/test/runtime.test.mjs @@ -141,17 +141,23 @@ test("rejects an expired deadline before dispatch and keeps the session usable", runtimeContext: context, }); + const deadline = Date.now() - 1; const expired = await runtime.invoke( { input: "expired" }, - { ...context, deadline_millis: Date.now() - 1 }, + { ...context, deadline_millis: deadline }, ); assert.deepEqual(expired, { status: "failed", output: null, error: { - code: "strands_deadline_exceeded", + code: "timeout", message: "The Strands invocation deadline has already elapsed", retryable: false, + extensions: { + owner: "adapter", + deadline_millis: deadline, + cause_code: "strands_deadline_exceeded", + }, }, }); assert.deepEqual(prompts, []); diff --git a/adapters/typescript/strands/test/strands-acp.test.mjs b/adapters/typescript/strands/test/strands-acp.test.mjs index dbdc1f051..5dff70f77 100644 --- a/adapters/typescript/strands/test/strands-acp.test.mjs +++ b/adapters/typescript/strands/test/strands-acp.test.mjs @@ -133,7 +133,13 @@ test("passes MCP servers into the first ACP session and cleans up on timeout", a const result = await session.prompt("mcp"); assert.equal(JSON.parse(result.text)[0].name, "probe"); await assert.rejects(session.prompt("hang", 25), { - code: "strands_prompt_timeout", + code: "timeout", + metadata: { + owner: "adapter", + timeout_seconds: 0.025, + cause_code: "strands_prompt_timeout", + native_code: "acp_prompt_timeout", + }, }); await session.stop(); }); @@ -238,7 +244,7 @@ test("retains a timed-out runtime session until process cleanup succeeds", { ski }); await assert.rejects( runtime.invoke({ input: "hang" }, { ...input.runtimeContext, deadline_millis: Date.now() + 50 }), - { code: "strands_prompt_timeout" }, + { code: "timeout" }, ); originalKill(child, 0); await assert.rejects(runtime.invoke({ input: "next" }, input.runtimeContext), { code: "strands_not_started" }); diff --git a/crates/fabric-core/src/error.rs b/crates/fabric-core/src/error.rs index 056c0c034..f6ac9ab76 100644 --- a/crates/fabric-core/src/error.rs +++ b/crates/fabric-core/src/error.rs @@ -226,6 +226,8 @@ pub enum FabricError { message: String, /// Bounded adapter-host diagnostics. diagnostics: String, + /// Structured adapter or runtime failure evidence. + metadata: std::collections::BTreeMap, }, /// A runtime handle was used with a different run plan than the one that created it. #[error( diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index d7041c9ed..a831b6e9a 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -2900,17 +2900,25 @@ fn exchange_lifecycle_message( let line = match host.responses.recv_timeout(timeout) { Ok(line) => line, Err(RecvTimeoutError::Timeout) => { - return Err(lifecycle_error( - operation, - runtime_id, - "host_timeout", - format!( - "persistent local adapter host did not complete {} within {} ms", + return Err(FabricError::AdapterLifecycleOperation { + operation: operation.as_str(), + runtime_id: runtime_id.to_string(), + code: "host_timeout".to_string(), + message: format!( + "Fabric runtime did not complete {} within {} seconds", operation.as_str(), - timeout.as_millis() + timeout.as_secs_f64() ), - local_host_diagnostics(host), - )); + diagnostics: local_host_diagnostics(host), + metadata: BTreeMap::from([ + ("owner".to_string(), serde_json::json!("fabric_runtime")), + ( + "timeout_seconds".to_string(), + serde_json::json!(timeout.as_secs_f64()), + ), + ("cause_code".to_string(), serde_json::json!("host_timeout")), + ]), + }); } Err(RecvTimeoutError::Disconnected) => { return Err(lifecycle_error( @@ -2999,13 +3007,14 @@ fn exchange_lifecycle_message( diagnostics.push_str("adapter metadata: "); diagnostics.push_str(&metadata); } - Err(lifecycle_error( - operation, - runtime_id, - error.code, - error.message, + Err(FabricError::AdapterLifecycleOperation { + operation: operation.as_str(), + runtime_id: runtime_id.to_string(), + code: error.code, + message: error.message, diagnostics, - )) + metadata: error.metadata, + }) } } } @@ -3023,6 +3032,7 @@ fn lifecycle_error( code: code.into(), message: message.into(), diagnostics: diagnostics.into(), + metadata: BTreeMap::new(), } } @@ -5791,6 +5801,12 @@ for line in sys.stdin: .. } if code == "host_timeout" && diagnostics.contains("invoke accepted without response") )); + let FabricError::AdapterLifecycleOperation { metadata, .. } = &error else { + panic!("expected lifecycle deadline error"); + }; + assert_eq!(metadata["owner"], "fabric_runtime"); + assert_eq!(metadata["timeout_seconds"], 0.1); + assert_eq!(metadata["cause_code"], "host_timeout"); assert!(!local_hosts().contains_key(&runtime.runtime_id)); assert!( host.lock() diff --git a/crates/fabric-python/src/lib.rs b/crates/fabric-python/src/lib.rs index a340a1301..126521e1c 100644 --- a/crates/fabric-python/src/lib.rs +++ b/crates/fabric-python/src/lib.rs @@ -401,8 +401,20 @@ where fn to_py_error(error: nemo_fabric_core::FabricError) -> PyErr { match &error { - FabricError::AdapterLifecycleOperation { code, .. } if code == "host_timeout" => { - PyTimeoutError::new_err(error.to_string()) + FabricError::AdapterLifecycleOperation { code, metadata, .. } + if code == "host_timeout" || code == "timeout" => + { + Python::attach(|py| { + let timeout = PyTimeoutError::new_err(error.to_string()); + let details = py.import("json")?.call_method1( + "loads", + (serde_json::to_string(metadata) + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?,), + )?; + timeout.value(py).setattr("details", details)?; + Ok::<_, PyErr>(timeout) + }) + .unwrap_or_else(|error| error) } FabricError::ResourceObservationUnavailable { .. } => { ResourceObservationUnavailableError::new_err(error.to_string()) diff --git a/docs/integrations/consumer/harbor.mdx b/docs/integrations/consumer/harbor.mdx index 76251e05c..101caa97b 100644 --- a/docs/integrations/consumer/harbor.mdx +++ b/docs/integrations/consumer/harbor.mdx @@ -77,3 +77,10 @@ The existing `FabricAgent` bridge follows this flow for `fabric_environment_env` `environment_env_names` is Harbor's existing transport field. `environment.env_names` is the core NeMo Fabric forwarding field described in [Set Environment Values Or Forward Them By Name](../../sdk/python.mdx#set-environment-values-or-forward-them-by-name). This addition does not migrate the Harbor bridge to the core field: Harbor's retained run specification contains names, but its runner still puts resolved values in `environment.env`, where values are serialized into the NeMo Fabric run plan. Keep using Harbor's managed environment and redaction rules for credentials. Core `env_names` does not replace Harbor's secret delivery or redaction. + + +## Execution Deadlines + +Fabric runtime and normalized adapter invocation deadlines use code `timeout` at stage `invoke`. The bridge raises Harbor's `AgentTimeoutError`, so both remain eligible for the original verifier. Startup and shutdown deadlines retain runtime failure handling and skip verification. The collected `fabric-result-*.json` retains the owning layer, actual limit, and native cause in `runner_error.details` or normalized error metadata. + +Harbor also needs its [phase timeout ownership fix](https://github.com/harbor-framework/harbor/pull/3376) to preserve the inner exception in the trial summary. Older Harbor versions can relabel an inner 300-second deadline as the outer 1200-second task timeout even when the outer context has not expired. Fabric evidence keeps the original deadline on those versions; upgrading Fabric alone does not fix that Harbor summary. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index f93512c2b..a36a33049 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1439,6 +1439,8 @@ Core host-operation deadlines (`host_timeout`) become Python `TimeoutError`, the If invocation succeeds in `Fabric.run()` but stop reaches this deadline, it returns a failed `RunResult` with `error.stage == "stop"` and `error.code == "timeout"`. Check the returned result as well as raised errors. +Adapter lifecycle failures with canonical `code == "timeout"` also preserve deadline identity. Raised `FabricRuntimeError.details` retains structured source metadata: runtime host deadlines report `owner: "fabric_runtime"`, `timeout_seconds`, and `cause_code: "host_timeout"`; adapter deadlines retain their adapter-owned limit and cause. Native diagnostic messages remain available through exception chaining. + Adapters can report execution deadlines in normalized results with `result.error.code == "timeout"`. Other adapter-specific timeout codes are not automatically reclassified. Diagnostic text is not a timeout classifier. diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py index fae1fdfb0..ea27434e5 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py @@ -66,6 +66,9 @@ def _runtime_error(error: Exception, *, stage: str) -> FabricRuntimeError: str(error), stage=stage, code="timeout" if isinstance(error, TimeoutError) else None, + details=getattr(error, "details", None) + if isinstance(error, TimeoutError) + else None, ) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/README.md b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/README.md index d7d6f1bc4..68ba3b037 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/README.md +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/README.md @@ -58,17 +58,13 @@ Deploy matching bridge and runner versions: a runner predating name-only transpo The task runner writes normalized evidence before exiting nonzero for failed or cancelled runs. The bridge downloads that evidence before reporting a Harbor execution failure or cancellation, including when the runner exits nonzero. A completed answer can still receive verifier reward zero; verifier scoring is not an execution failure. A missing or malformed result does not hide an available process failure. -Lifecycle failures before a normalized result is available produce a separate -`runner_error` record. Core host-operation deadlines (`host_timeout`) use code -`timeout`. The bridge classifies only this canonical code as a deadline, whether -it appears in a runner error or normalized result, and raises `TimeoutError`, -which Harbor translates into `AgentTimeoutError`. Cancellation takes precedence. -An invocation with status `cancelled` becomes a trial execution error and retains -its normalized status; it does not interrupt the Harbor job. Actual orchestration -cancellation still propagates as `asyncio.CancelledError`. -The bridge does not -inspect harness IDs, native error-code lists, or diagnostic text. Other -adapter-specific timeout codes are not automatically reclassified. +Lifecycle failures before a normalized result is available produce a separate `runner_error` record. Core host-operation deadlines (`host_timeout`) and adapter lifecycle deadlines reported with canonical code `timeout` retain the lifecycle stage and structured `details`. Runtime deadlines include `owner: "fabric_runtime"`, `timeout_seconds`, and `cause_code: "host_timeout"`. Adapters normalize native execution deadlines at their boundary, with their own owner, limit, and native cause; the bridge does not inspect harness IDs, native error-code lists, or diagnostic text. + +Only canonical `timeout` failures at stage `invoke` raise Harbor's `AgentTimeoutError` and remain eligible for verification. Startup and shutdown deadlines, ordinary runtime failures, and configuration errors raise `RuntimeError` and skip verification. Equivalent runtime and adapter invocation deadlines use the same policy. The verifier still owns the reward. + +Harbor versions with [phase timeout ownership](https://github.com/harbor-framework/harbor/pull/3376) preserve the inner exception and its actual deadline. Older Harbor versions catch all nested `TimeoutError` subclasses and can still overwrite the trial exception message with their outer budget, even when that budget did not expire. The downloaded Fabric evidence retains the original deadline and cause in either case; this Fabric change alone does not fix Harbor's phase ownership. + +Cancellation takes precedence. An invocation with status `cancelled` becomes a trial execution error and retains its normalized status; it does not interrupt the Harbor job. Actual orchestration cancellation still propagates as `asyncio.CancelledError`. ## Finalized Telemetry Artifacts diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/fabric_agent.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/fabric_agent.py index 4b24ae9c0..484d7da80 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/fabric_agent.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/fabric_agent.py @@ -1057,8 +1057,14 @@ def raise_run_failure(status: str, error: dict[str, Any] | None) -> NoReturn: # An invocation outcome must not cancel Harbor's job orchestration. # Actual task cancellation still propagates from the awaited operations. raise RuntimeError(message) - if error is not None and error.get("code") == "timeout": - raise TimeoutError(message) + if ( + error is not None + and error.get("code") == "timeout" + and error.get("stage") == "invoke" + ): + from harbor.trial.errors import AgentTimeoutError + + raise AgentTimeoutError(message) raise RuntimeError(message) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/models.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/models.py index 6e15270c6..f59db854f 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/models.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/models.py @@ -7,6 +7,7 @@ from pathlib import PurePosixPath import re +from typing import Any from typing import Literal from typing import Self @@ -75,6 +76,7 @@ class FabricRunnerError(BaseModel): code: str | None = None message: str retryable: bool = False + details: dict[str, Any] = Field(default_factory=dict) class FabricRunnerFailure(BaseModel): diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/runner.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/runner.py index cb19dfea5..eeb7bff4a 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/runner.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/runner.py @@ -107,6 +107,7 @@ def main() -> None: else f"Fabric runner failed ({type(error).__name__})" ), retryable=error.retryable if isinstance(error, FabricError) else False, + details=error.details if isinstance(error, FabricError) else {}, ) args.result.write_text( FabricRunnerFailure(runner_error=diagnostic).model_dump_json(indent=2), diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index f29e0bbad..c017592ee 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -1017,6 +1017,7 @@ def run() -> dict[str, Any]: or "runtime_stop_failed", "message": str(error) or "runtime shutdown failed", "retryable": False, + "metadata": _runtime_error(error, stage="stop").details, } stop_events = [] if result is not None and isinstance(stop_events, list): diff --git a/skills/nemo-fabric-build-adapter/SKILL.md b/skills/nemo-fabric-build-adapter/SKILL.md index 9dfc2389c..81c4b683c 100644 --- a/skills/nemo-fabric-build-adapter/SKILL.md +++ b/skills/nemo-fabric-build-adapter/SKILL.md @@ -139,6 +139,7 @@ one `stop` for each NeMo Fabric runtime. - Accept `AgentRunRequest` and `RuntimeContext`, then return one `AgentRunResult` from `invoke`. - Make `stop` safe after partial startup and failed invocation. +- Normalize execution deadlines to canonical error code `timeout` at the adapter boundary. Preserve the owner, enforced limit or absolute deadline, and stable native cause in lifecycle error metadata or normalized error extensions. Do not classify transport or setup failures as invocation deadlines. - Isolate mutable state between independent runtimes. - If the descriptor declares `capabilities.streaming`, implement `async invoke_openai_stream(request, context, emit)`. Execute the target exactly once, diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 1c8cc99e5..45fae5e5b 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -97,6 +97,8 @@ Core host-operation deadlines (`host_timeout`) become Python `TimeoutError`, the If invocation succeeds in `Fabric.run()` but stop reaches this deadline, it returns a failed `RunResult` with `error.stage == "stop"` and `error.code == "timeout"`. Check the returned result as well as raised errors. +Adapter lifecycle failures with canonical `code == "timeout"` also preserve deadline identity. Raised `FabricRuntimeError.details` retains structured source metadata: runtime host deadlines report `owner: "fabric_runtime"`, `timeout_seconds`, and `cause_code: "host_timeout"`; adapter deadlines retain their adapter-owned limit and cause. Native diagnostic messages remain available through exception chaining. + Adapters can report execution deadlines in normalized results with `result.error.code == "timeout"`. Other adapter-specific timeout codes are not automatically reclassified. Do not classify deadlines from error text. diff --git a/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py b/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py index 728ed1f4c..6642678f0 100644 --- a/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py +++ b/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py @@ -45,6 +45,16 @@ async def invoke( "hermes_runtime_not_started", "shim runtime is not started", ) + if request.context.get("normalized_deadline"): + raise lifecycle.LifecycleError( + "timeout", + "Adapter invocation exceeded 300 seconds", + metadata={ + "owner": "adapter", + "timeout_seconds": 300, + "cause_code": "strands_prompt_timeout", + }, + ) pid_file = request.context.get("child_pid_file") if pid_file is not None: # Start a direct child in its own session (POSIX) and a grandchild diff --git a/tests/python/test_harbor_outcomes.py b/tests/python/test_harbor_outcomes.py index 0602b8ec3..448b66c86 100644 --- a/tests/python/test_harbor_outcomes.py +++ b/tests/python/test_harbor_outcomes.py @@ -371,12 +371,14 @@ async def test_completed_wrong_answer_is_not_an_execution_failure(bridge): @pytest.mark.parametrize( - ("code", "exception"), - [("timeout", TimeoutError), ("connection_failed", RuntimeError)], + ("code", "is_timeout"), + [("timeout", True), ("connection_failed", False)], ) -async def test_bridge_classifies_only_canonical_deadline(bridge, code, exception): +async def test_bridge_classifies_only_canonical_deadline(bridge, code, is_timeout): from harbor.models.agent.context import AgentContext + from harbor.trial.errors import AgentTimeoutError + exception = AgentTimeoutError if is_timeout else RuntimeError agent, environment, document = bridge document["status"] = "failed" document["error"] = { @@ -392,7 +394,7 @@ async def test_bridge_classifies_only_canonical_deadline(bridge, code, exception @pytest.mark.parametrize( ("code", "exception"), - [("timeout", TimeoutError), ("configuration_failed", RuntimeError)], + [("timeout", RuntimeError), ("configuration_failed", RuntimeError)], ) async def test_bridge_retains_runner_failure_record(bridge, code, exception): from harbor.models.agent.context import AgentContext @@ -410,7 +412,10 @@ async def test_bridge_retains_runner_failure_record(bridge, code, exception): with pytest.raises(exception): await agent.run("test", environment, context) agent.populate_context_post_run(context) - assert context.metadata["fabric"]["runner_error"] == document["runner_error"] + assert context.metadata["fabric"]["runner_error"] == { + "details": {}, + **document["runner_error"], + } @pytest.mark.parametrize("runner_failure", [False, True]) @@ -455,8 +460,10 @@ async def test_inner_deadline_enters_harbor_agent_timeout_path( user=None, ) - assert type(caught.value.__cause__) is TimeoutError - assert "inner invocation deadline" in str(caught.value.__cause__) + # Older Harbor releases wrap this error; ownership-aware versions retain it. + inner = caught.value.__cause__ or caught.value + assert isinstance(inner, AgentTimeoutError) + assert "inner invocation deadline" in str(inner) assert target.agent_execution.finished_at is not None agent.populate_context_post_run(target.agent_result) failure = target.agent_result.metadata["fabric"] @@ -484,6 +491,7 @@ def test_runner_serializes_structured_deadline(runner_cli, monkeypatch, stage): "code": "timeout", "message": "deadline", "retryable": False, + "details": {}, } @@ -510,3 +518,100 @@ def test_runner_invalid_spec_preserves_safe_evidence(runner_cli): runner.main() assert "secret-sentinel" not in runner_cli.read_text() assert "ValidationError" in runner_cli.read_text() + + +@pytest.mark.parametrize("stage", ["start", "invoke", "stop"]) +async def test_bridge_deadline_policy_preserves_owner_and_cause(bridge, stage): + from harbor.models.agent.context import AgentContext + from harbor.trial.errors import AgentTimeoutError + + agent, environment, document = bridge + document.clear() + document["runner_error"] = { + "stage": stage, + "code": "timeout", + "message": "Fabric runtime invoke deadline expired after 300 seconds", + "retryable": False, + "details": { + "owner": "fabric_runtime", + "timeout_seconds": 300, + "cause_code": "host_timeout", + }, + } + context = AgentContext() + expected = AgentTimeoutError if stage == "invoke" else RuntimeError + with pytest.raises(expected) as caught: + async with asyncio.timeout(1200): + await agent.run("test", environment, context) + assert type(caught.value) is expected + assert "300 seconds" in str(caught.value) + assert "1200" not in str(caught.value) + agent.populate_context_post_run(context) + assert context.metadata["fabric"]["runner_error"] == { + "details": {}, + **document["runner_error"], + } + + +@pytest.mark.parametrize("runner_failure", [False, True]) +@pytest.mark.parametrize( + "code,stage,verifier_calls", + [ + ("timeout", "invoke", 1), + ("connection_failed", "invoke", 0), + ("timeout", "start", 0), + ("timeout", "stop", 0), + ], +) +async def test_deadline_verifier_policy_and_cleanup( + bridge, runner_failure, code, stage, verifier_calls +): + from functools import partial + from harbor.models.agent.context import AgentContext + from harbor.models.task.config import TaskConfig + from harbor.models.task.task import Task + from harbor.models.trial.result import TrialResult + from harbor.trial.single_step import SingleStepTrial + + agent, environment, document = bridge + error = { + "stage": stage, + "code": code, + "message": "adapter invocation exceeded 300 seconds", + "retryable": False, + } + if runner_failure: + document.clear() + document["runner_error"] = error + else: + document.update(status="failed", error=error) + trial = MagicMock(spec=SingleStepTrial) + trial.result = MagicMock(spec=TrialResult) + trial.result.agent_result = AgentContext() + trial.task = MagicMock(spec=Task) + trial.task.config = TaskConfig() + trial.task.instruction = "test" + trial._load_trajectory = None + trial._agent_timeout_sec = 1200 + trial._run_agent.side_effect = partial(SingleStepTrial._run_agent, trial) + + # Model Harbor's ownership-preserving phase contract. Its upstream fix + # is tracked separately in harbor-framework/harbor#3376. + async def phase(**kwargs): + async with asyncio.timeout(kwargs["timeout_sec"]): + await agent.run("test", environment, kwargs["target"].agent_result) + + trial._run_agent_phase.side_effect = phase + if verifier_calls: + await SingleStepTrial._run(trial) + else: + with pytest.raises(RuntimeError): + await SingleStepTrial._run(trial) + assert trial._run_verifier.await_count == verifier_calls + trial._sync_agent_output.assert_awaited_once() + if verifier_calls: + failure = trial._record_exception.call_args.args[0] + assert "300 seconds" in str(failure) + assert "1200" not in str(failure) + else: + trial._record_exception.assert_not_called() diff --git a/tests/python/test_native_sdk.py b/tests/python/test_native_sdk.py index f571159b7..6d36f8cf9 100644 --- a/tests/python/test_native_sdk.py +++ b/tests/python/test_native_sdk.py @@ -137,6 +137,11 @@ async def test_native_deadline_keeps_type_and_evicts_runtime( assert caught.value.code == "timeout" assert caught.value.stage == "invoke" assert isinstance(caught.value.__cause__, TimeoutError) + assert caught.value.details == { + "owner": "fabric_runtime", + "timeout_seconds": 5.0, + "cause_code": "host_timeout", + } assert runtime.status == RuntimeStatus.FAILED with pytest.raises(FabricStateError): await runtime.invoke(input="retry") @@ -188,6 +193,11 @@ def test_runner_real_inner_deadline_preserves_failure_evidence( failure = FabricRunnerFailure.model_validate_json(result.read_text()) assert failure.runner_error.code == "timeout" assert failure.runner_error.stage == "invoke" + assert failure.runner_error.details == { + "owner": "fabric_runtime", + "timeout_seconds": 1.0, + "cause_code": "host_timeout", + } assert not failure.runner_error.retryable @@ -437,3 +447,22 @@ async def smoke(client: Fabric, fixture_agent: Path) -> None: assert empty_chunks == [] assert empty_result.status == "succeeded" assert empty_result.output["openai_stream_invocation_count"] == 2 + + +async def test_adapter_deadline_keeps_normalized_code_and_native_cause( + hermes_shim_agent_dir, +): + with pytest.raises(FabricRuntimeError) as caught: + await Fabric().run( + hermes_shim_config(), + base_dir=hermes_shim_agent_dir, + request=RunRequest(input="test", context={"normalized_deadline": True}), + ) + assert caught.value.code == "timeout" + assert caught.value.stage == "invoke" + assert caught.value.details == { + "owner": "adapter", + "timeout_seconds": 300, + "cause_code": "strands_prompt_timeout", + } + assert isinstance(caught.value.__cause__, TimeoutError) diff --git a/tests/python/test_runtime.py b/tests/python/test_runtime.py index 0934a31fa..d2a514481 100644 --- a/tests/python/test_runtime.py +++ b/tests/python/test_runtime.py @@ -205,16 +205,25 @@ async def test_one_shot_deadline_preserves_stage_and_cleanup( async def test_one_shot_stop_deadline_has_canonical_result_error( native_client, mock_native, structured ): - mock_native.stop_runtime.side_effect = ( - FabricRuntimeError("deadline", stage="stop", code="timeout") + details = { + "owner": "fabric_runtime", + "timeout_seconds": 300, + "cause_code": "host_timeout", + } + error = ( + FabricRuntimeError("deadline", stage="stop", code="timeout", details=details) if structured else TimeoutError("deadline") ) + error.details = details + mock_native.stop_runtime.side_effect = error result = await native_client.run(_config(), input="test") assert result.status == "failed" assert result.error.stage == "stop" assert result.error.code == "timeout" + assert result.error.metadata == details + @pytest.mark.parametrize("stage", ["start", "invoke"]) async def test_one_shot_preserves_existing_structured_error( @@ -1158,3 +1167,20 @@ async def test_native_unavailable_uses_typed_error(monkeypatch: pytest.MonkeyPat with pytest.raises(FabricNativeUnavailableError, match="native extension"): Fabric().plan(_config()) + + +async def test_native_deadline_details_survive_sdk_and_cleanup( + native_client, mock_native +): + native_error = TimeoutError("host invocation exceeded 300 seconds") + native_error.details = { + "owner": "fabric_runtime", + "timeout_seconds": 300, + "cause_code": "host_timeout", + } + mock_native.invoke_runtime.side_effect = native_error + with pytest.raises(FabricRuntimeError) as caught: + await native_client.run(_config(), input="test") + assert caught.value.details == native_error.details + assert caught.value.__cause__ is native_error + mock_native.stop_runtime.assert_called_once() From d8363060cfad649836dfbb07a8bf0c002f1598fb Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Tue, 3 Nov 2026 21:58:11 +0000 Subject: [PATCH 02/10] fix: normalize Kiro and Kilo invocation deadlines Signed-off-by: Ajay Thorve --- adapters/typescript/kilo/README.md | 5 ++++ adapters/typescript/kilo/src/kilo-sdk.ts | 10 +++++++- adapters/typescript/kilo/src/runtime.ts | 9 ++++---- .../typescript/kilo/test/kilo-sdk.test.mjs | 10 ++++++-- .../typescript/kilo/test/runtime.test.mjs | 23 ++++++++++++++++++- adapters/typescript/kiro/README.md | 5 ++++ adapters/typescript/kiro/src/kiro-acp.ts | 17 ++++++++++---- adapters/typescript/kiro/src/runtime.ts | 7 +++++- .../typescript/kiro/test/kiro-acp.test.mjs | 9 ++++++++ .../typescript/kiro/test/runtime.test.mjs | 10 ++++++-- 10 files changed, 90 insertions(+), 15 deletions(-) diff --git a/adapters/typescript/kilo/README.md b/adapters/typescript/kilo/README.md index 473d01971..df967cf83 100644 --- a/adapters/typescript/kilo/README.md +++ b/adapters/typescript/kilo/README.md @@ -155,3 +155,8 @@ credentials, or MCP connectivity. Checks never send prompts or abort a turn. At most one native health request remains outstanding after a caller timeout. Refer to the [SDK health-check overview](../../../docs/sdk/health-checks.mdx) for shared behavior and a comparison of native evidence across adapters. + + +## Invocation deadlines + +Kilo invocation deadlines use canonical error code `timeout`. Error metadata retains `owner: "adapter"`, the native cause, and the enforced timeout in seconds or absolute `deadline_millis`. Setup and transport failures retain their existing classifications. diff --git a/adapters/typescript/kilo/src/kilo-sdk.ts b/adapters/typescript/kilo/src/kilo-sdk.ts index dc287bbcb..c2304d0a2 100644 --- a/adapters/typescript/kilo/src/kilo-sdk.ts +++ b/adapters/typescript/kilo/src/kilo-sdk.ts @@ -607,7 +607,15 @@ class KiloSdkSessionHandle implements KiloSessionHandle { } catch { if (timedOut) { await abortPromise; - throw new LifecycleError("kilo_prompt_timeout", "Timed out while waiting for Kilo Code to complete the prompt", {retryable: false}); + throw new LifecycleError( + "timeout", + `Kilo adapter invocation timed out after ${this.promptTimeoutMs / 1000} seconds`, + {metadata: { + owner: "adapter", + timeout_seconds: this.promptTimeoutMs / 1000, + cause_code: "kilo_prompt_timeout", + }}, + ); } throw new LifecycleError("kilo_session_failed", "Kilo Code session communication failed", {retryable: false}); } finally { diff --git a/adapters/typescript/kilo/src/runtime.ts b/adapters/typescript/kilo/src/runtime.ts index 24eb3f509..4328cf68b 100644 --- a/adapters/typescript/kilo/src/runtime.ts +++ b/adapters/typescript/kilo/src/runtime.ts @@ -7,6 +7,7 @@ import type { AgentRunRequest, AgentRunResult, AgentUsage, + JsonObject, RuntimeContext, } from "nemo-fabric-adapter-contract"; import { @@ -19,7 +20,7 @@ import { /** Failed communication retains its usage snapshot while the session is retired. */ export class KiloPromptError extends LifecycleError { constructor(error: LifecycleError, readonly usage: AgentUsage) { - super(error.code, error.message, { retryable: error.retryable }); + super(error.code, error.message, { retryable: error.retryable, metadata: error.metadata }); } } @@ -27,8 +28,8 @@ export interface KiloPromptOutcome { text?: string; error?: boolean; usage?: Age export interface KiloSessionHandle { id: string; check(level: CheckLevel): Promise; prompt(text: string, deadlineMillis?: number): Promise; stop(): Promise } export interface KiloSessionFactory { create(input: AdapterStartInput): Promise } -function failed(code: string, message: string, usage?: AgentUsage): AgentRunResult { - return {status: "failed", output: null, error: {code, message, retryable: false}, ...(usage === undefined ? {} : {usage})}; +function failed(code: string, message: string, usage?: AgentUsage, metadata?: JsonObject): AgentRunResult { + return {status: "failed", output: null, error: {code, message, retryable: false, ...(metadata === undefined ? {} : {extensions: metadata})}, ...(usage === undefined ? {} : {usage})}; } export class KiloAdapterRuntime implements AdapterRuntime { @@ -57,7 +58,7 @@ export class KiloAdapterRuntime implements AdapterRuntime { this.generation += 1; this.session = undefined; try { await session.stop(); } catch { /* Preserve the prompt failure. */ } - if (error instanceof KiloPromptError) return failed(error.code, error.message, error.usage); + if (error instanceof KiloPromptError) return failed(error.code, error.message, error.usage, error.metadata); if (error instanceof LifecycleError) throw error; throw new LifecycleError("kilo_session_failed", "Kilo Code session communication failed", {retryable: true}); } finally { diff --git a/adapters/typescript/kilo/test/kilo-sdk.test.mjs b/adapters/typescript/kilo/test/kilo-sdk.test.mjs index 141743f2a..2d2f7bcfb 100644 --- a/adapters/typescript/kilo/test/kilo-sdk.test.mjs +++ b/adapters/typescript/kilo/test/kilo-sdk.test.mjs @@ -252,7 +252,10 @@ test("aborts the Kilo session when a prompt reaches its deadline", async () => { }, }, }); - await assert.rejects(handle.prompt("wait forever"), (error) => error.code === "kilo_prompt_timeout"); + await assert.rejects(handle.prompt("wait forever"), { + code: "timeout", + metadata: {owner: "adapter", timeout_seconds: 0, cause_code: "kilo_prompt_timeout"}, + }); assert.equal(promptSignal.aborted, true); assert.deepEqual(abortParameters, {sessionID: "session-timeout", directory: workspace}); assert.ok(abortSignal instanceof AbortSignal); @@ -338,7 +341,10 @@ for (const phase of ["headers", "body"]) { }); const prompting = handle.prompt("wait for the result"); if (expire) { - const rejected = assert.rejects(prompting, (error) => error.code === "kilo_prompt_timeout"); + const rejected = assert.rejects(prompting, { + code: "timeout", + metadata: {owner: "adapter", timeout_seconds: 0.5, cause_code: "kilo_prompt_timeout"}, + }); await requestReady; await rejected; await Promise.all([sessionAborted, requestClosed]); diff --git a/adapters/typescript/kilo/test/runtime.test.mjs b/adapters/typescript/kilo/test/runtime.test.mjs index c33a50237..685f12273 100644 --- a/adapters/typescript/kilo/test/runtime.test.mjs +++ b/adapters/typescript/kilo/test/runtime.test.mjs @@ -4,7 +4,8 @@ import assert from "node:assert/strict"; import test from "node:test"; -import {KiloAdapterRuntime} from "../dist/runtime.js"; +import {KiloAdapterRuntime, KiloPromptError} from "../dist/runtime.js"; +import {LifecycleError} from "nemo-fabric-adapters-common"; const context = {runtime_id: "r", invocation_id: "i", request_id: "q", artifacts: {}, environment: {environment_id: "e", provider: "local", ownership: "caller_owned", control_location: "external_control", env: {}}}; @@ -36,3 +37,23 @@ test("invalidates the runtime after transport failure", async () => { await assert.rejects(runtime.invoke({input: "two"}, context), (error) => error.code === "kilo_not_started"); assert.equal(stopped, 1); }); + + +test("preserves deadline ownership and usage through normalized failure and cleanup", async () => { + const metadata = {owner: "adapter", timeout_seconds: 300, cause_code: "kilo_prompt_timeout"}; + const usage = {input_tokens: 1, output_tokens: 2}; + let stopped = 0; + const runtime = new KiloAdapterRuntime({create: async () => ({ + id: "s", + prompt: async () => { throw new KiloPromptError(new LifecycleError("timeout", "Kilo deadline after 300 seconds", {metadata}), usage); }, + stop: async () => { stopped += 1; }, + })}); + await runtime.start({agentName: "agent", baseDir: ".", config: {}, runtimeContext: context}); + const result = await runtime.invoke({input: "test"}, context); + assert.equal(result.error.code, "timeout"); + assert.deepEqual(result.error.extensions, metadata); + assert.deepEqual(result.usage, usage); + assert.equal(stopped, 1); + await runtime.stop(); + assert.equal(stopped, 1); +}); diff --git a/adapters/typescript/kiro/README.md b/adapters/typescript/kiro/README.md index 3c2c00696..2ad246890 100644 --- a/adapters/typescript/kiro/README.md +++ b/adapters/typescript/kiro/README.md @@ -73,3 +73,8 @@ live, and a stopped or exited process fails the liveness observation. `active` and `ready` are unsupported. Checks do not send prompts, change session settings, or cancel invocations. Refer to the [SDK health-check overview](../../../docs/sdk/health-checks.mdx) for shared behavior and a comparison of native evidence across adapters. + + +## Invocation deadlines + +Kiro invocation deadlines use canonical error code `timeout`. Error metadata retains `owner: "adapter"`, the native cause, and the enforced timeout in seconds or absolute `deadline_millis`. Setup and transport failures retain their existing classifications. diff --git a/adapters/typescript/kiro/src/kiro-acp.ts b/adapters/typescript/kiro/src/kiro-acp.ts index 2aeae706e..09acbbd0d 100644 --- a/adapters/typescript/kiro/src/kiro-acp.ts +++ b/adapters/typescript/kiro/src/kiro-acp.ts @@ -55,7 +55,7 @@ function childEnvironment(input: AdapterStartInput): NodeJS.ProcessEnv { return environment; } -function translateRunnerError(error: AcpRunnerError, stage: "start" | "prompt"): LifecycleError { +function translateRunnerError(error: AcpRunnerError, stage: "start" | "prompt", timeoutMs = 300_000): LifecycleError { if (error.code === "acp_process_unavailable") { return new LifecycleError("kiro_harness_unavailable", "The installed Kiro CLI could not be started"); } @@ -78,7 +78,16 @@ function translateRunnerError(error: AcpRunnerError, stage: "start" | "prompt"): ); } if (error.code === "acp_prompt_timeout") { - return new LifecycleError("kiro_prompt_timeout", "Kiro did not finish the invocation before the deadline"); + return new LifecycleError( + "timeout", + `Kiro adapter invocation timed out after ${timeoutMs / 1000} seconds`, + { metadata: { + owner: "adapter", + timeout_seconds: timeoutMs / 1000, + cause_code: "kiro_prompt_timeout", + native_code: error.code, + } }, + ); } return new LifecycleError( stage === "start" ? "kiro_start_failed" : "kiro_session_failed", @@ -185,12 +194,12 @@ class KiroAcpSession implements KiroSessionHandle { }; } - async prompt(text: string, timeoutMs?: number): Promise { + async prompt(text: string, timeoutMs = 300_000): Promise { let result: AcpPromptResult; try { result = await this.session.prompt(text, timeoutMs); } catch (error) { - if (error instanceof AcpRunnerError) throw translateRunnerError(error, "prompt"); + if (error instanceof AcpRunnerError) throw translateRunnerError(error, "prompt", timeoutMs); throw new LifecycleError("kiro_session_failed", "Kiro ACP session communication failed"); } let usage: AgentUsage | undefined; diff --git a/adapters/typescript/kiro/src/runtime.ts b/adapters/typescript/kiro/src/runtime.ts index 61d88875d..f66c2acce 100644 --- a/adapters/typescript/kiro/src/runtime.ts +++ b/adapters/typescript/kiro/src/runtime.ts @@ -89,9 +89,14 @@ export class KiroAdapterRuntime implements AdapterRuntime { status: "failed", output: null, error: { - code: "kiro_deadline_exceeded", + code: "timeout", message: "The Kiro invocation deadline has already elapsed", retryable: false, + extensions: { + owner: "adapter", + deadline_millis: context.deadline_millis, + cause_code: "kiro_deadline_exceeded", + }, }, }; } diff --git a/adapters/typescript/kiro/test/kiro-acp.test.mjs b/adapters/typescript/kiro/test/kiro-acp.test.mjs index 7bb481a31..6319444b9 100644 --- a/adapters/typescript/kiro/test/kiro-acp.test.mjs +++ b/adapters/typescript/kiro/test/kiro-acp.test.mjs @@ -108,6 +108,15 @@ for (const fixtureType of process.platform === "win32" ? ["executable"] : ["shel total_tokens: 5, }); } + await assert.rejects(session.prompt("hang", 0), { + code: "timeout", + metadata: { + owner: "adapter", + timeout_seconds: 0, + cause_code: "kiro_prompt_timeout", + native_code: "acp_prompt_timeout", + }, + }); } finally { await session.stop(); assert.equal((await session.check("live")).reason_code, "session_closed"); diff --git a/adapters/typescript/kiro/test/runtime.test.mjs b/adapters/typescript/kiro/test/runtime.test.mjs index 3f919069d..49fb47ba7 100644 --- a/adapters/typescript/kiro/test/runtime.test.mjs +++ b/adapters/typescript/kiro/test/runtime.test.mjs @@ -98,17 +98,23 @@ test("rejects an expired deadline before dispatch and keeps the session usable", }); await runtime.start({ agentName: "agent", baseDir: ".", config: {}, runtimeContext: context }); + const deadline = Date.now() - 1; const expired = await runtime.invoke( { input: "expired" }, - { ...context, deadline_millis: Date.now() - 1 }, + { ...context, deadline_millis: deadline }, ); assert.deepEqual(expired, { status: "failed", output: null, error: { - code: "kiro_deadline_exceeded", + code: "timeout", message: "The Kiro invocation deadline has already elapsed", retryable: false, + extensions: { + owner: "adapter", + deadline_millis: deadline, + cause_code: "kiro_deadline_exceeded", + }, }, }); assert.deepEqual(prompts, []); From 02796b66ac16ca69c66bb61c10b2e8cfae10d33a Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Tue, 3 Nov 2026 22:00:08 +0000 Subject: [PATCH 03/10] fix: name expired adapter deadlines in diagnostics Signed-off-by: Ajay Thorve --- adapters/typescript/kiro/src/runtime.ts | 2 +- adapters/typescript/kiro/test/runtime.test.mjs | 2 +- adapters/typescript/strands/src/runtime.ts | 2 +- adapters/typescript/strands/test/runtime.test.mjs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/adapters/typescript/kiro/src/runtime.ts b/adapters/typescript/kiro/src/runtime.ts index f66c2acce..f7f750544 100644 --- a/adapters/typescript/kiro/src/runtime.ts +++ b/adapters/typescript/kiro/src/runtime.ts @@ -90,7 +90,7 @@ export class KiroAdapterRuntime implements AdapterRuntime { output: null, error: { code: "timeout", - message: "The Kiro invocation deadline has already elapsed", + message: `Kiro adapter invocation deadline ${context.deadline_millis} (Unix milliseconds) has already elapsed`, retryable: false, extensions: { owner: "adapter", diff --git a/adapters/typescript/kiro/test/runtime.test.mjs b/adapters/typescript/kiro/test/runtime.test.mjs index 49fb47ba7..d9a779e27 100644 --- a/adapters/typescript/kiro/test/runtime.test.mjs +++ b/adapters/typescript/kiro/test/runtime.test.mjs @@ -108,7 +108,7 @@ test("rejects an expired deadline before dispatch and keeps the session usable", output: null, error: { code: "timeout", - message: "The Kiro invocation deadline has already elapsed", + message: `Kiro adapter invocation deadline ${deadline} (Unix milliseconds) has already elapsed`, retryable: false, extensions: { owner: "adapter", diff --git a/adapters/typescript/strands/src/runtime.ts b/adapters/typescript/strands/src/runtime.ts index 8c2ebc2be..238ef37b5 100644 --- a/adapters/typescript/strands/src/runtime.ts +++ b/adapters/typescript/strands/src/runtime.ts @@ -131,7 +131,7 @@ export class StrandsAdapterRuntime implements AdapterRuntime { output: null, error: { code: "timeout", - message: "The Strands invocation deadline has already elapsed", + message: `Strands adapter invocation deadline ${context.deadline_millis} (Unix milliseconds) has already elapsed`, retryable: false, extensions: { owner: "adapter", diff --git a/adapters/typescript/strands/test/runtime.test.mjs b/adapters/typescript/strands/test/runtime.test.mjs index fa6ca73d7..14752663c 100644 --- a/adapters/typescript/strands/test/runtime.test.mjs +++ b/adapters/typescript/strands/test/runtime.test.mjs @@ -151,7 +151,7 @@ test("rejects an expired deadline before dispatch and keeps the session usable", output: null, error: { code: "timeout", - message: "The Strands invocation deadline has already elapsed", + message: `Strands adapter invocation deadline ${deadline} (Unix milliseconds) has already elapsed`, retryable: false, extensions: { owner: "adapter", From e410de03c456d881a20318fb80e1dcbd78e31c64 Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Tue, 3 Nov 2026 22:06:39 +0000 Subject: [PATCH 04/10] fix: declare adapter deadline error extensions Signed-off-by: Ajay Thorve --- .../typescript/kilo/kilo.fabric-adapter.json | 22 ++++++ .../typescript/kiro/kiro.fabric-adapter.json | 22 ++++++ .../strands/strands.fabric-adapter.json | 22 ++++++ .../nemo_fabric_adapter_catalog/catalog.json | 72 ++++++++++++++++++- .../hermes_shim/adapter.py | 10 +++ tests/python/test_native_sdk.py | 48 +++++++++++++ 6 files changed, 193 insertions(+), 3 deletions(-) diff --git a/adapters/typescript/kilo/kilo.fabric-adapter.json b/adapters/typescript/kilo/kilo.fabric-adapter.json index 4ee63c7ef..815b58500 100644 --- a/adapters/typescript/kilo/kilo.fabric-adapter.json +++ b/adapters/typescript/kilo/kilo.fabric-adapter.json @@ -45,6 +45,28 @@ "additionalProperties": false }, "extension_schemas": { + "run_error": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "owner": { + "const": "adapter" + }, + "timeout_seconds": { + "type": "number", + "minimum": 0 + }, + "cause_code": { + "const": "kilo_prompt_timeout" + } + }, + "required": [ + "owner", + "timeout_seconds", + "cause_code" + ], + "additionalProperties": false + }, "usage": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", diff --git a/adapters/typescript/kiro/kiro.fabric-adapter.json b/adapters/typescript/kiro/kiro.fabric-adapter.json index ac16a5c5c..837946eac 100644 --- a/adapters/typescript/kiro/kiro.fabric-adapter.json +++ b/adapters/typescript/kiro/kiro.fabric-adapter.json @@ -32,6 +32,28 @@ "additionalProperties": false }, "extension_schemas": { + "run_error": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "owner": { + "const": "adapter" + }, + "deadline_millis": { + "type": "integer", + "minimum": 0 + }, + "cause_code": { + "const": "kiro_deadline_exceeded" + } + }, + "required": [ + "owner", + "deadline_millis", + "cause_code" + ], + "additionalProperties": false + }, "run_result": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", diff --git a/adapters/typescript/strands/strands.fabric-adapter.json b/adapters/typescript/strands/strands.fabric-adapter.json index 550648af6..ab7e4f09f 100644 --- a/adapters/typescript/strands/strands.fabric-adapter.json +++ b/adapters/typescript/strands/strands.fabric-adapter.json @@ -106,6 +106,28 @@ ] }, "extension_schemas": { + "run_error": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "owner": { + "const": "adapter" + }, + "deadline_millis": { + "type": "integer", + "minimum": 0 + }, + "cause_code": { + "const": "strands_deadline_exceeded" + } + }, + "required": [ + "owner", + "deadline_millis", + "cause_code" + ], + "additionalProperties": false + }, "run_result": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", diff --git a/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json b/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json index 2d6fc6b90..268be022e 100644 --- a/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json +++ b/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json @@ -956,6 +956,28 @@ }, "contract_version": "fabric.adapter/v1alpha2", "extension_schemas": { + "run_error": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "cause_code": { + "const": "kilo_prompt_timeout" + }, + "owner": { + "const": "adapter" + }, + "timeout_seconds": { + "minimum": 0, + "type": "number" + } + }, + "required": [ + "owner", + "timeout_seconds", + "cause_code" + ], + "type": "object" + }, "usage": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -1128,7 +1150,7 @@ "name": "nemo-fabric-adapters-kilo", "version": "0.7.0" }, - "sha256": "6bf1bf8c3df7c62c7942c23c5b1639df54764773e2f4d945c19daf04c701a7a1", + "sha256": "bb236376f76d986c5b087952dd815e9c100dfa9ccd235964e25e8bc9f610cedd", "source": "adapters/typescript/kilo/kilo.fabric-adapter.json" }, "nvidia.fabric.kiro": { @@ -1154,6 +1176,28 @@ }, "contract_version": "fabric.adapter/v1alpha2", "extension_schemas": { + "run_error": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "cause_code": { + "const": "kiro_deadline_exceeded" + }, + "deadline_millis": { + "minimum": 0, + "type": "integer" + }, + "owner": { + "const": "adapter" + } + }, + "required": [ + "owner", + "deadline_millis", + "cause_code" + ], + "type": "object" + }, "run_result": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -1243,7 +1287,7 @@ "name": "nemo-fabric-adapters-kiro", "version": "0.7.0" }, - "sha256": "e7393a07550505f23a3396128a29f5b078492ce7f1580f6f3b79d5872e4587a2", + "sha256": "7273f357f8f8f3d429f4127d06f29653c015d20fb4d82d3c8c0f4dee9f84ff2c", "source": "adapters/typescript/kiro/kiro.fabric-adapter.json" }, "nvidia.fabric.langchain.deepagents": { @@ -3038,6 +3082,28 @@ }, "contract_version": "fabric.adapter/v1alpha2", "extension_schemas": { + "run_error": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "cause_code": { + "const": "strands_deadline_exceeded" + }, + "deadline_millis": { + "minimum": 0, + "type": "integer" + }, + "owner": { + "const": "adapter" + } + }, + "required": [ + "owner", + "deadline_millis", + "cause_code" + ], + "type": "object" + }, "run_result": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -3201,7 +3267,7 @@ "name": "nemo-fabric-adapters-strands", "version": "0.7.0" }, - "sha256": "37982ca315b241b03fce3edb1d49d2f1663e3224ebcf89824d3a976055ccd949", + "sha256": "3b6e7f1dec2fc416c30c00e24225b21a161160252b6aba3846028319478ca3c9", "source": "adapters/typescript/strands/strands.fabric-adapter.json" } }, diff --git a/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py b/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py index 6642678f0..cb35973b5 100644 --- a/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py +++ b/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py @@ -45,6 +45,16 @@ async def invoke( "hermes_runtime_not_started", "shim runtime is not started", ) + if "normalized_deadline_result" in request.context: + return AgentRunResult( + status=AgentRunStatus.FAILED, + output=None, + error=AgentRunError( + code="timeout", + message="Adapter invocation deadline expired", + extensions=request.context["normalized_deadline_result"], + ), + ) if request.context.get("normalized_deadline"): raise lifecycle.LifecycleError( "timeout", diff --git a/tests/python/test_native_sdk.py b/tests/python/test_native_sdk.py index 6d36f8cf9..98aec332d 100644 --- a/tests/python/test_native_sdk.py +++ b/tests/python/test_native_sdk.py @@ -466,3 +466,51 @@ async def test_adapter_deadline_keeps_normalized_code_and_native_cause( "cause_code": "strands_prompt_timeout", } assert isinstance(caught.value.__cause__, TimeoutError) + + +@pytest.mark.parametrize("adapter", ["strands", "kiro", "kilo"]) +@pytest.mark.parametrize("invalid_metadata", [False, True]) +async def test_committed_deadline_schema_survives_core_result_validation( + hermes_shim_agent_dir, repo_root, adapter, invalid_metadata +): + source = json.loads( + ( + repo_root / f"adapters/typescript/{adapter}/{adapter}.fabric-adapter.json" + ).read_text() + ) + descriptor_path = ( + hermes_shim_agent_dir / "adapters/hermes-shim/hermes-shim.fabric-adapter.json" + ) + descriptor = json.loads(descriptor_path.read_text()) + descriptor["extension_schemas"].update(source["extension_schemas"]) + descriptor_path.write_text(json.dumps(descriptor)) + metadata = ( + { + "owner": "adapter", + "timeout_seconds": 300, + "cause_code": "kilo_prompt_timeout", + } + if adapter == "kilo" + else { + "owner": "adapter", + "deadline_millis": 300_000, + "cause_code": f"{adapter}_deadline_exceeded", + } + ) + if invalid_metadata: + metadata["undeclared"] = "must not cross the runtime boundary" + operation = Fabric().run( + hermes_shim_config(), + base_dir=hermes_shim_agent_dir, + request=RunRequest( + input="deadline", context={"normalized_deadline_result": metadata} + ), + ) + if invalid_metadata: + with pytest.raises(FabricRuntimeError, match="extensions"): + await operation + else: + result = await operation + assert result.status == "failed" + assert result.error.code == "timeout" + assert result.error.metadata == metadata From bfe5818094e1534d77af6947b171252def4d7b8e Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Tue, 3 Nov 2026 22:35:00 +0000 Subject: [PATCH 05/10] docs: regenerate Fabric error metadata reference Signed-off-by: Ajay Thorve --- .../nemo-fabric-core/error/enum-fabricerror.mdx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index 18ad7379a..4f4fb4e94 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentInUse {\n        environment_id: String,\n        runtime_id: String,\n    },\n    EnvironmentReleaseInProgress {\n        environment_id: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    ResourceObservationUnavailable {\n        runtime_id: String,\n        reason: String,\n    },\n    ResourceObservationUnsupported {\n        runtime_id: String,\n        reason: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
+
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n        metadata: BTreeMap<String, Value>,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentInUse {\n        environment_id: String,\n        runtime_id: String,\n    },\n    EnvironmentReleaseInProgress {\n        environment_id: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    ResourceObservationUnavailable {\n        runtime_id: String,\n        reason: String,\n    },\n    ResourceObservationUnsupported {\n        runtime_id: String,\n        reason: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
Errors raised by NeMo Fabric config loading and validation. @@ -405,6 +405,10 @@ Human-readable failure message. Bounded adapter-host diagnostics. +##### `metadata: BTreeMap` + +Structured adapter or runtime failure evidence. + ### `RuntimeHandleMismatch`
From d0eff413df48281b0bf0670200fcb74b5e21bbf4 Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Tue, 3 Nov 2026 22:58:48 +0000 Subject: [PATCH 06/10] fix: retain timeout evidence across thread and shutdown boundaries Signed-off-by: Ajay Thorve --- adapters/typescript/kilo/README.md | 2 +- crates/fabric-core/src/runtime.rs | 45 +++++++++++++------ .../src/nemo_fabric/runtime.py | 15 ++++++- tests/python/test_runtime.py | 17 +++++-- 4 files changed, 59 insertions(+), 20 deletions(-) diff --git a/adapters/typescript/kilo/README.md b/adapters/typescript/kilo/README.md index df967cf83..6d790f328 100644 --- a/adapters/typescript/kilo/README.md +++ b/adapters/typescript/kilo/README.md @@ -76,7 +76,7 @@ authentication and per-server tool filters are unsupported. The adapter accepts plain-text input and returns the terminal assistant text in `output.response`. Ordered invocations reuse one warm Kilo Code session and its conversation history. A prompt has a 30-minute deadline; on timeout the adapter -asks Kilo Code to abort the prompt and reports `kilo_prompt_timeout`. +asks Kilo Code to abort the prompt and reports `timeout`, retaining `kilo_prompt_timeout` as the cause. The adapter disables project Kilo Code configuration and uses temporary Kilo and XDG directories. The child process receives required operating-system diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index a831b6e9a..194ca0372 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -931,7 +931,10 @@ pub fn run_plan(plan: &RunPlan, request: RunRequest) -> Result { code: "runtime_stop_failed".to_string(), message: error.to_string(), retryable: false, - metadata: BTreeMap::new(), + metadata: match &error { + FabricError::AdapterLifecycleOperation { metadata, .. } => metadata.clone(), + _ => BTreeMap::new(), + }, }); } Err(_) => {} @@ -4538,6 +4541,13 @@ for line in sys.stdin: with open(os.environ["FABRIC_FAKE_HOST_STOP_MARKER"], "w", encoding="utf-8") as stream: stream.write("started") time.sleep(0.25) + if MODE == "stop_deadline": + error = failure("stop", "timeout", "adapter stop deadline expired") + error["metadata"] = { + "owner": "adapter", "timeout_seconds": 300, "cause_code": "adapter_stop_timeout", + } + response("stop", error=error) + sys.exit(18) if MODE == "stop_failure": response("stop", error=failure("stop", "fake_stop", "stop rejected")) sys.exit(18) @@ -6517,19 +6527,28 @@ for line in sys.stdin: #[test] fn run_plan_preserves_completed_result_when_stop_fails() { - let (root, plan) = local_host_plan("stop_failure"); - - let result = run_plan(&plan, RunRequest::text("completed")) - .expect("completed result with stop failure"); - - assert_eq!(result.status, RunStatus::Failed); - assert_eq!(result.output["input"], "completed"); - let error = result.error.expect("stop error"); - assert_eq!(error.stage, ErrorStage::Stop); - assert_eq!(error.code, "runtime_stop_failed"); - assert!(error.message.contains("fake_stop"), "{}", error.message); + for (mode, cause) in [("stop_failure", "fake_stop"), ("stop_deadline", "timeout")] { + let (root, plan) = local_host_plan(mode); + + let result = run_plan(&plan, RunRequest::text("completed")) + .expect("completed result with stop failure"); + + assert_eq!(result.status, RunStatus::Failed); + assert_eq!(result.output["input"], "completed"); + let error = result.error.expect("stop error"); + assert_eq!(error.stage, ErrorStage::Stop); + assert_eq!(error.code, "runtime_stop_failed"); + assert!(error.message.contains(cause), "{}", error.message); + if mode == "stop_deadline" { + assert_eq!(error.metadata["owner"], "adapter"); + assert_eq!(error.metadata["timeout_seconds"], 300); + assert_eq!(error.metadata["cause_code"], "adapter_stop_timeout"); + } else { + assert!(error.metadata.is_empty()); + } - let _ = fs::remove_dir_all(root); + let _ = fs::remove_dir_all(root); + } } #[test] fn default_environment_prepares_through_local_provider() { diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index c017592ee..96718c773 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -1052,9 +1052,20 @@ def _resource_snapshot_error(native: Any, error: Exception) -> FabricRuntimeErro async def _call_blocking(func: Any) -> Any: - worker = asyncio.create_task(asyncio.to_thread(func)) + def call() -> tuple[Any, TimeoutError | None]: + try: + return func(), None + except TimeoutError as error: + # Python 3.11/3.12's executor bridge recreates TimeoutError, losing + # its details. Carry the original as a result across that boundary. + return None, error + + worker = asyncio.create_task(asyncio.to_thread(call)) try: - return await asyncio.shield(worker) + result, timeout_error = await asyncio.shield(worker) + if timeout_error is not None: + raise timeout_error + return result except asyncio.CancelledError as cancelled: # Re-awaiting shield in a cancelled task can prevent the worker result # from becoming observable, so let the event loop finish it first. diff --git a/tests/python/test_runtime.py b/tests/python/test_runtime.py index d2a514481..99e528969 100644 --- a/tests/python/test_runtime.py +++ b/tests/python/test_runtime.py @@ -166,22 +166,31 @@ def _runtime_wrapper( async def test_deadline_identity_survives_stateful_sdk_boundaries( native_client, mock_native, stage, error_type, expected_code ): + error = error_type("deadline-looking text") + details = { + "owner": "fabric_runtime", + "timeout_seconds": 300, + "cause_code": "host_timeout", + } + if expected_code == "timeout": + error.details = details if stage == "start": - mock_native.start_runtime.side_effect = error_type("deadline-looking text") + mock_native.start_runtime.side_effect = error operation = native_client.start_runtime(_config()) else: runtime = await native_client.start_runtime(_config()) if stage == "invoke": - mock_native.invoke_runtime.side_effect = error_type("deadline-looking text") + mock_native.invoke_runtime.side_effect = error operation = runtime.invoke(input="test") else: - mock_native.stop_runtime.side_effect = error_type("deadline-looking text") + mock_native.stop_runtime.side_effect = error operation = runtime.stop() with pytest.raises(FabricRuntimeError) as caught: await operation assert caught.value.stage == stage assert caught.value.code == expected_code - assert isinstance(caught.value.__cause__, error_type) + assert caught.value.__cause__ is error + assert caught.value.details == (details if expected_code == "timeout" else {}) if stage == "invoke": await runtime.stop() From ff10bb23b14a38bf1246ad1c6fc4f5a67ab79e0d Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Wed, 4 Nov 2026 04:42:39 +0000 Subject: [PATCH 07/10] docs: separate consumer skill deadline guidance Signed-off-by: Ajay Thorve --- skills/nemo-fabric-build-adapter/SKILL.md | 1 - skills/nemo-fabric-integrate/references/results-and-errors.md | 2 -- 2 files changed, 3 deletions(-) diff --git a/skills/nemo-fabric-build-adapter/SKILL.md b/skills/nemo-fabric-build-adapter/SKILL.md index 81c4b683c..9dfc2389c 100644 --- a/skills/nemo-fabric-build-adapter/SKILL.md +++ b/skills/nemo-fabric-build-adapter/SKILL.md @@ -139,7 +139,6 @@ one `stop` for each NeMo Fabric runtime. - Accept `AgentRunRequest` and `RuntimeContext`, then return one `AgentRunResult` from `invoke`. - Make `stop` safe after partial startup and failed invocation. -- Normalize execution deadlines to canonical error code `timeout` at the adapter boundary. Preserve the owner, enforced limit or absolute deadline, and stable native cause in lifecycle error metadata or normalized error extensions. Do not classify transport or setup failures as invocation deadlines. - Isolate mutable state between independent runtimes. - If the descriptor declares `capabilities.streaming`, implement `async invoke_openai_stream(request, context, emit)`. Execute the target exactly once, diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 45fae5e5b..1c8cc99e5 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -97,8 +97,6 @@ Core host-operation deadlines (`host_timeout`) become Python `TimeoutError`, the If invocation succeeds in `Fabric.run()` but stop reaches this deadline, it returns a failed `RunResult` with `error.stage == "stop"` and `error.code == "timeout"`. Check the returned result as well as raised errors. -Adapter lifecycle failures with canonical `code == "timeout"` also preserve deadline identity. Raised `FabricRuntimeError.details` retains structured source metadata: runtime host deadlines report `owner: "fabric_runtime"`, `timeout_seconds`, and `cause_code: "host_timeout"`; adapter deadlines retain their adapter-owned limit and cause. Native diagnostic messages remain available through exception chaining. - Adapters can report execution deadlines in normalized results with `result.error.code == "timeout"`. Other adapter-specific timeout codes are not automatically reclassified. Do not classify deadlines from error text. From bb8ca31e2ee81ea8b73fe136e541ad1069d754d0 Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Fri, 6 Nov 2026 18:43:49 +0000 Subject: [PATCH 08/10] test: assert deadline evidence after cleanup failure Signed-off-by: Ajay Thorve --- adapters/typescript/strands/test/strands-acp.test.mjs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/adapters/typescript/strands/test/strands-acp.test.mjs b/adapters/typescript/strands/test/strands-acp.test.mjs index cac9a9101..083e74134 100644 --- a/adapters/typescript/strands/test/strands-acp.test.mjs +++ b/adapters/typescript/strands/test/strands-acp.test.mjs @@ -245,7 +245,11 @@ test("retains a timed-out runtime session until process cleanup succeeds", { ski }); await assert.rejects( runtime.invoke({ input: "hang" }, { ...input.runtimeContext, deadline_millis: Date.now() + 50 }), - { code: "timeout" }, + (error) => + error.code === "timeout" && + error.metadata?.owner === "adapter" && + error.metadata?.cause_code === "strands_prompt_timeout" && + error.metadata?.native_code === "acp_prompt_timeout", ); originalKill(child, 0); await assert.rejects(runtime.invoke({ input: "next" }, input.runtimeContext), { code: "strands_not_started" }); From 0f88ce31f5827c36e9ba3bb3240ef3a709648818 Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Tue, 10 Nov 2026 19:28:51 +0000 Subject: [PATCH 09/10] fix: preserve timeouts when deadline detail attachment fails Signed-off-by: Ajay Thorve --- crates/fabric-python/src/lib.rs | 25 +++++++------ .../src/nemo_fabric/runtime.py | 6 ++-- tests/python/test_native_sdk.py | 36 +++++++++++++++++++ 3 files changed, 54 insertions(+), 13 deletions(-) diff --git a/crates/fabric-python/src/lib.rs b/crates/fabric-python/src/lib.rs index 8f3e7b2ef..1b8ecbee9 100644 --- a/crates/fabric-python/src/lib.rs +++ b/crates/fabric-python/src/lib.rs @@ -16,7 +16,7 @@ use nemo_fabric_core::{ resolve_diagnostic_plan_from_config_with_adapter_directories, resolve_run_plan_from_config_with_adapter_directories, run_plan, }; -use pyo3::exceptions::{PyRuntimeError, PyTimeoutError}; +use pyo3::exceptions::{PyException, PyRuntimeError, PyTimeoutError}; use pyo3::prelude::*; pyo3::create_exception!( @@ -434,17 +434,22 @@ fn to_py_error(error: nemo_fabric_core::FabricError) -> PyErr { FabricError::AdapterLifecycleOperation { code, metadata, .. } if code == "host_timeout" || code == "timeout" => { + let timeout = PyTimeoutError::new_err(error.to_string()); Python::attach(|py| { - let timeout = PyTimeoutError::new_err(error.to_string()); - let details = py.import("json")?.call_method1( - "loads", - (serde_json::to_string(metadata) - .map_err(|error| PyRuntimeError::new_err(error.to_string()))?,), - )?; - timeout.value(py).setattr("details", details)?; - Ok::<_, PyErr>(timeout) + let attachment = (|| -> PyResult<()> { + let details = py.import("json")?.call_method1( + "loads", + (serde_json::to_string(metadata) + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?,), + )?; + timeout.value(py).setattr("details", details) + })(); + // Metadata enrichment must not change the timeout classification. + match attachment { + Err(error) if !error.is_instance_of::(py) => error, + _ => timeout, + } }) - .unwrap_or_else(|error| error) } FabricError::ResourceObservationUnavailable { .. } => { ResourceObservationUnavailableError::new_err(error.to_string()) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index 5533f882a..6b39f5d6c 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -1012,14 +1012,14 @@ def run() -> dict[str, Any]: if result is None: raise if result.get("status") == "succeeded": + stop_error = _runtime_error(error, stage="stop") result["status"] = "failed" result["error"] = { "stage": "stop", - "code": _runtime_error(error, stage="stop").code - or "runtime_stop_failed", + "code": stop_error.code or "runtime_stop_failed", "message": str(error) or "runtime shutdown failed", "retryable": False, - "metadata": _runtime_error(error, stage="stop").details, + "metadata": stop_error.details, } stop_events = [] if result is not None and isinstance(stop_events, list): diff --git a/tests/python/test_native_sdk.py b/tests/python/test_native_sdk.py index 98aec332d..e47ab4033 100644 --- a/tests/python/test_native_sdk.py +++ b/tests/python/test_native_sdk.py @@ -468,6 +468,42 @@ async def test_adapter_deadline_keeps_normalized_code_and_native_cause( assert isinstance(caught.value.__cause__, TimeoutError) +@pytest.mark.parametrize( + "attachment_error_type", [RecursionError, ValueError, KeyboardInterrupt, SystemExit] +) +def test_native_deadline_attachment_failure_preserves_exception_kind( + hermes_shim_agent_dir, monkeypatch, attachment_error_type +): + plan = native.plan_config( + json.dumps(hermes_shim_config().to_mapping()), str(hermes_shim_agent_dir) + ) + runtime = native.start_runtime(plan) + request = json.dumps( + RunRequest(input="test", context={"normalized_deadline": True}).to_mapping() + ) + attachment_error = attachment_error_type("metadata attachment failed") + ordinary_error = isinstance(attachment_error, Exception) + expected = TimeoutError if ordinary_error else attachment_error_type + calls = [] + + def fail_attachment(*args, **kwargs): + calls.append(args) + raise attachment_error + + try: + with monkeypatch.context() as patch: + patch.setattr(json, "loads", fail_attachment) + with pytest.raises(expected) as caught: + native.invoke_runtime(plan, runtime, request) + assert len(calls) == 1 + if ordinary_error: + assert "300 seconds" in str(caught.value) + else: + assert caught.value is attachment_error + finally: + native.stop_runtime(plan, runtime) + + @pytest.mark.parametrize("adapter", ["strands", "kiro", "kilo"]) @pytest.mark.parametrize("invalid_metadata", [False, True]) async def test_committed_deadline_schema_survives_core_result_validation( From 023fc67f6067a82a86117879a0cce8939c17f3df Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Tue, 10 Nov 2026 23:17:29 +0000 Subject: [PATCH 10/10] docs: use NeMo Fabric name in deadline guidance Signed-off-by: Ajay Thorve --- docs/integrations/consumer/harbor.mdx | 4 ++-- .../src/nemo_fabric/integrations/harbor/README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/integrations/consumer/harbor.mdx b/docs/integrations/consumer/harbor.mdx index 78ba05d4c..e625b5660 100644 --- a/docs/integrations/consumer/harbor.mdx +++ b/docs/integrations/consumer/harbor.mdx @@ -89,6 +89,6 @@ Keep using Harbor's managed environment and redaction rules for credentials. Cor ## Execution Deadlines -Fabric runtime and normalized adapter invocation deadlines use code `timeout` at stage `invoke`. The bridge raises Harbor's `AgentTimeoutError`, so both remain eligible for the original verifier. Startup and shutdown deadlines retain runtime failure handling and skip verification. The collected `fabric-result-*.json` retains the owning layer, actual limit, and native cause in `runner_error.details` or normalized error metadata. +NeMo Fabric runtime and normalized adapter invocation deadlines use code `timeout` at stage `invoke`. The bridge raises Harbor's `AgentTimeoutError`, so both remain eligible for the original verifier. Startup and shutdown deadlines retain runtime failure handling and skip verification. The collected `fabric-result-*.json` retains the owning layer, actual limit, and native cause in `runner_error.details` or normalized error metadata. -Harbor also needs its [phase timeout ownership fix](https://github.com/harbor-framework/harbor/pull/3376) to preserve the inner exception in the trial summary. Older Harbor versions can relabel an inner 300-second deadline as the outer 1200-second task timeout even when the outer context has not expired. Fabric evidence keeps the original deadline on those versions; upgrading Fabric alone does not fix that Harbor summary. +Harbor also needs its [phase timeout ownership fix](https://github.com/harbor-framework/harbor/pull/3376) to preserve the inner exception in the trial summary. Older Harbor versions can relabel an inner 300-second deadline as the outer 1200-second task timeout even when the outer context has not expired. NeMo Fabric evidence keeps the original deadline on those versions; upgrading NeMo Fabric alone does not fix that Harbor summary. diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/README.md b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/README.md index a9475910c..9b02bf0eb 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/README.md +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/README.md @@ -81,7 +81,7 @@ Lifecycle failures before a normalized result is available produce a separate `r Only canonical `timeout` failures at stage `invoke` raise Harbor's `AgentTimeoutError` and remain eligible for verification. Startup and shutdown deadlines, ordinary runtime failures, and configuration errors raise `RuntimeError` and skip verification. Equivalent runtime and adapter invocation deadlines use the same policy. The verifier still owns the reward. -Harbor versions with [phase timeout ownership](https://github.com/harbor-framework/harbor/pull/3376) preserve the inner exception and its actual deadline. Older Harbor versions catch all nested `TimeoutError` subclasses and can still overwrite the trial exception message with their outer budget, even when that budget did not expire. The downloaded Fabric evidence retains the original deadline and cause in either case; this Fabric change alone does not fix Harbor's phase ownership. +Harbor versions with [phase timeout ownership](https://github.com/harbor-framework/harbor/pull/3376) preserve the inner exception and its actual deadline. Older Harbor versions catch all nested `TimeoutError` subclasses and can still overwrite the trial exception message with their outer budget, even when that budget did not expire. The downloaded NeMo Fabric evidence retains the original deadline and cause in either case; this NeMo Fabric change alone does not fix Harbor's phase ownership. Cancellation takes precedence. An invocation with status `cancelled` becomes a trial execution error and retains its normalized status; it does not interrupt the Harbor job. Actual orchestration cancellation still propagates as `asyncio.CancelledError`.