From 51cb586324995a86f24251b1d054c17bcbc27fef Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Thu, 5 Nov 2026 09:20:43 -0700 Subject: [PATCH 1/6] feat: add runtime status lookup to the SDK and CLI Signed-off-by: Sara Tadayon --- .agents/skills/contribute-api/SKILL.md | 1 + README.md | 2 +- crates/fabric-core/src/environment.rs | 93 ++++- crates/fabric-core/src/lib.rs | 1 + crates/fabric-core/src/runtime.rs | 39 +++ crates/fabric-core/src/strict_json.rs | 65 ++++ crates/fabric-core/src/task_command.rs | 4 + crates/fabric-python/src/lib.rs | 30 ++ .../api/python-library-reference/index.md | 1 + .../nemo_fabric.client.md | 42 +++ .../nemo_fabric.types.md | 86 +++++ docs/sdk/python.mdx | 34 +- environment-providers/openshell/src/lib.rs | 92 ++++- .../openshell/src/strict_json.rs | 65 ++++ .../src/nemo_fabric/__init__.py | 2 + .../src/nemo_fabric/_native.pyi | 3 + .../src/nemo_fabric/client.py | 198 ++++++++++- .../src/nemo_fabric/runtime.py | 2 + .../src/nemo_fabric/runtime_server/README.md | 8 +- .../src/nemo_fabric/runtime_server/cli.py | 4 +- .../nemo_fabric/runtime_server/protocol.py | 29 +- .../src/nemo_fabric/runtime_server/state.py | 23 ++ .../nemo_fabric/runtime_server/transport.py | 7 + .../src/nemo_fabric/types.py | 88 +++++ skills/nemo-fabric-integrate/SKILL.md | 2 + tests/python/runtime_server/test_status.py | 210 +++++++++++ tests/python/test_runtime.py | 327 ++++++++++++++++++ tests/python/test_sdk_server_checks.py | 203 +++++++++++ tests/python/test_streaming.py | 11 +- 29 files changed, 1650 insertions(+), 22 deletions(-) create mode 100644 crates/fabric-core/src/strict_json.rs create mode 100644 environment-providers/openshell/src/strict_json.rs create mode 100644 tests/python/runtime_server/test_status.py diff --git a/.agents/skills/contribute-api/SKILL.md b/.agents/skills/contribute-api/SKILL.md index 1a2abb67f..460af359a 100644 --- a/.agents/skills/contribute-api/SKILL.md +++ b/.agents/skills/contribute-api/SKILL.md @@ -25,6 +25,7 @@ runtime or bindings. - Keep every affected public surface in parity - `RunRequest.relay_session_root` is forwarded as the typed `AgentRunRequest.relay_session_root` field. Keep Rust, Python, TypeScript, and schema representations in parity. Context keys do not control Relay propagation; preserve the adapter fallback for unusable UUIDs. - OpenShell task commands serialize public SDK config/request/handle/result mappings. Keep forwarding buffered and preserve task-generated IDs, normalized failed results, telemetry references, and declared artifacts. Do not normalize an SDK result as an adapter result. Task paths remain POSIX strings; host artifact destinations stay separate. Unknown outcomes bind the immutable sandbox identity through reattachment and must not be replayed. +- Runtime discovery uses the typed public `Fabric.runtime_status()` operation. Keep `fabric-agent-ctl status` delegated to that SDK operation and OpenShell status routed through the existing provider command transport. Preserve recorded lifecycle state separately from health observations, use null handles for stopped records, and exclude environment variables and arbitrary provider metadata from discovery snapshots. - Update docs and examples in the same branch ## Minimum Acceptance diff --git a/README.md b/README.md index c332857e7..9b3937ec1 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ NeMo Fabric separates consumers from execution targets through a stable, version - [Installation](docs/getting-started/install.mdx): Install the runtime, adapters, harness dependencies, and telemetry components, and choose a deployment scenario. - [Adapter catalog](sdk/python/nemo-fabric-adapter-catalog/README.md): Inspect bundled adapter metadata on a host without installing harness SDKs. - [Host admission](docs/sdk/python.mdx#inspect-metadata-on-a-separate-host): Validate requested capabilities and reject host/task descriptor drift before execution. -- [NeMo Fabric runtime server](sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md): Manage runtimes and check server responsiveness inside one sandbox through `fabric-agent-ctl` and one private local socket. Unconfirmed cleanup retains sandbox ownership and blocks replacement servers. +- [NeMo Fabric runtime server](sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md): Manage runtimes, rediscover their recorded status and safe handles, and check server responsiveness inside one sandbox through `fabric-agent-ctl` and one private local socket. Unconfirmed cleanup retains sandbox ownership and blocks replacement servers. - [Health checks](docs/sdk/health-checks.mdx): Use `Fabric.check()` for OpenShell runtime-server checks, agent runtime checks, and prepared or attached OpenClaw Gateway service checks. Refer to the [adapter support table](docs/sdk/health-checks.mdx#native-evidence-by-adapter) for supported levels and native evidence. OpenShell server and runtime requests map to `fabric-agent-ctl check`; service checks use the local SDK. - [Documentation overview](docs/about-nemo-fabric/overview.mdx): Understand the execution model and choose an interface. - [Adapter reference](adapters/README.md): Compare adapter configuration, runtime, and observability support. diff --git a/crates/fabric-core/src/environment.rs b/crates/fabric-core/src/environment.rs index edb3c0a58..63b0853fe 100644 --- a/crates/fabric-core/src/environment.rs +++ b/crates/fabric-core/src/environment.rs @@ -497,7 +497,7 @@ where } } -// A dedicated provider process handles each health check and exits after the response. +// A dedicated provider process handles each read-only command and exits after the response. fn request_check_provider_process( command: OsString, operation: ProviderOperation<'_>, @@ -506,6 +506,13 @@ fn request_check_provider_process( where T: for<'de> Deserialize<'de>, { + let is_check = matches!( + &operation, + ProviderOperation::RuntimeCommand { + command: TaskCommand::Check { .. }, + .. + } + ); let started = Instant::now(); let budget = Duration::try_from_secs_f64(timeout_seconds).unwrap_or(Duration::MAX); let remaining = || { @@ -531,7 +538,9 @@ where }) .map_err(FabricError::SerializeJson)?; let seconds = remaining()?.as_secs_f64(); - request["command"]["timeout_seconds"] = Value::from(seconds); + if is_check { + request["command"]["timeout_seconds"] = Value::from(seconds); + } request["timeout_seconds"] = Value::from(seconds.ceil().clamp(1.0, 320.0) as u64); let mut encoded = serde_json::to_vec(&request).map_err(FabricError::SerializeJson)?; if encoded.len() > MAX_PROVIDER_REQUEST_BYTES { @@ -567,13 +576,30 @@ where } }; remaining()?; - if output.is_empty() || output.len() > 16 * 1024 || output.last() != Some(&b'\n') { + if output.is_empty() + || output.len() + > if is_check { + 16 * 1024 + } else { + MAX_PROVIDER_RESPONSE_BYTES + } + || output.last() != Some(&b'\n') + { return provider_error( "runtime_command", "provider_protocol_error", "invalid health response framing".into(), ); } + if !is_check { + crate::strict_json::validate(&output).map_err(|_| { + provider_failure( + "runtime_command", + "provider_protocol_error", + "invalid status response".into(), + ) + })?; + } let response: ProviderResponse = serde_json::from_slice(&output).map_err(|_| { provider_failure( "runtime_command", @@ -635,11 +661,16 @@ where let budget = *timeout_seconds; return request_check_provider_process(command, operation, budget); } + if matches!(task_command, TaskCommand::Status { .. }) { + return request_check_provider_process(command, operation, 10.0); + } let operation_name = operation.name(); let request_id = new_id("environment-provider-request"); let environment_key = environment_resource_key(environment)?; let (identity, is_start, is_stop) = match task_command { - TaskCommand::Check { .. } => unreachable!("health uses a separate provider process"), + TaskCommand::Check { .. } | TaskCommand::Status { .. } => { + unreachable!("reads use a separate provider process") + } TaskCommand::Start { .. } => (request_id.clone(), true, false), TaskCommand::Invoke { runtime_id, .. } => (runtime_id.clone(), false, false), TaskCommand::Stop { runtime_id, .. } => (runtime_id.clone(), false, true), @@ -1449,6 +1480,60 @@ mod tests { fs::remove_dir_all(directory).expect("remove fixture"); } + #[test] + fn status_provider_does_not_wait_for_an_invocation_worker() { + let (directory, command) = provider_script( + r#" +import json, sys +for line in sys.stdin: + request = json.loads(line) + assert request['command']['operation'] == 'status' + assert set(request['command']) == {'operation', 'agent', 'runtime_id'} + print(json.dumps({'protocol_version': request['protocol_version'], + 'request_id': request['request_id'], 'status': 'succeeded', + 'output': {'seen': 'status'}}), flush=True) +"#, + ); + let environment: EnvironmentHandle = serde_json::from_value(serde_json::json!({ + "environment_id": "status-worker", "provider": "openshell", + "control_location": "in_env_control", "ownership": "caller_owned", + "connection": {"gateway": "https://status.invalid", "sandbox_id": new_id("sandbox")} + })) + .expect("environment fixture"); + let key = serde_json::json!([environment_resource_key(&environment).unwrap(), "runtime-1"]) + .to_string(); + let process = runtime_provider_process(&key, &command, "runtime_command").expect("worker"); + let guard = process.lock().unwrap(); + let (sender, receiver) = mpsc::channel(); + thread::scope(|scope| { + scope.spawn(|| { + let task = TaskCommand::Status { + agent: "demo".to_string(), + runtime_id: "runtime-1".to_string(), + }; + let result = request_runtime_provider_process::( + command, + ProviderOperation::RuntimeCommand { + environment: &environment, + command: &task, + timeout_seconds: 10, + }, + ); + sender.send(result).unwrap(); + }); + let result = receiver.recv_timeout(Duration::from_secs(2)); + drop(guard); + assert_eq!( + result + .expect("status must bypass the invocation mutex") + .expect("status output")["seen"], + "status" + ); + }); + retire_runtime_provider_process(&key, &process); + fs::remove_dir_all(directory).expect("remove fixture"); + } + #[test] fn health_provider_cleanup_is_bounded_with_inherited_pipes() { for pipe in ["stdout", "stderr", "both"] { diff --git a/crates/fabric-core/src/lib.rs b/crates/fabric-core/src/lib.rs index 56a68b44e..778276cca 100644 --- a/crates/fabric-core/src/lib.rs +++ b/crates/fabric-core/src/lib.rs @@ -15,6 +15,7 @@ pub mod provenance; pub mod resource_snapshot; pub mod runtime; pub mod schema; +mod strict_json; pub mod system_stats; mod task_command; mod time; diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 21ee30ae6..5185e73fb 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -47,6 +47,45 @@ pub(crate) fn remote_check_agent( openshell::check_agent(environment, runtime_id) } +// Internal transport entrypoint for the typed Python SDK status operation. +#[doc(hidden)] +pub fn runtime_status_environment( + environment: &EnvironmentHandle, + agent: &str, + runtime_id: &str, +) -> Result { + if environment.provider != "openshell" + || environment.control_location != ControlLocation::InEnvControl + || [agent, runtime_id] + .iter() + .any(|value| value.trim().is_empty() || value.contains('\0')) + { + return Err(FabricError::InvalidConfig { + field: "runtime_status".to_string(), + reason: "a valid runtime target and OpenShell environment are required".to_string(), + }); + } + let command = crate::task_command::TaskCommand::Status { + agent: agent.to_string(), + runtime_id: runtime_id.to_string(), + }; + match crate::environment::command_runtime(environment, &command, 10)? { + crate::task_command::TaskCommandResponse::Returned { value } => Ok(value), + crate::task_command::TaskCommandResponse::Rejected { error } => { + Err(FabricError::EnvironmentProviderOperation { + provider: "openshell".to_string(), + operation: "status".to_string(), + code: if error.effects == "none" { + error.code + } else { + "invalid_response".to_string() + }, + message: "The runtime status request was rejected.".to_string(), + }) + } + } +} + static NEXT_ID: AtomicU64 = AtomicU64::new(1); const ADAPTER_PYTHON_ENV: &str = "ADAPTER_PYTHON"; const VIRTUAL_ENV_ENV: &str = "VIRTUAL_ENV"; diff --git a/crates/fabric-core/src/strict_json.rs b/crates/fabric-core/src/strict_json.rs new file mode 100644 index 000000000..036ebe73d --- /dev/null +++ b/crates/fabric-core/src/strict_json.rs @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Validate JSON object keys before conversion to lossy map representations. + +use std::collections::BTreeSet; +use std::fmt; + +use serde::de::{self, MapAccess, SeqAccess, Visitor}; +use serde::{Deserialize, Deserializer}; + +pub(crate) fn validate(encoded: &[u8]) -> serde_json::Result<()> { + serde_json::from_slice::(encoded).map(|_| ()) +} + +struct UniqueKeys; + +impl<'de> Deserialize<'de> for UniqueKeys { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_any(Self) + } +} + +impl<'de> Visitor<'de> for UniqueKeys { + type Value = Self; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("JSON without duplicate object fields") + } + + fn visit_bool(self, _: bool) -> Result { + Ok(self) + } + fn visit_i64(self, _: i64) -> Result { + Ok(self) + } + fn visit_u64(self, _: u64) -> Result { + Ok(self) + } + fn visit_f64(self, _: f64) -> Result { + Ok(self) + } + fn visit_str(self, _: &str) -> Result { + Ok(self) + } + fn visit_unit(self) -> Result { + Ok(self) + } + + fn visit_seq>(self, mut sequence: A) -> Result { + while sequence.next_element::()?.is_some() {} + Ok(self) + } + + fn visit_map>(self, mut mapping: A) -> Result { + let mut keys = BTreeSet::new(); + while let Some(key) = mapping.next_key::()? { + if !keys.insert(key) { + return Err(de::Error::custom("duplicate JSON field")); + } + mapping.next_value::()?; + } + Ok(self) + } +} diff --git a/crates/fabric-core/src/task_command.rs b/crates/fabric-core/src/task_command.rs index 6e8d04614..e7618a63a 100644 --- a/crates/fabric-core/src/task_command.rs +++ b/crates/fabric-core/src/task_command.rs @@ -32,6 +32,10 @@ pub(crate) enum TaskCommand { runtime_id: String, request: Value, }, + Status { + agent: String, + runtime_id: String, + }, Stop { agent: String, runtime_id: String, diff --git a/crates/fabric-python/src/lib.rs b/crates/fabric-python/src/lib.rs index 3ec7569cb..2a26f8a6a 100644 --- a/crates/fabric-python/src/lib.rs +++ b/crates/fabric-python/src/lib.rs @@ -284,6 +284,35 @@ fn check_environment( to_json(&report) } +/// Read recorded runtime status through an OpenShell environment. +#[pyfunction] +fn runtime_status_environment( + py: Python<'_>, + environment_json: String, + agent: String, + runtime_id: String, +) -> PyResult { + let environment = parse_environment_handle(environment_json)?; + let report = py + .detach(|| { + nemo_fabric_core::runtime::runtime_status_environment(&environment, &agent, &runtime_id) + }) + .map_err(|error| { + let code = match &error { + FabricError::EnvironmentProviderOperation { code, .. } + if matches!(code.as_str(), "wrong_agent" | "runtime_unavailable") => + { + code.as_str() + } + _ => "status_failed", + }; + let exception = PyRuntimeError::new_err("The runtime status lookup failed."); + let _ = exception.value(py).setattr("code", code); + exception + })?; + to_json(&report) +} + /// Check health for a registered service. #[pyfunction] fn check_service(py: Python<'_>, service_json: String, request_json: String) -> PyResult { @@ -361,6 +390,7 @@ fn runtime_resource_snapshot( #[pymodule] fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(check_environment, m)?)?; + m.add_function(wrap_pyfunction!(runtime_status_environment, m)?)?; m.add_function(wrap_pyfunction!(check_service, m)?)?; m.add("ServiceInUseError", m.py().get_type::())?; m.add( diff --git a/docs/reference/api/python-library-reference/index.md b/docs/reference/api/python-library-reference/index.md index cdf4297b0..da2f126bd 100644 --- a/docs/reference/api/python-library-reference/index.md +++ b/docs/reference/api/python-library-reference/index.md @@ -92,6 +92,7 @@ SPDX-License-Identifier: Apache-2.0 --> - [`types.RuntimeCapabilities`](./nemo_fabric.types.md#class-runtimecapabilities): Operations declared by the resolved runtime and adapter. - [`types.RuntimeHandle`](./nemo_fabric.types.md#class-runtimehandle): Opaque identity and binding for one started runtime. - [`types.RuntimeResourceSnapshot`](./nemo_fabric.types.md#class-runtimeresourcesnapshot): Timestamped resource usage of one runtime's local adapter host process tree. +- [`types.RuntimeStatusReport`](./nemo_fabric.types.md#class-runtimestatusreport): Recorded lifecycle state and a safe handle snapshot for discovery. - [`types.SampleSummary`](./nemo_fabric.types.md#class-samplesummary): Retained native series, reductions, and collection-quality diagnostics. - [`types.ServiceHandle`](./nemo_fabric.types.md#class-servicehandle): Opaque identity and binding for one prepared or attached service. - [`types.ServiceReference`](./nemo_fabric.types.md#class-servicereference): Reference to an already-running caller-owned service. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.client.md b/docs/reference/api/python-library-reference/nemo_fabric.client.md index fa21b10c1..4f5012749 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.client.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.client.md @@ -377,6 +377,48 @@ Execute one complete start, invoke, and stop lifecycle. --- +### method `runtime_status` + +```python +async def runtime_status( + *, + runtime_id: str, + agent_name: str, + environment: EnvironmentHandle | None = None, +) -> RuntimeStatusReport +``` + +Read recorded runtime state and rediscover a safe handle by identifier. + +Without an environment, read runtimes created by this ``Fabric`` instance and retain each ``Runtime`` until successful stop. The most recent 128 successful stops retain lightweight records with a null handle. A local environment additionally selects its exact environment identifier. + +An OpenShell environment routes ``fabric-agent-ctl status`` to the owning server, including from a later client process. The returned handle belongs to that server's local environment. Records do not survive server restart. This operation never starts, invokes, stops, or probes a runtime. Recorded ``active`` state does not establish current harness health; use ``check()`` for a fresh observation. + + + +**Args:** + + - `runtime_id`: Runtime identifier returned by start. + - `agent_name`: Exact owning agent name. + - `environment`: Optional local selector or verified OpenShell binding. + + + +**Returns:** + An immutable ``RuntimeStatusReport`` with a discovery-only handle. Handle environment variables and arbitrary provider metadata are omitted. + + + +**Raises:** + + - `FabricConfigError`: If the target or returned report is invalid. + - `FabricStateError`: If the runtime or selected ownership is unavailable. + - `FabricNativeUnavailableError`: If remote routing needs a missing extension. + - `FabricRuntimeError`: If the status transport fails. + +--- + + ### method `start_runtime` ```python diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index f86dcc1ac..17e26f496 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -1062,6 +1062,92 @@ def __init__(mapping: Mapping[str, Any]) -> None +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- + + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. + + +--- + + +## class `RuntimeStatusReport` + +Recorded lifecycle state and a safe handle snapshot for discovery. + +This report reads existing ownership records without probing the harness. ``handle`` is null after stop. Otherwise the handle preserves runtime and environment identity but excludes environment variables, connection metadata, provider metadata, and unknown extensions. The snapshot is for inspection; use the original ``Runtime`` for invocation and cleanup. + + + +**Attributes:** + + - `runtime_id`: Selected runtime lifecycle identifier. + - `agent_name`: Exact agent name that owns the runtime. + - `status`: Recorded ``active``, ``stopped``, or ``failed`` SDK state. + - `handle`: Sanitized ``RuntimeHandle``, or null after stop. + - `server_state`: Recorded runtime-server state, or null for a direct local read. + + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `runtime_id` | `str` | +| `agent_name` | `str` | +| `status` | `Literal['active', 'stopped', 'failed']` | +| `handle` | `RuntimeHandle \| None` | +| `server_state` | `Literal['running', 'stopping', 'stopped', 'unknown'] \| None` | + +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + --- ### property extra_fields diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 259d53568..710c17dd2 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -233,7 +233,37 @@ root is separate. OpenShell invocation is buffered; streaming and shared-service hosting remain local SDK capabilities. Commands with uncertain outcomes are never replayed; an uncertain start blocks further starts in that sandbox. -### Runtime Health +### Recorded Runtime Status + +Use `Fabric.runtime_status()` to read the recorded lifecycle state and rediscover a runtime handle without starting, invoking, stopping, or probing the harness: + +```python +report = await fabric.runtime_status( + runtime_id=runtime.runtime_id, + agent_name=config.metadata.name, +) +print(report.status) # "active", "stopped", or "failed" +``` + +A direct local lookup selects a runtime started by the same `Fabric` instance. The client retains live runtime objects until successful stop, even when the caller drops the original Python reference. Callers remain responsible for stopping runtimes. An optional local `EnvironmentHandle` restricts the lookup to that environment. After successful stop, the latest 128 records remain available with `handle=None`; older records are unavailable. + +For OpenShell, pass a verified environment handle and the runtime identifier returned by start. The SDK sends `fabric-agent-ctl status` to the original owning server, so a later client process can perform the same read: + +```python +report = await Fabric().runtime_status( + environment=environment, + runtime_id=runtime_id, + agent_name="researcher", +) +``` + +`RuntimeStatusReport` contains `runtime_id`, `agent_name`, `status`, `handle`, and `server_state`. The server state is `running`, `stopping`, `stopped`, or `unknown`; direct local reads use `None`. Missing runtimes, wrong agents, and mismatched local environment identifiers raise `FabricStateError`. The SDK validates remote report identity and uses fixed diagnostics for transport failures. + +The returned handle is a discovery snapshot. The snapshot retains runtime identity, the opaque binding, adapter and service identifiers, and environment identity and paths. The snapshot excludes environment variable values, provider connection metadata, provider metadata, and unknown handle extensions. Use the original `Runtime` for invocation and cleanup. OpenShell reports the handle owned inside the sandbox, rather than a new host-side runtime binding. Stopped runtimes have no current handle. Records are held by the owning process and do not survive server restart. + +Recorded `active` state does not prove the harness is currently healthy. Use `Fabric.check()` for a fresh health observation. This status operation adds no native-interface endpoints or credential locations. + +## Runtime Health Use `Fabric.check(environment=..., request=...)` to check runtime health. Refer to the [health-check overview](health-checks.mdx) for scopes, outcomes, and deadlines, and the [adapter support table](health-checks.mdx#native-evidence-by-adapter) for supported levels and native evidence. @@ -336,7 +366,7 @@ Service checks evaluate earlier levels in order under the same three-second defa `Fabric.check()` and `fabric-agent-ctl check`. The executable ships with `nemo-fabric-runtime` and follows the SDK package release. -`fabric-agent-ctl serve` starts one runtime server per sandbox. The `start`, `invoke`, and `stop` commands call the existing `Fabric.start_runtime()`, `Runtime.invoke()`, and `Runtime.stop()` methods. Start accepts `FabricConfig` and returns the SDK `RuntimeHandle` mapping. Invoke accepts a full `RunRequest` through `--request` or any JSON SDK input through `--input`, and returns the SDK `RunResult` mapping. Stop returns JSON null. These payloads occupy the response envelope's `result` field; command diagnostics remain separate. Targeted commands require the returned runtime ID and exact agent name. The caller waits for the private socket before start. Caller-side preflight uses `Fabric.plan()` and `Fabric.doctor()`; startup validates configuration against installed descriptors. +`fabric-agent-ctl serve` starts one runtime server per sandbox. The `start`, `invoke`, `stop`, and `status` commands call `Fabric.start_runtime()`, `Runtime.invoke()`, `Runtime.stop()`, and `Fabric.runtime_status()`. Start accepts `FabricConfig` and returns the SDK `RuntimeHandle` mapping. Invoke accepts a full `RunRequest` through `--request` or any JSON SDK input through `--input`, and returns the SDK `RunResult` mapping. Stop returns JSON null. These payloads occupy the response envelope's `result` field; command diagnostics remain separate. Targeted commands require the returned runtime ID and exact agent name. The caller waits for the private socket before start. Caller-side preflight uses `Fabric.plan()` and `Fabric.doctor()`; startup validates configuration against installed descriptors. One sandbox can own multiple runtimes, each addressed by its returned runtime ID. Invoke and stop operations are independent across IDs; stopping one runtime leaves the others usable, and another runtime can be started after a stop. An uncertain start blocks further starts because the server cannot safely determine whether an unaddressable runtime exists. Ordinary SDK stop does not confirm descendant cleanup, so the runtime server retains `fabric.unclean` after runtime use and reports `cleanup_unconfirmed` if previous sandbox process cleanup cannot be confirmed. Refer to the [runtime server command guide](https://github.com/NVIDIA/NeMo-Fabric/blob/main/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md). diff --git a/environment-providers/openshell/src/lib.rs b/environment-providers/openshell/src/lib.rs index 75d8c0486..5cea41d91 100644 --- a/environment-providers/openshell/src/lib.rs +++ b/environment-providers/openshell/src/lib.rs @@ -8,6 +8,8 @@ //! identity and policy before operations and exposes bounded sandbox exec; //! agent runtime state is outside this provider. +mod strict_json; + use std::collections::HashMap; use std::future::Future; use std::io::{BufRead, Write}; @@ -344,6 +346,11 @@ async fn runtime_command( )); } if executed.stdout.is_empty() { + if operation == "status" { + strict_json::validate(&executed.stderr).map_err(|_| { + ProviderError::contract("invalid_response", "status command returned invalid JSON") + })?; + } let diagnostic: TaskDiagnosticEnvelope = serde_json::from_slice(&executed.stderr).map_err(|_| { ProviderError::contract("invalid_response", "task command outcome is unknown") @@ -353,9 +360,17 @@ async fn runtime_command( || diagnostic.operation != operation || !matches!(diagnostic.status.as_str(), "failed" | "unsupported") || !matches!(error.effects.as_str(), "none" | "applied" | "unknown") + || (operation == "status" && error.effects != "none") || !matches!( error.stage.as_str(), - "request" | "validation" | "start" | "invoke" | "stop" | "check" | "transport" + "request" + | "validation" + | "start" + | "invoke" + | "stop" + | "check" + | "status" + | "transport" ) || !valid_diagnostic_code(&error.code) { @@ -387,6 +402,11 @@ fn decode_sdk_command_response( "task command output was not one SDK response envelope", )); } + if operation == "status" { + strict_json::validate(stdout).map_err(|_| { + ProviderError::contract("invalid_response", "status command returned invalid JSON") + })?; + } let response: Value = serde_json::from_slice(stdout).map_err(|_| { ProviderError::contract("invalid_response", "task command did not return SDK JSON") })?; @@ -423,6 +443,14 @@ fn decode_sdk_command_response( let status = response["status"].as_str().unwrap_or_default(); let result = &response["result"]; let error = &response["error"]; + if operation == "status" + && (response["changed"] != false || (!error.is_null() && error["effects"] != "none")) + { + return Err(ProviderError::contract( + "invalid_response", + "status command reported lifecycle effects", + )); + } if operation == "check" { if response["changed"] != false || (!error.is_null() && error["effects"] != "none") { return Err(ProviderError::contract( @@ -501,7 +529,14 @@ fn decode_sdk_command_response( || !valid_diagnostic_code(code) || !matches!( stage, - "request" | "validation" | "start" | "invoke" | "stop" | "check" | "transport" + "request" + | "validation" + | "start" + | "invoke" + | "stop" + | "check" + | "status" + | "transport" ) || !matches!(effects, "none" | "applied" | "unknown") { @@ -529,6 +564,7 @@ fn valid_diagnostic_code(code: &str) -> bool { | "start_failed" | "start_uncertain" | "runtime_id_conflict" + | "status_failed" | "stop_failed" | "invoke_failed" | "host_stopping" @@ -1372,6 +1408,10 @@ enum TaskCommand { runtime_id: String, request: Value, }, + Status { + agent: String, + runtime_id: String, + }, Stop { agent: String, runtime_id: String, @@ -1439,6 +1479,7 @@ impl TaskCommand { runtime_id, request, } => ("invoke", agent, Some(runtime_id), Some(request)), + Self::Status { agent, runtime_id } => ("status", agent, Some(runtime_id), None), Self::Stop { agent, runtime_id } => ("stop", agent, Some(runtime_id), None), }; if agent.trim().is_empty() @@ -3703,6 +3744,53 @@ mod tests { ), } } + #[test] + fn status_command_names_the_agent_and_runtime_without_payload() { + let command: TaskCommand = serde_json::from_value(json!({ + "operation": "status", "agent": "researcher", "runtime_id": "runtime-1" + })) + .expect("status command"); + let (operation, arguments, payload) = command.arguments().expect("status arguments"); + assert_eq!(operation, "status"); + assert_eq!( + arguments, + [ + "fabric-agent-ctl", + "status", + "--agent", + "researcher", + "--runtime-id", + "runtime-1" + ] + ); + assert!(payload.is_none()); + } + + #[test] + fn status_response_rejects_duplicate_fields_before_normalizing_json() { + let response = br#"{"operation":"status","status":"succeeded","changed":false,"result":{"runtime_id":"wrong","runtime_id":"runtime-1","agent_name":"demo","status":"stopped","handle":null,"server_state":"stopped"},"error":null}"#; + assert!(decode_sdk_command_response("status", 0, response).is_err()); + } + + #[test] + fn status_responses_never_report_lifecycle_effects() { + let mut response = json!({"operation": "status", "status": "succeeded", + "changed": false, "result": {"runtime_id": "runtime-1", "agent_name": "researcher", + "status": "stopped", "handle": null, "server_state": "stopped"}, "error": null}); + assert!(decode_sdk_command_response("status", 0, response.to_string().as_bytes()).is_ok()); + response["changed"] = json!(true); + assert!(decode_sdk_command_response("status", 0, response.to_string().as_bytes()).is_err()); + response["changed"] = Value::Null; + assert!(decode_sdk_command_response("status", 0, response.to_string().as_bytes()).is_err()); + response["changed"] = json!(false); + response["status"] = json!("failed"); + response["result"] = Value::Null; + response["error"] = json!({"code": "runtime_unavailable", "stage": "status", + "message": "PRIVATE", "effects": "none"}); + assert!(decode_sdk_command_response("status", 1, response.to_string().as_bytes()).is_ok()); + response["error"]["effects"] = json!("unknown"); + assert!(decode_sdk_command_response("status", 1, response.to_string().as_bytes()).is_err()); + } } fn gateway_capture(mut report: Value, observed_at: Option) -> Value { diff --git a/environment-providers/openshell/src/strict_json.rs b/environment-providers/openshell/src/strict_json.rs new file mode 100644 index 000000000..036ebe73d --- /dev/null +++ b/environment-providers/openshell/src/strict_json.rs @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Validate JSON object keys before conversion to lossy map representations. + +use std::collections::BTreeSet; +use std::fmt; + +use serde::de::{self, MapAccess, SeqAccess, Visitor}; +use serde::{Deserialize, Deserializer}; + +pub(crate) fn validate(encoded: &[u8]) -> serde_json::Result<()> { + serde_json::from_slice::(encoded).map(|_| ()) +} + +struct UniqueKeys; + +impl<'de> Deserialize<'de> for UniqueKeys { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_any(Self) + } +} + +impl<'de> Visitor<'de> for UniqueKeys { + type Value = Self; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("JSON without duplicate object fields") + } + + fn visit_bool(self, _: bool) -> Result { + Ok(self) + } + fn visit_i64(self, _: i64) -> Result { + Ok(self) + } + fn visit_u64(self, _: u64) -> Result { + Ok(self) + } + fn visit_f64(self, _: f64) -> Result { + Ok(self) + } + fn visit_str(self, _: &str) -> Result { + Ok(self) + } + fn visit_unit(self) -> Result { + Ok(self) + } + + fn visit_seq>(self, mut sequence: A) -> Result { + while sequence.next_element::()?.is_some() {} + Ok(self) + } + + fn visit_map>(self, mut mapping: A) -> Result { + let mut keys = BTreeSet::new(); + while let Some(key) = mapping.next_key::()? { + if !keys.insert(key) { + return Err(de::Error::custom("duplicate JSON field")); + } + mapping.next_value::()?; + } + Ok(self) + } +} diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py index 78050ab22..1f6eec00e 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py @@ -83,6 +83,7 @@ from nemo_fabric.types import RunUsage from nemo_fabric.types import RuntimeCapabilities from nemo_fabric.types import RuntimeHandle +from nemo_fabric.types import RuntimeStatusReport from nemo_fabric.types import RuntimeResourceSnapshot from nemo_fabric.types import ServiceHandle from nemo_fabric.types import ServiceReference @@ -156,6 +157,7 @@ "RunUsage", "RuntimeCapabilities", "RuntimeHandle", + "RuntimeStatusReport", "RuntimeResourceSnapshot", "SystemStatsConfig", "SystemStatsSamplingConfig", diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi index 370bb5813..5b25ad9a9 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi @@ -45,6 +45,9 @@ def invoke_openai_stream( ) -> str: ... def stop_runtime(plan_json: str, runtime_json: str) -> str: ... def runtime_resource_snapshot(plan_json: str, runtime_json: str) -> str: ... +def runtime_status_environment( + environment_json: str, agent: str, runtime_id: str +) -> str: ... def check_environment(environment_json: str, request_json: str) -> str: ... def check_service(service_json: str, request_json: str) -> str: ... diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py index 95d7e16ec..ef912a5b9 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py @@ -13,6 +13,7 @@ import os import time from threading import Lock +from collections import OrderedDict from collections.abc import Mapping from contextlib import AsyncExitStack from concurrent.futures import ThreadPoolExecutor @@ -26,11 +27,13 @@ FabricError, FabricNativeUnavailableError, FabricRuntimeError, + FabricStateError, _runtime_error, ) from nemo_fabric.models import FabricConfig, RunRequest from nemo_fabric.runtime import ( Runtime, + RuntimeStatus, _PI_ADAPTER_ID, _call_blocking, _json_mapping, @@ -52,6 +55,7 @@ RunResult, ServiceHandle, ServiceReference, + RuntimeStatusReport, ) try: @@ -64,6 +68,7 @@ # Dedicated workers run health requests during invocations. _CHECK_EXECUTOR = ThreadPoolExecutor(max_workers=4, thread_name_prefix="fabric-health") logger = logging.getLogger(__name__) +_MAX_STOPPED_STATUS_RECORDS = 128 class Fabric: @@ -94,6 +99,182 @@ def __init__(self) -> None: self._execution_participant = "caller_sdk" self._software_snapshot = None self._software_snapshot_lock = Lock() + self._runtimes: dict[str, Runtime] = {} + self._stopped_status: OrderedDict[ + str, tuple[tuple[str, str], RuntimeStatusReport] + ] = OrderedDict() + + async def runtime_status( + self, + *, + runtime_id: str, + agent_name: str, + environment: EnvironmentHandle | None = None, + ) -> RuntimeStatusReport: + """Read recorded runtime state and rediscover a safe handle by identifier. + + Without an environment, read runtimes created by this ``Fabric`` instance + and retain each ``Runtime`` until successful stop. The most recent 128 + successful stops retain lightweight records with a null handle. A local + environment additionally selects its exact environment identifier. + + An OpenShell environment routes ``fabric-agent-ctl status`` to the owning + server, including from a later client process. The returned handle belongs + to that server's local environment. Records do not survive server restart. + This operation never starts, invokes, stops, or probes a runtime. Recorded + ``active`` state does not establish current harness health; use ``check()`` + for a fresh observation. + + Args: + runtime_id: Runtime identifier returned by start. + agent_name: Exact owning agent name. + environment: Optional local selector or verified OpenShell binding. + + Returns: + An immutable ``RuntimeStatusReport`` with a discovery-only handle. + Handle environment variables and arbitrary provider metadata are omitted. + + Raises: + FabricConfigError: If the target or returned report is invalid. + FabricStateError: If the runtime or selected ownership is unavailable. + FabricNativeUnavailableError: If remote routing needs a missing extension. + FabricRuntimeError: If the status transport fails. + """ + for value in (runtime_id, agent_name): + if not isinstance(value, str) or not value.strip() or "\0" in value: + raise FabricConfigError("The runtime status target is invalid.") + if environment is not None: + environment_json = _environment_json(environment) + if environment.provider == "openshell": + native = self._require_native_module("runtime_status") + try: + raw = await asyncio.get_running_loop().run_in_executor( + _CHECK_EXECUTOR, + native.runtime_status_environment, + environment_json, + agent_name, + runtime_id, + ) + except Exception as error: + code = getattr(error, "code", None) + if code in ("wrong_agent", "runtime_unavailable"): + raise FabricStateError( + "The selected runtime is unavailable.", + stage="status", + code=code, + ) from None + raise FabricRuntimeError( + "The runtime status transport failed.", stage="status" + ) from None + try: + encoded = raw.encode("utf-8") + if len(encoded) > 4 * 1024 * 1024: + raise ValueError() + + def unique(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate runtime status field") + result[key] = value + return result + + report = RuntimeStatusReport.from_mapping( + json.loads(encoded, object_pairs_hook=unique) + ) + if (report.runtime_id, report.agent_name) != ( + runtime_id, + agent_name, + ): + raise ValueError() + if report.server_state is None or ( + report.handle is not None + and report.handle.environment["provider"] != "local" + ): + raise ValueError() + return report + except ( + FabricConfigError, + ValueError, + TypeError, + AttributeError, + RecursionError, + ): + raise FabricConfigError( + "The runtime status report is invalid." + ) from None + if environment.provider != "local": + raise FabricConfigError( + "Runtime status supports local and OpenShell environments." + ) + runtime = self._runtimes.get(runtime_id) + if runtime is not None: + handle = runtime.handle + environment_identity = ( + handle.environment["provider"], + handle.environment["environment_id"], + ) + report = RuntimeStatusReport.from_mapping( + { + "runtime_id": runtime_id, + "agent_name": handle.agent_name, + "status": runtime.status.value, + "handle": None + if runtime.status is RuntimeStatus.STOPPED + else handle.to_mapping(), + "server_state": None, + } + ) + else: + record = self._stopped_status.get(runtime_id) + if record is None: + raise FabricStateError( + "The selected runtime is unavailable.", + stage="status", + code="runtime_unavailable", + ) + environment_identity, report = record + if report.agent_name != agent_name: + raise FabricStateError( + "The request does not identify this agent.", + stage="status", + code="wrong_agent", + ) + if ( + environment is not None + and (environment.provider, environment.environment_id) + != environment_identity + ): + raise FabricStateError( + "The selected environment does not own this runtime.", + stage="status", + code="runtime_unavailable", + ) + return report + + def _record_stopped_runtime(self, runtime: Runtime) -> None: + if ( + runtime.status is not RuntimeStatus.STOPPED + or self._runtimes.get(runtime.runtime_id) is not runtime + ): + return + handle = runtime.handle + self._stopped_status[runtime.runtime_id] = ( + (handle.environment["provider"], handle.environment["environment_id"]), + RuntimeStatusReport.from_mapping( + { + "runtime_id": runtime.runtime_id, + "agent_name": handle.agent_name, + "status": "stopped", + "handle": None, + "server_state": None, + } + ), + ) + self._stopped_status.move_to_end(runtime.runtime_id) + self._runtimes.pop(runtime.runtime_id, None) + while len(self._stopped_status) > _MAX_STOPPED_STATUS_RECORDS: + self._stopped_status.popitem(last=False) def _set_execution_participant(self, participant: str) -> None: """Relabel only this instance's own cached observations when embedded.""" @@ -844,7 +1025,7 @@ def start() -> dict[str, Any]: await close_start_resources() raise await service_guard.aclose() - return Runtime( + owned_runtime = Runtime( client=self, plan=plan, runtime=runtime, @@ -852,6 +1033,21 @@ def start() -> dict[str, Any]: collector=collector, collector_client=collector_client, ) + if ( + owned_runtime.runtime_id in self._runtimes + or owned_runtime.runtime_id in self._stopped_status + ): + # A duplicate native identity cannot safely replace an existing owner + # or be stopped independently. The enclosing server retains its + # uncertain-start marker so the supervisor can replace the sandbox. + await close_start_resources() + raise FabricStateError( + "Fabric returned a runtime identifier that is already owned.", + stage="start", + code="runtime_id_conflict", + ) + self._runtimes[owned_runtime.runtime_id] = owned_runtime + return owned_runtime async def prepare_service( self, diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index f29e0bbad..6a1eab728 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -308,6 +308,7 @@ def stop_after_cancel() -> Any: finally: if not invocation_claimed: self._current_task = None + self._client._record_stopped_runtime(self) def invoke_stream( self, @@ -837,6 +838,7 @@ def stop() -> Any: self._status = RuntimeStatus.STOPPED finally: self._closing = False + self._client._record_stopped_runtime(self) try: await self._deregister_requests() except Exception as cleanup_error: diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md index 3ab95c23c..f9fc55321 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md @@ -18,6 +18,7 @@ SPDX-License-Identifier: Apache-2.0 | `invoke --agent NAME --runtime-id ID --input FILE` | `Runtime.invoke(input=...)` | | `invoke --agent NAME --runtime-id ID --request FILE` | `Runtime.invoke(request=RunRequest(...))` | | `stop --agent NAME --runtime-id ID` | `Runtime.stop()` | +| `status --agent NAME --runtime-id ID` | `Fabric.runtime_status()` with recorded server state | Run the server under the sandbox supervisor: @@ -30,6 +31,7 @@ Wait for `/sandbox/fabric.sock` with a caller-owned deadline, then issue command ```sh fabric-agent-ctl check fabric-agent-ctl start --agent researcher --config /sandbox/config.json +fabric-agent-ctl status --agent researcher --runtime-id RUNTIME_ID fabric-agent-ctl invoke --agent researcher --runtime-id RUNTIME_ID --input /sandbox/input.json fabric-agent-ctl stop --agent researcher --runtime-id RUNTIME_ID ``` @@ -38,6 +40,10 @@ The response envelope separates command diagnostics from the public SDK payload Use `--request FILE` to forward a complete public `RunRequest`, including `request_id`, `relay_session_root`, `context`, `overrides`, and supported extensions. For example, a request file can contain `{"input": "Review calculator.py", "request_id": "review-1", "context": {"review_tag": "cedar-4827"}}`. `--input FILE` is a mutually exclusive convenience for any JSON-compatible SDK input: an object, array, string, number, boolean, or null. Null follows the SDK's normalization to an empty input string. Agent input fields named `stream` or `streaming` remain ordinary input data; this command calls `Runtime.invoke()`, never a streaming method. +`status --agent NAME --runtime-id ID` returns the existing SDK ownership record without starting, invoking, stopping, or probing the harness. The result is a `RuntimeStatusReport` mapping with `runtime_id`, `agent_name`, `status` (`active`, `stopped`, or `failed`), `handle`, and the server's recorded `server_state` (`running`, `stopping`, `stopped`, or `unknown`). Recorded active state does not establish current process health. Status reads bypass the invocation lock. A valid read exits 0, including a known stopped or failed runtime; missing or mismatched ownership exits 1. `changed` is always false, and status errors always report `effects: "none"`. + +The handle is a discovery snapshot with runtime identity, opaque binding, adapter and service identifiers, and environment identity and paths. Environment variables, provider connection metadata, provider metadata, and unknown handle extensions are excluded. The original `Runtime` remains the lifecycle handle for invocation and cleanup. Successful stop removes the current handle, so status returns `handle: null`; the most recent 128 stopped records remain available. A later CLI process can retrieve a record while the original server remains alive. Records do not survive server replacement. Native-interface metadata remains separate future work. + `check` reports server responsiveness. Add both `--agent` and `--runtime-id` to check one runtime. Runtime checks default to `live`; the mutually exclusive `--active` and `--ready` flags include preceding levels. Checks can run during invocations, and busy agents can pass. Each completed check preserves an immutable `fabric.check/v1alpha1` report in `result`. Server reports use `scope: "server"`, with null runtime fields and one server observation. Runtime reports identify the exact runtime and requested level and include ordered agent observations. `changed` is always false. Passed health exits 0; failed, unknown, and unsupported health exit 1. Check diagnostics have `effects: "none"`. Missing or invalid responses produce fixed transport errors with `result: null`. @@ -88,7 +94,7 @@ The bridge requires Unix sockets and `fcntl`; Windows imports remain safe and co ## Local Qualification And Migration -Follow the [OpenShell Code Review instructions](../../../../../../examples/code_review_agent/README.md#openshell-code-review) to build the image and create a sandbox. The image contains the current SDK, runtime server, adapter, and harness, and advertises `fabric-agent-ctl`, interface version 2, and `start/invoke/stop/serve/check`. The server implementation is `nemo_fabric.runtime_server.state.RuntimeServer`. Rebuild the image and create fresh sandboxes after updating the command interface. +Follow the [OpenShell Code Review instructions](../../../../../../examples/code_review_agent/README.md#openshell-code-review) to build the image and create a sandbox. The image contains the current SDK, runtime server, adapter, and harness, and advertises `fabric-agent-ctl`, interface version 2, and `start/invoke/stop/serve/check/status`. The server implementation is `nemo_fabric.runtime_server.state.RuntimeServer`. Rebuild the image and create fresh sandboxes after updating the command interface. The example starts the server, waits for its socket, starts a runtime, invokes twice, and stops through separate exec calls. The older NemoClaw client using `validate/prepare/configure` requires migration. Version 1 requests fail explicitly. Direct SDK lifecycle and streaming APIs retain their existing behavior. diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/cli.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/cli.py index 7f55a3e03..5742a30d2 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/cli.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/cli.py @@ -101,9 +101,9 @@ def parse_command(arguments: list[str]) -> dict[str, Any]: else: raise ProtocolError() required = set() - if operation in ("start", "invoke", "stop"): + if operation in ("start", "invoke", "stop", "status"): required.add("--agent") - if operation in ("invoke", "stop"): + if operation in ("invoke", "stop", "status"): required.add("--runtime-id") if operation in CONFIG_OPERATIONS: required.add("--config") diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/protocol.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/protocol.py index f5cb46f2f..5d1887bda 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/protocol.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/protocol.py @@ -8,7 +8,7 @@ import sys from typing import Any -from nemo_fabric import FabricConfigError, RunResult, RuntimeHandle +from nemo_fabric import FabricConfigError, RunResult, RuntimeHandle, RuntimeStatusReport from nemo_fabric.runtime_server.health import validate_check_response INTERFACE_VERSION = 2 @@ -18,7 +18,7 @@ READ_SECONDS = 10.0 SHUTDOWN_SECONDS = 4.0 SOCKET_PATH = "/sandbox/fabric.sock" -OPERATIONS = ("start", "invoke", "stop", "serve", "check") +OPERATIONS = ("start", "invoke", "stop", "serve", "check", "status") # A cause is an SDK or adapter error code, never its message or other values. CAUSE = re.compile(r"[a-z][a-z0-9_]{0,63}") CAUSE_SUFFIX = re.compile(rf" Cause: ({CAUSE.pattern})\.\Z") @@ -37,6 +37,7 @@ "start_failed": "Fabric could not establish an active runtime.", "start_uncertain": "A previous runtime start has an uncertain outcome; replace the sandbox before starting another.", "runtime_id_conflict": "Fabric returned a runtime ID that is already in use.", + "status_failed": "Fabric could not read the recorded runtime status.", "stop_failed": "Fabric could not complete the stop operation.", "invoke_failed": "Fabric did not report a successful invocation.", "host_stopping": "The Fabric host is shutting down.", @@ -123,9 +124,9 @@ def validate_request(request: Any) -> str: ): raise ProtocolError("interface_unsupported", unsupported=True) required = {"interface_version", "operation"} - if operation in ("start", "invoke", "stop"): + if operation in ("start", "invoke", "stop", "status"): required.add("agent") - if operation in ("invoke", "stop"): + if operation in ("invoke", "stop", "status"): required.add("runtime_id") if operation == "start": required.add("config") @@ -233,9 +234,11 @@ def encode_response(response: dict[str, Any]) -> bytes: return _encode( envelope( response["operation"], - changed=False if response["operation"] == "check" else None, + changed=False if response["operation"] in ("check", "status") else None, error=ProtocolError("response_too_large", "transport"), - effects="none" if response["operation"] == "check" else "unknown", + effects="none" + if response["operation"] in ("check", "status") + else "unknown", ) ) @@ -267,6 +270,7 @@ def validate_response(response: dict[str, Any], operation: str) -> None: "stop", "invoke", "check", + "status", "transport", ) or error["effects"] not in ("none", "applied", "unknown") @@ -281,12 +285,25 @@ def validate_response(response: dict[str, Any], operation: str) -> None: error["message"] = str( ProtocolError(error["code"], cause=cause[1] if cause else None) ) + if operation == "status": + if ( + response["changed"] is not False + or error["effects"] != "none" + or response["result"] is not None + ): + raise invalid + return if operation != "check": return result = response["result"] try: if operation == "check": validate_check_response(response) + elif operation == "status": + report = RuntimeStatusReport.from_mapping(result) + if response["changed"] is not False or report.server_state is None: + raise invalid + response["result"] = report.to_mapping() elif operation == "invoke": RunResult.from_mapping(result) if response["changed"] is not None: diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py index e809138ec..f42ffd1df 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py @@ -16,6 +16,7 @@ Fabric, FabricConfig, FabricConfigError, + FabricStateError, CheckRequest, EnvironmentHandle, RunRequest, @@ -120,6 +121,19 @@ async def handle(self, request: Any) -> dict[str, Any]: if self.shutting_down else None, ) + if operation == "status": + owned = self._owned(request) + stage = "status" + report = await self.fabric.runtime_status( + runtime_id=request["runtime_id"], agent_name=request["agent"] + ) + return envelope( + operation, + { + **report.to_mapping(), + "server_state": owned.state, + }, + ) if self.shutting_down: raise ProtocolError("host_stopping") @@ -200,6 +214,15 @@ async def handle(self, request: Any) -> dict[str, Any]: result = (await owned.runtime.invoke(**invoke_arguments)).to_mapping() return envelope(operation, result, changed=None) except Exception as error: + if isinstance(error, FabricStateError) and error.code in ( + "wrong_agent", + "runtime_unavailable", + "runtime_id_conflict", + ): + error = ProtocolError( + error.code, + "start" if error.code == "runtime_id_conflict" else "status", + ) if not isinstance(error, ProtocolError): if isinstance(error, (FabricConfigError, ValidationError)): code = ( diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/transport.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/transport.py index 1c0b4db84..c4237ce5a 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/transport.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/transport.py @@ -121,6 +121,13 @@ async def client( await writer.drain() response = await read_message(reader, RESPONSE_LIMIT) validate_response(response, operation) + if operation == "status" and response["status"] == "succeeded": + result = response["result"] + if (result["runtime_id"], result["agent_name"]) != ( + request["runtime_id"], + request["agent"], + ): + raise ProtocolError("invalid_response", "transport") if operation == "check": finished = time.monotonic() if finished >= deadline: diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 8e829f6cb..4e3f53c5a 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -16,6 +16,7 @@ from pathlib import Path from types import MappingProxyType from typing import Any +from typing import Literal from typing import Self from typing import TypeVar @@ -1846,6 +1847,93 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: return data +class RuntimeStatusReport(FabricMapping): + """Recorded lifecycle state and a safe handle snapshot for discovery. + + This report reads existing ownership records without probing the harness. + ``handle`` is null after stop. Otherwise the handle preserves runtime and + environment identity but excludes environment variables, connection metadata, + provider metadata, and unknown extensions. The snapshot is for inspection; + use the original ``Runtime`` for invocation and cleanup. + + Attributes: + runtime_id: Selected runtime lifecycle identifier. + agent_name: Exact agent name that owns the runtime. + status: Recorded ``active``, ``stopped``, or ``failed`` SDK state. + handle: Sanitized ``RuntimeHandle``, or null after stop. + server_state: Recorded runtime-server state, or null for a direct local read. + """ + + runtime_id: str + agent_name: str + status: Literal["active", "stopped", "failed"] + handle: RuntimeHandle | None + server_state: Literal["running", "stopping", "stopped", "unknown"] | None + _fields = frozenset( + {"runtime_id", "agent_name", "status", "handle", "server_state"} + ) + + def __getitem__(self, key: str) -> Any: + value = super().__getitem__(key) + if key == "handle" and value is not None: + return RuntimeHandle.from_mapping(value.to_mapping()) + return value + + def __getattr__(self, name: str) -> Any: + if name == "handle": + return self[name] + return super().__getattr__(name) + + def __setattr__(self, name: str, value: Any) -> None: + raise AttributeError("RuntimeStatusReport is immutable") + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + if set(data) != cls._fields: + raise FabricConfigError("The runtime status report has invalid fields.") + for field in ("runtime_id", "agent_name"): + data[field] = _required_text(data.get(field), field) + if "\0" in data[field]: + raise FabricConfigError("The runtime status target is invalid.") + if data["status"] not in ("active", "stopped", "failed"): + raise FabricConfigError("The recorded runtime status is invalid.") + if data["server_state"] not in ( + None, + "running", + "stopping", + "stopped", + "unknown", + ): + raise FabricConfigError("The recorded server state is invalid.") + if data["status"] == "stopped": + if data["handle"] is not None: + raise FabricConfigError( + "A stopped runtime cannot report a current handle." + ) + else: + handle = RuntimeHandle.from_mapping(data["handle"]) + if (handle.runtime_id, handle.agent_name) != ( + data["runtime_id"], + data["agent_name"], + ): + raise FabricConfigError( + "The runtime status handle does not match its target." + ) + environment = EnvironmentHandle.from_mapping( + handle.environment + ).to_mapping() + for field in ("env", "connection", "metadata"): + environment.pop(field, None) + values = { + field: value + for field, value in handle.to_mapping().items() + if field in RuntimeHandle._fields + } + values["environment"] = environment + data["handle"] = RuntimeHandle.from_mapping(values) + return data + + class ServiceReference(FabricMapping): """Reference to an already-running caller-owned service. diff --git a/skills/nemo-fabric-integrate/SKILL.md b/skills/nemo-fabric-integrate/SKILL.md index c24c7cc58..8c7c049d1 100644 --- a/skills/nemo-fabric-integrate/SKILL.md +++ b/skills/nemo-fabric-integrate/SKILL.md @@ -230,6 +230,8 @@ runtime before calling `release_environment(environment)` separately. Provider availability and reuse limits vary; task paths must remain strings for non-local environments. See the [environment lifecycle guide](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/sdk/python.mdx#explicit-environment-lifecycle). +Use `await fabric.runtime_status(runtime_id=runtime_id, agent_name=name)` to read recorded state and a safe discovery handle for a runtime started by the same `Fabric` instance. The client retains live runtime objects until successful stop. Add `environment=environment` for an OpenShell lookup through `fabric-agent-ctl status`, including from a later client process while the original server remains alive. `RuntimeStatusReport` reports `active`, `stopped`, or `failed`; `server_state` is present for server reads and null for direct local reads. Successful stops retain up to 128 lightweight records with a null handle. Discovery handles omit environment variable values and arbitrary provider metadata; use the original `Runtime` for invocation and cleanup. Status reads perform no lifecycle operation or health probe. Use `Fabric.check()` when the consumer needs fresh health evidence. + The selected adapter owns the execution topology. The bundled Claude, Codex, Deep Agents, and Hermes Agent adapters retain their native client, graph/checkpointer, or agent/database inside one local host for the full runtime. Local `process` diff --git a/tests/python/runtime_server/test_status.py b/tests/python/runtime_server/test_status.py new file mode 100644 index 000000000..cd07f41ca --- /dev/null +++ b/tests/python/runtime_server/test_status.py @@ -0,0 +1,210 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Read-only runtime discovery through the SDK and the command bridge.""" + +import asyncio +import json +from unittest.mock import AsyncMock + +import pytest + +from nemo_fabric import RuntimeStatus +from nemo_fabric.runtime_server import cli, protocol, transport +from nemo_fabric.runtime_server.state import RuntimeServer + + +@pytest.mark.parametrize("extra", [[], ["--live"], ["--config", "private.json"]]) +def test_status_command_requires_only_agent_and_runtime(extra): + arguments = ["status", "--agent", "researcher", "--runtime-id", "runtime-1", *extra] + if extra: + with pytest.raises(protocol.ProtocolError): + cli.parse_command(arguments) + else: + assert cli.parse_command(arguments) == { + "interface_version": 2, + "operation": "status", + "agent": "researcher", + "runtime_id": "runtime-1", + } + + +@pytest.mark.parametrize("state", [RuntimeStatus.ACTIVE, RuntimeStatus.FAILED]) +async def test_status_server_delegates_to_sdk_without_invocation_lock( + blocking_adapter_config, tmp_path, monkeypatch, state +): + host = RuntimeServer(directory=tmp_path) + config = blocking_adapter_config.to_mapping() + start = await host.handle( + { + "interface_version": 2, + "operation": "start", + "agent": "blocking", + "config": config, + } + ) + assert start["status"] == "succeeded", start + runtime_id = start["result"]["runtime_id"] + owned = host.runtimes[runtime_id] + owned.runtime._status = state + sdk_status = AsyncMock(wraps=host.fabric.runtime_status) + monkeypatch.setattr(host.fabric, "runtime_status", sdk_status) + try: + async with owned.lock: + response = await asyncio.wait_for( + host.handle( + { + "interface_version": 2, + "operation": "status", + "agent": "blocking", + "runtime_id": runtime_id, + } + ), + 0.5, + ) + assert response["status"] == "succeeded", response + assert response["changed"] is False + assert response["result"]["status"] == state + assert response["result"]["server_state"] == "running" + assert response["result"]["handle"]["runtime_id"] == runtime_id + sdk_status.assert_awaited_once_with( + runtime_id=runtime_id, agent_name="blocking" + ) + assert not (tmp_path / "entered").exists() + assert not (tmp_path / "stopped").exists() + finally: + await host.shutdown() + + +async def test_status_process_bridge_reads_stopped_record_and_does_not_start_server( + blocking_adapter_config, tmp_path, socket_path +): + host = RuntimeServer(directory=tmp_path) + ready, stop = asyncio.Event(), asyncio.Event() + serving = asyncio.create_task( + transport.serve( + host, + socket_path=socket_path, + ready_event=ready, + stop_event=stop, + ) + ) + await asyncio.wait_for(ready.wait(), 5) + try: + started = await host.handle( + { + "interface_version": 2, + "operation": "start", + "agent": "blocking", + "config": blocking_adapter_config.to_mapping(), + } + ) + assert started["status"] == "succeeded", started + request = { + "interface_version": 2, + "operation": "status", + "agent": "blocking", + "runtime_id": started["result"]["runtime_id"], + } + active = await transport.client(request, socket_path=socket_path) + assert active["status"] == "succeeded", active + assert active["result"]["status"] == "active" + assert (await host.handle({**request, "operation": "stop"}))[ + "status" + ] == "succeeded" + stopped = await transport.client(request, socket_path=socket_path) + assert stopped["result"]["status"] == "stopped" + assert stopped["result"]["server_state"] == "stopped" + assert stopped["result"]["handle"] is None + wrong = await transport.client( + {**request, "agent": "other"}, socket_path=socket_path + ) + assert wrong["error"]["code"] == "wrong_agent" + missing = await transport.client( + {**request, "runtime_id": "missing"}, socket_path=socket_path + ) + assert missing["error"]["code"] == "runtime_unavailable" + finally: + stop.set() + await asyncio.wait_for(serving, 5) + missing_server = await transport.client(request, socket_path=socket_path) + assert missing_server["error"]["effects"] == "none" + assert missing_server["changed"] is False + assert not socket_path.exists() + + +def test_oversized_status_response_has_no_effects(): + response = protocol.envelope("status", {"private": "x" * protocol.RESPONSE_LIMIT}) + decoded = json.loads(protocol.encode_response(response)) + assert decoded["changed"] is False + assert decoded["error"]["effects"] == "none" + + +@pytest.mark.parametrize( + "field,value", + [ + ("changed", True), + ("changed", None), + ("result", {"status": "active"}), + ( + "error", + { + "code": "runtime_unavailable", + "stage": "status", + "message": "PRIVATE", + "effects": "unknown", + }, + ), + ], +) +def test_status_rejects_malformed_or_effectful_responses(field, value): + response = protocol.envelope( + "status", + { + "runtime_id": "runtime-1", + "agent_name": "researcher", + "status": "stopped", + "handle": None, + "server_state": "stopped", + }, + ) + response[field] = value + if field == "error": + response["status"] = "failed" + response["result"] = None + with pytest.raises(protocol.ProtocolError): + protocol.validate_response(response, "status") + + +@pytest.mark.parametrize( + "field,value", [("runtime_id", "other"), ("agent_name", "other")] +) +async def test_status_transport_rejects_a_different_target(socket_path, field, value): + request = { + "interface_version": 2, + "operation": "status", + "agent": "researcher", + "runtime_id": "runtime-1", + } + + async def answer(reader, writer): + await reader.readline() + report = { + "runtime_id": "runtime-1", + "agent_name": "researcher", + "status": "stopped", + "handle": None, + "server_state": "stopped", + field: value, + } + writer.write(protocol.encode_response(protocol.envelope("status", report))) + await writer.drain() + writer.close() + + server = await asyncio.start_unix_server(answer, path=socket_path) + async with server: + response = await transport.client(request, socket_path=socket_path) + assert response["status"] == "failed" + assert response["error"]["code"] == "invalid_response" + assert response["error"]["effects"] == "none" + assert response["changed"] is False diff --git a/tests/python/test_runtime.py b/tests/python/test_runtime.py index 0934a31fa..cf5aaa624 100644 --- a/tests/python/test_runtime.py +++ b/tests/python/test_runtime.py @@ -1158,3 +1158,330 @@ async def test_native_unavailable_uses_typed_error(monkeypatch: pytest.MonkeyPat with pytest.raises(FabricNativeUnavailableError, match="native extension"): Fabric().plan(_config()) + + +@pytest.mark.parametrize("state", [RuntimeStatus.ACTIVE, RuntimeStatus.FAILED]) +async def test_runtime_status_reads_owned_state_without_native_work( + native_client, mock_native, state +): + runtime = await native_client.start_runtime(_config()) + runtime._status = state + before = list(mock_native.mock_calls) + report = await native_client.runtime_status( + runtime_id=runtime.runtime_id, agent_name="demo" + ) + assert report.runtime_id == runtime.runtime_id + assert report.agent_name == "demo" + assert report.status == state + assert report.handle.runtime_id == runtime.runtime_id + assert report.server_state is None + assert mock_native.mock_calls == before + with pytest.raises(AttributeError): + report.status = "stopped" + + +async def test_runtime_status_does_not_disclose_environment_secrets( + native_client, mock_native +): + handle = _runtime() + handle["environment"].update( + { + "env": {"API_KEY": "TEST_ONLY_ENV_SECRET"}, + "connection": {"token": "TEST_ONLY_CONNECTION_SECRET"}, + "metadata": {"token": "TEST_ONLY_METADATA_SECRET"}, + } + ) + handle["environment"]["workspace"] = "/workspace" + handle["environment"]["artifacts"] = "/artifacts" + handle["service_id"] = "service-1" + handle["unknown_token"] = "TEST_ONLY_EXTENSION_SECRET" + mock_native.start_runtime.return_value = json.dumps(handle) + runtime = await native_client.start_runtime(_config()) + report = await native_client.runtime_status( + runtime_id=runtime.runtime_id, agent_name="demo" + ) + serialized = report.to_mapping() + assert "TEST_ONLY_" not in json.dumps(serialized) + assert serialized["handle"]["runtime_binding"] == handle["runtime_binding"] + assert serialized["handle"]["environment"]["environment_id"] == "environment-1" + assert serialized["handle"]["service_id"] == "service-1" + assert serialized["handle"]["environment"]["workspace"] == "/workspace" + assert serialized["handle"]["environment"]["artifacts"] == "/artifacts" + assert runtime.handle.environment["env"]["API_KEY"] == "TEST_ONLY_ENV_SECRET" + + +async def test_runtime_status_is_scoped_to_sdk_owner_agent_and_environment( + native_client, mock_native +): + from nemo_fabric import EnvironmentHandle + + runtime = await native_client.start_runtime(_config()) + environment = EnvironmentHandle.from_mapping(runtime.handle.environment) + report = await native_client.runtime_status( + runtime_id=runtime.runtime_id, agent_name="demo", environment=environment + ) + assert report.status == "active" + for fabric, runtime_id, agent_name, selected_environment in ( + (Fabric(), runtime.runtime_id, "demo", None), + (native_client, "missing", "demo", None), + (native_client, runtime.runtime_id, "other", None), + ( + native_client, + runtime.runtime_id, + "demo", + EnvironmentHandle.from_mapping( + {**environment.to_mapping(), "environment_id": "different"} + ), + ), + ): + with pytest.raises(FabricStateError) as caught: + await fabric.runtime_status( + runtime_id=runtime_id, + agent_name=agent_name, + environment=selected_environment, + ) + assert caught.value.code in {"runtime_unavailable", "wrong_agent"} + await runtime.stop() + + +async def test_runtime_status_after_stop_has_no_live_handle_and_survives_drop( + native_client, mock_native +): + import gc + import weakref + + runtime = await native_client.start_runtime(_config()) + runtime_id = runtime.runtime_id + reference = weakref.ref(runtime) + await runtime.stop() + del runtime + gc.collect() + assert reference() is None + report = await native_client.runtime_status( + runtime_id=runtime_id, agent_name="demo" + ) + assert report.status == "stopped" + assert report.handle is None + + +@pytest.mark.parametrize("field", ["runtime_id", "agent_name"]) +@pytest.mark.parametrize("value", [None, "", " ", "\x00", 4]) +async def test_runtime_status_rejects_invalid_targets_before_native_work( + native_client, mock_native, field, value +): + arguments = {"runtime_id": "runtime-1", "agent_name": "demo", field: value} + before = list(mock_native.mock_calls) + with pytest.raises(FabricConfigError): + await native_client.runtime_status(**arguments) + assert mock_native.mock_calls == before + + +async def test_runtime_status_preserves_live_runtime_after_caller_drops_reference( + native_client, +): + import gc + import weakref + + runtime = await native_client.start_runtime(_config()) + runtime_id = runtime.runtime_id + reference = weakref.ref(runtime) + del runtime + gc.collect() + report = await native_client.runtime_status( + runtime_id=runtime_id, agent_name="demo" + ) + assert report.status == "active" + assert reference() is not None + await reference().stop() + + +async def test_runtime_status_stopped_records_are_bounded( + native_client, mock_native, monkeypatch +): + monkeypatch.setattr(client_mod, "_MAX_STOPPED_STATUS_RECORDS", 2) + for index in range(3): + mock_native.start_runtime.return_value = json.dumps( + _runtime(f"runtime-{index}") + ) + runtime = await native_client.start_runtime(_config()) + await runtime.stop() + with pytest.raises(FabricStateError, match="unavailable"): + await native_client.runtime_status(runtime_id="runtime-0", agent_name="demo") + for runtime_id in ("runtime-1", "runtime-2"): + report = await native_client.runtime_status( + runtime_id=runtime_id, agent_name="demo" + ) + assert report.status == "stopped" + assert report.handle is None + + +async def test_runtime_status_stopping_one_runtime_preserves_another( + native_client, mock_native +): + original = await native_client.start_runtime(_config()) + mock_native.start_runtime.return_value = json.dumps(_runtime("runtime-2")) + replacement = await native_client.start_runtime(_config()) + await original.stop() + report = await native_client.runtime_status( + runtime_id=replacement.runtime_id, agent_name="demo" + ) + assert report.status == "active" + await replacement.stop() + + +async def test_runtime_status_after_failed_stop_preserves_failure_without_retry( + native_client, mock_native +): + runtime = await native_client.start_runtime(_config()) + mock_native.stop_runtime.side_effect = RuntimeError("TEST_ONLY_PRIVATE") + with pytest.raises(FabricRuntimeError): + await runtime.stop() + before = list(mock_native.mock_calls) + report = await native_client.runtime_status( + runtime_id=runtime.runtime_id, agent_name="demo" + ) + assert report.status == "failed" + assert report.handle.runtime_id == runtime.runtime_id + assert mock_native.mock_calls == before + assert "TEST_ONLY_PRIVATE" not in json.dumps(report.to_mapping()) + mock_native.stop_runtime.side_effect = None + await runtime.stop() + + +async def test_runtime_status_cancelled_invocation_releases_live_index( + native_client, mock_native +): + import gc + import weakref + + entered, release = threading.Event(), threading.Event() + invoke = mock_native.invoke_runtime.side_effect + + def wait_then_invoke(*arguments): + entered.set() + assert release.wait(3) + return invoke(*arguments) + + mock_native.invoke_runtime.side_effect = wait_then_invoke + runtime = await native_client.start_runtime(_config()) + reference = weakref.ref(runtime) + task = asyncio.create_task(runtime.invoke(input="work")) + assert await _wait_for(entered) + task.cancel() + release.set() + with pytest.raises(asyncio.CancelledError): + await task + assert runtime.status == "stopped" + report = await native_client.runtime_status( + runtime_id=runtime.runtime_id, agent_name="demo" + ) + assert report.status == "stopped" + assert report.handle is None + del task + del runtime + await asyncio.sleep(0) + gc.collect() + assert reference() is None + + +async def test_runtime_status_duplicate_start_preserves_the_original_owner( + native_client, mock_native +): + original = await native_client.start_runtime(_config()) + duplicate = {**_runtime(), "runtime_binding": "replacement-binding"} + mock_native.start_runtime.return_value = json.dumps(duplicate) + with pytest.raises(FabricStateError) as caught: + await native_client.start_runtime(_config()) + assert caught.value.code == "runtime_id_conflict" + mock_native.stop_runtime.assert_not_called() + original._status = RuntimeStatus.FAILED + report = await native_client.runtime_status( + runtime_id=original.runtime_id, agent_name="demo" + ) + assert report.status == "failed" + assert report.handle.runtime_binding == original.handle.runtime_binding + await original.stop() + report = await native_client.runtime_status( + runtime_id=original.runtime_id, agent_name="demo" + ) + assert report.status == "stopped" + assert report.handle is None + + +async def test_runtime_status_server_collision_preserves_original_state( + native_client, mock_native, tmp_path +): + from nemo_fabric.runtime_server.state import RuntimeServer + + host = RuntimeServer(directory=tmp_path, fabric=native_client) + request = { + "interface_version": 2, + "operation": "start", + "agent": "demo", + "config": _config().to_mapping(), + } + first = await host.handle(request) + assert first["status"] == "succeeded", first + original = host.runtimes["runtime-1"].runtime + mock_native.start_runtime.return_value = json.dumps( + {**_runtime(), "runtime_binding": "replacement-binding"} + ) + duplicate = await host.handle(request) + assert duplicate["error"]["code"] == "runtime_id_conflict" + original._status = RuntimeStatus.FAILED + selected = { + "interface_version": 2, + "operation": "status", + "agent": "demo", + "runtime_id": "runtime-1", + } + failed = await host.handle(selected) + assert failed["result"]["status"] == "failed" + assert ( + failed["result"]["handle"]["runtime_binding"] + == first["result"]["runtime_binding"] + ) + assert (await host.handle({**selected, "operation": "stop"}))[ + "status" + ] == "succeeded" + stopped = await host.handle(selected) + assert stopped["result"]["status"] == "stopped" + assert stopped["result"]["handle"] is None + + +async def test_runtime_status_duplicate_start_cannot_replace_a_stopped_record( + native_client, mock_native +): + runtime = await native_client.start_runtime(_config()) + await runtime.stop() + mock_native.start_runtime.return_value = json.dumps( + {**_runtime(), "runtime_binding": "replacement-binding"} + ) + with pytest.raises(FabricStateError) as caught: + await native_client.start_runtime(_config()) + assert caught.value.code == "runtime_id_conflict" + report = await native_client.runtime_status( + runtime_id=runtime.runtime_id, agent_name="demo" + ) + assert report.status == "stopped" + assert report.handle is None + mock_native.stop_runtime.assert_called_once() + + +@pytest.mark.parametrize("access", ["attribute", "mapping"]) +async def test_runtime_status_handle_mutation_cannot_change_shared_report( + native_client, access +): + runtime = await native_client.start_runtime(_config()) + report = await native_client.runtime_status( + runtime_id=runtime.runtime_id, agent_name="demo" + ) + original = report.to_mapping() + snapshot = report.handle if access == "attribute" else report["handle"] + snapshot.runtime_id = "other-runtime" + snapshot.environment = {"env": {"API_KEY": "TEST_ONLY_MUTATION_SECRET"}} + assert report.handle.runtime_id == runtime.runtime_id + assert report["handle"].runtime_id == runtime.runtime_id + assert report.handle.environment == original["handle"]["environment"] + assert report.to_mapping() == original + await runtime.stop() diff --git a/tests/python/test_sdk_server_checks.py b/tests/python/test_sdk_server_checks.py index bfb1e0387..e0fdbd3f1 100644 --- a/tests/python/test_sdk_server_checks.py +++ b/tests/python/test_sdk_server_checks.py @@ -190,3 +190,206 @@ async def test_sdk_server_check_reaches_the_real_command_bridge( assert [command["operation"] for command in commands] == ["check"] assert "agent" not in commands[0] assert "runtime_id" not in commands[0] + + +async def test_runtime_status_remote_reaches_sdk_in_the_original_server( + blocking_adapter_config, server_environment, server_sdk_bridge, tmp_path +): + host, log = server_sdk_bridge + config = blocking_adapter_config.model_copy(deep=True) + config.environment.workspace = str(tmp_path) + config.discovery.local_paths = [str(tmp_path / "blocking.fabric-adapter.json")] + started = await host.handle( + { + "interface_version": 2, + "operation": "start", + "agent": "blocking", + "config": config.to_mapping(), + } + ) + assert started["status"] == "succeeded", started + runtime_id = started["result"]["runtime_id"] + try: + later_client = Fabric() + active = await later_client.runtime_status( + environment=server_environment, runtime_id=runtime_id, agent_name="blocking" + ) + assert active.status == "active" + assert active.server_state == "running" + assert active.handle.runtime_id == runtime_id + assert active.handle.environment["provider"] == "local" + stopped = await host.handle( + { + "interface_version": 2, + "operation": "stop", + "agent": "blocking", + "runtime_id": runtime_id, + } + ) + assert stopped["status"] == "succeeded" + report = await later_client.runtime_status( + environment=server_environment, runtime_id=runtime_id, agent_name="blocking" + ) + assert report.status == "stopped" + assert report.handle is None + commands = [json.loads(line) for line in log.read_text().splitlines()] + assert commands == [ + {"operation": "status", "agent": "blocking", "runtime_id": runtime_id}, + {"operation": "status", "agent": "blocking", "runtime_id": runtime_id}, + ] + assert not (tmp_path / "entered").exists() + finally: + await host.shutdown() + + +@pytest.mark.parametrize( + "change", + [ + {"runtime_id": "other"}, + {"agent_name": "other"}, + {"status": "unrecognized"}, + {"surprise": "TEST_ONLY_SECRET"}, + ], +) +async def test_runtime_status_remote_rejects_invalid_or_mismatched_reports( + server_environment, monkeypatch, change +): + from nemo_fabric import FabricConfigError + + native = MagicMock() + native.runtime_status_environment.return_value = json.dumps( + { + "runtime_id": "runtime-1", + "agent_name": "demo", + "status": "stopped", + "handle": None, + "server_state": "stopped", + **change, + } + ) + monkeypatch.setattr("nemo_fabric.client._native", native) + with pytest.raises(FabricConfigError): + await Fabric().runtime_status( + environment=server_environment, runtime_id="runtime-1", agent_name="demo" + ) + native.start_runtime.assert_not_called() + native.check_environment.assert_not_called() + + +async def test_runtime_status_remote_during_invocation_keeps_the_original_runtime( + blocking_adapter_config, server_environment, server_sdk_bridge, tmp_path +): + host, log = server_sdk_bridge + config = blocking_adapter_config.model_copy(deep=True) + config.environment.workspace = str(tmp_path) + config.discovery.local_paths = [str(tmp_path / "blocking.fabric-adapter.json")] + started = await host.handle( + { + "interface_version": 2, + "operation": "start", + "agent": "blocking", + "config": config.to_mapping(), + } + ) + assert started["status"] == "succeeded", started + runtime_id = started["result"]["runtime_id"] + invoking = asyncio.create_task( + host.handle( + { + "interface_version": 2, + "operation": "invoke", + "agent": "blocking", + "runtime_id": runtime_id, + "input": "work", + } + ) + ) + try: + async with asyncio.timeout(5): + while not (tmp_path / "entered").exists(): + await asyncio.sleep(0.01) + report = await asyncio.wait_for( + Fabric().runtime_status( + environment=server_environment, + runtime_id=runtime_id, + agent_name="blocking", + ), + 3, + ) + assert report.status == "active" + assert report.handle.runtime_id == runtime_id + assert not invoking.done() + assert not (tmp_path / "stopped").exists() + for agent_name, selected_id, expected in ( + ("other", runtime_id, "wrong_agent"), + ("blocking", "missing", "runtime_unavailable"), + ): + from nemo_fabric import FabricStateError + + with pytest.raises(FabricStateError) as caught: + await Fabric().runtime_status( + environment=server_environment, + runtime_id=selected_id, + agent_name=agent_name, + ) + assert caught.value.code == expected + finally: + (tmp_path / "release").write_text("release") + assert (await asyncio.wait_for(invoking, 5))["status"] == "succeeded" + await host.shutdown() + + +@pytest.mark.parametrize( + "raw", + [ + '{"runtime_id":"runtime-1","runtime_id":"other"}', + "null", + "[]", + "{", + "x" * (4 * 1024 * 1024 + 1), + ], +) +async def test_runtime_status_remote_rejects_malformed_or_oversized_json( + server_environment, monkeypatch, raw +): + from nemo_fabric import FabricConfigError + + native = MagicMock() + native.runtime_status_environment.return_value = raw + monkeypatch.setattr("nemo_fabric.client._native", native) + with pytest.raises(FabricConfigError, match="runtime status report is invalid"): + await Fabric().runtime_status( + environment=server_environment, runtime_id="runtime-1", agent_name="demo" + ) + native.start_runtime.assert_not_called() + native.invoke_runtime.assert_not_called() + native.stop_runtime.assert_not_called() + native.check_environment.assert_not_called() + + +@pytest.mark.parametrize( + "report_json", + [ + '{"runtime_id":"wrong-target","runtime_id":"runtime-1","agent_name":"demo","status":"stopped","handle":null,"server_state":"stopped"}', + '{"runtime_id":"runtime-1","agent_name":"demo","status":"failed","status":"stopped","handle":null,"server_state":"stopped"}', + ], +) +async def test_runtime_status_native_transport_rejects_duplicate_fields( + server_environment, tmp_path, report_json +): + from nemo_fabric import FabricRuntimeError + + provider = tmp_path / "duplicate-provider" + provider.write_text( + f"#!{sys.executable}\n" + + "import json, sys\n" + + "frame = json.loads(sys.stdin.readline())\n" + + "prefix = json.dumps({'protocol_version': frame['protocol_version'], 'request_id': frame['request_id'], 'status': 'succeeded'})[:-1]\n" + + f'print(prefix + \', "output": {{"status": "returned", "value": \' + {report_json!r} + \'}}}}\', flush=True)\n' + ) + provider.chmod(0o700) + os.environ["NEMO_FABRIC_OPEN_SHELL_PROVIDER"] = str(provider) + with pytest.raises(FabricRuntimeError, match="runtime status transport failed"): + await Fabric().runtime_status( + environment=server_environment, runtime_id="runtime-1", agent_name="demo" + ) diff --git a/tests/python/test_streaming.py b/tests/python/test_streaming.py index 790d9a14d..946391289 100644 --- a/tests/python/test_streaming.py +++ b/tests/python/test_streaming.py @@ -333,6 +333,12 @@ async def test_start_runtime_creates_runtime_owned_collectors( native_client: Fabric, mock_native: MagicMock, ): + mock_native.start_runtime.side_effect = [ + json.dumps(_runtime()), + json.dumps( + {**_runtime(), "runtime_id": "runtime-2", "runtime_binding": "binding-2"} + ), + ] first = await native_client.start_runtime(_config(relay=True), streaming=True) second = await native_client.start_runtime(_config(relay=True), streaming=True) try: @@ -342,6 +348,7 @@ async def test_start_runtime_creates_runtime_owned_collectors( first_sink = first_plan["relay"]["observability"]["atof"]["sinks"][-1] second_sink = second_plan["relay"]["observability"]["atof"]["sinks"][-1] + assert first.runtime_id != second.runtime_id assert first._collector is not second._collector assert first_sink["url"] != second_sink["url"] finally: @@ -792,9 +799,7 @@ def no_agent_run(plan_json: str, runtime_json: str, request_json: str) -> str: "message": "Pi stopped before starting an agent run", "retryable": False, }, - "metadata": { - "adapter": {"pi_turn_started": False, "pi_turn_count": 0} - }, + "metadata": {"adapter": {"pi_turn_started": False, "pi_turn_count": 0}}, } ) return json.dumps(result) From 18b214bbb90b86f7137af7f874d92a5fa2e40cf7 Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Thu, 5 Nov 2026 21:24:15 -0700 Subject: [PATCH 2/6] fix: bound runtime records and publish the status schema Signed-off-by: Sara Tadayon --- .agents/skills/contribute-api/SKILL.md | 2 +- README.md | 2 +- crates/fabric-core/src/schema.rs | 48 +++- .../nemo_fabric.client.md | 4 +- .../schema/enum-schemaname.mdx | 9 +- docs/sdk/python.mdx | 4 +- schemas/SCHEMA.md | 2 +- schemas/sdk/runtime-status-report.schema.json | 233 ++++++++++++++++++ .../src/nemo_fabric/client.py | 121 ++++++--- .../src/nemo_fabric/runtime_server/README.md | 2 +- .../nemo_fabric/runtime_server/protocol.py | 1 + .../src/nemo_fabric/runtime_server/state.py | 7 + skills/nemo-fabric-integrate/SKILL.md | 2 +- tests/conftest.py | 12 + tests/python/runtime_server/test_state.py | 23 +- tests/python/test_runtime.py | 207 +++++++++++++++- tests/python/test_sdk_server_checks.py | 46 +++- 17 files changed, 681 insertions(+), 44 deletions(-) create mode 100644 schemas/sdk/runtime-status-report.schema.json diff --git a/.agents/skills/contribute-api/SKILL.md b/.agents/skills/contribute-api/SKILL.md index 460af359a..dc1ba7a05 100644 --- a/.agents/skills/contribute-api/SKILL.md +++ b/.agents/skills/contribute-api/SKILL.md @@ -25,7 +25,7 @@ runtime or bindings. - Keep every affected public surface in parity - `RunRequest.relay_session_root` is forwarded as the typed `AgentRunRequest.relay_session_root` field. Keep Rust, Python, TypeScript, and schema representations in parity. Context keys do not control Relay propagation; preserve the adapter fallback for unusable UUIDs. - OpenShell task commands serialize public SDK config/request/handle/result mappings. Keep forwarding buffered and preserve task-generated IDs, normalized failed results, telemetry references, and declared artifacts. Do not normalize an SDK result as an adapter result. Task paths remain POSIX strings; host artifact destinations stay separate. Unknown outcomes bind the immutable sandbox identity through reattachment and must not be replayed. -- Runtime discovery uses the typed public `Fabric.runtime_status()` operation. Keep `fabric-agent-ctl status` delegated to that SDK operation and OpenShell status routed through the existing provider command transport. Preserve recorded lifecycle state separately from health observations, use null handles for stopped records, and exclude environment variables and arbitrary provider metadata from discovery snapshots. +- Runtime discovery uses the typed public `Fabric.runtime_status()` operation and `sdk/runtime-status-report` schema. Keep `fabric-agent-ctl status` delegated to that SDK operation and OpenShell status routed through the existing provider command transport. Preserve recorded lifecycle state separately from health observations, use null handles for stopped records, and exclude environment variables and arbitrary provider metadata from discovery snapshots. Each client caps all runtime records, including pending starts, at 128; evict only stopped records and reject full-capacity starts with `runtime_limit_reached` before runtime effects. - Update docs and examples in the same branch ## Minimum Acceptance diff --git a/README.md b/README.md index 9b3937ec1..d3430569a 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ NeMo Fabric separates consumers from execution targets through a stable, version - [Installation](docs/getting-started/install.mdx): Install the runtime, adapters, harness dependencies, and telemetry components, and choose a deployment scenario. - [Adapter catalog](sdk/python/nemo-fabric-adapter-catalog/README.md): Inspect bundled adapter metadata on a host without installing harness SDKs. - [Host admission](docs/sdk/python.mdx#inspect-metadata-on-a-separate-host): Validate requested capabilities and reject host/task descriptor drift before execution. -- [NeMo Fabric runtime server](sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md): Manage runtimes, rediscover their recorded status and safe handles, and check server responsiveness inside one sandbox through `fabric-agent-ctl` and one private local socket. Unconfirmed cleanup retains sandbox ownership and blocks replacement servers. +- [NeMo Fabric runtime server](sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md): Manage runtimes, rediscover their recorded status and safe handles, and check server responsiveness inside one sandbox through `fabric-agent-ctl` and one private local socket. Each SDK client keeps at most 128 runtime records, evicting stopped records first and rejecting new starts when active, failed, and pending runtimes fill the limit. Unconfirmed cleanup retains sandbox ownership and blocks replacement servers. - [Health checks](docs/sdk/health-checks.mdx): Use `Fabric.check()` for OpenShell runtime-server checks, agent runtime checks, and prepared or attached OpenClaw Gateway service checks. Refer to the [adapter support table](docs/sdk/health-checks.mdx#native-evidence-by-adapter) for supported levels and native evidence. OpenShell server and runtime requests map to `fabric-agent-ctl check`; service checks use the local SDK. - [Documentation overview](docs/about-nemo-fabric/overview.mdx): Understand the execution model and choose an interface. - [Adapter reference](adapters/README.md): Compare adapter configuration, runtime, and observability support. diff --git a/crates/fabric-core/src/schema.rs b/crates/fabric-core/src/schema.rs index 1c88436b2..159f15a47 100644 --- a/crates/fabric-core/src/schema.rs +++ b/crates/fabric-core/src/schema.rs @@ -62,6 +62,8 @@ pub enum SchemaName { EnvironmentReference, /// Runtime handle schema. RuntimeHandle, + /// Recorded SDK runtime state and sanitized discovery handle. + RuntimeStatusReport, /// Caller-supplied reference to an already-running service. ServiceReference, /// Prepared or attached service handle schema. @@ -84,7 +86,7 @@ pub enum SchemaName { impl SchemaName { /// All public schemas in stable output order. - pub const ALL: [Self; 28] = [ + pub const ALL: [Self; 29] = [ Self::ExecutionEvidence, Self::CheckRequest, Self::CheckReport, @@ -104,6 +106,7 @@ impl SchemaName { Self::EnvironmentHandle, Self::EnvironmentReference, Self::RuntimeHandle, + Self::RuntimeStatusReport, Self::ServiceReference, Self::ServiceHandle, Self::InvocationHandle, @@ -137,6 +140,7 @@ impl SchemaName { Self::EnvironmentHandle => "environment-handle", Self::EnvironmentReference => "environment-reference", Self::RuntimeHandle => "runtime-handle", + Self::RuntimeStatusReport => "runtime-status-report", Self::ServiceReference => "service-reference", Self::ServiceHandle => "service-handle", Self::InvocationHandle => "invocation-handle", @@ -200,6 +204,7 @@ impl SchemaName { "environment-handle" | "environment_handle" => Ok(Self::EnvironmentHandle), "environment-reference" | "environment_reference" => Ok(Self::EnvironmentReference), "runtime-handle" | "runtime_handle" => Ok(Self::RuntimeHandle), + "runtime-status-report" | "runtime_status_report" => Ok(Self::RuntimeStatusReport), "service-reference" | "service_reference" => Ok(Self::ServiceReference), "service-handle" | "service_handle" => Ok(Self::ServiceHandle), "invocation-handle" | "invocation_handle" => Ok(Self::InvocationHandle), @@ -246,6 +251,7 @@ pub fn generate_schema(schema: SchemaName) -> Result { SchemaName::EnvironmentHandle => to_value(schema_for!(EnvironmentHandle)), SchemaName::EnvironmentReference => to_value(schema_for!(EnvironmentReference)), SchemaName::RuntimeHandle => to_value(schema_for!(RuntimeHandle)), + SchemaName::RuntimeStatusReport => runtime_status_report_schema(), SchemaName::ServiceReference => to_value(schema_for!(ServiceReference)), SchemaName::ServiceHandle => to_value(schema_for!(ServiceHandle)), SchemaName::InvocationHandle => to_value(schema_for!(InvocationHandle)), @@ -258,6 +264,45 @@ pub fn generate_schema(schema: SchemaName) -> Result { } } +/// Generate the Python-owned status wire contract from the shared handle schema. +/// Status lookup remains an SDK operation; Rust only transports its JSON result. +fn runtime_status_report_schema() -> Result { + let mut handle = generate_schema(SchemaName::RuntimeHandle)?; + let mut definitions = handle.as_object_mut().unwrap().remove("$defs").unwrap(); + handle.as_object_mut().unwrap().remove("$schema"); + handle["additionalProperties"] = Value::Bool(false); + let environment = definitions["EnvironmentHandle"]["properties"] + .as_object_mut() + .unwrap(); + for field in ["env", "connection", "metadata"] { + environment.remove(field); + } + definitions["RuntimeHandle"] = handle; + let identity = serde_json::json!({ + "type": "string", "minLength": 1, "pattern": "\\S", + "not": {"pattern": "\\u0000"} + }); + Ok(serde_json::json!({ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "RuntimeStatusReport", + "description": "Recorded lifecycle state and a sanitized discovery handle, not a health observation.", + "type": "object", + "additionalProperties": false, + "required": ["runtime_id", "agent_name", "status", "handle", "server_state"], + "properties": { + "runtime_id": identity, + "agent_name": identity, + "status": {"enum": ["active", "stopped", "failed"]}, + "handle": {"anyOf": [{"$ref": "#/$defs/RuntimeHandle"}, {"type": "null"}]}, + "server_state": {"enum": ["running", "stopping", "stopped", "unknown", null]} + }, + "if": {"properties": {"status": {"const": "stopped"}}}, + "then": {"properties": {"handle": {"type": "null"}}}, + "else": {"properties": {"handle": {"$ref": "#/$defs/RuntimeHandle"}}}, + "$defs": definitions + })) +} + /// Generate all schemas keyed by stable schema name. pub fn generate_all_schemas() -> Result> { SchemaName::ALL @@ -369,6 +414,7 @@ mod tests { SchemaName::EnvironmentHandle, SchemaName::EnvironmentReference, SchemaName::RuntimeHandle, + SchemaName::RuntimeStatusReport, SchemaName::InvocationHandle, SchemaName::RunRequest, SchemaName::RunResult, diff --git a/docs/reference/api/python-library-reference/nemo_fabric.client.md b/docs/reference/api/python-library-reference/nemo_fabric.client.md index 4f5012749..274814359 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.client.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.client.md @@ -390,7 +390,7 @@ async def runtime_status( Read recorded runtime state and rediscover a safe handle by identifier. -Without an environment, read runtimes created by this ``Fabric`` instance and retain each ``Runtime`` until successful stop. The most recent 128 successful stops retain lightweight records with a null handle. A local environment additionally selects its exact environment identifier. +Without an environment, read runtimes created by this ``Fabric`` instance and retain each ``Runtime`` until successful stop. Each client holds at most 128 runtime records, including pending starts. New starts evict the oldest stopped records first and fail with ``runtime_limit_reached`` if active, failed, and pending runtimes fill the limit. A stopped record has a null handle. A local environment also selects its exact identifier. An OpenShell environment routes ``fabric-agent-ctl status`` to the owning server, including from a later client process. The returned handle belongs to that server's local environment. Records do not survive server restart. This operation never starts, invokes, stops, or probes a runtime. Recorded ``active`` state does not establish current harness health; use ``check()`` for a fresh observation. @@ -462,6 +462,7 @@ Each call starts a new logical runtime. Runtime-scoped overrides are recursively **Raises:** - `FabricConfigError`: If inputs or overrides are invalid, streaming is requested without NeMo Relay enabled, ``launch_collector`` is set without streaming, Pi is configured with an external collector, or an external collector has no sink. + - `FabricStateError`: If active, failed, and pending runtimes fill the client's 128-record limit (``runtime_limit_reached``). - `FabricNativeUnavailableError`: If the native extension is not installed. - `FabricRuntimeError`: If runtime startup fails. @@ -505,6 +506,7 @@ Starting or stopping the runtime does not release ``environment``. Runtime and r **Raises:** - `FabricConfigError`: If inputs are invalid or the handle does not match the plan. + - `FabricStateError`: If active, failed, and pending runtimes fill the client's 128-record limit (``runtime_limit_reached``). - `FabricNativeUnavailableError`: If the native extension is not installed. - `FabricRuntimeError`: If runtime startup fails. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx index 266b81990..921f4ad04 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx @@ -30,6 +30,7 @@ pub enum SchemaName { EnvironmentHandle, EnvironmentReference, RuntimeHandle, + RuntimeStatusReport, ServiceReference, ServiceHandle, InvocationHandle, @@ -160,6 +161,12 @@ Existing environment reference schema. Runtime handle schema. +### `RuntimeStatusReport` + +
+ +Recorded SDK runtime state and sanitized discovery handle. + ### `ServiceReference`
@@ -222,7 +229,7 @@ Runtime resource snapshot schema. #### `ALL` -
28]"}} />
+
29]"}} />
All public schemas in stable output order. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 710c17dd2..c7223516b 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -245,7 +245,7 @@ report = await fabric.runtime_status( print(report.status) # "active", "stopped", or "failed" ``` -A direct local lookup selects a runtime started by the same `Fabric` instance. The client retains live runtime objects until successful stop, even when the caller drops the original Python reference. Callers remain responsible for stopping runtimes. An optional local `EnvironmentHandle` restricts the lookup to that environment. After successful stop, the latest 128 records remain available with `handle=None`; older records are unavailable. +A direct local lookup selects a runtime started by the same `Fabric` instance. Each client holds at most 128 records across active, failed, stopped, and pending runtimes. New starts evict the oldest successfully stopped records first. When active, failed, and pending runtimes fill the limit, startup raises `FabricStateError` with `code="runtime_limit_reached"` before creating runtime resources. The client retains active and failed runtime objects until successful stop, even when the caller drops the original Python reference. Callers remain responsible for stopping runtimes. An optional local `EnvironmentHandle` restricts the lookup to that environment. Retained stopped records have `handle=None`; evicted records are unavailable. For OpenShell, pass a verified environment handle and the runtime identifier returned by start. The SDK sends `fabric-agent-ctl status` to the original owning server, so a later client process can perform the same read: @@ -257,7 +257,7 @@ report = await Fabric().runtime_status( ) ``` -`RuntimeStatusReport` contains `runtime_id`, `agent_name`, `status`, `handle`, and `server_state`. The server state is `running`, `stopping`, `stopped`, or `unknown`; direct local reads use `None`. Missing runtimes, wrong agents, and mismatched local environment identifiers raise `FabricStateError`. The SDK validates remote report identity and uses fixed diagnostics for transport failures. +`RuntimeStatusReport` contains `runtime_id`, `agent_name`, `status`, `handle`, and `server_state`. The [runtime status schema](https://github.com/NVIDIA/NeMo-Fabric/blob/main/schemas/sdk/runtime-status-report.schema.json) defines the JSON shape for SDK and command consumers. The server state is `running`, `stopping`, `stopped`, or `unknown`; direct local reads use `None`. Missing runtimes, wrong agents, and mismatched local environment identifiers raise `FabricStateError`. The SDK validates remote report identity and uses fixed diagnostics for transport failures. The returned handle is a discovery snapshot. The snapshot retains runtime identity, the opaque binding, adapter and service identifiers, and environment identity and paths. The snapshot excludes environment variable values, provider connection metadata, provider metadata, and unknown handle extensions. Use the original `Runtime` for invocation and cleanup. OpenShell reports the handle owned inside the sandbox, rather than a new host-side runtime binding. Stopped runtimes have no current handle. Records are held by the owning process and do not survive server restart. diff --git a/schemas/SCHEMA.md b/schemas/SCHEMA.md index b42206626..ec246bade 100644 --- a/schemas/SCHEMA.md +++ b/schemas/SCHEMA.md @@ -6,7 +6,7 @@ SPDX-License-Identifier: Apache-2.0 # NVIDIA NeMo Fabric JSON Schemas This directory contains committed JSON Schema snapshots for the public NeMo Fabric -contract. The files are generated from the Rust core types, not edited by hand. +contract. The files are generated by the Rust core schema generator, not edited by hand. Most schemas derive from Rust types; `sdk/runtime-status-report` describes the Python-owned status operation and reuses the generated runtime handle shape with discovery-only redaction. The Rust status transport passes JSON through without adding a public Rust lifecycle API. The Python adapter-contract package exposes standard-library dataclasses with optional Pydantic interoperability. Shared evidence types and validation live in diff --git a/schemas/sdk/runtime-status-report.schema.json b/schemas/sdk/runtime-status-report.schema.json new file mode 100644 index 000000000..d13d699e3 --- /dev/null +++ b/schemas/sdk/runtime-status-report.schema.json @@ -0,0 +1,233 @@ +{ + "$defs": { + "AdapterKind": { + "description": "Adapter implementation kind.", + "oneOf": [ + { + "const": "process", + "description": "Launch and supervise a persistent adapter process.", + "type": "string" + }, + { + "const": "http", + "description": "Connect to a service or HTTP-backed harness.", + "type": "string" + }, + { + "const": "python", + "description": "Launch and supervise a persistent Python adapter host.", + "type": "string" + }, + { + "const": "native_plugin", + "description": "Delegate to a harness-native plugin package.", + "type": "string" + } + ] + }, + "ControlLocation": { + "description": "Where NeMo Fabric control code runs relative to the environment.", + "oneOf": [ + { + "const": "external_control", + "description": "NeMo Fabric runs on the host/control plane and starts or connects to the harness in the environment.", + "type": "string" + }, + { + "const": "in_env_control", + "description": "NeMo Fabric runs inside the prepared environment with the harness.", + "type": "string" + } + ] + }, + "EnvironmentHandle": { + "additionalProperties": false, + "description": "Resolved execution environment context.", + "properties": { + "artifacts": { + "description": "Artifact root visible to the harness runtime. For non-local providers this\nis an environment-local path and must not be resolved on the host.", + "type": [ + "string", + "null" + ] + }, + "control_location": { + "$ref": "#/$defs/ControlLocation", + "description": "Where NeMo Fabric control code runs." + }, + "environment_id": { + "description": "Environment handle id.", + "type": "string" + }, + "ownership": { + "$ref": "#/$defs/EnvironmentOwnership", + "description": "Whether NeMo Fabric owns the environment resource." + }, + "provider": { + "description": "Environment provider.", + "type": "string" + }, + "workspace": { + "description": "Workspace visible to the harness runtime. For non-local providers this is\nan environment-local path and must not be resolved on the host.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "environment_id", + "provider", + "control_location", + "ownership" + ], + "type": "object" + }, + "EnvironmentOwnership": { + "description": "Whether NeMo Fabric owns the underlying environment resource.", + "oneOf": [ + { + "const": "caller_owned", + "description": "The caller or a surrounding system owns the environment resource.", + "type": "string" + }, + { + "const": "fabric_owned", + "description": "NeMo Fabric created or leased the environment resource and may release it.", + "type": "string" + } + ] + }, + "RuntimeHandle": { + "additionalProperties": false, + "description": "Active or resumable harness runtime.", + "properties": { + "adapter_id": { + "description": "Adapter implementation id.", + "type": [ + "string", + "null" + ] + }, + "adapter_kind": { + "$ref": "#/$defs/AdapterKind", + "description": "Adapter kind." + }, + "agent_name": { + "description": "Agent name.", + "type": "string" + }, + "environment": { + "$ref": "#/$defs/EnvironmentHandle", + "description": "Prepared environment." + }, + "harness": { + "description": "Stable machine-readable harness identifier.", + "type": "string" + }, + "runtime_binding": { + "description": "NeMo Fabric-owned opaque binding for this runtime handle.", + "type": "string" + }, + "runtime_id": { + "description": "Runtime handle id.", + "type": "string" + }, + "service_id": { + "description": "Service used by this runtime, when it is connected to a shared service.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "runtime_id", + "runtime_binding", + "agent_name", + "harness", + "adapter_kind", + "environment" + ], + "title": "RuntimeHandle", + "type": "object" + } + }, + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "description": "Recorded lifecycle state and a sanitized discovery handle, not a health observation.", + "else": { + "properties": { + "handle": { + "$ref": "#/$defs/RuntimeHandle" + } + } + }, + "if": { + "properties": { + "status": { + "const": "stopped" + } + } + }, + "properties": { + "agent_name": { + "minLength": 1, + "not": { + "pattern": "\\u0000" + }, + "pattern": "\\S", + "type": "string" + }, + "handle": { + "anyOf": [ + { + "$ref": "#/$defs/RuntimeHandle" + }, + { + "type": "null" + } + ] + }, + "runtime_id": { + "minLength": 1, + "not": { + "pattern": "\\u0000" + }, + "pattern": "\\S", + "type": "string" + }, + "server_state": { + "enum": [ + "running", + "stopping", + "stopped", + "unknown", + null + ] + }, + "status": { + "enum": [ + "active", + "stopped", + "failed" + ] + } + }, + "required": [ + "runtime_id", + "agent_name", + "status", + "handle", + "server_state" + ], + "then": { + "properties": { + "handle": { + "type": "null" + } + } + }, + "title": "RuntimeStatusReport", + "type": "object" +} \ No newline at end of file diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py index ef912a5b9..4e4e289cb 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py @@ -68,7 +68,7 @@ # Dedicated workers run health requests during invocations. _CHECK_EXECUTOR = ThreadPoolExecutor(max_workers=4, thread_name_prefix="fabric-health") logger = logging.getLogger(__name__) -_MAX_STOPPED_STATUS_RECORDS = 128 +_MAX_RUNTIME_RECORDS = 128 class Fabric: @@ -99,6 +99,8 @@ def __init__(self) -> None: self._execution_participant = "caller_sdk" self._software_snapshot = None self._software_snapshot_lock = Lock() + self._runtime_records_lock = Lock() + self._pending_runtime_starts: set[object] = set() self._runtimes: dict[str, Runtime] = {} self._stopped_status: OrderedDict[ str, tuple[tuple[str, str], RuntimeStatusReport] @@ -114,9 +116,11 @@ async def runtime_status( """Read recorded runtime state and rediscover a safe handle by identifier. Without an environment, read runtimes created by this ``Fabric`` instance - and retain each ``Runtime`` until successful stop. The most recent 128 - successful stops retain lightweight records with a null handle. A local - environment additionally selects its exact environment identifier. + and retain each ``Runtime`` until successful stop. Each client holds at + most 128 runtime records, including pending starts. New starts evict the + oldest stopped records first and fail with ``runtime_limit_reached`` if + active, failed, and pending runtimes fill the limit. A stopped record has + a null handle. A local environment also selects its exact identifier. An OpenShell environment routes ``fabric-agent-ctl status`` to the owning server, including from a later client process. The returned handle belongs @@ -253,28 +257,27 @@ def unique(pairs): return report def _record_stopped_runtime(self, runtime: Runtime) -> None: - if ( - runtime.status is not RuntimeStatus.STOPPED - or self._runtimes.get(runtime.runtime_id) is not runtime - ): - return - handle = runtime.handle - self._stopped_status[runtime.runtime_id] = ( - (handle.environment["provider"], handle.environment["environment_id"]), - RuntimeStatusReport.from_mapping( - { - "runtime_id": runtime.runtime_id, - "agent_name": handle.agent_name, - "status": "stopped", - "handle": None, - "server_state": None, - } - ), - ) - self._stopped_status.move_to_end(runtime.runtime_id) - self._runtimes.pop(runtime.runtime_id, None) - while len(self._stopped_status) > _MAX_STOPPED_STATUS_RECORDS: - self._stopped_status.popitem(last=False) + with self._runtime_records_lock: + if ( + runtime.status is not RuntimeStatus.STOPPED + or self._runtimes.get(runtime.runtime_id) is not runtime + ): + return + handle = runtime.handle + self._stopped_status[runtime.runtime_id] = ( + (handle.environment["provider"], handle.environment["environment_id"]), + RuntimeStatusReport.from_mapping( + { + "runtime_id": runtime.runtime_id, + "agent_name": handle.agent_name, + "status": "stopped", + "handle": None, + "server_state": None, + } + ), + ) + self._stopped_status.move_to_end(runtime.runtime_id) + self._runtimes.pop(runtime.runtime_id, None) def _set_execution_participant(self, participant: str) -> None: """Relabel only this instance's own cached observations when embedded.""" @@ -610,6 +613,8 @@ async def start_runtime( requested without NeMo Relay enabled, ``launch_collector`` is set without streaming, Pi is configured with an external collector, or an external collector has no sink. + FabricStateError: If active, failed, and pending runtimes fill the + client's 128-record limit (``runtime_limit_reached``). FabricNativeUnavailableError: If the native extension is not installed. FabricRuntimeError: If runtime startup fails. @@ -785,6 +790,8 @@ async def start_runtime_in( Raises: FabricConfigError: If inputs are invalid or the handle does not match the plan. + FabricStateError: If active, failed, and pending runtimes fill the + client's 128-record limit (``runtime_limit_reached``). FabricNativeUnavailableError: If the native extension is not installed. FabricRuntimeError: If runtime startup fails. """ @@ -840,6 +847,54 @@ async def _start_runtime( completion_wait_timeout: float = 1.0, service: Service | None = None, expected_descriptor_sha256: str | None = None, + ) -> Runtime: + reservation = object() + with self._runtime_records_lock: + occupied = ( + len(self._runtimes) + + len(self._stopped_status) + + len(self._pending_runtime_starts) + ) + while occupied >= _MAX_RUNTIME_RECORDS and self._stopped_status: + self._stopped_status.popitem(last=False) + occupied -= 1 + if occupied >= _MAX_RUNTIME_RECORDS: + raise FabricStateError( + "The client runtime limit has been reached; stop an owned runtime before starting another.", + stage="start", + code="runtime_limit_reached", + ) + self._pending_runtime_starts.add(reservation) + try: + return await self._start_runtime_reserved( + config, + reservation=reservation, + environment=environment, + base_dir=base_dir, + overrides=overrides, + streaming=streaming, + launch_collector=launch_collector, + completion_wait_timeout=completion_wait_timeout, + service=service, + expected_descriptor_sha256=expected_descriptor_sha256, + ) + finally: + with self._runtime_records_lock: + self._pending_runtime_starts.discard(reservation) + + async def _start_runtime_reserved( + self, + config: FabricConfig, + *, + reservation: object, + environment: EnvironmentHandle | None = None, + base_dir: str | os.PathLike[str] | None = None, + overrides: Mapping[str, Any] | None = None, + streaming: bool = False, + launch_collector: bool | None = None, + completion_wait_timeout: float = 1.0, + service: Service | None = None, + expected_descriptor_sha256: str | None = None, ) -> Runtime: runtime_overrides = _json_mapping(overrides, "runtime overrides") if service is not None and not isinstance(service, Service): @@ -1033,10 +1088,15 @@ def start() -> dict[str, Any]: collector=collector, collector_client=collector_client, ) - if ( - owned_runtime.runtime_id in self._runtimes - or owned_runtime.runtime_id in self._stopped_status - ): + with self._runtime_records_lock: + duplicate = ( + owned_runtime.runtime_id in self._runtimes + or owned_runtime.runtime_id in self._stopped_status + ) + if not duplicate: + self._runtimes[owned_runtime.runtime_id] = owned_runtime + self._pending_runtime_starts.remove(reservation) + if duplicate: # A duplicate native identity cannot safely replace an existing owner # or be stopped independently. The enclosing server retains its # uncertain-start marker so the supervisor can replace the sandbox. @@ -1046,7 +1106,6 @@ def start() -> dict[str, Any]: stage="start", code="runtime_id_conflict", ) - self._runtimes[owned_runtime.runtime_id] = owned_runtime return owned_runtime async def prepare_service( diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md index f9fc55321..f67c18433 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md @@ -42,7 +42,7 @@ Use `--request FILE` to forward a complete public `RunRequest`, including `reque `status --agent NAME --runtime-id ID` returns the existing SDK ownership record without starting, invoking, stopping, or probing the harness. The result is a `RuntimeStatusReport` mapping with `runtime_id`, `agent_name`, `status` (`active`, `stopped`, or `failed`), `handle`, and the server's recorded `server_state` (`running`, `stopping`, `stopped`, or `unknown`). Recorded active state does not establish current process health. Status reads bypass the invocation lock. A valid read exits 0, including a known stopped or failed runtime; missing or mismatched ownership exits 1. `changed` is always false, and status errors always report `effects: "none"`. -The handle is a discovery snapshot with runtime identity, opaque binding, adapter and service identifiers, and environment identity and paths. Environment variables, provider connection metadata, provider metadata, and unknown handle extensions are excluded. The original `Runtime` remains the lifecycle handle for invocation and cleanup. Successful stop removes the current handle, so status returns `handle: null`; the most recent 128 stopped records remain available. A later CLI process can retrieve a record while the original server remains alive. Records do not survive server replacement. Native-interface metadata remains separate future work. +The handle is a discovery snapshot with runtime identity, opaque binding, adapter and service identifiers, and environment identity and paths. Environment variables, provider connection metadata, provider metadata, and unknown handle extensions are excluded. The original `Runtime` remains the lifecycle handle for invocation and cleanup. Successful stop removes the current handle, so status returns `handle: null`. The owning SDK client holds at most 128 runtime records, including pending starts. New starts evict the oldest stopped records first; if active, failed, and pending runtimes fill the limit, startup returns `runtime_limit_reached` with no effects and can retry after successful cleanup. A later CLI process can retrieve a retained record while the original server remains alive. Records do not survive server replacement. Native-interface metadata remains separate future work. `check` reports server responsiveness. Add both `--agent` and `--runtime-id` to check one runtime. Runtime checks default to `live`; the mutually exclusive `--active` and `--ready` flags include preceding levels. Checks can run during invocations, and busy agents can pass. diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/protocol.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/protocol.py index 5d1887bda..fb9463bd2 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/protocol.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/protocol.py @@ -37,6 +37,7 @@ "start_failed": "Fabric could not establish an active runtime.", "start_uncertain": "A previous runtime start has an uncertain outcome; replace the sandbox before starting another.", "runtime_id_conflict": "Fabric returned a runtime ID that is already in use.", + "runtime_limit_reached": "The client runtime limit has been reached; stop an owned runtime before starting another.", "status_failed": "Fabric could not read the recorded runtime status.", "stop_failed": "Fabric could not complete the stop operation.", "invoke_failed": "Fabric did not report a successful invocation.", diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py index f42ffd1df..7f975bfd9 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py @@ -156,6 +156,13 @@ async def handle(self, request: Any) -> dict[str, Any]: runtime = await self.fabric.start_runtime( config, base_dir=self.directory ) + except FabricStateError as error: + if error.code != "runtime_limit_reached": + raise + self.has_used_runtime = previously_used + self.start_uncertain = False + changed, effects = False, "none" + raise ProtocolError("runtime_limit_reached", "start") from None except FabricConfigError: self.has_used_runtime = previously_used self.start_uncertain = False diff --git a/skills/nemo-fabric-integrate/SKILL.md b/skills/nemo-fabric-integrate/SKILL.md index 8c7c049d1..53d2e3060 100644 --- a/skills/nemo-fabric-integrate/SKILL.md +++ b/skills/nemo-fabric-integrate/SKILL.md @@ -230,7 +230,7 @@ runtime before calling `release_environment(environment)` separately. Provider availability and reuse limits vary; task paths must remain strings for non-local environments. See the [environment lifecycle guide](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/sdk/python.mdx#explicit-environment-lifecycle). -Use `await fabric.runtime_status(runtime_id=runtime_id, agent_name=name)` to read recorded state and a safe discovery handle for a runtime started by the same `Fabric` instance. The client retains live runtime objects until successful stop. Add `environment=environment` for an OpenShell lookup through `fabric-agent-ctl status`, including from a later client process while the original server remains alive. `RuntimeStatusReport` reports `active`, `stopped`, or `failed`; `server_state` is present for server reads and null for direct local reads. Successful stops retain up to 128 lightweight records with a null handle. Discovery handles omit environment variable values and arbitrary provider metadata; use the original `Runtime` for invocation and cleanup. Status reads perform no lifecycle operation or health probe. Use `Fabric.check()` when the consumer needs fresh health evidence. +Use `await fabric.runtime_status(runtime_id=runtime_id, agent_name=name)` to read recorded state and a safe discovery handle for a runtime started by the same `Fabric` instance. The client retains live runtime objects until successful stop. Add `environment=environment` for an OpenShell lookup through `fabric-agent-ctl status`, including from a later client process while the original server remains alive. `RuntimeStatusReport` reports `active`, `stopped`, or `failed`; `server_state` is present for server reads and null for direct local reads. Each client holds at most 128 runtime records, including pending starts. New starts evict the oldest successfully stopped records first; when active, failed, and pending runtimes fill the limit, startup raises `FabricStateError` with `code="runtime_limit_reached"`. Retained stopped records have a null handle. Discovery handles omit environment variable values and arbitrary provider metadata; use the original `Runtime` for invocation and cleanup. Status reads perform no lifecycle operation or health probe. Use `Fabric.check()` when the consumer needs fresh health evidence. The selected adapter owns the execution topology. The bundled Claude, Codex, Deep Agents, and Hermes Agent adapters retain their native client, graph/checkpointer, diff --git a/tests/conftest.py b/tests/conftest.py index c481186a0..1a770d8f8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -69,6 +69,18 @@ def repo_root_fixture() -> Path: return CUR_DIR.parent.resolve() +@pytest.fixture(name="runtime_status_validator") +def runtime_status_validator_fixture(repo_root): + import json + from jsonschema import Draft202012Validator + + schema = json.loads( + (repo_root / "schemas/sdk/runtime-status-report.schema.json").read_text() + ) + Draft202012Validator.check_schema(schema) + return Draft202012Validator(schema) + + @pytest.fixture(name="default_skill") def default_skill_fixture() -> Path: return CUR_DIR / "fixtures" / "default" diff --git a/tests/python/runtime_server/test_state.py b/tests/python/runtime_server/test_state.py index 61949721b..65f720657 100644 --- a/tests/python/runtime_server/test_state.py +++ b/tests/python/runtime_server/test_state.py @@ -6,7 +6,7 @@ import asyncio import pytest -from nemo_fabric import FabricConfigError, RunRequest, RuntimeStatus +from nemo_fabric import FabricConfigError, FabricStateError, RunRequest, RuntimeStatus from nemo_fabric.errors import FabricRuntimeError from nemo_fabric.runtime_server.protocol import ProtocolError @@ -33,6 +33,27 @@ async def test_start_delegates_to_existing_sdk( mock_fabric.start_runtime.assert_awaited_once() +@pytest.mark.parametrize("previously_used", [False, True]) +async def test_capacity_rejection_preserves_server_ownership_and_fixed_diagnostics( + host, host_request, mock_fabric, runtime_factory, previously_used +): + if previously_used: + assert (await host.handle(host_request("start")))["status"] == "succeeded" + mock_fabric.start_runtime.side_effect = FabricStateError( + "TEST_ONLY_PRIVATE", stage="start", code="runtime_limit_reached" + ) + response = await host.handle(host_request("start")) + assert response["error"]["code"] == "runtime_limit_reached" + assert response["error"]["effects"] == "none" + assert response["changed"] is False + assert not host.start_uncertain + assert host.has_used_runtime is previously_used + assert "TEST_ONLY_PRIVATE" not in str(response) + mock_fabric.start_runtime.side_effect = None + mock_fabric.start_runtime.return_value = runtime_factory("replacement") + assert (await host.handle(host_request("start")))["status"] == "succeeded" + + @pytest.mark.parametrize( "field", ["expected_descriptor_fingerprint", "expected_generation", "surprise"] ) diff --git a/tests/python/test_runtime.py b/tests/python/test_runtime.py index cf5aaa624..d2799e911 100644 --- a/tests/python/test_runtime.py +++ b/tests/python/test_runtime.py @@ -19,6 +19,7 @@ Fabric, FabricConfig, FabricConfigError, + FabricError, FabricNativeUnavailableError, FabricRuntimeError, FabricStateError, @@ -1298,7 +1299,7 @@ async def test_runtime_status_preserves_live_runtime_after_caller_drops_referenc async def test_runtime_status_stopped_records_are_bounded( native_client, mock_native, monkeypatch ): - monkeypatch.setattr(client_mod, "_MAX_STOPPED_STATUS_RECORDS", 2) + monkeypatch.setattr(client_mod, "_MAX_RUNTIME_RECORDS", 2) for index in range(3): mock_native.start_runtime.return_value = json.dumps( _runtime(f"runtime-{index}") @@ -1315,6 +1316,210 @@ async def test_runtime_status_stopped_records_are_bounded( assert report.handle is None +@pytest.mark.parametrize("state", ["active", "failed"]) +async def test_runtime_limit_preserves_owners_and_rejects_before_native_work( + native_client, mock_native, monkeypatch, state +): + monkeypatch.setattr(client_mod, "_MAX_RUNTIME_RECORDS", 2) + runtimes = [] + for index in range(2): + mock_native.start_runtime.return_value = json.dumps( + _runtime(f"runtime-{index}") + ) + runtime = await native_client.start_runtime(_config()) + runtimes.append(runtime) + if state == "failed": + mock_native.stop_runtime.side_effect = RuntimeError("TEST_ONLY_PRIVATE") + with pytest.raises(FabricRuntimeError): + await runtime.stop() + before = list(mock_native.mock_calls) + with pytest.raises(FabricStateError) as caught: + await native_client.start_runtime(_config()) + assert caught.value.code == "runtime_limit_reached" + assert caught.value.stage == "start" + assert mock_native.mock_calls == before + assert "TEST_ONLY_PRIVATE" not in str(caught.value) + mock_native.stop_runtime.side_effect = None + for runtime in runtimes: + report = await native_client.runtime_status( + runtime_id=runtime.runtime_id, agent_name="demo" + ) + assert report.status == state + await runtime.stop() + + +async def test_runtime_limit_evicts_stopped_records_before_active_and_failed_owners( + native_client, mock_native, monkeypatch +): + monkeypatch.setattr(client_mod, "_MAX_RUNTIME_RECORDS", 3) + stopped = await native_client.start_runtime(_config()) + await stopped.stop() + mock_native.start_runtime.return_value = json.dumps(_runtime("failed")) + failed = await native_client.start_runtime(_config()) + mock_native.stop_runtime.side_effect = RuntimeError("stop failed") + with pytest.raises(FabricRuntimeError): + await failed.stop() + mock_native.stop_runtime.side_effect = None + mock_native.start_runtime.return_value = json.dumps(_runtime("active")) + active = await native_client.start_runtime(_config()) + mock_native.start_runtime.return_value = json.dumps(_runtime("new")) + new = await native_client.start_runtime(_config()) + with pytest.raises(FabricStateError) as caught: + await native_client.runtime_status( + runtime_id=stopped.runtime_id, agent_name="demo" + ) + assert caught.value.code == "runtime_unavailable" + for runtime, status in [(failed, "failed"), (active, "active"), (new, "active")]: + report = await native_client.runtime_status( + runtime_id=runtime.runtime_id, agent_name="demo" + ) + assert report.status == status + await runtime.stop() + + +@pytest.mark.parametrize("failure", ["validation", "planning", "start"]) +async def test_failed_start_releases_runtime_capacity( + native_client, mock_native, monkeypatch, failure +): + monkeypatch.setattr(client_mod, "_MAX_RUNTIME_RECORDS", 1) + plan = mock_native.plan_config.side_effect + if failure == "planning": + mock_native.plan_config.side_effect = RuntimeError("planning failed") + if failure == "start": + mock_native.start_runtime.side_effect = RuntimeError("start failed") + with pytest.raises(FabricError): + await native_client.start_runtime( + _config(), launch_collector=True if failure == "validation" else None + ) + mock_native.plan_config.side_effect = plan + mock_native.start_runtime.side_effect = None + runtime = await native_client.start_runtime(_config()) + await runtime.stop() + + +async def test_pending_start_uses_capacity_and_cancellation_releases_the_reservation( + native_client, mock_native, monkeypatch +): + monkeypatch.setattr(client_mod, "_MAX_RUNTIME_RECORDS", 1) + entered, release = threading.Event(), threading.Event() + plan = mock_native.plan_config.side_effect + + def delayed_plan(*arguments): + entered.set() + assert release.wait(3) + return plan(*arguments) + + mock_native.plan_config.side_effect = delayed_plan + pending = asyncio.create_task(native_client.start_runtime(_config())) + try: + assert await _wait_for(entered) + with pytest.raises(FabricStateError) as caught: + await native_client.start_runtime(_config()) + assert caught.value.code == "runtime_limit_reached" + mock_native.start_runtime.assert_not_called() + pending.cancel() + release.set() + with pytest.raises(asyncio.CancelledError): + await pending + finally: + release.set() + await asyncio.gather(pending, return_exceptions=True) + mock_native.plan_config.side_effect = plan + runtime = await native_client.start_runtime(_config()) + await runtime.stop() + + +async def test_runtime_capacity_is_scoped_to_each_client(native_client, monkeypatch): + monkeypatch.setattr(client_mod, "_MAX_RUNTIME_RECORDS", 1) + first = await native_client.start_runtime(_config()) + second = await Fabric().start_runtime(_config()) + await first.stop() + await second.stop() + + +@pytest.mark.parametrize("state", ["active", "failed", "stopped"]) +@pytest.mark.parametrize( + "server_state", [None, "running", "stopping", "stopped", "unknown"] +) +async def test_runtime_status_output_matches_the_published_schema( + native_client, mock_native, runtime_status_validator, state, server_state +): + runtime = await native_client.start_runtime(_config()) + if state == "failed": + mock_native.stop_runtime.side_effect = RuntimeError("stop failed") + with pytest.raises(FabricRuntimeError): + await runtime.stop() + mock_native.stop_runtime.side_effect = None + if state == "stopped": + await runtime.stop() + report = ( + await native_client.runtime_status( + runtime_id=runtime.runtime_id, agent_name="demo" + ) + ).to_mapping() + report["server_state"] = server_state + runtime_status_validator.validate(report) + await runtime.stop() + + +@pytest.mark.parametrize( + "field", ["runtime_id", "agent_name", "status", "handle", "server_state"] +) +async def test_runtime_status_schema_requires_every_field( + native_client, runtime_status_validator, field +): + runtime = await native_client.start_runtime(_config()) + report = ( + await native_client.runtime_status( + runtime_id=runtime.runtime_id, agent_name="demo" + ) + ).to_mapping() + del report[field] + assert not runtime_status_validator.is_valid(report) + await runtime.stop() + + +@pytest.mark.parametrize( + "mutation", + [ + "null_handle", + "stopped_handle", + "invalid_status", + "invalid_server_state", + "extra_field", + "extra_handle_field", + "env", + "connection", + "metadata", + ], +) +async def test_runtime_status_schema_rejects_invalid_shapes_and_private_environment_fields( + native_client, runtime_status_validator, mutation +): + runtime = await native_client.start_runtime(_config()) + report = ( + await native_client.runtime_status( + runtime_id=runtime.runtime_id, agent_name="demo" + ) + ).to_mapping() + if mutation == "null_handle": + report["handle"] = None + elif mutation == "stopped_handle": + report["status"] = "stopped" + elif mutation == "invalid_status": + report["status"] = "running" + elif mutation == "invalid_server_state": + report["server_state"] = "responding" + elif mutation == "extra_field": + report["private"] = "TEST_ONLY_PRIVATE" + elif mutation == "extra_handle_field": + report["handle"]["private"] = "TEST_ONLY_PRIVATE" + else: + report["handle"]["environment"][mutation] = {"private": "TEST_ONLY_PRIVATE"} + assert not runtime_status_validator.is_valid(report) + await runtime.stop() + + async def test_runtime_status_stopping_one_runtime_preserves_another( native_client, mock_native ): diff --git a/tests/python/test_sdk_server_checks.py b/tests/python/test_sdk_server_checks.py index e0fdbd3f1..abea83c71 100644 --- a/tests/python/test_sdk_server_checks.py +++ b/tests/python/test_sdk_server_checks.py @@ -193,7 +193,11 @@ async def test_sdk_server_check_reaches_the_real_command_bridge( async def test_runtime_status_remote_reaches_sdk_in_the_original_server( - blocking_adapter_config, server_environment, server_sdk_bridge, tmp_path + blocking_adapter_config, + server_environment, + server_sdk_bridge, + tmp_path, + runtime_status_validator, ): host, log = server_sdk_bridge config = blocking_adapter_config.model_copy(deep=True) @@ -218,6 +222,7 @@ async def test_runtime_status_remote_reaches_sdk_in_the_original_server( assert active.server_state == "running" assert active.handle.runtime_id == runtime_id assert active.handle.environment["provider"] == "local" + runtime_status_validator.validate(active.to_mapping()) stopped = await host.handle( { "interface_version": 2, @@ -232,6 +237,7 @@ async def test_runtime_status_remote_reaches_sdk_in_the_original_server( ) assert report.status == "stopped" assert report.handle is None + runtime_status_validator.validate(report.to_mapping()) commands = [json.loads(line) for line in log.read_text().splitlines()] assert commands == [ {"operation": "status", "agent": "blocking", "runtime_id": runtime_id}, @@ -242,6 +248,44 @@ async def test_runtime_status_remote_reaches_sdk_in_the_original_server( await host.shutdown() +async def test_server_runtime_limit_has_no_effects_and_start_recovers_after_stop( + blocking_adapter_config, server_sdk_bridge, tmp_path, monkeypatch +): + monkeypatch.setattr("nemo_fabric.client._MAX_RUNTIME_RECORDS", 1) + host, _ = server_sdk_bridge + config = blocking_adapter_config.model_copy(deep=True) + config.environment.workspace = str(tmp_path) + config.discovery.local_paths = [str(tmp_path / "blocking.fabric-adapter.json")] + command = { + "interface_version": 2, + "operation": "start", + "agent": "blocking", + "config": config.to_mapping(), + } + try: + first = await host.handle(command) + assert first["status"] == "succeeded", first + rejected = await host.handle(command) + assert rejected["error"]["code"] == "runtime_limit_reached", rejected + assert rejected["error"]["effects"] == "none" + assert rejected["changed"] is False + assert not host.start_uncertain + stopped = await host.handle( + { + "interface_version": 2, + "operation": "stop", + "agent": "blocking", + "runtime_id": first["result"]["runtime_id"], + } + ) + assert stopped["status"] == "succeeded", stopped + replacement = await host.handle(command) + assert replacement["status"] == "succeeded", replacement + assert replacement["result"]["runtime_id"] != first["result"]["runtime_id"] + finally: + await host.shutdown() + + @pytest.mark.parametrize( "change", [ From b7020f69d2bd2140010dff92c8601ce025501ff6 Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Fri, 6 Nov 2026 10:33:14 -0700 Subject: [PATCH 3/6] test: restrict runtime status socket cases to Unix Signed-off-by: Sara Tadayon --- tests/python/runtime_server/test_platform.py | 6 +++++- tests/python/runtime_server/test_status.py | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/python/runtime_server/test_platform.py b/tests/python/runtime_server/test_platform.py index bfc9bb60d..4d8e6c1d1 100644 --- a/tests/python/runtime_server/test_platform.py +++ b/tests/python/runtime_server/test_platform.py @@ -10,7 +10,11 @@ @pytest.mark.parametrize( "arguments", - [["stop", "--agent", "review", "--runtime-id", "runtime-1"], ["check"]], + [ + ["stop", "--agent", "review", "--runtime-id", "runtime-1"], + ["check"], + ["status", "--agent", "review", "--runtime-id", "runtime-1"], + ], ) def test_windows_import_without_fcntl_and_deliberate_unsupported_command(arguments): code = """ diff --git a/tests/python/runtime_server/test_status.py b/tests/python/runtime_server/test_status.py index cd07f41ca..2f9f28df7 100644 --- a/tests/python/runtime_server/test_status.py +++ b/tests/python/runtime_server/test_status.py @@ -5,6 +5,7 @@ import asyncio import json +import sys from unittest.mock import AsyncMock import pytest @@ -76,6 +77,7 @@ async def test_status_server_delegates_to_sdk_without_invocation_lock( await host.shutdown() +@pytest.mark.skipif(sys.platform == "win32", reason="Unix command bridge") async def test_status_process_bridge_reads_stopped_record_and_does_not_start_server( blocking_adapter_config, tmp_path, socket_path ): @@ -176,6 +178,7 @@ def test_status_rejects_malformed_or_effectful_responses(field, value): protocol.validate_response(response, "status") +@pytest.mark.skipif(sys.platform == "win32", reason="Unix socket transport") @pytest.mark.parametrize( "field,value", [("runtime_id", "other"), ("agent_name", "other")] ) From 95dd1719c7b3e16541374b56a8b3960d56aef384 Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Mon, 9 Nov 2026 11:24:20 -0700 Subject: [PATCH 4/6] fix: reject recursive runtime status responses Signed-off-by: Sara Tadayon --- .../nemo_fabric/runtime_server/protocol.py | 8 ++- tests/python/runtime_server/test_status.py | 58 +++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/protocol.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/protocol.py index fb9463bd2..a72492fb4 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/protocol.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/protocol.py @@ -318,5 +318,11 @@ def validate_response(response: dict[str, Any], operation: str) -> None: raise invalid else: raise invalid - except (FabricConfigError, TypeError, ValueError, KeyError) as error: + except ( + FabricConfigError, + TypeError, + ValueError, + KeyError, + RecursionError, + ) as error: raise invalid from error diff --git a/tests/python/runtime_server/test_status.py b/tests/python/runtime_server/test_status.py index 2f9f28df7..0a3a1e4d9 100644 --- a/tests/python/runtime_server/test_status.py +++ b/tests/python/runtime_server/test_status.py @@ -15,6 +15,64 @@ from nemo_fabric.runtime_server.state import RuntimeServer +@pytest.fixture(name="recursive_status_response") +def recursive_status_response_fixture(runtime_factory): + private = "TEST_ONLY_PRIVATE" + for _ in range(600): + private = [private] + handle = runtime_factory("runtime-1").handle.to_mapping() + handle["environment"]["metadata"] = {"private": private} + response = protocol.envelope( + "status", + { + "runtime_id": "runtime-1", + "agent_name": "researcher", + "status": "active", + "handle": handle, + "server_state": "running", + }, + ) + return protocol.encode_response(response) + + +def test_status_validation_rejects_recursive_metadata(recursive_status_response): + response = protocol.decode_object(recursive_status_response) + with pytest.raises(protocol.ProtocolError) as caught: + protocol.validate_response(response, "status") + assert caught.value.code == "invalid_response" + assert caught.value.stage == "transport" + assert str(caught.value) == protocol.MESSAGES["invalid_response"] + + +@pytest.mark.skipif(sys.platform == "win32", reason="Unix socket transport") +async def test_status_transport_rejects_recursive_metadata( + socket_path, recursive_status_response +): + request = { + "interface_version": 2, + "operation": "status", + "agent": "researcher", + "runtime_id": "runtime-1", + } + + async def answer(reader, writer): + await reader.readline() + writer.write(recursive_status_response) + await writer.drain() + writer.close() + await writer.wait_closed() + + server = await asyncio.start_unix_server(answer, path=socket_path) + async with server: + response = await transport.client(request, socket_path=socket_path, timeout=1) + assert response["status"] == "failed" + assert response["result"] is None + assert response["error"]["code"] == "invalid_response" + assert response["error"]["effects"] == "none" + assert response["changed"] is False + assert "TEST_ONLY_PRIVATE" not in json.dumps(response) + + @pytest.mark.parametrize("extra", [[], ["--live"], ["--config", "private.json"]]) def test_status_command_requires_only_agent_and_runtime(extra): arguments = ["status", "--agent", "researcher", "--runtime-id", "runtime-1", *extra] From a7c1d0cd38c30e7b5f8401720841d8d69245669f Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Mon, 9 Nov 2026 12:45:34 -0700 Subject: [PATCH 5/6] test: shorten runtime status test IDs for Windows Signed-off-by: Sara Tadayon --- tests/python/test_sdk_server_checks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/python/test_sdk_server_checks.py b/tests/python/test_sdk_server_checks.py index abea83c71..73cb3caa1 100644 --- a/tests/python/test_sdk_server_checks.py +++ b/tests/python/test_sdk_server_checks.py @@ -392,6 +392,7 @@ async def test_runtime_status_remote_during_invocation_keeps_the_original_runtim "{", "x" * (4 * 1024 * 1024 + 1), ], + ids=["duplicate-fields", "null", "array", "malformed", "oversized"], ) async def test_runtime_status_remote_rejects_malformed_or_oversized_json( server_environment, monkeypatch, raw From ada6281a14e5c61be8c0156060d299070ddc9c96 Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Mon, 9 Nov 2026 13:11:24 -0700 Subject: [PATCH 6/6] test: use platform-native paths in runtime status checks Signed-off-by: Sara Tadayon --- tests/python/test_runtime.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/python/test_runtime.py b/tests/python/test_runtime.py index d2799e911..124189499 100644 --- a/tests/python/test_runtime.py +++ b/tests/python/test_runtime.py @@ -1182,7 +1182,7 @@ async def test_runtime_status_reads_owned_state_without_native_work( async def test_runtime_status_does_not_disclose_environment_secrets( - native_client, mock_native + native_client, mock_native, tmp_path ): handle = _runtime() handle["environment"].update( @@ -1192,8 +1192,10 @@ async def test_runtime_status_does_not_disclose_environment_secrets( "metadata": {"token": "TEST_ONLY_METADATA_SECRET"}, } ) - handle["environment"]["workspace"] = "/workspace" - handle["environment"]["artifacts"] = "/artifacts" + workspace = str(tmp_path / "workspace") + artifacts = str(tmp_path / "artifacts") + handle["environment"]["workspace"] = workspace + handle["environment"]["artifacts"] = artifacts handle["service_id"] = "service-1" handle["unknown_token"] = "TEST_ONLY_EXTENSION_SECRET" mock_native.start_runtime.return_value = json.dumps(handle) @@ -1206,8 +1208,8 @@ async def test_runtime_status_does_not_disclose_environment_secrets( assert serialized["handle"]["runtime_binding"] == handle["runtime_binding"] assert serialized["handle"]["environment"]["environment_id"] == "environment-1" assert serialized["handle"]["service_id"] == "service-1" - assert serialized["handle"]["environment"]["workspace"] == "/workspace" - assert serialized["handle"]["environment"]["artifacts"] == "/artifacts" + assert serialized["handle"]["environment"]["workspace"] == workspace + assert serialized["handle"]["environment"]["artifacts"] == artifacts assert runtime.handle.environment["env"]["API_KEY"] == "TEST_ONLY_ENV_SECRET"