From 8c1dd4f38bc902fb00fd00dfd4a7fae74fdc796d Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Thu, 12 Nov 2026 11:43:30 -0700 Subject: [PATCH 01/10] fix: persist invocation results for SDK and CLI recovery Signed-off-by: Sara Tadayon --- crates/fabric-core/src/environment.rs | 7 +- crates/fabric-core/src/runtime.rs | 196 +++++++++++++- crates/fabric-core/src/runtime/results.rs | 244 ++++++++++++++++++ crates/fabric-core/src/task_command.rs | 5 + crates/fabric-python/src/lib.rs | 53 ++++ .../nemo_fabric.client.md | 20 ++ docs/sdk/python.mdx | 22 ++ environment-providers/openshell/src/lib.rs | 92 ++++++- .../src/nemo_fabric/_native.pyi | 7 + .../src/nemo_fabric/client.py | 85 ++++++ .../src/nemo_fabric/runtime.py | 6 + .../src/nemo_fabric/runtime_server/README.md | 22 ++ .../src/nemo_fabric/runtime_server/cli.py | 12 +- .../nemo_fabric/runtime_server/protocol.py | 26 +- .../src/nemo_fabric/runtime_server/state.py | 22 ++ .../nemo_fabric/runtime_server/transport.py | 53 +++- tests/fixtures/runtime_check_provider.py | 11 +- tests/python/runtime_server/test_commands.py | 22 ++ tests/python/runtime_server/test_sdk.py | 149 +++++++++++ tests/python/runtime_server/test_transport.py | 32 +++ tests/python/test_sdk_server_checks.py | 92 +++++++ 21 files changed, 1156 insertions(+), 22 deletions(-) create mode 100644 crates/fabric-core/src/runtime/results.rs diff --git a/crates/fabric-core/src/environment.rs b/crates/fabric-core/src/environment.rs index 63b0853fe..7e289a016 100644 --- a/crates/fabric-core/src/environment.rs +++ b/crates/fabric-core/src/environment.rs @@ -661,14 +661,17 @@ where let budget = *timeout_seconds; return request_check_provider_process(command, operation, budget); } - if matches!(task_command, TaskCommand::Status { .. }) { + if matches!( + task_command, + TaskCommand::Status { .. } | TaskCommand::Result { .. } + ) { return request_check_provider_process(command, operation, 10.0); } let operation_name = operation.name(); let request_id = new_id("environment-provider-request"); let environment_key = environment_resource_key(environment)?; let (identity, is_start, is_stop) = match task_command { - TaskCommand::Check { .. } | TaskCommand::Status { .. } => { + TaskCommand::Check { .. } | TaskCommand::Status { .. } | TaskCommand::Result { .. } => { unreachable!("reads use a separate provider process") } TaskCommand::Start { .. } => (request_id.clone(), true, false), diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 54eeefd27..38814cdf4 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -39,7 +39,10 @@ use crate::system_stats::ResourceObservation; use crate::system_stats::{CgroupStats, InvocationSystemStats, StatsSession, SystemReader}; mod openshell; +mod results; use openshell::OpenShellRuntimeAdapter; +#[doc(hidden)] +pub use results::invocation_result; pub(crate) fn remote_check_agent( environment: &EnvironmentHandle, @@ -87,6 +90,43 @@ pub fn runtime_status_environment( } } +/// Read a saved task-side invocation result through its verified OpenShell environment. +#[doc(hidden)] +pub fn invocation_result_environment( + environment: &EnvironmentHandle, + agent: &str, + runtime_id: &str, + request_id: &str, +) -> Result { + if environment.provider != "openshell" + || environment.control_location != ControlLocation::InEnvControl + || [agent, runtime_id, request_id] + .iter() + .any(|value| value.trim().is_empty() || value.contains('\0')) + { + return Err(FabricError::InvalidConfig { + field: "invocation_result".into(), + reason: "a valid invocation target and verified OpenShell environment are required" + .into(), + }); + } + let command = crate::task_command::TaskCommand::Result { + agent: agent.into(), + runtime_id: runtime_id.into(), + request_id: request_id.into(), + }; + match crate::environment::command_runtime(environment, &command, 10)? { + crate::task_command::TaskCommandResponse::Returned { value } => Ok(value), + crate::task_command::TaskCommandResponse::Rejected { .. } => { + Err(FabricError::InvalidConfig { + field: "invocation_result".into(), + reason: "the result lookup could not be completed; do not replay uncertain work" + .into(), + }) + } + } +} + static NEXT_ID: AtomicU64 = AtomicU64::new(1); const ADAPTER_PYTHON_ENV: &str = "ADAPTER_PYTHON"; const VIRTUAL_ENV_ENV: &str = "VIRTUAL_ENV"; @@ -2777,6 +2817,12 @@ fn run_local_host_invocation_with_timeout( request_id: request.request_id.clone(), runtime_id: runtime.runtime_id.clone(), }; + let root = artifact_manifest(plan)? + .root + .unwrap_or_else(|| std::env::temp_dir().join("nemo-fabric")); + if let Some(result) = results::reserve(&root, runtime, &request, &invocation)? { + return Ok(result); + } let host = local_hosts() .get(&runtime.runtime_id) .cloned() @@ -3192,7 +3238,7 @@ fn run_local_host_invocation_with_timeout( serde_json::to_value(&agent_result.extensions).map_err(FabricError::SerializeJson)?, ); } - Ok(RunResult { + let result = RunResult { session_ref: runtime.session_ref.clone(), agent_name: plan.agent_name.clone(), harness: harness(plan), @@ -3214,7 +3260,9 @@ fn run_local_host_invocation_with_timeout( .collect(), events, metadata, - }) + }; + results::save(&root, &result)?; + Ok(result) } fn invalidate_timed_out_local_host( @@ -6614,6 +6662,150 @@ else: fs::remove_dir_all(root).unwrap(); } + #[test] + fn completed_invocation_persists_the_full_result() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).expect("start"); + let result = invoke_runtime(&plan, &runtime, RunRequest::text("hello")).expect("invoke"); + let path = + PathBuf::from(result.metadata["fabric_home"].as_str().unwrap()).join("run-result.json"); + let saved: RunResult = + serde_json::from_slice(&fs::read(path).expect("saved result")).expect("valid result"); + assert_eq!(saved, result); + stop_runtime(&plan, &runtime).expect("stop"); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn result_lookup_and_duplicate_request_preserve_one_invocation() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).expect("start"); + let mut request = RunRequest::text("hello"); + request.request_id = "same-request".into(); + let first = invoke_runtime(&plan, &runtime, request.clone()).expect("invoke"); + let second = invoke_runtime(&plan, &runtime, request.clone()).expect("retry"); + assert_eq!(first, second); + let artifacts = root.join("artifacts"); + assert_eq!( + invocation_result( + &artifacts, + &plan.agent_name, + &runtime.runtime_id, + "same-request" + ) + .unwrap(), + Some(first.clone()) + ); + request.input = serde_json::json!("different input"); + assert!(invoke_runtime(&plan, &runtime, request).is_err()); + stop_runtime(&plan, &runtime).expect("stop"); + assert_eq!( + invocation_result( + &artifacts, + &plan.agent_name, + &runtime.runtime_id, + "same-request" + ) + .unwrap(), + Some(first) + ); + assert!( + invocation_result( + &artifacts, + "wrong-agent", + &runtime.runtime_id, + "same-request" + ) + .is_err() + ); + assert!( + invocation_result(&artifacts, &plan.agent_name, "../../escape", "same-request") + .is_err() + ); + fs::remove_dir_all(root).unwrap(); + } + + #[cfg(unix)] + #[test] + fn result_lookup_rejects_symlinked_result_files() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).expect("start"); + let mut request = RunRequest::text("hello"); + request.request_id = "private-request".into(); + let result = invoke_runtime(&plan, &runtime, request).expect("invoke"); + let path = + PathBuf::from(result.metadata["fabric_home"].as_str().unwrap()).join("run-result.json"); + let outside = root.join("outside-result.json"); + fs::rename(&path, &outside).unwrap(); + std::os::unix::fs::symlink(&outside, &path).unwrap(); + assert!( + invocation_result( + &root.join("artifacts"), + &plan.agent_name, + &runtime.runtime_id, + "private-request" + ) + .is_err() + ); + stop_runtime(&plan, &runtime).expect("stop"); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn duplicate_pending_request_is_not_submitted_again() { + let (root, plan) = local_host_plan("invoke_timeout"); + let runtime = start_runtime(&plan).unwrap(); + let stderr = local_hosts() + .get(&runtime.runtime_id) + .unwrap() + .lock() + .unwrap() + .stderr_path + .clone(); + let mut request = RunRequest::text("slow"); + request.request_id = "pending-request".into(); + let worker_plan = plan.clone(); + let worker_runtime = runtime.clone(); + let worker_request = request.clone(); + let worker = thread::spawn(move || { + run_local_host_adapter_with_timeout( + &worker_plan, + &worker_runtime, + worker_request, + Duration::from_millis(500), + ) + }); + let deadline = Instant::now() + Duration::from_secs(3); + while !fs::read_to_string(&stderr) + .unwrap() + .contains("invoke accepted without response") + { + assert!(Instant::now() < deadline); + thread::sleep(Duration::from_millis(10)); + } + let error = + invoke_runtime(&plan, &runtime, request).expect_err("pending request must not replay"); + assert!( + matches!(error, FabricError::AdapterLifecycleOperation { code, .. } if code == "invocation_pending") + ); + worker + .join() + .unwrap() + .expect_err("original invocation times out"); + assert!( + invocation_result( + &root.join("artifacts"), + &plan.agent_name, + &runtime.runtime_id, + "pending-request" + ) + .unwrap() + .is_none() + ); + stop_runtime(&plan, &runtime).unwrap(); + fs::remove_dir_all(root).unwrap(); + } + #[test] fn local_host_reuses_one_process_and_stops_idempotently() { let (root, plan) = local_host_plan("success"); diff --git a/crates/fabric-core/src/runtime/results.rs b/crates/fabric-core/src/runtime/results.rs new file mode 100644 index 000000000..bbbb69136 --- /dev/null +++ b/crates/fabric-core/src/runtime/results.rs @@ -0,0 +1,244 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Private, per-runtime request reservations and completed invocation results. + +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use super::AdapterLifecycleOperation; +use super::{InvocationHandle, RunRequest, RunResult, RuntimeHandle, lifecycle_error}; +use crate::Result; + +const MAX_RESULT_BYTES: u64 = 4 * 1024 * 1024; + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Reservation { + agent_name: String, + runtime_id: String, + request_id: String, + invocation_id: String, + fingerprint: String, +} + +fn failure(runtime_id: &str, code: &str) -> crate::FabricError { + lifecycle_error( + AdapterLifecycleOperation::Invoke, + runtime_id, + code, + "The invocation record is unavailable or conflicts with this request; do not replay uncertain work.", + "", + ) +} + +fn leaf(value: &str) -> bool { + !value.is_empty() + && value + .bytes() + .all(|c| c.is_ascii_alphanumeric() || c == b'-' || c == b'_') +} + +fn directory(parent: &Path, name: &str, create: bool, runtime_id: &str) -> Result { + let path = parent.join(name); + if create { + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + if let Err(error) = builder.create(&path) + && error.kind() != std::io::ErrorKind::AlreadyExists + { + return Err(failure(runtime_id, "result_storage_failed")); + } + } + let metadata = + fs::symlink_metadata(&path).map_err(|_| failure(runtime_id, "result_storage_failed"))?; + if !metadata.is_dir() || metadata.file_type().is_symlink() { + return Err(failure(runtime_id, "result_storage_failed")); + } + Ok(path) +} + +fn runtime_directory(root: &Path, runtime_id: &str, create: bool) -> Result { + if !leaf(runtime_id) { + return Err(failure(runtime_id, "invalid_result_target")); + } + if create { + fs::create_dir_all(root).map_err(|_| failure(runtime_id, "result_storage_failed"))?; + } + let root = fs::canonicalize(root).map_err(|_| failure(runtime_id, "result_storage_failed"))?; + let fabric = directory(&root, ".fabric", create, runtime_id)?; + directory(&fabric, runtime_id, create, runtime_id) +} + +fn options() -> OpenOptions { + let mut options = OpenOptions::new(); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); + } + options +} + +fn read(path: &Path, runtime_id: &str) -> Result> { + let file = match options().read(true).open(path) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(_) => return Err(failure(runtime_id, "result_storage_failed")), + }; + let metadata = file + .metadata() + .map_err(|_| failure(runtime_id, "result_storage_failed"))?; + if !metadata.is_file() || metadata.len() > MAX_RESULT_BYTES { + return Err(failure(runtime_id, "result_storage_failed")); + } + let mut bytes = Vec::new(); + file.take(MAX_RESULT_BYTES + 1) + .read_to_end(&mut bytes) + .map_err(|_| failure(runtime_id, "result_storage_failed"))?; + if bytes.len() as u64 > MAX_RESULT_BYTES { + return Err(failure(runtime_id, "result_storage_failed")); + } + serde_json::from_slice(&bytes) + .map(Some) + .map_err(|_| failure(runtime_id, "result_storage_failed")) +} + +fn index_name(request_id: &str) -> String { + format!("{:x}.json", Sha256::digest(request_id.as_bytes())) +} + +fn completed(runtime: &Path, record: &Reservation) -> Result> { + if !leaf(&record.invocation_id) { + return Err(failure(&record.runtime_id, "invalid_result_target")); + } + let invocation = runtime.join(&record.invocation_id); + if !invocation.exists() { + return Ok(None); + } + let invocation = directory(runtime, &record.invocation_id, false, &record.runtime_id)?; + let result: Option = read(&invocation.join("run-result.json"), &record.runtime_id)?; + if let Some(result) = &result + && (result.agent_name != record.agent_name + || result.runtime_id != record.runtime_id + || result.request_id != record.request_id + || result.invocation_id != record.invocation_id) + { + return Err(failure(&record.runtime_id, "invalid_result_target")); + } + Ok(result) +} + +pub(super) fn reserve( + root: &Path, + runtime: &RuntimeHandle, + request: &RunRequest, + invocation: &InvocationHandle, +) -> Result> { + let runtime_dir = runtime_directory(root, &runtime.runtime_id, true)?; + let requests = directory(&runtime_dir, "requests", true, &runtime.runtime_id)?; + let path = requests.join(index_name(&request.request_id)); + let record = Reservation { + agent_name: runtime.agent_name.clone(), + runtime_id: runtime.runtime_id.clone(), + request_id: request.request_id.clone(), + invocation_id: invocation.invocation_id.clone(), + fingerprint: format!( + "{:x}", + Sha256::digest(serde_json::to_vec(request).map_err(crate::FabricError::SerializeJson)?) + ), + }; + match options().write(true).create_new(true).open(&path) { + Ok(mut file) => { + let bytes = serde_json::to_vec(&record).map_err(crate::FabricError::SerializeJson)?; + file.write_all(&bytes) + .and_then(|_| file.sync_all()) + .map_err(|_| failure(&runtime.runtime_id, "result_storage_failed"))?; + Ok(None) + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + let previous: Reservation = read(&path, &runtime.runtime_id)? + .ok_or_else(|| failure(&runtime.runtime_id, "result_storage_failed"))?; + if previous.agent_name != record.agent_name + || previous.runtime_id != record.runtime_id + || previous.request_id != record.request_id + || previous.fingerprint != record.fingerprint + { + return Err(failure(&runtime.runtime_id, "request_conflict")); + } + completed(&runtime_dir, &previous)? + .map(Some) + .ok_or_else(|| failure(&runtime.runtime_id, "invocation_pending")) + } + Err(_) => Err(failure(&runtime.runtime_id, "result_storage_failed")), + } +} + +pub(super) fn save(root: &Path, result: &RunResult) -> Result<()> { + let runtime = runtime_directory(root, &result.runtime_id, true)?; + let invocation = directory(&runtime, &result.invocation_id, true, &result.runtime_id)?; + let temporary = invocation.join(format!("{}.tmp", super::new_id("result"))); + let mut file: File = options() + .write(true) + .create_new(true) + .open(&temporary) + .map_err(|_| failure(&result.runtime_id, "result_storage_failed"))?; + let bytes = serde_json::to_vec(result).map_err(crate::FabricError::SerializeJson)?; + file.write_all(&bytes) + .and_then(|_| file.sync_all()) + .and_then(|_| fs::rename(&temporary, invocation.join("run-result.json"))) + .map_err(|_| failure(&result.runtime_id, "result_storage_failed"))?; + Ok(()) +} + +/// Read a saved result. Absence does not establish whether the request ran. +#[doc(hidden)] +pub fn invocation_result( + root: &Path, + agent: &str, + runtime_id: &str, + request_id: &str, +) -> Result> { + if request_id.trim().is_empty() || agent.trim().is_empty() { + return Err(failure(runtime_id, "invalid_result_target")); + } + if !leaf(runtime_id) { + return Err(failure(runtime_id, "invalid_result_target")); + } + let root = match fs::canonicalize(root) { + Ok(root) => root, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(_) => return Err(failure(runtime_id, "result_storage_failed")), + }; + let fabric = root.join(".fabric"); + let runtime = fabric.join(runtime_id); + let requests = runtime.join("requests"); + for path in [&fabric, &runtime, &requests] { + match fs::symlink_metadata(path) { + Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + _ => return Err(failure(runtime_id, "result_storage_failed")), + } + } + let record: Option = read(&requests.join(index_name(request_id)), runtime_id)?; + let Some(record) = record else { + return Ok(None); + }; + if record.agent_name != agent + || record.runtime_id != runtime_id + || record.request_id != request_id + { + return Err(failure(runtime_id, "invalid_result_target")); + } + completed(&runtime, &record) +} diff --git a/crates/fabric-core/src/task_command.rs b/crates/fabric-core/src/task_command.rs index 2919d45d9..4ee21f380 100644 --- a/crates/fabric-core/src/task_command.rs +++ b/crates/fabric-core/src/task_command.rs @@ -38,6 +38,11 @@ pub(crate) enum TaskCommand { agent: String, runtime_id: String, }, + Result { + agent: String, + runtime_id: String, + request_id: String, + }, Stop { agent: String, runtime_id: String, diff --git a/crates/fabric-python/src/lib.rs b/crates/fabric-python/src/lib.rs index 975ba7825..26879566d 100644 --- a/crates/fabric-python/src/lib.rs +++ b/crates/fabric-python/src/lib.rs @@ -284,6 +284,57 @@ fn check_environment( to_json(&report) } +/// Read a saved result from the configured artifact root. +#[pyfunction] +fn invocation_result( + py: Python<'_>, + artifacts: Option, + agent: String, + runtime_id: String, + request_id: String, +) -> PyResult { + let root = artifacts + .map(PathBuf::from) + .unwrap_or_else(|| std::env::temp_dir().join("nemo-fabric")); + let result = py + .detach(|| { + nemo_fabric_core::runtime::invocation_result(&root, &agent, &runtime_id, &request_id) + }) + .map_err(|_| { + PyRuntimeError::new_err( + "The invocation result is unavailable; do not replay uncertain work.", + ) + })?; + to_json(&result) +} + +/// Read a saved result through an OpenShell environment. +#[pyfunction] +fn invocation_result_environment( + py: Python<'_>, + environment_json: String, + agent: String, + runtime_id: String, + request_id: String, +) -> PyResult { + let environment = parse_environment_handle(environment_json)?; + let result = py + .detach(|| { + nemo_fabric_core::runtime::invocation_result_environment( + &environment, + &agent, + &runtime_id, + &request_id, + ) + }) + .map_err(|_| { + PyRuntimeError::new_err( + "The invocation result is unavailable; do not replay uncertain work.", + ) + })?; + to_json(&result) +} + /// Read recorded runtime status through an OpenShell environment. #[pyfunction] fn runtime_status_environment( @@ -391,6 +442,8 @@ fn runtime_resource_snapshot( fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(check_environment, m)?)?; m.add_function(wrap_pyfunction!(runtime_status_environment, m)?)?; + m.add_function(wrap_pyfunction!(invocation_result, m)?)?; + m.add_function(wrap_pyfunction!(invocation_result_environment, m)?)?; m.add_function(wrap_pyfunction!(check_service, m)?)?; m.add("ServiceInUseError", m.py().get_type::())?; m.add( diff --git a/docs/reference/api/python-library-reference/nemo_fabric.client.md b/docs/reference/api/python-library-reference/nemo_fabric.client.md index 815b74b20..3ecec93c5 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.client.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.client.md @@ -197,6 +197,26 @@ Doctor checks the resolved adapter, capability mappings, and declared environmen --- +### method `invocation_result` + +```python +async def invocation_result( + *, + agent_name: str, + runtime_id: str, + request_id: str, + artifacts: str | os.PathLike[str] | None = None, + environment: EnvironmentHandle | None = None, +) -> RunResult | None +``` + +Retrieve a saved invocation without executing or repeating work. + +Supply the original request identifier. A null result means no completed result is available; it does not prove that work is running or safe to repeat. Local reads use the configured artifact root. OpenShell reads use the owning runtime server through ``fabric-agent-ctl result``. Files survive only while their artifact storage survives. + +--- + + ### method `plan` ```python diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index a0303b03d..65cfb9e63 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1707,3 +1707,25 @@ OpenClaw service startup reports retain their original source and capture time w Calling applications and environment owners supply typed `ProvenanceConfig.reports` snapshots through the configuration already sent to NeMo Fabric. Capture them at application initialization and preserve their original `source`, `origin`, availability and `observed_at`. A guest never inspects its caller. Harbor implements this seam with `fabric_provenance`, whose shape matches the SDK control; `--ak 'fabric_provenance={"enabled":false}'` disables collection before host initialization and propagates to the guest. The [software evidence example](https://github.com/NVIDIA/NeMo-Fabric/tree/main/examples/provenance) provides six execution-path checks, separate-venv runs, saved RunResults and private comparison fixtures. Its local provider/service fixtures are labeled separately from live deployments. Safe configuration capture is the P0 follow-up and is outside this software-version contract. + + +### Recover a buffered invocation result + +Fabric saves each completed `RunResult` as `/.fabric///run-result.json` before returning the result. On Unix, result and request-record files are readable only by their owner. The artifact root comes from `runtime.artifacts` or the environment; without either setting, Fabric uses its temporary artifact directory. Files remain until their storage is removed; no automatic retention or eviction is added. + +Choose a stable `RunRequest.request_id` before submitting work. After a command timeout, `invocation_timeout` reports unknown effects and returns the runtime and request identifiers. The invocation can continue. Retrieve the completed result with: + +```python +result = await fabric.invocation_result( + agent_name="researcher", runtime_id=runtime_id, + request_id=request_id, artifacts="/path/to/artifacts", +) +``` + +```sh +fabric-agent-ctl result --agent researcher --runtime-id RUNTIME_ID --request-id REQUEST_ID +``` + +A null result means no completed result is available; it does not prove the request is still running or safe to repeat. A retry of the same request identifier and input returns the saved result, or reports that the original invocation is pending. Reusing that identifier with different input is rejected. Failed or interrupted reservations are not automatically replayed. + +For OpenShell, pass `environment=environment` instead of `artifacts` to query the original server through the provider. The CLI requires a retained ownership record on that server. A later local SDK client can read retained files using the explicit artifact root even after runtime stop. Neither path reconstructs a missing result. Remote results retain task-side artifact paths. diff --git a/environment-providers/openshell/src/lib.rs b/environment-providers/openshell/src/lib.rs index 5e7525e6d..537534b66 100644 --- a/environment-providers/openshell/src/lib.rs +++ b/environment-providers/openshell/src/lib.rs @@ -346,7 +346,7 @@ async fn runtime_command( )); } if executed.stdout.is_empty() { - if operation == "status" { + if matches!(operation, "status" | "result") { strict_json::validate(&executed.stderr).map_err(|_| { ProviderError::contract("invalid_response", "status command returned invalid JSON") })?; @@ -360,7 +360,7 @@ async fn runtime_command( || diagnostic.operation != operation || !matches!(diagnostic.status.as_str(), "failed" | "unsupported") || !matches!(error.effects.as_str(), "none" | "applied" | "unknown") - || (operation == "status" && error.effects != "none") + || (matches!(operation, "status" | "result") && error.effects != "none") || !matches!( error.stage.as_str(), "request" @@ -368,6 +368,7 @@ async fn runtime_command( | "start" | "invoke" | "stop" + | "result" | "check" | "status" | "transport" @@ -402,7 +403,7 @@ fn decode_sdk_command_response( "task command output was not one SDK response envelope", )); } - if operation == "status" { + if matches!(operation, "status" | "result") { strict_json::validate(stdout).map_err(|_| { ProviderError::contract("invalid_response", "status command returned invalid JSON") })?; @@ -443,7 +444,7 @@ fn decode_sdk_command_response( let status = response["status"].as_str().unwrap_or_default(); let result = &response["result"]; let error = &response["error"]; - if operation == "status" + if matches!(operation, "status" | "result") && (response["changed"] != false || (!error.is_null() && error["effects"] != "none")) { return Err(ProviderError::contract( @@ -494,7 +495,9 @@ fn decode_sdk_command_response( if exit_code != 0 || !error.is_null() || (operation == "stop" && !result.is_null()) - || (operation != "stop" && !result.is_object()) + || (operation != "stop" + && !(operation == "result" && result.is_null()) + && !result.is_object()) { return Err(ProviderError::contract( "invalid_response", @@ -536,6 +539,7 @@ fn decode_sdk_command_response( | "stop" | "check" | "status" + | "result" | "transport" ) || !matches!(effects, "none" | "applied" | "unknown") @@ -573,6 +577,8 @@ fn valid_diagnostic_code(code: &str) -> bool { | "start_uncertain" | "runtime_id_conflict" | "status_failed" + | "result_failed" + | "invocation_timeout" | "stop_failed" | "invoke_failed" | "host_stopping" @@ -1422,6 +1428,11 @@ enum TaskCommand { agent: String, runtime_id: String, }, + Result { + agent: String, + runtime_id: String, + request_id: String, + }, Stop { agent: String, runtime_id: String, @@ -1510,6 +1521,31 @@ impl TaskCommand { runtime_id, request, } => ("invoke", agent, Some(runtime_id), Some(request)), + Self::Result { + agent, + runtime_id, + request_id, + } => { + if [agent.as_str(), runtime_id.as_str(), request_id.as_str()] + .iter() + .any(|value| value.trim().is_empty() || value.contains('\0')) + { + return Err(ProviderError::contract( + "invalid_runtime_command", + "invalid result target", + )); + } + arguments.extend([ + "result".into(), + "--agent".into(), + agent, + "--runtime-id".into(), + runtime_id, + "--request-id".into(), + request_id, + ]); + return Ok(("result", arguments, None)); + } Self::Status { agent, runtime_id } => ("status", agent, Some(runtime_id), None), Self::Stop { agent, runtime_id } => ("stop", agent, Some(runtime_id), None), }; @@ -3852,6 +3888,52 @@ mod tests { ), } } + #[test] + fn result_command_is_a_read_with_request_identity() { + let command: TaskCommand = serde_json::from_value(json!({"operation": "result", "agent": "researcher", "runtime_id": "runtime-1", "request_id": "request-1"})).unwrap(); + let (operation, arguments, payload) = command.arguments().unwrap(); + assert_eq!(operation, "result"); + assert_eq!( + arguments, + vec![ + "fabric-agent-ctl", + "result", + "--agent", + "researcher", + "--runtime-id", + "runtime-1", + "--request-id", + "request-1" + ] + ); + assert!(payload.is_none()); + let response = json!({"operation": "result", "status": "succeeded", "changed": false, "result": null, "error": null}); + assert_eq!( + decode_sdk_command_response("result", 0, response.to_string().as_bytes()).unwrap(), + json!({"status": "returned", "value": null}) + ); + let duplicate = br#"{"operation":"result","status":"succeeded","changed":false,"result":null,"result":{},"error":null}"#; + assert!(decode_sdk_command_response("result", 0, duplicate).is_err()); + } + + #[test] + fn result_and_timeout_diagnostics_keep_their_specific_codes() { + for (operation, code, stage, effects) in [ + ("result", "result_failed", "result", "none"), + ("invoke", "invocation_timeout", "transport", "unknown"), + ] { + let response = json!({"operation": operation, "status": "failed", "changed": if operation == "result" { json!(false) } else { Value::Null }, "result": null, + "error": {"code": code, "stage": stage, "effects": effects, "message": "TEST_ONLY_PRIVATE"}}); + let decoded = + decode_sdk_command_response(operation, 1, response.to_string().as_bytes()).unwrap(); + assert_eq!( + decoded, + json!({"status": "rejected", "error": {"code": code, "stage": stage, "effects": effects}}) + ); + assert!(!decoded.to_string().contains("TEST_ONLY_PRIVATE")); + } + } + #[test] fn status_command_names_the_agent_and_runtime_without_payload() { let command: TaskCommand = serde_json::from_value(json!({ diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi index 5b25ad9a9..e3e8f4180 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi @@ -53,3 +53,10 @@ def check_service(service_json: str, request_json: str) -> str: ... class ResourceObservationUnavailableError(RuntimeError): ... class ResourceObservationUnsupportedError(RuntimeError): ... + +def invocation_result( + artifacts: str | None, agent: str, runtime_id: str, request_id: str +) -> str: ... +def invocation_result_environment( + environment_json: str, agent: str, runtime_id: str, request_id: str +) -> str: ... diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py index c50068e25..3d7f55643 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py @@ -107,6 +107,91 @@ def __init__(self) -> None: str, tuple[tuple[str, str], RuntimeStatusReport] ] = OrderedDict() + async def invocation_result( + self, + *, + agent_name: str, + runtime_id: str, + request_id: str, + artifacts: str | os.PathLike[str] | None = None, + environment: EnvironmentHandle | None = None, + ) -> RunResult | None: + """Retrieve a saved invocation without executing or repeating work. + + Supply the original request identifier. A null result means no completed + result is available; it does not prove that work is running or safe to + repeat. Local reads use the configured artifact root. OpenShell reads + use the owning runtime server through ``fabric-agent-ctl result``. + Files survive only while their artifact storage survives. + """ + if any( + not isinstance(value, str) or not value.strip() or "\0" in value + for value in (agent_name, runtime_id, request_id) + ): + raise FabricConfigError("The invocation result target is invalid.") + native = self._require_native_module("invocation_result") + if environment is not None and environment.provider == "openshell": + if artifacts is not None: + raise FabricConfigError( + "Remote lookup uses the runtime's artifact storage." + ) + operation = lambda: native.invocation_result_environment( + _environment_json(environment), agent_name, runtime_id, request_id + ) + else: + if environment is not None and environment.provider != "local": + raise FabricConfigError( + "Result lookup supports local and OpenShell environments." + ) + runtime = self._runtimes.get(runtime_id) + if environment is not None and ( + runtime is None + or ( + runtime.handle.environment["provider"], + runtime.handle.environment["environment_id"], + ) + != (environment.provider, environment.environment_id) + ): + raise FabricStateError( + "The selected environment does not own this runtime.", + stage="result", + code="runtime_unavailable", + ) + if artifacts is None and runtime is not None: + mapping = runtime._plan.to_mapping() + configured = mapping["config"].get("runtime", {}).get( + "artifacts" + ) or runtime.handle.environment.get("artifacts") + if configured is not None: + from pathlib import Path + + artifacts = str(Path(mapping["base_dir"]) / configured) + artifact_path = os.fspath(artifacts) if artifacts is not None else None + operation = lambda: native.invocation_result( + artifact_path, agent_name, runtime_id, request_id + ) + try: + encoded = await asyncio.get_running_loop().run_in_executor( + _CHECK_EXECUTOR, operation + ) + raw = json.loads(encoded) + if raw is None: + return None + result = RunResult.from_mapping(raw) + if (result.agent_name, result.runtime_id, result.request_id) != ( + agent_name, + runtime_id, + request_id, + ): + raise ValueError() + return result + except Exception: + raise FabricRuntimeError( + "The invocation result could not be retrieved; do not replay uncertain work.", + stage="result", + code="result_unavailable", + ) from None + async def runtime_status( self, *, diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index d70f53615..cdde84895 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -915,6 +915,12 @@ async def _close_streaming_resources(self) -> None: await self._collector.aclose() def _absorb(self, result: RunResult) -> None: + identity = (result.runtime_id, result.invocation_id, result.request_id) + if any( + (item["runtime_id"], item["invocation_id"], item["request_id"]) == identity + for item in self._invocations + ): + return self._invocations.append( { "request_id": result.request_id, diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md index 624226d08..35c5c09e9 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md @@ -108,3 +108,25 @@ Follow the [OpenShell Code Review instructions](../../../../../../examples/code_ The example starts the server, waits for its socket, starts a runtime, invokes twice, and stops through separate exec calls. The older NemoClaw client using `validate/prepare/configure` requires migration. Version 1 requests fail explicitly. Direct SDK lifecycle and streaming APIs retain their existing behavior. The task configuration carries the default-on `provenance` collector control and typed caller reports through the existing start request. `enabled=false` skips guest SDK and adapter collection. The guest records its own installation as `guest_sdk`; it never inspects caller packages. Returned RunResults preserve the caller reports and guest-owned snapshots. The host keeps its own snapshot independently so missing evidence from older intermediaries is explicit. + + +### Recover a buffered invocation result + +Fabric saves each completed `RunResult` as `/.fabric///run-result.json` before returning the result. On Unix, result and request-record files are readable only by their owner. The artifact root comes from `runtime.artifacts` or the environment; without either setting, Fabric uses its temporary artifact directory. Files remain until their storage is removed; no automatic retention or eviction is added. + +Choose a stable `RunRequest.request_id` before submitting work. After a command timeout, `invocation_timeout` reports unknown effects and returns the runtime and request identifiers. The invocation can continue. Retrieve the completed result with: + +```python +result = await fabric.invocation_result( + agent_name="researcher", runtime_id=runtime_id, + request_id=request_id, artifacts="/path/to/artifacts", +) +``` + +```sh +fabric-agent-ctl result --agent researcher --runtime-id RUNTIME_ID --request-id REQUEST_ID +``` + +A null result means no completed result is available; it does not prove the request is still running or safe to repeat. A retry of the same request identifier and input returns the saved result, or reports that the original invocation is pending. Reusing that identifier with different input is rejected. Failed or interrupted reservations are not automatically replayed. + +For OpenShell, pass `environment=environment` instead of `artifacts` to query the original server through the provider. The CLI requires a retained ownership record on that server. A later local SDK client can read retained files using the explicit artifact root even after runtime stop. Neither path reconstructs a missing result. Remote results retain task-side artifact paths. diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/cli.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/cli.py index 56ba97d49..b31343ecd 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/cli.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/cli.py @@ -89,28 +89,33 @@ def parse_command(arguments: list[str]) -> dict[str, Any]: if flag in ( "--agent", "--runtime-id", + "--request-id", "--config", "--session-ref", "--input", "--request", "--timeout-seconds", ): - if index + 1 == len(arguments) or arguments[index + 1].startswith("--"): + if index + 1 == len(arguments) or ( + flag != "--request-id" and arguments[index + 1].startswith("--") + ): raise ProtocolError() flags[flag] = arguments[index + 1] index += 2 else: raise ProtocolError() required = set() - if operation in ("start", "invoke", "stop", "status"): + if operation in ("start", "invoke", "stop", "status", "result"): required.add("--agent") - if operation in ("invoke", "stop", "status"): + if operation in ("invoke", "stop", "status", "result"): required.add("--runtime-id") if operation in CONFIG_OPERATIONS: config_flags = flags.keys() & {"--config", "--request"} if len(config_flags) != 1: raise ProtocolError() required.update(config_flags) + if operation == "result": + required.add("--request-id") allowed = required.copy() if operation == "start" and "--config" in flags: allowed.add("--session-ref") @@ -144,6 +149,7 @@ def parse_command(arguments: list[str]) -> dict[str, Any]: for flag in ( "--agent", "--runtime-id", + "--request-id", ): if flag in flags: request[flag[2:].replace("-", "_")] = flags[flag] diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/protocol.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/protocol.py index da339f683..c850c2579 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/protocol.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/protocol.py @@ -18,7 +18,7 @@ READ_SECONDS = 10.0 SHUTDOWN_SECONDS = 4.0 SOCKET_PATH = "/sandbox/fabric.sock" -OPERATIONS = ("start", "invoke", "stop", "serve", "check", "status") +OPERATIONS = ("start", "invoke", "stop", "serve", "check", "status", "result") # A cause is an SDK or adapter error code, never its message or other values. CAUSE = re.compile(r"[a-z][a-z0-9_]{0,63}") CAUSE_SUFFIX = re.compile(rf" Cause: ({CAUSE.pattern})\.\Z") @@ -47,6 +47,8 @@ "runtime_limit_reached": "The client runtime limit has been reached; stop an owned runtime before starting another.", "status_failed": "Fabric could not read the recorded runtime status.", "stop_failed": "Fabric could not complete the stop operation.", + "result_failed": "Fabric could not retrieve the saved invocation result.", + "invocation_timeout": "The invocation reply exceeded its deadline; work may continue. Retrieve the result using the original request ID; do not submit it again with a new ID.", "invoke_failed": "Fabric did not report a successful invocation.", "host_stopping": "The Fabric host is shutting down.", "runtime_unavailable": "No active Fabric runtime is available.", @@ -132,9 +134,9 @@ def validate_request(request: Any) -> str: ): raise ProtocolError("interface_unsupported", unsupported=True) required = {"interface_version", "operation"} - if operation in ("start", "invoke", "stop", "status"): + if operation in ("start", "invoke", "stop", "status", "result"): required.add("agent") - if operation in ("invoke", "stop", "status"): + if operation in ("invoke", "stop", "status", "result"): required.add("runtime_id") if operation == "start": required.add("config") @@ -143,6 +145,8 @@ def validate_request(request: Any) -> str: if len(payload_fields) != 1: raise ProtocolError() required.update(payload_fields) + if operation == "result": + required.add("request_id") allowed = required.copy() if operation == "start": allowed.add("session_ref") @@ -165,7 +169,7 @@ def validate_request(request: Any) -> str: raise ProtocolError() if not required <= request.keys() or not request.keys() <= allowed: raise ProtocolError() - for field in ("agent", "runtime_id"): + for field in ("agent", "runtime_id", "request_id"): if field in request and ( not isinstance(request[field], str) or not request[field].strip() @@ -244,10 +248,12 @@ def encode_response(response: dict[str, Any]) -> bytes: return _encode( envelope( response["operation"], - changed=False if response["operation"] in ("check", "status") else None, + changed=False + if response["operation"] in ("check", "status", "result") + else None, error=ProtocolError("response_too_large", "transport"), effects="none" - if response["operation"] in ("check", "status") + if response["operation"] in ("check", "status", "result") else "unknown", ) ) @@ -281,6 +287,7 @@ def validate_response(response: dict[str, Any], operation: str) -> None: "invoke", "check", "status", + "result", "transport", ) or error["effects"] not in ("none", "applied", "unknown") @@ -295,7 +302,7 @@ def validate_response(response: dict[str, Any], operation: str) -> None: error["message"] = str( ProtocolError(error["code"], cause=cause[1] if cause else None) ) - if operation == "status": + if operation in ("status", "result"): if ( response["changed"] is not False or error["effects"] != "none" @@ -314,6 +321,11 @@ def validate_response(response: dict[str, Any], operation: str) -> None: if response["changed"] is not False or report.server_state is None: raise invalid response["result"] = report.to_mapping() + elif operation == "result": + if result is not None: + RunResult.from_mapping(result) + if response["changed"] is not False: + raise invalid elif operation == "invoke": RunResult.from_mapping(result) if response["changed"] is not None: diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py index b46e31bf3..4ce3a8c0a 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py @@ -59,6 +59,7 @@ class OwnedRuntime: lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False) generation: int = 0 stop_lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False) + artifacts: str | None = None class RuntimeServer: @@ -139,6 +140,18 @@ async def handle(self, request: Any) -> dict[str, Any]: if self.shutting_down else None, ) + if operation == "result": + owned = self._owned(request) + stage = "result" + result = await self.fabric.invocation_result( + agent_name=request["agent"], + runtime_id=request["runtime_id"], + request_id=request["request_id"], + artifacts=owned.artifacts, + ) + return envelope( + operation, result.to_mapping() if result is not None else None + ) if operation == "status": owned = self._owned(request) stage = "status" @@ -211,6 +224,15 @@ async def handle(self, request: Any) -> dict[str, Any]: # uncertain and retain sandbox ownership. raise ProtocolError("runtime_id_conflict", "start") owned = OwnedRuntime(runtime, request["agent"], "unknown") + configured = ( + config.runtime.artifacts + or runtime.handle.environment.get("artifacts") + ) + owned.artifacts = ( + str(self.directory / configured) + if configured is not None + else None + ) self.runtimes[runtime.runtime_id] = owned # The runtime is addressable; stop can clean it up. self.start_uncertain = False diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/transport.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/transport.py index c4237ce5a..9c616ac85 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/transport.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/transport.py @@ -101,6 +101,21 @@ async def client( writer = None try: encoded = encode_request(request) + if operation == "invoke": + from nemo_fabric.models import RunRequest + + payload = ( + RunRequest.model_validate(request["request"]) + if "request" in request + else RunRequest(input=request["input"]) + ) + request = { + key: value + for key, value in request.items() + if key not in ("request", "input") + } + request["request"] = payload.to_mapping() + encoded = encode_request(request) if operation == "check": timeout = min(timeout, request.get("timeout_seconds", CHECK_SECONDS)) deadline = started + timeout @@ -121,6 +136,20 @@ async def client( await writer.drain() response = await read_message(reader, RESPONSE_LIMIT) validate_response(response, operation) + if ( + operation == "result" + and response["status"] == "succeeded" + and response["result"] is not None + ): + if any( + response["result"][key] != request[request_key] + for key, request_key in ( + ("agent_name", "agent"), + ("runtime_id", "runtime_id"), + ("request_id", "request_id"), + ) + ): + raise ProtocolError("invalid_response", "transport") if operation == "status" and response["status"] == "succeeded": result = response["result"] if (result["runtime_id"], result["agent_name"]) != ( @@ -152,6 +181,15 @@ async def client( report["completed_at"] = ( datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") ) + if ( + operation == "invoke" + and response["error"] is not None + and response["error"]["effects"] == "unknown" + ): + response["result"] = { + "runtime_id": request["runtime_id"], + "request_id": request["request"]["request_id"], + } return response except (OSError, ValueError, TypeError, KeyError, TimeoutError) as error: if operation == "check" and isinstance(error, TimeoutError): @@ -177,12 +215,23 @@ async def client( ) if not isinstance(error, ProtocolError): error = ProtocolError( - "invalid_response" if sent else "host_unavailable", "transport" + "invocation_timeout" + if sent and operation == "invoke" and isinstance(error, TimeoutError) + else "invalid_response" + if sent + else "host_unavailable", + "transport", ) uncertain = sent and operation in (*LIFECYCLE_OPERATIONS, "invoke") + recovery = None + if uncertain and operation == "invoke" and "request" in request: + recovery = { + "runtime_id": request["runtime_id"], + "request_id": request["request"]["request_id"], + } return envelope( operation, - None, + recovery, changed=None if uncertain else False, error=error, effects="unknown" if uncertain else "none", diff --git a/tests/fixtures/runtime_check_provider.py b/tests/fixtures/runtime_check_provider.py index 009a50576..8b2e7cae1 100644 --- a/tests/fixtures/runtime_check_provider.py +++ b/tests/fixtures/runtime_check_provider.py @@ -77,6 +77,8 @@ arguments.extend(["--agent", command["agent"]]) if "runtime_id" in command: arguments.extend(["--runtime-id", command["runtime_id"]]) + if name == "result": + arguments.extend(["--request-id", command["request_id"]]) field = ( "config" if name == "start" else "request" if name == "invoke" else None ) @@ -88,7 +90,14 @@ key: value for key, value in command.items() if key - in {"operation", "agent", "runtime_id", "level", "timeout_seconds"} + in { + "operation", + "agent", + "runtime_id", + "request_id", + "level", + "timeout_seconds", + } } with Path(path).open("a") as stream: stream.write(json.dumps(routing) + "\n") diff --git a/tests/python/runtime_server/test_commands.py b/tests/python/runtime_server/test_commands.py index 01ca6c7c2..1c0313811 100644 --- a/tests/python/runtime_server/test_commands.py +++ b/tests/python/runtime_server/test_commands.py @@ -296,3 +296,25 @@ def test_unusable_input_preserves_existing_bridge_error_code( assert response["error"]["code"] == "invalid_input" assert response["error"]["effects"] == "none" assert response["changed"] is False + + +def test_result_command_targets_one_request(): + from nemo_fabric.runtime_server.cli import parse_command + + assert parse_command( + [ + "result", + "--agent", + "researcher", + "--runtime-id", + "runtime-1", + "--request-id", + "request-1", + ] + ) == { + "interface_version": 2, + "operation": "result", + "agent": "researcher", + "runtime_id": "runtime-1", + "request_id": "request-1", + } diff --git a/tests/python/runtime_server/test_sdk.py b/tests/python/runtime_server/test_sdk.py index 3dc8d2811..340949967 100644 --- a/tests/python/runtime_server/test_sdk.py +++ b/tests/python/runtime_server/test_sdk.py @@ -322,3 +322,152 @@ async def test_real_sdk_busy_stop_exits_host_and_native_child( with suppress(Exception): await task await owned.stop() + + +async def test_result_is_saved_and_retrievable_after_caller_timeout( + blocking_adapter_config, tmp_path, socket_path +): + from nemo_fabric import RunRequest + from nemo_fabric.runtime_server import transport + + host = RuntimeServer(directory=tmp_path) + command = {"interface_version": 2, "agent": "blocking"} + started = await host.handle( + { + **command, + "operation": "start", + "config": blocking_adapter_config.to_mapping(), + } + ) + assert started["status"] == "succeeded", started + command["runtime_id"] = started["result"]["runtime_id"] + stop, ready = asyncio.Event(), asyncio.Event() + server = asyncio.create_task( + transport.serve( + host, socket_path=socket_path, stop_event=stop, ready_event=ready + ) + ) + try: + await asyncio.wait_for(ready.wait(), 3) + request = RunRequest(input="slow", request_id="recover-this-request") + timeout = await transport.client( + {**command, "operation": "invoke", "request": request.to_mapping()}, + socket_path=socket_path, + timeout=0.1, + ) + assert timeout["error"]["code"] == "invocation_timeout" + assert timeout["error"]["effects"] == "unknown" + assert timeout["result"] == { + "runtime_id": command["runtime_id"], + "request_id": request.request_id, + } + (tmp_path / "release").touch() + async with asyncio.timeout(5): + while True: + response = await transport.client( + { + **command, + "operation": "result", + "request_id": request.request_id, + }, + socket_path=socket_path, + ) + assert response["status"] == "succeeded", response + if response["result"] is not None: + break + await asyncio.sleep(0.01) + result = response["result"] + assert result["request_id"] == request.request_id + assert result["output"] == {"response": "released"} + saved = ( + tmp_path + / "artifacts" + / ".fabric" + / result["runtime_id"] + / result["invocation_id"] + / "run-result.json" + ) + from nemo_fabric import RunResult + + assert ( + RunResult.from_mapping(json.loads(saved.read_text())).to_mapping() == result + ) + assert saved.stat().st_mode & 0o077 == 0 + sdk_result = await host.fabric.invocation_result( + agent_name="blocking", + runtime_id=command["runtime_id"], + request_id=request.request_id, + artifacts=tmp_path / "artifacts", + ) + assert sdk_result.to_mapping() == result + assert (await host.handle({**command, "operation": "stop"}))[ + "status" + ] == "succeeded" + assert ( + await host.handle( + {**command, "operation": "result", "request_id": request.request_id} + ) + )["result"] == result + finally: + (tmp_path / "release").touch() + stop.set() + await asyncio.wait_for(server, 5) + + +async def test_cached_retry_preserves_later_sdk_history( + blocking_adapter_config, tmp_path +): + from nemo_fabric import RunRequest + + (tmp_path / "release").touch() + script = tmp_path / "adapter.py" + text = script.read_text().replace("harness=None", "history=[]\nharness=None") + text = text.replace( + "output={'status':'succeeded','output':{'response':'released'}}", + "history.append({'role':'assistant','content':str(len(history)+1)})\n output={'status':'succeeded','output':{'messages':history}}", + ) + script.write_text(text) + runtime = await Fabric().start_runtime(blocking_adapter_config, base_dir=tmp_path) + try: + request = RunRequest(input="A", request_id="request-a") + first = await runtime.invoke(request=request) + await runtime.invoke(request=RunRequest(input="B", request_id="request-b")) + history = runtime.messages + invocations = runtime.invocations + cached = await runtime.invoke(request=request) + assert cached.invocation_id == first.invocation_id + assert len(history) == 2 + assert runtime.messages == history + assert runtime.invocations == invocations + finally: + await runtime.stop() + + +async def test_lookup_before_first_invocation_returns_no_completed_result( + blocking_adapter_config, tmp_path +): + host = RuntimeServer(directory=tmp_path) + command = {"interface_version": 2, "agent": "blocking"} + started = await host.handle( + { + **command, + "operation": "start", + "config": blocking_adapter_config.to_mapping(), + } + ) + assert started["status"] == "succeeded", started + try: + response = await host.handle( + { + **command, + "operation": "result", + "runtime_id": started["result"]["runtime_id"], + "request_id": "not-submitted", + } + ) + assert response["status"] == "succeeded", response + assert response["result"] is None + assert response["changed"] is False + assert not (tmp_path / "entered").exists() + finally: + await host.shutdown() diff --git a/tests/python/runtime_server/test_transport.py b/tests/python/runtime_server/test_transport.py index 8faa170e8..0c250ebce 100644 --- a/tests/python/runtime_server/test_transport.py +++ b/tests/python/runtime_server/test_transport.py @@ -600,3 +600,35 @@ async def test_startup_preserves_caller_file_at_socket_path(host, socket_path): with pytest.raises(OSError): await transport.serve(host, socket_path=socket_path, stop_event=asyncio.Event()) assert socket_path.read_text() == "keep" + + +async def test_uncertain_response_preserves_generated_request_identity( + host_request, monkeypatch +): + from unittest.mock import AsyncMock + + reader = MagicMock(spec=asyncio.StreamReader) + reader.readline = AsyncMock( + return_value=protocol.encode_response( + protocol.envelope( + "invoke", + error=protocol.ProtocolError("response_too_large", "transport"), + changed=None, + effects="unknown", + ) + ) + ) + writer = MagicMock(spec=asyncio.StreamWriter) + writer.drain = AsyncMock() + writer.wait_closed = AsyncMock() + monkeypatch.setattr( + asyncio, "open_unix_connection", AsyncMock(return_value=(reader, writer)) + ) + response = await transport.client(host_request("invoke", input={"prompt": "hello"})) + sent = json.loads(writer.write.call_args.args[0]) + assert response["error"]["code"] == "response_too_large" + assert response["error"]["effects"] == "unknown" + assert response["result"] == { + "runtime_id": sent["runtime_id"], + "request_id": sent["request"]["request_id"], + } diff --git a/tests/python/test_sdk_server_checks.py b/tests/python/test_sdk_server_checks.py index 73cb3caa1..f8c9bc5d9 100644 --- a/tests/python/test_sdk_server_checks.py +++ b/tests/python/test_sdk_server_checks.py @@ -438,3 +438,95 @@ async def test_runtime_status_native_transport_rejects_duplicate_fields( await Fabric().runtime_status( environment=server_environment, runtime_id="runtime-1", agent_name="demo" ) + + +async def test_remote_result_lookup_during_and_after_invocation( + blocking_adapter_config, server_environment, server_sdk_bridge, tmp_path +): + from nemo_fabric import RunRequest + + host, log = server_sdk_bridge + config = blocking_adapter_config.model_copy(deep=True) + config.environment.workspace = str(tmp_path) + config.runtime.artifacts = str(tmp_path / "artifacts") + config.discovery.local_paths = [str(tmp_path / "blocking.fabric-adapter.json")] + command = {"interface_version": 2, "agent": "blocking"} + started = await host.handle( + {**command, "operation": "start", "config": config.to_mapping()} + ) + assert started["status"] == "succeeded", started + runtime_id = started["result"]["runtime_id"] + request = RunRequest(input="slow", request_id="recover-remote") + turn = asyncio.create_task( + host.handle( + { + **command, + "operation": "invoke", + "runtime_id": runtime_id, + "request": request.to_mapping(), + } + ) + ) + try: + async with asyncio.timeout(5): + while not (tmp_path / "entered").exists(): + await asyncio.sleep(0.01) + client = Fabric() + assert ( + await client.invocation_result( + environment=server_environment, + agent_name="blocking", + runtime_id=runtime_id, + request_id=request.request_id, + ) + is None + ) + (tmp_path / "release").touch() + completed = await asyncio.wait_for(turn, 5) + result = await client.invocation_result( + environment=server_environment, + agent_name="blocking", + runtime_id=runtime_id, + request_id=request.request_id, + ) + assert result.to_mapping() == completed["result"] + commands = [json.loads(line) for line in log.read_text().splitlines()] + assert ( + commands + == [ + { + "operation": "result", + "agent": "blocking", + "runtime_id": runtime_id, + "request_id": request.request_id, + } + ] + * 2 + ) + finally: + (tmp_path / "release").touch() + await turn + await host.shutdown() + + +async def test_remote_result_errors_do_not_expose_provider_details( + server_environment, monkeypatch +): + from nemo_fabric import FabricRuntimeError + + native = MagicMock() + native.invocation_result_environment.side_effect = RuntimeError( + "TEST_ONLY_PRIVATE provider token" + ) + monkeypatch.setattr("nemo_fabric.client._native", native) + with pytest.raises(FabricRuntimeError) as caught: + await Fabric().invocation_result( + environment=server_environment, + agent_name="researcher", + runtime_id="runtime-1", + request_id="request-1", + ) + assert caught.value.code == "result_unavailable" + assert "TEST_ONLY_PRIVATE" not in str(caught.value) + native.invoke_runtime.assert_not_called() + native.stop_runtime.assert_not_called() From c886676b938581e6b5eabe23a95db51175430b39 Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Thu, 12 Nov 2026 12:57:26 -0700 Subject: [PATCH 02/10] fix: compile invocation result storage on Windows Signed-off-by: Sara Tadayon --- crates/fabric-core/src/runtime/results.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/crates/fabric-core/src/runtime/results.rs b/crates/fabric-core/src/runtime/results.rs index bbbb69136..877a38361 100644 --- a/crates/fabric-core/src/runtime/results.rs +++ b/crates/fabric-core/src/runtime/results.rs @@ -46,12 +46,14 @@ fn leaf(value: &str) -> bool { fn directory(parent: &Path, name: &str, create: bool, runtime_id: &str) -> Result { let path = parent.join(name); if create { - let mut builder = fs::DirBuilder::new(); + let builder = fs::DirBuilder::new(); #[cfg(unix)] - { + let builder = { use std::os::unix::fs::DirBuilderExt; + let mut builder = builder; builder.mode(0o700); - } + builder + }; if let Err(error) = builder.create(&path) && error.kind() != std::io::ErrorKind::AlreadyExists { @@ -79,14 +81,16 @@ fn runtime_directory(root: &Path, runtime_id: &str, create: bool) -> Result OpenOptions { - let mut options = OpenOptions::new(); + let options = OpenOptions::new(); #[cfg(unix)] - { + let options = { use std::os::unix::fs::OpenOptionsExt; + let mut options = options; options .mode(0o600) .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); - } + options + }; options } From 789ee81556ee1e03bcac86acb61f4294e4289026 Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Thu, 12 Nov 2026 13:13:33 -0700 Subject: [PATCH 03/10] test: restrict Unix socket recovery checks to Unix Signed-off-by: Sara Tadayon --- tests/python/runtime_server/test_sdk.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/python/runtime_server/test_sdk.py b/tests/python/runtime_server/test_sdk.py index 340949967..2849edf64 100644 --- a/tests/python/runtime_server/test_sdk.py +++ b/tests/python/runtime_server/test_sdk.py @@ -324,6 +324,7 @@ async def test_real_sdk_busy_stop_exits_host_and_native_child( await owned.stop() +@pytest.mark.skipif(sys.platform == "win32", reason="Unix socket transport") async def test_result_is_saved_and_retrievable_after_caller_timeout( blocking_adapter_config, tmp_path, socket_path ): From 766199b132fe0b8f77120b1ee0afd15fc4b6bace Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Thu, 12 Nov 2026 14:58:16 -0700 Subject: [PATCH 04/10] fix: validate result reservations and persist finalized runs Signed-off-by: Sara Tadayon --- crates/fabric-core/src/runtime.rs | 324 +++++++++++++++++- crates/fabric-core/src/runtime/results.rs | 141 +++++++- crates/fabric-python/src/lib.rs | 17 + .../nemo_fabric.models.md | 2 + docs/sdk/python.mdx | 4 +- .../src/nemo_fabric/_native.pyi | 1 + .../src/nemo_fabric/client.py | 5 +- .../src/nemo_fabric/models.py | 12 +- .../src/nemo_fabric/runtime.py | 13 + .../src/nemo_fabric/runtime_server/README.md | 4 +- .../nemo_fabric/runtime_server/transport.py | 13 +- tests/python/runtime_server/test_commands.py | 24 ++ tests/python/runtime_server/test_transport.py | 26 ++ tests/python/test_native_sdk.py | 66 ++++ tests/python/test_runtime.py | 25 ++ tests/python/test_sdk_contract.py | 12 + tests/python/test_sdk_server_checks.py | 26 ++ 17 files changed, 677 insertions(+), 38 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 38814cdf4..2567a35f2 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -1083,9 +1083,47 @@ pub fn run_plan(plan: &RunPlan, request: RunRequest) -> Result { } Err(_) => {} } + if uses_local_host(plan) && runtime.environment.provider == "local" { + save_run_result(plan, &runtime, &result)?; + } Ok(result) } +/// Persist the final one-shot result after Python lifecycle cleanup. +#[doc(hidden)] +pub fn save_run_result(plan: &RunPlan, runtime: &RuntimeHandle, result: &RunResult) -> Result<()> { + validate_runtime_handle(plan, runtime)?; + let invalid = || { + lifecycle_error( + AdapterLifecycleOperation::Invoke, + &runtime.runtime_id, + "invalid_result_target", + "The completed result does not match its reservation.", + "", + ) + }; + if runtime.environment.provider != "local" + || result.runtime_id != runtime.runtime_id + || result.agent_name != runtime.agent_name + { + return Err(invalid()); + } + let root = artifact_manifest(plan)? + .root + .unwrap_or_else(|| std::env::temp_dir().join("nemo-fabric")); + let saved = results::invocation_result( + &root, + &result.agent_name, + &result.runtime_id, + &result.request_id, + )? + .ok_or_else(invalid)?; + if saved.invocation_id != result.invocation_id { + return Err(invalid()); + } + results::save(&root, result) +} + /// Prepare an execution environment for a run plan. /// /// This block implements local environment preparation. Other providers return @@ -2812,6 +2850,14 @@ fn run_local_host_invocation_with_timeout( if request.request_id.is_empty() { request.request_id = new_id("request"); } + if request.request_id.trim().is_empty() || request.request_id.contains('\0') { + return Err(FabricError::InvalidConfig { + field: "request_id".into(), + reason: "must not be blank or contain NUL".into(), + }); + } + let projected = project_agent_run_request(&request)?; + validate_agent_run_request_extensions(&projected, plan.adapter_descriptor.as_ref())?; let invocation = InvocationHandle { invocation_id: new_id("invocation"), request_id: request.request_id.clone(), @@ -2820,9 +2866,16 @@ fn run_local_host_invocation_with_timeout( let root = artifact_manifest(plan)? .root .unwrap_or_else(|| std::env::temp_dir().join("nemo-fabric")); - if let Some(result) = results::reserve(&root, runtime, &request, &invocation)? { - return Ok(result); - } + let mut reservation = match results::reserve( + &root, + runtime, + &request, + &invocation, + matches!(&invocation_kind, LocalHostInvocation::OpenAiStream(_)), + )? { + results::RequestReservation::Cached(result) => return Ok(*result), + results::RequestReservation::Pending(pending) => pending, + }; let host = local_hosts() .get(&runtime.runtime_id) .cloned() @@ -2938,11 +2991,12 @@ fn run_local_host_invocation_with_timeout( if let Some(sampling) = &sampling { sampling.start(invocation_started); } - let exchange = exchange_lifecycle_message( + let exchange = exchange_lifecycle_message_with_dispatch( &mut host_guard, &runtime.runtime_id, &lifecycle_request, invoke_timeout, + || reservation.mark_dispatched(), ); // Close the memory-only sampling window before reading the clock. No // OS collection or sample-file I/O runs under this registration lock. @@ -3864,6 +3918,16 @@ fn exchange_lifecycle_message( runtime_id: &str, request: &AdapterLifecycleRequest, timeout: Duration, +) -> Result { + exchange_lifecycle_message_with_dispatch(host, runtime_id, request, timeout, || {}) +} + +fn exchange_lifecycle_message_with_dispatch( + host: &mut LocalAdapterHost, + runtime_id: &str, + request: &AdapterLifecycleRequest, + timeout: Duration, + before_dispatch: impl FnOnce(), ) -> Result { let operation = request.operation(); if let Some(status) = host.child.try_wait().map_err(|source| { @@ -3888,6 +3952,9 @@ fn exchange_lifecycle_message( } let mut message = serde_json::to_string(request).map_err(FabricError::SerializeJson)?; message.push('\n'); + // A write can fail after partial delivery, so retain the reservation from + // the first possible dispatch rather than treating a write error as safe. + before_dispatch(); if let Err(source) = host .stdin .write_all(message.as_bytes()) @@ -5093,7 +5160,7 @@ fn artifact_manifest(plan: &RunPlan) -> Result { .and_then(|environment| environment.artifacts.clone()) }); if let Some(root) = &root { - std::fs::create_dir_all(root).map_err(|source| FabricError::Write { + results::create_artifact_root(root).map_err(|source| FabricError::Write { path: root.clone(), source, })?; @@ -5113,15 +5180,7 @@ fn prepare_fabric_home( .root .clone() .unwrap_or_else(|| std::env::temp_dir().join("nemo-fabric")); - let fabric_home = root - .join(".fabric") - .join(&runtime.runtime_id) - .join(&invocation.invocation_id); - std::fs::create_dir_all(&fabric_home).map_err(|source| FabricError::Write { - path: fabric_home.clone(), - source, - })?; - Ok(fabric_home) + results::invocation_directory(&root, &runtime.runtime_id, &invocation.invocation_id) } fn write_fabric_invocation(fabric_home: &Path, payload: &str) -> Result { @@ -6676,6 +6735,232 @@ else: fs::remove_dir_all(root).unwrap(); } + #[test] + fn invalid_overrides_do_not_reserve_a_request() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let mut request = RunRequest::text("hello"); + request.request_id = "validation-retry".into(); + request.overrides = Some(serde_json::json!([])); + assert!(invoke_runtime(&plan, &runtime, request.clone()).is_err()); + request.overrides = None; + let retried = invoke_runtime(&plan, &runtime, request); + stop_runtime(&plan, &runtime).unwrap(); + assert_eq!(retried.unwrap().output["invocation_count"], 1); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn pre_dispatch_host_failure_does_not_block_a_corrected_retry() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let host = local_hosts().remove(&runtime.runtime_id).unwrap(); + let mut request = RunRequest::text("original"); + request.request_id = "not-dispatched".into(); + let rejected = invoke_runtime(&plan, &runtime, request.clone()); + local_hosts().insert(runtime.runtime_id.clone(), host); + assert!(rejected.is_err()); + request.input = serde_json::json!("corrected"); + let retried = invoke_runtime(&plan, &runtime, request); + stop_runtime(&plan, &runtime).unwrap(); + assert_eq!(retried.unwrap().output["invocation_count"], 1); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn invalid_request_ids_are_rejected_before_invocation() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + for id in [" ", "\t\n", "request\0id"] { + let mut request = RunRequest::text("hello"); + request.request_id = id.into(); + assert!(matches!(invoke_runtime(&plan, &runtime, request), + Err(FabricError::InvalidConfig { field, .. }) if field == "request_id")); + assert!(matches!( + invocation_result(&root.join("artifacts"), &runtime.agent_name, &runtime.runtime_id, id), + Err(FabricError::AdapterLifecycleOperation { code, .. }) if code == "invalid_result_target" + )); + } + let result = invoke_runtime(&plan, &runtime, RunRequest::text("valid")).unwrap(); + stop_runtime(&plan, &runtime).unwrap(); + assert_eq!(result.output["invocation_count"], 1); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn one_shot_lookup_returns_the_final_result_after_stop() { + for mode in ["success", "stop_failure"] { + let (root, plan) = local_host_plan(mode); + let result = run_plan(&plan, RunRequest::text("hello")).unwrap(); + let saved = invocation_result( + &root.join("artifacts"), + &result.agent_name, + &result.runtime_id, + &result.request_id, + ) + .unwrap(); + assert_eq!(saved, Some(result)); + fs::remove_dir_all(root).unwrap(); + } + } + + #[test] + fn final_result_save_requires_original_runtime_and_reserved_identity() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let result = invoke_runtime(&plan, &runtime, RunRequest::text("hello")).unwrap(); + stop_runtime(&plan, &runtime).unwrap(); + let mut wrong_runtime = runtime.clone(); + wrong_runtime.runtime_binding = "wrong-binding".into(); + assert!(save_run_result(&plan, &wrong_runtime, &result).is_err()); + for field in ["agent", "runtime", "request", "invocation"] { + let mut wrong_result = result.clone(); + match field { + "agent" => wrong_result.agent_name.push_str("-other"), + "runtime" => wrong_result.runtime_id.push_str("-other"), + "request" => wrong_result.request_id.push_str("-other"), + _ => wrong_result.invocation_id = "../../escape".into(), + } + assert!(save_run_result(&plan, &runtime, &wrong_result).is_err()); + } + assert_eq!( + invocation_result( + &root.join("artifacts"), + &result.agent_name, + &result.runtime_id, + &result.request_id + ) + .unwrap(), + Some(result) + ); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn oversized_results_do_not_replace_a_readable_saved_result() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let result = invoke_runtime(&plan, &runtime, RunRequest::text("hello")).unwrap(); + let mut oversized = result.clone(); + oversized.output = Value::String("x".repeat(4 * 1024 * 1024)); + let saved = results::save(&root.join("artifacts"), &oversized); + stop_runtime(&plan, &runtime).unwrap(); + assert!(saved.is_err()); + assert_eq!( + invocation_result( + &root.join("artifacts"), + &result.agent_name, + &result.runtime_id, + &result.request_id, + ) + .unwrap(), + Some(result) + ); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn buffered_results_cannot_satisfy_a_stream_request() { + let (root, mut plan) = local_host_plan("success"); + plan.capabilities.streaming = true; + plan.adapter_descriptor + .as_mut() + .unwrap() + .descriptor + .capabilities + .streaming = true; + let runtime = start_runtime(&plan).unwrap(); + let request = RunRequest::text("hello"); + let result = invoke_runtime(&plan, &runtime, request.clone()).unwrap(); + let streamed = invoke_openai_stream( + &plan, + &runtime, + request, + OpenAiStreamTransport { + port: 1, + token: "test-stream-token".into(), + }, + ); + stop_runtime(&plan, &runtime).unwrap(); + assert!( + matches!(streamed, Err(FabricError::AdapterLifecycleOperation { code, .. }) + if code == "request_conflict") + ); + assert_eq!( + invocation_result( + &root.join("artifacts"), + &result.agent_name, + &result.runtime_id, + &result.request_id, + ) + .unwrap(), + Some(result) + ); + fs::remove_dir_all(root).unwrap(); + } + + #[cfg(unix)] + #[test] + fn result_lookup_rejects_a_writable_storage_root() { + use std::os::unix::fs::PermissionsExt; + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let result = invoke_runtime(&plan, &runtime, RunRequest::text("hello")).unwrap(); + let artifacts = root.join("artifacts"); + fs::set_permissions(&artifacts, fs::Permissions::from_mode(0o777)).unwrap(); + let lookup = invocation_result( + &artifacts, + &result.agent_name, + &result.runtime_id, + &result.request_id, + ); + fs::set_permissions(&artifacts, fs::Permissions::from_mode(0o700)).unwrap(); + stop_runtime(&plan, &runtime).unwrap(); + assert!(lookup.is_err()); + fs::remove_dir_all(root).unwrap(); + } + + #[cfg(unix)] + #[test] + fn result_lookup_rejects_a_writable_storage_ancestor() { + use std::os::unix::fs::PermissionsExt; + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let result = invoke_runtime(&plan, &runtime, RunRequest::text("hello")).unwrap(); + fs::set_permissions(&root, fs::Permissions::from_mode(0o777)).unwrap(); + let lookup = invocation_result( + &root.join("artifacts"), + &result.agent_name, + &result.runtime_id, + &result.request_id, + ); + fs::set_permissions(&root, fs::Permissions::from_mode(0o700)).unwrap(); + stop_runtime(&plan, &runtime).unwrap(); + assert!(lookup.is_err()); + fs::remove_dir_all(root).unwrap(); + } + + #[cfg(unix)] + #[test] + fn result_lookup_rejects_an_unsafe_existing_namespace() { + use std::os::unix::fs::PermissionsExt; + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let result = invoke_runtime(&plan, &runtime, RunRequest::text("hello")).unwrap(); + let namespace = root.join("artifacts/.fabric"); + fs::set_permissions(&namespace, fs::Permissions::from_mode(0o777)).unwrap(); + let lookup = invocation_result( + &root.join("artifacts"), + &result.agent_name, + &result.runtime_id, + &result.request_id, + ); + fs::set_permissions(&namespace, fs::Permissions::from_mode(0o700)).unwrap(); + stop_runtime(&plan, &runtime).unwrap(); + assert!(lookup.is_err()); + fs::remove_dir_all(root).unwrap(); + } + #[test] fn result_lookup_and_duplicate_request_preserve_one_invocation() { let (root, plan) = local_host_plan("success"); @@ -7869,7 +8154,8 @@ else: let runtime = start_runtime(&plan).expect("start local host"); let (transport, capture) = openai_stream_listener("stream-secret"); - let result = invoke_openai_stream(&plan, &runtime, RunRequest::text("stream"), transport) + let request = RunRequest::text("stream"); + let result = invoke_openai_stream(&plan, &runtime, request.clone(), transport.clone()) .expect("stream invocation"); let records = capture.join().expect("stream capture"); @@ -7911,6 +8197,14 @@ else: assert_eq!(persisted["stream"]["invocation_id"], result.invocation_id); assert_eq!(persisted["stream"]["request_id"], result.request_id); + let replayed = invoke_openai_stream(&plan, &runtime, request.clone(), transport); + assert!( + matches!(replayed, Err(FabricError::AdapterLifecycleOperation { code, .. }) + if code == "stream_replay_unsupported") + ); + assert!(matches!(invoke_runtime(&plan, &runtime, request), + Err(FabricError::AdapterLifecycleOperation { code, .. }) + if code == "request_conflict")); stop_runtime(&plan, &runtime).expect("stop local host"); let _ = fs::remove_dir_all(root); } diff --git a/crates/fabric-core/src/runtime/results.rs b/crates/fabric-core/src/runtime/results.rs index 877a38361..551c95013 100644 --- a/crates/fabric-core/src/runtime/results.rs +++ b/crates/fabric-core/src/runtime/results.rs @@ -24,6 +24,34 @@ struct Reservation { request_id: String, invocation_id: String, fingerprint: String, + #[serde(default)] + streaming: Option, +} + +pub(super) enum RequestReservation { + Cached(Box), + Pending(PendingReservation), +} + +pub(super) struct PendingReservation { + path: PathBuf, + dispatched: bool, +} + +impl PendingReservation { + pub(super) fn mark_dispatched(&mut self) { + self.dispatched = true; + } +} + +impl Drop for PendingReservation { + fn drop(&mut self) { + if !self.dispatched { + // This guard owns the file created with create_new. Other requests + // cannot acquire this request ID until the file has been removed. + let _ = fs::remove_file(&self.path); + } + } } fn failure(runtime_id: &str, code: &str) -> crate::FabricError { @@ -43,6 +71,54 @@ fn leaf(value: &str) -> bool { .all(|c| c.is_ascii_alphanumeric() || c == b'-' || c == b'_') } +fn trusted_ancestry(root: &Path, runtime_id: &str) -> Result<()> { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + // SAFETY: geteuid has no preconditions or side effects. + let uid = unsafe { libc::geteuid() }; + for ancestor in root.ancestors() { + let metadata = fs::symlink_metadata(ancestor) + .map_err(|_| failure(runtime_id, "result_storage_failed"))?; + if !metadata.is_dir() + || !matches!(metadata.uid(), owner if owner == uid || owner == 0) + || (metadata.mode() & 0o022 != 0 && metadata.mode() & 0o1000 == 0) + { + return Err(failure(runtime_id, "result_storage_failed")); + } + } + } + #[cfg(not(unix))] + let _ = (root, runtime_id); + Ok(()) +} + +fn owned_metadata(metadata: &fs::Metadata, file: bool) -> bool { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + // SAFETY: geteuid has no preconditions or side effects. + let forbidden = if file { 0o077 } else { 0o022 }; + metadata.uid() == unsafe { libc::geteuid() } && metadata.mode() & forbidden == 0 + } + #[cfg(not(unix))] + { + let _ = (metadata, file); + true + } +} + +pub(super) fn create_artifact_root(root: &Path) -> std::io::Result<()> { + let mut builder = fs::DirBuilder::new(); + builder.recursive(true); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder.create(root) +} + fn directory(parent: &Path, name: &str, create: bool, runtime_id: &str) -> Result { let path = parent.join(name); if create { @@ -62,7 +138,8 @@ fn directory(parent: &Path, name: &str, create: bool, runtime_id: &str) -> Resul } let metadata = fs::symlink_metadata(&path).map_err(|_| failure(runtime_id, "result_storage_failed"))?; - if !metadata.is_dir() || metadata.file_type().is_symlink() { + if !metadata.is_dir() || metadata.file_type().is_symlink() || !owned_metadata(&metadata, false) + { return Err(failure(runtime_id, "result_storage_failed")); } Ok(path) @@ -73,13 +150,26 @@ fn runtime_directory(root: &Path, runtime_id: &str, create: bool) -> Result Result { + if !leaf(invocation_id) { + return Err(failure(runtime_id, "invalid_result_target")); + } + let runtime = runtime_directory(root, runtime_id, true)?; + directory(&runtime, invocation_id, true, runtime_id) +} + fn options() -> OpenOptions { let options = OpenOptions::new(); #[cfg(unix)] @@ -103,7 +193,8 @@ fn read(path: &Path, runtime_id: &str) -> Result let metadata = file .metadata() .map_err(|_| failure(runtime_id, "result_storage_failed"))?; - if !metadata.is_file() || metadata.len() > MAX_RESULT_BYTES { + if !metadata.is_file() || metadata.len() > MAX_RESULT_BYTES || !owned_metadata(&metadata, true) + { return Err(failure(runtime_id, "result_storage_failed")); } let mut bytes = Vec::new(); @@ -148,7 +239,8 @@ pub(super) fn reserve( runtime: &RuntimeHandle, request: &RunRequest, invocation: &InvocationHandle, -) -> Result> { + streaming: bool, +) -> Result { let runtime_dir = runtime_directory(root, &runtime.runtime_id, true)?; let requests = directory(&runtime_dir, "requests", true, &runtime.runtime_id)?; let path = requests.join(index_name(&request.request_id)); @@ -161,14 +253,22 @@ pub(super) fn reserve( "{:x}", Sha256::digest(serde_json::to_vec(request).map_err(crate::FabricError::SerializeJson)?) ), + streaming: Some(streaming), }; + let bytes = serde_json::to_vec(&record).map_err(crate::FabricError::SerializeJson)?; + if bytes.len() as u64 > MAX_RESULT_BYTES { + return Err(failure(&runtime.runtime_id, "result_storage_failed")); + } match options().write(true).create_new(true).open(&path) { Ok(mut file) => { - let bytes = serde_json::to_vec(&record).map_err(crate::FabricError::SerializeJson)?; + let pending = PendingReservation { + path, + dispatched: false, + }; file.write_all(&bytes) .and_then(|_| file.sync_all()) .map_err(|_| failure(&runtime.runtime_id, "result_storage_failed"))?; - Ok(None) + Ok(RequestReservation::Pending(pending)) } Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { let previous: Reservation = read(&path, &runtime.runtime_id)? @@ -177,27 +277,33 @@ pub(super) fn reserve( || previous.runtime_id != record.runtime_id || previous.request_id != record.request_id || previous.fingerprint != record.fingerprint + || previous.streaming != record.streaming { return Err(failure(&runtime.runtime_id, "request_conflict")); } - completed(&runtime_dir, &previous)? - .map(Some) - .ok_or_else(|| failure(&runtime.runtime_id, "invocation_pending")) + let result = completed(&runtime_dir, &previous)? + .ok_or_else(|| failure(&runtime.runtime_id, "invocation_pending"))?; + if streaming { + return Err(failure(&runtime.runtime_id, "stream_replay_unsupported")); + } + Ok(RequestReservation::Cached(Box::new(result))) } Err(_) => Err(failure(&runtime.runtime_id, "result_storage_failed")), } } pub(super) fn save(root: &Path, result: &RunResult) -> Result<()> { - let runtime = runtime_directory(root, &result.runtime_id, true)?; - let invocation = directory(&runtime, &result.invocation_id, true, &result.runtime_id)?; + let bytes = serde_json::to_vec(result).map_err(crate::FabricError::SerializeJson)?; + if bytes.len() as u64 > MAX_RESULT_BYTES { + return Err(failure(&result.runtime_id, "result_storage_failed")); + } + let invocation = invocation_directory(root, &result.runtime_id, &result.invocation_id)?; let temporary = invocation.join(format!("{}.tmp", super::new_id("result"))); let mut file: File = options() .write(true) .create_new(true) .open(&temporary) .map_err(|_| failure(&result.runtime_id, "result_storage_failed"))?; - let bytes = serde_json::to_vec(result).map_err(crate::FabricError::SerializeJson)?; file.write_all(&bytes) .and_then(|_| file.sync_all()) .and_then(|_| fs::rename(&temporary, invocation.join("run-result.json"))) @@ -213,7 +319,10 @@ pub fn invocation_result( runtime_id: &str, request_id: &str, ) -> Result> { - if request_id.trim().is_empty() || agent.trim().is_empty() { + if [request_id, agent] + .iter() + .any(|value| value.trim().is_empty() || value.contains('')) + { return Err(failure(runtime_id, "invalid_result_target")); } if !leaf(runtime_id) { @@ -224,12 +333,16 @@ pub fn invocation_result( Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), Err(_) => return Err(failure(runtime_id, "result_storage_failed")), }; + trusted_ancestry(&root, runtime_id)?; let fabric = root.join(".fabric"); let runtime = fabric.join(runtime_id); let requests = runtime.join("requests"); for path in [&fabric, &runtime, &requests] { match fs::symlink_metadata(path) { - Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => {} + Ok(metadata) + if metadata.is_dir() + && !metadata.file_type().is_symlink() + && owned_metadata(&metadata, false) => {} Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), _ => return Err(failure(runtime_id, "result_storage_failed")), } diff --git a/crates/fabric-python/src/lib.rs b/crates/fabric-python/src/lib.rs index 26879566d..34790749d 100644 --- a/crates/fabric-python/src/lib.rs +++ b/crates/fabric-python/src/lib.rs @@ -308,6 +308,22 @@ fn invocation_result( to_json(&result) } +/// Persist a result after the Python one-shot lifecycle finishes cleanup. +#[pyfunction] +fn _save_run_result( + py: Python<'_>, + plan_json: String, + runtime_json: String, + result_json: String, +) -> PyResult<()> { + let plan = parse_run_plan(plan_json)?; + let runtime = parse_runtime_handle(runtime_json)?; + let result: nemo_fabric_core::RunResult = serde_json::from_str(&result_json) + .map_err(|_| pyo3::exceptions::PyValueError::new_err("The completed result is invalid."))?; + py.detach(|| nemo_fabric_core::runtime::save_run_result(&plan, &runtime, &result)) + .map_err(|_| PyRuntimeError::new_err("The final invocation result could not be saved.")) +} + /// Read a saved result through an OpenShell environment. #[pyfunction] fn invocation_result_environment( @@ -443,6 +459,7 @@ fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(check_environment, m)?)?; m.add_function(wrap_pyfunction!(runtime_status_environment, m)?)?; m.add_function(wrap_pyfunction!(invocation_result, m)?)?; + m.add_function(wrap_pyfunction!(_save_run_result, m)?)?; m.add_function(wrap_pyfunction!(invocation_result_environment, m)?)?; m.add_function(wrap_pyfunction!(check_service, m)?)?; m.add("ServiceInUseError", m.py().get_type::())?; diff --git a/docs/reference/api/python-library-reference/nemo_fabric.models.md b/docs/reference/api/python-library-reference/nemo_fabric.models.md index 65356de00..13a26a990 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.models.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.models.md @@ -2761,6 +2761,8 @@ Return a detached mapping matching the Rust ``FabricConfig`` schema. One validated NeMo Fabric invocation request. +Request identifiers must not be blank or contain NUL. + ### Fields diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 65cfb9e63..bf3cca3bb 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1711,7 +1711,9 @@ The [software evidence example](https://github.com/NVIDIA/NeMo-Fabric/tree/main/ ### Recover a buffered invocation result -Fabric saves each completed `RunResult` as `/.fabric///run-result.json` before returning the result. On Unix, result and request-record files are readable only by their owner. The artifact root comes from `runtime.artifacts` or the environment; without either setting, Fabric uses its temporary artifact directory. Files remain until their storage is removed; no automatic retention or eviction is added. +NeMo Fabric saves each completed `RunResult` as `/.fabric///run-result.json` before returning the result. On Unix, result and request-record files are readable only by their owner. The artifact root comes from `runtime.artifacts` or the environment; without either setting, NeMo Fabric uses its temporary artifact directory. Files remain until their storage is removed; no automatic retention or eviction is added. + +Use a trusted artifact directory. On Unix, result storage rejects unsafe directory ownership, writable ancestry, and record-file permissions. Serialized results must fit within 4 MiB. Request identifiers must not be blank or contain NUL. Buffered retries can return the saved result; a request ID cannot switch between buffered and streaming invocations. A completed streaming request returns `stream_replay_unsupported` on replay; retrieve its terminal result with `invocation_result()` instead. Choose a stable `RunRequest.request_id` before submitting work. After a command timeout, `invocation_timeout` reports unknown effects and returns the runtime and request identifiers. The invocation can continue. Retrieve the completed result with: diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi index e3e8f4180..f2d7a8318 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi @@ -60,3 +60,4 @@ def invocation_result( def invocation_result_environment( environment_json: str, agent: str, runtime_id: str, request_id: str ) -> str: ... +def _save_run_result(plan_json: str, runtime_json: str, result_json: str) -> None: ... diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py index 3d7f55643..719aae113 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py @@ -129,6 +129,9 @@ async def invocation_result( for value in (agent_name, runtime_id, request_id) ): raise FabricConfigError("The invocation result target is invalid.") + environment_json = ( + _environment_json(environment) if environment is not None else None + ) native = self._require_native_module("invocation_result") if environment is not None and environment.provider == "openshell": if artifacts is not None: @@ -136,7 +139,7 @@ async def invocation_result( "Remote lookup uses the runtime's artifact storage." ) operation = lambda: native.invocation_result_environment( - _environment_json(environment), agent_name, runtime_id, request_id + environment_json, agent_name, runtime_id, request_id ) else: if environment is not None and environment.provider != "local": diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py index 19dacbe56..e5780ec55 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py @@ -1361,7 +1361,10 @@ def enable_relay( class RunRequest(FabricBaseModel): - """One validated NeMo Fabric invocation request.""" + """One validated NeMo Fabric invocation request. + + Request identifiers must not be blank or contain NUL. + """ acknowledge_recovery: bool = Field( default=False, @@ -1380,6 +1383,13 @@ class RunRequest(FabricBaseModel): context: dict[str, Any] = Field(default_factory=dict) overrides: dict[str, Any] | None = None + @field_validator("request_id") + @classmethod + def _validate_request_id(cls, value: str) -> str: + if not value.strip() or "\0" in value: + raise ValueError("request_id must not be blank or contain NUL") + return value + @field_validator("input", mode="before") @classmethod def _validate_input(cls, value: Any) -> Any: diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index cdde84895..8421f8134 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -1103,6 +1103,19 @@ def run() -> dict[str, Any]: ) if result is not None: observations["partial_result"] = result + if runtime.get("environment", {}).get("provider") == "local": + try: + native._save_run_result( + plan_json, runtime_json, json.dumps(result) + ) + except Exception as error: + failure = FabricRuntimeError( + "The final invocation result could not be saved.", + stage="result", + code="result_storage_failed", + ) + _retain_runtime_evidence(failure, observations) + raise failure from error if ( stop_error is not None and _runtime_error(stop_error, stage="stop").code diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md index 35c5c09e9..dcdd71bee 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md @@ -112,7 +112,9 @@ The task configuration carries the default-on `provenance` collector control and ### Recover a buffered invocation result -Fabric saves each completed `RunResult` as `/.fabric///run-result.json` before returning the result. On Unix, result and request-record files are readable only by their owner. The artifact root comes from `runtime.artifacts` or the environment; without either setting, Fabric uses its temporary artifact directory. Files remain until their storage is removed; no automatic retention or eviction is added. +NeMo Fabric saves each completed `RunResult` as `/.fabric///run-result.json` before returning the result. On Unix, result and request-record files are readable only by their owner. The artifact root comes from `runtime.artifacts` or the environment; without either setting, NeMo Fabric uses its temporary artifact directory. Files remain until their storage is removed; no automatic retention or eviction is added. + +Use a trusted artifact directory. On Unix, result storage rejects unsafe directory ownership, writable ancestry, and record-file permissions. Serialized results must fit within 4 MiB. Request identifiers must not be blank or contain NUL. Buffered retries can return the saved result; a request ID cannot switch between buffered and streaming invocations. A completed streaming request returns `stream_replay_unsupported` on replay; retrieve its terminal result with `invocation_result()` instead. Choose a stable `RunRequest.request_id` before submitting work. After a command timeout, `invocation_timeout` reports unknown effects and returns the runtime and request identifiers. The invocation can continue. Retrieve the completed result with: diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/transport.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/transport.py index 9c616ac85..dbc906789 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/transport.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/transport.py @@ -104,11 +104,14 @@ async def client( if operation == "invoke": from nemo_fabric.models import RunRequest - payload = ( - RunRequest.model_validate(request["request"]) - if "request" in request - else RunRequest(input=request["input"]) - ) + try: + payload = ( + RunRequest.model_validate(request["request"]) + if "request" in request + else RunRequest(input=request["input"]) + ) + except (TypeError, ValueError): + raise ProtocolError("invalid_request") from None request = { key: value for key, value in request.items() diff --git a/tests/python/runtime_server/test_commands.py b/tests/python/runtime_server/test_commands.py index 1c0313811..c8c8614a8 100644 --- a/tests/python/runtime_server/test_commands.py +++ b/tests/python/runtime_server/test_commands.py @@ -298,6 +298,30 @@ def test_unusable_input_preserves_existing_bridge_error_code( assert response["changed"] is False +@pytest.mark.parametrize( + "flags", + [[], ["--request-id", ""], ["--request-id", "first", "--request-id", "second"]], +) +def test_result_command_rejects_invalid_request_id_before_transport( + flags, monkeypatch, capsys +): + from unittest.mock import AsyncMock + + client = AsyncMock() + monkeypatch.setattr(cli.transport, "client", client) + assert ( + cli.main( + ["result", "--agent", "researcher", "--runtime-id", "runtime-1", *flags] + ) + == 1 + ) + response = json.loads(capsys.readouterr().out) + assert response["error"]["code"] == "invalid_request" + assert response["error"]["effects"] == "none" + assert response["changed"] is False + client.assert_not_awaited() + + def test_result_command_targets_one_request(): from nemo_fabric.runtime_server.cli import parse_command diff --git a/tests/python/runtime_server/test_transport.py b/tests/python/runtime_server/test_transport.py index 0c250ebce..9c8fb2acd 100644 --- a/tests/python/runtime_server/test_transport.py +++ b/tests/python/runtime_server/test_transport.py @@ -23,6 +23,32 @@ from nemo_fabric.runtime_server import protocol, transport +@pytest.mark.parametrize( + "payload", [{"context": []}, {"overrides": []}, {"request_id": " "}] +) +async def test_invalid_invocation_payload_is_rejected_before_connection( + monkeypatch, socket_path, payload +): + from unittest.mock import AsyncMock + + connect = AsyncMock() + monkeypatch.setattr(asyncio, "open_unix_connection", connect) + response = await transport.client( + { + "interface_version": 2, + "operation": "invoke", + "agent": "researcher", + "runtime_id": "runtime-1", + "request": {"input": "hello", **payload}, + }, + socket_path=socket_path, + ) + assert response["error"]["code"] == "invalid_request" + assert response["error"]["effects"] == "none" + assert response["changed"] is False + connect.assert_not_awaited() + + @pytest.fixture(name="lifecycle_files") def lifecycle_files_fixture(monkeypatch): files = { diff --git a/tests/python/test_native_sdk.py b/tests/python/test_native_sdk.py index 89de856c3..32084bc23 100644 --- a/tests/python/test_native_sdk.py +++ b/tests/python/test_native_sdk.py @@ -26,6 +26,72 @@ from nemo_fabric import RuntimeStatus +@pytest.mark.parametrize("stop_fails", [False, True]) +async def test_one_shot_lookup_matches_cleanup_result( + hermes_shim_agent_dir, monkeypatch, stop_fails +): + monkeypatch.setenv("ADAPTER_PYTHON", sys.executable) + original_stop = native.stop_runtime + + def stop(*args): + events = original_stop(*args) + if stop_fails: + raise RuntimeError("test-only stop failure after confirmed cleanup") + return events + + monkeypatch.setattr(native, "stop_runtime", stop) + client = Fabric() + result = await client.run( + hermes_shim_config(), base_dir=hermes_shim_agent_dir, input="final result" + ) + saved = await client.invocation_result( + artifacts=hermes_shim_agent_dir / "artifacts", + agent_name=result.agent_name, + runtime_id=result.runtime_id, + request_id=result.request_id, + ) + assert result.status == ("failed" if stop_fails else "succeeded") + assert saved == result + + +@pytest.mark.skipif(sys.platform == "win32", reason="Unix umask permissions") +@pytest.mark.parametrize("existing_root", [False, True]) +def test_result_storage_supports_group_writable_umask( + hermes_shim_agent_dir, monkeypatch, existing_root +): + monkeypatch.setenv("ADAPTER_PYTHON", sys.executable) + artifacts = hermes_shim_agent_dir / "artifacts" + if existing_root: + artifacts.mkdir(mode=0o700) + script = """ +import asyncio, json, os, sys +from nemo_fabric import Fabric, FabricConfig +os.umask(0o002) +async def main(): + config = FabricConfig.from_mapping(json.loads(sys.argv[1])) + runtime = await Fabric().start_runtime(config, base_dir=sys.argv[2]) + try: + result = await runtime.invoke(input="private result storage") + assert result.status == "succeeded" + finally: + await runtime.stop() +asyncio.run(main()) +""" + completed = subprocess.run( + [ + sys.executable, + "-c", + script, + hermes_shim_config().model_dump_json(exclude_none=True), + str(hermes_shim_agent_dir), + ], + capture_output=True, + text=True, + timeout=30, + ) + assert completed.returncode == 0, completed.stderr + + async def test_native_sdk(hermes_shim_agent_dir: Path): assert native.version() diff --git a/tests/python/test_runtime.py b/tests/python/test_runtime.py index 5071280c9..91c391ca7 100644 --- a/tests/python/test_runtime.py +++ b/tests/python/test_runtime.py @@ -308,6 +308,31 @@ def occupy_worker(): mock_native.stop_runtime.assert_called_once() +async def test_remote_one_shot_does_not_write_to_local_result_storage(mock_native): + runtime = _runtime() + runtime["environment"]["provider"] = "openshell" + mock_native.start_runtime.return_value = json.dumps(runtime) + result = await runtime_mod._run_native_lifecycle( + mock_native, _plan(), {"request_id": "remote-request", "input": "remote work"} + ) + assert result["status"] == "succeeded" + mock_native._save_run_result.assert_not_called() + + +async def test_final_save_failure_retains_one_shot_recovery_evidence(mock_native): + mock_native._save_run_result.side_effect = RuntimeError("storage unavailable") + with pytest.raises(FabricRuntimeError) as caught: + await runtime_mod._run_native_lifecycle( + mock_native, _plan(), {"request_id": "saved-request", "input": "hello"} + ) + assert caught.value.stage == "result" + evidence = caught.value.details["runtime_evidence"] + assert evidence["runtime_id"] == "runtime-1" + assert evidence["request_id"] == "saved-request" + assert evidence["cleanup"] == "completed" + assert evidence["partial_result"]["status"] == "succeeded" + + @pytest.mark.parametrize("stage", ["start", "invoke", "stop"]) @pytest.mark.parametrize( ("error_type", "native_code", "expected_code"), diff --git a/tests/python/test_sdk_contract.py b/tests/python/test_sdk_contract.py index 65d1e707e..aa8734da1 100644 --- a/tests/python/test_sdk_contract.py +++ b/tests/python/test_sdk_contract.py @@ -1751,6 +1751,7 @@ def _fabric_config() -> FabricConfig: class NativeRecorder: def __init__(self) -> None: self.requests: list[dict[str, Any]] = [] + self.saved_results: list[dict[str, Any]] = [] self.config_base_dir_calls: list[str | None] = [] self.stopped = 0 self.fail_invoke = False @@ -1838,6 +1839,16 @@ def stop_runtime(self, plan_json: str, runtime_json: str) -> str: self.stopped += 1 return json.dumps([]) + def _save_run_result( + self, plan_json: str, runtime_json: str, result_json: str + ) -> None: + result = json.loads(result_json) + assert self.stopped > 0 + assert json.loads(plan_json)["agent_name"] == result["agent_name"] + assert json.loads(runtime_json)["runtime_id"] == result["runtime_id"] + assert self.requests[-1]["request_id"] == result["request_id"] + self.saved_results.append(result) + class NativeClient(Fabric): def __init__(self, native: NativeRecorder) -> None: @@ -2205,6 +2216,7 @@ async def test_run_accepts_full_run_request(): assert isinstance(result, RunResult) assert result.status == "succeeded" + assert RunResult.from_mapping(native.saved_results[0]) == result assert native.requests[0] == { "input": "hello", "request_id": "request-4", diff --git a/tests/python/test_sdk_server_checks.py b/tests/python/test_sdk_server_checks.py index f8c9bc5d9..5c765e81e 100644 --- a/tests/python/test_sdk_server_checks.py +++ b/tests/python/test_sdk_server_checks.py @@ -530,3 +530,29 @@ async def test_remote_result_errors_do_not_expose_provider_details( assert "TEST_ONLY_PRIVATE" not in str(caught.value) native.invoke_runtime.assert_not_called() native.stop_runtime.assert_not_called() + + +@pytest.mark.parametrize("request_id", [" ", "\t\n", "request\0id"]) +def test_recovery_request_ids_are_validated_before_invocation(request_id): + from nemo_fabric import RunRequest + + with pytest.raises(ValueError): + RunRequest(input="hello", request_id=request_id) + + +async def test_result_lookup_rejects_untyped_environment_before_native_dispatch( + monkeypatch, +): + from nemo_fabric import FabricConfigError + + native = MagicMock() + monkeypatch.setattr("nemo_fabric.client._native", native) + with pytest.raises(FabricConfigError): + await Fabric().invocation_result( + agent_name="researcher", + runtime_id="runtime-1", + request_id="request-1", + environment={"provider": "openshell"}, + ) + native.invocation_result.assert_not_called() + native.invocation_result_environment.assert_not_called() From 7b63ac4da4ee5ff989fcfb6fc2aff91d66d6a9dd Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Thu, 12 Nov 2026 17:21:14 -0700 Subject: [PATCH 05/10] fix: publish invocation results after cleanup finalizes Signed-off-by: Sara Tadayon --- crates/fabric-core/src/runtime.rs | 239 +++++++++++++++--- crates/fabric-core/src/runtime/results.rs | 104 +++++++- crates/fabric-python/src/lib.rs | 34 ++- .../runtime/struct-runrequest.mdx | 4 +- docs/sdk/python.mdx | 2 + schemas/sdk/run-request.schema.json | 4 +- .../src/nemo_fabric/_native.pyi | 1 + .../src/nemo_fabric/client.py | 11 +- .../src/nemo_fabric/runtime.py | 4 + .../src/nemo_fabric/runtime_server/README.md | 2 + tests/python/test_native_sdk.py | 58 +++++ tests/python/test_sdk_contract.py | 7 + tests/python/test_sdk_server_checks.py | 30 +++ 13 files changed, 456 insertions(+), 44 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 2567a35f2..b5e3adc29 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -115,12 +115,35 @@ pub fn invocation_result_environment( runtime_id: runtime_id.into(), request_id: request_id.into(), }; - match crate::environment::command_runtime(environment, &command, 10)? { + result_lookup_response(crate::environment::command_runtime( + environment, + &command, + 10, + )?) +} + +fn result_lookup_response(response: crate::task_command::TaskCommandResponse) -> Result { + match response { crate::task_command::TaskCommandResponse::Returned { value } => Ok(value), - crate::task_command::TaskCommandResponse::Rejected { .. } => { - Err(FabricError::InvalidConfig { - field: "invocation_result".into(), - reason: "the result lookup could not be completed; do not replay uncertain work" + crate::task_command::TaskCommandResponse::Rejected { error } => { + Err(FabricError::EnvironmentProviderOperation { + provider: "openshell".into(), + operation: "invocation_result".into(), + code: if error.effects == "none" + && error.code.len() <= 64 + && error + .code + .bytes() + .next() + .is_some_and(|first| first.is_ascii_lowercase()) + && error.code.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_' + }) { + error.code + } else { + "invalid_response".into() + }, + message: "The result lookup could not be completed; do not replay uncertain work." .into(), }) } @@ -178,12 +201,13 @@ static LOCAL_SERVICES: LazyLock, } +impl Default for RunRequest { + fn default() -> Self { + Self { + request_id: new_id("request"), + relay_session_root: None, + input: Value::Null, + context: BTreeMap::new(), + overrides: None, + } + } +} + impl RunRequest { /// Build a text request. pub fn text(input: impl Into) -> Self { @@ -1049,7 +1085,9 @@ enum LocalServiceState { /// any completed result. Retry [`stop_runtime`] with that handle and this plan. pub fn run_plan(plan: &RunPlan, request: RunRequest) -> Result { let runtime = start_runtime(plan)?; - let mut result = match invoke_runtime(plan, &runtime, request) { + let invoked = begin_run_result(plan, &runtime, &request) + .and_then(|()| invoke_runtime(plan, &runtime, request)); + let mut result = match invoked { Ok(result) => result, Err(error) => { let _ = stop_runtime(plan, &runtime); @@ -1089,6 +1127,51 @@ pub fn run_plan(plan: &RunPlan, request: RunRequest) -> Result { Ok(result) } +/// Keep local one-shot output unavailable until cleanup is finalized. +#[doc(hidden)] +pub fn begin_run_result( + plan: &RunPlan, + runtime: &RuntimeHandle, + request: &RunRequest, +) -> Result<()> { + validate_runtime_handle(plan, runtime)?; + if request.request_id.trim().is_empty() || request.request_id.contains('\0') { + return Err(FabricError::InvalidConfig { + field: "request_id".into(), + reason: "must not be blank or contain NUL".into(), + }); + } + if runtime.environment.provider == "local" { + let root = artifact_manifest(plan)? + .root + .unwrap_or_else(default_artifact_root); + results::begin_finalization( + &root, + &runtime.agent_name, + &runtime.runtime_id, + &request.request_id, + )?; + } + Ok(()) +} + +/// Default result and host storage; Unix users have separate namespaces. +#[doc(hidden)] +pub fn default_artifact_root() -> PathBuf { + #[cfg(unix)] + { + // SAFETY: geteuid has no preconditions or side effects. + default_artifact_root_for_user(unsafe { libc::geteuid() }) + } + #[cfg(not(unix))] + std::env::temp_dir().join("nemo-fabric") +} + +#[cfg(unix)] +fn default_artifact_root_for_user(uid: u32) -> PathBuf { + std::env::temp_dir().join(format!("nemo-fabric-{uid}")) +} + /// Persist the final one-shot result after Python lifecycle cleanup. #[doc(hidden)] pub fn save_run_result(plan: &RunPlan, runtime: &RuntimeHandle, result: &RunResult) -> Result<()> { @@ -1110,18 +1193,8 @@ pub fn save_run_result(plan: &RunPlan, runtime: &RuntimeHandle, result: &RunResu } let root = artifact_manifest(plan)? .root - .unwrap_or_else(|| std::env::temp_dir().join("nemo-fabric")); - let saved = results::invocation_result( - &root, - &result.agent_name, - &result.runtime_id, - &result.request_id, - )? - .ok_or_else(invalid)?; - if saved.invocation_id != result.invocation_id { - return Err(invalid()); - } - results::save(&root, result) + .unwrap_or_else(default_artifact_root); + results::finalize(&root, result) } /// Prepare an execution environment for a run plan. @@ -2842,14 +2915,11 @@ impl LocalHostInvocation { fn run_local_host_invocation_with_timeout( plan: &RunPlan, runtime: &RuntimeHandle, - mut request: RunRequest, + request: RunRequest, invocation_kind: LocalHostInvocation, invoke_timeout: Duration, ) -> Result { let operation = invocation_kind.operation(); - if request.request_id.is_empty() { - request.request_id = new_id("request"); - } if request.request_id.trim().is_empty() || request.request_id.contains('\0') { return Err(FabricError::InvalidConfig { field: "request_id".into(), @@ -2865,7 +2935,7 @@ fn run_local_host_invocation_with_timeout( }; let root = artifact_manifest(plan)? .root - .unwrap_or_else(|| std::env::temp_dir().join("nemo-fabric")); + .unwrap_or_else(default_artifact_root); let mut reservation = match results::reserve( &root, runtime, @@ -3649,10 +3719,7 @@ fn local_services() } pub(crate) fn local_host_directory(id: &str) -> PathBuf { - std::env::temp_dir() - .join("nemo-fabric") - .join("hosts") - .join(id) + default_artifact_root().join("hosts").join(id) } fn spawn_local_host( @@ -5176,10 +5243,7 @@ fn prepare_fabric_home( runtime: &RuntimeHandle, invocation: &InvocationHandle, ) -> Result { - let root = manifest - .root - .clone() - .unwrap_or_else(|| std::env::temp_dir().join("nemo-fabric")); + let root = manifest.root.clone().unwrap_or_else(default_artifact_root); results::invocation_directory(&root, &runtime.runtime_id, &invocation.invocation_id) } @@ -6771,7 +6835,7 @@ else: fn invalid_request_ids_are_rejected_before_invocation() { let (root, plan) = local_host_plan("success"); let runtime = start_runtime(&plan).unwrap(); - for id in [" ", "\t\n", "request\0id"] { + for id in ["", " ", "\t\n", "request\0id"] { let mut request = RunRequest::text("hello"); request.request_id = id.into(); assert!(matches!(invoke_runtime(&plan, &runtime, request), @@ -6787,6 +6851,113 @@ else: fs::remove_dir_all(root).unwrap(); } + #[test] + fn default_request_identity_is_known_before_invocation() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let request = RunRequest::default(); + let request_id = request.request_id.clone(); + let result = invoke_runtime(&plan, &runtime, request).unwrap(); + stop_runtime(&plan, &runtime).unwrap(); + assert!(!request_id.is_empty()); + assert_eq!(result.request_id, request_id); + assert_eq!( + invocation_result( + &root.join("artifacts"), + &result.agent_name, + &runtime.runtime_id, + &request_id + ) + .unwrap(), + Some(result) + ); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn one_shot_result_is_unavailable_until_cleanup_is_finalized() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let request = RunRequest::text("hello"); + begin_run_result(&plan, &runtime, &request).unwrap(); + let mut result = invoke_runtime(&plan, &runtime, request.clone()).unwrap(); + assert!( + invocation_result( + &root.join("artifacts"), + &result.agent_name, + &runtime.runtime_id, + &request.request_id + ) + .unwrap() + .is_none() + ); + assert!( + matches!(invoke_runtime(&plan, &runtime, request), Err(FabricError::AdapterLifecycleOperation {code, ..}) if code == "invocation_pending") + ); + result.events.extend(stop_runtime(&plan, &runtime).unwrap()); + save_run_result(&plan, &runtime, &result).unwrap(); + assert_eq!( + invocation_result( + &root.join("artifacts"), + &result.agent_name, + &runtime.runtime_id, + &result.request_id + ) + .unwrap(), + Some(result) + ); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn failed_final_save_cannot_replay_a_pre_cleanup_success() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let request = RunRequest::text("hello"); + begin_run_result(&plan, &runtime, &request).unwrap(); + let mut result = invoke_runtime(&plan, &runtime, request.clone()).unwrap(); + stop_runtime(&plan, &runtime).unwrap(); + result.status = RunStatus::Failed; + result.output = Value::String("x".repeat(4 * 1024 * 1024)); + assert!(save_run_result(&plan, &runtime, &result).is_err()); + assert!( + invocation_result( + &root.join("artifacts"), + &result.agent_name, + &runtime.runtime_id, + &result.request_id + ) + .unwrap() + .is_none() + ); + assert!( + matches!(invoke_runtime(&plan, &runtime, request), Err(FabricError::AdapterLifecycleOperation {code, ..}) if code == "invocation_pending") + ); + fs::remove_dir_all(root).unwrap(); + } + + #[cfg(unix)] + #[test] + fn default_artifact_storage_is_separate_for_each_unix_user() { + let first = default_artifact_root_for_user(1001); + let second = default_artifact_root_for_user(1002); + assert_ne!(first, second); + assert_eq!(first.parent(), second.parent()); + assert_ne!(first, std::env::temp_dir().join("nemo-fabric")); + } + + #[test] + fn result_lookup_preserves_rejection_codes_without_uncertain_effects() { + for (effects, expected) in [("none", "wrong_agent"), ("unknown", "invalid_response")] { + let response = serde_json::from_value(serde_json::json!({ + "status": "rejected", "error": {"code": "wrong_agent", "stage": "result", "effects": effects} + })).unwrap(); + assert!( + matches!(result_lookup_response(response), Err(FabricError::EnvironmentProviderOperation {operation, code, ..}) if operation == "invocation_result" && code == expected) + ); + } + } + #[test] fn one_shot_lookup_returns_the_final_result_after_stop() { for mode in ["success", "stop_failure"] { diff --git a/crates/fabric-core/src/runtime/results.rs b/crates/fabric-core/src/runtime/results.rs index 551c95013..c8680a758 100644 --- a/crates/fabric-core/src/runtime/results.rs +++ b/crates/fabric-core/src/runtime/results.rs @@ -28,6 +28,14 @@ struct Reservation { streaming: Option, } +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Finalization { + agent_name: String, + runtime_id: String, + request_id: String, +} + pub(super) enum RequestReservation { Cached(Box), Pending(PendingReservation), @@ -213,7 +221,72 @@ fn index_name(request_id: &str) -> String { format!("{:x}.json", Sha256::digest(request_id.as_bytes())) } +fn finalization_path(requests: &Path, request_id: &str) -> PathBuf { + requests.join(format!("{}.finalizing", index_name(request_id))) +} + +pub(super) fn begin_finalization( + root: &Path, + agent: &str, + runtime_id: &str, + request_id: &str, +) -> Result<()> { + let runtime = runtime_directory(root, runtime_id, true)?; + let requests = directory(&runtime, "requests", true, runtime_id)?; + let path = finalization_path(&requests, request_id); + let marker = Finalization { + agent_name: agent.into(), + runtime_id: runtime_id.into(), + request_id: request_id.into(), + }; + let bytes = serde_json::to_vec(&marker).map_err(crate::FabricError::SerializeJson)?; + if bytes.len() as u64 > MAX_RESULT_BYTES { + return Err(failure(runtime_id, "result_storage_failed")); + } + match options().write(true).create_new(true).open(&path) { + Ok(mut file) => { + if file + .write_all(&bytes) + .and_then(|_| file.sync_all()) + .is_err() + { + let _ = fs::remove_file(path); + return Err(failure(runtime_id, "result_storage_failed")); + } + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + let existing: Finalization = read(&path, runtime_id)? + .ok_or_else(|| failure(runtime_id, "result_storage_failed"))?; + if existing.agent_name != agent + || existing.runtime_id != runtime_id + || existing.request_id != request_id + { + return Err(failure(runtime_id, "invalid_result_target")); + } + } + Err(_) => return Err(failure(runtime_id, "result_storage_failed")), + } + Ok(()) +} + fn completed(runtime: &Path, record: &Reservation) -> Result> { + if let Some(marker) = read::( + &finalization_path(&runtime.join("requests"), &record.request_id), + &record.runtime_id, + )? { + if marker.agent_name != record.agent_name + || marker.runtime_id != record.runtime_id + || marker.request_id != record.request_id + { + return Err(failure(&record.runtime_id, "invalid_result_target")); + } + // Invocation output alone does not establish one-shot cleanup finality. + return Ok(None); + } + completed_result(runtime, record) +} + +fn completed_result(runtime: &Path, record: &Reservation) -> Result> { if !leaf(&record.invocation_id) { return Err(failure(&record.runtime_id, "invalid_result_target")); } @@ -311,6 +384,35 @@ pub(super) fn save(root: &Path, result: &RunResult) -> Result<()> { Ok(()) } +pub(super) fn finalize(root: &Path, result: &RunResult) -> Result<()> { + let runtime = runtime_directory(root, &result.runtime_id, false)?; + let requests = directory(&runtime, "requests", false, &result.runtime_id)?; + let record: Reservation = read( + &requests.join(index_name(&result.request_id)), + &result.runtime_id, + )? + .ok_or_else(|| failure(&result.runtime_id, "invalid_result_target"))?; + if record.agent_name != result.agent_name + || record.runtime_id != result.runtime_id + || record.request_id != result.request_id + || record.invocation_id != result.invocation_id + { + return Err(failure(&result.runtime_id, "invalid_result_target")); + } + completed_result(&runtime, &record)? + .ok_or_else(|| failure(&result.runtime_id, "invalid_result_target"))?; + begin_finalization( + root, + &result.agent_name, + &result.runtime_id, + &result.request_id, + )?; + save(root, result)?; + // Publish cleanup finality only after the replacement result is saved. + fs::remove_file(finalization_path(&requests, &result.request_id)) + .map_err(|_| failure(&result.runtime_id, "result_storage_failed")) +} + /// Read a saved result. Absence does not establish whether the request ran. #[doc(hidden)] pub fn invocation_result( @@ -321,7 +423,7 @@ pub fn invocation_result( ) -> Result> { if [request_id, agent] .iter() - .any(|value| value.trim().is_empty() || value.contains('')) + .any(|value| value.trim().is_empty() || value.contains('\0')) { return Err(failure(runtime_id, "invalid_result_target")); } diff --git a/crates/fabric-python/src/lib.rs b/crates/fabric-python/src/lib.rs index 34790749d..8c6d92578 100644 --- a/crates/fabric-python/src/lib.rs +++ b/crates/fabric-python/src/lib.rs @@ -295,7 +295,7 @@ fn invocation_result( ) -> PyResult { let root = artifacts .map(PathBuf::from) - .unwrap_or_else(|| std::env::temp_dir().join("nemo-fabric")); + .unwrap_or_else(nemo_fabric_core::runtime::default_artifact_root); let result = py .detach(|| { nemo_fabric_core::runtime::invocation_result(&root, &agent, &runtime_id, &request_id) @@ -308,6 +308,23 @@ fn invocation_result( to_json(&result) } +/// Mark a local one-shot result as awaiting lifecycle finalization. +#[pyfunction] +fn _begin_run_result( + py: Python<'_>, + plan_json: String, + runtime_json: String, + request_json: String, +) -> PyResult<()> { + let plan = parse_run_plan(plan_json)?; + let runtime = parse_runtime_handle(runtime_json)?; + let request: RunRequest = serde_json::from_str(&request_json).map_err(|_| { + pyo3::exceptions::PyValueError::new_err("The invocation request is invalid.") + })?; + py.detach(|| nemo_fabric_core::runtime::begin_run_result(&plan, &runtime, &request)) + .map_err(to_py_error) +} + /// Persist a result after the Python one-shot lifecycle finishes cleanup. #[pyfunction] fn _save_run_result( @@ -343,10 +360,18 @@ fn invocation_result_environment( &request_id, ) }) - .map_err(|_| { - PyRuntimeError::new_err( + .map_err(|error| { + let failure = PyRuntimeError::new_err( "The invocation result is unavailable; do not replay uncertain work.", - ) + ); + if let FabricError::EnvironmentProviderOperation { code, .. } = error { + match Python::attach(|py| failure.value(py).setattr("code", code)) { + Ok(()) => failure, + Err(error) => error, + } + } else { + failure + } })?; to_json(&result) } @@ -460,6 +485,7 @@ fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(runtime_status_environment, m)?)?; m.add_function(wrap_pyfunction!(invocation_result, m)?)?; m.add_function(wrap_pyfunction!(_save_run_result, m)?)?; + m.add_function(wrap_pyfunction!(_begin_run_result, m)?)?; m.add_function(wrap_pyfunction!(invocation_result_environment, m)?)?; m.add_function(wrap_pyfunction!(check_service, m)?)?; m.add("ServiceInUseError", m.py().get_type::())?; diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx index e117661b6..8d582a921 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx @@ -21,7 +21,7 @@ Explicitly accept recovery uncertainty before submitting this new turn. ### `request_id: String` -Request id. +Nonblank request identifier. `Default` and `text` generate an identifier before dispatch. ### `relay_session_root: Option` @@ -79,7 +79,7 @@ Build a text request. #### `default` -
default() -> RunRequest"}} />
+
default() -> Self"}} />
### `impl<'de> Deserialize<'de> for RunRequest` diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index bf3cca3bb..b05746134 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1713,6 +1713,8 @@ The [software evidence example](https://github.com/NVIDIA/NeMo-Fabric/tree/main/ NeMo Fabric saves each completed `RunResult` as `/.fabric///run-result.json` before returning the result. On Unix, result and request-record files are readable only by their owner. The artifact root comes from `runtime.artifacts` or the environment; without either setting, NeMo Fabric uses its temporary artifact directory. Files remain until their storage is removed; no automatic retention or eviction is added. +On Unix, default temporary storage uses `nemo-fabric-` under the system temporary directory, so users have separate roots. Native writes and SDK lookup select the same root. Local one-shot results remain unavailable until the shutdown outcome is saved. A failed final save or interruption keeps the request reserved and prevents lookup from returning an earlier success. Rust `RunRequest::default()` generates a request ID before dispatch; explicitly supplied blank IDs are rejected. + Use a trusted artifact directory. On Unix, result storage rejects unsafe directory ownership, writable ancestry, and record-file permissions. Serialized results must fit within 4 MiB. Request identifiers must not be blank or contain NUL. Buffered retries can return the saved result; a request ID cannot switch between buffered and streaming invocations. A completed streaming request returns `stream_replay_unsupported` on replay; retrieve its terminal result with `invocation_result()` instead. Choose a stable `RunRequest.request_id` before submitting work. After a command timeout, `invocation_timeout` reports unknown effects and returns the runtime and request identifiers. The invocation can continue. Retrieve the completed result with: diff --git a/schemas/sdk/run-request.schema.json b/schemas/sdk/run-request.schema.json index 6f5235228..b3c85a026 100644 --- a/schemas/sdk/run-request.schema.json +++ b/schemas/sdk/run-request.schema.json @@ -26,7 +26,9 @@ ] }, "request_id": { - "description": "Request id.", + "description": "Nonblank request identifier. `Default` and `text` generate an identifier before dispatch.", + "minLength": 1, + "pattern": "^[^\\x00]*[^\\s\\x00][^\\x00]*$", "type": "string" } }, diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi index f2d7a8318..c0b3ce6ea 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi @@ -61,3 +61,4 @@ def invocation_result_environment( environment_json: str, agent: str, runtime_id: str, request_id: str ) -> str: ... def _save_run_result(plan_json: str, runtime_json: str, result_json: str) -> None: ... +def _begin_run_result(plan_json: str, runtime_json: str, request_json: str) -> None: ... diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py index 719aae113..c54af430b 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py @@ -11,6 +11,7 @@ import logging import math import os +import re import time from threading import Lock from collections import OrderedDict @@ -188,11 +189,17 @@ async def invocation_result( ): raise ValueError() return result - except Exception: + except Exception as error: + code = getattr(error, "code", None) + if ( + not isinstance(code, str) + or re.fullmatch(r"[a-z][a-z0-9_]{0,63}", code) is None + ): + code = "result_unavailable" raise FabricRuntimeError( "The invocation result could not be retrieved; do not replay uncertain work.", stage="result", - code="result_unavailable", + code=code, ) from None async def runtime_status( diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index 8421f8134..127f80d18 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -1061,6 +1061,10 @@ def run() -> dict[str, Any]: invoke_error: FabricError | None = None try: try: + if runtime.get("environment", {}).get("provider") == "local": + native._begin_run_result( + plan_json, runtime_json, json.dumps(dict(request)) + ) result = json.loads( native.invoke_runtime( plan_json, runtime_json, json.dumps(dict(request)) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md index dcdd71bee..3b0dec16e 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md @@ -132,3 +132,5 @@ fabric-agent-ctl result --agent researcher --runtime-id RUNTIME_ID --request-id A null result means no completed result is available; it does not prove the request is still running or safe to repeat. A retry of the same request identifier and input returns the saved result, or reports that the original invocation is pending. Reusing that identifier with different input is rejected. Failed or interrupted reservations are not automatically replayed. For OpenShell, pass `environment=environment` instead of `artifacts` to query the original server through the provider. The CLI requires a retained ownership record on that server. A later local SDK client can read retained files using the explicit artifact root even after runtime stop. Neither path reconstructs a missing result. Remote results retain task-side artifact paths. + +The Unix default result root is specific to the effective user. Local one-shot lookup remains unavailable until the shutdown outcome is saved; interrupted or failed finalization does not expose an earlier success. diff --git a/tests/python/test_native_sdk.py b/tests/python/test_native_sdk.py index 32084bc23..5a4c1055a 100644 --- a/tests/python/test_native_sdk.py +++ b/tests/python/test_native_sdk.py @@ -675,3 +675,61 @@ async def test_native_failed_start_exposes_a_private_cleanup_retry( directory.mkdir(mode=0o700) await recovery.stop() assert recovery.status is RuntimeStatus.STOPPED + + +async def test_failed_final_save_never_exposes_pre_cleanup_success( + hermes_shim_agent_dir, monkeypatch +): + os.environ["ADAPTER_PYTHON"] = sys.executable + original_stop = native.stop_runtime + + def stop(*args): + original_stop(*args) + raise RuntimeError("test-only stop failure") + + def save(*args): + raise RuntimeError("test-only final save failure") + + monkeypatch.setattr(native, "stop_runtime", stop) + monkeypatch.setattr(native, "_save_run_result", save) + client = Fabric() + with pytest.raises(FabricRuntimeError) as caught: + await client.run( + hermes_shim_config(), base_dir=hermes_shim_agent_dir, input="hello" + ) + assert caught.value.code == "result_storage_failed" + partial = caught.value.details["runtime_evidence"]["partial_result"] + assert partial["status"] == "failed" + saved = await client.invocation_result( + artifacts=hermes_shim_agent_dir / "artifacts", + agent_name=partial["agent_name"], + runtime_id=partial["runtime_id"], + request_id=partial["request_id"], + ) + assert saved is None + + +async def test_default_result_root_matches_native_writes_and_lookup( + hermes_shim_agent_dir, +): + import shutil + + os.environ["ADAPTER_PYTHON"] = sys.executable + config = hermes_shim_config() + config.runtime.artifacts = None + config.environment.artifacts = None + client = Fabric() + result = await client.run( + config, base_dir=hermes_shim_agent_dir, input="default storage" + ) + try: + saved = await client.invocation_result( + agent_name=result.agent_name, + runtime_id=result.runtime_id, + request_id=result.request_id, + ) + assert saved == result + finally: + directory = Path(result.metadata["fabric_home"]).parent + assert directory.name == result.runtime_id + shutil.rmtree(directory) diff --git a/tests/python/test_sdk_contract.py b/tests/python/test_sdk_contract.py index aa8734da1..d471ae479 100644 --- a/tests/python/test_sdk_contract.py +++ b/tests/python/test_sdk_contract.py @@ -1839,6 +1839,13 @@ def stop_runtime(self, plan_json: str, runtime_json: str) -> str: self.stopped += 1 return json.dumps([]) + def _begin_run_result( + self, plan_json: str, runtime_json: str, request_json: str + ) -> None: + assert json.loads(plan_json)["agent_name"] == "demo" + assert json.loads(runtime_json)["runtime_id"] == "runtime-1" + assert json.loads(request_json)["request_id"] + def _save_run_result( self, plan_json: str, runtime_json: str, result_json: str ) -> None: diff --git a/tests/python/test_sdk_server_checks.py b/tests/python/test_sdk_server_checks.py index 5c765e81e..84289023d 100644 --- a/tests/python/test_sdk_server_checks.py +++ b/tests/python/test_sdk_server_checks.py @@ -556,3 +556,33 @@ async def test_result_lookup_rejects_untyped_environment_before_native_dispatch( ) native.invocation_result.assert_not_called() native.invocation_result_environment.assert_not_called() + + +@pytest.mark.parametrize( + "code, expected", + [ + ("wrong_agent", "wrong_agent"), + ("invalid_response", "invalid_response"), + ("private value with spaces", "result_unavailable"), + ], +) +async def test_result_lookup_preserves_safe_rejection_codes( + server_environment, monkeypatch, code, expected +): + from nemo_fabric import FabricRuntimeError + + native = MagicMock() + native.invocation_result_environment.side_effect = FabricRuntimeError( + "private native message", code=code + ) + monkeypatch.setattr("nemo_fabric.client._native", native) + with pytest.raises(FabricRuntimeError) as caught: + await Fabric().invocation_result( + environment=server_environment, + agent_name="demo", + runtime_id="runtime-1", + request_id="request-1", + ) + assert caught.value.code == expected + assert "private native message" not in str(caught.value) + assert "private value with spaces" not in repr(caught.value.details) From 7cf99c8d193f45da1d0ba19e9f8c3d292846835d Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Thu, 12 Nov 2026 21:21:11 -0700 Subject: [PATCH 06/10] fix: retain stopped result context and protect default storage Signed-off-by: Sara Tadayon --- crates/fabric-core/src/check.rs | 9 +- crates/fabric-core/src/runtime.rs | 71 ++++++++++----- crates/fabric-core/src/runtime/results.rs | 11 +++ crates/fabric-python/src/lib.rs | 3 +- docs/sdk/python.mdx | 4 +- .../src/nemo_fabric/client.py | 43 +++++---- .../src/nemo_fabric/runtime_server/README.md | 2 +- tests/python/test_native_sdk.py | 88 +++++++++++++++++++ tests/python/test_sdk_server_checks.py | 6 +- 9 files changed, 189 insertions(+), 48 deletions(-) diff --git a/crates/fabric-core/src/check.rs b/crates/fabric-core/src/check.rs index da1a572b3..dd462a6b6 100644 --- a/crates/fabric-core/src/check.rs +++ b/crates/fabric-core/src/check.rs @@ -1117,7 +1117,9 @@ fn probe_target( let budget = Duration::try_from_secs_f64(timeout).unwrap_or(Duration::MAX); let remaining = || budget.saturating_sub(started.elapsed()); let result = (|| -> std::io::Result { - let path = crate::runtime::local_host_directory(runtime_id).join("health.endpoint.json"); + let path = crate::runtime::local_host_directory(runtime_id) + .map_err(|_| std::io::Error::other("default host storage unavailable"))? + .join("health.endpoint.json"); let mut options = OpenOptions::new(); options.read(true); #[cfg(unix)] @@ -1283,7 +1285,7 @@ mod tests { fn endpoint(runtime: &RuntimeHandle) -> TcpListener { let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); - let directory = crate::runtime::local_host_directory(&runtime.runtime_id); + let directory = crate::runtime::local_host_directory(&runtime.runtime_id).unwrap(); std::fs::create_dir_all(&directory).unwrap(); let endpoint = directory.join("health.endpoint.json"); std::fs::write( @@ -1379,7 +1381,8 @@ mod tests { fn cleanup(runtime: &RuntimeHandle) { invalidate_runtime(runtime); - std::fs::remove_dir_all(crate::runtime::local_host_directory(&runtime.runtime_id)).unwrap(); + std::fs::remove_dir_all(crate::runtime::local_host_directory(&runtime.runtime_id).unwrap()) + .unwrap(); } fn gateway_service(runtime: &RuntimeHandle) -> crate::ServiceHandle { diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index b5e3adc29..19e76ed7b 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -1144,7 +1144,7 @@ pub fn begin_run_result( if runtime.environment.provider == "local" { let root = artifact_manifest(plan)? .root - .unwrap_or_else(default_artifact_root); + .map_or_else(default_artifact_root, Ok)?; results::begin_finalization( &root, &runtime.agent_name, @@ -1157,19 +1157,34 @@ pub fn begin_run_result( /// Default result and host storage; Unix users have separate namespaces. #[doc(hidden)] -pub fn default_artifact_root() -> PathBuf { +pub fn default_artifact_root() -> Result { #[cfg(unix)] { + let home = std::env::home_dir().ok_or_else(default_storage_unavailable)?; // SAFETY: geteuid has no preconditions or side effects. - default_artifact_root_for_user(unsafe { libc::geteuid() }) + default_artifact_root_for_user(&home, unsafe { libc::geteuid() }) } #[cfg(not(unix))] - std::env::temp_dir().join("nemo-fabric") + Ok(std::env::temp_dir().join("nemo-fabric")) } #[cfg(unix)] -fn default_artifact_root_for_user(uid: u32) -> PathBuf { - std::env::temp_dir().join(format!("nemo-fabric-{uid}")) +fn default_storage_unavailable() -> FabricError { + FabricError::InvalidConfig { + field: "local_storage".into(), + reason: "Default local storage requires a trusted user home directory.".into(), + } +} + +#[cfg(unix)] +fn default_artifact_root_for_user(home: &Path, uid: u32) -> Result { + use std::os::unix::fs::MetadataExt; + let home = std::fs::canonicalize(home).map_err(|_| default_storage_unavailable())?; + let metadata = std::fs::metadata(&home).map_err(|_| default_storage_unavailable())?; + if !metadata.is_dir() || metadata.uid() != uid || metadata.mode() & 0o022 != 0 { + return Err(default_storage_unavailable()); + } + Ok(home.join(".cache").join(format!("nemo-fabric-{uid}"))) } /// Persist the final one-shot result after Python lifecycle cleanup. @@ -1193,7 +1208,7 @@ pub fn save_run_result(plan: &RunPlan, runtime: &RuntimeHandle, result: &RunResu } let root = artifact_manifest(plan)? .root - .unwrap_or_else(default_artifact_root); + .map_or_else(default_artifact_root, Ok)?; results::finalize(&root, result) } @@ -1425,7 +1440,7 @@ fn start_service( let mut artifacts = artifact_manifest(plan)?; let fabric_home = prepare_fabric_home(&artifacts, &runtime, &invocation)?; let relay_config = prepare_relay_runtime_config(plan, &runtime, &fabric_home, &mut artifacts)?; - let connection_path = local_host_directory(&service_id).join("service-connection.json"); + let connection_path = local_host_directory(&service_id)?.join("service-connection.json"); let service_context = match reference { Some(reference) => AdapterServiceContext::Attach { connection_path: connection_path.clone(), @@ -2935,7 +2950,7 @@ fn run_local_host_invocation_with_timeout( }; let root = artifact_manifest(plan)? .root - .unwrap_or_else(default_artifact_root); + .map_or_else(default_artifact_root, Ok)?; let mut reservation = match results::reserve( &root, runtime, @@ -3718,8 +3733,8 @@ fn local_services() .unwrap_or_else(|error| error.into_inner()) } -pub(crate) fn local_host_directory(id: &str) -> PathBuf { - default_artifact_root().join("hosts").join(id) +pub(crate) fn local_host_directory(id: &str) -> Result { + Ok(default_artifact_root()?.join("hosts").join(id)) } fn spawn_local_host( @@ -3742,11 +3757,7 @@ fn spawn_local_host( }; // Lock acquisition creates retained storage, allowing its physical path to resolve. let scope = cleanup_scope(plan, state_dir.as_deref(), runtime.service_id.as_deref()); - let runtime_dir = local_host_directory(&runtime.runtime_id); - std::fs::create_dir_all(&runtime_dir).map_err(|source| FabricError::Write { - path: runtime_dir.clone(), - source, - })?; + let runtime_dir = results::host_directory(&default_artifact_root()?, &runtime.runtime_id)?; let stderr_path = runtime_dir.join("host.stderr.log"); let stderr = File::create(&stderr_path).map_err(|source| FabricError::Write { path: stderr_path.clone(), @@ -5243,7 +5254,10 @@ fn prepare_fabric_home( runtime: &RuntimeHandle, invocation: &InvocationHandle, ) -> Result { - let root = manifest.root.clone().unwrap_or_else(default_artifact_root); + let root = manifest + .root + .clone() + .map_or_else(default_artifact_root, Ok)?; results::invocation_directory(&root, &runtime.runtime_id, &invocation.invocation_id) } @@ -6938,12 +6952,19 @@ else: #[cfg(unix)] #[test] - fn default_artifact_storage_is_separate_for_each_unix_user() { - let first = default_artifact_root_for_user(1001); - let second = default_artifact_root_for_user(1002); - assert_ne!(first, second); - assert_eq!(first.parent(), second.parent()); - assert_ne!(first, std::env::temp_dir().join("nemo-fabric")); + fn default_artifact_storage_requires_a_protected_user_home() { + use std::os::unix::fs::{DirBuilderExt, PermissionsExt}; + let home = std::env::temp_dir().join(new_id("fabric-private-home-test")); + fs::DirBuilder::new().mode(0o700).create(&home).unwrap(); + // SAFETY: geteuid has no preconditions or side effects. + let uid = unsafe { libc::geteuid() }; + let root = default_artifact_root_for_user(&home, uid).unwrap(); + assert!(root.starts_with(fs::canonicalize(&home).unwrap())); + assert!(default_artifact_root_for_user(&home, uid.wrapping_add(1)).is_err()); + fs::set_permissions(&home, fs::Permissions::from_mode(0o777)).unwrap(); + let unsafe_home = default_artifact_root_for_user(&home, uid); + fs::remove_dir_all(home).unwrap(); + assert!(unsafe_home.is_err()); } #[test] @@ -9955,7 +9976,9 @@ else: .unwrap() .clone(); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let endpoint = local_host_directory(&runtime.runtime_id).join("health.endpoint.json"); + let endpoint = local_host_directory(&runtime.runtime_id) + .unwrap() + .join("health.endpoint.json"); fs::write( &endpoint, serde_json::to_vec(&serde_json::json!({ diff --git a/crates/fabric-core/src/runtime/results.rs b/crates/fabric-core/src/runtime/results.rs index c8680a758..d7adcd9d8 100644 --- a/crates/fabric-core/src/runtime/results.rs +++ b/crates/fabric-core/src/runtime/results.rs @@ -166,6 +166,17 @@ fn runtime_directory(root: &Path, runtime_id: &str, create: bool) -> Result Result { + if !leaf(runtime_id) { + return Err(failure(runtime_id, "invalid_result_target")); + } + create_artifact_root(root).map_err(|_| failure(runtime_id, "result_storage_failed"))?; + let root = fs::canonicalize(root).map_err(|_| failure(runtime_id, "result_storage_failed"))?; + trusted_ancestry(&root, runtime_id)?; + let hosts = directory(&root, "hosts", true, runtime_id)?; + directory(&hosts, runtime_id, true, runtime_id) +} + pub(super) fn invocation_directory( root: &Path, runtime_id: &str, diff --git a/crates/fabric-python/src/lib.rs b/crates/fabric-python/src/lib.rs index 8c6d92578..022aa2d9e 100644 --- a/crates/fabric-python/src/lib.rs +++ b/crates/fabric-python/src/lib.rs @@ -295,7 +295,8 @@ fn invocation_result( ) -> PyResult { let root = artifacts .map(PathBuf::from) - .unwrap_or_else(nemo_fabric_core::runtime::default_artifact_root); + .map_or_else(nemo_fabric_core::runtime::default_artifact_root, Ok) + .map_err(to_py_error)?; let result = py .detach(|| { nemo_fabric_core::runtime::invocation_result(&root, &agent, &runtime_id, &request_id) diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index b05746134..b409bb70a 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1711,9 +1711,9 @@ The [software evidence example](https://github.com/NVIDIA/NeMo-Fabric/tree/main/ ### Recover a buffered invocation result -NeMo Fabric saves each completed `RunResult` as `/.fabric///run-result.json` before returning the result. On Unix, result and request-record files are readable only by their owner. The artifact root comes from `runtime.artifacts` or the environment; without either setting, NeMo Fabric uses its temporary artifact directory. Files remain until their storage is removed; no automatic retention or eviction is added. +NeMo Fabric saves each completed `RunResult` as `/.fabric///run-result.json` before returning the result. On Unix, result and request-record files are readable only by their owner. The artifact root comes from `runtime.artifacts` or the environment; without either setting, NeMo Fabric uses its default artifact directory. Files remain until their storage is removed; no automatic retention or eviction is added. The owning `Fabric` client retains the artifact location and environment identity in its bounded stopped-runtime records, so that client can look up results after `Runtime.stop()` without repeating the artifact path. After record eviction or from a new client, provide the original explicit artifact root unless the default root applies. -On Unix, default temporary storage uses `nemo-fabric-` under the system temporary directory, so users have separate roots. Native writes and SDK lookup select the same root. Local one-shot results remain unavailable until the shutdown outcome is saved. A failed final save or interruption keeps the request reserved and prevents lookup from returning an earlier success. Rust `RunRequest::default()` generates a request ID before dispatch; explicitly supplied blank IDs are rejected. +On Unix, default storage uses `~/.cache/nemo-fabric-` beneath a home directory owned by that user and not writable by other users. Local host directories use the same protected root. A missing or unsafe home directory produces a storage error. Native writes and SDK lookup select the same root. Local one-shot results remain unavailable until the shutdown outcome is saved. A failed final save or interruption keeps the request reserved and prevents lookup from returning an earlier success. Rust `RunRequest::default()` generates a request ID before dispatch; explicitly supplied blank IDs are rejected. Use a trusted artifact directory. On Unix, result storage rejects unsafe directory ownership, writable ancestry, and record-file permissions. Serialized results must fit within 4 MiB. Request identifiers must not be blank or contain NUL. Buffered retries can return the saved result; a request ID cannot switch between buffered and streaming invocations. A completed streaming request returns `stream_replay_unsupported` on replay; retrieve its terminal result with `invocation_result()` instead. diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py index c54af430b..7f1d9b374 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py @@ -105,7 +105,7 @@ def __init__(self) -> None: self._pending_runtime_starts: set[object] = set() self._runtimes: dict[str, Runtime] = {} self._stopped_status: OrderedDict[ - str, tuple[tuple[str, str], RuntimeStatusReport] + str, tuple[tuple[str, str], RuntimeStatusReport, str | None] ] = OrderedDict() async def invocation_result( @@ -148,28 +148,28 @@ async def invocation_result( "Result lookup supports local and OpenShell environments." ) runtime = self._runtimes.get(runtime_id) - if environment is not None and ( - runtime is None - or ( + record = self._stopped_status.get(runtime_id) + if runtime is not None: + environment_identity = ( runtime.handle.environment["provider"], runtime.handle.environment["environment_id"], ) - != (environment.provider, environment.environment_id) + configured_artifacts = self._runtime_artifacts(runtime) + elif record is not None: + environment_identity, _, configured_artifacts = record + else: + environment_identity, configured_artifacts = None, None + if environment is not None and environment_identity != ( + environment.provider, + environment.environment_id, ): raise FabricStateError( "The selected environment does not own this runtime.", stage="result", code="runtime_unavailable", ) - if artifacts is None and runtime is not None: - mapping = runtime._plan.to_mapping() - configured = mapping["config"].get("runtime", {}).get( - "artifacts" - ) or runtime.handle.environment.get("artifacts") - if configured is not None: - from pathlib import Path - - artifacts = str(Path(mapping["base_dir"]) / configured) + if artifacts is None: + artifacts = configured_artifacts artifact_path = os.fspath(artifacts) if artifacts is not None else None operation = lambda: native.invocation_result( artifact_path, agent_name, runtime_id, request_id @@ -333,7 +333,7 @@ def unique(pairs): stage="status", code="runtime_unavailable", ) - environment_identity, report = record + environment_identity, report, _ = record if report.agent_name != agent_name: raise FabricStateError( "The request does not identify this agent.", @@ -352,6 +352,18 @@ def unique(pairs): ) return report + @staticmethod + def _runtime_artifacts(runtime: Runtime) -> str | None: + mapping = runtime._plan.to_mapping() + configured = mapping["config"].get("runtime", {}).get( + "artifacts" + ) or runtime.handle.environment.get("artifacts") + if configured is None: + return None + from pathlib import Path + + return str(Path(mapping["base_dir"]) / configured) + def _record_stopped_runtime(self, runtime: Runtime) -> None: with self._runtime_records_lock: if ( @@ -371,6 +383,7 @@ def _record_stopped_runtime(self, runtime: Runtime) -> None: "server_state": None, } ), + self._runtime_artifacts(runtime), ) self._stopped_status.move_to_end(runtime.runtime_id) self._runtimes.pop(runtime.runtime_id, None) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md index 3b0dec16e..2c2ea721c 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md @@ -133,4 +133,4 @@ A null result means no completed result is available; it does not prove the requ For OpenShell, pass `environment=environment` instead of `artifacts` to query the original server through the provider. The CLI requires a retained ownership record on that server. A later local SDK client can read retained files using the explicit artifact root even after runtime stop. Neither path reconstructs a missing result. Remote results retain task-side artifact paths. -The Unix default result root is specific to the effective user. Local one-shot lookup remains unavailable until the shutdown outcome is saved; interrupted or failed finalization does not expose an earlier success. +The Unix default result root is `~/.cache/nemo-fabric-` under a user-owned home directory that other users cannot write. Local host directories use the same protected root. Local one-shot lookup remains unavailable until the shutdown outcome is saved; interrupted or failed finalization does not expose an earlier success. diff --git a/tests/python/test_native_sdk.py b/tests/python/test_native_sdk.py index 5a4c1055a..503b60b73 100644 --- a/tests/python/test_native_sdk.py +++ b/tests/python/test_native_sdk.py @@ -733,3 +733,91 @@ async def test_default_result_root_matches_native_writes_and_lookup( directory = Path(result.metadata["fabric_home"]).parent assert directory.name == result.runtime_id shutil.rmtree(directory) + + +@pytest.mark.parametrize("artifact_source", ["runtime", "environment"]) +@pytest.mark.parametrize("select_environment", [False, True]) +async def test_stopped_runtime_result_lookup_retains_storage_and_environment( + hermes_shim_agent_dir, artifact_source, select_environment +): + from nemo_fabric import EnvironmentHandle + + os.environ["ADAPTER_PYTHON"] = sys.executable + config = hermes_shim_config() + config.runtime.artifacts = None + config.environment.artifacts = None + getattr(config, artifact_source).artifacts = "retained-results" + client = Fabric() + runtime = await client.start_runtime(config, base_dir=hermes_shim_agent_dir) + try: + result = await runtime.invoke(input="retain lookup context") + finally: + await runtime.stop() + environment = EnvironmentHandle.from_mapping( + runtime.handle.to_mapping()["environment"] + ) + saved = await client.invocation_result( + agent_name=result.agent_name, + runtime_id=result.runtime_id, + request_id=result.request_id, + environment=environment if select_environment else None, + ) + assert saved == result + assert runtime.runtime_id not in client._runtimes + if select_environment: + wrong = environment.to_mapping() + wrong["environment_id"] = "another-environment" + with pytest.raises(FabricStateError) as caught: + await client.invocation_result( + agent_name=result.agent_name, + runtime_id=result.runtime_id, + request_id=result.request_id, + environment=EnvironmentHandle.from_mapping(wrong), + artifacts=hermes_shim_agent_dir / "retained-results", + ) + assert caught.value.code == "runtime_unavailable" + + +@pytest.mark.skipif(sys.platform == "win32", reason="Unix shared temporary root") +def test_default_storage_ignores_a_precreated_shared_temporary_name( + hermes_shim_agent_dir, tmp_path +): + shared = tmp_path / "shared" + shared.mkdir(mode=0o777) + # Another user can occupy this predictable name before Fabric's first run. + (shared / f"nemo-fabric-{os.geteuid()}").write_text("occupied") + config = hermes_shim_config() + config.runtime.artifacts = None + config.environment.artifacts = None + script = """ +import asyncio, json, os, shutil, sys +from pathlib import Path +from nemo_fabric import Fabric, FabricConfig +async def main(): + client = Fabric() + result = await client.run(FabricConfig.from_mapping(json.loads(sys.argv[1])), base_dir=sys.argv[2], input="private default storage") + directory = Path(result.metadata["fabric_home"]).parent + try: + assert Path(sys.argv[3]) not in directory.parents + saved = await client.invocation_result(agent_name=result.agent_name, runtime_id=result.runtime_id, request_id=result.request_id) + assert saved == result + finally: + shutil.rmtree(directory) +asyncio.run(main()) +""" + environment = dict(os.environ, TMPDIR=str(shared), ADAPTER_PYTHON=sys.executable) + completed = subprocess.run( + [ + sys.executable, + "-c", + script, + config.model_dump_json(exclude_none=True), + str(hermes_shim_agent_dir), + str(shared), + ], + env=environment, + capture_output=True, + text=True, + timeout=30, + ) + assert completed.returncode == 0, completed.stderr diff --git a/tests/python/test_sdk_server_checks.py b/tests/python/test_sdk_server_checks.py index 84289023d..0718cbbec 100644 --- a/tests/python/test_sdk_server_checks.py +++ b/tests/python/test_sdk_server_checks.py @@ -505,8 +505,10 @@ async def test_remote_result_lookup_during_and_after_invocation( ) finally: (tmp_path / "release").touch() - await turn - await host.shutdown() + try: + await turn + finally: + await host.shutdown() async def test_remote_result_errors_do_not_expose_provider_details( From 152b38c3c649ebbf371901765dc2d74b398fda34 Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Fri, 13 Nov 2026 13:17:44 -0700 Subject: [PATCH 07/10] fix: recover dispatched failures and support shared artifact roots Signed-off-by: Sara Tadayon --- crates/fabric-core/src/runtime.rs | 332 +++++++++++++++--- crates/fabric-core/src/runtime/results.rs | 33 +- crates/fabric-python/src/lib.rs | 23 ++ docs/sdk/python.mdx | 6 +- .../src/nemo_fabric/_native.pyi | 3 + .../src/nemo_fabric/runtime.py | 18 + .../src/nemo_fabric/runtime_server/README.md | 6 +- skills/nemo-fabric-integrate/SKILL.md | 2 + tests/python/test_native_sdk.py | 124 +++++++ tests/python/test_runtime.py | 17 + tests/python/test_sdk_contract.py | 7 + 11 files changed, 506 insertions(+), 65 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 19e76ed7b..a8f86ae48 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -226,6 +226,7 @@ pub struct RunRequest { impl Default for RunRequest { fn default() -> Self { Self { + acknowledge_recovery: false, request_id: new_id("request"), relay_session_root: None, input: Value::Null, @@ -1085,12 +1086,14 @@ enum LocalServiceState { /// any completed result. Retry [`stop_runtime`] with that handle and this plan. pub fn run_plan(plan: &RunPlan, request: RunRequest) -> Result { let runtime = start_runtime(plan)?; + let original_request = request.clone(); let invoked = begin_run_result(plan, &runtime, &request) .and_then(|()| invoke_runtime(plan, &runtime, request)); let mut result = match invoked { Ok(result) => result, Err(error) => { - let _ = stop_runtime(plan, &runtime); + let events = stop_runtime(plan, &runtime).unwrap_or_default(); + finish_failed_run_result(plan, &runtime, &original_request, events)?; if local_runtime_is_retained(&runtime.runtime_id) { return Err(FabricError::RuntimeCleanupPending(Box::new( crate::RuntimeCleanupPending::new(runtime, None, error), @@ -1160,7 +1163,7 @@ pub fn begin_run_result( pub fn default_artifact_root() -> Result { #[cfg(unix)] { - let home = std::env::home_dir().ok_or_else(default_storage_unavailable)?; + let home = std::env::home_dir().unwrap_or_default(); // SAFETY: geteuid has no preconditions or side effects. default_artifact_root_for_user(&home, unsafe { libc::geteuid() }) } @@ -1172,19 +1175,52 @@ pub fn default_artifact_root() -> Result { fn default_storage_unavailable() -> FabricError { FabricError::InvalidConfig { field: "local_storage".into(), - reason: "Default local storage requires a trusted user home directory.".into(), + reason: "Default local storage requires a private directory owned by the current user." + .into(), } } #[cfg(unix)] fn default_artifact_root_for_user(home: &Path, uid: u32) -> Result { use std::os::unix::fs::MetadataExt; - let home = std::fs::canonicalize(home).map_err(|_| default_storage_unavailable())?; - let metadata = std::fs::metadata(&home).map_err(|_| default_storage_unavailable())?; - if !metadata.is_dir() || metadata.uid() != uid || metadata.mode() & 0o022 != 0 { + // SAFETY: geteuid has no preconditions or side effects. + if uid != unsafe { libc::geteuid() } { return Err(default_storage_unavailable()); } - Ok(home.join(".cache").join(format!("nemo-fabric-{uid}"))) + if let Ok(home) = std::fs::canonicalize(home) + && let Ok(metadata) = std::fs::metadata(&home) + && metadata.is_dir() + && metadata.uid() == uid + && metadata.mode() & 0o022 == 0 + { + return Ok(home.join(".cache").join(format!("nemo-fabric-{uid}"))); + } + temporary_artifact_root_for_user(&std::env::temp_dir(), uid) +} + +#[cfg(unix)] +fn temporary_artifact_root_for_user(temporary: &Path, uid: u32) -> Result { + use std::os::unix::fs::MetadataExt; + results::trusted_ancestry(temporary, "local-storage") + .map_err(|_| default_storage_unavailable())?; + let root = temporary.join(format!("nemo-fabric-{uid}-private")); + // Atomic creation and lstat reject pre-created symlinks, other owners, + // and writable directories. Never chmod a path supplied by another user. + use std::os::unix::fs::DirBuilderExt; + if let Err(error) = std::fs::DirBuilder::new().mode(0o700).create(&root) + && error.kind() != std::io::ErrorKind::AlreadyExists + { + return Err(default_storage_unavailable()); + } + let metadata = std::fs::symlink_metadata(&root).map_err(|_| default_storage_unavailable())?; + if !metadata.is_dir() + || metadata.file_type().is_symlink() + || metadata.uid() != uid + || metadata.mode() & 0o077 != 0 + { + return Err(default_storage_unavailable()); + } + Ok(root) } /// Persist the final one-shot result after Python lifecycle cleanup. @@ -1212,6 +1248,32 @@ pub fn save_run_result(plan: &RunPlan, runtime: &RuntimeHandle, result: &RunResu results::finalize(&root, result) } +/// Finalize a dispatched one-shot failure after lifecycle cleanup. +#[doc(hidden)] +pub fn finish_failed_run_result( + plan: &RunPlan, + runtime: &RuntimeHandle, + request: &RunRequest, + events: Vec, +) -> Result<()> { + validate_runtime_handle(plan, runtime)?; + if runtime.environment.provider == "local" && uses_local_host(plan) { + let root = artifact_manifest(plan)? + .root + .map_or_else(default_artifact_root, Ok)?; + if let Some(mut result) = results::unfinalized_result( + &root, + &runtime.agent_name, + &runtime.runtime_id, + &request.request_id, + )? { + result.events.extend(events); + save_run_result(plan, runtime, &result)?; + } + } + Ok(()) +} + /// Prepare an execution environment for a run plan. /// /// This block implements local environment preparation. Other providers return @@ -2934,7 +2996,6 @@ fn run_local_host_invocation_with_timeout( invocation_kind: LocalHostInvocation, invoke_timeout: Duration, ) -> Result { - let operation = invocation_kind.operation(); if request.request_id.trim().is_empty() || request.request_id.contains('\0') { return Err(FabricError::InvalidConfig { field: "request_id".into(), @@ -2961,6 +3022,80 @@ fn run_local_host_invocation_with_timeout( results::RequestReservation::Cached(result) => return Ok(*result), results::RequestReservation::Pending(pending) => pending, }; + let outcome = execute_local_host_invocation( + plan, + runtime, + request, + invocation_kind, + invoke_timeout, + invocation.clone(), + &mut reservation, + ) + .and_then(|result| { + results::save(&root, &result)?; + Ok(result) + }); + if let Err(error) = &outcome + && reservation.dispatched() + { + let code = match error { + FabricError::AdapterLifecycleOperation { code, .. } + if matches!( + code.as_str(), + "host_crashed" | "host_timeout" | "interrupted" | "invalid_agent_run_result" + ) => + { + code.as_str() + } + _ => "invocation_failed", + }; + let result = RunResult { + session_ref: None, + agent_name: plan.agent_name.clone(), + harness: harness(plan), + adapter_kind: adapter_kind(plan), + adapter_id: adapter_id(plan), + runtime_id: runtime.runtime_id.clone(), + invocation_id: invocation.invocation_id, + request_id: invocation.request_id, + status: if code == "interrupted" { RunStatus::Cancelled } else { RunStatus::Failed }, + output: Value::Null, + error: Some(ErrorInfo { + stage: ErrorStage::Invoke, + code: code.into(), + message: "The dispatched invocation ended without a valid result; effects may be unknown. Do not replay the work.".into(), + retryable: false, + metadata: BTreeMap::from([("effects".into(), Value::String("unknown".into()))]), + }), + usage: None, + execution: None, + resource_snapshot: None, + system_stats: None, + artifacts: ArtifactManifest { root: Some(root.clone()), ..ArtifactManifest::default() }, + telemetry: vec![], + events: vec![event_with_metadata( + "invocation_end", + "The dispatched invocation ended without a valid result.", + BTreeMap::from([("code".into(), Value::String(code.into()))]), + )], + metadata: BTreeMap::new(), + }; + // Retain the dispatched reservation even if this write fails. + results::save(&root, &result)?; + } + outcome +} + +fn execute_local_host_invocation( + plan: &RunPlan, + runtime: &RuntimeHandle, + request: RunRequest, + invocation_kind: LocalHostInvocation, + invoke_timeout: Duration, + invocation: InvocationHandle, + reservation: &mut results::PendingReservation, +) -> Result { + let operation = invocation_kind.operation(); let host = local_hosts() .get(&runtime.runtime_id) .cloned() @@ -3400,7 +3535,6 @@ fn run_local_host_invocation_with_timeout( events, metadata, }; - results::save(&root, &result)?; Ok(result) } @@ -5861,6 +5995,8 @@ for line in sys.stdin: invocations += 1 if MODE == "invoke_stderr": print(f"diagnostic-{invocations}", file=sys.stderr, flush=True) + if MODE == "invoke_crash": + os._exit(17) if MODE == "invoke_timeout": print("invoke accepted without response", file=sys.stderr, flush=True) continue @@ -6964,7 +7100,34 @@ else: fs::set_permissions(&home, fs::Permissions::from_mode(0o777)).unwrap(); let unsafe_home = default_artifact_root_for_user(&home, uid); fs::remove_dir_all(home).unwrap(); - assert!(unsafe_home.is_err()); + let fallback = unsafe_home.expect("shared home uses private temporary storage"); + assert_eq!( + fallback, + std::env::temp_dir().join(format!("nemo-fabric-{uid}-private")) + ); + assert_eq!( + fs::metadata(&fallback).unwrap().permissions().mode() & 0o777, + 0o700 + ); + } + + #[cfg(unix)] + #[test] + fn temporary_storage_rejects_unsafe_precreated_namespaces() { + use std::os::unix::fs::{PermissionsExt, symlink}; + let temporary = std::env::temp_dir().join(new_id("fabric-fallback-test")); + fs::create_dir(&temporary).unwrap(); + // SAFETY: geteuid has no preconditions or side effects. + let uid = unsafe { libc::geteuid() }; + let root = temporary.join(format!("nemo-fabric-{uid}-private")); + fs::create_dir(&root).unwrap(); + fs::set_permissions(&root, fs::Permissions::from_mode(0o777)).unwrap(); + assert!(temporary_artifact_root_for_user(&temporary, uid).is_err()); + fs::remove_dir(&root).unwrap(); + symlink(&temporary, &root).unwrap(); + assert!(temporary_artifact_root_for_user(&temporary, uid).is_err()); + fs::remove_file(&root).unwrap(); + fs::remove_dir(&temporary).unwrap(); } #[test] @@ -6979,6 +7142,72 @@ else: } } + #[test] + fn dispatched_failures_have_terminal_results_without_reexecution() { + for (mode, expected) in [ + ("invoke_timeout", "host_timeout"), + ("invoke_crash", "host_crashed"), + ("blank_error_code", "invalid_agent_run_result"), + ("invoke_failure", "invocation_failed"), + ] { + let (root, plan) = local_host_plan(mode); + let runtime = start_runtime(&plan).unwrap(); + let request = RunRequest::text("hello"); + let error = run_local_host_adapter_with_timeout( + &plan, + &runtime, + request.clone(), + Duration::from_millis(100), + ) + .unwrap_err(); + assert!(!error.to_string().is_empty()); + let saved = invocation_result( + &root.join("artifacts"), + &runtime.agent_name, + &runtime.runtime_id, + &request.request_id, + ) + .unwrap(); + stop_runtime(&plan, &runtime).unwrap(); + let saved = saved.expect("dispatched failure has a terminal result"); + assert_eq!(saved.status, RunStatus::Failed); + assert!(saved.output.is_null()); + assert_eq!(saved.error.as_ref().unwrap().code, expected); + assert!(!saved.error.as_ref().unwrap().retryable); + assert_eq!(saved.error.as_ref().unwrap().metadata["effects"], "unknown"); + assert_eq!(invoke_runtime(&plan, &runtime, request).unwrap(), saved); + fs::remove_dir_all(root).unwrap(); + } + } + + #[test] + fn one_shot_dispatched_failure_is_finalized_after_cleanup() { + let (root, plan) = local_host_plan("blank_error_code"); + let request = RunRequest::text("hello"); + run_plan(&plan, request.clone()).unwrap_err(); + let runtimes = fs::read_dir(root.join("artifacts/.fabric")).unwrap(); + let runtime = runtimes + .map(|entry| entry.unwrap().file_name().into_string().unwrap()) + .find(|name| name.starts_with("runtime-")) + .unwrap(); + let saved = invocation_result( + &root.join("artifacts"), + &plan.agent_name, + &runtime, + &request.request_id, + ) + .unwrap() + .expect("one-shot failure must be retrievable"); + assert_eq!(saved.status, RunStatus::Failed); + assert!( + saved + .events + .iter() + .any(|event| event.kind == "runtime_stop") + ); + fs::remove_dir_all(root).unwrap(); + } + #[test] fn one_shot_lookup_returns_the_final_result_after_stop() { for mode in ["success", "stop_failure"] { @@ -7093,43 +7322,33 @@ else: #[cfg(unix)] #[test] - fn result_lookup_rejects_a_writable_storage_root() { - use std::os::unix::fs::PermissionsExt; - let (root, plan) = local_host_plan("success"); - let runtime = start_runtime(&plan).unwrap(); - let result = invoke_runtime(&plan, &runtime, RunRequest::text("hello")).unwrap(); - let artifacts = root.join("artifacts"); - fs::set_permissions(&artifacts, fs::Permissions::from_mode(0o777)).unwrap(); - let lookup = invocation_result( - &artifacts, - &result.agent_name, - &result.runtime_id, - &result.request_id, - ); - fs::set_permissions(&artifacts, fs::Permissions::from_mode(0o700)).unwrap(); - stop_runtime(&plan, &runtime).unwrap(); - assert!(lookup.is_err()); - fs::remove_dir_all(root).unwrap(); - } - - #[cfg(unix)] - #[test] - fn result_lookup_rejects_a_writable_storage_ancestor() { - use std::os::unix::fs::PermissionsExt; - let (root, plan) = local_host_plan("success"); - let runtime = start_runtime(&plan).unwrap(); - let result = invoke_runtime(&plan, &runtime, RunRequest::text("hello")).unwrap(); - fs::set_permissions(&root, fs::Permissions::from_mode(0o777)).unwrap(); - let lookup = invocation_result( - &root.join("artifacts"), - &result.agent_name, - &result.runtime_id, - &result.request_id, - ); - fs::set_permissions(&root, fs::Permissions::from_mode(0o700)).unwrap(); - stop_runtime(&plan, &runtime).unwrap(); - assert!(lookup.is_err()); - fs::remove_dir_all(root).unwrap(); + fn explicit_shared_artifact_roots_allow_private_results() { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + for mode in [0o775, 0o777] { + let (root, plan) = local_host_plan("success"); + fs::set_permissions(&root, fs::Permissions::from_mode(mode)).unwrap(); + let artifacts = root.join("artifacts"); + fs::create_dir_all(&artifacts).unwrap(); + fs::set_permissions(&artifacts, fs::Permissions::from_mode(mode)).unwrap(); + let runtime = start_runtime(&plan).unwrap(); + let result = invoke_runtime(&plan, &runtime, RunRequest::text("hello")).unwrap(); + stop_runtime(&plan, &runtime).unwrap(); + assert_eq!( + invocation_result( + &artifacts, + &result.agent_name, + &runtime.runtime_id, + &result.request_id + ) + .unwrap(), + Some(result) + ); + assert_eq!( + fs::metadata(artifacts.join(".fabric")).unwrap().mode() & 0o777, + 0o700 + ); + fs::remove_dir_all(root).unwrap(); + } } #[cfg(unix)] @@ -7269,16 +7488,15 @@ else: .join() .unwrap() .expect_err("original invocation times out"); - assert!( - invocation_result( - &root.join("artifacts"), - &plan.agent_name, - &runtime.runtime_id, - "pending-request" - ) - .unwrap() - .is_none() - ); + let terminal = invocation_result( + &root.join("artifacts"), + &plan.agent_name, + &runtime.runtime_id, + "pending-request", + ) + .unwrap() + .expect("timed-out invocation has a terminal result"); + assert_eq!(terminal.error.unwrap().code, "host_timeout"); stop_runtime(&plan, &runtime).unwrap(); fs::remove_dir_all(root).unwrap(); } diff --git a/crates/fabric-core/src/runtime/results.rs b/crates/fabric-core/src/runtime/results.rs index d7adcd9d8..d0c4ac262 100644 --- a/crates/fabric-core/src/runtime/results.rs +++ b/crates/fabric-core/src/runtime/results.rs @@ -47,6 +47,10 @@ pub(super) struct PendingReservation { } impl PendingReservation { + pub(super) fn dispatched(&self) -> bool { + self.dispatched + } + pub(super) fn mark_dispatched(&mut self) { self.dispatched = true; } @@ -79,7 +83,7 @@ fn leaf(value: &str) -> bool { .all(|c| c.is_ascii_alphanumeric() || c == b'-' || c == b'_') } -fn trusted_ancestry(root: &Path, runtime_id: &str) -> Result<()> { +pub(super) fn trusted_ancestry(root: &Path, runtime_id: &str) -> Result<()> { #[cfg(unix)] { use std::os::unix::fs::MetadataExt; @@ -161,7 +165,6 @@ fn runtime_directory(root: &Path, runtime_id: &str, create: bool) -> Result Result> { + lookup(root, agent, runtime_id, request_id, true) +} + +pub(super) fn unfinalized_result( + root: &Path, + agent: &str, + runtime_id: &str, + request_id: &str, +) -> Result> { + lookup(root, agent, runtime_id, request_id, false) +} + +fn lookup( + root: &Path, + agent: &str, + runtime_id: &str, + request_id: &str, + finalized: bool, ) -> Result> { if [request_id, agent] .iter() @@ -446,7 +468,6 @@ pub fn invocation_result( Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), Err(_) => return Err(failure(runtime_id, "result_storage_failed")), }; - trusted_ancestry(&root, runtime_id)?; let fabric = root.join(".fabric"); let runtime = fabric.join(runtime_id); let requests = runtime.join("requests"); @@ -470,5 +491,9 @@ pub fn invocation_result( { return Err(failure(runtime_id, "invalid_result_target")); } - completed(&runtime, &record) + if finalized { + completed(&runtime, &record) + } else { + completed_result(&runtime, &record) + } } diff --git a/crates/fabric-python/src/lib.rs b/crates/fabric-python/src/lib.rs index 022aa2d9e..21d5dd802 100644 --- a/crates/fabric-python/src/lib.rs +++ b/crates/fabric-python/src/lib.rs @@ -326,6 +326,28 @@ fn _begin_run_result( .map_err(to_py_error) } +/// Finalize a saved one-shot protocol failure after cleanup. +#[pyfunction] +fn _finish_failed_run_result( + py: Python<'_>, + plan_json: String, + runtime_json: String, + request_json: String, + events_json: String, +) -> PyResult<()> { + let plan = parse_run_plan(plan_json)?; + let runtime = parse_runtime_handle(runtime_json)?; + let request: RunRequest = serde_json::from_str(&request_json).map_err(|_| { + pyo3::exceptions::PyValueError::new_err("The invocation request is invalid.") + })?; + let events: Vec = serde_json::from_str(&events_json) + .map_err(|_| pyo3::exceptions::PyValueError::new_err("The cleanup events are invalid."))?; + py.detach(|| { + nemo_fabric_core::runtime::finish_failed_run_result(&plan, &runtime, &request, events) + }) + .map_err(to_py_error) +} + /// Persist a result after the Python one-shot lifecycle finishes cleanup. #[pyfunction] fn _save_run_result( @@ -486,6 +508,7 @@ fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(runtime_status_environment, m)?)?; m.add_function(wrap_pyfunction!(invocation_result, m)?)?; m.add_function(wrap_pyfunction!(_save_run_result, m)?)?; + m.add_function(wrap_pyfunction!(_finish_failed_run_result, m)?)?; m.add_function(wrap_pyfunction!(_begin_run_result, m)?)?; m.add_function(wrap_pyfunction!(invocation_result_environment, m)?)?; m.add_function(wrap_pyfunction!(check_service, m)?)?; diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index b409bb70a..0601506e4 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1713,9 +1713,9 @@ The [software evidence example](https://github.com/NVIDIA/NeMo-Fabric/tree/main/ NeMo Fabric saves each completed `RunResult` as `/.fabric///run-result.json` before returning the result. On Unix, result and request-record files are readable only by their owner. The artifact root comes from `runtime.artifacts` or the environment; without either setting, NeMo Fabric uses its default artifact directory. Files remain until their storage is removed; no automatic retention or eviction is added. The owning `Fabric` client retains the artifact location and environment identity in its bounded stopped-runtime records, so that client can look up results after `Runtime.stop()` without repeating the artifact path. After record eviction or from a new client, provide the original explicit artifact root unless the default root applies. -On Unix, default storage uses `~/.cache/nemo-fabric-` beneath a home directory owned by that user and not writable by other users. Local host directories use the same protected root. A missing or unsafe home directory produces a storage error. Native writes and SDK lookup select the same root. Local one-shot results remain unavailable until the shutdown outcome is saved. A failed final save or interruption keeps the request reserved and prevents lookup from returning an earlier success. Rust `RunRequest::default()` generates a request ID before dispatch; explicitly supplied blank IDs are rejected. +On Unix, default storage uses `~/.cache/nemo-fabric-` beneath a home directory owned by that user and not writable by other users. Local host directories use the same protected root. When no usable user-owned home directory exists, NeMo Fabric uses `/nemo-fabric--private` with mode 0700. An existing fallback directory must belong to that user, have private permissions, and not be a symlink. Native writes and SDK lookup select the same root. Local one-shot results remain unavailable until the shutdown outcome is saved. A failed final save or interruption keeps the request reserved and prevents lookup from returning an earlier success. Rust `RunRequest::default()` generates a request ID before dispatch; explicitly supplied blank IDs are rejected. -Use a trusted artifact directory. On Unix, result storage rejects unsafe directory ownership, writable ancestry, and record-file permissions. Serialized results must fit within 4 MiB. Request identifiers must not be blank or contain NUL. Buffered retries can return the saved result; a request ID cannot switch between buffered and streaming invocations. A completed streaming request returns `stream_replay_unsupported` on replay; retrieve its terminal result with `invocation_result()` instead. +Use a trusted artifact directory. On Unix, result storage validates ownership, permissions, and symlinks within the private `.fabric` subtree, and requires private record-file permissions. Explicit caller-configured artifact roots can have shared or group-writable ancestors. Serialized results must fit within 4 MiB. Request identifiers must not be blank or contain NUL. Buffered retries can return the saved result; a request ID cannot switch between buffered and streaming invocations. A completed streaming request returns `stream_replay_unsupported` on replay; retrieve its terminal result with `invocation_result()` instead. Choose a stable `RunRequest.request_id` before submitting work. After a command timeout, `invocation_timeout` reports unknown effects and returns the runtime and request identifiers. The invocation can continue. Retrieve the completed result with: @@ -1733,3 +1733,5 @@ fabric-agent-ctl result --agent researcher --runtime-id RUNTIME_ID --request-id A null result means no completed result is available; it does not prove the request is still running or safe to repeat. A retry of the same request identifier and input returns the saved result, or reports that the original invocation is pending. Reusing that identifier with different input is rejected. Failed or interrupted reservations are not automatically replayed. For OpenShell, pass `environment=environment` instead of `artifacts` to query the original server through the provider. The CLI requires a retained ownership record on that server. A later local SDK client can read retained files using the explicit artifact root even after runtime stop. Neither path reconstructs a missing result. Remote results retain task-side artifact paths. + +Dispatched invocations that end in a host crash, host timeout, or invalid adapter response save a terminal failed result. The original call still raises its error. The saved result uses fixed diagnostic text, records unknown effects, and disables retry. A missing result never authorizes replay. diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi index c0b3ce6ea..276eac274 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi @@ -62,3 +62,6 @@ def invocation_result_environment( ) -> str: ... def _save_run_result(plan_json: str, runtime_json: str, result_json: str) -> None: ... def _begin_run_result(plan_json: str, runtime_json: str, request_json: str) -> None: ... +def _finish_failed_run_result( + plan_json: str, runtime_json: str, request_json: str, events_json: str +) -> None: ... diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index 127f80d18..77bce7d2f 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -1136,6 +1136,24 @@ def run() -> dict[str, Any]: if invoke_error is None: raise failure from stop_error if invoke_error is not None: + if runtime.get("environment", {}).get("provider") == "local": + try: + native._finish_failed_run_result( + plan_json, + runtime_json, + json.dumps(dict(request)), + json.dumps( + stop_events if isinstance(stop_events, list) else [] + ), + ) + except Exception as error: + failure = FabricRuntimeError( + "The final invocation result could not be saved.", + stage="result", + code="result_storage_failed", + ) + _retain_runtime_evidence(failure, observations) + raise failure from error # Keep obtained identity and cleanup evidence beside the original # exception. A protocol failure is not a normalized invocation. _retain_runtime_evidence(invoke_error, observations) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md index 2c2ea721c..d753a8497 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md @@ -114,7 +114,7 @@ The task configuration carries the default-on `provenance` collector control and NeMo Fabric saves each completed `RunResult` as `/.fabric///run-result.json` before returning the result. On Unix, result and request-record files are readable only by their owner. The artifact root comes from `runtime.artifacts` or the environment; without either setting, NeMo Fabric uses its temporary artifact directory. Files remain until their storage is removed; no automatic retention or eviction is added. -Use a trusted artifact directory. On Unix, result storage rejects unsafe directory ownership, writable ancestry, and record-file permissions. Serialized results must fit within 4 MiB. Request identifiers must not be blank or contain NUL. Buffered retries can return the saved result; a request ID cannot switch between buffered and streaming invocations. A completed streaming request returns `stream_replay_unsupported` on replay; retrieve its terminal result with `invocation_result()` instead. +Use a trusted artifact directory. On Unix, result storage validates ownership, permissions, and symlinks within the private `.fabric` subtree, and requires private record-file permissions. Explicit caller-configured artifact roots can have shared or group-writable ancestors. Serialized results must fit within 4 MiB. Request identifiers must not be blank or contain NUL. Buffered retries can return the saved result; a request ID cannot switch between buffered and streaming invocations. A completed streaming request returns `stream_replay_unsupported` on replay; retrieve its terminal result with `invocation_result()` instead. Choose a stable `RunRequest.request_id` before submitting work. After a command timeout, `invocation_timeout` reports unknown effects and returns the runtime and request identifiers. The invocation can continue. Retrieve the completed result with: @@ -133,4 +133,6 @@ A null result means no completed result is available; it does not prove the requ For OpenShell, pass `environment=environment` instead of `artifacts` to query the original server through the provider. The CLI requires a retained ownership record on that server. A later local SDK client can read retained files using the explicit artifact root even after runtime stop. Neither path reconstructs a missing result. Remote results retain task-side artifact paths. -The Unix default result root is `~/.cache/nemo-fabric-` under a user-owned home directory that other users cannot write. Local host directories use the same protected root. Local one-shot lookup remains unavailable until the shutdown outcome is saved; interrupted or failed finalization does not expose an earlier success. +The Unix default result root is `~/.cache/nemo-fabric-` under a user-owned home directory that other users cannot write. Local host directories use the same protected root. If no usable user-owned home exists, the root is `/nemo-fabric--private`, validated for ownership, mode 0700, and symlink safety. Local one-shot lookup remains unavailable until the shutdown outcome is saved; interrupted or failed finalization does not expose an earlier success. + +Dispatched invocations that end in a host crash, host timeout, or invalid adapter response save a terminal failed result. The original call still raises its error. The saved result uses fixed diagnostic text, records unknown effects, and disables retry. A missing result never authorizes replay. diff --git a/skills/nemo-fabric-integrate/SKILL.md b/skills/nemo-fabric-integrate/SKILL.md index 8e0788d80..55dacab86 100644 --- a/skills/nemo-fabric-integrate/SKILL.md +++ b/skills/nemo-fabric-integrate/SKILL.md @@ -250,6 +250,8 @@ environments. See the [environment lifecycle guide](https://github.com/NVIDIA/Ne Use `await fabric.runtime_status(runtime_id=runtime_id, agent_name=name)` to read recorded state and a safe discovery handle for a runtime started by the same `Fabric` instance. The client retains live runtime objects until successful stop. Add `environment=environment` for an OpenShell lookup through `fabric-agent-ctl status`, including from a later client process while the original server remains alive. `RuntimeStatusReport` reports `active`, `stopped`, or `failed`; `server_state` is present for server reads and null for direct local reads. Each client holds at most 128 runtime records, including pending starts. New starts evict the oldest successfully stopped records first; when active, failed, and pending runtimes fill the limit, startup raises `FabricStateError` with `code="runtime_limit_reached"`. Retained stopped records have a null handle. Discovery handles omit environment variable values and arbitrary provider metadata; use the original `Runtime` for invocation and cleanup. Status reads perform no lifecycle operation or health probe. Use `Fabric.check()` when the consumer needs fresh health evidence. +After a caller timeout, use `await fabric.invocation_result(agent_name=name, runtime_id=runtime_id, request_id=request_id)` to retrieve the original invocation result. Set a stable `RunRequest.request_id` before dispatch. A timeout reported as `invocation_timeout` leaves effects unknown and does not stop the invocation. Pass the original `artifacts` directory from a new local client, or `environment=environment` for OpenShell lookup. The CLI equivalent is `fabric-agent-ctl result --agent --runtime-id --request-id `. Lookup does not invoke or probe the harness. A null result can mean pending or unknown work and never authorizes replay. A matching buffered retry returns a saved result; changed input with the same ID is rejected. Dispatched failures save a terminal result with unknown effects and `retryable=False`. Results remain until their storage is removed. + The selected adapter owns the execution topology. The bundled Claude, Codex, Deep Agents, and Hermes Agent adapters retain their native client, graph/checkpointer, or agent/database inside one local host for the full runtime. Local `process` diff --git a/tests/python/test_native_sdk.py b/tests/python/test_native_sdk.py index 503b60b73..4b73974ce 100644 --- a/tests/python/test_native_sdk.py +++ b/tests/python/test_native_sdk.py @@ -821,3 +821,127 @@ async def main(): timeout=30, ) assert completed.returncode == 0, completed.stderr + + +@pytest.mark.parametrize("one_shot", [False, True]) +@pytest.mark.parametrize("failure", ["crash", "timeout", "invalid"]) +async def test_dispatched_failure_result_is_retrievable( + blocking_adapter_config, tmp_path, one_shot, failure +): + config = blocking_adapter_config + config.runtime.timeout_seconds = 0.2 + adapter = tmp_path / "adapter.py" + source = adapter.read_text().replace( + " if operation=='invoke':", + " if operation=='invoke':\n" + " with (root/'executions').open('a') as count: count.write('once\\n')\n" + " if (root/'crash').exists(): os._exit(17)", + ) + if failure == "invalid": + source = source.replace( + "output={'status':'succeeded','output':{'response':'released'}}", + "output={'status':'invalid','private':'synthetic-sensitive-value'}", + ) + (tmp_path / "release").touch() + elif failure == "crash": + (tmp_path / "crash").touch() + adapter.write_text(source) + client = Fabric() + request = RunRequest(request_id="terminal-failure", input="synthetic test") + runtime = None + try: + if one_shot: + with pytest.raises(FabricRuntimeError) as caught: + await client.run(config, base_dir=tmp_path, request=request) + runtime_id = caught.value.details["runtime_evidence"]["runtime_id"] + else: + runtime = await client.start_runtime(config, base_dir=tmp_path) + runtime_id = runtime.runtime_id + with pytest.raises(FabricRuntimeError): + await runtime.invoke(request=request) + saved = await client.invocation_result( + agent_name=config.metadata.name, + runtime_id=runtime_id, + request_id=request.request_id, + artifacts=tmp_path / "artifacts", + ) + assert saved is not None + assert saved.status == "failed" + assert saved.output is None + assert ( + saved.error.code + == { + "crash": "host_crashed", + "timeout": "host_timeout", + "invalid": "invalid_agent_run_result", + }[failure] + ) + assert saved.error.retryable is False + assert saved.error.metadata["effects"] == "unknown" + assert "synthetic-sensitive-value" not in json.dumps(saved.to_mapping()) + if runtime is not None: + replayed = json.loads( + native.invoke_runtime( + json.dumps(runtime._plan.to_mapping()), + json.dumps(runtime.handle.to_mapping()), + json.dumps(request.to_mapping()), + ) + ) + assert replayed["invocation_id"] == saved.invocation_id + assert replayed["error"] == saved.error.to_mapping() + with pytest.raises(FabricStateError): + await runtime.invoke(request=request) + assert (tmp_path / "executions").read_text() == "once\n" + finally: + if runtime is not None: + await runtime.stop() + + +@pytest.mark.skipif(sys.platform == "win32", reason="Unix storage ownership") +@pytest.mark.parametrize("one_shot", [False, True]) +def test_unusable_home_uses_private_temporary_storage( + hermes_shim_agent_dir, tmp_path, one_shot +): + temporary = tmp_path / "fallback" + temporary.mkdir() + config = hermes_shim_config() + config.runtime.artifacts = None + config.environment.artifacts = None + script = """ +import asyncio, json, os, pathlib, sys +from nemo_fabric import Fabric, FabricConfig +async def main(): + client = Fabric() + config = FabricConfig.from_mapping(json.loads(sys.argv[1])) + if sys.argv[3] == "True": + result = await client.run(config, base_dir=sys.argv[2], input="home fallback") + else: + runtime = await client.start_runtime(config, base_dir=sys.argv[2]) + try: + result = await runtime.invoke(input="home fallback") + finally: + await runtime.stop() + saved = await client.invocation_result(agent_name=result.agent_name, runtime_id=result.runtime_id, request_id=result.request_id) + assert saved == result + root = pathlib.Path(os.environ['TMPDIR']) / f'nemo-fabric-{os.geteuid()}-private' + assert root.stat().st_mode & 0o777 == 0o700 + assert pathlib.Path(result.metadata['fabric_home']).is_relative_to(root) +asyncio.run(main()) +""" + completed = subprocess.run( + [ + sys.executable, + "-c", + script, + config.model_dump_json(exclude_none=True), + str(hermes_shim_agent_dir), + str(one_shot), + ], + env=dict( + os.environ, HOME="/", TMPDIR=str(temporary), ADAPTER_PYTHON=sys.executable + ), + capture_output=True, + text=True, + timeout=30, + ) + assert completed.returncode == 0, completed.stderr diff --git a/tests/python/test_runtime.py b/tests/python/test_runtime.py index 91c391ca7..a0ce4387b 100644 --- a/tests/python/test_runtime.py +++ b/tests/python/test_runtime.py @@ -2008,3 +2008,20 @@ def start(*args): finally: release.set() await asyncio.gather(task, return_exceptions=True) + + +async def test_failed_result_finalization_retains_runtime_evidence(mock_native): + mock_native.invoke_runtime.side_effect = FabricRuntimeError( + "synthetic invocation failure", stage="invoke", code="host_crashed" + ) + mock_native._finish_failed_run_result.side_effect = RuntimeError( + "synthetic storage failure" + ) + with pytest.raises(FabricRuntimeError) as caught: + await runtime_mod._run_native_lifecycle( + mock_native, _plan(), {"request_id": "failed-request", "input": "synthetic"} + ) + assert caught.value.code == "result_storage_failed" + assert caught.value.stage == "result" + assert caught.value.details["runtime_evidence"]["runtime_id"] == "runtime-1" + assert caught.value.details["runtime_evidence"]["request_id"] == "failed-request" diff --git a/tests/python/test_sdk_contract.py b/tests/python/test_sdk_contract.py index d471ae479..c1ccfbcf6 100644 --- a/tests/python/test_sdk_contract.py +++ b/tests/python/test_sdk_contract.py @@ -1846,6 +1846,13 @@ def _begin_run_result( assert json.loads(runtime_json)["runtime_id"] == "runtime-1" assert json.loads(request_json)["request_id"] + def _finish_failed_run_result( + self, plan_json: str, runtime_json: str, request_json: str, events_json: str + ) -> None: + self._begin_run_result(plan_json, runtime_json, request_json) + assert self.stopped > 0 + assert isinstance(json.loads(events_json), list) + def _save_run_result( self, plan_json: str, runtime_json: str, result_json: str ) -> None: From 718acce9df2c4d2f202e93427aa1c1a0c7ea1829 Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Mon, 16 Nov 2026 09:22:43 -0700 Subject: [PATCH 08/10] fix: preserve active runtimes after rejected requests Signed-off-by: Sara Tadayon --- docs/sdk/python.mdx | 2 + .../src/nemo_fabric/runtime.py | 18 +++++++-- .../src/nemo_fabric/runtime_server/README.md | 2 + tests/python/test_native_sdk.py | 22 +++++++++++ tests/python/test_runtime.py | 37 ++++++++++++++++++- 5 files changed, 76 insertions(+), 5 deletions(-) diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 0601506e4..c0edbca10 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1717,6 +1717,8 @@ On Unix, default storage uses `~/.cache/nemo-fabric-` beneath Use a trusted artifact directory. On Unix, result storage validates ownership, permissions, and symlinks within the private `.fabric` subtree, and requires private record-file permissions. Explicit caller-configured artifact roots can have shared or group-writable ancestors. Serialized results must fit within 4 MiB. Request identifiers must not be blank or contain NUL. Buffered retries can return the saved result; a request ID cannot switch between buffered and streaming invocations. A completed streaming request returns `stream_replay_unsupported` on replay; retrieve its terminal result with `invocation_result()` instead. +The SDK keeps the runtime active after `request_conflict`, `invocation_pending`, and `stream_replay_unsupported`. These request rejections do not dispatch a new turn. + Choose a stable `RunRequest.request_id` before submitting work. After a command timeout, `invocation_timeout` reports unknown effects and returns the runtime and request identifiers. The invocation can continue. Retrieve the completed result with: ```python diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index 77bce7d2f..a6b591257 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -46,6 +46,12 @@ _PI_ADAPTER_ID = "nvidia.fabric.pi" _PI_STREAM_CORRELATION_MODE = "pi_turn_window" +# Result-store rejections do not dispatch a new turn to the adapter. +_INVOCATION_REJECTION_CODES = ( + "request_conflict", + "invocation_pending", + "stream_replay_unsupported", +) class RuntimeStatus(str, Enum): @@ -321,12 +327,18 @@ def stop_after_cancel() -> Any: else: self._status = RuntimeStatus.STOPPED raise - except FabricError: - if self._status is not RuntimeStatus.STOPPED: + except FabricError as error: + if ( + self._status is not RuntimeStatus.STOPPED + and error.code not in _INVOCATION_REJECTION_CODES + ): self._status = RuntimeStatus.FAILED raise except Exception as error: - if self._status is not RuntimeStatus.STOPPED: + if ( + self._status is not RuntimeStatus.STOPPED + and getattr(error, "code", None) not in _INVOCATION_REJECTION_CODES + ): self._status = RuntimeStatus.FAILED raise _runtime_error(error, stage="invoke") from error if absorb_result: diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md index d753a8497..3fe93e127 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md @@ -116,6 +116,8 @@ NeMo Fabric saves each completed `RunResult` as `/.fabric/ Date: Mon, 16 Nov 2026 10:12:13 -0700 Subject: [PATCH 09/10] fix: preserve result finalization and cleanup recovery Signed-off-by: Sara Tadayon --- crates/fabric-core/src/runtime.rs | 72 +++++++++++++++++-- .../src/nemo_fabric/runtime.py | 13 ++-- tests/python/test_runtime.py | 23 ++++++ 3 files changed, 94 insertions(+), 14 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index a8f86ae48..08d27b86a 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -1093,7 +1093,9 @@ pub fn run_plan(plan: &RunPlan, request: RunRequest) -> Result { Ok(result) => result, Err(error) => { let events = stop_runtime(plan, &runtime).unwrap_or_default(); - finish_failed_run_result(plan, &runtime, &original_request, events)?; + let error = finish_failed_run_result(plan, &runtime, &original_request, events) + .err() + .unwrap_or(error); if local_runtime_is_retained(&runtime.runtime_id) { return Err(FabricError::RuntimeCleanupPending(Box::new( crate::RuntimeCleanupPending::new(runtime, None, error), @@ -1104,6 +1106,21 @@ pub fn run_plan(plan: &RunPlan, request: RunRequest) -> Result { }; match stop_runtime(plan, &runtime) { Err(error) if local_runtime_is_retained(&runtime.runtime_id) => { + if result.status == RunStatus::Succeeded { + result.status = RunStatus::Failed; + result.error = Some(ErrorInfo { + stage: ErrorStage::Stop, + code: "cleanup_unconfirmed".into(), + message: + "Runtime cleanup is unconfirmed; retry cleanup with the original runtime." + .into(), + retryable: false, + metadata: BTreeMap::new(), + }); + } + let error = save_run_result(plan, &runtime, &result) + .err() + .unwrap_or(error); return Err(FabricError::RuntimeCleanupPending(Box::new( crate::RuntimeCleanupPending::new(runtime, Some(result), error), ))); @@ -6140,6 +6157,9 @@ for line in sys.stdin: shutil.rmtree(directory) with open(directory, "w", encoding="utf-8") as stream: stream.write("cleanup requires a retry") + root = os.path.dirname(os.environ["FABRIC_FAKE_HOST_STOP_MARKER"]) + if os.path.exists(os.path.join(root, "fail-result-storage")): + os.chmod(os.path.join(root, "artifacts", ".fabric"), 0o777) if MODE == "stop_delay": with open(os.environ["FABRIC_FAKE_HOST_STOP_MARKER"], "w", encoding="utf-8") as stream: stream.write("started") @@ -6455,7 +6475,7 @@ else: }; assert_eq!(pending.runtime(), &runtime); assert!(local_hosts().contains_key(&runtime.runtime_id)); - assert!(local_host_directory(&runtime.runtime_id).exists()); + assert!(local_host_directory(&runtime.runtime_id).unwrap().exists()); let denied = invoke_runtime(&plan, &runtime, RunRequest::text("must not invoke")); stop_runtime(&plan, pending.runtime()).unwrap(); #[cfg(unix)] @@ -6487,7 +6507,7 @@ else: matches!(release, Err(FabricError::AdapterLifecycleOperation {code, ..}) if code == "cleanup_unconfirmed") ); assert!(local_hosts().contains_key(&context.runtime().runtime_id)); - let directory = local_host_directory(&context.runtime().runtime_id); + let directory = local_host_directory(&context.runtime().runtime_id).unwrap(); assert!(directory.is_file()); fs::remove_file(&directory).unwrap(); fs::create_dir(&directory).unwrap(); @@ -6667,9 +6687,19 @@ else: #[cfg(unix)] #[test] fn one_shot_cleanup_failures_keep_a_safe_retry_handle() { - for mode in ["cleanup_failure", "invoke_failure_cleanup_failure"] { + use std::os::unix::fs::PermissionsExt; + for (mode, storage_fails) in [ + ("cleanup_failure", true), + ("invoke_failure_cleanup_failure", true), + ("cleanup_failure", false), + ("invoke_failure_cleanup_failure", false), + ] { let (root, plan) = local_host_plan(mode); - let error = run_plan(&plan, RunRequest::text("TEST_ONLY_PRIVATE_RESULT")) + if storage_fails { + fs::write(root.join("fail-result-storage"), "").unwrap(); + } + let request = RunRequest::text("TEST_ONLY_PRIVATE_RESULT"); + let error = run_plan(&plan, request.clone()) .expect_err("unconfirmed cleanup must retain a retry handle"); let debug = format!("{error:?}"); let FabricError::RuntimeCleanupPending(pending) = error else { @@ -6677,13 +6707,40 @@ else: }; assert_eq!(pending.result().is_some(), mode == "cleanup_failure"); let runtime = pending.runtime(); - let directory = local_host_directory(&runtime.runtime_id); + let directory = local_host_directory(&runtime.runtime_id).unwrap(); fs::remove_file(&directory).unwrap(); fs::create_dir_all(&directory).unwrap(); stop_runtime(&plan, runtime).expect("retry cleanup with the original handle"); assert!(!local_hosts().contains_key(&runtime.runtime_id)); assert!(!debug.contains("TEST_ONLY_PRIVATE_RESULT")); assert!(!debug.contains(&runtime.runtime_binding)); + if storage_fails { + assert!( + matches!(pending.cause(), FabricError::AdapterLifecycleOperation { code, .. } if code == "result_storage_failed") + ); + fs::set_permissions( + root.join("artifacts/.fabric"), + fs::Permissions::from_mode(0o700), + ) + .unwrap(); + } + let saved = invocation_result( + &root.join("artifacts"), + &plan.agent_name, + &runtime.runtime_id, + &request.request_id, + ) + .unwrap(); + assert_eq!( + saved.is_some(), + !storage_fails, + "publish only finalized results" + ); + if !storage_fails && let Some(result) = pending.result() { + assert_eq!(saved.as_ref().unwrap(), result); + assert_eq!(result.status, RunStatus::Failed); + assert_eq!(result.error.as_ref().unwrap().code, "cleanup_unconfirmed"); + } fs::remove_dir_all(root).unwrap(); } } @@ -7750,7 +7807,8 @@ else: let result = pending .result() .expect("completed invocation remains available"); - assert_eq!(result.status, RunStatus::Succeeded); + assert_eq!(result.status, RunStatus::Failed); + assert_eq!(result.error.as_ref().unwrap().stage, ErrorStage::Stop); assert_eq!(result.runtime_id, pending.runtime().runtime_id); assert!( matches!(pending.cause(), FabricError::AdapterLifecycleOperation {operation: "stop", code, ..} if code == "cleanup_unconfirmed") diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index a6b591257..e5235f216 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -1097,6 +1097,8 @@ def run() -> dict[str, Any]: except Exception as error: stop_error = error cleanup = "failed" + if _runtime_error(error, stage="stop").code == "cleanup_unconfirmed": + cleanup_runtime_handle = runtime if invoke_error is None: if result is None: raise @@ -1132,12 +1134,7 @@ def run() -> dict[str, Any]: ) _retain_runtime_evidence(failure, observations) raise failure from error - if ( - stop_error is not None - and _runtime_error(stop_error, stage="stop").code - == "cleanup_unconfirmed" - ): - cleanup_runtime_handle = runtime + if cleanup_runtime_handle is not None: failure = invoke_error or FabricRuntimeError( "Runtime cleanup is unconfirmed; retry cleanup_runtime.stop().", stage="stop", @@ -1178,7 +1175,9 @@ def run() -> dict[str, Any]: if cleanup_runtime_handle is not None: error._fabric_cleanup_runtime_handle = cleanup_runtime_handle raise - except FabricError: + except FabricError as error: + if cleanup_runtime_handle is not None: + error._fabric_cleanup_runtime_handle = cleanup_runtime_handle raise except Exception as error: raise _runtime_error(error, stage="run") from error diff --git a/tests/python/test_runtime.py b/tests/python/test_runtime.py index 42ee75f67..db52a833c 100644 --- a/tests/python/test_runtime.py +++ b/tests/python/test_runtime.py @@ -247,6 +247,29 @@ async def test_one_shot_retains_a_cleanup_runtime_without_exposing_its_handle( assert mock_native.stop_runtime.call_count == 2 +@pytest.mark.parametrize("invoke_fails", [False, True]) +async def test_result_storage_failure_preserves_pending_cleanup( + native_client, mock_native, invoke_fails +): + if invoke_fails: + mock_native.invoke_runtime.side_effect = FabricRuntimeError( + "synthetic invoke failure", stage="invoke", code="host_crashed" + ) + mock_native.stop_runtime.side_effect = FabricRuntimeError( + "synthetic cleanup failure", stage="stop", code="cleanup_unconfirmed" + ) + method = "_finish_failed_run_result" if invoke_fails else "_save_run_result" + getattr(mock_native, method).side_effect = RuntimeError("synthetic storage failure") + with pytest.raises(FabricRuntimeError) as caught: + await native_client.run(_config(), input="synthetic work") + assert caught.value.code == "result_storage_failed" + recovery = caught.value.cleanup_runtime + assert isinstance(recovery, Runtime) + mock_native.stop_runtime.side_effect = None + await recovery.stop() + assert recovery.status is RuntimeStatus.STOPPED + + async def test_cancelled_one_shot_retains_a_cleanup_runtime(native_client, mock_native): entered, release = threading.Event(), threading.Event() From 77eea39abc364af3f194419c56851eb8ca32f239 Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Mon, 16 Nov 2026 11:40:03 -0700 Subject: [PATCH 10/10] fix: preserve deadline details in recovered results Signed-off-by: Sara Tadayon --- crates/fabric-core/src/runtime.rs | 19 +++++++++++++++++-- tests/python/test_native_sdk.py | 26 +++++++++++++++++++++----- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 08d27b86a..c3b3a1c86 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -3059,13 +3059,22 @@ fn run_local_host_invocation_with_timeout( FabricError::AdapterLifecycleOperation { code, .. } if matches!( code.as_str(), - "host_crashed" | "host_timeout" | "interrupted" | "invalid_agent_run_result" + "host_crashed" + | "host_timeout" + | "timeout" + | "interrupted" + | "invalid_agent_run_result" ) => { code.as_str() } _ => "invocation_failed", }; + let mut metadata = match error { + FabricError::AdapterLifecycleOperation { metadata, .. } => metadata.clone(), + _ => BTreeMap::new(), + }; + metadata.insert("effects".into(), Value::String("unknown".into())); let result = RunResult { session_ref: None, agent_name: plan.agent_name.clone(), @@ -3082,7 +3091,7 @@ fn run_local_host_invocation_with_timeout( code: code.into(), message: "The dispatched invocation ended without a valid result; effects may be unknown. Do not replay the work.".into(), retryable: false, - metadata: BTreeMap::from([("effects".into(), Value::String("unknown".into()))]), + metadata, }), usage: None, execution: None, @@ -7232,6 +7241,12 @@ else: assert_eq!(saved.error.as_ref().unwrap().code, expected); assert!(!saved.error.as_ref().unwrap().retryable); assert_eq!(saved.error.as_ref().unwrap().metadata["effects"], "unknown"); + if expected == "host_timeout" { + let metadata = &saved.error.as_ref().unwrap().metadata; + assert_eq!(metadata["owner"], "fabric_runtime"); + assert_eq!(metadata["timeout_seconds"], 0.1); + assert_eq!(metadata["cause_code"], "host_timeout"); + } assert_eq!(invoke_runtime(&plan, &runtime, request).unwrap(), saved); fs::remove_dir_all(root).unwrap(); } diff --git a/tests/python/test_native_sdk.py b/tests/python/test_native_sdk.py index 9a38485ee..26f732b3a 100644 --- a/tests/python/test_native_sdk.py +++ b/tests/python/test_native_sdk.py @@ -562,12 +562,12 @@ async def smoke(client: Fabric, fixture_agent: Path) -> None: async def test_adapter_deadline_keeps_normalized_code_and_native_cause( hermes_shim_agent_dir, ): + client = Fabric() + config = hermes_shim_config() + config.runtime.artifacts = str(hermes_shim_agent_dir / "deadline-results") + request = RunRequest(input="test", context={"normalized_deadline": True}) with pytest.raises(FabricRuntimeError) as caught: - await Fabric().run( - hermes_shim_config(), - base_dir=hermes_shim_agent_dir, - request=RunRequest(input="test", context={"normalized_deadline": True}), - ) + await client.run(config, base_dir=hermes_shim_agent_dir, request=request) assert caught.value.code == "timeout" assert caught.value.stage == "invoke" assert caught.value.details["owner"] == "adapter" @@ -575,6 +575,18 @@ async def test_adapter_deadline_keeps_normalized_code_and_native_cause( assert caught.value.details["cause_code"] == "strands_prompt_timeout" assert caught.value.details["runtime_evidence"]["cleanup"] == "completed" assert isinstance(caught.value.__cause__, TimeoutError) + saved = await client.invocation_result( + agent_name=config.metadata.name, + runtime_id=caught.value.details["runtime_evidence"]["runtime_id"], + request_id=request.request_id, + artifacts=config.runtime.artifacts, + ) + assert saved is not None + assert saved.error.code == "timeout" + assert saved.error.metadata["owner"] == "adapter" + assert saved.error.metadata["timeout_seconds"] == 300 + assert saved.error.metadata["cause_code"] == "strands_prompt_timeout" + assert saved.error.metadata["effects"] == "unknown" @pytest.mark.parametrize( @@ -900,6 +912,10 @@ async def test_dispatched_failure_result_is_retrievable( ) assert saved.error.retryable is False assert saved.error.metadata["effects"] == "unknown" + if failure == "timeout": + assert saved.error.metadata["owner"] == "fabric_runtime" + assert saved.error.metadata["timeout_seconds"] == 0.2 + assert saved.error.metadata["cause_code"] == "host_timeout" assert "synthetic-sensitive-value" not in json.dumps(saved.to_mapping()) if runtime is not None: replayed = json.loads(