diff --git a/crates/fabric-core/src/check.rs b/crates/fabric-core/src/check.rs index da1a572b3..dd462a6b6 100644 --- a/crates/fabric-core/src/check.rs +++ b/crates/fabric-core/src/check.rs @@ -1117,7 +1117,9 @@ fn probe_target( let budget = Duration::try_from_secs_f64(timeout).unwrap_or(Duration::MAX); let remaining = || budget.saturating_sub(started.elapsed()); let result = (|| -> std::io::Result { - let path = crate::runtime::local_host_directory(runtime_id).join("health.endpoint.json"); + let path = crate::runtime::local_host_directory(runtime_id) + .map_err(|_| std::io::Error::other("default host storage unavailable"))? + .join("health.endpoint.json"); let mut options = OpenOptions::new(); options.read(true); #[cfg(unix)] @@ -1283,7 +1285,7 @@ mod tests { fn endpoint(runtime: &RuntimeHandle) -> TcpListener { let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); - let directory = crate::runtime::local_host_directory(&runtime.runtime_id); + let directory = crate::runtime::local_host_directory(&runtime.runtime_id).unwrap(); std::fs::create_dir_all(&directory).unwrap(); let endpoint = directory.join("health.endpoint.json"); std::fs::write( @@ -1379,7 +1381,8 @@ mod tests { fn cleanup(runtime: &RuntimeHandle) { invalidate_runtime(runtime); - std::fs::remove_dir_all(crate::runtime::local_host_directory(&runtime.runtime_id)).unwrap(); + std::fs::remove_dir_all(crate::runtime::local_host_directory(&runtime.runtime_id).unwrap()) + .unwrap(); } fn gateway_service(runtime: &RuntimeHandle) -> crate::ServiceHandle { diff --git a/crates/fabric-core/src/environment.rs b/crates/fabric-core/src/environment.rs index 63b0853fe..7e289a016 100644 --- a/crates/fabric-core/src/environment.rs +++ b/crates/fabric-core/src/environment.rs @@ -661,14 +661,17 @@ where let budget = *timeout_seconds; return request_check_provider_process(command, operation, budget); } - if matches!(task_command, TaskCommand::Status { .. }) { + if matches!( + task_command, + TaskCommand::Status { .. } | TaskCommand::Result { .. } + ) { return request_check_provider_process(command, operation, 10.0); } let operation_name = operation.name(); let request_id = new_id("environment-provider-request"); let environment_key = environment_resource_key(environment)?; let (identity, is_start, is_stop) = match task_command { - TaskCommand::Check { .. } | TaskCommand::Status { .. } => { + TaskCommand::Check { .. } | TaskCommand::Status { .. } | TaskCommand::Result { .. } => { unreachable!("reads use a separate provider process") } TaskCommand::Start { .. } => (request_id.clone(), true, false), diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 54eeefd27..c3b3a1c86 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -39,7 +39,10 @@ use crate::system_stats::ResourceObservation; use crate::system_stats::{CgroupStats, InvocationSystemStats, StatsSession, SystemReader}; mod openshell; +mod results; use openshell::OpenShellRuntimeAdapter; +#[doc(hidden)] +pub use results::invocation_result; pub(crate) fn remote_check_agent( environment: &EnvironmentHandle, @@ -87,6 +90,66 @@ pub fn runtime_status_environment( } } +/// Read a saved task-side invocation result through its verified OpenShell environment. +#[doc(hidden)] +pub fn invocation_result_environment( + environment: &EnvironmentHandle, + agent: &str, + runtime_id: &str, + request_id: &str, +) -> Result { + if environment.provider != "openshell" + || environment.control_location != ControlLocation::InEnvControl + || [agent, runtime_id, request_id] + .iter() + .any(|value| value.trim().is_empty() || value.contains('\0')) + { + return Err(FabricError::InvalidConfig { + field: "invocation_result".into(), + reason: "a valid invocation target and verified OpenShell environment are required" + .into(), + }); + } + let command = crate::task_command::TaskCommand::Result { + agent: agent.into(), + runtime_id: runtime_id.into(), + request_id: request_id.into(), + }; + result_lookup_response(crate::environment::command_runtime( + environment, + &command, + 10, + )?) +} + +fn result_lookup_response(response: crate::task_command::TaskCommandResponse) -> Result { + match response { + crate::task_command::TaskCommandResponse::Returned { value } => Ok(value), + crate::task_command::TaskCommandResponse::Rejected { error } => { + Err(FabricError::EnvironmentProviderOperation { + provider: "openshell".into(), + operation: "invocation_result".into(), + code: if error.effects == "none" + && error.code.len() <= 64 + && error + .code + .bytes() + .next() + .is_some_and(|first| first.is_ascii_lowercase()) + && error.code.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_' + }) { + error.code + } else { + "invalid_response".into() + }, + message: "The result lookup could not be completed; do not replay uncertain work." + .into(), + }) + } + } +} + static NEXT_ID: AtomicU64 = AtomicU64::new(1); const ADAPTER_PYTHON_ENV: &str = "ADAPTER_PYTHON"; const VIRTUAL_ENV_ENV: &str = "VIRTUAL_ENV"; @@ -138,12 +201,13 @@ static LOCAL_SERVICES: LazyLock, } +impl Default for RunRequest { + fn default() -> Self { + Self { + acknowledge_recovery: false, + request_id: new_id("request"), + relay_session_root: None, + input: Value::Null, + context: BTreeMap::new(), + overrides: None, + } + } +} + impl RunRequest { /// Build a text request. pub fn text(input: impl Into) -> Self { @@ -1009,10 +1086,16 @@ enum LocalServiceState { /// any completed result. Retry [`stop_runtime`] with that handle and this plan. pub fn run_plan(plan: &RunPlan, request: RunRequest) -> Result { let runtime = start_runtime(plan)?; - let mut result = match invoke_runtime(plan, &runtime, request) { + let original_request = request.clone(); + let invoked = begin_run_result(plan, &runtime, &request) + .and_then(|()| invoke_runtime(plan, &runtime, request)); + let mut result = match invoked { Ok(result) => result, Err(error) => { - let _ = stop_runtime(plan, &runtime); + let events = stop_runtime(plan, &runtime).unwrap_or_default(); + let error = finish_failed_run_result(plan, &runtime, &original_request, events) + .err() + .unwrap_or(error); if local_runtime_is_retained(&runtime.runtime_id) { return Err(FabricError::RuntimeCleanupPending(Box::new( crate::RuntimeCleanupPending::new(runtime, None, error), @@ -1023,6 +1106,21 @@ pub fn run_plan(plan: &RunPlan, request: RunRequest) -> Result { }; match stop_runtime(plan, &runtime) { Err(error) if local_runtime_is_retained(&runtime.runtime_id) => { + if result.status == RunStatus::Succeeded { + result.status = RunStatus::Failed; + result.error = Some(ErrorInfo { + stage: ErrorStage::Stop, + code: "cleanup_unconfirmed".into(), + message: + "Runtime cleanup is unconfirmed; retry cleanup with the original runtime." + .into(), + retryable: false, + metadata: BTreeMap::new(), + }); + } + let error = save_run_result(plan, &runtime, &result) + .err() + .unwrap_or(error); return Err(FabricError::RuntimeCleanupPending(Box::new( crate::RuntimeCleanupPending::new(runtime, Some(result), error), ))); @@ -1043,9 +1141,156 @@ pub fn run_plan(plan: &RunPlan, request: RunRequest) -> Result { } Err(_) => {} } + if uses_local_host(plan) && runtime.environment.provider == "local" { + save_run_result(plan, &runtime, &result)?; + } Ok(result) } +/// Keep local one-shot output unavailable until cleanup is finalized. +#[doc(hidden)] +pub fn begin_run_result( + plan: &RunPlan, + runtime: &RuntimeHandle, + request: &RunRequest, +) -> Result<()> { + validate_runtime_handle(plan, runtime)?; + if request.request_id.trim().is_empty() || request.request_id.contains('\0') { + return Err(FabricError::InvalidConfig { + field: "request_id".into(), + reason: "must not be blank or contain NUL".into(), + }); + } + if runtime.environment.provider == "local" { + let root = artifact_manifest(plan)? + .root + .map_or_else(default_artifact_root, Ok)?; + results::begin_finalization( + &root, + &runtime.agent_name, + &runtime.runtime_id, + &request.request_id, + )?; + } + Ok(()) +} + +/// Default result and host storage; Unix users have separate namespaces. +#[doc(hidden)] +pub fn default_artifact_root() -> Result { + #[cfg(unix)] + { + let home = std::env::home_dir().unwrap_or_default(); + // SAFETY: geteuid has no preconditions or side effects. + default_artifact_root_for_user(&home, unsafe { libc::geteuid() }) + } + #[cfg(not(unix))] + Ok(std::env::temp_dir().join("nemo-fabric")) +} + +#[cfg(unix)] +fn default_storage_unavailable() -> FabricError { + FabricError::InvalidConfig { + field: "local_storage".into(), + reason: "Default local storage requires a private directory owned by the current user." + .into(), + } +} + +#[cfg(unix)] +fn default_artifact_root_for_user(home: &Path, uid: u32) -> Result { + use std::os::unix::fs::MetadataExt; + // SAFETY: geteuid has no preconditions or side effects. + if uid != unsafe { libc::geteuid() } { + return Err(default_storage_unavailable()); + } + if let Ok(home) = std::fs::canonicalize(home) + && let Ok(metadata) = std::fs::metadata(&home) + && metadata.is_dir() + && metadata.uid() == uid + && metadata.mode() & 0o022 == 0 + { + return Ok(home.join(".cache").join(format!("nemo-fabric-{uid}"))); + } + temporary_artifact_root_for_user(&std::env::temp_dir(), uid) +} + +#[cfg(unix)] +fn temporary_artifact_root_for_user(temporary: &Path, uid: u32) -> Result { + use std::os::unix::fs::MetadataExt; + results::trusted_ancestry(temporary, "local-storage") + .map_err(|_| default_storage_unavailable())?; + let root = temporary.join(format!("nemo-fabric-{uid}-private")); + // Atomic creation and lstat reject pre-created symlinks, other owners, + // and writable directories. Never chmod a path supplied by another user. + use std::os::unix::fs::DirBuilderExt; + if let Err(error) = std::fs::DirBuilder::new().mode(0o700).create(&root) + && error.kind() != std::io::ErrorKind::AlreadyExists + { + return Err(default_storage_unavailable()); + } + let metadata = std::fs::symlink_metadata(&root).map_err(|_| default_storage_unavailable())?; + if !metadata.is_dir() + || metadata.file_type().is_symlink() + || metadata.uid() != uid + || metadata.mode() & 0o077 != 0 + { + return Err(default_storage_unavailable()); + } + Ok(root) +} + +/// Persist the final one-shot result after Python lifecycle cleanup. +#[doc(hidden)] +pub fn save_run_result(plan: &RunPlan, runtime: &RuntimeHandle, result: &RunResult) -> Result<()> { + validate_runtime_handle(plan, runtime)?; + let invalid = || { + lifecycle_error( + AdapterLifecycleOperation::Invoke, + &runtime.runtime_id, + "invalid_result_target", + "The completed result does not match its reservation.", + "", + ) + }; + if runtime.environment.provider != "local" + || result.runtime_id != runtime.runtime_id + || result.agent_name != runtime.agent_name + { + return Err(invalid()); + } + let root = artifact_manifest(plan)? + .root + .map_or_else(default_artifact_root, Ok)?; + results::finalize(&root, result) +} + +/// Finalize a dispatched one-shot failure after lifecycle cleanup. +#[doc(hidden)] +pub fn finish_failed_run_result( + plan: &RunPlan, + runtime: &RuntimeHandle, + request: &RunRequest, + events: Vec, +) -> Result<()> { + validate_runtime_handle(plan, runtime)?; + if runtime.environment.provider == "local" && uses_local_host(plan) { + let root = artifact_manifest(plan)? + .root + .map_or_else(default_artifact_root, Ok)?; + if let Some(mut result) = results::unfinalized_result( + &root, + &runtime.agent_name, + &runtime.runtime_id, + &request.request_id, + )? { + result.events.extend(events); + save_run_result(plan, runtime, &result)?; + } + } + Ok(()) +} + /// Prepare an execution environment for a run plan. /// /// This block implements local environment preparation. Other providers return @@ -1274,7 +1519,7 @@ fn start_service( let mut artifacts = artifact_manifest(plan)?; let fabric_home = prepare_fabric_home(&artifacts, &runtime, &invocation)?; let relay_config = prepare_relay_runtime_config(plan, &runtime, &fabric_home, &mut artifacts)?; - let connection_path = local_host_directory(&service_id).join("service-connection.json"); + let connection_path = local_host_directory(&service_id)?.join("service-connection.json"); let service_context = match reference { Some(reference) => AdapterServiceContext::Attach { connection_path: connection_path.clone(), @@ -2764,19 +3009,119 @@ impl LocalHostInvocation { fn run_local_host_invocation_with_timeout( plan: &RunPlan, runtime: &RuntimeHandle, - mut request: RunRequest, + request: RunRequest, invocation_kind: LocalHostInvocation, invoke_timeout: Duration, ) -> Result { - let operation = invocation_kind.operation(); - if request.request_id.is_empty() { - request.request_id = new_id("request"); + if request.request_id.trim().is_empty() || request.request_id.contains('\0') { + return Err(FabricError::InvalidConfig { + field: "request_id".into(), + reason: "must not be blank or contain NUL".into(), + }); } + let projected = project_agent_run_request(&request)?; + validate_agent_run_request_extensions(&projected, plan.adapter_descriptor.as_ref())?; let invocation = InvocationHandle { invocation_id: new_id("invocation"), request_id: request.request_id.clone(), runtime_id: runtime.runtime_id.clone(), }; + let root = artifact_manifest(plan)? + .root + .map_or_else(default_artifact_root, Ok)?; + let mut reservation = match results::reserve( + &root, + runtime, + &request, + &invocation, + matches!(&invocation_kind, LocalHostInvocation::OpenAiStream(_)), + )? { + results::RequestReservation::Cached(result) => return Ok(*result), + results::RequestReservation::Pending(pending) => pending, + }; + let outcome = execute_local_host_invocation( + plan, + runtime, + request, + invocation_kind, + invoke_timeout, + invocation.clone(), + &mut reservation, + ) + .and_then(|result| { + results::save(&root, &result)?; + Ok(result) + }); + if let Err(error) = &outcome + && reservation.dispatched() + { + let code = match error { + FabricError::AdapterLifecycleOperation { code, .. } + if matches!( + code.as_str(), + "host_crashed" + | "host_timeout" + | "timeout" + | "interrupted" + | "invalid_agent_run_result" + ) => + { + code.as_str() + } + _ => "invocation_failed", + }; + let mut metadata = match error { + FabricError::AdapterLifecycleOperation { metadata, .. } => metadata.clone(), + _ => BTreeMap::new(), + }; + metadata.insert("effects".into(), Value::String("unknown".into())); + let result = RunResult { + session_ref: None, + agent_name: plan.agent_name.clone(), + harness: harness(plan), + adapter_kind: adapter_kind(plan), + adapter_id: adapter_id(plan), + runtime_id: runtime.runtime_id.clone(), + invocation_id: invocation.invocation_id, + request_id: invocation.request_id, + status: if code == "interrupted" { RunStatus::Cancelled } else { RunStatus::Failed }, + output: Value::Null, + error: Some(ErrorInfo { + stage: ErrorStage::Invoke, + code: code.into(), + message: "The dispatched invocation ended without a valid result; effects may be unknown. Do not replay the work.".into(), + retryable: false, + metadata, + }), + usage: None, + execution: None, + resource_snapshot: None, + system_stats: None, + artifacts: ArtifactManifest { root: Some(root.clone()), ..ArtifactManifest::default() }, + telemetry: vec![], + events: vec![event_with_metadata( + "invocation_end", + "The dispatched invocation ended without a valid result.", + BTreeMap::from([("code".into(), Value::String(code.into()))]), + )], + metadata: BTreeMap::new(), + }; + // Retain the dispatched reservation even if this write fails. + results::save(&root, &result)?; + } + outcome +} + +fn execute_local_host_invocation( + plan: &RunPlan, + runtime: &RuntimeHandle, + request: RunRequest, + invocation_kind: LocalHostInvocation, + invoke_timeout: Duration, + invocation: InvocationHandle, + reservation: &mut results::PendingReservation, +) -> Result { + let operation = invocation_kind.operation(); let host = local_hosts() .get(&runtime.runtime_id) .cloned() @@ -2892,11 +3237,12 @@ fn run_local_host_invocation_with_timeout( if let Some(sampling) = &sampling { sampling.start(invocation_started); } - let exchange = exchange_lifecycle_message( + let exchange = exchange_lifecycle_message_with_dispatch( &mut host_guard, &runtime.runtime_id, &lifecycle_request, invoke_timeout, + || reservation.mark_dispatched(), ); // Close the memory-only sampling window before reading the clock. No // OS collection or sample-file I/O runs under this registration lock. @@ -3192,7 +3538,7 @@ fn run_local_host_invocation_with_timeout( serde_json::to_value(&agent_result.extensions).map_err(FabricError::SerializeJson)?, ); } - Ok(RunResult { + let result = RunResult { session_ref: runtime.session_ref.clone(), agent_name: plan.agent_name.clone(), harness: harness(plan), @@ -3214,7 +3560,8 @@ fn run_local_host_invocation_with_timeout( .collect(), events, metadata, - }) + }; + Ok(result) } fn invalidate_timed_out_local_host( @@ -3546,11 +3893,8 @@ fn local_services() .unwrap_or_else(|error| error.into_inner()) } -pub(crate) fn local_host_directory(id: &str) -> PathBuf { - std::env::temp_dir() - .join("nemo-fabric") - .join("hosts") - .join(id) +pub(crate) fn local_host_directory(id: &str) -> Result { + Ok(default_artifact_root()?.join("hosts").join(id)) } fn spawn_local_host( @@ -3573,11 +3917,7 @@ fn spawn_local_host( }; // Lock acquisition creates retained storage, allowing its physical path to resolve. let scope = cleanup_scope(plan, state_dir.as_deref(), runtime.service_id.as_deref()); - let runtime_dir = local_host_directory(&runtime.runtime_id); - std::fs::create_dir_all(&runtime_dir).map_err(|source| FabricError::Write { - path: runtime_dir.clone(), - source, - })?; + let runtime_dir = results::host_directory(&default_artifact_root()?, &runtime.runtime_id)?; let stderr_path = runtime_dir.join("host.stderr.log"); let stderr = File::create(&stderr_path).map_err(|source| FabricError::Write { path: stderr_path.clone(), @@ -3816,6 +4156,16 @@ fn exchange_lifecycle_message( runtime_id: &str, request: &AdapterLifecycleRequest, timeout: Duration, +) -> Result { + exchange_lifecycle_message_with_dispatch(host, runtime_id, request, timeout, || {}) +} + +fn exchange_lifecycle_message_with_dispatch( + host: &mut LocalAdapterHost, + runtime_id: &str, + request: &AdapterLifecycleRequest, + timeout: Duration, + before_dispatch: impl FnOnce(), ) -> Result { let operation = request.operation(); if let Some(status) = host.child.try_wait().map_err(|source| { @@ -3840,6 +4190,9 @@ fn exchange_lifecycle_message( } let mut message = serde_json::to_string(request).map_err(FabricError::SerializeJson)?; message.push('\n'); + // A write can fail after partial delivery, so retain the reservation from + // the first possible dispatch rather than treating a write error as safe. + before_dispatch(); if let Err(source) = host .stdin .write_all(message.as_bytes()) @@ -5045,7 +5398,7 @@ fn artifact_manifest(plan: &RunPlan) -> Result { .and_then(|environment| environment.artifacts.clone()) }); if let Some(root) = &root { - std::fs::create_dir_all(root).map_err(|source| FabricError::Write { + results::create_artifact_root(root).map_err(|source| FabricError::Write { path: root.clone(), source, })?; @@ -5064,16 +5417,8 @@ fn prepare_fabric_home( let root = manifest .root .clone() - .unwrap_or_else(|| std::env::temp_dir().join("nemo-fabric")); - let fabric_home = root - .join(".fabric") - .join(&runtime.runtime_id) - .join(&invocation.invocation_id); - std::fs::create_dir_all(&fabric_home).map_err(|source| FabricError::Write { - path: fabric_home.clone(), - source, - })?; - Ok(fabric_home) + .map_or_else(default_artifact_root, Ok)?; + results::invocation_directory(&root, &runtime.runtime_id, &invocation.invocation_id) } fn write_fabric_invocation(fabric_home: &Path, payload: &str) -> Result { @@ -5676,6 +6021,8 @@ for line in sys.stdin: invocations += 1 if MODE == "invoke_stderr": print(f"diagnostic-{invocations}", file=sys.stderr, flush=True) + if MODE == "invoke_crash": + os._exit(17) if MODE == "invoke_timeout": print("invoke accepted without response", file=sys.stderr, flush=True) continue @@ -5819,6 +6166,9 @@ for line in sys.stdin: shutil.rmtree(directory) with open(directory, "w", encoding="utf-8") as stream: stream.write("cleanup requires a retry") + root = os.path.dirname(os.environ["FABRIC_FAKE_HOST_STOP_MARKER"]) + if os.path.exists(os.path.join(root, "fail-result-storage")): + os.chmod(os.path.join(root, "artifacts", ".fabric"), 0o777) if MODE == "stop_delay": with open(os.environ["FABRIC_FAKE_HOST_STOP_MARKER"], "w", encoding="utf-8") as stream: stream.write("started") @@ -6134,7 +6484,7 @@ else: }; assert_eq!(pending.runtime(), &runtime); assert!(local_hosts().contains_key(&runtime.runtime_id)); - assert!(local_host_directory(&runtime.runtime_id).exists()); + assert!(local_host_directory(&runtime.runtime_id).unwrap().exists()); let denied = invoke_runtime(&plan, &runtime, RunRequest::text("must not invoke")); stop_runtime(&plan, pending.runtime()).unwrap(); #[cfg(unix)] @@ -6166,7 +6516,7 @@ else: matches!(release, Err(FabricError::AdapterLifecycleOperation {code, ..}) if code == "cleanup_unconfirmed") ); assert!(local_hosts().contains_key(&context.runtime().runtime_id)); - let directory = local_host_directory(&context.runtime().runtime_id); + let directory = local_host_directory(&context.runtime().runtime_id).unwrap(); assert!(directory.is_file()); fs::remove_file(&directory).unwrap(); fs::create_dir(&directory).unwrap(); @@ -6346,9 +6696,19 @@ else: #[cfg(unix)] #[test] fn one_shot_cleanup_failures_keep_a_safe_retry_handle() { - for mode in ["cleanup_failure", "invoke_failure_cleanup_failure"] { + use std::os::unix::fs::PermissionsExt; + for (mode, storage_fails) in [ + ("cleanup_failure", true), + ("invoke_failure_cleanup_failure", true), + ("cleanup_failure", false), + ("invoke_failure_cleanup_failure", false), + ] { let (root, plan) = local_host_plan(mode); - let error = run_plan(&plan, RunRequest::text("TEST_ONLY_PRIVATE_RESULT")) + if storage_fails { + fs::write(root.join("fail-result-storage"), "").unwrap(); + } + let request = RunRequest::text("TEST_ONLY_PRIVATE_RESULT"); + let error = run_plan(&plan, request.clone()) .expect_err("unconfirmed cleanup must retain a retry handle"); let debug = format!("{error:?}"); let FabricError::RuntimeCleanupPending(pending) = error else { @@ -6356,13 +6716,40 @@ else: }; assert_eq!(pending.result().is_some(), mode == "cleanup_failure"); let runtime = pending.runtime(); - let directory = local_host_directory(&runtime.runtime_id); + let directory = local_host_directory(&runtime.runtime_id).unwrap(); fs::remove_file(&directory).unwrap(); fs::create_dir_all(&directory).unwrap(); stop_runtime(&plan, runtime).expect("retry cleanup with the original handle"); assert!(!local_hosts().contains_key(&runtime.runtime_id)); assert!(!debug.contains("TEST_ONLY_PRIVATE_RESULT")); assert!(!debug.contains(&runtime.runtime_binding)); + if storage_fails { + assert!( + matches!(pending.cause(), FabricError::AdapterLifecycleOperation { code, .. } if code == "result_storage_failed") + ); + fs::set_permissions( + root.join("artifacts/.fabric"), + fs::Permissions::from_mode(0o700), + ) + .unwrap(); + } + let saved = invocation_result( + &root.join("artifacts"), + &plan.agent_name, + &runtime.runtime_id, + &request.request_id, + ) + .unwrap(); + assert_eq!( + saved.is_some(), + !storage_fails, + "publish only finalized results" + ); + if !storage_fails && let Some(result) = pending.result() { + assert_eq!(saved.as_ref().unwrap(), result); + assert_eq!(result.status, RunStatus::Failed); + assert_eq!(result.error.as_ref().unwrap().code, "cleanup_unconfirmed"); + } fs::remove_dir_all(root).unwrap(); } } @@ -6614,6 +7001,578 @@ else: fs::remove_dir_all(root).unwrap(); } + #[test] + fn completed_invocation_persists_the_full_result() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).expect("start"); + let result = invoke_runtime(&plan, &runtime, RunRequest::text("hello")).expect("invoke"); + let path = + PathBuf::from(result.metadata["fabric_home"].as_str().unwrap()).join("run-result.json"); + let saved: RunResult = + serde_json::from_slice(&fs::read(path).expect("saved result")).expect("valid result"); + assert_eq!(saved, result); + stop_runtime(&plan, &runtime).expect("stop"); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn invalid_overrides_do_not_reserve_a_request() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let mut request = RunRequest::text("hello"); + request.request_id = "validation-retry".into(); + request.overrides = Some(serde_json::json!([])); + assert!(invoke_runtime(&plan, &runtime, request.clone()).is_err()); + request.overrides = None; + let retried = invoke_runtime(&plan, &runtime, request); + stop_runtime(&plan, &runtime).unwrap(); + assert_eq!(retried.unwrap().output["invocation_count"], 1); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn pre_dispatch_host_failure_does_not_block_a_corrected_retry() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let host = local_hosts().remove(&runtime.runtime_id).unwrap(); + let mut request = RunRequest::text("original"); + request.request_id = "not-dispatched".into(); + let rejected = invoke_runtime(&plan, &runtime, request.clone()); + local_hosts().insert(runtime.runtime_id.clone(), host); + assert!(rejected.is_err()); + request.input = serde_json::json!("corrected"); + let retried = invoke_runtime(&plan, &runtime, request); + stop_runtime(&plan, &runtime).unwrap(); + assert_eq!(retried.unwrap().output["invocation_count"], 1); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn invalid_request_ids_are_rejected_before_invocation() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + for id in ["", " ", "\t\n", "request\0id"] { + let mut request = RunRequest::text("hello"); + request.request_id = id.into(); + assert!(matches!(invoke_runtime(&plan, &runtime, request), + Err(FabricError::InvalidConfig { field, .. }) if field == "request_id")); + assert!(matches!( + invocation_result(&root.join("artifacts"), &runtime.agent_name, &runtime.runtime_id, id), + Err(FabricError::AdapterLifecycleOperation { code, .. }) if code == "invalid_result_target" + )); + } + let result = invoke_runtime(&plan, &runtime, RunRequest::text("valid")).unwrap(); + stop_runtime(&plan, &runtime).unwrap(); + assert_eq!(result.output["invocation_count"], 1); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn default_request_identity_is_known_before_invocation() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let request = RunRequest::default(); + let request_id = request.request_id.clone(); + let result = invoke_runtime(&plan, &runtime, request).unwrap(); + stop_runtime(&plan, &runtime).unwrap(); + assert!(!request_id.is_empty()); + assert_eq!(result.request_id, request_id); + assert_eq!( + invocation_result( + &root.join("artifacts"), + &result.agent_name, + &runtime.runtime_id, + &request_id + ) + .unwrap(), + Some(result) + ); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn one_shot_result_is_unavailable_until_cleanup_is_finalized() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let request = RunRequest::text("hello"); + begin_run_result(&plan, &runtime, &request).unwrap(); + let mut result = invoke_runtime(&plan, &runtime, request.clone()).unwrap(); + assert!( + invocation_result( + &root.join("artifacts"), + &result.agent_name, + &runtime.runtime_id, + &request.request_id + ) + .unwrap() + .is_none() + ); + assert!( + matches!(invoke_runtime(&plan, &runtime, request), Err(FabricError::AdapterLifecycleOperation {code, ..}) if code == "invocation_pending") + ); + result.events.extend(stop_runtime(&plan, &runtime).unwrap()); + save_run_result(&plan, &runtime, &result).unwrap(); + assert_eq!( + invocation_result( + &root.join("artifacts"), + &result.agent_name, + &runtime.runtime_id, + &result.request_id + ) + .unwrap(), + Some(result) + ); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn failed_final_save_cannot_replay_a_pre_cleanup_success() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let request = RunRequest::text("hello"); + begin_run_result(&plan, &runtime, &request).unwrap(); + let mut result = invoke_runtime(&plan, &runtime, request.clone()).unwrap(); + stop_runtime(&plan, &runtime).unwrap(); + result.status = RunStatus::Failed; + result.output = Value::String("x".repeat(4 * 1024 * 1024)); + assert!(save_run_result(&plan, &runtime, &result).is_err()); + assert!( + invocation_result( + &root.join("artifacts"), + &result.agent_name, + &runtime.runtime_id, + &result.request_id + ) + .unwrap() + .is_none() + ); + assert!( + matches!(invoke_runtime(&plan, &runtime, request), Err(FabricError::AdapterLifecycleOperation {code, ..}) if code == "invocation_pending") + ); + fs::remove_dir_all(root).unwrap(); + } + + #[cfg(unix)] + #[test] + fn default_artifact_storage_requires_a_protected_user_home() { + use std::os::unix::fs::{DirBuilderExt, PermissionsExt}; + let home = std::env::temp_dir().join(new_id("fabric-private-home-test")); + fs::DirBuilder::new().mode(0o700).create(&home).unwrap(); + // SAFETY: geteuid has no preconditions or side effects. + let uid = unsafe { libc::geteuid() }; + let root = default_artifact_root_for_user(&home, uid).unwrap(); + assert!(root.starts_with(fs::canonicalize(&home).unwrap())); + assert!(default_artifact_root_for_user(&home, uid.wrapping_add(1)).is_err()); + fs::set_permissions(&home, fs::Permissions::from_mode(0o777)).unwrap(); + let unsafe_home = default_artifact_root_for_user(&home, uid); + fs::remove_dir_all(home).unwrap(); + let fallback = unsafe_home.expect("shared home uses private temporary storage"); + assert_eq!( + fallback, + std::env::temp_dir().join(format!("nemo-fabric-{uid}-private")) + ); + assert_eq!( + fs::metadata(&fallback).unwrap().permissions().mode() & 0o777, + 0o700 + ); + } + + #[cfg(unix)] + #[test] + fn temporary_storage_rejects_unsafe_precreated_namespaces() { + use std::os::unix::fs::{PermissionsExt, symlink}; + let temporary = std::env::temp_dir().join(new_id("fabric-fallback-test")); + fs::create_dir(&temporary).unwrap(); + // SAFETY: geteuid has no preconditions or side effects. + let uid = unsafe { libc::geteuid() }; + let root = temporary.join(format!("nemo-fabric-{uid}-private")); + fs::create_dir(&root).unwrap(); + fs::set_permissions(&root, fs::Permissions::from_mode(0o777)).unwrap(); + assert!(temporary_artifact_root_for_user(&temporary, uid).is_err()); + fs::remove_dir(&root).unwrap(); + symlink(&temporary, &root).unwrap(); + assert!(temporary_artifact_root_for_user(&temporary, uid).is_err()); + fs::remove_file(&root).unwrap(); + fs::remove_dir(&temporary).unwrap(); + } + + #[test] + fn result_lookup_preserves_rejection_codes_without_uncertain_effects() { + for (effects, expected) in [("none", "wrong_agent"), ("unknown", "invalid_response")] { + let response = serde_json::from_value(serde_json::json!({ + "status": "rejected", "error": {"code": "wrong_agent", "stage": "result", "effects": effects} + })).unwrap(); + assert!( + matches!(result_lookup_response(response), Err(FabricError::EnvironmentProviderOperation {operation, code, ..}) if operation == "invocation_result" && code == expected) + ); + } + } + + #[test] + fn dispatched_failures_have_terminal_results_without_reexecution() { + for (mode, expected) in [ + ("invoke_timeout", "host_timeout"), + ("invoke_crash", "host_crashed"), + ("blank_error_code", "invalid_agent_run_result"), + ("invoke_failure", "invocation_failed"), + ] { + let (root, plan) = local_host_plan(mode); + let runtime = start_runtime(&plan).unwrap(); + let request = RunRequest::text("hello"); + let error = run_local_host_adapter_with_timeout( + &plan, + &runtime, + request.clone(), + Duration::from_millis(100), + ) + .unwrap_err(); + assert!(!error.to_string().is_empty()); + let saved = invocation_result( + &root.join("artifacts"), + &runtime.agent_name, + &runtime.runtime_id, + &request.request_id, + ) + .unwrap(); + stop_runtime(&plan, &runtime).unwrap(); + let saved = saved.expect("dispatched failure has a terminal result"); + assert_eq!(saved.status, RunStatus::Failed); + assert!(saved.output.is_null()); + assert_eq!(saved.error.as_ref().unwrap().code, expected); + assert!(!saved.error.as_ref().unwrap().retryable); + assert_eq!(saved.error.as_ref().unwrap().metadata["effects"], "unknown"); + if expected == "host_timeout" { + let metadata = &saved.error.as_ref().unwrap().metadata; + assert_eq!(metadata["owner"], "fabric_runtime"); + assert_eq!(metadata["timeout_seconds"], 0.1); + assert_eq!(metadata["cause_code"], "host_timeout"); + } + assert_eq!(invoke_runtime(&plan, &runtime, request).unwrap(), saved); + fs::remove_dir_all(root).unwrap(); + } + } + + #[test] + fn one_shot_dispatched_failure_is_finalized_after_cleanup() { + let (root, plan) = local_host_plan("blank_error_code"); + let request = RunRequest::text("hello"); + run_plan(&plan, request.clone()).unwrap_err(); + let runtimes = fs::read_dir(root.join("artifacts/.fabric")).unwrap(); + let runtime = runtimes + .map(|entry| entry.unwrap().file_name().into_string().unwrap()) + .find(|name| name.starts_with("runtime-")) + .unwrap(); + let saved = invocation_result( + &root.join("artifacts"), + &plan.agent_name, + &runtime, + &request.request_id, + ) + .unwrap() + .expect("one-shot failure must be retrievable"); + assert_eq!(saved.status, RunStatus::Failed); + assert!( + saved + .events + .iter() + .any(|event| event.kind == "runtime_stop") + ); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn one_shot_lookup_returns_the_final_result_after_stop() { + for mode in ["success", "stop_failure"] { + let (root, plan) = local_host_plan(mode); + let result = run_plan(&plan, RunRequest::text("hello")).unwrap(); + let saved = invocation_result( + &root.join("artifacts"), + &result.agent_name, + &result.runtime_id, + &result.request_id, + ) + .unwrap(); + assert_eq!(saved, Some(result)); + fs::remove_dir_all(root).unwrap(); + } + } + + #[test] + fn final_result_save_requires_original_runtime_and_reserved_identity() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let result = invoke_runtime(&plan, &runtime, RunRequest::text("hello")).unwrap(); + stop_runtime(&plan, &runtime).unwrap(); + let mut wrong_runtime = runtime.clone(); + wrong_runtime.runtime_binding = "wrong-binding".into(); + assert!(save_run_result(&plan, &wrong_runtime, &result).is_err()); + for field in ["agent", "runtime", "request", "invocation"] { + let mut wrong_result = result.clone(); + match field { + "agent" => wrong_result.agent_name.push_str("-other"), + "runtime" => wrong_result.runtime_id.push_str("-other"), + "request" => wrong_result.request_id.push_str("-other"), + _ => wrong_result.invocation_id = "../../escape".into(), + } + assert!(save_run_result(&plan, &runtime, &wrong_result).is_err()); + } + assert_eq!( + invocation_result( + &root.join("artifacts"), + &result.agent_name, + &result.runtime_id, + &result.request_id + ) + .unwrap(), + Some(result) + ); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn oversized_results_do_not_replace_a_readable_saved_result() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let result = invoke_runtime(&plan, &runtime, RunRequest::text("hello")).unwrap(); + let mut oversized = result.clone(); + oversized.output = Value::String("x".repeat(4 * 1024 * 1024)); + let saved = results::save(&root.join("artifacts"), &oversized); + stop_runtime(&plan, &runtime).unwrap(); + assert!(saved.is_err()); + assert_eq!( + invocation_result( + &root.join("artifacts"), + &result.agent_name, + &result.runtime_id, + &result.request_id, + ) + .unwrap(), + Some(result) + ); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn buffered_results_cannot_satisfy_a_stream_request() { + let (root, mut plan) = local_host_plan("success"); + plan.capabilities.streaming = true; + plan.adapter_descriptor + .as_mut() + .unwrap() + .descriptor + .capabilities + .streaming = true; + let runtime = start_runtime(&plan).unwrap(); + let request = RunRequest::text("hello"); + let result = invoke_runtime(&plan, &runtime, request.clone()).unwrap(); + let streamed = invoke_openai_stream( + &plan, + &runtime, + request, + OpenAiStreamTransport { + port: 1, + token: "test-stream-token".into(), + }, + ); + stop_runtime(&plan, &runtime).unwrap(); + assert!( + matches!(streamed, Err(FabricError::AdapterLifecycleOperation { code, .. }) + if code == "request_conflict") + ); + assert_eq!( + invocation_result( + &root.join("artifacts"), + &result.agent_name, + &result.runtime_id, + &result.request_id, + ) + .unwrap(), + Some(result) + ); + fs::remove_dir_all(root).unwrap(); + } + + #[cfg(unix)] + #[test] + fn explicit_shared_artifact_roots_allow_private_results() { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + for mode in [0o775, 0o777] { + let (root, plan) = local_host_plan("success"); + fs::set_permissions(&root, fs::Permissions::from_mode(mode)).unwrap(); + let artifacts = root.join("artifacts"); + fs::create_dir_all(&artifacts).unwrap(); + fs::set_permissions(&artifacts, fs::Permissions::from_mode(mode)).unwrap(); + let runtime = start_runtime(&plan).unwrap(); + let result = invoke_runtime(&plan, &runtime, RunRequest::text("hello")).unwrap(); + stop_runtime(&plan, &runtime).unwrap(); + assert_eq!( + invocation_result( + &artifacts, + &result.agent_name, + &runtime.runtime_id, + &result.request_id + ) + .unwrap(), + Some(result) + ); + assert_eq!( + fs::metadata(artifacts.join(".fabric")).unwrap().mode() & 0o777, + 0o700 + ); + fs::remove_dir_all(root).unwrap(); + } + } + + #[cfg(unix)] + #[test] + fn result_lookup_rejects_an_unsafe_existing_namespace() { + use std::os::unix::fs::PermissionsExt; + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let result = invoke_runtime(&plan, &runtime, RunRequest::text("hello")).unwrap(); + let namespace = root.join("artifacts/.fabric"); + fs::set_permissions(&namespace, fs::Permissions::from_mode(0o777)).unwrap(); + let lookup = invocation_result( + &root.join("artifacts"), + &result.agent_name, + &result.runtime_id, + &result.request_id, + ); + fs::set_permissions(&namespace, fs::Permissions::from_mode(0o700)).unwrap(); + stop_runtime(&plan, &runtime).unwrap(); + assert!(lookup.is_err()); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn result_lookup_and_duplicate_request_preserve_one_invocation() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).expect("start"); + let mut request = RunRequest::text("hello"); + request.request_id = "same-request".into(); + let first = invoke_runtime(&plan, &runtime, request.clone()).expect("invoke"); + let second = invoke_runtime(&plan, &runtime, request.clone()).expect("retry"); + assert_eq!(first, second); + let artifacts = root.join("artifacts"); + assert_eq!( + invocation_result( + &artifacts, + &plan.agent_name, + &runtime.runtime_id, + "same-request" + ) + .unwrap(), + Some(first.clone()) + ); + request.input = serde_json::json!("different input"); + assert!(invoke_runtime(&plan, &runtime, request).is_err()); + stop_runtime(&plan, &runtime).expect("stop"); + assert_eq!( + invocation_result( + &artifacts, + &plan.agent_name, + &runtime.runtime_id, + "same-request" + ) + .unwrap(), + Some(first) + ); + assert!( + invocation_result( + &artifacts, + "wrong-agent", + &runtime.runtime_id, + "same-request" + ) + .is_err() + ); + assert!( + invocation_result(&artifacts, &plan.agent_name, "../../escape", "same-request") + .is_err() + ); + fs::remove_dir_all(root).unwrap(); + } + + #[cfg(unix)] + #[test] + fn result_lookup_rejects_symlinked_result_files() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).expect("start"); + let mut request = RunRequest::text("hello"); + request.request_id = "private-request".into(); + let result = invoke_runtime(&plan, &runtime, request).expect("invoke"); + let path = + PathBuf::from(result.metadata["fabric_home"].as_str().unwrap()).join("run-result.json"); + let outside = root.join("outside-result.json"); + fs::rename(&path, &outside).unwrap(); + std::os::unix::fs::symlink(&outside, &path).unwrap(); + assert!( + invocation_result( + &root.join("artifacts"), + &plan.agent_name, + &runtime.runtime_id, + "private-request" + ) + .is_err() + ); + stop_runtime(&plan, &runtime).expect("stop"); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn duplicate_pending_request_is_not_submitted_again() { + let (root, plan) = local_host_plan("invoke_timeout"); + let runtime = start_runtime(&plan).unwrap(); + let stderr = local_hosts() + .get(&runtime.runtime_id) + .unwrap() + .lock() + .unwrap() + .stderr_path + .clone(); + let mut request = RunRequest::text("slow"); + request.request_id = "pending-request".into(); + let worker_plan = plan.clone(); + let worker_runtime = runtime.clone(); + let worker_request = request.clone(); + let worker = thread::spawn(move || { + run_local_host_adapter_with_timeout( + &worker_plan, + &worker_runtime, + worker_request, + Duration::from_millis(500), + ) + }); + let deadline = Instant::now() + Duration::from_secs(3); + while !fs::read_to_string(&stderr) + .unwrap() + .contains("invoke accepted without response") + { + assert!(Instant::now() < deadline); + thread::sleep(Duration::from_millis(10)); + } + let error = + invoke_runtime(&plan, &runtime, request).expect_err("pending request must not replay"); + assert!( + matches!(error, FabricError::AdapterLifecycleOperation { code, .. } if code == "invocation_pending") + ); + worker + .join() + .unwrap() + .expect_err("original invocation times out"); + let terminal = invocation_result( + &root.join("artifacts"), + &plan.agent_name, + &runtime.runtime_id, + "pending-request", + ) + .unwrap() + .expect("timed-out invocation has a terminal result"); + assert_eq!(terminal.error.unwrap().code, "host_timeout"); + stop_runtime(&plan, &runtime).unwrap(); + fs::remove_dir_all(root).unwrap(); + } + #[test] fn local_host_reuses_one_process_and_stops_idempotently() { let (root, plan) = local_host_plan("success"); @@ -6863,7 +7822,8 @@ else: let result = pending .result() .expect("completed invocation remains available"); - assert_eq!(result.status, RunStatus::Succeeded); + assert_eq!(result.status, RunStatus::Failed); + assert_eq!(result.error.as_ref().unwrap().stage, ErrorStage::Stop); assert_eq!(result.runtime_id, pending.runtime().runtime_id); assert!( matches!(pending.cause(), FabricError::AdapterLifecycleOperation {operation: "stop", code, ..} if code == "cleanup_unconfirmed") @@ -7677,7 +8637,8 @@ else: let runtime = start_runtime(&plan).expect("start local host"); let (transport, capture) = openai_stream_listener("stream-secret"); - let result = invoke_openai_stream(&plan, &runtime, RunRequest::text("stream"), transport) + let request = RunRequest::text("stream"); + let result = invoke_openai_stream(&plan, &runtime, request.clone(), transport.clone()) .expect("stream invocation"); let records = capture.join().expect("stream capture"); @@ -7719,6 +8680,14 @@ else: assert_eq!(persisted["stream"]["invocation_id"], result.invocation_id); assert_eq!(persisted["stream"]["request_id"], result.request_id); + let replayed = invoke_openai_stream(&plan, &runtime, request.clone(), transport); + assert!( + matches!(replayed, Err(FabricError::AdapterLifecycleOperation { code, .. }) + if code == "stream_replay_unsupported") + ); + assert!(matches!(invoke_runtime(&plan, &runtime, request), + Err(FabricError::AdapterLifecycleOperation { code, .. }) + if code == "request_conflict")); stop_runtime(&plan, &runtime).expect("stop local host"); let _ = fs::remove_dir_all(root); } @@ -9298,7 +10267,9 @@ else: .unwrap() .clone(); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let endpoint = local_host_directory(&runtime.runtime_id).join("health.endpoint.json"); + let endpoint = local_host_directory(&runtime.runtime_id) + .unwrap() + .join("health.endpoint.json"); fs::write( &endpoint, serde_json::to_vec(&serde_json::json!({ diff --git a/crates/fabric-core/src/runtime/results.rs b/crates/fabric-core/src/runtime/results.rs new file mode 100644 index 000000000..d0c4ac262 --- /dev/null +++ b/crates/fabric-core/src/runtime/results.rs @@ -0,0 +1,499 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Private, per-runtime request reservations and completed invocation results. + +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use super::AdapterLifecycleOperation; +use super::{InvocationHandle, RunRequest, RunResult, RuntimeHandle, lifecycle_error}; +use crate::Result; + +const MAX_RESULT_BYTES: u64 = 4 * 1024 * 1024; + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Reservation { + agent_name: String, + runtime_id: String, + request_id: String, + invocation_id: String, + fingerprint: String, + #[serde(default)] + streaming: Option, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Finalization { + agent_name: String, + runtime_id: String, + request_id: String, +} + +pub(super) enum RequestReservation { + Cached(Box), + Pending(PendingReservation), +} + +pub(super) struct PendingReservation { + path: PathBuf, + dispatched: bool, +} + +impl PendingReservation { + pub(super) fn dispatched(&self) -> bool { + self.dispatched + } + + pub(super) fn mark_dispatched(&mut self) { + self.dispatched = true; + } +} + +impl Drop for PendingReservation { + fn drop(&mut self) { + if !self.dispatched { + // This guard owns the file created with create_new. Other requests + // cannot acquire this request ID until the file has been removed. + let _ = fs::remove_file(&self.path); + } + } +} + +fn failure(runtime_id: &str, code: &str) -> crate::FabricError { + lifecycle_error( + AdapterLifecycleOperation::Invoke, + runtime_id, + code, + "The invocation record is unavailable or conflicts with this request; do not replay uncertain work.", + "", + ) +} + +fn leaf(value: &str) -> bool { + !value.is_empty() + && value + .bytes() + .all(|c| c.is_ascii_alphanumeric() || c == b'-' || c == b'_') +} + +pub(super) fn trusted_ancestry(root: &Path, runtime_id: &str) -> Result<()> { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + // SAFETY: geteuid has no preconditions or side effects. + let uid = unsafe { libc::geteuid() }; + for ancestor in root.ancestors() { + let metadata = fs::symlink_metadata(ancestor) + .map_err(|_| failure(runtime_id, "result_storage_failed"))?; + if !metadata.is_dir() + || !matches!(metadata.uid(), owner if owner == uid || owner == 0) + || (metadata.mode() & 0o022 != 0 && metadata.mode() & 0o1000 == 0) + { + return Err(failure(runtime_id, "result_storage_failed")); + } + } + } + #[cfg(not(unix))] + let _ = (root, runtime_id); + Ok(()) +} + +fn owned_metadata(metadata: &fs::Metadata, file: bool) -> bool { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + // SAFETY: geteuid has no preconditions or side effects. + let forbidden = if file { 0o077 } else { 0o022 }; + metadata.uid() == unsafe { libc::geteuid() } && metadata.mode() & forbidden == 0 + } + #[cfg(not(unix))] + { + let _ = (metadata, file); + true + } +} + +pub(super) fn create_artifact_root(root: &Path) -> std::io::Result<()> { + let mut builder = fs::DirBuilder::new(); + builder.recursive(true); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder.create(root) +} + +fn directory(parent: &Path, name: &str, create: bool, runtime_id: &str) -> Result { + let path = parent.join(name); + if create { + let builder = fs::DirBuilder::new(); + #[cfg(unix)] + let builder = { + use std::os::unix::fs::DirBuilderExt; + let mut builder = builder; + builder.mode(0o700); + builder + }; + if let Err(error) = builder.create(&path) + && error.kind() != std::io::ErrorKind::AlreadyExists + { + return Err(failure(runtime_id, "result_storage_failed")); + } + } + let metadata = + fs::symlink_metadata(&path).map_err(|_| failure(runtime_id, "result_storage_failed"))?; + if !metadata.is_dir() || metadata.file_type().is_symlink() || !owned_metadata(&metadata, false) + { + return Err(failure(runtime_id, "result_storage_failed")); + } + Ok(path) +} + +fn runtime_directory(root: &Path, runtime_id: &str, create: bool) -> Result { + if !leaf(runtime_id) { + return Err(failure(runtime_id, "invalid_result_target")); + } + if create { + create_artifact_root(root).map_err(|_| failure(runtime_id, "result_storage_failed"))?; + } + let root = fs::canonicalize(root).map_err(|_| failure(runtime_id, "result_storage_failed"))?; + let fabric = directory(&root, ".fabric", create, runtime_id)?; + directory(&fabric, runtime_id, create, runtime_id) +} + +pub(super) fn host_directory(root: &Path, runtime_id: &str) -> Result { + if !leaf(runtime_id) { + return Err(failure(runtime_id, "invalid_result_target")); + } + create_artifact_root(root).map_err(|_| failure(runtime_id, "result_storage_failed"))?; + let root = fs::canonicalize(root).map_err(|_| failure(runtime_id, "result_storage_failed"))?; + trusted_ancestry(&root, runtime_id)?; + let hosts = directory(&root, "hosts", true, runtime_id)?; + directory(&hosts, runtime_id, true, runtime_id) +} + +pub(super) fn invocation_directory( + root: &Path, + runtime_id: &str, + invocation_id: &str, +) -> Result { + if !leaf(invocation_id) { + return Err(failure(runtime_id, "invalid_result_target")); + } + let runtime = runtime_directory(root, runtime_id, true)?; + directory(&runtime, invocation_id, true, runtime_id) +} + +fn options() -> OpenOptions { + let options = OpenOptions::new(); + #[cfg(unix)] + let options = { + use std::os::unix::fs::OpenOptionsExt; + let mut options = options; + options + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); + options + }; + options +} + +fn read(path: &Path, runtime_id: &str) -> Result> { + let file = match options().read(true).open(path) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(_) => return Err(failure(runtime_id, "result_storage_failed")), + }; + let metadata = file + .metadata() + .map_err(|_| failure(runtime_id, "result_storage_failed"))?; + if !metadata.is_file() || metadata.len() > MAX_RESULT_BYTES || !owned_metadata(&metadata, true) + { + return Err(failure(runtime_id, "result_storage_failed")); + } + let mut bytes = Vec::new(); + file.take(MAX_RESULT_BYTES + 1) + .read_to_end(&mut bytes) + .map_err(|_| failure(runtime_id, "result_storage_failed"))?; + if bytes.len() as u64 > MAX_RESULT_BYTES { + return Err(failure(runtime_id, "result_storage_failed")); + } + serde_json::from_slice(&bytes) + .map(Some) + .map_err(|_| failure(runtime_id, "result_storage_failed")) +} + +fn index_name(request_id: &str) -> String { + format!("{:x}.json", Sha256::digest(request_id.as_bytes())) +} + +fn finalization_path(requests: &Path, request_id: &str) -> PathBuf { + requests.join(format!("{}.finalizing", index_name(request_id))) +} + +pub(super) fn begin_finalization( + root: &Path, + agent: &str, + runtime_id: &str, + request_id: &str, +) -> Result<()> { + let runtime = runtime_directory(root, runtime_id, true)?; + let requests = directory(&runtime, "requests", true, runtime_id)?; + let path = finalization_path(&requests, request_id); + let marker = Finalization { + agent_name: agent.into(), + runtime_id: runtime_id.into(), + request_id: request_id.into(), + }; + let bytes = serde_json::to_vec(&marker).map_err(crate::FabricError::SerializeJson)?; + if bytes.len() as u64 > MAX_RESULT_BYTES { + return Err(failure(runtime_id, "result_storage_failed")); + } + match options().write(true).create_new(true).open(&path) { + Ok(mut file) => { + if file + .write_all(&bytes) + .and_then(|_| file.sync_all()) + .is_err() + { + let _ = fs::remove_file(path); + return Err(failure(runtime_id, "result_storage_failed")); + } + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + let existing: Finalization = read(&path, runtime_id)? + .ok_or_else(|| failure(runtime_id, "result_storage_failed"))?; + if existing.agent_name != agent + || existing.runtime_id != runtime_id + || existing.request_id != request_id + { + return Err(failure(runtime_id, "invalid_result_target")); + } + } + Err(_) => return Err(failure(runtime_id, "result_storage_failed")), + } + Ok(()) +} + +fn completed(runtime: &Path, record: &Reservation) -> Result> { + if let Some(marker) = read::( + &finalization_path(&runtime.join("requests"), &record.request_id), + &record.runtime_id, + )? { + if marker.agent_name != record.agent_name + || marker.runtime_id != record.runtime_id + || marker.request_id != record.request_id + { + return Err(failure(&record.runtime_id, "invalid_result_target")); + } + // Invocation output alone does not establish one-shot cleanup finality. + return Ok(None); + } + completed_result(runtime, record) +} + +fn completed_result(runtime: &Path, record: &Reservation) -> Result> { + if !leaf(&record.invocation_id) { + return Err(failure(&record.runtime_id, "invalid_result_target")); + } + let invocation = runtime.join(&record.invocation_id); + if !invocation.exists() { + return Ok(None); + } + let invocation = directory(runtime, &record.invocation_id, false, &record.runtime_id)?; + let result: Option = read(&invocation.join("run-result.json"), &record.runtime_id)?; + if let Some(result) = &result + && (result.agent_name != record.agent_name + || result.runtime_id != record.runtime_id + || result.request_id != record.request_id + || result.invocation_id != record.invocation_id) + { + return Err(failure(&record.runtime_id, "invalid_result_target")); + } + Ok(result) +} + +pub(super) fn reserve( + root: &Path, + runtime: &RuntimeHandle, + request: &RunRequest, + invocation: &InvocationHandle, + streaming: bool, +) -> Result { + let runtime_dir = runtime_directory(root, &runtime.runtime_id, true)?; + let requests = directory(&runtime_dir, "requests", true, &runtime.runtime_id)?; + let path = requests.join(index_name(&request.request_id)); + let record = Reservation { + agent_name: runtime.agent_name.clone(), + runtime_id: runtime.runtime_id.clone(), + request_id: request.request_id.clone(), + invocation_id: invocation.invocation_id.clone(), + fingerprint: format!( + "{:x}", + Sha256::digest(serde_json::to_vec(request).map_err(crate::FabricError::SerializeJson)?) + ), + streaming: Some(streaming), + }; + let bytes = serde_json::to_vec(&record).map_err(crate::FabricError::SerializeJson)?; + if bytes.len() as u64 > MAX_RESULT_BYTES { + return Err(failure(&runtime.runtime_id, "result_storage_failed")); + } + match options().write(true).create_new(true).open(&path) { + Ok(mut file) => { + let pending = PendingReservation { + path, + dispatched: false, + }; + file.write_all(&bytes) + .and_then(|_| file.sync_all()) + .map_err(|_| failure(&runtime.runtime_id, "result_storage_failed"))?; + Ok(RequestReservation::Pending(pending)) + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + let previous: Reservation = read(&path, &runtime.runtime_id)? + .ok_or_else(|| failure(&runtime.runtime_id, "result_storage_failed"))?; + if previous.agent_name != record.agent_name + || previous.runtime_id != record.runtime_id + || previous.request_id != record.request_id + || previous.fingerprint != record.fingerprint + || previous.streaming != record.streaming + { + return Err(failure(&runtime.runtime_id, "request_conflict")); + } + let result = completed(&runtime_dir, &previous)? + .ok_or_else(|| failure(&runtime.runtime_id, "invocation_pending"))?; + if streaming { + return Err(failure(&runtime.runtime_id, "stream_replay_unsupported")); + } + Ok(RequestReservation::Cached(Box::new(result))) + } + Err(_) => Err(failure(&runtime.runtime_id, "result_storage_failed")), + } +} + +pub(super) fn save(root: &Path, result: &RunResult) -> Result<()> { + let bytes = serde_json::to_vec(result).map_err(crate::FabricError::SerializeJson)?; + if bytes.len() as u64 > MAX_RESULT_BYTES { + return Err(failure(&result.runtime_id, "result_storage_failed")); + } + let invocation = invocation_directory(root, &result.runtime_id, &result.invocation_id)?; + let temporary = invocation.join(format!("{}.tmp", super::new_id("result"))); + let mut file: File = options() + .write(true) + .create_new(true) + .open(&temporary) + .map_err(|_| failure(&result.runtime_id, "result_storage_failed"))?; + file.write_all(&bytes) + .and_then(|_| file.sync_all()) + .and_then(|_| fs::rename(&temporary, invocation.join("run-result.json"))) + .map_err(|_| failure(&result.runtime_id, "result_storage_failed"))?; + Ok(()) +} + +pub(super) fn finalize(root: &Path, result: &RunResult) -> Result<()> { + let runtime = runtime_directory(root, &result.runtime_id, false)?; + let requests = directory(&runtime, "requests", false, &result.runtime_id)?; + let record: Reservation = read( + &requests.join(index_name(&result.request_id)), + &result.runtime_id, + )? + .ok_or_else(|| failure(&result.runtime_id, "invalid_result_target"))?; + if record.agent_name != result.agent_name + || record.runtime_id != result.runtime_id + || record.request_id != result.request_id + || record.invocation_id != result.invocation_id + { + return Err(failure(&result.runtime_id, "invalid_result_target")); + } + completed_result(&runtime, &record)? + .ok_or_else(|| failure(&result.runtime_id, "invalid_result_target"))?; + begin_finalization( + root, + &result.agent_name, + &result.runtime_id, + &result.request_id, + )?; + save(root, result)?; + // Publish cleanup finality only after the replacement result is saved. + fs::remove_file(finalization_path(&requests, &result.request_id)) + .map_err(|_| failure(&result.runtime_id, "result_storage_failed")) +} + +/// Read a saved result. Absence does not establish whether the request ran. +#[doc(hidden)] +pub fn invocation_result( + root: &Path, + agent: &str, + runtime_id: &str, + request_id: &str, +) -> Result> { + lookup(root, agent, runtime_id, request_id, true) +} + +pub(super) fn unfinalized_result( + root: &Path, + agent: &str, + runtime_id: &str, + request_id: &str, +) -> Result> { + lookup(root, agent, runtime_id, request_id, false) +} + +fn lookup( + root: &Path, + agent: &str, + runtime_id: &str, + request_id: &str, + finalized: bool, +) -> Result> { + if [request_id, agent] + .iter() + .any(|value| value.trim().is_empty() || value.contains('\0')) + { + return Err(failure(runtime_id, "invalid_result_target")); + } + if !leaf(runtime_id) { + return Err(failure(runtime_id, "invalid_result_target")); + } + let root = match fs::canonicalize(root) { + Ok(root) => root, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(_) => return Err(failure(runtime_id, "result_storage_failed")), + }; + let fabric = root.join(".fabric"); + let runtime = fabric.join(runtime_id); + let requests = runtime.join("requests"); + for path in [&fabric, &runtime, &requests] { + match fs::symlink_metadata(path) { + Ok(metadata) + if metadata.is_dir() + && !metadata.file_type().is_symlink() + && owned_metadata(&metadata, false) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + _ => return Err(failure(runtime_id, "result_storage_failed")), + } + } + let record: Option = read(&requests.join(index_name(request_id)), runtime_id)?; + let Some(record) = record else { + return Ok(None); + }; + if record.agent_name != agent + || record.runtime_id != runtime_id + || record.request_id != request_id + { + return Err(failure(runtime_id, "invalid_result_target")); + } + if finalized { + completed(&runtime, &record) + } else { + completed_result(&runtime, &record) + } +} diff --git a/crates/fabric-core/src/task_command.rs b/crates/fabric-core/src/task_command.rs index 2919d45d9..4ee21f380 100644 --- a/crates/fabric-core/src/task_command.rs +++ b/crates/fabric-core/src/task_command.rs @@ -38,6 +38,11 @@ pub(crate) enum TaskCommand { agent: String, runtime_id: String, }, + Result { + agent: String, + runtime_id: String, + request_id: String, + }, Stop { agent: String, runtime_id: String, diff --git a/crates/fabric-python/src/lib.rs b/crates/fabric-python/src/lib.rs index 975ba7825..21d5dd802 100644 --- a/crates/fabric-python/src/lib.rs +++ b/crates/fabric-python/src/lib.rs @@ -284,6 +284,121 @@ fn check_environment( to_json(&report) } +/// Read a saved result from the configured artifact root. +#[pyfunction] +fn invocation_result( + py: Python<'_>, + artifacts: Option, + agent: String, + runtime_id: String, + request_id: String, +) -> PyResult { + let root = artifacts + .map(PathBuf::from) + .map_or_else(nemo_fabric_core::runtime::default_artifact_root, Ok) + .map_err(to_py_error)?; + let result = py + .detach(|| { + nemo_fabric_core::runtime::invocation_result(&root, &agent, &runtime_id, &request_id) + }) + .map_err(|_| { + PyRuntimeError::new_err( + "The invocation result is unavailable; do not replay uncertain work.", + ) + })?; + to_json(&result) +} + +/// Mark a local one-shot result as awaiting lifecycle finalization. +#[pyfunction] +fn _begin_run_result( + py: Python<'_>, + plan_json: String, + runtime_json: String, + request_json: String, +) -> PyResult<()> { + let plan = parse_run_plan(plan_json)?; + let runtime = parse_runtime_handle(runtime_json)?; + let request: RunRequest = serde_json::from_str(&request_json).map_err(|_| { + pyo3::exceptions::PyValueError::new_err("The invocation request is invalid.") + })?; + py.detach(|| nemo_fabric_core::runtime::begin_run_result(&plan, &runtime, &request)) + .map_err(to_py_error) +} + +/// Finalize a saved one-shot protocol failure after cleanup. +#[pyfunction] +fn _finish_failed_run_result( + py: Python<'_>, + plan_json: String, + runtime_json: String, + request_json: String, + events_json: String, +) -> PyResult<()> { + let plan = parse_run_plan(plan_json)?; + let runtime = parse_runtime_handle(runtime_json)?; + let request: RunRequest = serde_json::from_str(&request_json).map_err(|_| { + pyo3::exceptions::PyValueError::new_err("The invocation request is invalid.") + })?; + let events: Vec = serde_json::from_str(&events_json) + .map_err(|_| pyo3::exceptions::PyValueError::new_err("The cleanup events are invalid."))?; + py.detach(|| { + nemo_fabric_core::runtime::finish_failed_run_result(&plan, &runtime, &request, events) + }) + .map_err(to_py_error) +} + +/// Persist a result after the Python one-shot lifecycle finishes cleanup. +#[pyfunction] +fn _save_run_result( + py: Python<'_>, + plan_json: String, + runtime_json: String, + result_json: String, +) -> PyResult<()> { + let plan = parse_run_plan(plan_json)?; + let runtime = parse_runtime_handle(runtime_json)?; + let result: nemo_fabric_core::RunResult = serde_json::from_str(&result_json) + .map_err(|_| pyo3::exceptions::PyValueError::new_err("The completed result is invalid."))?; + py.detach(|| nemo_fabric_core::runtime::save_run_result(&plan, &runtime, &result)) + .map_err(|_| PyRuntimeError::new_err("The final invocation result could not be saved.")) +} + +/// Read a saved result through an OpenShell environment. +#[pyfunction] +fn invocation_result_environment( + py: Python<'_>, + environment_json: String, + agent: String, + runtime_id: String, + request_id: String, +) -> PyResult { + let environment = parse_environment_handle(environment_json)?; + let result = py + .detach(|| { + nemo_fabric_core::runtime::invocation_result_environment( + &environment, + &agent, + &runtime_id, + &request_id, + ) + }) + .map_err(|error| { + let failure = PyRuntimeError::new_err( + "The invocation result is unavailable; do not replay uncertain work.", + ); + if let FabricError::EnvironmentProviderOperation { code, .. } = error { + match Python::attach(|py| failure.value(py).setattr("code", code)) { + Ok(()) => failure, + Err(error) => error, + } + } else { + failure + } + })?; + to_json(&result) +} + /// Read recorded runtime status through an OpenShell environment. #[pyfunction] fn runtime_status_environment( @@ -391,6 +506,11 @@ fn runtime_resource_snapshot( fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(check_environment, m)?)?; m.add_function(wrap_pyfunction!(runtime_status_environment, m)?)?; + m.add_function(wrap_pyfunction!(invocation_result, m)?)?; + m.add_function(wrap_pyfunction!(_save_run_result, m)?)?; + m.add_function(wrap_pyfunction!(_finish_failed_run_result, m)?)?; + m.add_function(wrap_pyfunction!(_begin_run_result, m)?)?; + m.add_function(wrap_pyfunction!(invocation_result_environment, m)?)?; m.add_function(wrap_pyfunction!(check_service, m)?)?; m.add("ServiceInUseError", m.py().get_type::())?; m.add( diff --git a/docs/reference/api/python-library-reference/nemo_fabric.client.md b/docs/reference/api/python-library-reference/nemo_fabric.client.md index 815b74b20..3ecec93c5 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.client.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.client.md @@ -197,6 +197,26 @@ Doctor checks the resolved adapter, capability mappings, and declared environmen --- +### method `invocation_result` + +```python +async def invocation_result( + *, + agent_name: str, + runtime_id: str, + request_id: str, + artifacts: str | os.PathLike[str] | None = None, + environment: EnvironmentHandle | None = None, +) -> RunResult | None +``` + +Retrieve a saved invocation without executing or repeating work. + +Supply the original request identifier. A null result means no completed result is available; it does not prove that work is running or safe to repeat. Local reads use the configured artifact root. OpenShell reads use the owning runtime server through ``fabric-agent-ctl result``. Files survive only while their artifact storage survives. + +--- + + ### method `plan` ```python diff --git a/docs/reference/api/python-library-reference/nemo_fabric.models.md b/docs/reference/api/python-library-reference/nemo_fabric.models.md index 65356de00..13a26a990 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.models.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.models.md @@ -2761,6 +2761,8 @@ Return a detached mapping matching the Rust ``FabricConfig`` schema. One validated NeMo Fabric invocation request. +Request identifiers must not be blank or contain NUL. + ### Fields diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx index e117661b6..8d582a921 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx @@ -21,7 +21,7 @@ Explicitly accept recovery uncertainty before submitting this new turn. ### `request_id: String` -Request id. +Nonblank request identifier. `Default` and `text` generate an identifier before dispatch. ### `relay_session_root: Option` @@ -79,7 +79,7 @@ Build a text request. #### `default` -
default() -> RunRequest"}} />
+
default() -> Self"}} />
### `impl<'de> Deserialize<'de> for RunRequest` diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index a0303b03d..c0edbca10 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1707,3 +1707,33 @@ OpenClaw service startup reports retain their original source and capture time w Calling applications and environment owners supply typed `ProvenanceConfig.reports` snapshots through the configuration already sent to NeMo Fabric. Capture them at application initialization and preserve their original `source`, `origin`, availability and `observed_at`. A guest never inspects its caller. Harbor implements this seam with `fabric_provenance`, whose shape matches the SDK control; `--ak 'fabric_provenance={"enabled":false}'` disables collection before host initialization and propagates to the guest. The [software evidence example](https://github.com/NVIDIA/NeMo-Fabric/tree/main/examples/provenance) provides six execution-path checks, separate-venv runs, saved RunResults and private comparison fixtures. Its local provider/service fixtures are labeled separately from live deployments. Safe configuration capture is the P0 follow-up and is outside this software-version contract. + + +### Recover a buffered invocation result + +NeMo Fabric saves each completed `RunResult` as `/.fabric///run-result.json` before returning the result. On Unix, result and request-record files are readable only by their owner. The artifact root comes from `runtime.artifacts` or the environment; without either setting, NeMo Fabric uses its default artifact directory. Files remain until their storage is removed; no automatic retention or eviction is added. The owning `Fabric` client retains the artifact location and environment identity in its bounded stopped-runtime records, so that client can look up results after `Runtime.stop()` without repeating the artifact path. After record eviction or from a new client, provide the original explicit artifact root unless the default root applies. + +On Unix, default storage uses `~/.cache/nemo-fabric-` beneath a home directory owned by that user and not writable by other users. Local host directories use the same protected root. When no usable user-owned home directory exists, NeMo Fabric uses `/nemo-fabric--private` with mode 0700. An existing fallback directory must belong to that user, have private permissions, and not be a symlink. Native writes and SDK lookup select the same root. Local one-shot results remain unavailable until the shutdown outcome is saved. A failed final save or interruption keeps the request reserved and prevents lookup from returning an earlier success. Rust `RunRequest::default()` generates a request ID before dispatch; explicitly supplied blank IDs are rejected. + +Use a trusted artifact directory. On Unix, result storage validates ownership, permissions, and symlinks within the private `.fabric` subtree, and requires private record-file permissions. Explicit caller-configured artifact roots can have shared or group-writable ancestors. Serialized results must fit within 4 MiB. Request identifiers must not be blank or contain NUL. Buffered retries can return the saved result; a request ID cannot switch between buffered and streaming invocations. A completed streaming request returns `stream_replay_unsupported` on replay; retrieve its terminal result with `invocation_result()` instead. + +The SDK keeps the runtime active after `request_conflict`, `invocation_pending`, and `stream_replay_unsupported`. These request rejections do not dispatch a new turn. + +Choose a stable `RunRequest.request_id` before submitting work. After a command timeout, `invocation_timeout` reports unknown effects and returns the runtime and request identifiers. The invocation can continue. Retrieve the completed result with: + +```python +result = await fabric.invocation_result( + agent_name="researcher", runtime_id=runtime_id, + request_id=request_id, artifacts="/path/to/artifacts", +) +``` + +```sh +fabric-agent-ctl result --agent researcher --runtime-id RUNTIME_ID --request-id REQUEST_ID +``` + +A null result means no completed result is available; it does not prove the request is still running or safe to repeat. A retry of the same request identifier and input returns the saved result, or reports that the original invocation is pending. Reusing that identifier with different input is rejected. Failed or interrupted reservations are not automatically replayed. + +For OpenShell, pass `environment=environment` instead of `artifacts` to query the original server through the provider. The CLI requires a retained ownership record on that server. A later local SDK client can read retained files using the explicit artifact root even after runtime stop. Neither path reconstructs a missing result. Remote results retain task-side artifact paths. + +Dispatched invocations that end in a host crash, host timeout, or invalid adapter response save a terminal failed result. The original call still raises its error. The saved result uses fixed diagnostic text, records unknown effects, and disables retry. A missing result never authorizes replay. diff --git a/environment-providers/openshell/src/lib.rs b/environment-providers/openshell/src/lib.rs index 5e7525e6d..537534b66 100644 --- a/environment-providers/openshell/src/lib.rs +++ b/environment-providers/openshell/src/lib.rs @@ -346,7 +346,7 @@ async fn runtime_command( )); } if executed.stdout.is_empty() { - if operation == "status" { + if matches!(operation, "status" | "result") { strict_json::validate(&executed.stderr).map_err(|_| { ProviderError::contract("invalid_response", "status command returned invalid JSON") })?; @@ -360,7 +360,7 @@ async fn runtime_command( || diagnostic.operation != operation || !matches!(diagnostic.status.as_str(), "failed" | "unsupported") || !matches!(error.effects.as_str(), "none" | "applied" | "unknown") - || (operation == "status" && error.effects != "none") + || (matches!(operation, "status" | "result") && error.effects != "none") || !matches!( error.stage.as_str(), "request" @@ -368,6 +368,7 @@ async fn runtime_command( | "start" | "invoke" | "stop" + | "result" | "check" | "status" | "transport" @@ -402,7 +403,7 @@ fn decode_sdk_command_response( "task command output was not one SDK response envelope", )); } - if operation == "status" { + if matches!(operation, "status" | "result") { strict_json::validate(stdout).map_err(|_| { ProviderError::contract("invalid_response", "status command returned invalid JSON") })?; @@ -443,7 +444,7 @@ fn decode_sdk_command_response( let status = response["status"].as_str().unwrap_or_default(); let result = &response["result"]; let error = &response["error"]; - if operation == "status" + if matches!(operation, "status" | "result") && (response["changed"] != false || (!error.is_null() && error["effects"] != "none")) { return Err(ProviderError::contract( @@ -494,7 +495,9 @@ fn decode_sdk_command_response( if exit_code != 0 || !error.is_null() || (operation == "stop" && !result.is_null()) - || (operation != "stop" && !result.is_object()) + || (operation != "stop" + && !(operation == "result" && result.is_null()) + && !result.is_object()) { return Err(ProviderError::contract( "invalid_response", @@ -536,6 +539,7 @@ fn decode_sdk_command_response( | "stop" | "check" | "status" + | "result" | "transport" ) || !matches!(effects, "none" | "applied" | "unknown") @@ -573,6 +577,8 @@ fn valid_diagnostic_code(code: &str) -> bool { | "start_uncertain" | "runtime_id_conflict" | "status_failed" + | "result_failed" + | "invocation_timeout" | "stop_failed" | "invoke_failed" | "host_stopping" @@ -1422,6 +1428,11 @@ enum TaskCommand { agent: String, runtime_id: String, }, + Result { + agent: String, + runtime_id: String, + request_id: String, + }, Stop { agent: String, runtime_id: String, @@ -1510,6 +1521,31 @@ impl TaskCommand { runtime_id, request, } => ("invoke", agent, Some(runtime_id), Some(request)), + Self::Result { + agent, + runtime_id, + request_id, + } => { + if [agent.as_str(), runtime_id.as_str(), request_id.as_str()] + .iter() + .any(|value| value.trim().is_empty() || value.contains('\0')) + { + return Err(ProviderError::contract( + "invalid_runtime_command", + "invalid result target", + )); + } + arguments.extend([ + "result".into(), + "--agent".into(), + agent, + "--runtime-id".into(), + runtime_id, + "--request-id".into(), + request_id, + ]); + return Ok(("result", arguments, None)); + } Self::Status { agent, runtime_id } => ("status", agent, Some(runtime_id), None), Self::Stop { agent, runtime_id } => ("stop", agent, Some(runtime_id), None), }; @@ -3852,6 +3888,52 @@ mod tests { ), } } + #[test] + fn result_command_is_a_read_with_request_identity() { + let command: TaskCommand = serde_json::from_value(json!({"operation": "result", "agent": "researcher", "runtime_id": "runtime-1", "request_id": "request-1"})).unwrap(); + let (operation, arguments, payload) = command.arguments().unwrap(); + assert_eq!(operation, "result"); + assert_eq!( + arguments, + vec![ + "fabric-agent-ctl", + "result", + "--agent", + "researcher", + "--runtime-id", + "runtime-1", + "--request-id", + "request-1" + ] + ); + assert!(payload.is_none()); + let response = json!({"operation": "result", "status": "succeeded", "changed": false, "result": null, "error": null}); + assert_eq!( + decode_sdk_command_response("result", 0, response.to_string().as_bytes()).unwrap(), + json!({"status": "returned", "value": null}) + ); + let duplicate = br#"{"operation":"result","status":"succeeded","changed":false,"result":null,"result":{},"error":null}"#; + assert!(decode_sdk_command_response("result", 0, duplicate).is_err()); + } + + #[test] + fn result_and_timeout_diagnostics_keep_their_specific_codes() { + for (operation, code, stage, effects) in [ + ("result", "result_failed", "result", "none"), + ("invoke", "invocation_timeout", "transport", "unknown"), + ] { + let response = json!({"operation": operation, "status": "failed", "changed": if operation == "result" { json!(false) } else { Value::Null }, "result": null, + "error": {"code": code, "stage": stage, "effects": effects, "message": "TEST_ONLY_PRIVATE"}}); + let decoded = + decode_sdk_command_response(operation, 1, response.to_string().as_bytes()).unwrap(); + assert_eq!( + decoded, + json!({"status": "rejected", "error": {"code": code, "stage": stage, "effects": effects}}) + ); + assert!(!decoded.to_string().contains("TEST_ONLY_PRIVATE")); + } + } + #[test] fn status_command_names_the_agent_and_runtime_without_payload() { let command: TaskCommand = serde_json::from_value(json!({ diff --git a/schemas/sdk/run-request.schema.json b/schemas/sdk/run-request.schema.json index 6f5235228..b3c85a026 100644 --- a/schemas/sdk/run-request.schema.json +++ b/schemas/sdk/run-request.schema.json @@ -26,7 +26,9 @@ ] }, "request_id": { - "description": "Request id.", + "description": "Nonblank request identifier. `Default` and `text` generate an identifier before dispatch.", + "minLength": 1, + "pattern": "^[^\\x00]*[^\\s\\x00][^\\x00]*$", "type": "string" } }, diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi index 5b25ad9a9..276eac274 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi @@ -53,3 +53,15 @@ def check_service(service_json: str, request_json: str) -> str: ... class ResourceObservationUnavailableError(RuntimeError): ... class ResourceObservationUnsupportedError(RuntimeError): ... + +def invocation_result( + artifacts: str | None, agent: str, runtime_id: str, request_id: str +) -> str: ... +def invocation_result_environment( + environment_json: str, agent: str, runtime_id: str, request_id: str +) -> str: ... +def _save_run_result(plan_json: str, runtime_json: str, result_json: str) -> None: ... +def _begin_run_result(plan_json: str, runtime_json: str, request_json: str) -> None: ... +def _finish_failed_run_result( + plan_json: str, runtime_json: str, request_json: str, events_json: str +) -> None: ... diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py index c50068e25..7f1d9b374 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py @@ -11,6 +11,7 @@ import logging import math import os +import re import time from threading import Lock from collections import OrderedDict @@ -104,9 +105,103 @@ def __init__(self) -> None: self._pending_runtime_starts: set[object] = set() self._runtimes: dict[str, Runtime] = {} self._stopped_status: OrderedDict[ - str, tuple[tuple[str, str], RuntimeStatusReport] + str, tuple[tuple[str, str], RuntimeStatusReport, str | None] ] = OrderedDict() + async def invocation_result( + self, + *, + agent_name: str, + runtime_id: str, + request_id: str, + artifacts: str | os.PathLike[str] | None = None, + environment: EnvironmentHandle | None = None, + ) -> RunResult | None: + """Retrieve a saved invocation without executing or repeating work. + + Supply the original request identifier. A null result means no completed + result is available; it does not prove that work is running or safe to + repeat. Local reads use the configured artifact root. OpenShell reads + use the owning runtime server through ``fabric-agent-ctl result``. + Files survive only while their artifact storage survives. + """ + if any( + not isinstance(value, str) or not value.strip() or "\0" in value + for value in (agent_name, runtime_id, request_id) + ): + raise FabricConfigError("The invocation result target is invalid.") + environment_json = ( + _environment_json(environment) if environment is not None else None + ) + native = self._require_native_module("invocation_result") + if environment is not None and environment.provider == "openshell": + if artifacts is not None: + raise FabricConfigError( + "Remote lookup uses the runtime's artifact storage." + ) + operation = lambda: native.invocation_result_environment( + environment_json, agent_name, runtime_id, request_id + ) + else: + if environment is not None and environment.provider != "local": + raise FabricConfigError( + "Result lookup supports local and OpenShell environments." + ) + runtime = self._runtimes.get(runtime_id) + record = self._stopped_status.get(runtime_id) + if runtime is not None: + environment_identity = ( + runtime.handle.environment["provider"], + runtime.handle.environment["environment_id"], + ) + configured_artifacts = self._runtime_artifacts(runtime) + elif record is not None: + environment_identity, _, configured_artifacts = record + else: + environment_identity, configured_artifacts = None, None + if environment is not None and environment_identity != ( + environment.provider, + environment.environment_id, + ): + raise FabricStateError( + "The selected environment does not own this runtime.", + stage="result", + code="runtime_unavailable", + ) + if artifacts is None: + artifacts = configured_artifacts + artifact_path = os.fspath(artifacts) if artifacts is not None else None + operation = lambda: native.invocation_result( + artifact_path, agent_name, runtime_id, request_id + ) + try: + encoded = await asyncio.get_running_loop().run_in_executor( + _CHECK_EXECUTOR, operation + ) + raw = json.loads(encoded) + if raw is None: + return None + result = RunResult.from_mapping(raw) + if (result.agent_name, result.runtime_id, result.request_id) != ( + agent_name, + runtime_id, + request_id, + ): + raise ValueError() + return result + except Exception as error: + code = getattr(error, "code", None) + if ( + not isinstance(code, str) + or re.fullmatch(r"[a-z][a-z0-9_]{0,63}", code) is None + ): + code = "result_unavailable" + raise FabricRuntimeError( + "The invocation result could not be retrieved; do not replay uncertain work.", + stage="result", + code=code, + ) from None + async def runtime_status( self, *, @@ -238,7 +333,7 @@ def unique(pairs): stage="status", code="runtime_unavailable", ) - environment_identity, report = record + environment_identity, report, _ = record if report.agent_name != agent_name: raise FabricStateError( "The request does not identify this agent.", @@ -257,6 +352,18 @@ def unique(pairs): ) return report + @staticmethod + def _runtime_artifacts(runtime: Runtime) -> str | None: + mapping = runtime._plan.to_mapping() + configured = mapping["config"].get("runtime", {}).get( + "artifacts" + ) or runtime.handle.environment.get("artifacts") + if configured is None: + return None + from pathlib import Path + + return str(Path(mapping["base_dir"]) / configured) + def _record_stopped_runtime(self, runtime: Runtime) -> None: with self._runtime_records_lock: if ( @@ -276,6 +383,7 @@ def _record_stopped_runtime(self, runtime: Runtime) -> None: "server_state": None, } ), + self._runtime_artifacts(runtime), ) self._stopped_status.move_to_end(runtime.runtime_id) self._runtimes.pop(runtime.runtime_id, None) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py index 19dacbe56..e5780ec55 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py @@ -1361,7 +1361,10 @@ def enable_relay( class RunRequest(FabricBaseModel): - """One validated NeMo Fabric invocation request.""" + """One validated NeMo Fabric invocation request. + + Request identifiers must not be blank or contain NUL. + """ acknowledge_recovery: bool = Field( default=False, @@ -1380,6 +1383,13 @@ class RunRequest(FabricBaseModel): context: dict[str, Any] = Field(default_factory=dict) overrides: dict[str, Any] | None = None + @field_validator("request_id") + @classmethod + def _validate_request_id(cls, value: str) -> str: + if not value.strip() or "\0" in value: + raise ValueError("request_id must not be blank or contain NUL") + return value + @field_validator("input", mode="before") @classmethod def _validate_input(cls, value: Any) -> Any: diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index d70f53615..e5235f216 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -46,6 +46,12 @@ _PI_ADAPTER_ID = "nvidia.fabric.pi" _PI_STREAM_CORRELATION_MODE = "pi_turn_window" +# Result-store rejections do not dispatch a new turn to the adapter. +_INVOCATION_REJECTION_CODES = ( + "request_conflict", + "invocation_pending", + "stream_replay_unsupported", +) class RuntimeStatus(str, Enum): @@ -321,12 +327,18 @@ def stop_after_cancel() -> Any: else: self._status = RuntimeStatus.STOPPED raise - except FabricError: - if self._status is not RuntimeStatus.STOPPED: + except FabricError as error: + if ( + self._status is not RuntimeStatus.STOPPED + and error.code not in _INVOCATION_REJECTION_CODES + ): self._status = RuntimeStatus.FAILED raise except Exception as error: - if self._status is not RuntimeStatus.STOPPED: + if ( + self._status is not RuntimeStatus.STOPPED + and getattr(error, "code", None) not in _INVOCATION_REJECTION_CODES + ): self._status = RuntimeStatus.FAILED raise _runtime_error(error, stage="invoke") from error if absorb_result: @@ -915,6 +927,12 @@ async def _close_streaming_resources(self) -> None: await self._collector.aclose() def _absorb(self, result: RunResult) -> None: + identity = (result.runtime_id, result.invocation_id, result.request_id) + if any( + (item["runtime_id"], item["invocation_id"], item["request_id"]) == identity + for item in self._invocations + ): + return self._invocations.append( { "request_id": result.request_id, @@ -1055,6 +1073,10 @@ def run() -> dict[str, Any]: invoke_error: FabricError | None = None try: try: + if runtime.get("environment", {}).get("provider") == "local": + native._begin_run_result( + plan_json, runtime_json, json.dumps(dict(request)) + ) result = json.loads( native.invoke_runtime( plan_json, runtime_json, json.dumps(dict(request)) @@ -1075,6 +1097,8 @@ def run() -> dict[str, Any]: except Exception as error: stop_error = error cleanup = "failed" + if _runtime_error(error, stage="stop").code == "cleanup_unconfirmed": + cleanup_runtime_handle = runtime if invoke_error is None: if result is None: raise @@ -1097,12 +1121,20 @@ def run() -> dict[str, Any]: ) if result is not None: observations["partial_result"] = result - if ( - stop_error is not None - and _runtime_error(stop_error, stage="stop").code - == "cleanup_unconfirmed" - ): - cleanup_runtime_handle = runtime + if runtime.get("environment", {}).get("provider") == "local": + try: + native._save_run_result( + plan_json, runtime_json, json.dumps(result) + ) + except Exception as error: + failure = FabricRuntimeError( + "The final invocation result could not be saved.", + stage="result", + code="result_storage_failed", + ) + _retain_runtime_evidence(failure, observations) + raise failure from error + if cleanup_runtime_handle is not None: failure = invoke_error or FabricRuntimeError( "Runtime cleanup is unconfirmed; retry cleanup_runtime.stop().", stage="stop", @@ -1113,6 +1145,24 @@ def run() -> dict[str, Any]: if invoke_error is None: raise failure from stop_error if invoke_error is not None: + if runtime.get("environment", {}).get("provider") == "local": + try: + native._finish_failed_run_result( + plan_json, + runtime_json, + json.dumps(dict(request)), + json.dumps( + stop_events if isinstance(stop_events, list) else [] + ), + ) + except Exception as error: + failure = FabricRuntimeError( + "The final invocation result could not be saved.", + stage="result", + code="result_storage_failed", + ) + _retain_runtime_evidence(failure, observations) + raise failure from error # Keep obtained identity and cleanup evidence beside the original # exception. A protocol failure is not a normalized invocation. _retain_runtime_evidence(invoke_error, observations) @@ -1125,7 +1175,9 @@ def run() -> dict[str, Any]: if cleanup_runtime_handle is not None: error._fabric_cleanup_runtime_handle = cleanup_runtime_handle raise - except FabricError: + except FabricError as error: + if cleanup_runtime_handle is not None: + error._fabric_cleanup_runtime_handle = cleanup_runtime_handle raise except Exception as error: raise _runtime_error(error, stage="run") from error diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md index 624226d08..3fe93e127 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md @@ -108,3 +108,33 @@ Follow the [OpenShell Code Review instructions](../../../../../../examples/code_ The example starts the server, waits for its socket, starts a runtime, invokes twice, and stops through separate exec calls. The older NemoClaw client using `validate/prepare/configure` requires migration. Version 1 requests fail explicitly. Direct SDK lifecycle and streaming APIs retain their existing behavior. The task configuration carries the default-on `provenance` collector control and typed caller reports through the existing start request. `enabled=false` skips guest SDK and adapter collection. The guest records its own installation as `guest_sdk`; it never inspects caller packages. Returned RunResults preserve the caller reports and guest-owned snapshots. The host keeps its own snapshot independently so missing evidence from older intermediaries is explicit. + + +### Recover a buffered invocation result + +NeMo Fabric saves each completed `RunResult` as `/.fabric///run-result.json` before returning the result. On Unix, result and request-record files are readable only by their owner. The artifact root comes from `runtime.artifacts` or the environment; without either setting, NeMo Fabric uses its temporary artifact directory. Files remain until their storage is removed; no automatic retention or eviction is added. + +Use a trusted artifact directory. On Unix, result storage validates ownership, permissions, and symlinks within the private `.fabric` subtree, and requires private record-file permissions. Explicit caller-configured artifact roots can have shared or group-writable ancestors. Serialized results must fit within 4 MiB. Request identifiers must not be blank or contain NUL. Buffered retries can return the saved result; a request ID cannot switch between buffered and streaming invocations. A completed streaming request returns `stream_replay_unsupported` on replay; retrieve its terminal result with `invocation_result()` instead. + +The SDK keeps the runtime active after `request_conflict`, `invocation_pending`, and `stream_replay_unsupported`. These request rejections do not dispatch a new turn. + +Choose a stable `RunRequest.request_id` before submitting work. After a command timeout, `invocation_timeout` reports unknown effects and returns the runtime and request identifiers. The invocation can continue. Retrieve the completed result with: + +```python +result = await fabric.invocation_result( + agent_name="researcher", runtime_id=runtime_id, + request_id=request_id, artifacts="/path/to/artifacts", +) +``` + +```sh +fabric-agent-ctl result --agent researcher --runtime-id RUNTIME_ID --request-id REQUEST_ID +``` + +A null result means no completed result is available; it does not prove the request is still running or safe to repeat. A retry of the same request identifier and input returns the saved result, or reports that the original invocation is pending. Reusing that identifier with different input is rejected. Failed or interrupted reservations are not automatically replayed. + +For OpenShell, pass `environment=environment` instead of `artifacts` to query the original server through the provider. The CLI requires a retained ownership record on that server. A later local SDK client can read retained files using the explicit artifact root even after runtime stop. Neither path reconstructs a missing result. Remote results retain task-side artifact paths. + +The Unix default result root is `~/.cache/nemo-fabric-` under a user-owned home directory that other users cannot write. Local host directories use the same protected root. If no usable user-owned home exists, the root is `/nemo-fabric--private`, validated for ownership, mode 0700, and symlink safety. Local one-shot lookup remains unavailable until the shutdown outcome is saved; interrupted or failed finalization does not expose an earlier success. + +Dispatched invocations that end in a host crash, host timeout, or invalid adapter response save a terminal failed result. The original call still raises its error. The saved result uses fixed diagnostic text, records unknown effects, and disables retry. A missing result never authorizes replay. diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/cli.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/cli.py index 56ba97d49..b31343ecd 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/cli.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/cli.py @@ -89,28 +89,33 @@ def parse_command(arguments: list[str]) -> dict[str, Any]: if flag in ( "--agent", "--runtime-id", + "--request-id", "--config", "--session-ref", "--input", "--request", "--timeout-seconds", ): - if index + 1 == len(arguments) or arguments[index + 1].startswith("--"): + if index + 1 == len(arguments) or ( + flag != "--request-id" and arguments[index + 1].startswith("--") + ): raise ProtocolError() flags[flag] = arguments[index + 1] index += 2 else: raise ProtocolError() required = set() - if operation in ("start", "invoke", "stop", "status"): + if operation in ("start", "invoke", "stop", "status", "result"): required.add("--agent") - if operation in ("invoke", "stop", "status"): + if operation in ("invoke", "stop", "status", "result"): required.add("--runtime-id") if operation in CONFIG_OPERATIONS: config_flags = flags.keys() & {"--config", "--request"} if len(config_flags) != 1: raise ProtocolError() required.update(config_flags) + if operation == "result": + required.add("--request-id") allowed = required.copy() if operation == "start" and "--config" in flags: allowed.add("--session-ref") @@ -144,6 +149,7 @@ def parse_command(arguments: list[str]) -> dict[str, Any]: for flag in ( "--agent", "--runtime-id", + "--request-id", ): if flag in flags: request[flag[2:].replace("-", "_")] = flags[flag] diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/protocol.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/protocol.py index da339f683..c850c2579 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/protocol.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/protocol.py @@ -18,7 +18,7 @@ READ_SECONDS = 10.0 SHUTDOWN_SECONDS = 4.0 SOCKET_PATH = "/sandbox/fabric.sock" -OPERATIONS = ("start", "invoke", "stop", "serve", "check", "status") +OPERATIONS = ("start", "invoke", "stop", "serve", "check", "status", "result") # A cause is an SDK or adapter error code, never its message or other values. CAUSE = re.compile(r"[a-z][a-z0-9_]{0,63}") CAUSE_SUFFIX = re.compile(rf" Cause: ({CAUSE.pattern})\.\Z") @@ -47,6 +47,8 @@ "runtime_limit_reached": "The client runtime limit has been reached; stop an owned runtime before starting another.", "status_failed": "Fabric could not read the recorded runtime status.", "stop_failed": "Fabric could not complete the stop operation.", + "result_failed": "Fabric could not retrieve the saved invocation result.", + "invocation_timeout": "The invocation reply exceeded its deadline; work may continue. Retrieve the result using the original request ID; do not submit it again with a new ID.", "invoke_failed": "Fabric did not report a successful invocation.", "host_stopping": "The Fabric host is shutting down.", "runtime_unavailable": "No active Fabric runtime is available.", @@ -132,9 +134,9 @@ def validate_request(request: Any) -> str: ): raise ProtocolError("interface_unsupported", unsupported=True) required = {"interface_version", "operation"} - if operation in ("start", "invoke", "stop", "status"): + if operation in ("start", "invoke", "stop", "status", "result"): required.add("agent") - if operation in ("invoke", "stop", "status"): + if operation in ("invoke", "stop", "status", "result"): required.add("runtime_id") if operation == "start": required.add("config") @@ -143,6 +145,8 @@ def validate_request(request: Any) -> str: if len(payload_fields) != 1: raise ProtocolError() required.update(payload_fields) + if operation == "result": + required.add("request_id") allowed = required.copy() if operation == "start": allowed.add("session_ref") @@ -165,7 +169,7 @@ def validate_request(request: Any) -> str: raise ProtocolError() if not required <= request.keys() or not request.keys() <= allowed: raise ProtocolError() - for field in ("agent", "runtime_id"): + for field in ("agent", "runtime_id", "request_id"): if field in request and ( not isinstance(request[field], str) or not request[field].strip() @@ -244,10 +248,12 @@ def encode_response(response: dict[str, Any]) -> bytes: return _encode( envelope( response["operation"], - changed=False if response["operation"] in ("check", "status") else None, + changed=False + if response["operation"] in ("check", "status", "result") + else None, error=ProtocolError("response_too_large", "transport"), effects="none" - if response["operation"] in ("check", "status") + if response["operation"] in ("check", "status", "result") else "unknown", ) ) @@ -281,6 +287,7 @@ def validate_response(response: dict[str, Any], operation: str) -> None: "invoke", "check", "status", + "result", "transport", ) or error["effects"] not in ("none", "applied", "unknown") @@ -295,7 +302,7 @@ def validate_response(response: dict[str, Any], operation: str) -> None: error["message"] = str( ProtocolError(error["code"], cause=cause[1] if cause else None) ) - if operation == "status": + if operation in ("status", "result"): if ( response["changed"] is not False or error["effects"] != "none" @@ -314,6 +321,11 @@ def validate_response(response: dict[str, Any], operation: str) -> None: if response["changed"] is not False or report.server_state is None: raise invalid response["result"] = report.to_mapping() + elif operation == "result": + if result is not None: + RunResult.from_mapping(result) + if response["changed"] is not False: + raise invalid elif operation == "invoke": RunResult.from_mapping(result) if response["changed"] is not None: diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py index b46e31bf3..4ce3a8c0a 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py @@ -59,6 +59,7 @@ class OwnedRuntime: lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False) generation: int = 0 stop_lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False) + artifacts: str | None = None class RuntimeServer: @@ -139,6 +140,18 @@ async def handle(self, request: Any) -> dict[str, Any]: if self.shutting_down else None, ) + if operation == "result": + owned = self._owned(request) + stage = "result" + result = await self.fabric.invocation_result( + agent_name=request["agent"], + runtime_id=request["runtime_id"], + request_id=request["request_id"], + artifacts=owned.artifacts, + ) + return envelope( + operation, result.to_mapping() if result is not None else None + ) if operation == "status": owned = self._owned(request) stage = "status" @@ -211,6 +224,15 @@ async def handle(self, request: Any) -> dict[str, Any]: # uncertain and retain sandbox ownership. raise ProtocolError("runtime_id_conflict", "start") owned = OwnedRuntime(runtime, request["agent"], "unknown") + configured = ( + config.runtime.artifacts + or runtime.handle.environment.get("artifacts") + ) + owned.artifacts = ( + str(self.directory / configured) + if configured is not None + else None + ) self.runtimes[runtime.runtime_id] = owned # The runtime is addressable; stop can clean it up. self.start_uncertain = False diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/transport.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/transport.py index c4237ce5a..dbc906789 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/transport.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/transport.py @@ -101,6 +101,24 @@ async def client( writer = None try: encoded = encode_request(request) + if operation == "invoke": + from nemo_fabric.models import RunRequest + + try: + payload = ( + RunRequest.model_validate(request["request"]) + if "request" in request + else RunRequest(input=request["input"]) + ) + except (TypeError, ValueError): + raise ProtocolError("invalid_request") from None + request = { + key: value + for key, value in request.items() + if key not in ("request", "input") + } + request["request"] = payload.to_mapping() + encoded = encode_request(request) if operation == "check": timeout = min(timeout, request.get("timeout_seconds", CHECK_SECONDS)) deadline = started + timeout @@ -121,6 +139,20 @@ async def client( await writer.drain() response = await read_message(reader, RESPONSE_LIMIT) validate_response(response, operation) + if ( + operation == "result" + and response["status"] == "succeeded" + and response["result"] is not None + ): + if any( + response["result"][key] != request[request_key] + for key, request_key in ( + ("agent_name", "agent"), + ("runtime_id", "runtime_id"), + ("request_id", "request_id"), + ) + ): + raise ProtocolError("invalid_response", "transport") if operation == "status" and response["status"] == "succeeded": result = response["result"] if (result["runtime_id"], result["agent_name"]) != ( @@ -152,6 +184,15 @@ async def client( report["completed_at"] = ( datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") ) + if ( + operation == "invoke" + and response["error"] is not None + and response["error"]["effects"] == "unknown" + ): + response["result"] = { + "runtime_id": request["runtime_id"], + "request_id": request["request"]["request_id"], + } return response except (OSError, ValueError, TypeError, KeyError, TimeoutError) as error: if operation == "check" and isinstance(error, TimeoutError): @@ -177,12 +218,23 @@ async def client( ) if not isinstance(error, ProtocolError): error = ProtocolError( - "invalid_response" if sent else "host_unavailable", "transport" + "invocation_timeout" + if sent and operation == "invoke" and isinstance(error, TimeoutError) + else "invalid_response" + if sent + else "host_unavailable", + "transport", ) uncertain = sent and operation in (*LIFECYCLE_OPERATIONS, "invoke") + recovery = None + if uncertain and operation == "invoke" and "request" in request: + recovery = { + "runtime_id": request["runtime_id"], + "request_id": request["request"]["request_id"], + } return envelope( operation, - None, + recovery, changed=None if uncertain else False, error=error, effects="unknown" if uncertain else "none", diff --git a/skills/nemo-fabric-integrate/SKILL.md b/skills/nemo-fabric-integrate/SKILL.md index 8e0788d80..55dacab86 100644 --- a/skills/nemo-fabric-integrate/SKILL.md +++ b/skills/nemo-fabric-integrate/SKILL.md @@ -250,6 +250,8 @@ environments. See the [environment lifecycle guide](https://github.com/NVIDIA/Ne Use `await fabric.runtime_status(runtime_id=runtime_id, agent_name=name)` to read recorded state and a safe discovery handle for a runtime started by the same `Fabric` instance. The client retains live runtime objects until successful stop. Add `environment=environment` for an OpenShell lookup through `fabric-agent-ctl status`, including from a later client process while the original server remains alive. `RuntimeStatusReport` reports `active`, `stopped`, or `failed`; `server_state` is present for server reads and null for direct local reads. Each client holds at most 128 runtime records, including pending starts. New starts evict the oldest successfully stopped records first; when active, failed, and pending runtimes fill the limit, startup raises `FabricStateError` with `code="runtime_limit_reached"`. Retained stopped records have a null handle. Discovery handles omit environment variable values and arbitrary provider metadata; use the original `Runtime` for invocation and cleanup. Status reads perform no lifecycle operation or health probe. Use `Fabric.check()` when the consumer needs fresh health evidence. +After a caller timeout, use `await fabric.invocation_result(agent_name=name, runtime_id=runtime_id, request_id=request_id)` to retrieve the original invocation result. Set a stable `RunRequest.request_id` before dispatch. A timeout reported as `invocation_timeout` leaves effects unknown and does not stop the invocation. Pass the original `artifacts` directory from a new local client, or `environment=environment` for OpenShell lookup. The CLI equivalent is `fabric-agent-ctl result --agent --runtime-id --request-id `. Lookup does not invoke or probe the harness. A null result can mean pending or unknown work and never authorizes replay. A matching buffered retry returns a saved result; changed input with the same ID is rejected. Dispatched failures save a terminal result with unknown effects and `retryable=False`. Results remain until their storage is removed. + The selected adapter owns the execution topology. The bundled Claude, Codex, Deep Agents, and Hermes Agent adapters retain their native client, graph/checkpointer, or agent/database inside one local host for the full runtime. Local `process` diff --git a/tests/fixtures/runtime_check_provider.py b/tests/fixtures/runtime_check_provider.py index 009a50576..8b2e7cae1 100644 --- a/tests/fixtures/runtime_check_provider.py +++ b/tests/fixtures/runtime_check_provider.py @@ -77,6 +77,8 @@ arguments.extend(["--agent", command["agent"]]) if "runtime_id" in command: arguments.extend(["--runtime-id", command["runtime_id"]]) + if name == "result": + arguments.extend(["--request-id", command["request_id"]]) field = ( "config" if name == "start" else "request" if name == "invoke" else None ) @@ -88,7 +90,14 @@ key: value for key, value in command.items() if key - in {"operation", "agent", "runtime_id", "level", "timeout_seconds"} + in { + "operation", + "agent", + "runtime_id", + "request_id", + "level", + "timeout_seconds", + } } with Path(path).open("a") as stream: stream.write(json.dumps(routing) + "\n") diff --git a/tests/python/runtime_server/test_commands.py b/tests/python/runtime_server/test_commands.py index 01ca6c7c2..c8c8614a8 100644 --- a/tests/python/runtime_server/test_commands.py +++ b/tests/python/runtime_server/test_commands.py @@ -296,3 +296,49 @@ def test_unusable_input_preserves_existing_bridge_error_code( assert response["error"]["code"] == "invalid_input" assert response["error"]["effects"] == "none" assert response["changed"] is False + + +@pytest.mark.parametrize( + "flags", + [[], ["--request-id", ""], ["--request-id", "first", "--request-id", "second"]], +) +def test_result_command_rejects_invalid_request_id_before_transport( + flags, monkeypatch, capsys +): + from unittest.mock import AsyncMock + + client = AsyncMock() + monkeypatch.setattr(cli.transport, "client", client) + assert ( + cli.main( + ["result", "--agent", "researcher", "--runtime-id", "runtime-1", *flags] + ) + == 1 + ) + response = json.loads(capsys.readouterr().out) + assert response["error"]["code"] == "invalid_request" + assert response["error"]["effects"] == "none" + assert response["changed"] is False + client.assert_not_awaited() + + +def test_result_command_targets_one_request(): + from nemo_fabric.runtime_server.cli import parse_command + + assert parse_command( + [ + "result", + "--agent", + "researcher", + "--runtime-id", + "runtime-1", + "--request-id", + "request-1", + ] + ) == { + "interface_version": 2, + "operation": "result", + "agent": "researcher", + "runtime_id": "runtime-1", + "request_id": "request-1", + } diff --git a/tests/python/runtime_server/test_sdk.py b/tests/python/runtime_server/test_sdk.py index 3dc8d2811..2849edf64 100644 --- a/tests/python/runtime_server/test_sdk.py +++ b/tests/python/runtime_server/test_sdk.py @@ -322,3 +322,153 @@ async def test_real_sdk_busy_stop_exits_host_and_native_child( with suppress(Exception): await task await owned.stop() + + +@pytest.mark.skipif(sys.platform == "win32", reason="Unix socket transport") +async def test_result_is_saved_and_retrievable_after_caller_timeout( + blocking_adapter_config, tmp_path, socket_path +): + from nemo_fabric import RunRequest + from nemo_fabric.runtime_server import transport + + host = RuntimeServer(directory=tmp_path) + command = {"interface_version": 2, "agent": "blocking"} + started = await host.handle( + { + **command, + "operation": "start", + "config": blocking_adapter_config.to_mapping(), + } + ) + assert started["status"] == "succeeded", started + command["runtime_id"] = started["result"]["runtime_id"] + stop, ready = asyncio.Event(), asyncio.Event() + server = asyncio.create_task( + transport.serve( + host, socket_path=socket_path, stop_event=stop, ready_event=ready + ) + ) + try: + await asyncio.wait_for(ready.wait(), 3) + request = RunRequest(input="slow", request_id="recover-this-request") + timeout = await transport.client( + {**command, "operation": "invoke", "request": request.to_mapping()}, + socket_path=socket_path, + timeout=0.1, + ) + assert timeout["error"]["code"] == "invocation_timeout" + assert timeout["error"]["effects"] == "unknown" + assert timeout["result"] == { + "runtime_id": command["runtime_id"], + "request_id": request.request_id, + } + (tmp_path / "release").touch() + async with asyncio.timeout(5): + while True: + response = await transport.client( + { + **command, + "operation": "result", + "request_id": request.request_id, + }, + socket_path=socket_path, + ) + assert response["status"] == "succeeded", response + if response["result"] is not None: + break + await asyncio.sleep(0.01) + result = response["result"] + assert result["request_id"] == request.request_id + assert result["output"] == {"response": "released"} + saved = ( + tmp_path + / "artifacts" + / ".fabric" + / result["runtime_id"] + / result["invocation_id"] + / "run-result.json" + ) + from nemo_fabric import RunResult + + assert ( + RunResult.from_mapping(json.loads(saved.read_text())).to_mapping() == result + ) + assert saved.stat().st_mode & 0o077 == 0 + sdk_result = await host.fabric.invocation_result( + agent_name="blocking", + runtime_id=command["runtime_id"], + request_id=request.request_id, + artifacts=tmp_path / "artifacts", + ) + assert sdk_result.to_mapping() == result + assert (await host.handle({**command, "operation": "stop"}))[ + "status" + ] == "succeeded" + assert ( + await host.handle( + {**command, "operation": "result", "request_id": request.request_id} + ) + )["result"] == result + finally: + (tmp_path / "release").touch() + stop.set() + await asyncio.wait_for(server, 5) + + +async def test_cached_retry_preserves_later_sdk_history( + blocking_adapter_config, tmp_path +): + from nemo_fabric import RunRequest + + (tmp_path / "release").touch() + script = tmp_path / "adapter.py" + text = script.read_text().replace("harness=None", "history=[]\nharness=None") + text = text.replace( + "output={'status':'succeeded','output':{'response':'released'}}", + "history.append({'role':'assistant','content':str(len(history)+1)})\n output={'status':'succeeded','output':{'messages':history}}", + ) + script.write_text(text) + runtime = await Fabric().start_runtime(blocking_adapter_config, base_dir=tmp_path) + try: + request = RunRequest(input="A", request_id="request-a") + first = await runtime.invoke(request=request) + await runtime.invoke(request=RunRequest(input="B", request_id="request-b")) + history = runtime.messages + invocations = runtime.invocations + cached = await runtime.invoke(request=request) + assert cached.invocation_id == first.invocation_id + assert len(history) == 2 + assert runtime.messages == history + assert runtime.invocations == invocations + finally: + await runtime.stop() + + +async def test_lookup_before_first_invocation_returns_no_completed_result( + blocking_adapter_config, tmp_path +): + host = RuntimeServer(directory=tmp_path) + command = {"interface_version": 2, "agent": "blocking"} + started = await host.handle( + { + **command, + "operation": "start", + "config": blocking_adapter_config.to_mapping(), + } + ) + assert started["status"] == "succeeded", started + try: + response = await host.handle( + { + **command, + "operation": "result", + "runtime_id": started["result"]["runtime_id"], + "request_id": "not-submitted", + } + ) + assert response["status"] == "succeeded", response + assert response["result"] is None + assert response["changed"] is False + assert not (tmp_path / "entered").exists() + finally: + await host.shutdown() diff --git a/tests/python/runtime_server/test_transport.py b/tests/python/runtime_server/test_transport.py index 8faa170e8..9c8fb2acd 100644 --- a/tests/python/runtime_server/test_transport.py +++ b/tests/python/runtime_server/test_transport.py @@ -23,6 +23,32 @@ from nemo_fabric.runtime_server import protocol, transport +@pytest.mark.parametrize( + "payload", [{"context": []}, {"overrides": []}, {"request_id": " "}] +) +async def test_invalid_invocation_payload_is_rejected_before_connection( + monkeypatch, socket_path, payload +): + from unittest.mock import AsyncMock + + connect = AsyncMock() + monkeypatch.setattr(asyncio, "open_unix_connection", connect) + response = await transport.client( + { + "interface_version": 2, + "operation": "invoke", + "agent": "researcher", + "runtime_id": "runtime-1", + "request": {"input": "hello", **payload}, + }, + socket_path=socket_path, + ) + assert response["error"]["code"] == "invalid_request" + assert response["error"]["effects"] == "none" + assert response["changed"] is False + connect.assert_not_awaited() + + @pytest.fixture(name="lifecycle_files") def lifecycle_files_fixture(monkeypatch): files = { @@ -600,3 +626,35 @@ async def test_startup_preserves_caller_file_at_socket_path(host, socket_path): with pytest.raises(OSError): await transport.serve(host, socket_path=socket_path, stop_event=asyncio.Event()) assert socket_path.read_text() == "keep" + + +async def test_uncertain_response_preserves_generated_request_identity( + host_request, monkeypatch +): + from unittest.mock import AsyncMock + + reader = MagicMock(spec=asyncio.StreamReader) + reader.readline = AsyncMock( + return_value=protocol.encode_response( + protocol.envelope( + "invoke", + error=protocol.ProtocolError("response_too_large", "transport"), + changed=None, + effects="unknown", + ) + ) + ) + writer = MagicMock(spec=asyncio.StreamWriter) + writer.drain = AsyncMock() + writer.wait_closed = AsyncMock() + monkeypatch.setattr( + asyncio, "open_unix_connection", AsyncMock(return_value=(reader, writer)) + ) + response = await transport.client(host_request("invoke", input={"prompt": "hello"})) + sent = json.loads(writer.write.call_args.args[0]) + assert response["error"]["code"] == "response_too_large" + assert response["error"]["effects"] == "unknown" + assert response["result"] == { + "runtime_id": sent["runtime_id"], + "request_id": sent["request"]["request_id"], + } diff --git a/tests/python/test_native_sdk.py b/tests/python/test_native_sdk.py index 89de856c3..26f732b3a 100644 --- a/tests/python/test_native_sdk.py +++ b/tests/python/test_native_sdk.py @@ -26,6 +26,94 @@ from nemo_fabric import RuntimeStatus +async def test_conflicting_request_keeps_runtime_usable( + blocking_adapter_config, tmp_path +): + (tmp_path / "release").touch() + runtime = await Fabric().start_runtime(blocking_adapter_config, base_dir=tmp_path) + try: + await runtime.invoke(request=RunRequest(request_id="original", input="first")) + with pytest.raises(FabricRuntimeError) as caught: + await runtime.invoke( + request=RunRequest(request_id="original", input="changed") + ) + assert caught.value.code == "request_conflict" + assert runtime.status is RuntimeStatus.ACTIVE + fresh = await runtime.invoke( + request=RunRequest(request_id="fresh", input="new work") + ) + assert fresh.status == "succeeded" + assert fresh.request_id == "fresh" + finally: + await runtime.stop() + + +@pytest.mark.parametrize("stop_fails", [False, True]) +async def test_one_shot_lookup_matches_cleanup_result( + hermes_shim_agent_dir, monkeypatch, stop_fails +): + monkeypatch.setenv("ADAPTER_PYTHON", sys.executable) + original_stop = native.stop_runtime + + def stop(*args): + events = original_stop(*args) + if stop_fails: + raise RuntimeError("test-only stop failure after confirmed cleanup") + return events + + monkeypatch.setattr(native, "stop_runtime", stop) + client = Fabric() + result = await client.run( + hermes_shim_config(), base_dir=hermes_shim_agent_dir, input="final result" + ) + saved = await client.invocation_result( + artifacts=hermes_shim_agent_dir / "artifacts", + agent_name=result.agent_name, + runtime_id=result.runtime_id, + request_id=result.request_id, + ) + assert result.status == ("failed" if stop_fails else "succeeded") + assert saved == result + + +@pytest.mark.skipif(sys.platform == "win32", reason="Unix umask permissions") +@pytest.mark.parametrize("existing_root", [False, True]) +def test_result_storage_supports_group_writable_umask( + hermes_shim_agent_dir, monkeypatch, existing_root +): + monkeypatch.setenv("ADAPTER_PYTHON", sys.executable) + artifacts = hermes_shim_agent_dir / "artifacts" + if existing_root: + artifacts.mkdir(mode=0o700) + script = """ +import asyncio, json, os, sys +from nemo_fabric import Fabric, FabricConfig +os.umask(0o002) +async def main(): + config = FabricConfig.from_mapping(json.loads(sys.argv[1])) + runtime = await Fabric().start_runtime(config, base_dir=sys.argv[2]) + try: + result = await runtime.invoke(input="private result storage") + assert result.status == "succeeded" + finally: + await runtime.stop() +asyncio.run(main()) +""" + completed = subprocess.run( + [ + sys.executable, + "-c", + script, + hermes_shim_config().model_dump_json(exclude_none=True), + str(hermes_shim_agent_dir), + ], + capture_output=True, + text=True, + timeout=30, + ) + assert completed.returncode == 0, completed.stderr + + async def test_native_sdk(hermes_shim_agent_dir: Path): assert native.version() @@ -474,12 +562,12 @@ async def smoke(client: Fabric, fixture_agent: Path) -> None: async def test_adapter_deadline_keeps_normalized_code_and_native_cause( hermes_shim_agent_dir, ): + client = Fabric() + config = hermes_shim_config() + config.runtime.artifacts = str(hermes_shim_agent_dir / "deadline-results") + request = RunRequest(input="test", context={"normalized_deadline": True}) with pytest.raises(FabricRuntimeError) as caught: - await Fabric().run( - hermes_shim_config(), - base_dir=hermes_shim_agent_dir, - request=RunRequest(input="test", context={"normalized_deadline": True}), - ) + await client.run(config, base_dir=hermes_shim_agent_dir, request=request) assert caught.value.code == "timeout" assert caught.value.stage == "invoke" assert caught.value.details["owner"] == "adapter" @@ -487,6 +575,18 @@ async def test_adapter_deadline_keeps_normalized_code_and_native_cause( assert caught.value.details["cause_code"] == "strands_prompt_timeout" assert caught.value.details["runtime_evidence"]["cleanup"] == "completed" assert isinstance(caught.value.__cause__, TimeoutError) + saved = await client.invocation_result( + agent_name=config.metadata.name, + runtime_id=caught.value.details["runtime_evidence"]["runtime_id"], + request_id=request.request_id, + artifacts=config.runtime.artifacts, + ) + assert saved is not None + assert saved.error.code == "timeout" + assert saved.error.metadata["owner"] == "adapter" + assert saved.error.metadata["timeout_seconds"] == 300 + assert saved.error.metadata["cause_code"] == "strands_prompt_timeout" + assert saved.error.metadata["effects"] == "unknown" @pytest.mark.parametrize( @@ -609,3 +709,277 @@ async def test_native_failed_start_exposes_a_private_cleanup_retry( directory.mkdir(mode=0o700) await recovery.stop() assert recovery.status is RuntimeStatus.STOPPED + + +async def test_failed_final_save_never_exposes_pre_cleanup_success( + hermes_shim_agent_dir, monkeypatch +): + os.environ["ADAPTER_PYTHON"] = sys.executable + original_stop = native.stop_runtime + + def stop(*args): + original_stop(*args) + raise RuntimeError("test-only stop failure") + + def save(*args): + raise RuntimeError("test-only final save failure") + + monkeypatch.setattr(native, "stop_runtime", stop) + monkeypatch.setattr(native, "_save_run_result", save) + client = Fabric() + with pytest.raises(FabricRuntimeError) as caught: + await client.run( + hermes_shim_config(), base_dir=hermes_shim_agent_dir, input="hello" + ) + assert caught.value.code == "result_storage_failed" + partial = caught.value.details["runtime_evidence"]["partial_result"] + assert partial["status"] == "failed" + saved = await client.invocation_result( + artifacts=hermes_shim_agent_dir / "artifacts", + agent_name=partial["agent_name"], + runtime_id=partial["runtime_id"], + request_id=partial["request_id"], + ) + assert saved is None + + +async def test_default_result_root_matches_native_writes_and_lookup( + hermes_shim_agent_dir, +): + import shutil + + os.environ["ADAPTER_PYTHON"] = sys.executable + config = hermes_shim_config() + config.runtime.artifacts = None + config.environment.artifacts = None + client = Fabric() + result = await client.run( + config, base_dir=hermes_shim_agent_dir, input="default storage" + ) + try: + saved = await client.invocation_result( + agent_name=result.agent_name, + runtime_id=result.runtime_id, + request_id=result.request_id, + ) + assert saved == result + finally: + directory = Path(result.metadata["fabric_home"]).parent + assert directory.name == result.runtime_id + shutil.rmtree(directory) + + +@pytest.mark.parametrize("artifact_source", ["runtime", "environment"]) +@pytest.mark.parametrize("select_environment", [False, True]) +async def test_stopped_runtime_result_lookup_retains_storage_and_environment( + hermes_shim_agent_dir, artifact_source, select_environment +): + from nemo_fabric import EnvironmentHandle + + os.environ["ADAPTER_PYTHON"] = sys.executable + config = hermes_shim_config() + config.runtime.artifacts = None + config.environment.artifacts = None + getattr(config, artifact_source).artifacts = "retained-results" + client = Fabric() + runtime = await client.start_runtime(config, base_dir=hermes_shim_agent_dir) + try: + result = await runtime.invoke(input="retain lookup context") + finally: + await runtime.stop() + environment = EnvironmentHandle.from_mapping( + runtime.handle.to_mapping()["environment"] + ) + saved = await client.invocation_result( + agent_name=result.agent_name, + runtime_id=result.runtime_id, + request_id=result.request_id, + environment=environment if select_environment else None, + ) + assert saved == result + assert runtime.runtime_id not in client._runtimes + if select_environment: + wrong = environment.to_mapping() + wrong["environment_id"] = "another-environment" + with pytest.raises(FabricStateError) as caught: + await client.invocation_result( + agent_name=result.agent_name, + runtime_id=result.runtime_id, + request_id=result.request_id, + environment=EnvironmentHandle.from_mapping(wrong), + artifacts=hermes_shim_agent_dir / "retained-results", + ) + assert caught.value.code == "runtime_unavailable" + + +@pytest.mark.skipif(sys.platform == "win32", reason="Unix shared temporary root") +def test_default_storage_ignores_a_precreated_shared_temporary_name( + hermes_shim_agent_dir, tmp_path +): + shared = tmp_path / "shared" + shared.mkdir(mode=0o777) + # Another user can occupy this predictable name before Fabric's first run. + (shared / f"nemo-fabric-{os.geteuid()}").write_text("occupied") + config = hermes_shim_config() + config.runtime.artifacts = None + config.environment.artifacts = None + script = """ +import asyncio, json, os, shutil, sys +from pathlib import Path +from nemo_fabric import Fabric, FabricConfig +async def main(): + client = Fabric() + result = await client.run(FabricConfig.from_mapping(json.loads(sys.argv[1])), base_dir=sys.argv[2], input="private default storage") + directory = Path(result.metadata["fabric_home"]).parent + try: + assert Path(sys.argv[3]) not in directory.parents + saved = await client.invocation_result(agent_name=result.agent_name, runtime_id=result.runtime_id, request_id=result.request_id) + assert saved == result + finally: + shutil.rmtree(directory) +asyncio.run(main()) +""" + environment = dict(os.environ, TMPDIR=str(shared), ADAPTER_PYTHON=sys.executable) + completed = subprocess.run( + [ + sys.executable, + "-c", + script, + config.model_dump_json(exclude_none=True), + str(hermes_shim_agent_dir), + str(shared), + ], + env=environment, + capture_output=True, + text=True, + timeout=30, + ) + assert completed.returncode == 0, completed.stderr + + +@pytest.mark.parametrize("one_shot", [False, True]) +@pytest.mark.parametrize("failure", ["crash", "timeout", "invalid"]) +async def test_dispatched_failure_result_is_retrievable( + blocking_adapter_config, tmp_path, one_shot, failure +): + config = blocking_adapter_config + config.runtime.timeout_seconds = 0.2 + adapter = tmp_path / "adapter.py" + source = adapter.read_text().replace( + " if operation=='invoke':", + " if operation=='invoke':\n" + " with (root/'executions').open('a') as count: count.write('once\\n')\n" + " if (root/'crash').exists(): os._exit(17)", + ) + if failure == "invalid": + source = source.replace( + "output={'status':'succeeded','output':{'response':'released'}}", + "output={'status':'invalid','private':'synthetic-sensitive-value'}", + ) + (tmp_path / "release").touch() + elif failure == "crash": + (tmp_path / "crash").touch() + adapter.write_text(source) + client = Fabric() + request = RunRequest(request_id="terminal-failure", input="synthetic test") + runtime = None + try: + if one_shot: + with pytest.raises(FabricRuntimeError) as caught: + await client.run(config, base_dir=tmp_path, request=request) + runtime_id = caught.value.details["runtime_evidence"]["runtime_id"] + else: + runtime = await client.start_runtime(config, base_dir=tmp_path) + runtime_id = runtime.runtime_id + with pytest.raises(FabricRuntimeError): + await runtime.invoke(request=request) + saved = await client.invocation_result( + agent_name=config.metadata.name, + runtime_id=runtime_id, + request_id=request.request_id, + artifacts=tmp_path / "artifacts", + ) + assert saved is not None + assert saved.status == "failed" + assert saved.output is None + assert ( + saved.error.code + == { + "crash": "host_crashed", + "timeout": "host_timeout", + "invalid": "invalid_agent_run_result", + }[failure] + ) + assert saved.error.retryable is False + assert saved.error.metadata["effects"] == "unknown" + if failure == "timeout": + assert saved.error.metadata["owner"] == "fabric_runtime" + assert saved.error.metadata["timeout_seconds"] == 0.2 + assert saved.error.metadata["cause_code"] == "host_timeout" + assert "synthetic-sensitive-value" not in json.dumps(saved.to_mapping()) + if runtime is not None: + replayed = json.loads( + native.invoke_runtime( + json.dumps(runtime._plan.to_mapping()), + json.dumps(runtime.handle.to_mapping()), + json.dumps(request.to_mapping()), + ) + ) + assert replayed["invocation_id"] == saved.invocation_id + assert replayed["error"] == saved.error.to_mapping() + with pytest.raises(FabricStateError): + await runtime.invoke(request=request) + assert (tmp_path / "executions").read_text() == "once\n" + finally: + if runtime is not None: + await runtime.stop() + + +@pytest.mark.skipif(sys.platform == "win32", reason="Unix storage ownership") +@pytest.mark.parametrize("one_shot", [False, True]) +def test_unusable_home_uses_private_temporary_storage( + hermes_shim_agent_dir, tmp_path, one_shot +): + temporary = tmp_path / "fallback" + temporary.mkdir() + config = hermes_shim_config() + config.runtime.artifacts = None + config.environment.artifacts = None + script = """ +import asyncio, json, os, pathlib, sys +from nemo_fabric import Fabric, FabricConfig +async def main(): + client = Fabric() + config = FabricConfig.from_mapping(json.loads(sys.argv[1])) + if sys.argv[3] == "True": + result = await client.run(config, base_dir=sys.argv[2], input="home fallback") + else: + runtime = await client.start_runtime(config, base_dir=sys.argv[2]) + try: + result = await runtime.invoke(input="home fallback") + finally: + await runtime.stop() + saved = await client.invocation_result(agent_name=result.agent_name, runtime_id=result.runtime_id, request_id=result.request_id) + assert saved == result + root = pathlib.Path(os.environ['TMPDIR']) / f'nemo-fabric-{os.geteuid()}-private' + assert root.stat().st_mode & 0o777 == 0o700 + assert pathlib.Path(result.metadata['fabric_home']).is_relative_to(root) +asyncio.run(main()) +""" + completed = subprocess.run( + [ + sys.executable, + "-c", + script, + config.model_dump_json(exclude_none=True), + str(hermes_shim_agent_dir), + str(one_shot), + ], + env=dict( + os.environ, HOME="/", TMPDIR=str(temporary), ADAPTER_PYTHON=sys.executable + ), + capture_output=True, + text=True, + timeout=30, + ) + assert completed.returncode == 0, completed.stderr diff --git a/tests/python/test_runtime.py b/tests/python/test_runtime.py index 5071280c9..db52a833c 100644 --- a/tests/python/test_runtime.py +++ b/tests/python/test_runtime.py @@ -247,6 +247,29 @@ async def test_one_shot_retains_a_cleanup_runtime_without_exposing_its_handle( assert mock_native.stop_runtime.call_count == 2 +@pytest.mark.parametrize("invoke_fails", [False, True]) +async def test_result_storage_failure_preserves_pending_cleanup( + native_client, mock_native, invoke_fails +): + if invoke_fails: + mock_native.invoke_runtime.side_effect = FabricRuntimeError( + "synthetic invoke failure", stage="invoke", code="host_crashed" + ) + mock_native.stop_runtime.side_effect = FabricRuntimeError( + "synthetic cleanup failure", stage="stop", code="cleanup_unconfirmed" + ) + method = "_finish_failed_run_result" if invoke_fails else "_save_run_result" + getattr(mock_native, method).side_effect = RuntimeError("synthetic storage failure") + with pytest.raises(FabricRuntimeError) as caught: + await native_client.run(_config(), input="synthetic work") + assert caught.value.code == "result_storage_failed" + recovery = caught.value.cleanup_runtime + assert isinstance(recovery, Runtime) + mock_native.stop_runtime.side_effect = None + await recovery.stop() + assert recovery.status is RuntimeStatus.STOPPED + + async def test_cancelled_one_shot_retains_a_cleanup_runtime(native_client, mock_native): entered, release = threading.Event(), threading.Event() @@ -308,6 +331,31 @@ def occupy_worker(): mock_native.stop_runtime.assert_called_once() +async def test_remote_one_shot_does_not_write_to_local_result_storage(mock_native): + runtime = _runtime() + runtime["environment"]["provider"] = "openshell" + mock_native.start_runtime.return_value = json.dumps(runtime) + result = await runtime_mod._run_native_lifecycle( + mock_native, _plan(), {"request_id": "remote-request", "input": "remote work"} + ) + assert result["status"] == "succeeded" + mock_native._save_run_result.assert_not_called() + + +async def test_final_save_failure_retains_one_shot_recovery_evidence(mock_native): + mock_native._save_run_result.side_effect = RuntimeError("storage unavailable") + with pytest.raises(FabricRuntimeError) as caught: + await runtime_mod._run_native_lifecycle( + mock_native, _plan(), {"request_id": "saved-request", "input": "hello"} + ) + assert caught.value.stage == "result" + evidence = caught.value.details["runtime_evidence"] + assert evidence["runtime_id"] == "runtime-1" + assert evidence["request_id"] == "saved-request" + assert evidence["cleanup"] == "completed" + assert evidence["partial_result"]["status"] == "succeeded" + + @pytest.mark.parametrize("stage", ["start", "invoke", "stop"]) @pytest.mark.parametrize( ("error_type", "native_code", "expected_code"), @@ -465,8 +513,41 @@ async def test_runtime_reuses_runtime_and_orders_turns(mock_native: MagicMock): assert len(runtime.invocations) == 2 -async def test_native_invoke_failure_marks_runtime_failed(mock_native: MagicMock): - mock_native.invoke_runtime.side_effect = RuntimeError("invoke failed") +@pytest.mark.parametrize( + "code", ["request_conflict", "invocation_pending", "stream_replay_unsupported"] +) +@pytest.mark.parametrize("structured", [False, True]) +async def test_request_rejection_keeps_runtime_usable(mock_native, code, structured): + error = ( + FabricRuntimeError("request rejected", stage="invoke", code=code) + if structured + else RuntimeError("request rejected") + ) + error.code = code + invoke = mock_native.invoke_runtime.side_effect + mock_native.invoke_runtime.side_effect = error + runtime = _runtime_wrapper(mock_native) + try: + with pytest.raises(FabricRuntimeError) as caught: + await runtime.invoke(input="rejected") + assert caught.value.code == code + assert runtime.status is RuntimeStatus.ACTIVE + if structured: + assert caught.value is error + else: + assert caught.value.__cause__ is error + mock_native.invoke_runtime.side_effect = invoke + fresh = await runtime.invoke(input="fresh") + assert fresh.status == "succeeded" + finally: + await runtime.stop() + + +@pytest.mark.parametrize("code", [None, "host_crashed", "result_storage_failed"]) +async def test_native_invoke_failure_marks_runtime_failed(mock_native: MagicMock, code): + error = RuntimeError("invoke failed") + error.code = code + mock_native.invoke_runtime.side_effect = error runtime = _runtime_wrapper(mock_native) with pytest.raises(FabricRuntimeError, match="invoke failed"): @@ -1983,3 +2064,20 @@ def start(*args): finally: release.set() await asyncio.gather(task, return_exceptions=True) + + +async def test_failed_result_finalization_retains_runtime_evidence(mock_native): + mock_native.invoke_runtime.side_effect = FabricRuntimeError( + "synthetic invocation failure", stage="invoke", code="host_crashed" + ) + mock_native._finish_failed_run_result.side_effect = RuntimeError( + "synthetic storage failure" + ) + with pytest.raises(FabricRuntimeError) as caught: + await runtime_mod._run_native_lifecycle( + mock_native, _plan(), {"request_id": "failed-request", "input": "synthetic"} + ) + assert caught.value.code == "result_storage_failed" + assert caught.value.stage == "result" + assert caught.value.details["runtime_evidence"]["runtime_id"] == "runtime-1" + assert caught.value.details["runtime_evidence"]["request_id"] == "failed-request" diff --git a/tests/python/test_sdk_contract.py b/tests/python/test_sdk_contract.py index 65d1e707e..c1ccfbcf6 100644 --- a/tests/python/test_sdk_contract.py +++ b/tests/python/test_sdk_contract.py @@ -1751,6 +1751,7 @@ def _fabric_config() -> FabricConfig: class NativeRecorder: def __init__(self) -> None: self.requests: list[dict[str, Any]] = [] + self.saved_results: list[dict[str, Any]] = [] self.config_base_dir_calls: list[str | None] = [] self.stopped = 0 self.fail_invoke = False @@ -1838,6 +1839,30 @@ def stop_runtime(self, plan_json: str, runtime_json: str) -> str: self.stopped += 1 return json.dumps([]) + def _begin_run_result( + self, plan_json: str, runtime_json: str, request_json: str + ) -> None: + assert json.loads(plan_json)["agent_name"] == "demo" + assert json.loads(runtime_json)["runtime_id"] == "runtime-1" + assert json.loads(request_json)["request_id"] + + def _finish_failed_run_result( + self, plan_json: str, runtime_json: str, request_json: str, events_json: str + ) -> None: + self._begin_run_result(plan_json, runtime_json, request_json) + assert self.stopped > 0 + assert isinstance(json.loads(events_json), list) + + def _save_run_result( + self, plan_json: str, runtime_json: str, result_json: str + ) -> None: + result = json.loads(result_json) + assert self.stopped > 0 + assert json.loads(plan_json)["agent_name"] == result["agent_name"] + assert json.loads(runtime_json)["runtime_id"] == result["runtime_id"] + assert self.requests[-1]["request_id"] == result["request_id"] + self.saved_results.append(result) + class NativeClient(Fabric): def __init__(self, native: NativeRecorder) -> None: @@ -2205,6 +2230,7 @@ async def test_run_accepts_full_run_request(): assert isinstance(result, RunResult) assert result.status == "succeeded" + assert RunResult.from_mapping(native.saved_results[0]) == result assert native.requests[0] == { "input": "hello", "request_id": "request-4", diff --git a/tests/python/test_sdk_server_checks.py b/tests/python/test_sdk_server_checks.py index 73cb3caa1..0718cbbec 100644 --- a/tests/python/test_sdk_server_checks.py +++ b/tests/python/test_sdk_server_checks.py @@ -438,3 +438,153 @@ async def test_runtime_status_native_transport_rejects_duplicate_fields( await Fabric().runtime_status( environment=server_environment, runtime_id="runtime-1", agent_name="demo" ) + + +async def test_remote_result_lookup_during_and_after_invocation( + blocking_adapter_config, server_environment, server_sdk_bridge, tmp_path +): + from nemo_fabric import RunRequest + + host, log = server_sdk_bridge + config = blocking_adapter_config.model_copy(deep=True) + config.environment.workspace = str(tmp_path) + config.runtime.artifacts = str(tmp_path / "artifacts") + config.discovery.local_paths = [str(tmp_path / "blocking.fabric-adapter.json")] + command = {"interface_version": 2, "agent": "blocking"} + started = await host.handle( + {**command, "operation": "start", "config": config.to_mapping()} + ) + assert started["status"] == "succeeded", started + runtime_id = started["result"]["runtime_id"] + request = RunRequest(input="slow", request_id="recover-remote") + turn = asyncio.create_task( + host.handle( + { + **command, + "operation": "invoke", + "runtime_id": runtime_id, + "request": request.to_mapping(), + } + ) + ) + try: + async with asyncio.timeout(5): + while not (tmp_path / "entered").exists(): + await asyncio.sleep(0.01) + client = Fabric() + assert ( + await client.invocation_result( + environment=server_environment, + agent_name="blocking", + runtime_id=runtime_id, + request_id=request.request_id, + ) + is None + ) + (tmp_path / "release").touch() + completed = await asyncio.wait_for(turn, 5) + result = await client.invocation_result( + environment=server_environment, + agent_name="blocking", + runtime_id=runtime_id, + request_id=request.request_id, + ) + assert result.to_mapping() == completed["result"] + commands = [json.loads(line) for line in log.read_text().splitlines()] + assert ( + commands + == [ + { + "operation": "result", + "agent": "blocking", + "runtime_id": runtime_id, + "request_id": request.request_id, + } + ] + * 2 + ) + finally: + (tmp_path / "release").touch() + try: + await turn + finally: + await host.shutdown() + + +async def test_remote_result_errors_do_not_expose_provider_details( + server_environment, monkeypatch +): + from nemo_fabric import FabricRuntimeError + + native = MagicMock() + native.invocation_result_environment.side_effect = RuntimeError( + "TEST_ONLY_PRIVATE provider token" + ) + monkeypatch.setattr("nemo_fabric.client._native", native) + with pytest.raises(FabricRuntimeError) as caught: + await Fabric().invocation_result( + environment=server_environment, + agent_name="researcher", + runtime_id="runtime-1", + request_id="request-1", + ) + assert caught.value.code == "result_unavailable" + assert "TEST_ONLY_PRIVATE" not in str(caught.value) + native.invoke_runtime.assert_not_called() + native.stop_runtime.assert_not_called() + + +@pytest.mark.parametrize("request_id", [" ", "\t\n", "request\0id"]) +def test_recovery_request_ids_are_validated_before_invocation(request_id): + from nemo_fabric import RunRequest + + with pytest.raises(ValueError): + RunRequest(input="hello", request_id=request_id) + + +async def test_result_lookup_rejects_untyped_environment_before_native_dispatch( + monkeypatch, +): + from nemo_fabric import FabricConfigError + + native = MagicMock() + monkeypatch.setattr("nemo_fabric.client._native", native) + with pytest.raises(FabricConfigError): + await Fabric().invocation_result( + agent_name="researcher", + runtime_id="runtime-1", + request_id="request-1", + environment={"provider": "openshell"}, + ) + native.invocation_result.assert_not_called() + native.invocation_result_environment.assert_not_called() + + +@pytest.mark.parametrize( + "code, expected", + [ + ("wrong_agent", "wrong_agent"), + ("invalid_response", "invalid_response"), + ("private value with spaces", "result_unavailable"), + ], +) +async def test_result_lookup_preserves_safe_rejection_codes( + server_environment, monkeypatch, code, expected +): + from nemo_fabric import FabricRuntimeError + + native = MagicMock() + native.invocation_result_environment.side_effect = FabricRuntimeError( + "private native message", code=code + ) + monkeypatch.setattr("nemo_fabric.client._native", native) + with pytest.raises(FabricRuntimeError) as caught: + await Fabric().invocation_result( + environment=server_environment, + agent_name="demo", + runtime_id="runtime-1", + request_id="request-1", + ) + assert caught.value.code == expected + assert "private native message" not in str(caught.value) + assert "private value with spaces" not in repr(caught.value.details)