From 2f55ad412429572ea3ba43136d0d12abf0a3c87c Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 12 Nov 2026 15:20:41 -0500 Subject: [PATCH 1/3] test: cover middleware request effects and CLI export --- .../nemoclaw-build/src/run_ci/live_docker.rs | 23 ++- crates/nemoclaw-e2e/cases/docker-dummy.yaml | 54 ++++++- .../tests/e2e/hcl/middleware/main.tf | 68 +++++++++ .../e2e/hcl/tests/docker-dummy.tftest.hcl | 90 +++++++++++ crates/nemoclaw-e2e/tests/e2e/hcl_live.rs | 51 ++++++- .../tests/integration/rust/deployment.rs | 26 +++- crates/nemoclaw-test-fixtures/src/lib.rs | 1 + .../nemoclaw-test-fixtures/src/middleware.rs | 144 ++++++++++++++++++ crates/test-support/http.rs | 9 +- 9 files changed, 457 insertions(+), 9 deletions(-) create mode 100644 crates/nemoclaw-e2e/tests/e2e/hcl/middleware/main.tf create mode 100644 crates/nemoclaw-test-fixtures/src/middleware.rs diff --git a/crates/nemoclaw-build/src/run_ci/live_docker.rs b/crates/nemoclaw-build/src/run_ci/live_docker.rs index 94006b3e82e..3b0bd9871d1 100644 --- a/crates/nemoclaw-build/src/run_ci/live_docker.rs +++ b/crates/nemoclaw-build/src/run_ci/live_docker.rs @@ -315,7 +315,28 @@ fn prepare( let document = match case.as_str() { // The dummy reports native health, so its whole HCL E2E lifecycle // completes. - DUMMY_CASE => dummy_case(&template, &uid, &endpoint, &subnet, &dummy)?, + DUMMY_CASE => { + let host = subnet + .strip_suffix(".0/24") + .ok_or("a gateway subnet is a /24")?; + let bypass = docker_output(&[ + "network", + "inspect", + "bridge", + "--format", + "{{(index .IPAM.Config 0).Gateway}}", + ])?; + let document = dummy_case(&template, &uid, &endpoint, &subnet, &dummy)?; + live::bind_case( + &document, + &[ + ("172.30.122.1", &format!("{host}.1")), + ("172.17.0.1", &bypass), + ("17701", &live::free_port()?.to_string()), + ("17702", &live::free_port()?.to_string()), + ], + )? + } INTERFACES_CASE => { let host = subnet .strip_suffix(".0/24") diff --git a/crates/nemoclaw-e2e/cases/docker-dummy.yaml b/crates/nemoclaw-e2e/cases/docker-dummy.yaml index 010dd045b3b..f3b69590517 100644 --- a/crates/nemoclaw-e2e/cases/docker-dummy.yaml +++ b/crates/nemoclaw-e2e/cases/docker-dummy.yaml @@ -2,9 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 # The reference dummy agent behind a managed Docker gateway. The dummy -# implements the image's bridge contract, including native health, without -# Fabric, a model, or credentials, and replies with its reply setting. Its +# runs Fabric's runtime with the test adapter, including native health, +# without a model or credentials, and replies with its reply setting. Its # workspace is a named volume, which the gateway allows sandboxes to mount. +# The case runner serves an external middleware and HTTP effect recorder on +# host ports. The gateway bridge selects middleware; the default Docker bridge +# is an otherwise-permitted excluded destination to the same recorder. apiVersion: nemoclaw.nvidia.com/v1alpha1 kind: NemoClawConfig metadata: @@ -22,6 +25,13 @@ spec: config: openshell.drivers.docker: allow_driver_config: true + openshell.supervisor.middleware: + - name: live-guard + grpc_endpoint: http://172.30.122.1:17701 + allow_insecure_transport: true + max_payload_bytes: 4096 + binding_policy: exact + bindings: [http_request_pre_credentials] inferenceProviders: hosted: provider: openai @@ -37,7 +47,45 @@ spec: settings: reply: dummy-reply network: - tier: isolated + policy: + explicit: + version: 1 + filesystem_policy: + include_workdir: false + read_only: [/usr, /bin, /sbin, /lib, /lib64, /etc, /app, /opt, /proc] + read_write: [/sandbox, /tmp, /dev/null, /dev/urandom, /home/node] + landlock: + compatibility: best_effort + process: + run_as_user: "10001" + run_as_group: "10001" + network_policies: + fixture: + name: HTTP effect recorder + endpoints: + - host: 172.30.122.1 + port: 17702 + allowed_ips: [172.30.122.1/32] + protocol: rest + enforcement: enforce + rules: + - allow: {method: GET, path: "/**"} + - host: 172.17.0.1 + port: 17702 + allowed_ips: [172.17.0.1/32] + protocol: rest + enforcement: enforce + rules: + - allow: {method: GET, path: "/**"} + binaries: + - path: /usr/local/bin/python3.14 + network_middlewares: + guard: + middleware: live-guard + on_error: fail_closed + endpoints: + include: [172.30.122.1, 172.17.0.1] + exclude: [172.17.0.1] driverConfig: docker: mounts: diff --git a/crates/nemoclaw-e2e/tests/e2e/hcl/middleware/main.tf b/crates/nemoclaw-e2e/tests/e2e/hcl/middleware/main.tf new file mode 100644 index 00000000000..89be1f7180e --- /dev/null +++ b/crates/nemoclaw-e2e/tests/e2e/hcl/middleware/main.tf @@ -0,0 +1,68 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Observe the external fixture after sandbox requests, and expose its bound URLs. +terraform { + required_version = "= 1.12.12" +} + +variable "context" { + type = object({ + config = string + state_dir = string + probe_executable = string + evidence_dir = string + }) +} + +variable "baseline" { + type = string + default = null +} + +locals { + policy = yamldecode(file(var.context.config)).spec.sandboxes.assistant.network.policy.explicit + endpoints = local.policy.network_policies.fixture.endpoints + port = local.endpoints[0].port +} + +resource "terraform_data" "observation" { + triggers_replace = uuid() + provisioner "local-exec" { + command = "exec \"$PROBE\" hcl_live::middleware_observation --exact --ignored --quiet" + environment = { + PROBE = var.context.probe_executable + NEMOCLAW_HCL_HTTP_PORT = tostring(local.port) + NEMOCLAW_HCL_STATE = var.context.state_dir + NEMOCLAW_HCL_REPORT = "${var.context.evidence_dir}/middleware-${self.id}.json" + } + } +} + +locals { + report = jsondecode(file("${var.context.evidence_dir}/middleware-${terraform_data.observation.id}.json")) +} + +output "sandbox" { + value = local.report.sandbox +} +output "same_sandbox" { + value = var.baseline == null || local.report.sandbox == var.baseline +} +output "effects" { + value = local.report.effects +} +output "targets" { + value = { for path, index in { allow = 0, deny = 0, bypass = 1 } : path => { + sandbox = "assistant" + command = ["python3", "-c", <<-PY + import urllib.request, urllib.error + try: + print(urllib.request.urlopen("http://${local.endpoints[index].host}:${local.port}/${path}", timeout=10).status) + except urllib.error.HTTPError as error: + print(error.code) + PY + ] + contains = path == "deny" ? "403" : "200" + } } +} diff --git a/crates/nemoclaw-e2e/tests/e2e/hcl/tests/docker-dummy.tftest.hcl b/crates/nemoclaw-e2e/tests/e2e/hcl/tests/docker-dummy.tftest.hcl index 98d12e9c1f8..a77187c23e7 100644 --- a/crates/nemoclaw-e2e/tests/e2e/hcl/tests/docker-dummy.tftest.hcl +++ b/crates/nemoclaw-e2e/tests/e2e/hcl/tests/docker-dummy.tftest.hcl @@ -7,6 +7,9 @@ # storage, then a redeploy that reuses that storage with a new gateway # container, the marker read back in the new sandbox, the reply again, and # destroy. It needs no model or credentials; cargo ci live-docker runs it. +# Before destroy, actual sandbox HTTP requests prove middleware selection, +# denial without a destination effect, and exclusion. Reapply must preserve +# the sandbox identity without changing resources. # The workspace volume, which a platform provides before apply. The agent # image runs agents as 10001:10001 (image/fabric/Dockerfile). @@ -81,6 +84,93 @@ run "write_marker" { } } +# Capture the identity and the bound request URLs before issuing traffic. +run "middleware_before" { + module { + source = "./middleware" + } +} + +run "middleware_requests" { + module { + source = "./command" + } + variables { + connection = run.apply.result.connection + targets = run.middleware_before.targets + } + assert { + condition = length(output.failed) == 0 + error_message = "Middleware request statuses failed: ${join(", ", output.failed)}" + } +} + +run "middleware_effects" { + module { + source = "./middleware" + } + variables { + baseline = run.middleware_before.sandbox + } + assert { + condition = try(output.effects.middleware["/allow"], 0) == 1 + error_message = "The allowed request did not reach middleware exactly once." + } + assert { + condition = try(output.effects.destination["/allow"], 0) == 1 + error_message = "The allowed request did not produce exactly one destination effect." + } + assert { + condition = try(output.effects.middleware["/deny"], 0) == 1 + error_message = "The denied request did not reach middleware exactly once." + } + assert { + condition = try(output.effects.destination["/deny"], 0) == 0 + error_message = "The denied request reached the destination." + } + assert { + condition = try(output.effects.middleware["/bypass"], 0) == 0 + error_message = "The excluded destination reached middleware." + } + assert { + condition = try(output.effects.destination["/bypass"], 0) == 1 + error_message = "The otherwise-permitted excluded destination was not reached exactly once." + } +} + +run "unchanged_apply" { + module { + source = "./cli" + } + variables { + operation = "apply" + } + assert { + condition = output.exit_code == 0 && output.changes == {} + error_message = "Unchanged reconciliation changed resources or failed: ${output.error}" + } +} + +run "middleware_after_reapply" { + module { + source = "./middleware" + } + variables { + baseline = run.middleware_effects.sandbox + } + assert { + condition = output.same_sandbox + error_message = "Unchanged reconciliation replaced the sandbox." + } +} + +# Leave this module's final variables independent of earlier run outputs for cleanup. +run "middleware_cleanup_inputs" { + module { + source = "./middleware" + } +} + run "unchanged_plan" { module { source = "./cli" diff --git a/crates/nemoclaw-e2e/tests/e2e/hcl_live.rs b/crates/nemoclaw-e2e/tests/e2e/hcl_live.rs index 5f20cf22d68..1fae76ea245 100644 --- a/crates/nemoclaw-e2e/tests/e2e/hcl_live.rs +++ b/crates/nemoclaw-e2e/tests/e2e/hcl_live.rs @@ -27,9 +27,9 @@ use std::{ env::var, fs, io::Write as _, path::PathBuf, process::Command, sync::Arc, time::Duration, }; -#[test] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[ignore = "live: needs NEMOCLAW_TEST_BUNDLE, NEMOCLAW_HCL_CASE, NEMOCLAW_HCL_RUN, Docker, and the case's images and credentials"] -fn case() { +async fn case() { let bundle = Bundle::from_env(); let case = var("NEMOCLAW_HCL_CASE").unwrap(); let run = PathBuf::from(var("NEMOCLAW_HCL_RUN").unwrap()); @@ -39,6 +39,23 @@ fn case() { fs::create_dir_all(directory).unwrap(); } let config = var("NEMOCLAW_HCL_DOCUMENT").unwrap_or_else(|_| format!("cases/{case}.yaml")); + // These listeners live exactly as long as this case, including failed HCL runs. + let _middleware = if case == "docker-dummy" { + let document: Value = + serde_saphyr::from_str(&fs::read_to_string(&config).unwrap()).unwrap(); + let endpoint = document["spec"]["gateway"]["config"]["openshell.supervisor.middleware"][0] + ["grpc_endpoint"] + .as_str() + .unwrap(); + let grpc = url::Url::parse(endpoint).unwrap().port().unwrap(); + let http = document["spec"]["sandboxes"]["assistant"]["network"]["policy"]["explicit"]["network_policies"]["fixture"]["endpoints"][0]["port"].as_u64().unwrap(); + Some( + nemoclaw_test_fixtures::middleware::Fixture::start(grpc, u16::try_from(http).unwrap()) + .await, + ) + } else { + None + }; let context = json!({ "config": fs::canonicalize(config).unwrap(), "state_dir": run.join("deployment"), @@ -141,6 +158,36 @@ async fn probe() { fs::write(input("REPORT"), report.to_string()).unwrap(); } +/// Read independent middleware/destination effects and the sandbox identity +/// after commands have completed. HCL compares these facts across reapply. +#[tokio::test] +#[ignore = "run by tests/e2e/hcl/middleware"] +async fn middleware_observation() { + let input = |name: &str| var(format!("NEMOCLAW_HCL_{name}")).unwrap(); + let port = input("HTTP_PORT"); + let effects = reqwest::Client::new() + .get(format!("http://127.0.0.1:{port}/effects")) + .send() + .await + .unwrap() + .error_for_status() + .unwrap() + .bytes() + .await + .unwrap(); + let effects: Value = serde_json::from_slice(&effects).unwrap(); + let binding = sandbox_binding(&PathBuf::from(input("STATE")), "assistant"); + fs::write( + input("REPORT"), + json!({ + "effects": effects, + "sandbox": binding["id"], + }) + .to_string(), + ) + .unwrap(); +} + /// Request one service a sandbox exposes, as a person would with what apply /// reports: read the interface's token in the sandbox and forward its port /// with the pinned OpenShell CLI, then send one request, with the token as a diff --git a/crates/nemoclaw-e2e/tests/integration/rust/deployment.rs b/crates/nemoclaw-e2e/tests/integration/rust/deployment.rs index ed1b59aa7f5..c1d13f9ec9c 100644 --- a/crates/nemoclaw-e2e/tests/integration/rust/deployment.rs +++ b/crates/nemoclaw-e2e/tests/integration/rust/deployment.rs @@ -367,8 +367,8 @@ async fn shell_restriction_and_restore_replace_only_the_target_sandbox() { /// One document carries the authored settings that export must reproduce: /// web search and API integrations, an authored inference API, native harness /// settings, network grants, service exposures, and a second sandbox's tool -/// limit. The bundle's CLI exports what the SDK applied, and reapplying the -/// export changes nothing. Each kind of observed drift then stops plan and +/// limit and middleware binding. The bundle's CLI exports what the SDK applied, +/// and reapplying the export changes nothing. Each kind of observed drift stops plan and /// export without gateway effects or a change to saved state. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[ignore = "requires a verified NEMOCLAW_TEST_BUNDLE"] @@ -393,6 +393,28 @@ async fn authored_settings_survive_cli_export_and_drift_stops_plan_and_export() reader.agent.tools = Some(nemoclaw_sdk::config::AgentTools { allow: vec!["read".into()], }); + reader.network = serde_json::from_value(serde_json::json!({ + "policy": {"explicit": { + "version": 1, + "filesystem_policy": {"include_workdir": false, + "read_only": ["/usr", "/opt"], "read_write": ["/sandbox", "/tmp"]}, + "landlock": {"compatibility": "best_effort"}, + "process": {"run_as_user": "10001", "run_as_group": "10001"}, + "network_policies": {"docs": {"name": "docs", "endpoints": [{ + "host": "docs.example.com", "port": 443, + "protocol": "rest", "enforcement": "enforce", + "rules": [{"allow": {"method": "GET", "path": "/**"}}] + }], "binaries": [{"path": "/usr/bin/curl"}]}}, + "network_middlewares": {"guard": { + "name": "Authored guard", "middleware": "decision-guard", + "order": 10, "on_error": "fail_closed", + "config": {"mode": "block", "terms": ["forbidden"]}, + "endpoints": {"include": ["docs.example.com"], + "exclude": ["trusted.example.com"]} + }} + }} + })) + .unwrap(); let sandbox = document.spec.sandboxes.get_mut("assistant").unwrap(); sandbox.agent.integration_refs = vec!["search".into(), "bookings".into()]; sandbox.harness.as_mut().unwrap().settings = Some( diff --git a/crates/nemoclaw-test-fixtures/src/lib.rs b/crates/nemoclaw-test-fixtures/src/lib.rs index 3257e20dbe7..63f48e09979 100644 --- a/crates/nemoclaw-test-fixtures/src/lib.rs +++ b/crates/nemoclaw-test-fixtures/src/lib.rs @@ -7,6 +7,7 @@ pub mod bundle; #[path = "../../test-support/http.rs"] pub mod http; +pub mod middleware; pub mod openshell; pub mod schema; pub mod ssh; diff --git a/crates/nemoclaw-test-fixtures/src/middleware.rs b/crates/nemoclaw-test-fixtures/src/middleware.rs new file mode 100644 index 00000000000..933610e56d9 --- /dev/null +++ b/crates/nemoclaw-test-fixtures/src/middleware.rs @@ -0,0 +1,144 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +//! A live case's external middleware and HTTP destination, recording independent +//! request effects. Dropping the fixture stops both listeners. +use openshell_core::{ + extension_protocol::{ExtensionFamily, extension_metadata}, + middleware::WebSocketResponseStream, + proto::{ + Decision, HttpRequestEvaluation, HttpRequestResult, MiddlewareBinding, + MiddlewareDescribeRequest, MiddlewareManifest, SupervisorMiddlewareOperation, + SupervisorMiddlewarePhase, ValidateConfigRequest, ValidateConfigResponse, + WebSocketSessionEvent, + middleware::v1::supervisor_middleware_server::{ + SupervisorMiddleware, SupervisorMiddlewareServer, + }, + }, +}; +use serde_json::json; +use std::{ + collections::BTreeMap, + sync::{Arc, Mutex}, +}; +use tokio_stream::wrappers::TcpListenerStream; +use tonic::{Request, Response, Status}; + +#[derive(Default)] +struct Effects { + middleware: BTreeMap, + destination: BTreeMap, +} + +#[derive(Clone)] +struct Guard(Arc>); + +#[tonic::async_trait] +impl SupervisorMiddleware for Guard { + type EvaluateWebSocketSessionStream = WebSocketResponseStream; + + async fn describe( + &self, + _: Request, + ) -> Result, Status> { + Ok(Response::new(MiddlewareManifest { + name: "nemoclaw-live-guard".into(), + bindings: vec![MiddlewareBinding { + operation: SupervisorMiddlewareOperation::HttpRequest as i32, + phase: SupervisorMiddlewarePhase::PreCredentials as i32, + max_payload_bytes: 4096, + ..Default::default() + }], + extension: Some(extension_metadata( + ExtensionFamily::SupervisorMiddleware, + "nemoclaw-live-guard", + openshell_core::VERSION, + [], + )), + ..Default::default() + })) + } + + async fn validate_config( + &self, + _: Request, + ) -> Result, Status> { + Ok(Response::new(ValidateConfigResponse { + valid: true, + reason: String::new(), + })) + } + + async fn evaluate_http_request( + &self, + request: Request, + ) -> Result, Status> { + let target = request + .into_inner() + .target + .ok_or_else(|| Status::invalid_argument("missing target"))?; + *self + .0 + .lock() + .unwrap() + .middleware + .entry(target.path.clone()) + .or_default() += 1; + Ok(Response::new(HttpRequestResult { + decision: if target.path == "/deny" { + Decision::Deny + } else { + Decision::Allow + } as i32, + ..Default::default() + })) + } + + async fn evaluate_web_socket_session( + &self, + _: Request>, + ) -> Result, Status> { + Err(Status::unimplemented("the live case uses HTTP requests")) + } +} + +pub struct Fixture { + _http: crate::http::Fixture, + grpc: tokio::task::JoinHandle<()>, +} + +impl Fixture { + /// Listen on the test host; the gateway and sandbox reach its bridge addresses. + pub async fn start(grpc_port: u16, http_port: u16) -> Self { + let effects = Arc::new(Mutex::new(Effects::default())); + let guard = Guard(effects.clone()); + let listener = tokio::net::TcpListener::bind(("0.0.0.0", grpc_port)) + .await + .unwrap(); + let grpc = tokio::spawn(async move { + tonic::transport::Server::builder() + .add_service(SupervisorMiddlewareServer::new(guard)) + .serve_with_incoming(TcpListenerStream::new(listener)) + .await + .unwrap(); + }); + let http = + crate::http::Fixture::start_tcp_at(&format!("0.0.0.0:{http_port}"), move |request| { + let mut effects = effects.lock().unwrap(); + let body = if request.path == "/effects" { + json!({"middleware": effects.middleware, "destination": effects.destination}) + } else { + *effects.destination.entry(request.path).or_default() += 1; + json!({"reached": true}) + }; + Some((200, serde_json::to_vec(&body).unwrap())) + }) + .await; + Self { _http: http, grpc } + } +} + +impl Drop for Fixture { + fn drop(&mut self) { + self.grpc.abort(); + } +} diff --git a/crates/test-support/http.rs b/crates/test-support/http.rs index 0616063e472..3421453f273 100644 --- a/crates/test-support/http.rs +++ b/crates/test-support/http.rs @@ -94,9 +94,16 @@ impl Fixture { } /// Serve on an ephemeral loopback port; `endpoint` is `http://127.0.0.1:PORT`. pub async fn start_tcp( + handler: impl FnMut(Request) -> Option<(u16, Vec)> + Send + 'static, + ) -> Self { + Self::start_tcp_at("127.0.0.1:0", handler).await + } + /// Serve on a caller-owned address, including a live case's host port. + pub async fn start_tcp_at( + address: &str, mut handler: impl FnMut(Request) -> Option<(u16, Vec)> + Send + 'static, ) -> Self { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let listener = tokio::net::TcpListener::bind(address).await.unwrap(); let endpoint = format!("http://{}", listener.local_addr().unwrap()); let task = tokio::spawn(async move { loop { From b033521495435dfca2fdadcd85e1a313e0e0a180 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 16 Nov 2026 14:46:27 -0700 Subject: [PATCH 2/3] test: check middleware effects in one run and drop the reapply - The case's fixture writes its request counts to the run's reports, so the docker-dummy run that sends the requests asserts on them directly; the middleware module and its observation probe go. - One sandbox command sends the three requests. - Drop the unchanged reapply and sandbox identity check: the case's unchanged plan already shows no changes. - Bind the middleware ports and addresses inside dummy_case, before the UID and digest. - The export journey's middleware policy keeps only the fields the binding needs. - Document the middleware checks in the live test guide. Signed-off-by: Carlos Villela --- .../nemoclaw-build/src/run_ci/live_docker.rs | 45 ++++---- crates/nemoclaw-e2e/cases/docker-dummy.yaml | 7 +- .../tests/e2e/hcl/middleware/main.tf | 68 ------------ .../e2e/hcl/tests/docker-dummy.tftest.hcl | 103 +++++------------- crates/nemoclaw-e2e/tests/e2e/hcl_live.rs | 58 +++------- .../tests/integration/rust/deployment.rs | 11 +- .../nemoclaw-test-fixtures/src/middleware.rs | 57 +++++----- internal-docs/contributing/live-tests.md | 9 +- 8 files changed, 115 insertions(+), 243 deletions(-) delete mode 100644 crates/nemoclaw-e2e/tests/e2e/hcl/middleware/main.tf diff --git a/crates/nemoclaw-build/src/run_ci/live_docker.rs b/crates/nemoclaw-build/src/run_ci/live_docker.rs index 31079d57605..87f1ff9bbf0 100644 --- a/crates/nemoclaw-build/src/run_ci/live_docker.rs +++ b/crates/nemoclaw-build/src/run_ci/live_docker.rs @@ -301,28 +301,7 @@ fn prepare( let document = match case.as_str() { // The dummy reports native health, so its whole HCL E2E lifecycle // completes. - DUMMY_CASE => { - let host = subnet - .strip_suffix(".0/24") - .ok_or("a gateway subnet is a /24")?; - let bypass = docker_output(&[ - "network", - "inspect", - "bridge", - "--format", - "{{(index .IPAM.Config 0).Gateway}}", - ])?; - let document = dummy_case(&template, &uid, &endpoint, &subnet, &dummy)?; - live::bind_case( - &document, - &[ - ("172.30.122.1", &format!("{host}.1")), - ("172.17.0.1", &bypass), - ("17701", &live::free_port()?.to_string()), - ("17702", &live::free_port()?.to_string()), - ], - )? - } + DUMMY_CASE => dummy_case(&template, &uid, &endpoint, &subnet, &dummy)?, INTERFACES_CASE => { let host = subnet .strip_suffix(".0/24") @@ -377,7 +356,11 @@ fn prepare( const DUMMY_CASE: &str = "docker-dummy"; /// The dummy case bound to one run's UID, gateway endpoint, subnet, and image, -/// and a workspace volume named for the UID, so runs never share one. +/// a workspace volume named for the UID, so runs never share one, and free +/// ports for the middleware and effect recorder that `hcl_live::case` serves. +/// The sandbox reaches the recorder at the gateway network's host address, +/// which the middleware binding selects, and at the default bridge's gateway, +/// which it excludes. fn dummy_case( template: &str, uid: &str, @@ -386,9 +369,25 @@ fn dummy_case( image: &str, ) -> Result { let zero = format!("nc-live-dummy@sha256:{}", "0".repeat(64)); + let host = subnet + .strip_suffix(".0/24") + .map(|prefix| format!("{prefix}.1")) + .ok_or("a gateway subnet is a /24")?; + let bridge = docker_output(&[ + "network", + "inspect", + "bridge", + "--format", + "{{(index .IPAM.Config 0).Gateway}}", + ])?; + // The ports go first, before a random UID or digest could hold one. Ok(live::bind_case( template, &[ + ("17701", &live::free_port()?.to_string()), + ("17702", &live::free_port()?.to_string()), + ("172.30.122.1", &host), + ("172.17.0.1", &bridge), ("0b3f1c52-7d0e-4c6a-9a51-2f6c8e4d7a10", uid), ("http://127.0.0.1:17697", endpoint), ("172.30.122.0/24", subnet), diff --git a/crates/nemoclaw-e2e/cases/docker-dummy.yaml b/crates/nemoclaw-e2e/cases/docker-dummy.yaml index f3b69590517..536b221ff34 100644 --- a/crates/nemoclaw-e2e/cases/docker-dummy.yaml +++ b/crates/nemoclaw-e2e/cases/docker-dummy.yaml @@ -5,9 +5,10 @@ # runs Fabric's runtime with the test adapter, including native health, # without a model or credentials, and replies with its reply setting. Its # workspace is a named volume, which the gateway allows sandboxes to mount. -# The case runner serves an external middleware and HTTP effect recorder on -# host ports. The gateway bridge selects middleware; the default Docker bridge -# is an otherwise-permitted excluded destination to the same recorder. +# hcl_live::case serves an external middleware and an HTTP effect recorder on +# host ports. The sandbox reaches the recorder at the gateway network's host +# address, which the middleware binding selects, and at the default Docker +# bridge's gateway, which the policy permits and the binding excludes. apiVersion: nemoclaw.nvidia.com/v1alpha1 kind: NemoClawConfig metadata: diff --git a/crates/nemoclaw-e2e/tests/e2e/hcl/middleware/main.tf b/crates/nemoclaw-e2e/tests/e2e/hcl/middleware/main.tf deleted file mode 100644 index 498007970a8..00000000000 --- a/crates/nemoclaw-e2e/tests/e2e/hcl/middleware/main.tf +++ /dev/null @@ -1,68 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Observe the external fixture after sandbox requests, and expose its bound URLs. -terraform { - required_version = "= 1.13.9" -} - -variable "context" { - type = object({ - config = string - state_dir = string - probe_executable = string - evidence_dir = string - }) -} - -variable "baseline" { - type = string - default = null -} - -locals { - policy = yamldecode(file(var.context.config)).spec.sandboxes.assistant.network.policy.explicit - endpoints = local.policy.network_policies.fixture.endpoints - port = local.endpoints[0].port -} - -resource "terraform_data" "observation" { - triggers_replace = uuid() - provisioner "local-exec" { - command = "exec \"$PROBE\" hcl_live::middleware_observation --exact --ignored --quiet" - environment = { - PROBE = var.context.probe_executable - NEMOCLAW_HCL_HTTP_PORT = tostring(local.port) - NEMOCLAW_HCL_STATE = var.context.state_dir - NEMOCLAW_HCL_REPORT = "${var.context.evidence_dir}/middleware-${self.id}.json" - } - } -} - -locals { - report = jsondecode(file("${var.context.evidence_dir}/middleware-${terraform_data.observation.id}.json")) -} - -output "sandbox" { - value = local.report.sandbox -} -output "same_sandbox" { - value = var.baseline == null || local.report.sandbox == var.baseline -} -output "effects" { - value = local.report.effects -} -output "targets" { - value = { for path, index in { allow = 0, deny = 0, bypass = 1 } : path => { - sandbox = "assistant" - command = ["python3", "-c", <<-PY - import urllib.request, urllib.error - try: - print(urllib.request.urlopen("http://${local.endpoints[index].host}:${local.port}/${path}", timeout=10).status) - except urllib.error.HTTPError as error: - print(error.code) - PY - ] - contains = path == "deny" ? "403" : "200" - } } -} diff --git a/crates/nemoclaw-e2e/tests/e2e/hcl/tests/docker-dummy.tftest.hcl b/crates/nemoclaw-e2e/tests/e2e/hcl/tests/docker-dummy.tftest.hcl index a77187c23e7..e9445d6ec5a 100644 --- a/crates/nemoclaw-e2e/tests/e2e/hcl/tests/docker-dummy.tftest.hcl +++ b/crates/nemoclaw-e2e/tests/e2e/hcl/tests/docker-dummy.tftest.hcl @@ -7,9 +7,10 @@ # storage, then a redeploy that reuses that storage with a new gateway # container, the marker read back in the new sandbox, the reply again, and # destroy. It needs no model or credentials; cargo ci live-docker runs it. -# Before destroy, actual sandbox HTTP requests prove middleware selection, -# denial without a destination effect, and exclusion. Reapply must preserve -# the sandbox identity without changing resources. +# Before the first destroy, sandbox requests check that the gateway's +# external middleware sees the requests its binding selects and blocks the +# one it denies before the destination, and that an excluded destination +# bypasses it. # The workspace volume, which a platform provides before apply. The agent # image runs agents as 10001:10001 (image/fabric/Dockerfile). @@ -84,90 +85,46 @@ run "write_marker" { } } -# Capture the identity and the bound request URLs before issuing traffic. -run "middleware_before" { - module { - source = "./middleware" - } -} - -run "middleware_requests" { +# The sandbox requests the case's effect recorder at the gateway network's +# host address, which the middleware binding selects, and at the default +# bridge's gateway, which it excludes. The middleware denies /deny. +run "middleware" { module { source = "./command" } + variables { connection = run.apply.result.connection - targets = run.middleware_before.targets - } - assert { - condition = length(output.failed) == 0 - error_message = "Middleware request statuses failed: ${join(", ", output.failed)}" + targets = { + requests = { + sandbox = "assistant" + command = ["python3", "-c", <<-EOT + import json, sys, urllib.error, urllib.request + selected, excluded = (f"http://{e['host']}:{e['port']}" for e in json.loads(sys.argv[1])) + for url in (f"{selected}/allow", f"{selected}/deny", f"{excluded}/bypass"): + try: print(urllib.request.urlopen(url, timeout=10).status) + except urllib.error.HTTPError as error: print(error.status) + EOT + , jsonencode(yamldecode(file(jsondecode(var.context).config)).spec.sandboxes.assistant.network.policy.explicit.network_policies.fixture.endpoints)] + contains = "200\n403\n200\n" + } + } } -} -run "middleware_effects" { - module { - source = "./middleware" - } - variables { - baseline = run.middleware_before.sandbox - } - assert { - condition = try(output.effects.middleware["/allow"], 0) == 1 - error_message = "The allowed request did not reach middleware exactly once." - } - assert { - condition = try(output.effects.destination["/allow"], 0) == 1 - error_message = "The allowed request did not produce exactly one destination effect." - } - assert { - condition = try(output.effects.middleware["/deny"], 0) == 1 - error_message = "The denied request did not reach middleware exactly once." - } assert { - condition = try(output.effects.destination["/deny"], 0) == 0 - error_message = "The denied request reached the destination." - } - assert { - condition = try(output.effects.middleware["/bypass"], 0) == 0 - error_message = "The excluded destination reached middleware." - } - assert { - condition = try(output.effects.destination["/bypass"], 0) == 1 - error_message = "The otherwise-permitted excluded destination was not reached exactly once." + condition = length(output.failed) == 0 + error_message = "A request returned an unexpected status: ${join(", ", output.failed)}" } -} -run "unchanged_apply" { - module { - source = "./cli" - } - variables { - operation = "apply" - } + # hcl_live::case's fixture records each request before answering it. assert { - condition = output.exit_code == 0 && output.changes == {} - error_message = "Unchanged reconciliation changed resources or failed: ${output.error}" + condition = jsondecode(file("${jsondecode(var.context).evidence_dir}/middleware-effects.json")).middleware == { "/allow" = 1, "/deny" = 1 } + error_message = "Middleware must see the selected requests once each and not the excluded one: ${file("${jsondecode(var.context).evidence_dir}/middleware-effects.json")}" } -} -run "middleware_after_reapply" { - module { - source = "./middleware" - } - variables { - baseline = run.middleware_effects.sandbox - } assert { - condition = output.same_sandbox - error_message = "Unchanged reconciliation replaced the sandbox." - } -} - -# Leave this module's final variables independent of earlier run outputs for cleanup. -run "middleware_cleanup_inputs" { - module { - source = "./middleware" + condition = jsondecode(file("${jsondecode(var.context).evidence_dir}/middleware-effects.json")).destination == { "/allow" = 1, "/bypass" = 1 } + error_message = "The allowed and excluded requests must reach the destination once each, and the denied one never: ${file("${jsondecode(var.context).evidence_dir}/middleware-effects.json")}" } } diff --git a/crates/nemoclaw-e2e/tests/e2e/hcl_live.rs b/crates/nemoclaw-e2e/tests/e2e/hcl_live.rs index 79fe64065c5..c94ea494fd2 100644 --- a/crates/nemoclaw-e2e/tests/e2e/hcl_live.rs +++ b/crates/nemoclaw-e2e/tests/e2e/hcl_live.rs @@ -8,7 +8,10 @@ //! a kind case's fixtures, from the bundle. A kind case's `tests/e2e/hcl/kind` //! module runs `issuer` once for its existing OIDC issuer, and //! `tests/e2e/hcl/admission` runs `admission` to refuse, then allow, -//! StatefulSets in a kind case's namespace. The caller +//! StatefulSets in a kind case's namespace. For `docker-dummy`, `case` also +//! serves the external middleware and HTTP effect recorder that the case's +//! gateway and sandbox reach, which write the requests they see to +//! `middleware-effects.json` in the run's reports. The caller //! destroys the deployment, so cleanup also runs after a failed, cancelled, or //! timed-out job: `nemoclaw destroy --state-dir "$NEMOCLAW_HCL_RUN/deployment"`. //! After `tofu test`, `case` reports how long each run block took, on stderr @@ -80,19 +83,24 @@ async fn case() { fs::write(&config, serde_json::to_vec_pretty(&document).unwrap()).unwrap(); fs::write(reports.join("identity.json"), json!({"candidate":var("GITHUB_SHA").unwrap(),"uid":document["metadata"]["uid"],"case":case}).to_string()).unwrap(); } - // These listeners live exactly as long as this case, including failed HCL runs. + // The dummy case's middleware and effect recorder listen on the ports its + // document names until the case ends, including after a failed run block. let _middleware = if case == "docker-dummy" { let document: Value = serde_saphyr::from_str(&fs::read_to_string(&config).unwrap()).unwrap(); - let endpoint = document["spec"]["gateway"]["config"]["openshell.supervisor.middleware"][0] - ["grpc_endpoint"] - .as_str() - .unwrap(); - let grpc = url::Url::parse(endpoint).unwrap().port().unwrap(); - let http = document["spec"]["sandboxes"]["assistant"]["network"]["policy"]["explicit"]["network_policies"]["fixture"]["endpoints"][0]["port"].as_u64().unwrap(); + let grpc = &document["spec"]["gateway"]["config"]["openshell.supervisor.middleware"][0]["grpc_endpoint"]; + let recorder = &document["spec"]["sandboxes"]["assistant"]["network"]["policy"]["explicit"] + ["network_policies"]["fixture"]["endpoints"][0]["port"]; Some( - nemoclaw_test_fixtures::middleware::Fixture::start(grpc, u16::try_from(http).unwrap()) - .await, + nemoclaw_test_fixtures::middleware::Fixture::start( + url::Url::parse(grpc.as_str().unwrap()) + .unwrap() + .port() + .unwrap(), + recorder.as_u64().unwrap().try_into().unwrap(), + reports.join("middleware-effects.json"), + ) + .await, ) } else { None @@ -267,36 +275,6 @@ async fn probe() { fs::write(input("REPORT"), report.to_string()).unwrap(); } -/// Read independent middleware/destination effects and the sandbox identity -/// after commands have completed. HCL compares these facts across reapply. -#[tokio::test] -#[ignore = "run by tests/e2e/hcl/middleware"] -async fn middleware_observation() { - let input = |name: &str| var(format!("NEMOCLAW_HCL_{name}")).unwrap(); - let port = input("HTTP_PORT"); - let effects = reqwest::Client::new() - .get(format!("http://127.0.0.1:{port}/effects")) - .send() - .await - .unwrap() - .error_for_status() - .unwrap() - .bytes() - .await - .unwrap(); - let effects: Value = serde_json::from_slice(&effects).unwrap(); - let binding = sandbox_binding(&PathBuf::from(input("STATE")), "assistant"); - fs::write( - input("REPORT"), - json!({ - "effects": effects, - "sandbox": binding["id"], - }) - .to_string(), - ) - .unwrap(); -} - /// Request one service a sandbox exposes, as a person would with what apply /// reports: read the interface's token in the sandbox and forward its port /// with the pinned OpenShell CLI, then send one request, with the token as a diff --git a/crates/nemoclaw-e2e/tests/integration/rust/deployment.rs b/crates/nemoclaw-e2e/tests/integration/rust/deployment.rs index a59cb9ec27b..abc418b26f4 100644 --- a/crates/nemoclaw-e2e/tests/integration/rust/deployment.rs +++ b/crates/nemoclaw-e2e/tests/integration/rust/deployment.rs @@ -368,9 +368,10 @@ async fn shell_restriction_and_restore_replace_only_the_target_sandbox() { /// One document carries the authored settings that export must reproduce: /// web search and API integrations, an authored inference API, native harness /// settings, network grants, service exposures, and a second sandbox's tool -/// limit and middleware binding. The bundle's CLI exports what the SDK applied, -/// and reapplying the export changes nothing. Each kind of observed drift stops plan and -/// export without gateway effects or a change to saved state. +/// limit and middleware binding. The bundle's CLI exports what the SDK +/// applied, and reapplying the export changes nothing. Each kind of observed +/// drift stops plan and export without gateway effects or a change to saved +/// state. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[ignore = "requires a verified NEMOCLAW_TEST_BUNDLE"] async fn authored_settings_survive_cli_export_and_drift_stops_plan_and_export() { @@ -397,10 +398,6 @@ async fn authored_settings_survive_cli_export_and_drift_stops_plan_and_export() reader.network = serde_json::from_value(serde_json::json!({ "policy": {"explicit": { "version": 1, - "filesystem_policy": {"include_workdir": false, - "read_only": ["/usr", "/opt"], "read_write": ["/sandbox", "/tmp"]}, - "landlock": {"compatibility": "best_effort"}, - "process": {"run_as_user": "10001", "run_as_group": "10001"}, "network_policies": {"docs": {"name": "docs", "endpoints": [{ "host": "docs.example.com", "port": 443, "protocol": "rest", "enforcement": "enforce", diff --git a/crates/nemoclaw-test-fixtures/src/middleware.rs b/crates/nemoclaw-test-fixtures/src/middleware.rs index 933610e56d9..f102f5236f2 100644 --- a/crates/nemoclaw-test-fixtures/src/middleware.rs +++ b/crates/nemoclaw-test-fixtures/src/middleware.rs @@ -1,7 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! A live case's external middleware and HTTP destination, recording independent -//! request effects. Dropping the fixture stops both listeners. +//! The `docker-dummy` live case's external middleware and HTTP destination. +//! Each counts the requests it sees by path and rewrites the counts to a file +//! that the case's assertions read. Dropping the fixture stops both listeners. use openshell_core::{ extension_protocol::{ExtensionFamily, extension_metadata}, middleware::WebSocketResponseStream, @@ -15,22 +16,33 @@ use openshell_core::{ }, }, }; -use serde_json::json; +use serde_json::{Value, json}; use std::{ - collections::BTreeMap, + path::PathBuf, sync::{Arc, Mutex}, }; use tokio_stream::wrappers::TcpListenerStream; use tonic::{Request, Response, Status}; -#[derive(Default)] +/// Requests seen by path: `{"middleware": {PATH: N}, "destination": {PATH: N}}`. +#[derive(Clone)] struct Effects { - middleware: BTreeMap, - destination: BTreeMap, + file: PathBuf, + counts: Arc>, +} + +impl Effects { + fn record(&self, side: &str, path: &str) { + let mut counts = self.counts.lock().unwrap(); + let count = &mut counts[side][path]; + *count = json!(count.as_u64().unwrap_or(0) + 1); + std::fs::write(&self.file, counts.to_string()).unwrap(); + } } +/// Allows every request except one to `/deny`. #[derive(Clone)] -struct Guard(Arc>); +struct Guard(Effects); #[tonic::async_trait] impl SupervisorMiddleware for Guard { @@ -76,13 +88,7 @@ impl SupervisorMiddleware for Guard { .into_inner() .target .ok_or_else(|| Status::invalid_argument("missing target"))?; - *self - .0 - .lock() - .unwrap() - .middleware - .entry(target.path.clone()) - .or_default() += 1; + self.0.record("middleware", &target.path); Ok(Response::new(HttpRequestResult { decision: if target.path == "/deny" { Decision::Deny @@ -107,9 +113,14 @@ pub struct Fixture { } impl Fixture { - /// Listen on the test host; the gateway and sandbox reach its bridge addresses. - pub async fn start(grpc_port: u16, http_port: u16) -> Self { - let effects = Arc::new(Mutex::new(Effects::default())); + /// Listen on every address of the test host, which the gateway and sandbox + /// reach through their bridges, and record effects in `file`. + pub async fn start(grpc_port: u16, http_port: u16, file: PathBuf) -> Self { + let effects = Effects { + file, + counts: Arc::new(Mutex::new(json!({"middleware": {}, "destination": {}}))), + }; + std::fs::write(&effects.file, effects.counts.lock().unwrap().to_string()).unwrap(); let guard = Guard(effects.clone()); let listener = tokio::net::TcpListener::bind(("0.0.0.0", grpc_port)) .await @@ -123,14 +134,8 @@ impl Fixture { }); let http = crate::http::Fixture::start_tcp_at(&format!("0.0.0.0:{http_port}"), move |request| { - let mut effects = effects.lock().unwrap(); - let body = if request.path == "/effects" { - json!({"middleware": effects.middleware, "destination": effects.destination}) - } else { - *effects.destination.entry(request.path).or_default() += 1; - json!({"reached": true}) - }; - Some((200, serde_json::to_vec(&body).unwrap())) + effects.record("destination", &request.path); + Some((200, Vec::new())) }) .await; Self { _http: http, grpc } diff --git a/internal-docs/contributing/live-tests.md b/internal-docs/contributing/live-tests.md index 23b5a46e4f3..c3ddc6ded91 100644 --- a/internal-docs/contributing/live-tests.md +++ b/internal-docs/contributing/live-tests.md @@ -124,7 +124,7 @@ Run blocks call these modules: | `docker-openclaw-vllm-shared` | Live / HCL, NVIDIA GPU runner | Two OpenClaw agents sharing one vLLM model with bearer authentication; any reply from each. | | `docker-openclaw-hermes-ollama` | Live / HCL, NVIDIA GPU runner | OpenClaw on NVIDIA hosted inference, `FOUR`; Hermes on a managed Ollama model, any reply. | | `docker-openclaw-hermes-interfaces` | `cargo ci live-docker` | OpenClaw's Control UI and Hermes's API server and dashboard through the reported forward and token, and OpenClaw's terminal UI through `openshell sandbox exec --tty`, on a stub model that requires the run's key, with replies that name the skills each harness offers from its mounted volumes, and Hermes's dashboard opening a skill as its volume holds it; in each sandbox, a writable workspace, skill volumes that are writable or read-only as mounted, and no model key in the environment; in Hermes's sandbox, a pseudo-terminal; in OpenClaw's sandbox, a granted endpoint reached and an undeclared host refused ([Agent Interfaces](#agent-interfaces)). | -| `docker-dummy` | `cargo ci live-docker` | The [dummy agent](#dummy-agent-lifecycle); its configured reply before and after a redeploy that reuses the gateway's storage, and a workspace file on a named volume that survives the destroy and redeploy. | +| `docker-dummy` | `cargo ci live-docker` | The [dummy agent](#dummy-agent-lifecycle); its configured reply before and after a redeploy that reuses the gateway's storage, a workspace file on a named volume that survives the destroy and redeploy, and sandbox requests that an external middleware allows, denies, or, for an excluded destination, never sees. | | `kind-kubernetes-development` | `cargo ci live-kind` | One agent with the default [kubernetes runtime and development authentication profiles on kind](#kubernetes-gateway-without-the-helm-cli), applied with no `PATH`; it answers with the stub's reply before and after a redeploy that reuses the gateway's storage, and its sandbox's explicit policy lets `python3` reach the stub's `/health` and refuses an undeclared in-cluster address. | | `kind-kubernetes-oidc` | `cargo ci live-kind` | Two agents [on kind](#kubernetes-gateway-without-the-helm-cli) in a namespace with OpenShift's UID range, from a private registry's image, behind a gateway that accepts an existing OIDC issuer's tokens and that the chart's NodePort Service publishes at the node's address; apply goes through the port-forward, and each agent answers the stub's reply through the published address. | | `openshift-oidc` | By hand on OpenShift | The same two agents with the `openshift` runtime profile [on OpenShift](#openshift-gateway), whose `restricted-v2` constraint assigns the gateway its UID and `fsGroup`; each answers with the stub's reply. | @@ -273,11 +273,14 @@ To run one of them by hand, supply its inputs as its section describes. The `docker-dummy` [HCL E2E scenario](#hcl-e2e-scenarios) deploys an image derived from the [reference dummy image](agent-image-maintenance.md#qualify-the-reference-contract-image) through a managed Docker gateway. The dummy runs Fabric's real `fabric-agent-ctl` with NemoClaw's test adapter, so it needs no model or credentials. Its gateway allows driver configuration, and its sandbox mounts a named volume on `/sandbox/workspace`, as [Keep Agent Files on Mounted Volumes](../../docs/build-deploy/state.md#keep-agent-files-on-mounted-volumes) describes; the `tests/e2e/hcl/volume` module creates the volume before the first apply and gives it to the agent image's user, UID 10001. -The agent must reply with the configured `reply` setting, the `tests/e2e/hcl/command` module must write a marker file in the workspace as that user, a plan must report no changes, and destroy must keep only the gateway's storage. +The agent must reply with the configured `reply` setting, and the `tests/e2e/hcl/command` module must write a marker file in the workspace as that user. +The gateway registers an external middleware that `hcl_live::case` serves on the host, beside an HTTP recorder that counts the requests it receives; the sandbox's explicit policy permits the recorder at the gateway network's host address, which the middleware binding selects, and at the default Docker bridge's gateway, which the binding excludes. +From the sandbox, a request the middleware allows must reach the recorder once, a request it denies must return 403 without reaching the recorder, and a request to the excluded address must reach the recorder without reaching the middleware. +A plan must then report no changes, and destroy must keep only the gateway's storage. A second apply must then reuse that storage, refreshing rather than creating it again, and run the gateway in a new container; the new sandbox must read the marker from the kept volume, and the agent must reply again before a final destroy. It requests no inference and does not qualify a production adapter. -`cargo ci live-docker` binds `cases/docker-dummy.yaml` to a fresh UID, free gateway port and subnet, a volume name with the UID, and the derived image by digest, then runs `hcl_live::case` with `NEMOCLAW_HCL_DOCUMENT` naming the bound copy. +`cargo ci live-docker` binds `cases/docker-dummy.yaml` to a fresh UID, free gateway port and subnet, a volume name with the UID, the derived image by digest, free ports for the middleware and recorder, and the default bridge's gateway address, then runs `hcl_live::case` with `NEMOCLAW_HCL_DOCUMENT` naming the bound copy. After the suite, it removes every resource labeled with the document's UID, which includes the volume, and the gateway's sandbox objects; when the tests pass, it fails if OpenShell left a sandbox object. To run the scenario by hand, bind the template the same way and follow [HCL E2E Scenarios](#hcl-e2e-scenarios) with `NEMOCLAW_HCL_CASE=docker-dummy` and `NEMOCLAW_HCL_DOCUMENT=/absolute/path/to/bound.yaml`. From 5f0934d23ae588e00eb2956d8db00aabd9b71811 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 16 Nov 2026 15:11:23 -0700 Subject: [PATCH 3/3] test: read the bridge address outside dummy_case The dummy case binding ran docker network inspect, so its unit test failed on hosts without Docker, such as the macOS runner. prepare reads the address and passes it in, and the test checks that the binding selects both recorder addresses. Signed-off-by: Carlos Villela --- .../nemoclaw-build/src/run_ci/live_docker.rs | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/crates/nemoclaw-build/src/run_ci/live_docker.rs b/crates/nemoclaw-build/src/run_ci/live_docker.rs index 87f1ff9bbf0..02192f07003 100644 --- a/crates/nemoclaw-build/src/run_ci/live_docker.rs +++ b/crates/nemoclaw-build/src/run_ci/live_docker.rs @@ -301,7 +301,16 @@ fn prepare( let document = match case.as_str() { // The dummy reports native health, so its whole HCL E2E lifecycle // completes. - DUMMY_CASE => dummy_case(&template, &uid, &endpoint, &subnet, &dummy)?, + DUMMY_CASE => { + let bridge = docker_output(&[ + "network", + "inspect", + "bridge", + "--format", + "{{(index .IPAM.Config 0).Gateway}}", + ])?; + dummy_case(&template, &uid, &endpoint, &subnet, &dummy, &bridge)? + } INTERFACES_CASE => { let host = subnet .strip_suffix(".0/24") @@ -359,27 +368,21 @@ const DUMMY_CASE: &str = "docker-dummy"; /// a workspace volume named for the UID, so runs never share one, and free /// ports for the middleware and effect recorder that `hcl_live::case` serves. /// The sandbox reaches the recorder at the gateway network's host address, -/// which the middleware binding selects, and at the default bridge's gateway, -/// which it excludes. +/// which the middleware binding selects, and at `bridge`, the default Docker +/// bridge's gateway, which it excludes. fn dummy_case( template: &str, uid: &str, endpoint: &str, subnet: &str, image: &str, + bridge: &str, ) -> Result { let zero = format!("nc-live-dummy@sha256:{}", "0".repeat(64)); let host = subnet .strip_suffix(".0/24") .map(|prefix| format!("{prefix}.1")) .ok_or("a gateway subnet is a /24")?; - let bridge = docker_output(&[ - "network", - "inspect", - "bridge", - "--format", - "{{(index .IPAM.Config 0).Gateway}}", - ])?; // The ports go first, before a random UID or digest could hold one. Ok(live::bind_case( template, @@ -387,7 +390,7 @@ fn dummy_case( ("17701", &live::free_port()?.to_string()), ("17702", &live::free_port()?.to_string()), ("172.30.122.1", &host), - ("172.17.0.1", &bridge), + ("172.17.0.1", bridge), ("0b3f1c52-7d0e-4c6a-9a51-2f6c8e4d7a10", uid), ("http://127.0.0.1:17697", endpoint), ("172.30.122.0/24", subnet), @@ -588,6 +591,7 @@ mod tests { "http://127.0.0.1:1", "172.30.9.0/24", "i@1", + "172.17.9.1", ) .unwrap(); for line in [ @@ -595,6 +599,7 @@ mod tests { " networkCIDR: 172.30.9.0/24", " ref: i@1", " source: hcl-dummy-workspace-u-1", + " include: [172.30.9.1, 172.17.9.1]", ] { assert!(bound.lines().any(|bound| bound == line), "{line}"); }