Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 27 additions & 2 deletions crates/nemoclaw-build/src/run_ci/live_docker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,16 @@ fn prepare(
let document = match case.as_str() {
// The dummy reports native health, so its whole HCL E2E lifecycle
// completes.
DUMMY_CASE => dummy_case(&template, &uid, &endpoint, &subnet, &dummy)?,
DUMMY_CASE => {
let bridge = docker_output(&[
"network",
"inspect",
"bridge",
"--format",
"{{(index .IPAM.Config 0).Gateway}}",
])?;
dummy_case(&template, &uid, &endpoint, &subnet, &dummy, &bridge)?
}
INTERFACES_CASE => {
let host = subnet
.strip_suffix(".0/24")
Expand Down Expand Up @@ -356,18 +365,32 @@ fn prepare(
const DUMMY_CASE: &str = "docker-dummy";

/// The dummy case bound to one run's UID, gateway endpoint, subnet, and image,
/// and a workspace volume named for the UID, so runs never share one.
/// a workspace volume named for the UID, so runs never share one, and free
/// ports for the middleware and effect recorder that `hcl_live::case` serves.
/// The sandbox reaches the recorder at the gateway network's host address,
/// which the middleware binding selects, and at `bridge`, the default Docker
/// bridge's gateway, which it excludes.
fn dummy_case(
template: &str,
uid: &str,
endpoint: &str,
subnet: &str,
image: &str,
bridge: &str,
) -> Result<String> {
let zero = format!("nc-live-dummy@sha256:{}", "0".repeat(64));
let host = subnet
.strip_suffix(".0/24")
.map(|prefix| format!("{prefix}.1"))
.ok_or("a gateway subnet is a /24")?;
// The ports go first, before a random UID or digest could hold one.
Ok(live::bind_case(
template,
&[
("17701", &live::free_port()?.to_string()),
("17702", &live::free_port()?.to_string()),
("172.30.122.1", &host),
("172.17.0.1", bridge),
("0b3f1c52-7d0e-4c6a-9a51-2f6c8e4d7a10", uid),
("http://127.0.0.1:17697", endpoint),
("172.30.122.0/24", subnet),
Expand Down Expand Up @@ -568,13 +591,15 @@ mod tests {
"http://127.0.0.1:1",
"172.30.9.0/24",
"i@1",
"172.17.9.1",
)
.unwrap();
for line in [
" uid: u-1",
" networkCIDR: 172.30.9.0/24",
" ref: i@1",
" source: hcl-dummy-workspace-u-1",
" include: [172.30.9.1, 172.17.9.1]",
] {
assert!(bound.lines().any(|bound| bound == line), "{line}");
}
Expand Down
55 changes: 52 additions & 3 deletions crates/nemoclaw-e2e/cases/docker-dummy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,13 @@
# SPDX-License-Identifier: Apache-2.0

# The reference dummy agent behind a managed Docker gateway. The dummy
# implements the image's bridge contract, including native health, without
# Fabric, a model, or credentials, and replies with its reply setting. Its
# runs Fabric's runtime with the test adapter, including native health,
# without a model or credentials, and replies with its reply setting. Its
# workspace is a named volume, which the gateway allows sandboxes to mount.
# hcl_live::case serves an external middleware and an HTTP effect recorder on
# host ports. The sandbox reaches the recorder at the gateway network's host
# address, which the middleware binding selects, and at the default Docker
# bridge's gateway, which the policy permits and the binding excludes.
apiVersion: nemoclaw.nvidia.com/v1alpha1
kind: NemoClawConfig
metadata:
Expand All @@ -22,6 +26,13 @@ spec:
config:
openshell.drivers.docker:
allow_driver_config: true
openshell.supervisor.middleware:
- name: live-guard
grpc_endpoint: http://172.30.122.1:17701
allow_insecure_transport: true
max_payload_bytes: 4096
binding_policy: exact
bindings: [http_request_pre_credentials]
inferenceProviders:
hosted:
provider: openai
Expand All @@ -37,7 +48,45 @@ spec:
settings:
reply: dummy-reply
network:
tier: isolated
policy:
explicit:
version: 1
filesystem_policy:
include_workdir: false
read_only: [/usr, /bin, /sbin, /lib, /lib64, /etc, /app, /opt, /proc]
read_write: [/sandbox, /tmp, /dev/null, /dev/urandom, /home/node]
landlock:
compatibility: best_effort
process:
run_as_user: "10001"
run_as_group: "10001"
network_policies:
fixture:
name: HTTP effect recorder
endpoints:
- host: 172.30.122.1
port: 17702
allowed_ips: [172.30.122.1/32]
protocol: rest
enforcement: enforce
rules:
- allow: {method: GET, path: "/**"}
- host: 172.17.0.1
port: 17702
allowed_ips: [172.17.0.1/32]
protocol: rest
enforcement: enforce
rules:
- allow: {method: GET, path: "/**"}
binaries:
- path: /usr/local/bin/python3.14
network_middlewares:
guard:
middleware: live-guard
on_error: fail_closed
endpoints:
include: [172.30.122.1, 172.17.0.1]
exclude: [172.17.0.1]
driverConfig:
docker:
mounts:
Expand Down
47 changes: 47 additions & 0 deletions crates/nemoclaw-e2e/tests/e2e/hcl/tests/docker-dummy.tftest.hcl
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@
# storage, then a redeploy that reuses that storage with a new gateway
# container, the marker read back in the new sandbox, the reply again, and
# destroy. It needs no model or credentials; cargo ci live-docker runs it.
# Before the first destroy, sandbox requests check that the gateway's
# external middleware sees the requests its binding selects and blocks the
# one it denies before the destination, and that an excluded destination
# bypasses it.

# The workspace volume, which a platform provides before apply. The agent
# image runs agents as 10001:10001 (image/fabric/Dockerfile).
Expand Down Expand Up @@ -81,6 +85,49 @@ run "write_marker" {
}
}

# The sandbox requests the case's effect recorder at the gateway network's
# host address, which the middleware binding selects, and at the default
# bridge's gateway, which it excludes. The middleware denies /deny.
run "middleware" {
module {
source = "./command"
}

variables {
connection = run.apply.result.connection
targets = {
requests = {
sandbox = "assistant"
command = ["python3", "-c", <<-EOT
import json, sys, urllib.error, urllib.request
selected, excluded = (f"http://{e['host']}:{e['port']}" for e in json.loads(sys.argv[1]))
for url in (f"{selected}/allow", f"{selected}/deny", f"{excluded}/bypass"):
try: print(urllib.request.urlopen(url, timeout=10).status)
except urllib.error.HTTPError as error: print(error.status)
EOT
, jsonencode(yamldecode(file(jsondecode(var.context).config)).spec.sandboxes.assistant.network.policy.explicit.network_policies.fixture.endpoints)]
contains = "200\n403\n200\n"
}
}
}

assert {
condition = length(output.failed) == 0
error_message = "A request returned an unexpected status: ${join(", ", output.failed)}"
}

# hcl_live::case's fixture records each request before answering it.
assert {
condition = jsondecode(file("${jsondecode(var.context).evidence_dir}/middleware-effects.json")).middleware == { "/allow" = 1, "/deny" = 1 }
error_message = "Middleware must see the selected requests once each and not the excluded one: ${file("${jsondecode(var.context).evidence_dir}/middleware-effects.json")}"
}

assert {
condition = jsondecode(file("${jsondecode(var.context).evidence_dir}/middleware-effects.json")).destination == { "/allow" = 1, "/bypass" = 1 }
error_message = "The allowed and excluded requests must reach the destination once each, and the denied one never: ${file("${jsondecode(var.context).evidence_dir}/middleware-effects.json")}"
}
}

run "unchanged_plan" {
module {
source = "./cli"
Expand Down
31 changes: 28 additions & 3 deletions crates/nemoclaw-e2e/tests/e2e/hcl_live.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,10 @@
//! a kind case's fixtures, from the bundle. A kind case's `tests/e2e/hcl/kind`
//! module runs `issuer` once for its existing OIDC issuer, and
//! `tests/e2e/hcl/admission` runs `admission` to refuse, then allow,
//! StatefulSets in a kind case's namespace. The caller
//! StatefulSets in a kind case's namespace. For `docker-dummy`, `case` also
//! serves the external middleware and HTTP effect recorder that the case's
//! gateway and sandbox reach, which write the requests they see to
//! `middleware-effects.json` in the run's reports. The caller
//! destroys the deployment, so cleanup also runs after a failed, cancelled, or
//! timed-out job: `nemoclaw destroy --state-dir "$NEMOCLAW_HCL_RUN/deployment"`.
//! After `tofu test`, `case` reports how long each run block took, on stderr
Expand All @@ -27,9 +30,9 @@ use std::{
env::var, fs, io::Write as _, path::PathBuf, process::Command, sync::Arc, time::Duration,
};

#[test]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[ignore = "live: needs NEMOCLAW_TEST_BUNDLE, NEMOCLAW_HCL_CASE, NEMOCLAW_HCL_RUN, Docker, and the case's images and credentials"]
fn case() {
async fn case() {
let bundle = Bundle::from_env();
let case = var("NEMOCLAW_HCL_CASE").unwrap();
let run = PathBuf::from(var("NEMOCLAW_HCL_RUN").unwrap());
Expand Down Expand Up @@ -80,6 +83,28 @@ fn case() {
fs::write(&config, serde_json::to_vec_pretty(&document).unwrap()).unwrap();
fs::write(reports.join("identity.json"), json!({"candidate":var("GITHUB_SHA").unwrap(),"uid":document["metadata"]["uid"],"case":case}).to_string()).unwrap();
}
// The dummy case's middleware and effect recorder listen on the ports its
// document names until the case ends, including after a failed run block.
let _middleware = if case == "docker-dummy" {
let document: Value =
serde_saphyr::from_str(&fs::read_to_string(&config).unwrap()).unwrap();
let grpc = &document["spec"]["gateway"]["config"]["openshell.supervisor.middleware"][0]["grpc_endpoint"];
let recorder = &document["spec"]["sandboxes"]["assistant"]["network"]["policy"]["explicit"]
["network_policies"]["fixture"]["endpoints"][0]["port"];
Some(
nemoclaw_test_fixtures::middleware::Fixture::start(
url::Url::parse(grpc.as_str().unwrap())
.unwrap()
.port()
.unwrap(),
recorder.as_u64().unwrap().try_into().unwrap(),
reports.join("middleware-effects.json"),
)
.await,
)
} else {
None
};
let context = json!({
"config": fs::canonicalize(config).unwrap(),
"state_dir": run.join("deployment"),
Expand Down
25 changes: 22 additions & 3 deletions crates/nemoclaw-e2e/tests/integration/rust/deployment.rs
Original file line number Diff line number Diff line change
Expand Up @@ -368,9 +368,10 @@ async fn shell_restriction_and_restore_replace_only_the_target_sandbox() {
/// One document carries the authored settings that export must reproduce:
/// web search and API integrations, an authored inference API, native harness
/// settings, network grants, service exposures, and a second sandbox's tool
/// limit. The bundle's CLI exports what the SDK applied, and reapplying the
/// export changes nothing. Each kind of observed drift then stops plan and
/// export without gateway effects or a change to saved state.
/// limit and middleware binding. The bundle's CLI exports what the SDK
/// applied, and reapplying the export changes nothing. Each kind of observed
/// drift stops plan and export without gateway effects or a change to saved
/// state.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[ignore = "requires a verified NEMOCLAW_TEST_BUNDLE"]
async fn authored_settings_survive_cli_export_and_drift_stops_plan_and_export() {
Expand All @@ -394,6 +395,24 @@ async fn authored_settings_survive_cli_export_and_drift_stops_plan_and_export()
reader.agent.tools = serde_json::json!({"enabled": ["read"]})
.as_object()
.cloned();
reader.network = serde_json::from_value(serde_json::json!({
"policy": {"explicit": {
"version": 1,
"network_policies": {"docs": {"name": "docs", "endpoints": [{
"host": "docs.example.com", "port": 443,
"protocol": "rest", "enforcement": "enforce",
"rules": [{"allow": {"method": "GET", "path": "/**"}}]
}], "binaries": [{"path": "/usr/bin/curl"}]}},
"network_middlewares": {"guard": {
"name": "Authored guard", "middleware": "decision-guard",
"order": 10, "on_error": "fail_closed",
"config": {"mode": "block", "terms": ["forbidden"]},
"endpoints": {"include": ["docs.example.com"],
"exclude": ["trusted.example.com"]}
}}
}}
}))
.unwrap();
let sandbox = document.spec.sandboxes.get_mut("assistant").unwrap();
sandbox.agent.integration_refs = vec!["search".into(), "bookings".into()];
sandbox.harness.as_mut().unwrap().settings = Some(
Expand Down
1 change: 1 addition & 0 deletions crates/nemoclaw-test-fixtures/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
pub mod bundle;
#[path = "../../test-support/http.rs"]
pub mod http;
pub mod middleware;
pub mod openshell;
pub mod schema;
pub mod ssh;
Expand Down
Loading
Loading