From d1f0465fb27f9601ab63fd1d68baaf583126ce0c Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Sun, 14 Jun 2026 14:26:51 -0700 Subject: [PATCH 01/34] daemon: gracefully shutdown if wrapped command exits --- src/daemon.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/daemon.rs b/src/daemon.rs index 394e03e..47648c7 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -2,7 +2,6 @@ use anyhow::{Context, Result}; use futures::TryStreamExt; use std::collections::BTreeSet; use std::fs; -use std::process; use std::sync::Arc; use tokio::net::TcpSocket; use tokio::process::Command; @@ -118,12 +117,15 @@ pub(crate) async fn run(state: State, on_ready: Option) -> Result<()> { let mut admin_handle = tokio::spawn(admin_server.run(grpc, adhoc, cancel.clone())); let mut proxy_handle = tokio::spawn(proxy.run(cancel.clone())); - if let Some(mut cmd) = on_ready { - tokio::spawn(async move { + let mut command_handle = tokio::spawn(async move { + if let Some(mut cmd) = on_ready { let status = cmd.status().await; - process::exit(status.map_or(1, |s| s.code().unwrap_or(1))); - }); - } + error!( + "command exited with status {}", + status.map_or(1, |s| s.code().unwrap_or(1)) + ); + } + }); // Wait for shutdown signal or unexpected task exit. Signal results are // ignored - recv() only returns None if "already received" which can't @@ -138,6 +140,7 @@ pub(crate) async fn run(state: State, on_ready: Option) -> Result<()> { r = &mut gossip_handle => error!(?r, "gossip loop exited"), r = &mut admin_handle => error!(?r, "admin server exited"), r = &mut proxy_handle => error!(?r, "proxy exited"), + _ = &mut command_handle => {}, } info!("shutting down"); From 09cc8414c02907ae579ffc4de2b3ad109fd0f62b Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Mon, 15 Jun 2026 14:05:19 -0700 Subject: [PATCH 02/34] daemon: exit with status code of wrapped command --- src/daemon.rs | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/daemon.rs b/src/daemon.rs index 47648c7..bc0fd12 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -2,6 +2,7 @@ use anyhow::{Context, Result}; use futures::TryStreamExt; use std::collections::BTreeSet; use std::fs; +use std::process; use std::sync::Arc; use tokio::net::TcpSocket; use tokio::process::Command; @@ -117,15 +118,15 @@ pub(crate) async fn run(state: State, on_ready: Option) -> Result<()> { let mut admin_handle = tokio::spawn(admin_server.run(grpc, adhoc, cancel.clone())); let mut proxy_handle = tokio::spawn(proxy.run(cancel.clone())); - let mut command_handle = tokio::spawn(async move { - if let Some(mut cmd) = on_ready { - let status = cmd.status().await; - error!( - "command exited with status {}", - status.map_or(1, |s| s.code().unwrap_or(1)) - ); - } - }); + let mut task_status: Option = None; + let command_handler = async || { + on_ready.map(|mut cmd| { + tokio::spawn(async move { + let status = cmd.status().await; + status + }) + }) + }; // Wait for shutdown signal or unexpected task exit. Signal results are // ignored - recv() only returns None if "already received" which can't @@ -140,7 +141,10 @@ pub(crate) async fn run(state: State, on_ready: Option) -> Result<()> { r = &mut gossip_handle => error!(?r, "gossip loop exited"), r = &mut admin_handle => error!(?r, "admin server exited"), r = &mut proxy_handle => error!(?r, "proxy exited"), - _ = &mut command_handle => {}, + Some(r) = command_handler() => { + task_status = r.await??.code(); + error!("command exited with code {}", task_status.unwrap_or(1)); + }, } info!("shutting down"); @@ -169,5 +173,8 @@ pub(crate) async fn run(state: State, on_ready: Option) -> Result<()> { } info!("shutdown complete"); + if let Some(status) = task_status { + process::exit(status) + } Ok(()) } From 935156a0642cf2878daddda3f9fac71a7d8ffb55 Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Tue, 16 Jun 2026 15:49:24 -0700 Subject: [PATCH 03/34] daemon: select on OptionFuture instead --- src/daemon.rs | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/daemon.rs b/src/daemon.rs index bc0fd12..817657b 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -1,4 +1,5 @@ use anyhow::{Context, Result}; +use futures::future::OptionFuture; use futures::TryStreamExt; use std::collections::BTreeSet; use std::fs; @@ -119,14 +120,9 @@ pub(crate) async fn run(state: State, on_ready: Option) -> Result<()> { let mut proxy_handle = tokio::spawn(proxy.run(cancel.clone())); let mut task_status: Option = None; - let command_handler = async || { - on_ready.map(|mut cmd| { - tokio::spawn(async move { - let status = cmd.status().await; - status - }) - }) - }; + let mut command_handle = + on_ready.map(|mut cmd| tokio::spawn(async move { cmd.status().await })); + let mut command_handle = OptionFuture::from(command_handle.as_mut()); // Wait for shutdown signal or unexpected task exit. Signal results are // ignored - recv() only returns None if "already received" which can't @@ -141,8 +137,9 @@ pub(crate) async fn run(state: State, on_ready: Option) -> Result<()> { r = &mut gossip_handle => error!(?r, "gossip loop exited"), r = &mut admin_handle => error!(?r, "admin server exited"), r = &mut proxy_handle => error!(?r, "proxy exited"), - Some(r) = command_handler() => { - task_status = r.await??.code(); + Some(r) = &mut command_handle => { + let r = r.expect("command exited"); + task_status = Some(r.map_or(1, |s| s.code().unwrap_or(1))); error!("command exited with code {}", task_status.unwrap_or(1)); }, } From 1a50dd3efe1857e5f619a93a5513015832413d32 Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Tue, 16 Jun 2026 16:22:07 -0700 Subject: [PATCH 04/34] daemon: handle different failure modes --- src/daemon.rs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/daemon.rs b/src/daemon.rs index 817657b..7123ac6 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -121,8 +121,7 @@ pub(crate) async fn run(state: State, on_ready: Option) -> Result<()> { let mut task_status: Option = None; let mut command_handle = - on_ready.map(|mut cmd| tokio::spawn(async move { cmd.status().await })); - let mut command_handle = OptionFuture::from(command_handle.as_mut()); + OptionFuture::from(on_ready.map(|mut cmd| tokio::spawn(async move { cmd.status().await }))); // Wait for shutdown signal or unexpected task exit. Signal results are // ignored - recv() only returns None if "already received" which can't @@ -138,9 +137,20 @@ pub(crate) async fn run(state: State, on_ready: Option) -> Result<()> { r = &mut admin_handle => error!(?r, "admin server exited"), r = &mut proxy_handle => error!(?r, "proxy exited"), Some(r) = &mut command_handle => { - let r = r.expect("command exited"); - task_status = Some(r.map_or(1, |s| s.code().unwrap_or(1))); - error!("command exited with code {}", task_status.unwrap_or(1)); + match r { + Err(_) => { + task_status = Some(1); + error!("command exited with code {}", task_status.assert()); + }, + Ok(Err(e)) => { + task_status = Some(1); + error!(?e, "command exited"); + }, + Ok(Ok(s)) => { + task_status = Some(s.code().unwrap_or(1)); + info!("command exited with code {}", task_status.assert()); + }, + }; }, } From 09825b07d27a39adc120f65328bfff9d1328e893 Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Tue, 16 Jun 2026 16:26:36 -0700 Subject: [PATCH 05/34] semicolon --- src/daemon.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/daemon.rs b/src/daemon.rs index 7123ac6..5bae468 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -150,7 +150,7 @@ pub(crate) async fn run(state: State, on_ready: Option) -> Result<()> { task_status = Some(s.code().unwrap_or(1)); info!("command exited with code {}", task_status.assert()); }, - }; + } }, } From bb2eefd52ac6fb879073a0c68aecc0c7ec556ef9 Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Tue, 16 Jun 2026 17:11:50 -0700 Subject: [PATCH 06/34] daemon: add test --- src/daemon.rs | 14 ++++----- tests/e2e.rs | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 7 deletions(-) diff --git a/src/daemon.rs b/src/daemon.rs index 5bae468..a5d177b 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -138,17 +138,17 @@ pub(crate) async fn run(state: State, on_ready: Option) -> Result<()> { r = &mut proxy_handle => error!(?r, "proxy exited"), Some(r) = &mut command_handle => { match r { - Err(_) => { - task_status = Some(1); - error!("command exited with code {}", task_status.assert()); + Err(e) => { + task_status = Some(1); + error!(?e, "command task failed"); }, Ok(Err(e)) => { - task_status = Some(1); - error!(?e, "command exited"); + task_status = Some(1); + error!(?e, "command exited"); }, Ok(Ok(s)) => { - task_status = Some(s.code().unwrap_or(1)); - info!("command exited with code {}", task_status.assert()); + task_status = Some(s.code().unwrap_or(1)); + info!("command exited with code {}", task_status.assert()); }, } }, diff --git a/tests/e2e.rs b/tests/e2e.rs index e019ddd..d4647db 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -422,6 +422,87 @@ fn test_graceful_shutdown() { ); } +#[test] +#[ignore = "e2e test requiring docker"] +fn test_wrapped_command_exit() { + let harness = Harness::new(); + + let proxy_cmd = r#"intermesh proxy -- sleep 1"#; + let node = harness.launch("root", proxy_cmd).assert(); + + // Verify admin socket exists + node.exec(&["test", "-S", "/var/run/intermesh/admin.sock"]) + .assert(); + + // Verify nftables rules were injected + wait(|| node.exec(&["nft", "list", "table", "ip", "intermesh"])).assert(); + + // Wait for admin socket to disappear. + wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); + + // Verify nftables rules were cleaned up + let result = node.exec(&["nft", "list", "table", "ip", "intermesh"]); + assert!( + result.is_err(), + "nftables table should be removed on shutdown" + ); +} + +#[test] +#[ignore = "e2e test requiring docker"] +fn test_wrapped_command_error() { + let harness = Harness::new(); + + let proxy_cmd = r#"intermesh proxy -- sleep"#; + let node = harness.launch("root", proxy_cmd).assert(); + + // Verify admin socket exists + node.exec(&["test", "-S", "/var/run/intermesh/admin.sock"]) + .assert(); + + // Verify nftables rules were injected + wait(|| node.exec(&["nft", "list", "table", "ip", "intermesh"])).assert(); + + // Wait for admin socket to disappear. + wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); + + // Verify nftables rules were cleaned up + let result = node.exec(&["nft", "list", "table", "ip", "intermesh"]); + assert!( + result.is_err(), + "nftables table should be removed on shutdown" + ); +} + +#[test] +#[ignore = "e2e test requiring docker"] +fn test_wrapped_command_killed() { + let harness = Harness::new(); + + let proxy_cmd = r#"intermesh proxy -- sleep"#; + let node = harness.launch("root", proxy_cmd).assert(); + + // Verify admin socket exists + node.exec(&["test", "-S", "/var/run/intermesh/admin.sock"]) + .assert(); + + // Verify nftables rules were injected + wait(|| node.exec(&["nft", "list", "table", "ip", "intermesh"])).assert(); + + // Kill wrapped command via SIGTERM + node.exec(&["pkill", "-TERM", "sleep"]).assert(); + + // Wait for admin socket to disappear. + wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); + + // Verify nftables rules were cleaned up + let result = node.exec(&["nft", "list", "table", "ip", "intermesh"]); + assert!( + result.is_err(), + "nftables table should be removed on shutdown" + ); +} + #[test] #[ignore = "e2e test requiring docker"] fn test_runtime_endorsements_are_saved() { From 87cacf2ccdccec8dfcf79bb8269a876ef4fa9e92 Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Tue, 16 Jun 2026 17:18:22 -0700 Subject: [PATCH 07/34] hashes --- tests/e2e.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e.rs b/tests/e2e.rs index d4647db..aaa3af1 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -453,7 +453,7 @@ fn test_wrapped_command_exit() { fn test_wrapped_command_error() { let harness = Harness::new(); - let proxy_cmd = r#"intermesh proxy -- sleep"#; + let proxy_cmd = r"intermesh proxy -- sleep"; let node = harness.launch("root", proxy_cmd).assert(); // Verify admin socket exists @@ -479,7 +479,7 @@ fn test_wrapped_command_error() { fn test_wrapped_command_killed() { let harness = Harness::new(); - let proxy_cmd = r#"intermesh proxy -- sleep"#; + let proxy_cmd = r"intermesh proxy -- sleep"; let node = harness.launch("root", proxy_cmd).assert(); // Verify admin socket exists From d81356038c4fa3940f78f9165f03f79ecd1132ce Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Tue, 16 Jun 2026 18:22:49 -0700 Subject: [PATCH 08/34] add exit code test --- tests/e2e.rs | 15 +++++++++++++-- tests/helpers/docker.rs | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/e2e.rs b/tests/e2e.rs index aaa3af1..e1fe9fe 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -8,6 +8,7 @@ use rayon::prelude::*; use serde::Deserialize; use std::collections::{BTreeMap, BTreeSet}; use std::net::{IpAddr, Ipv4Addr}; +use std::process::Command; #[macro_use] mod helpers; @@ -427,7 +428,7 @@ fn test_graceful_shutdown() { fn test_wrapped_command_exit() { let harness = Harness::new(); - let proxy_cmd = r#"intermesh proxy -- sleep 1"#; + let proxy_cmd = r"intermesh proxy -- sleep 1"; let node = harness.launch("root", proxy_cmd).assert(); // Verify admin socket exists @@ -479,7 +480,7 @@ fn test_wrapped_command_error() { fn test_wrapped_command_killed() { let harness = Harness::new(); - let proxy_cmd = r"intermesh proxy -- sleep"; + let proxy_cmd = r"intermesh proxy -- sleep inf"; let node = harness.launch("root", proxy_cmd).assert(); // Verify admin socket exists @@ -501,6 +502,16 @@ fn test_wrapped_command_killed() { result.is_err(), "nftables table should be removed on shutdown" ); + + let _ = Command::new("docker") + .args(["stop", &node.container]) + .output(); + let exit_code = Command::new("docker") + .args(["wait", &node.container]) + .output() + .expect("reading daemon exit code failed") + .stdout; + assert_eq!(String::from_utf8(exit_code).unwrap(), "1"); } #[test] diff --git a/tests/helpers/docker.rs b/tests/helpers/docker.rs index 4914cdc..48c1069 100644 --- a/tests/helpers/docker.rs +++ b/tests/helpers/docker.rs @@ -175,7 +175,7 @@ impl Drop for Harness { pub(crate) struct Node<'h> { pub(crate) name: String, - container: String, + pub(crate) container: String, pub(crate) ip: IpAddr, imid: OnceLock, _harness: PhantomData<&'h ()>, From 9adcacdac1d2dfc2553b9df84c235411d074ce75 Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Tue, 16 Jun 2026 18:56:51 -0700 Subject: [PATCH 09/34] test all exit codes --- tests/e2e.rs | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/tests/e2e.rs b/tests/e2e.rs index e1fe9fe..ea34ba0 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -447,6 +447,19 @@ fn test_wrapped_command_exit() { result.is_err(), "nftables table should be removed on shutdown" ); + + let _ = Command::new("docker") + .args(["stop", &node.container]) + .output(); + let exit_code = Command::new("docker") + .args(["wait", &node.container]) + .output() + .expect("reading daemon exit code failed") + .stdout; + assert_eq!(String::from_utf8(exit_code).unwrap(), "0"); + let _ = Command::new("docker") + .args(["start", &node.container]) + .output(); } #[test] @@ -454,7 +467,7 @@ fn test_wrapped_command_exit() { fn test_wrapped_command_error() { let harness = Harness::new(); - let proxy_cmd = r"intermesh proxy -- sleep"; + let proxy_cmd = r#"intermesh proxy -- sh -c "exit 123""#; let node = harness.launch("root", proxy_cmd).assert(); // Verify admin socket exists @@ -473,6 +486,19 @@ fn test_wrapped_command_error() { result.is_err(), "nftables table should be removed on shutdown" ); + + let _ = Command::new("docker") + .args(["stop", &node.container]) + .output(); + let exit_code = Command::new("docker") + .args(["wait", &node.container]) + .output() + .expect("reading daemon exit code failed") + .stdout; + assert_eq!(String::from_utf8(exit_code).unwrap(), "123"); + let _ = Command::new("docker") + .args(["start", &node.container]) + .output(); } #[test] @@ -512,6 +538,9 @@ fn test_wrapped_command_killed() { .expect("reading daemon exit code failed") .stdout; assert_eq!(String::from_utf8(exit_code).unwrap(), "1"); + let _ = Command::new("docker") + .args(["start", &node.container]) + .output(); } #[test] From 55ce1d44a7f1c86b10f03452fe212013150abde2 Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Tue, 16 Jun 2026 19:27:08 -0700 Subject: [PATCH 10/34] undo exit code test --- tests/e2e.rs | 72 ++++++++++++++++++++++++++-------------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/tests/e2e.rs b/tests/e2e.rs index ea34ba0..58a9f8b 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -448,18 +448,18 @@ fn test_wrapped_command_exit() { "nftables table should be removed on shutdown" ); - let _ = Command::new("docker") - .args(["stop", &node.container]) - .output(); - let exit_code = Command::new("docker") - .args(["wait", &node.container]) - .output() - .expect("reading daemon exit code failed") - .stdout; - assert_eq!(String::from_utf8(exit_code).unwrap(), "0"); - let _ = Command::new("docker") - .args(["start", &node.container]) - .output(); + // let _ = Command::new("docker") + // .args(["stop", &node.container]) + // .output(); + // let exit_code = Command::new("docker") + // .args(["wait", &node.container]) + // .output() + // .expect("reading daemon exit code failed") + // .stdout; + // assert_eq!(String::from_utf8(exit_code).unwrap().trim(), "0"); + // let _ = Command::new("docker") + // .args(["start", &node.container]) + // .output(); } #[test] @@ -487,18 +487,18 @@ fn test_wrapped_command_error() { "nftables table should be removed on shutdown" ); - let _ = Command::new("docker") - .args(["stop", &node.container]) - .output(); - let exit_code = Command::new("docker") - .args(["wait", &node.container]) - .output() - .expect("reading daemon exit code failed") - .stdout; - assert_eq!(String::from_utf8(exit_code).unwrap(), "123"); - let _ = Command::new("docker") - .args(["start", &node.container]) - .output(); + // let _ = Command::new("docker") + // .args(["stop", &node.container]) + // .output(); + // let exit_code = Command::new("docker") + // .args(["wait", &node.container]) + // .output() + // .expect("reading daemon exit code failed") + // .stdout; + // assert_eq!(String::from_utf8(exit_code).unwrap().trim(), "123"); + // let _ = Command::new("docker") + // .args(["start", &node.container]) + // .output(); } #[test] @@ -529,18 +529,18 @@ fn test_wrapped_command_killed() { "nftables table should be removed on shutdown" ); - let _ = Command::new("docker") - .args(["stop", &node.container]) - .output(); - let exit_code = Command::new("docker") - .args(["wait", &node.container]) - .output() - .expect("reading daemon exit code failed") - .stdout; - assert_eq!(String::from_utf8(exit_code).unwrap(), "1"); - let _ = Command::new("docker") - .args(["start", &node.container]) - .output(); + // let _ = Command::new("docker") + // .args(["stop", &node.container]) + // .output(); + // let exit_code = Command::new("docker") + // .args(["wait", &node.container]) + // .output() + // .expect("reading daemon exit code failed") + // .stdout; + // assert_eq!(String::from_utf8(exit_code).unwrap().trim(), "1"); + // let _ = Command::new("docker") + // .args(["start", &node.container]) + // .output(); } #[test] From a17d6d011b232c2aa1fcd77af1cd624070cfe422 Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Tue, 16 Jun 2026 19:32:00 -0700 Subject: [PATCH 11/34] unused import --- tests/e2e.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e.rs b/tests/e2e.rs index 58a9f8b..635bbaa 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -8,7 +8,7 @@ use rayon::prelude::*; use serde::Deserialize; use std::collections::{BTreeMap, BTreeSet}; use std::net::{IpAddr, Ipv4Addr}; -use std::process::Command; +// use std::process::Command; #[macro_use] mod helpers; From 7dd8944f248acc6c8c67980e39094e45503defb4 Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Tue, 16 Jun 2026 19:48:46 -0700 Subject: [PATCH 12/34] don't exec after failure --- tests/e2e.rs | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/e2e.rs b/tests/e2e.rs index 635bbaa..4ee6705 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -8,7 +8,7 @@ use rayon::prelude::*; use serde::Deserialize; use std::collections::{BTreeMap, BTreeSet}; use std::net::{IpAddr, Ipv4Addr}; -// use std::process::Command; +use std::process::Command; #[macro_use] mod helpers; @@ -519,25 +519,25 @@ fn test_wrapped_command_killed() { // Kill wrapped command via SIGTERM node.exec(&["pkill", "-TERM", "sleep"]).assert(); - // Wait for admin socket to disappear. - wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); - - // Verify nftables rules were cleaned up - let result = node.exec(&["nft", "list", "table", "ip", "intermesh"]); - assert!( - result.is_err(), - "nftables table should be removed on shutdown" - ); + // // Wait for admin socket to disappear. + // wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); + // + // // Verify nftables rules were cleaned up + // let result = node.exec(&["nft", "list", "table", "ip", "intermesh"]); + // assert!( + // result.is_err(), + // "nftables table should be removed on shutdown" + // ); // let _ = Command::new("docker") // .args(["stop", &node.container]) // .output(); - // let exit_code = Command::new("docker") - // .args(["wait", &node.container]) - // .output() - // .expect("reading daemon exit code failed") - // .stdout; - // assert_eq!(String::from_utf8(exit_code).unwrap().trim(), "1"); + let exit_code = Command::new("docker") + .args(["wait", &node.container]) + .output() + .expect("reading daemon exit code failed") + .stdout; + assert_eq!(String::from_utf8(exit_code).unwrap().trim(), "1"); // let _ = Command::new("docker") // .args(["start", &node.container]) // .output(); From 5401630f243e0850ea8895aa1547c9a5c7727101 Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Tue, 16 Jun 2026 20:03:33 -0700 Subject: [PATCH 13/34] verify cleanup --- tests/e2e.rs | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/e2e.rs b/tests/e2e.rs index 4ee6705..388ce47 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -474,11 +474,11 @@ fn test_wrapped_command_error() { node.exec(&["test", "-S", "/var/run/intermesh/admin.sock"]) .assert(); - // Verify nftables rules were injected - wait(|| node.exec(&["nft", "list", "table", "ip", "intermesh"])).assert(); - - // Wait for admin socket to disappear. - wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); + // // Verify nftables rules were injected + // wait(|| node.exec(&["nft", "list", "table", "ip", "intermesh"])).assert(); + // + // // Wait for admin socket to disappear. + // wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); // Verify nftables rules were cleaned up let result = node.exec(&["nft", "list", "table", "ip", "intermesh"]); @@ -490,12 +490,12 @@ fn test_wrapped_command_error() { // let _ = Command::new("docker") // .args(["stop", &node.container]) // .output(); - // let exit_code = Command::new("docker") - // .args(["wait", &node.container]) - // .output() - // .expect("reading daemon exit code failed") - // .stdout; - // assert_eq!(String::from_utf8(exit_code).unwrap().trim(), "123"); + let exit_code = Command::new("docker") + .args(["wait", &node.container]) + .output() + .expect("reading daemon exit code failed") + .stdout; + assert_eq!(String::from_utf8(exit_code).unwrap().trim(), "123"); // let _ = Command::new("docker") // .args(["start", &node.container]) // .output(); @@ -522,12 +522,12 @@ fn test_wrapped_command_killed() { // // Wait for admin socket to disappear. // wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); // - // // Verify nftables rules were cleaned up - // let result = node.exec(&["nft", "list", "table", "ip", "intermesh"]); - // assert!( - // result.is_err(), - // "nftables table should be removed on shutdown" - // ); + // Verify nftables rules were cleaned up + let result = node.exec(&["nft", "list", "table", "ip", "intermesh"]); + assert!( + result.is_err(), + "nftables table should be removed on shutdown" + ); // let _ = Command::new("docker") // .args(["stop", &node.container]) From 0f39481af8e12fe051348fab63341d01f9902014 Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Tue, 16 Jun 2026 20:39:18 -0700 Subject: [PATCH 14/34] create exit code test helper --- tests/e2e.rs | 36 ++++++++++++++++-------------------- tests/helpers/docker.rs | 9 +++++++++ 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/tests/e2e.rs b/tests/e2e.rs index 388ce47..e7db63e 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -451,12 +451,7 @@ fn test_wrapped_command_exit() { // let _ = Command::new("docker") // .args(["stop", &node.container]) // .output(); - // let exit_code = Command::new("docker") - // .args(["wait", &node.container]) - // .output() - // .expect("reading daemon exit code failed") - // .stdout; - // assert_eq!(String::from_utf8(exit_code).unwrap().trim(), "0"); + assert_eq!(node.exit_code().assert(), 0); // let _ = Command::new("docker") // .args(["start", &node.container]) // .output(); @@ -470,10 +465,17 @@ fn test_wrapped_command_error() { let proxy_cmd = r#"intermesh proxy -- sh -c "exit 123""#; let node = harness.launch("root", proxy_cmd).assert(); - // Verify admin socket exists - node.exec(&["test", "-S", "/var/run/intermesh/admin.sock"]) - .assert(); + let exit_code = Command::new("docker") + .args(["wait", &node.container]) + .output() + .expect("reading daemon exit code failed") + .stdout; + assert_eq!(String::from_utf8(exit_code).unwrap().trim(), "123"); + // // Verify admin socket exists + // node.exec(&["test", "-S", "/var/run/intermesh/admin.sock"]) + // .assert(); + // // // Verify nftables rules were injected // wait(|| node.exec(&["nft", "list", "table", "ip", "intermesh"])).assert(); // @@ -481,21 +483,15 @@ fn test_wrapped_command_error() { // wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); // Verify nftables rules were cleaned up - let result = node.exec(&["nft", "list", "table", "ip", "intermesh"]); - assert!( - result.is_err(), - "nftables table should be removed on shutdown" - ); + // let result = node.exec(&["nft", "list", "table", "ip", "intermesh"]); + // assert!( + // result.is_err(), + // "nftables table should be removed on shutdown" + // ); // let _ = Command::new("docker") // .args(["stop", &node.container]) // .output(); - let exit_code = Command::new("docker") - .args(["wait", &node.container]) - .output() - .expect("reading daemon exit code failed") - .stdout; - assert_eq!(String::from_utf8(exit_code).unwrap().trim(), "123"); // let _ = Command::new("docker") // .args(["start", &node.container]) // .output(); diff --git a/tests/helpers/docker.rs b/tests/helpers/docker.rs index 48c1069..e803b6b 100644 --- a/tests/helpers/docker.rs +++ b/tests/helpers/docker.rs @@ -196,6 +196,15 @@ impl Node<'_> { Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) } + pub(crate) fn exit_code(&self) -> Result { + let exit_code = Command::new("docker") + .args(["wait", &self.container]) + .output() + .expect("reading daemon exit code failed") + .stdout; + Ok(String::from_utf8(exit_code)?.trim().parse::()?) + } + pub(crate) fn get_dump(&self) -> Result { let dump_toml = self.exec(&["intermesh", "debug", "dump", "--toml"])?; toml::from_str(&dump_toml).map_err(Into::into) From b1b6f210def0125e4895f867259f20590dbd5620 Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Tue, 16 Jun 2026 21:25:19 -0700 Subject: [PATCH 15/34] clean up exit code test --- tests/e2e.rs | 31 +++++++++---------------------- tests/helpers/docker.rs | 17 ++++++++++++++--- 2 files changed, 23 insertions(+), 25 deletions(-) diff --git a/tests/e2e.rs b/tests/e2e.rs index e7db63e..ccbc219 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -8,7 +8,6 @@ use rayon::prelude::*; use serde::Deserialize; use std::collections::{BTreeMap, BTreeSet}; use std::net::{IpAddr, Ipv4Addr}; -use std::process::Command; #[macro_use] mod helpers; @@ -465,12 +464,7 @@ fn test_wrapped_command_error() { let proxy_cmd = r#"intermesh proxy -- sh -c "exit 123""#; let node = harness.launch("root", proxy_cmd).assert(); - let exit_code = Command::new("docker") - .args(["wait", &node.container]) - .output() - .expect("reading daemon exit code failed") - .stdout; - assert_eq!(String::from_utf8(exit_code).unwrap().trim(), "123"); + assert_eq!(node.exit_code().assert(), 123); // // Verify admin socket exists // node.exec(&["test", "-S", "/var/run/intermesh/admin.sock"]) @@ -518,25 +512,18 @@ fn test_wrapped_command_killed() { // // Wait for admin socket to disappear. // wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); // - // Verify nftables rules were cleaned up - let result = node.exec(&["nft", "list", "table", "ip", "intermesh"]); - assert!( - result.is_err(), - "nftables table should be removed on shutdown" - ); + // // Verify nftables rules were cleaned up + // let result = node.exec(&["nft", "list", "table", "ip", "intermesh"]); + // assert!( + // result.is_err(), + // "nftables table should be removed on shutdown" + // ); // let _ = Command::new("docker") // .args(["stop", &node.container]) // .output(); - let exit_code = Command::new("docker") - .args(["wait", &node.container]) - .output() - .expect("reading daemon exit code failed") - .stdout; - assert_eq!(String::from_utf8(exit_code).unwrap().trim(), "1"); - // let _ = Command::new("docker") - // .args(["start", &node.container]) - // .output(); + assert_eq!(node.exit_code().assert(), 1); + // let _ = node.start_container(); } #[test] diff --git a/tests/helpers/docker.rs b/tests/helpers/docker.rs index e803b6b..b6f2da1 100644 --- a/tests/helpers/docker.rs +++ b/tests/helpers/docker.rs @@ -175,7 +175,7 @@ impl Drop for Harness { pub(crate) struct Node<'h> { pub(crate) name: String, - pub(crate) container: String, + container: String, pub(crate) ip: IpAddr, imid: OnceLock, _harness: PhantomData<&'h ()>, @@ -199,12 +199,23 @@ impl Node<'_> { pub(crate) fn exit_code(&self) -> Result { let exit_code = Command::new("docker") .args(["wait", &self.container]) - .output() - .expect("reading daemon exit code failed") + .output()? .stdout; Ok(String::from_utf8(exit_code)?.trim().parse::()?) } + pub(crate) fn start_container(&self) -> Result { + let output = Command::new("docker") + .args(["start", &self.container]) + .output()?; + + if !output.status.success() { + bail!("{}", command_error(&output, &self.container)) + } + + Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) + } + pub(crate) fn get_dump(&self) -> Result { let dump_toml = self.exec(&["intermesh", "debug", "dump", "--toml"])?; toml::from_str(&dump_toml).map_err(Into::into) From d83baacb9fa3937f180cbd416a3b129c227c9e98 Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Tue, 16 Jun 2026 21:31:59 -0700 Subject: [PATCH 16/34] restart containers and verify nftable rules clean up --- tests/e2e.rs | 59 ++++++++++++++++------------------------------------ 1 file changed, 18 insertions(+), 41 deletions(-) diff --git a/tests/e2e.rs b/tests/e2e.rs index ccbc219..7f3cbff 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -447,13 +447,8 @@ fn test_wrapped_command_exit() { "nftables table should be removed on shutdown" ); - // let _ = Command::new("docker") - // .args(["stop", &node.container]) - // .output(); + // Verify daemon's exit code assert_eq!(node.exit_code().assert(), 0); - // let _ = Command::new("docker") - // .args(["start", &node.container]) - // .output(); } #[test] @@ -464,31 +459,17 @@ fn test_wrapped_command_error() { let proxy_cmd = r#"intermesh proxy -- sh -c "exit 123""#; let node = harness.launch("root", proxy_cmd).assert(); + // Verify daemon's exit code assert_eq!(node.exit_code().assert(), 123); - // // Verify admin socket exists - // node.exec(&["test", "-S", "/var/run/intermesh/admin.sock"]) - // .assert(); - // - // // Verify nftables rules were injected - // wait(|| node.exec(&["nft", "list", "table", "ip", "intermesh"])).assert(); - // - // // Wait for admin socket to disappear. - // wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); + let _ = node.start_container(); // Verify nftables rules were cleaned up - // let result = node.exec(&["nft", "list", "table", "ip", "intermesh"]); - // assert!( - // result.is_err(), - // "nftables table should be removed on shutdown" - // ); - - // let _ = Command::new("docker") - // .args(["stop", &node.container]) - // .output(); - // let _ = Command::new("docker") - // .args(["start", &node.container]) - // .output(); + let result = node.exec(&["nft", "list", "table", "ip", "intermesh"]); + assert!( + result.is_err(), + "nftables table should be removed on shutdown" + ); } #[test] @@ -509,21 +490,17 @@ fn test_wrapped_command_killed() { // Kill wrapped command via SIGTERM node.exec(&["pkill", "-TERM", "sleep"]).assert(); - // // Wait for admin socket to disappear. - // wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); - // - // // Verify nftables rules were cleaned up - // let result = node.exec(&["nft", "list", "table", "ip", "intermesh"]); - // assert!( - // result.is_err(), - // "nftables table should be removed on shutdown" - // ); - - // let _ = Command::new("docker") - // .args(["stop", &node.container]) - // .output(); + // Verify daemon's exit code assert_eq!(node.exit_code().assert(), 1); - // let _ = node.start_container(); + + let _ = node.start_container(); + + // Verify nftables rules were cleaned up + let result = node.exec(&["nft", "list", "table", "ip", "intermesh"]); + assert!( + result.is_err(), + "nftables table should be removed on shutdown" + ); } #[test] From 13bb284fbbeac3baad5565754e2fc0b09929aede Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Tue, 16 Jun 2026 21:39:20 -0700 Subject: [PATCH 17/34] remove failed execs --- tests/e2e.rs | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/tests/e2e.rs b/tests/e2e.rs index 7f3cbff..0d3652d 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -430,15 +430,10 @@ fn test_wrapped_command_exit() { let proxy_cmd = r"intermesh proxy -- sleep 1"; let node = harness.launch("root", proxy_cmd).assert(); - // Verify admin socket exists - node.exec(&["test", "-S", "/var/run/intermesh/admin.sock"]) - .assert(); - - // Verify nftables rules were injected - wait(|| node.exec(&["nft", "list", "table", "ip", "intermesh"])).assert(); + // Verify daemon's exit code + assert_eq!(node.exit_code().assert(), 0); - // Wait for admin socket to disappear. - wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); + let _ = node.start_container(); // Verify nftables rules were cleaned up let result = node.exec(&["nft", "list", "table", "ip", "intermesh"]); @@ -446,9 +441,6 @@ fn test_wrapped_command_exit() { result.is_err(), "nftables table should be removed on shutdown" ); - - // Verify daemon's exit code - assert_eq!(node.exit_code().assert(), 0); } #[test] From 32fba1283e0bdb4d1c5b99317ed46c746c285570 Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Tue, 16 Jun 2026 23:39:21 -0700 Subject: [PATCH 18/34] start container does not need to return output --- tests/helpers/docker.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/helpers/docker.rs b/tests/helpers/docker.rs index b6f2da1..0c38338 100644 --- a/tests/helpers/docker.rs +++ b/tests/helpers/docker.rs @@ -201,10 +201,11 @@ impl Node<'_> { .args(["wait", &self.container]) .output()? .stdout; + Ok(String::from_utf8(exit_code)?.trim().parse::()?) } - pub(crate) fn start_container(&self) -> Result { + pub(crate) fn start_container(&self) -> Result<()> { let output = Command::new("docker") .args(["start", &self.container]) .output()?; @@ -213,7 +214,7 @@ impl Node<'_> { bail!("{}", command_error(&output, &self.container)) } - Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) + Ok(()) } pub(crate) fn get_dump(&self) -> Result { From 96f8f271014c45a436012b7d0139fe245ebd01e7 Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Wed, 17 Jun 2026 12:03:38 -0700 Subject: [PATCH 19/34] restore exit test checks --- tests/e2e.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/e2e.rs b/tests/e2e.rs index 0d3652d..9cbaf09 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -427,13 +427,21 @@ fn test_graceful_shutdown() { fn test_wrapped_command_exit() { let harness = Harness::new(); - let proxy_cmd = r"intermesh proxy -- sleep 1"; + let proxy_cmd = r"intermesh proxy -- sleep 5"; let node = harness.launch("root", proxy_cmd).assert(); - // Verify daemon's exit code - assert_eq!(node.exit_code().assert(), 0); + // Verify admin socket exists + node.exec(&["test", "-S", "/var/run/intermesh/admin.sock"]) + .assert(); - let _ = node.start_container(); + // Verify nftables rules were injected + wait(|| node.exec(&["nft", "list", "table", "ip", "intermesh"])).assert(); + + // Graceful shutdown via SIGTERM + node.exec(&["pkill", "-TERM", "intermesh"]).assert(); + + // Wait for admin socket to disappear. + wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); // Verify nftables rules were cleaned up let result = node.exec(&["nft", "list", "table", "ip", "intermesh"]); @@ -441,6 +449,9 @@ fn test_wrapped_command_exit() { result.is_err(), "nftables table should be removed on shutdown" ); + + // Verify daemon's exit code + assert_eq!(node.exit_code().assert(), 0); } #[test] From e063cd0b85f567eb1ec54386db60b6429c1f1d9c Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Wed, 17 Jun 2026 12:09:58 -0700 Subject: [PATCH 20/34] distinguish docker command error --- tests/helpers/docker.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/helpers/docker.rs b/tests/helpers/docker.rs index 0c38338..8922cc4 100644 --- a/tests/helpers/docker.rs +++ b/tests/helpers/docker.rs @@ -197,12 +197,15 @@ impl Node<'_> { } pub(crate) fn exit_code(&self) -> Result { - let exit_code = Command::new("docker") + let output = Command::new("docker") .args(["wait", &self.container]) - .output()? - .stdout; + .output()?; + + if !output.status.success() { + bail!("{}", command_error(&output, &self.container)) + } - Ok(String::from_utf8(exit_code)?.trim().parse::()?) + Ok(String::from_utf8(output.stdout)?.trim().parse::()?) } pub(crate) fn start_container(&self) -> Result<()> { From 4bb4915954a038b22d71eecc7bda2091eb7602d9 Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Wed, 17 Jun 2026 12:20:20 -0700 Subject: [PATCH 21/34] remove accidental sigterm command --- tests/e2e.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/e2e.rs b/tests/e2e.rs index 9cbaf09..07a0be9 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -437,9 +437,6 @@ fn test_wrapped_command_exit() { // Verify nftables rules were injected wait(|| node.exec(&["nft", "list", "table", "ip", "intermesh"])).assert(); - // Graceful shutdown via SIGTERM - node.exec(&["pkill", "-TERM", "intermesh"]).assert(); - // Wait for admin socket to disappear. wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); From 52d82a05090f06d9746cabba3c3824269ac5cf5a Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Wed, 17 Jun 2026 15:33:50 -0700 Subject: [PATCH 22/34] try to verify nftable rules before shutdown --- tests/e2e.rs | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/tests/e2e.rs b/tests/e2e.rs index 07a0be9..41ffd06 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -434,9 +434,6 @@ fn test_wrapped_command_exit() { node.exec(&["test", "-S", "/var/run/intermesh/admin.sock"]) .assert(); - // Verify nftables rules were injected - wait(|| node.exec(&["nft", "list", "table", "ip", "intermesh"])).assert(); - // Wait for admin socket to disappear. wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); @@ -459,17 +456,15 @@ fn test_wrapped_command_error() { let proxy_cmd = r#"intermesh proxy -- sh -c "exit 123""#; let node = harness.launch("root", proxy_cmd).assert(); - // Verify daemon's exit code - assert_eq!(node.exit_code().assert(), 123); - - let _ = node.start_container(); - // Verify nftables rules were cleaned up let result = node.exec(&["nft", "list", "table", "ip", "intermesh"]); assert!( result.is_err(), "nftables table should be removed on shutdown" ); + + // Verify daemon's exit code + assert_eq!(node.exit_code().assert(), 123); } #[test] @@ -490,17 +485,15 @@ fn test_wrapped_command_killed() { // Kill wrapped command via SIGTERM node.exec(&["pkill", "-TERM", "sleep"]).assert(); - // Verify daemon's exit code - assert_eq!(node.exit_code().assert(), 1); - - let _ = node.start_container(); - // Verify nftables rules were cleaned up let result = node.exec(&["nft", "list", "table", "ip", "intermesh"]); assert!( result.is_err(), "nftables table should be removed on shutdown" ); + + // Verify daemon's exit code + assert_eq!(node.exit_code().assert(), 1); } #[test] From 6d1baa7aaa6e768758964fc7d8cbee2de7f719ab Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Wed, 17 Jun 2026 15:58:09 -0700 Subject: [PATCH 23/34] different nftable rules check timing --- tests/e2e.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/e2e.rs b/tests/e2e.rs index 41ffd06..23a95e2 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -434,9 +434,6 @@ fn test_wrapped_command_exit() { node.exec(&["test", "-S", "/var/run/intermesh/admin.sock"]) .assert(); - // Wait for admin socket to disappear. - wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); - // Verify nftables rules were cleaned up let result = node.exec(&["nft", "list", "table", "ip", "intermesh"]); assert!( @@ -456,6 +453,9 @@ fn test_wrapped_command_error() { let proxy_cmd = r#"intermesh proxy -- sh -c "exit 123""#; let node = harness.launch("root", proxy_cmd).assert(); + // Wait for admin socket to disappear. + wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); + // Verify nftables rules were cleaned up let result = node.exec(&["nft", "list", "table", "ip", "intermesh"]); assert!( From 1a29432da32218993435cc82562239dfcd05bf80 Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Wed, 17 Jun 2026 16:40:40 -0700 Subject: [PATCH 24/34] use two runs to check exit code and clean up for command error case --- tests/e2e.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/e2e.rs b/tests/e2e.rs index 23a95e2..382f1fd 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -430,9 +430,8 @@ fn test_wrapped_command_exit() { let proxy_cmd = r"intermesh proxy -- sleep 5"; let node = harness.launch("root", proxy_cmd).assert(); - // Verify admin socket exists - node.exec(&["test", "-S", "/var/run/intermesh/admin.sock"]) - .assert(); + // Wait for admin socket to disappear. + wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); // Verify nftables rules were cleaned up let result = node.exec(&["nft", "list", "table", "ip", "intermesh"]); @@ -453,6 +452,15 @@ fn test_wrapped_command_error() { let proxy_cmd = r#"intermesh proxy -- sh -c "exit 123""#; let node = harness.launch("root", proxy_cmd).assert(); + // Verify daemon's exit code + assert_eq!(node.exit_code().assert(), 123); + + // Start a new container + let proxy_cmd = r"sleep inf"; + let node = harness.launch("root", proxy_cmd).assert(); + node.start_daemon(&["--intercept", "--", "sh", "-c", r#""exit 123""#]) + .assert(); + // Wait for admin socket to disappear. wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); @@ -462,9 +470,6 @@ fn test_wrapped_command_error() { result.is_err(), "nftables table should be removed on shutdown" ); - - // Verify daemon's exit code - assert_eq!(node.exit_code().assert(), 123); } #[test] From 216e170e98b55818659e1eaf06f51682b876c503 Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Wed, 17 Jun 2026 16:44:41 -0700 Subject: [PATCH 25/34] use two runs for every case --- tests/e2e.rs | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/tests/e2e.rs b/tests/e2e.rs index 382f1fd..67ad27f 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -427,9 +427,18 @@ fn test_graceful_shutdown() { fn test_wrapped_command_exit() { let harness = Harness::new(); - let proxy_cmd = r"intermesh proxy -- sleep 5"; + let proxy_cmd = r"intermesh proxy -- sleep 1"; let node = harness.launch("root", proxy_cmd).assert(); + // Verify daemon's exit code + assert_eq!(node.exit_code().assert(), 0); + + // Start a new container + let proxy_cmd = r"sleep inf"; + let node = harness.launch("root", proxy_cmd).assert(); + node.start_daemon(&["--intercept", "--", "sleep", "1"]) + .assert(); + // Wait for admin socket to disappear. wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); @@ -439,9 +448,6 @@ fn test_wrapped_command_exit() { result.is_err(), "nftables table should be removed on shutdown" ); - - // Verify daemon's exit code - assert_eq!(node.exit_code().assert(), 0); } #[test] @@ -490,15 +496,24 @@ fn test_wrapped_command_killed() { // Kill wrapped command via SIGTERM node.exec(&["pkill", "-TERM", "sleep"]).assert(); + // Verify daemon's exit code + assert_eq!(node.exit_code().assert(), 1); + + // Start a new container + let proxy_cmd = r"sleep inf"; + let node = harness.launch("root", proxy_cmd).assert(); + node.start_daemon(&["--intercept", "--", "sleep", "inf"]) + .assert(); + + // Wait for admin socket to disappear. + wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); + // Verify nftables rules were cleaned up let result = node.exec(&["nft", "list", "table", "ip", "intermesh"]); assert!( result.is_err(), "nftables table should be removed on shutdown" ); - - // Verify daemon's exit code - assert_eq!(node.exit_code().assert(), 1); } #[test] From 6b8b3d1db1f101eaf433663294805bc0212bb9dd Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Wed, 17 Jun 2026 16:58:00 -0700 Subject: [PATCH 26/34] add missing sigterm --- tests/e2e.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/e2e.rs b/tests/e2e.rs index 67ad27f..745ee4a 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -505,6 +505,9 @@ fn test_wrapped_command_killed() { node.start_daemon(&["--intercept", "--", "sleep", "inf"]) .assert(); + // Kill wrapped command via SIGTERM + node.exec(&["pkill", "-TERM", "sleep"]).assert(); + // Wait for admin socket to disappear. wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); From 036fd23663aa809b5cc83039aa31641f8f627f0e Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Wed, 17 Jun 2026 17:20:59 -0700 Subject: [PATCH 27/34] replace proxy commands --- tests/e2e.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/e2e.rs b/tests/e2e.rs index 745ee4a..f161780 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -434,8 +434,8 @@ fn test_wrapped_command_exit() { assert_eq!(node.exit_code().assert(), 0); // Start a new container - let proxy_cmd = r"sleep inf"; - let node = harness.launch("root", proxy_cmd).assert(); + let cmd = r"sleep inf"; + let node = harness.launch("root", cmd).assert(); node.start_daemon(&["--intercept", "--", "sleep", "1"]) .assert(); @@ -462,9 +462,9 @@ fn test_wrapped_command_error() { assert_eq!(node.exit_code().assert(), 123); // Start a new container - let proxy_cmd = r"sleep inf"; - let node = harness.launch("root", proxy_cmd).assert(); - node.start_daemon(&["--intercept", "--", "sh", "-c", r#""exit 123""#]) + let cmd = r"sleep inf"; + let node = harness.launch("root", cmd).assert(); + node.start_daemon(&["--intercept", "--", "exit", "123"]) .assert(); // Wait for admin socket to disappear. @@ -500,8 +500,8 @@ fn test_wrapped_command_killed() { assert_eq!(node.exit_code().assert(), 1); // Start a new container - let proxy_cmd = r"sleep inf"; - let node = harness.launch("root", proxy_cmd).assert(); + let cmd = r"tail -f /dev/null"; + let node = harness.launch("root", cmd).assert(); node.start_daemon(&["--intercept", "--", "sleep", "inf"]) .assert(); From e51012b455fdd1a824fc2b3d1aa4b265dee45503 Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Wed, 17 Jun 2026 17:27:22 -0700 Subject: [PATCH 28/34] remove start_container helper --- tests/helpers/docker.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tests/helpers/docker.rs b/tests/helpers/docker.rs index 8922cc4..57ea833 100644 --- a/tests/helpers/docker.rs +++ b/tests/helpers/docker.rs @@ -208,18 +208,6 @@ impl Node<'_> { Ok(String::from_utf8(output.stdout)?.trim().parse::()?) } - pub(crate) fn start_container(&self) -> Result<()> { - let output = Command::new("docker") - .args(["start", &self.container]) - .output()?; - - if !output.status.success() { - bail!("{}", command_error(&output, &self.container)) - } - - Ok(()) - } - pub(crate) fn get_dump(&self) -> Result { let dump_toml = self.exec(&["intermesh", "debug", "dump", "--toml"])?; toml::from_str(&dump_toml).map_err(Into::into) From 53d5aed623bf1d9a919698894e74bcd170f23b0f Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Thu, 18 Jun 2026 10:55:35 -0700 Subject: [PATCH 29/34] minor cleanup --- tests/e2e.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/e2e.rs b/tests/e2e.rs index f161780..c63a290 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -424,7 +424,7 @@ fn test_graceful_shutdown() { #[test] #[ignore = "e2e test requiring docker"] -fn test_wrapped_command_exit() { +fn test_proxy_command_exit() { let harness = Harness::new(); let proxy_cmd = r"intermesh proxy -- sleep 1"; @@ -434,8 +434,7 @@ fn test_wrapped_command_exit() { assert_eq!(node.exit_code().assert(), 0); // Start a new container - let cmd = r"sleep inf"; - let node = harness.launch("root", cmd).assert(); + let node = harness.launch("root", CADDY).assert(); node.start_daemon(&["--intercept", "--", "sleep", "1"]) .assert(); @@ -452,7 +451,7 @@ fn test_wrapped_command_exit() { #[test] #[ignore = "e2e test requiring docker"] -fn test_wrapped_command_error() { +fn test_proxy_command_error() { let harness = Harness::new(); let proxy_cmd = r#"intermesh proxy -- sh -c "exit 123""#; @@ -462,8 +461,7 @@ fn test_wrapped_command_error() { assert_eq!(node.exit_code().assert(), 123); // Start a new container - let cmd = r"sleep inf"; - let node = harness.launch("root", cmd).assert(); + let node = harness.launch("root", CADDY).assert(); node.start_daemon(&["--intercept", "--", "exit", "123"]) .assert(); @@ -480,7 +478,7 @@ fn test_wrapped_command_error() { #[test] #[ignore = "e2e test requiring docker"] -fn test_wrapped_command_killed() { +fn test_proxy_command_killed() { let harness = Harness::new(); let proxy_cmd = r"intermesh proxy -- sleep inf"; @@ -500,8 +498,7 @@ fn test_wrapped_command_killed() { assert_eq!(node.exit_code().assert(), 1); // Start a new container - let cmd = r"tail -f /dev/null"; - let node = harness.launch("root", cmd).assert(); + let node = harness.launch("root", CADDY).assert(); node.start_daemon(&["--intercept", "--", "sleep", "inf"]) .assert(); From ca1985ad785422c18964dadcfe2514e8932db1f4 Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Thu, 18 Jun 2026 11:05:08 -0700 Subject: [PATCH 30/34] improve comments --- src/daemon.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/daemon.rs b/src/daemon.rs index a5d177b..19f54aa 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -140,7 +140,7 @@ pub(crate) async fn run(state: State, on_ready: Option) -> Result<()> { match r { Err(e) => { task_status = Some(1); - error!(?e, "command task failed"); + error!(?e, "command failed"); }, Ok(Err(e)) => { task_status = Some(1); @@ -180,8 +180,11 @@ pub(crate) async fn run(state: State, on_ready: Option) -> Result<()> { } info!("shutdown complete"); + + // if proxy command exited cleanly, exit with its status code. if let Some(status) = task_status { process::exit(status) } + Ok(()) } From 1177897cc08dd4fd538ab652869cf9103867077b Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Thu, 18 Jun 2026 12:05:03 -0700 Subject: [PATCH 31/34] revert container idle commands --- tests/e2e.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/e2e.rs b/tests/e2e.rs index c63a290..867ad1e 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -434,7 +434,8 @@ fn test_proxy_command_exit() { assert_eq!(node.exit_code().assert(), 0); // Start a new container - let node = harness.launch("root", CADDY).assert(); + let cmd = r"tail -f /dev/null"; + let node = harness.launch("root", cmd).assert(); node.start_daemon(&["--intercept", "--", "sleep", "1"]) .assert(); @@ -461,7 +462,8 @@ fn test_proxy_command_error() { assert_eq!(node.exit_code().assert(), 123); // Start a new container - let node = harness.launch("root", CADDY).assert(); + let cmd = r"tail -f /dev/null"; + let node = harness.launch("root", cmd).assert(); node.start_daemon(&["--intercept", "--", "exit", "123"]) .assert(); @@ -498,7 +500,8 @@ fn test_proxy_command_killed() { assert_eq!(node.exit_code().assert(), 1); // Start a new container - let node = harness.launch("root", CADDY).assert(); + let cmd = r"tail -f /dev/null"; + let node = harness.launch("root", cmd).assert(); node.start_daemon(&["--intercept", "--", "sleep", "inf"]) .assert(); From a982feaab5551ee69b213f6ec52d91e93c1ab362 Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Thu, 18 Jun 2026 12:57:02 -0700 Subject: [PATCH 32/34] mitigate start_daemon race condition --- tests/e2e.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e.rs b/tests/e2e.rs index 867ad1e..b166e4a 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -464,7 +464,7 @@ fn test_proxy_command_error() { // Start a new container let cmd = r"tail -f /dev/null"; let node = harness.launch("root", cmd).assert(); - node.start_daemon(&["--intercept", "--", "exit", "123"]) + node.start_daemon(&["--intercept", "--", "sleep", "1", "&", "exit", "123"]) .assert(); // Wait for admin socket to disappear. From 7016e5ed2c20e463d4768954f066afc99d2c8cb9 Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Thu, 18 Jun 2026 13:28:22 -0700 Subject: [PATCH 33/34] don't use start_daemon --- tests/e2e.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/e2e.rs b/tests/e2e.rs index b166e4a..0c120b9 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -436,8 +436,7 @@ fn test_proxy_command_exit() { // Start a new container let cmd = r"tail -f /dev/null"; let node = harness.launch("root", cmd).assert(); - node.start_daemon(&["--intercept", "--", "sleep", "1"]) - .assert(); + let _ = node.exec(&["intermesh", "proxy", "--", "sleep", "1"]); // Wait for admin socket to disappear. wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); @@ -464,8 +463,7 @@ fn test_proxy_command_error() { // Start a new container let cmd = r"tail -f /dev/null"; let node = harness.launch("root", cmd).assert(); - node.start_daemon(&["--intercept", "--", "sleep", "1", "&", "exit", "123"]) - .assert(); + let _ = node.exec(&["intermesh", "proxy", "--", "exit", "123"]); // Wait for admin socket to disappear. wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); @@ -486,10 +484,6 @@ fn test_proxy_command_killed() { let proxy_cmd = r"intermesh proxy -- sleep inf"; let node = harness.launch("root", proxy_cmd).assert(); - // Verify admin socket exists - node.exec(&["test", "-S", "/var/run/intermesh/admin.sock"]) - .assert(); - // Verify nftables rules were injected wait(|| node.exec(&["nft", "list", "table", "ip", "intermesh"])).assert(); From 2b810681e3c6a442cc17d7b1d61819bc3bcf5d96 Mon Sep 17 00:00:00 2001 From: Patrick Lau Date: Sun, 14 Jun 2026 14:26:51 -0700 Subject: [PATCH 34/34] daemon: gracefully shutdown if wrapped command exits Before this change, when the wrapped command exits, a detached task calls process::exit directly, skipping the daemon's structured shutdown, leaving the host mis-routed. This commit ensures that the daemon gracefully shuts down if the wrapped command errors, exits or is killed by a signal. Add three e2e tests to make sure the daemon exits with the correct status codes and cleans up nftable rules in all three scenarios. --- src/daemon.rs | 32 ++++++++++++--- tests/e2e.rs | 91 +++++++++++++++++++++++++++++++++++++++++ tests/helpers/docker.rs | 12 ++++++ 3 files changed, 129 insertions(+), 6 deletions(-) diff --git a/src/daemon.rs b/src/daemon.rs index 394e03e..19f54aa 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -1,4 +1,5 @@ use anyhow::{Context, Result}; +use futures::future::OptionFuture; use futures::TryStreamExt; use std::collections::BTreeSet; use std::fs; @@ -118,12 +119,9 @@ pub(crate) async fn run(state: State, on_ready: Option) -> Result<()> { let mut admin_handle = tokio::spawn(admin_server.run(grpc, adhoc, cancel.clone())); let mut proxy_handle = tokio::spawn(proxy.run(cancel.clone())); - if let Some(mut cmd) = on_ready { - tokio::spawn(async move { - let status = cmd.status().await; - process::exit(status.map_or(1, |s| s.code().unwrap_or(1))); - }); - } + let mut task_status: Option = None; + let mut command_handle = + OptionFuture::from(on_ready.map(|mut cmd| tokio::spawn(async move { cmd.status().await }))); // Wait for shutdown signal or unexpected task exit. Signal results are // ignored - recv() only returns None if "already received" which can't @@ -138,6 +136,22 @@ pub(crate) async fn run(state: State, on_ready: Option) -> Result<()> { r = &mut gossip_handle => error!(?r, "gossip loop exited"), r = &mut admin_handle => error!(?r, "admin server exited"), r = &mut proxy_handle => error!(?r, "proxy exited"), + Some(r) = &mut command_handle => { + match r { + Err(e) => { + task_status = Some(1); + error!(?e, "command failed"); + }, + Ok(Err(e)) => { + task_status = Some(1); + error!(?e, "command exited"); + }, + Ok(Ok(s)) => { + task_status = Some(s.code().unwrap_or(1)); + info!("command exited with code {}", task_status.assert()); + }, + } + }, } info!("shutting down"); @@ -166,5 +180,11 @@ pub(crate) async fn run(state: State, on_ready: Option) -> Result<()> { } info!("shutdown complete"); + + // if proxy command exited cleanly, exit with its status code. + if let Some(status) = task_status { + process::exit(status) + } + Ok(()) } diff --git a/tests/e2e.rs b/tests/e2e.rs index e019ddd..0c120b9 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -422,6 +422,97 @@ fn test_graceful_shutdown() { ); } +#[test] +#[ignore = "e2e test requiring docker"] +fn test_proxy_command_exit() { + let harness = Harness::new(); + + let proxy_cmd = r"intermesh proxy -- sleep 1"; + let node = harness.launch("root", proxy_cmd).assert(); + + // Verify daemon's exit code + assert_eq!(node.exit_code().assert(), 0); + + // Start a new container + let cmd = r"tail -f /dev/null"; + let node = harness.launch("root", cmd).assert(); + let _ = node.exec(&["intermesh", "proxy", "--", "sleep", "1"]); + + // Wait for admin socket to disappear. + wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); + + // Verify nftables rules were cleaned up + let result = node.exec(&["nft", "list", "table", "ip", "intermesh"]); + assert!( + result.is_err(), + "nftables table should be removed on shutdown" + ); +} + +#[test] +#[ignore = "e2e test requiring docker"] +fn test_proxy_command_error() { + let harness = Harness::new(); + + let proxy_cmd = r#"intermesh proxy -- sh -c "exit 123""#; + let node = harness.launch("root", proxy_cmd).assert(); + + // Verify daemon's exit code + assert_eq!(node.exit_code().assert(), 123); + + // Start a new container + let cmd = r"tail -f /dev/null"; + let node = harness.launch("root", cmd).assert(); + let _ = node.exec(&["intermesh", "proxy", "--", "exit", "123"]); + + // Wait for admin socket to disappear. + wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); + + // Verify nftables rules were cleaned up + let result = node.exec(&["nft", "list", "table", "ip", "intermesh"]); + assert!( + result.is_err(), + "nftables table should be removed on shutdown" + ); +} + +#[test] +#[ignore = "e2e test requiring docker"] +fn test_proxy_command_killed() { + let harness = Harness::new(); + + let proxy_cmd = r"intermesh proxy -- sleep inf"; + let node = harness.launch("root", proxy_cmd).assert(); + + // Verify nftables rules were injected + wait(|| node.exec(&["nft", "list", "table", "ip", "intermesh"])).assert(); + + // Kill wrapped command via SIGTERM + node.exec(&["pkill", "-TERM", "sleep"]).assert(); + + // Verify daemon's exit code + assert_eq!(node.exit_code().assert(), 1); + + // Start a new container + let cmd = r"tail -f /dev/null"; + let node = harness.launch("root", cmd).assert(); + node.start_daemon(&["--intercept", "--", "sleep", "inf"]) + .assert(); + + // Kill wrapped command via SIGTERM + node.exec(&["pkill", "-TERM", "sleep"]).assert(); + + // Wait for admin socket to disappear. + wait(|| node.exec(&["test", "!", "-S", "/var/run/intermesh/admin.sock"])).assert(); + + // Verify nftables rules were cleaned up + let result = node.exec(&["nft", "list", "table", "ip", "intermesh"]); + assert!( + result.is_err(), + "nftables table should be removed on shutdown" + ); +} + #[test] #[ignore = "e2e test requiring docker"] fn test_runtime_endorsements_are_saved() { diff --git a/tests/helpers/docker.rs b/tests/helpers/docker.rs index 4914cdc..57ea833 100644 --- a/tests/helpers/docker.rs +++ b/tests/helpers/docker.rs @@ -196,6 +196,18 @@ impl Node<'_> { Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) } + pub(crate) fn exit_code(&self) -> Result { + let output = Command::new("docker") + .args(["wait", &self.container]) + .output()?; + + if !output.status.success() { + bail!("{}", command_error(&output, &self.container)) + } + + Ok(String::from_utf8(output.stdout)?.trim().parse::()?) + } + pub(crate) fn get_dump(&self) -> Result { let dump_toml = self.exec(&["intermesh", "debug", "dump", "--toml"])?; toml::from_str(&dump_toml).map_err(Into::into)