Hub spoke networks

.github/workflows/integration.yml

@@ -65,6 +65,7 @@ jobs:
             --test lifecycle \
             --test volumes \
             --test sidecars \
+            --test networks \
             -- --test-threads=1
 
       - name: Stop daemon

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rooz"
-version = "0.152.0"
+version = "0.153.0"
 edition = "2024"
 
 [dependencies]

README.md

@@ -266,7 +266,36 @@ rooz enter secrets-test
       image: my:tools
 
   ```
-  All containers within a workspace are connected to a workspace-wide network. They can *talk* to each other using sidecar names. In the above examples that would be `sql` and `tools`. Also the usual container ID and IP works too, but it is not as convenient.
+  `image` may be omitted in individual config layers (e.g. an overlay extending a sidecar defined in a base via `bases:`) but must be set for each sidecar once all layers are merged, otherwise creating the workspace fails.
+
+### Networking
+
+Workspace networking follows a hub-and-spoke model:
+
+* the work container can reach every sidecar — it shares a dedicated pair network with each one
+* a sidecar can always reach the work container (pair-network reachability is symmetric; keep listeners in the work container minimal)
+* sidecars cannot reach each other unless explicitly connected via `peers:`
+* workspaces are isolated from each other
+
+`peers:` lists other sidecar names a sidecar may talk to. Each declared relation gets its own network, `a: peers [b]` and `b: peers [a]` are equivalent, and peer names must refer to sidecars defined in the same workspace. Sidecars resolve each other by name over their peer network:
+
+  ```yaml
+  sidecars:
+    claude:
+      peers: [proxy]   # claude's only egress path is the proxy
+    proxy:
+      egress: true     # reachable only by the work container and claude
+    dkr:
+      peers: [images]  # docker-in-docker pulling via the mirror
+    images:
+      egress: true     # registry pull-through cache
+  ```
+
+  Unlike other list fields (replaced by higher config layers), `peers` merge as a deduplicated union across layers: an overlay can add reachability edges but cannot remove inherited ones.
+
+  :warning: rooz creates one network per sidecar plus one per peer relation. Docker's default address pools allow only ~31 networks host-wide; if network creation fails with an address-pool error, configure `default-address-pools` with a smaller subnet size (e.g. `"size": 24`) in `daemon.json`. Podman (netavark) is not affected.
+
+  The work container addresses sidecars by their names (`sql` and `tools` in the first example above). The usual container ID and IP work too, but are not as convenient.
 
 * the `enter` command lets you specify `--container` to enter (otherwise it enters the work container).
 
@@ -291,6 +320,7 @@ Supported keywords:
 ```
 
 * `ports` - port bindings in the `"8080:8080"` format
+* `peers` - names of other sidecars this sidecar may reach (see [Networking](#networking); merged as a union across config layers)
 * `work_dir` - set working directory
 * `mount_work` (`bool`) - if true then the work volume is mounted at `/work`
 

scripts/test-daemon.sh

@@ -7,7 +7,7 @@
 #
 # After "up", eval the printed exports to configure the test environment:
 #   eval "$(scripts/test-daemon.sh up docker)"
-#   cargo test --test smoke --test lifecycle --test volumes --test sidecars -- --test-threads=1
+#   cargo test --test smoke --test lifecycle --test volumes --test sidecars --test networks -- --test-threads=1
 #
 # Requires:
 #   - docker available on PATH

src/api/container.rs

@@ -123,6 +123,7 @@ impl<'a> ContainerApi<'a> {
         {
             Ok(_) => {
                 log::debug!("Removed container: {}{}", &container_id, &force_display);
+                self.wait_removed(container_id).await;
                 Ok(())
             }
             Err(DockerResponseServerError {
@@ -143,6 +144,7 @@ impl<'a> ContainerApi<'a> {
                     &container_id,
                     &force_display
                 );
+                self.wait_removed(container_id).await;
                 Ok(())
             }
             Err(DockerResponseServerError {
@@ -158,6 +160,22 @@ impl<'a> ContainerApi<'a> {
         }
     }
 
+    async fn wait_removed(&self, container_id: &str) {
+        let _ = timeout(Duration::from_secs(30), async {
+            loop {
+                match self
+                    .client
+                    .inspect_container(container_id, None::<InspectContainerOptions>)
+                    .await
+                {
+                    Ok(_) => sleep(Duration::from_millis(100)).await,
+                    Err(_) => break,
+                }
+            }
+        })
+        .await;
+    }
+
     pub async fn kill(&self, container_id: &str, wait_for_remove: bool) -> Result<(), AnyError> {
         match self
             .client

src/api/sidecar.rs

@@ -24,6 +24,7 @@ impl<'a> WorkspaceApi<'a> {
     ) -> Result<RuntimeConfig, AnyError> {
         let mut cfg = config.clone();
         let labels = Labels::from(&[Labels::workspace(workspace_key)]);
+        let (_, peer_relations) = RuntimeConfig::workspace_networks(&config.sidecars);
 
         for (name, s) in &mut cfg.sidecars {
             log::debug!("Process sidecar: {}", name);
@@ -79,8 +80,16 @@ impl<'a> WorkspaceApi<'a> {
                 .map(|x| x.to_string())
                 .unwrap_or(constants::ROOT_UID.to_string());
 
-            let internal_network = &constants::internal_network(workspace_key);
+            let pair_network = &constants::pair_network(workspace_key, name);
             let egress_network = &constants::egress_network(workspace_key);
+            let mut extra_networks = peer_relations
+                .iter()
+                .filter(|(a, b)| a == name || b == name)
+                .map(|(a, b)| constants::peer_network(workspace_key, a, b))
+                .collect::<Vec<_>>();
+            if s.egress {
+                extra_networks.push(egress_network.to_string());
+            }
             let run_spec = RunSpec {
                 reason: &container_name,
                 container_name: &container_name,
@@ -90,11 +99,11 @@ impl<'a> WorkspaceApi<'a> {
                 workspace_key: &workspace_key,
                 labels: labels.clone(),
                 env: Some(s.env.clone()),
-                default_network: Some(internal_network),
-                additional_networks: if s.egress {
-                    Some(vec![egress_network])
-                } else {
+                default_network: Some(pair_network),
+                additional_networks: if extra_networks.is_empty() {
                     None
+                } else {
+                    Some(extra_networks.iter().map(|n| n.as_str()).collect())
                 },
                 network_aliases: Some(vec![name.into()]),
                 command: if cmd.is_empty() {
@@ -128,6 +137,7 @@ impl<'a> WorkspaceApi<'a> {
                         .create(RunSpec {
                             run_mode: RunMode::SidecarInstall,
                             default_network: Some(egress_network),
+                            additional_networks: None,
                             // IMPORTANT: do not inject the sidecar env so it doesn't get baked into
                             // the runtime image. It also ensures unaltered behavior of the base image
                             // during the installation stage

src/api/volume.rs

@@ -627,7 +627,7 @@ mod tests {
 
         let mut mounts = HashMap::new();
         mounts.insert(
-            ("main".to_string(), "/mydir".to_string()),
+            ("work".to_string(), "/mydir".to_string()),
             MountSource::DataEntryReference(DataEntryKey("mydir".to_string())),
         );
 
@@ -640,7 +640,7 @@ mod tests {
     fn inline_file_mount_gets_inline_fallback_volume() {
         let mut mounts = HashMap::new();
         mounts.insert(
-            ("main".to_string(), "/config".to_string()),
+            ("work".to_string(), "/config".to_string()),
             MountSource::InlineDataValue(inline("hello")),
         );
 
@@ -654,11 +654,11 @@ mod tests {
     fn multiple_inline_files_share_inline_volume() {
         let mut mounts = HashMap::new();
         mounts.insert(
-            ("main".to_string(), "/file-a".to_string()),
+            ("work".to_string(), "/file-a".to_string()),
             MountSource::InlineDataValue(inline("aaa")),
         );
         mounts.insert(
-            ("main".to_string(), "/file-b".to_string()),
+            ("work".to_string(), "/file-b".to_string()),
             MountSource::InlineDataValue(inline("bbb")),
         );
 

src/cli.rs

@@ -208,7 +208,7 @@ pub struct StartParams {
 }
 
 #[derive(Parser, Debug)]
-#[command(about = "Restarts a workspace's main container")]
+#[command(about = "Restarts a workspace's work container")]
 pub struct RestartParams {
     pub name: String,
     #[arg(long, default_value = "false", help = "")]

src/cmd/new.rs

@@ -22,6 +22,44 @@ use std::collections::HashMap;
 use std::fs;
 
 impl<'a> WorkspaceApi<'a> {
+    async fn ensure_network(
+        &self,
+        name: &str,
+        internal: bool,
+        labels: &Labels,
+    ) -> Result<(), AnyError> {
+        match self
+            .api
+            .client
+            .create_network(NetworkCreateRequest {
+                name: name.to_string(),
+                internal: if internal { Some(true) } else { None },
+                labels: Some(labels.clone().into()),
+                ..Default::default()
+            })
+            .await
+        {
+            Ok(_) => Ok(()),
+            Err(Error::DockerResponseServerError {
+                status_code: 409, ..
+            }) => {
+                log::debug!("Network already exists: {}. Skipping", name);
+                Ok(())
+            }
+            Err(e) if e.to_string().contains("non-overlapping IPv4 address pool") => {
+                Err(format!(
+                    "Could not create network '{}': the daemon ran out of address pools. \
+                     Rooz creates one network per sidecar plus one per peer relation. \
+                     Configure 'default-address-pools' with a smaller subnet size (e.g. \"size\": 24) \
+                     in the daemon config to allow more networks. Original error: {}",
+                    name, e
+                )
+                .into())
+            }
+            Err(e) => Err(e.into()),
+        }
+    }
+
     async fn new_core(
         &self,
         cfg_builder: &mut RoozCfg,
@@ -45,7 +83,7 @@ impl<'a> WorkspaceApi<'a> {
             .await?;
         cfg_builder.expand_vars()?;
 
-        let cfg = RuntimeConfig::from(&*cfg_builder);
+        let cfg = RuntimeConfig::try_from(&*cfg_builder)?;
 
         self.api
             .image
@@ -92,48 +130,21 @@ impl<'a> WorkspaceApi<'a> {
 
         let mut labels = work_spec.labels.clone();
 
-        let internal_network = &constants::internal_network(workspace_key);
         let egress_network = &constants::egress_network(workspace_key);
 
-        match self
-            .api
-            .client
-            .create_network(NetworkCreateRequest {
-                name: egress_network.to_string(),
-                labels: Some(labels.clone().into()),
-                ..Default::default()
-            })
-            .await
-        {
-            Ok(_) => {}
-            Err(Error::DockerResponseServerError {
-                status_code: 409, ..
-            }) => {
-                log::debug!("Network already exists: {}. Skipping", egress_network);
-            }
-            Err(e) => return Err(e.into()),
-        };
+        self.ensure_network(egress_network, false, &labels).await?;
 
-        if !cfg2.sidecars.is_empty() {
-            match self
-                .api
-                .client
-                .create_network(NetworkCreateRequest {
-                    name: internal_network.to_string(),
-                    internal: Some(true),
-                    labels: Some(labels.clone().into()),
-                    ..Default::default()
-                })
-                .await
-            {
-                Ok(_) => {}
-                Err(Error::DockerResponseServerError {
-                    status_code: 409, ..
-                }) => {
-                    log::debug!("Network already exists: {}. Skipping", internal_network);
-                }
-                Err(e) => return Err(e.into()),
-            };
+        let (pair_keys, peer_keys) = RuntimeConfig::workspace_networks(&cfg2.sidecars);
+        let pair_networks = pair_keys
+            .iter()
+            .map(|s| constants::pair_network(workspace_key, s))
+            .collect::<Vec<_>>();
+        for n in &pair_networks {
+            self.ensure_network(n, true, &labels).await?;
+        }
+        for (a, b) in &peer_keys {
+            self.ensure_network(&constants::peer_network(workspace_key, a, b), true, &labels)
+                .await?;
         }
 
         let cfg2 = self
@@ -157,8 +168,8 @@ impl<'a> WorkspaceApi<'a> {
                 .map(|r| r.dir)
                 .unwrap_or(constants::WORK_DIR.to_string()),
             default_network: Some(egress_network.as_str()),
-            additional_networks: if !cfg2.sidecars.is_empty() {
-                Some(vec![internal_network.as_str()])
+            additional_networks: if !pair_networks.is_empty() {
+                Some(pair_networks.iter().map(|n| n.as_str()).collect())
             } else {
                 None
             },
@@ -435,10 +446,10 @@ impl<'a> WorkspaceApi<'a> {
             .map(|v| (&v).dir.to_string())
             .or(Some(workspace.working_dir));
 
-        let cfg = RuntimeConfig::from(&RoozCfg {
+        let cfg = RuntimeConfig::try_from(&RoozCfg {
             shell: Some(vec![shell.into()]),
             ..config
-        });
+        })?;
 
         let container_id = self
             .enter(