diff --git a/.github/workflows/SDK.Dotnet.Test.Job.yml b/.github/workflows/SDK.Dotnet.Test.Job.yml index 0701cd3fb..0a7846de0 100644 --- a/.github/workflows/SDK.Dotnet.Test.Job.yml +++ b/.github/workflows/SDK.Dotnet.Test.Job.yml @@ -98,6 +98,22 @@ jobs: sudo DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::Retries=3 install -y \ lxc lxc-templates lxc-utils iptables debootstrap uidmap bridge-utils + - name: Configure unprivileged LXC + if: matrix.os_label == 'linux' + shell: bash + run: | + set -euo pipefail + for file in /etc/lxc/default.conf /etc/subuid /etc/subgid; do + sudo cp -a "$file" "$RUNNER_TEMP/mxc-lxc-$(basename "$file")" + done + sudo bash -c 'echo "root:100000:65536" >> /etc/subuid' + sudo bash -c 'echo "root:100000:65536" >> /etc/subgid' + + # Remove existing mappings before adding the test-only UID/GID range. + sudo sed -i '/idmap/d' /etc/lxc/default.conf + sudo bash -c 'echo "lxc.idmap = u 0 100000 65536" >> /etc/lxc/default.conf' + sudo bash -c 'echo "lxc.idmap = g 0 100000 65536" >> /etc/lxc/default.conf' + # The policy these tests use permits no network, so the container starts # with no interface and the runner's Docker-set FORWARD policy cannot # reach them. @@ -133,6 +149,18 @@ jobs: fi done + - name: Restore LXC configuration + if: always() && matrix.os_label == 'linux' + shell: bash + run: | + set -euo pipefail + for file in /etc/lxc/default.conf /etc/subuid /etc/subgid; do + backup="$RUNNER_TEMP/mxc-lxc-$(basename "$file")" + if [ -f "$backup" ]; then + sudo mv "$backup" "$file" + fi + done + # This regression requires Linux and a usable Bubblewrap. Install it in # the Linux lane and make an unavailable backend fail instead of skip. - name: Install Bubblewrap diff --git a/.github/workflows/SDK.Integration.Test.Job.yml b/.github/workflows/SDK.Integration.Test.Job.yml index 958526fd3..d163dbc28 100644 --- a/.github/workflows/SDK.Integration.Test.Job.yml +++ b/.github/workflows/SDK.Integration.Test.Job.yml @@ -110,8 +110,8 @@ jobs: # installs a similar but narrower set and sets the same skip # env var for the same reason (compounded by 1ES Hosted Pool # egress filtering). - packages: bubblewrap lxc lxc-utils dnsmasq-base iptables bridge-utils - version: 1.4 + packages: bubblewrap lxc lxc-utils uidmap dnsmasq-base iptables bridge-utils + version: 1.5 - name: Verify or repair Linux containment dependencies if: matrix.os_label == 'linux' @@ -123,6 +123,7 @@ jobs: bubblewrap lxc lxc-utils + uidmap dnsmasq-base iptables bridge-utils @@ -132,6 +133,8 @@ jobs: bwrap lxc-start lxc-create + newuidmap + newgidmap dnsmasq iptables brctl @@ -174,6 +177,22 @@ jobs: command -v "$command" done + - name: Configure unprivileged LXC + if: matrix.os_label == 'linux' + shell: bash + run: | + set -euo pipefail + for file in /etc/lxc/default.conf /etc/subuid /etc/subgid; do + sudo cp -a "$file" "$RUNNER_TEMP/mxc-lxc-$(basename "$file")" + done + sudo bash -c 'echo "root:100000:65536" >> /etc/subuid' + sudo bash -c 'echo "root:100000:65536" >> /etc/subgid' + + # Remove existing mappings before adding the test-only UID/GID range. + sudo sed -i '/idmap/d' /etc/lxc/default.conf + sudo bash -c 'echo "lxc.idmap = u 0 100000 65536" >> /etc/lxc/default.conf' + sudo bash -c 'echo "lxc.idmap = g 0 100000 65536" >> /etc/lxc/default.conf' + - name: Start LXC services (cache restore skips postinst hooks) if: matrix.os_label == 'linux' shell: bash @@ -217,3 +236,15 @@ jobs: else npm test fi + + - name: Restore LXC configuration + if: always() && matrix.os_label == 'linux' + shell: bash + run: | + set -euo pipefail + for file in /etc/lxc/default.conf /etc/subuid /etc/subgid; do + backup="$RUNNER_TEMP/mxc-lxc-$(basename "$file")" + if [ -f "$backup" ]; then + sudo mv "$backup" "$file" + fi + done diff --git a/.github/workflows/lxc-e2e.yml b/.github/workflows/lxc-e2e.yml index 9fe298c21..217337fc2 100644 --- a/.github/workflows/lxc-e2e.yml +++ b/.github/workflows/lxc-e2e.yml @@ -36,6 +36,20 @@ jobs: sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ lxc lxc-templates lxc-utils iptables debootstrap uidmap bridge-utils + - name: Configure unprivileged LXC + run: | + set -euo pipefail + for file in /etc/lxc/default.conf /etc/subuid /etc/subgid; do + sudo cp -a "$file" "$RUNNER_TEMP/mxc-lxc-$(basename "$file")" + done + sudo bash -c 'echo "root:100000:65536" >> /etc/subuid' + sudo bash -c 'echo "root:100000:65536" >> /etc/subgid' + + # Remove existing mappings before adding the test-only UID/GID range. + sudo sed -i '/idmap/d' /etc/lxc/default.conf + sudo bash -c 'echo "lxc.idmap = u 0 100000 65536" >> /etc/lxc/default.conf' + sudo bash -c 'echo "lxc.idmap = g 0 100000 65536" >> /etc/lxc/default.conf' + # GitHub-hosted runners ship Docker, and Docker sets the IPv4 FORWARD # policy to DROP. That breaks these tests twice over. # @@ -123,6 +137,17 @@ jobs: MXC_LXC_TESTS_REQUIRE_EXECUTION: "1" run: sudo --preserve-env=MXC_LXC_TESTS_REQUIRE_EXECUTION,PATH,HOME "$(command -v cargo)" test -p mxc-sdk --test streaming_lxc --test sdk_helpers + - name: Restore LXC configuration + if: always() + run: | + set -euo pipefail + for file in /etc/lxc/default.conf /etc/subuid /etc/subgid; do + backup="$RUNNER_TEMP/mxc-lxc-$(basename "$file")" + if [ -f "$backup" ]; then + sudo mv "$backup" "$file" + fi + done + - name: Show leftover firewall state on failure if: failure() run: | diff --git a/docs/backends/lxc/lxc-backend.md b/docs/backends/lxc/lxc-backend.md index 80fe22ea8..ebfbf10f6 100644 --- a/docs/backends/lxc/lxc-backend.md +++ b/docs/backends/lxc/lxc-backend.md @@ -18,34 +18,43 @@ Creates an LXC container to provide: - **Process isolation** via Linux namespaces (PID, mount, network, user) - **Filesystem isolation** via bind mounts with read-only/read-write/denied enforcement -- **Network isolation** via iptables rules inside the container's own network namespace +- **Network isolation** via host-owned nftables bridge rules on the container's network interfaces ## Prerequisites -- Linux kernel >= 2.6.32, or >= 3.12 to run unprivileged -- LXC >= 5.0 installed (`liblxc-dev` for building, `lxc-utils` for runtime) -- Root privileges, or [unprivileged LXC](#running-without-root) for a policy - that asks for no network. - Filtering egress or ingress installs iptables rules in the container's - network namespace, which requires root. +- Linux kernel >= 3.12. +- LXC >= 5.0 installed (`liblxc-dev` for building, `lxc-utils` for runtime). +- Bash and nftables (`nft`) for filtered networking. +- Host kernel support for nftables bridge filtering and connection tracking + (`CONFIG_NF_TABLES_BRIDGE` and `CONFIG_NF_CONNTRACK_BRIDGE`). + Installing the nftables package alone does not provide missing kernel support. +- Root access to run MXC and to configure LXC. + Filtering egress or ingress installs rules in the host's network namespace, + which requires root. +- Valid UID and GID root mappings to run LXC containers unprivileged. See [LXC Getting Started](https://linuxcontainers.org/lxc/getting-started/#create-unprivileged-containers-as-root-with-shared-uid-and-gid-ranges). + +MXC requires LXC containers to run unprivileged. After starting the container, +MXC reads `/proc//uid_map` for the container's init process directly on the +host before running the requested command. MXC accepts the container when a +readable range maps container root to a nonzero parent UID. ### Installation **Debian/Ubuntu:** ```bash -sudo apt install lxc lxc-utils liblxc-dev +sudo apt install lxc lxc-utils liblxc-dev nftables ``` **Fedora/RHEL:** ```bash -sudo dnf install lxc lxc-templates dnsmasq lxc-devel +sudo dnf install lxc lxc-templates dnsmasq lxc-devel nftables ``` RHEL needs EPEL enabled first — see [Red Hat Enterprise Linux setup](#red-hat-enterprise-linux-setup). **Arch Linux:** ```bash -sudo pacman -S lxc +sudo pacman -S lxc nftables ``` ### Running without root @@ -55,8 +64,6 @@ for unprivileged LXC. Everything else needs root. 1. Follow LXC's [Create Unprivileged Containers as a User](https://linuxcontainers.org/lxc/getting-started/#create-unprivileged-containers-as-a-user). -2. Let the container's root user reach its files under your home directory: - `chmod o+x ~`. ### Container networking @@ -155,13 +162,40 @@ not reachable through `process.env`. ## Network Policy -Directional `network.egress` rules govern port 53 like any other destination; -there is no implicit DNS exemption. - -`preservePolicy` leaves the inbound and outbound chains in place after the run. -The chains live in the container's network namespace, so they last only as long -as the container keeps running; stopping or destroying it takes them with it. A -partially installed chain from a failed run is torn down regardless. +MXC filters traffic on the host-side interfaces of the container, not inside +the container. For each bridge-attached veth interface, MXC generates a +host-owned Bash script and registers it through `lxc.net..script.up`. +Existing interface-up hooks run first. The generated hook installs inbound +and outbound nftables bridge rules before container init starts. +Both IPv4 and IPv6 traffic pass through these rules. Unsupported interface +topologies or unavailable host filtering reject startup. + +MXC does not remove `CAP_NET_ADMIN` or `CAP_NET_RAW` from container processes. +Init, attached commands, and workloads can use the capabilities their container +provides without controlling the host-owned filters. Flushing the guest's +firewall does not remove those filters. + +Directional `network.egress` rules govern DNS and DHCP like other traffic; +neither receives an implicit exemption. DHCP replies are admitted only for +requests allowed by the policy. Denying DHCP prevents DHCP address assignment; +other permitted traffic needs an address and route configured separately. + +`preservePolicy` retains installed filtering, startup hooks, and generated +scripts after the run, including when the container is stopped. Restarting +the retained container reinstalls filtering before init starts. A subsequent +MXC run replaces retained filtering with the new request's policy. + +| `preservePolicy` | `destroyOnExit` | After the run | +|---|---|---| +| `false` | `false` | Leave the container running; remove owned filtering, hooks, and scripts. | +| `true` | `false` | Leave the container running; retain owned filtering, hooks, and scripts. | +| `false` | `true` | Destroy the container, then remove owned host filtering and support files. | +| `true` | `true` | Destroy the container, then remove owned host filtering and support files. | + +A timeout or `kill()` stops the whole container; preservation still retains +its installed policy. Failed destruction reports an error and retains the +surviving container's filtering and support files. An incompletely installed +policy is not preserved. ### Proxy @@ -275,8 +309,9 @@ container init, where nothing aimed at the host `lxc-attach` process reaches it. LXC reads a run's network section only when the container starts. Omit `containerId` for a generated name. The claim is released when the handle drops. -**Teardown is owed on every terminal path,** including a drop without `wait`: -the network chains come down, and the container is released. A reused container +**Teardown is owed on every terminal path,** including a drop without `wait`, +and follows the [network policy lifecycle settings](#network-policy). +A reused container also has any stale proxy address mapping in its hosts file cleared before its new workload launches. MXC checks for marked lines inside the running container using `lxc-attach` and removes them if present. A failed check @@ -284,9 +319,10 @@ refuses to run the workload. A teardown failure is reported through `Sandbox::warnings`, which a bare drop leaves no handle to read. -**The container stays up for as long as the handle lives.** `lxc-exec` is -short-lived by comparison. Weigh that against the threat model before embedding -streaming in a long-lived service. +**An ordinary run keeps the container up while the handle lives.** A timeout +or `kill()` can stop it earlier. `lxc-exec` is short-lived by comparison. +Weigh that against the threat model before embedding streaming in a long-lived +service. ## Building @@ -313,7 +349,7 @@ streaming in a long-lived service. 2. **Install LXC and the pieces it needs at runtime.** ```bash - sudo dnf install -y lxc lxc-templates dnsmasq iptables + sudo dnf install -y lxc lxc-templates dnsmasq iptables nftables ``` Add `lxc-devel` as well if you are building MXC against liblxc rather than @@ -342,24 +378,13 @@ These failures carry the `backend_error` code and exit `-1`. ## Limitations -- **The workload can send only ICMP Echo.** A `protocol: "icmp"` rule lets every - ICMP type through the firewall, but the command MXC runs can originate only Echo - requests, the ones `ping` sends. -- **The network restrictions bind only the command MXC runs and its children.** - MXC takes away that command's ability to change the firewall or send raw - packets, but the container's init process (PID 1) keeps both. The command - runs as root inside the container, so it can ask init to start a new process, - for example with `systemd-run` on Ubuntu. That process is not a child of the - command, and it can change the firewall and send raw packets. -- **Network access stops after about an hour.** The stock `lxcbr0` hands out - DHCP leases that last one hour, and MXC's firewall blocks the DHCP request the - container sends to renew one. When the lease expires the container loses its - address and the workload can no longer reach the network. A run that finishes - within the hour is unaffected. -- **IPv6 egress is not supported.** An `egress` rule naming an IPv6 destination - installs and reports success, but no IPv6 traffic reaches that destination. - Stock `lxcbr0` gives the container no IPv6 address, so this surfaces only on a - host that provides one. +- **Cross-container reply isolation has an unfixed defect.** A container that + adopts another container's IP and Ethernet addresses can receive that + container's permitted reply despite its own policy denying the endpoint. + This was reproduced with UDP on a shared bridge. The host filter recognizes + tracked replies without verifying that the receiving interface belongs to + the container that originated the request. Host-owned filtering does not + yet provide the required reply isolation between those containers. - **A container reaches the proxy only while a run lasts.** See [Proxy](#proxy). - **No container lifecycle operations.** LXC implements `ScriptRunner` (create-and-run execution) and `SandboxBackend` (streaming over pipes or a caller-controlled PTY), not diff --git a/scripts/ci/prepare-linux-host.sh b/scripts/ci/prepare-linux-host.sh index 9611adb3c..a1b8cee75 100644 --- a/scripts/ci/prepare-linux-host.sh +++ b/scripts/ci/prepare-linux-host.sh @@ -123,7 +123,8 @@ load_conntrack_module() { } install_lxc() { - if command -v lxc-start >/dev/null 2>&1; then + if command -v lxc-start >/dev/null 2>&1 && + command -v nft >/dev/null 2>&1; then return fi @@ -135,14 +136,14 @@ install_lxc() { local packages if [[ "$package_manager" == "apt-get" ]]; then apt_update - packages=(lxc dnsmasq-base iptables bridge-utils) + packages=(lxc dnsmasq-base iptables nftables bridge-utils) # Debian dropped lxc-utils; Ubuntu still ships it. if apt-cache show lxc-utils >/dev/null 2>&1; then packages+=(lxc-utils) fi else install_epel - packages=(lxc lxc-templates dnsmasq iptables) + packages=(lxc lxc-templates dnsmasq iptables nftables) fi install_packages "${packages[@]}" } @@ -302,6 +303,7 @@ case "$backend" in lxc) install_lxc command -v lxc-start + command -v nft sudo -n true # Package installs may not activate the AppArmor profile, which # lxc-start needs. diff --git a/sdk/node/tests/integration/linux-process-container.test.ts b/sdk/node/tests/integration/linux-process-container.test.ts index b07e2c320..a9e2f0a7f 100644 --- a/sdk/node/tests/integration/linux-process-container.test.ts +++ b/sdk/node/tests/integration/linux-process-container.test.ts @@ -125,6 +125,8 @@ describe(`Linux LXC Container (schema ${schemaVersion})`, { it('should mount readwrite filesystem path', async () => { tempDir = createTempDir('mxc-lxc-test'); fs.writeFileSync(path.join(tempDir, 'test.txt'), 'original'); + fs.chmodSync(tempDir, 0o777); + fs.chmodSync(path.join(tempDir, 'test.txt'), 0o666); const policy = { filesystem: { readwritePaths: [tempDir] } }; const script = `cat ${tempDir}/test.txt && echo 'overwritten' > ${tempDir}/test.txt && cat ${tempDir}/test.txt`; const result = await runLxc(script, policy, `lxc-rw-${schemaVersion}`); @@ -135,6 +137,8 @@ describe(`Linux LXC Container (schema ${schemaVersion})`, { it('should mount readonly filesystem path', async () => { tempDir = createTempDir('mxc-lxc-test'); fs.writeFileSync(path.join(tempDir, 'data.txt'), 'readonly content'); + fs.chmodSync(tempDir, 0o755); + fs.chmodSync(path.join(tempDir, 'data.txt'), 0o644); const policy = { filesystem: { readonlyPaths: [tempDir] } }; const result = await runLxc( `cat ${tempDir}/data.txt && echo 'Read succeeded'`, @@ -147,6 +151,7 @@ describe(`Linux LXC Container (schema ${schemaVersion})`, { it('should download file to writable mount', { skip: lxcNetworkSkipReason }, async () => { tempDir = createTempDir('mxc-lxc-test'); + fs.chmodSync(tempDir, 0o777); const policy = { filesystem: { readwritePaths: [tempDir] }, network: outboundNetwork(), diff --git a/src/mxc-sdk/src/backends/lxc/common/host_network.rs b/src/mxc-sdk/src/backends/lxc/common/host_network.rs new file mode 100644 index 000000000..1eb9ec574 --- /dev/null +++ b/src/mxc-sdk/src/backends/lxc/common/host_network.rs @@ -0,0 +1,781 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::collections::BTreeMap; +use std::fs::{self, File, OpenOptions}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::{Arc, Mutex}; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use super::lxc_bindings::LxcContainer; +use super::network_ingress::{IngressManager, ICMPV6_ALLOW_TYPES}; +use super::network_iptables::NetworkIptablesManager; +use crate::mxc_common::logger::Logger; +use crate::mxc_common::models::ContainerPolicy; + +const HOOK_BEGIN: &str = "# BEGIN MXC host network"; +const HOOK_END: &str = "# END MXC host network"; + +#[derive(Default)] +struct NetworkConfig { + interfaces: BTreeMap>, + kept_network_namespace: bool, + shared_network_namespace: bool, + cloned_network_namespace: Option, +} + +impl NetworkConfig { + fn read(path: &Path) -> Result { + let mut config = Self::default(); + config.include(path, &mut Vec::new())?; + if config.kept_network_namespace + || config.shared_network_namespace + || config.cloned_network_namespace == Some(false) + { + return Err("LXC host filtering does not support a shared network namespace".into()); + } + Ok(config) + } + + fn include(&mut self, path: &Path, stack: &mut Vec) -> Result<(), String> { + let canonical = path + .canonicalize() + .map_err(|error| format!("Cannot read LXC config {}: {error}", path.display()))?; + if stack.contains(&canonical) { + return Err(format!("Recursive LXC include: {}", path.display())); + } + stack.push(canonical); + let contents = fs::read_to_string(path) + .map_err(|error| format!("Cannot read LXC config {}: {error}", path.display()))?; + for line in contents.lines() { + let line = line.trim(); + if !line.starts_with("lxc.") { + continue; + } + let Some((key, value)) = line.split_once('=') else { + return Err(format!("Unsupported LXC configuration line: {line}")); + }; + let key = key.trim(); + let value = value.trim(); + let value = if value.len() >= 2 + && ((value.starts_with('"') && value.ends_with('"')) + || (value.starts_with('\'') && value.ends_with('\''))) + { + &value[1..value.len() - 1] + } else { + value + }; + if key == "lxc.include" && !value.is_empty() { + let included = Path::new(value); + if value.ends_with('/') || included.is_dir() { + let entries = match fs::read_dir(included) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => return Err(error.to_string()), + }; + for entry in entries { + let entry = entry.map_err(|error| error.to_string())?; + if entry.path().extension().is_some_and(|ext| ext == "conf") { + self.include(&entry.path(), stack)?; + } + } + } else { + self.include(included, stack)?; + } + } else if key == "lxc.namespace.share.net" { + self.shared_network_namespace = !value.is_empty(); + } else if key == "lxc.namespace.keep" { + if value.is_empty() { + self.kept_network_namespace = false; + } else { + self.kept_network_namespace |= + value.split_whitespace().any(|name| name == "net"); + } + } else if key == "lxc.namespace.clone" { + if value.is_empty() { + self.cloned_network_namespace = None; + } else { + *self.cloned_network_namespace.get_or_insert(false) |= + value.split_whitespace().any(|name| name == "net"); + } + } else if key == "lxc.net" && value.is_empty() { + self.interfaces.clear(); + } else if let Some(suffix) = key.strip_prefix("lxc.net.") { + let Some((index, field)) = suffix.split_once('.') else { + return Err(format!("Unsupported LXC network key: {key}")); + }; + let index = index.parse::().map_err(|_| { + format!("LXC host filtering requires numeric network indices: {key}") + })?; + self.interfaces + .entry(index) + .or_default() + .insert(field.to_string(), value.to_string()); + } else if key.starts_with("lxc.network") { + return Err( + "LXC host filtering requires numeric lxc.net. configuration".into(), + ); + } + } + stack.pop(); + Ok(()) + } +} + +#[derive(Serialize, Deserialize)] +struct InstalledPolicy { + config: PathBuf, + token: String, + indices: Vec, +} + +#[derive(Clone)] +pub(crate) struct HostNetwork { + directory: PathBuf, + _lock: Arc, + operations: Arc>, +} + +impl HostNetwork { + pub(crate) fn claim(container: &LxcContainer) -> Result { + let root = Path::new(container.lxc_path()); + fs::create_dir_all(root).map_err(|error| error.to_string())?; + let root = root.canonicalize().map_err(|error| error.to_string())?; + let identity = format!("{}\0{}", root.display(), container.name()); + let digest: String = Sha256::digest(identity.as_bytes()) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + let directory = root.join(".mxc-network").join(digest); + fs::create_dir_all(&directory).map_err(|error| error.to_string())?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&directory, fs::Permissions::from_mode(0o700)) + .map_err(|error| error.to_string())?; + } + let lock = OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(directory.join("lock")) + .map_err(|error| error.to_string())?; + lock.try_lock().map_err(|_| { + format!( + "LXC container '{}' is already owned by another run", + container.name() + ) + })?; + Ok(Self { + directory, + _lock: Arc::new(lock), + operations: Arc::new(Mutex::new(())), + }) + } + + pub(crate) fn configure( + &mut self, + container: &LxcContainer, + policy: &ContainerPolicy, + no_interface: bool, + logger: &mut Logger, + ) -> Result<(), String> { + self.remove()?; + let _operation = self + .operations + .lock() + .unwrap_or_else(|error| error.into_inner()); + let config_path = Path::new(container.lxc_path()) + .join(container.name()) + .join("config") + .canonicalize() + .map_err(|error| error.to_string())?; + let config = NetworkConfig::read(&config_path)?; + if no_interface { + return Ok(()); + } + if let Some(field) = IngressManager::permissive_inbound_field(policy) { + return Err(format!( + "{field} asks for permissive inbound, which is not yet implemented for the LXC firewall path" + )); + } + let rules = NetworkIptablesManager::host_egress_rules(policy, logger)?; + let mut interfaces = Vec::new(); + for (index, fields) in &config.interfaces { + match fields.get("type").map(String::as_str) { + Some("empty") => continue, + Some("veth") => {} + other => return Err(format!( + "LXC host filtering requires bridge-attached veth interfaces; lxc.net.{index}.type is {other:?}" + )), + } + if fields + .get("veth.mode") + .is_some_and(|mode| mode != "bridge" && !mode.is_empty()) + || fields.get("link").is_none_or(String::is_empty) + { + return Err(format!( + "LXC host filtering requires lxc.net.{index} to attach to a bridge" + )); + } + interfaces.push((*index, fields)); + } + if interfaces.is_empty() { + return Err( + "LXC network policy needs at least one bridge-attached veth interface".into(), + ); + } + let token = uuid::Uuid::new_v4().simple().to_string(); + let installed = InstalledPolicy { + config: config_path.clone(), + token, + indices: interfaces.iter().map(|(index, _)| *index).collect(), + }; + fs::write( + self.directory.join("policy.json"), + serde_json::to_vec(&installed).map_err(|error| error.to_string())?, + ) + .map_err(|error| error.to_string())?; + let mut hooks = format!("\n{HOOK_BEGIN} {}\n", installed.token); + for (index, fields) in interfaces { + let script = self.directory.join(format!("up-{index}.sh")); + let ready = self.directory.join(format!("up-{index}.ready")); + let body = startup_script( + &installed.token, + index, + &fields["link"], + fields.get("script.up").map_or("", String::as_str), + &ready, + &rules, + ); + fs::write(&script, body).map_err(|error| error.to_string())?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&script, fs::Permissions::from_mode(0o700)) + .map_err(|error| error.to_string())?; + } + hooks.push_str(&format!( + "lxc.net.{index}.script.up = /bin/bash {}\n", + shell_quote(&script.to_string_lossy()) + )); + } + hooks.push_str(&format!("{HOOK_END} {}\n", installed.token)); + let mut contents = fs::read_to_string(&config_path).map_err(|error| error.to_string())?; + contents.push_str(&hooks); + fs::write(config_path, contents).map_err(|error| error.to_string()) + } + + pub(crate) fn verify_started(&self) -> Result<(), String> { + let Some(installed) = self.installed()? else { + return Ok(()); + }; + for index in installed.indices { + if !self.directory.join(format!("up-{index}.ready")).is_file() { + return Err(format!( + "LXC host policy hook did not finish for interface {index}" + )); + } + } + Ok(()) + } + + pub(crate) fn startup_failure(&self, reason: &str) -> String { + let mut message = reason.to_string(); + if let Ok(Some(installed)) = self.installed() { + for index in installed.indices { + if let Ok(output) = + fs::read_to_string(self.directory.join(format!("up-{index}.log"))) + { + if !output.trim().is_empty() { + message + .push_str(&format!("\nHost network hook {index}: {}", output.trim())); + } + } + } + } + message + } + + fn installed(&self) -> Result, String> { + let path = self.directory.join("policy.json"); + match fs::read(path) { + Ok(contents) => serde_json::from_slice(&contents) + .map(Some) + .map_err(|error| format!("Cannot read owned LXC host policy: {error}")), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error.to_string()), + } + } + + pub(crate) fn remove(&mut self) -> Result<(), String> { + let _operation = self + .operations + .lock() + .unwrap_or_else(|error| error.into_inner()); + let Some(installed) = self.installed()? else { + return Ok(()); + }; + #[cfg(test)] + if let Some(result) = + super::network_iptables::test_firewall::intercept("nft", &["remove-owned-host-policy"]) + { + return result; + } + let tables = Command::new("nft") + .args(["--json", "list", "tables"]) + .output() + .map_err(|error| format!("Cannot inspect owned host firewall tables: {error}"))?; + if !tables.status.success() { + return Err(format!( + "Cannot inspect owned host firewall tables: {}", + String::from_utf8_lossy(&tables.stderr) + )); + } + let tables: serde_json::Value = + serde_json::from_slice(&tables.stdout).map_err(|error| error.to_string())?; + let mut deletions = String::new(); + for index in &installed.indices { + let table = format!("mxc_{}_{index}", installed.token); + let exists = tables["nftables"].as_array().is_some_and(|entries| { + entries.iter().any(|entry| { + entry["table"]["family"] == "bridge" && entry["table"]["name"] == table + }) + }); + if exists { + deletions.push_str(&format!("delete table bridge {table}\n")); + } + } + if !deletions.is_empty() { + let mut child = Command::new("nft") + .args(["-f", "-"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|error| error.to_string())?; + let written = child.stdin.take().unwrap().write_all(deletions.as_bytes()); + let output = child + .wait_with_output() + .map_err(|error| error.to_string())?; + if !output.status.success() { + return Err(format!( + "Cannot remove owned host firewall tables: {}", + String::from_utf8_lossy(&output.stderr) + )); + } + written.map_err(|error| error.to_string())?; + } + match fs::read_to_string(&installed.config) { + Ok(contents) => { + fs::write( + &installed.config, + without_owned_hooks(&contents, &installed.token)?, + ) + .map_err(|error| error.to_string())?; + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.to_string()), + } + for index in installed.indices { + for suffix in ["sh", "ready", "log"] { + let path = self.directory.join(format!("up-{index}.{suffix}")); + match fs::remove_file(path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.to_string()), + } + } + } + fs::remove_file(self.directory.join("policy.json")).map_err(|error| error.to_string()) + } +} + +fn without_owned_hooks(contents: &str, token: &str) -> Result { + let begin = format!("{HOOK_BEGIN} {token}"); + let end = format!("{HOOK_END} {token}"); + let mut owned = false; + let mut result = String::new(); + for line in contents.split_inclusive('\n') { + let marker = line.trim_end_matches(['\r', '\n']); + if marker == begin { + owned = true; + } else if marker == end { + owned = false; + } else if !owned { + result.push_str(line); + } + } + if owned { + return Err("Incomplete MXC host network hook block".into()); + } + Ok(result) +} + +fn shell_quote(value: &str) -> String { + format!("'{}'", value.replace('\'', "'\\''")) +} + +fn startup_script( + token: &str, + index: u32, + bridge: &str, + prior_hook: &str, + ready: &Path, + egress: &str, +) -> String { + let prior = if prior_hook.is_empty() { + String::new() + } else { + format!( + "/bin/sh -c {} -- \"$@\"\n", + shell_quote(&format!("{prior_hook} \"$@\"")) + ) + }; + let types = ICMPV6_ALLOW_TYPES.join(", "); + format!( + r#"#!/bin/bash +set -euo pipefail +exec > {log} 2>&1 +{prior}export PATH=/usr/sbin:/usr/bin:/sbin:/bin +if [[ "${{LXC_HOOK_VERSION:-0}}" == 1 ]]; then + peer="${{LXC_NET_PEER:?missing host veth peer}}" +else + peer="${{6:?missing host veth peer}}" +fi +[[ "$peer" =~ ^[a-zA-Z0-9_.:-]{{1,15}}$ ]] || exit 1 +bridge={bridge} +[[ -d "/sys/class/net/$bridge/bridge" ]] || {{ echo "MXC requires a Linux bridge: $bridge"; exit 1; }} +[[ "$(readlink -f "/sys/class/net/$peer/master")" == "$(readlink -f "/sys/class/net/$bridge")" ]] || {{ echo "MXC host peer $peer is not attached to $bridge"; exit 1; }} +replace="" +if nft list table bridge mxc_{token}_{index} >/dev/null 2>&1; then + replace="delete table bridge mxc_{token}_{index}" +fi + +# Publish both directions in one nftables transaction before guest init. +nft -f - < {ready} +"#, + bridge = shell_quote(bridge), + ready = shell_quote(&ready.to_string_lossy()), + log = shell_quote(&ready.with_extension("log").to_string_lossy()), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mxc_common::logger::Mode; + + fn container_config(contents: &str) -> (PathBuf, LxcContainer) { + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("target") + .join("host-network-tests") + .join(uuid::Uuid::new_v4().simple().to_string()); + fs::create_dir_all(root.join("guest")).unwrap(); + fs::write(root.join("guest/config"), contents).unwrap(); + let container = LxcContainer::new("guest", Some(root.to_str().unwrap())); + (root, container) + } + + #[test] + fn includes_and_resets_determine_the_effective_numeric_interfaces() { + let (root, _) = container_config(""); + let included = root.join("included.conf"); + fs::write(&included, "lxc.net.7.type = veth\nlxc.net.7.link = old\n").unwrap(); + let path = root.join("guest/config"); + fs::write( + &path, + format!( + "lxc.include = {}\nlxc.net =\nlxc.net.3.type = veth\nlxc.net.3.link = bridge3\nlxc.net.9.type = empty\n", + included.display() + ), + ).unwrap(); + + let config = NetworkConfig::read(&path).unwrap(); + assert_eq!( + config.interfaces.keys().copied().collect::>(), + [3, 9] + ); + assert_eq!(config.interfaces[&3]["link"], "bridge3"); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn missing_directory_includes_are_optional_but_missing_file_includes_fail() { + let (root, _) = container_config(""); + let path = root.join("guest/config"); + let missing = root.join("missing"); + fs::write(&path, format!("lxc.include = {}/\n", missing.display())).unwrap(); + assert!(NetworkConfig::read(&path).is_ok()); + + fs::write(&path, format!("lxc.include = {}\n", missing.display())).unwrap(); + assert!(NetworkConfig::read(&path).is_err()); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn directory_includes_read_conf_entries_and_ignore_unrelated_lines() { + let (root, _) = container_config(""); + let directory = root.join("config.d"); + fs::create_dir(&directory).unwrap(); + fs::write( + directory.join("network.conf"), + "unrelated text\nlxc.net.4.type = veth\nlxc.net.4.link = bridge4\n", + ) + .unwrap(); + fs::write(directory.join("ignored.txt"), "lxc.net.9.type = phys\n").unwrap(); + let path = root.join("guest/config"); + fs::write(&path, format!("lxc.include = {}\n", directory.display())).unwrap(); + let config = NetworkConfig::read(&path).unwrap(); + assert_eq!(config.interfaces.keys().copied().collect::>(), [4]); + assert_eq!(config.interfaces[&4]["link"], "bridge4"); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn shared_and_kept_network_namespaces_are_independent() { + let (root, _) = container_config(""); + let path = root.join("guest/config"); + for contents in [ + "lxc.namespace.keep = net\nlxc.namespace.share.net =\n", + "lxc.namespace.share.net = 123\nlxc.namespace.keep =\n", + "lxc.namespace.keep = net\nlxc.namespace.keep = mnt\n", + "lxc.namespace.clone = mnt pid\n", + ] { + fs::write(&path, contents).unwrap(); + assert!(NetworkConfig::read(&path) + .err() + .unwrap() + .contains("shared network")); + } + fs::write(&path, "lxc.namespace.keep = net\nlxc.namespace.keep =\n").unwrap(); + assert!(NetworkConfig::read(&path).is_ok()); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn unsupported_topologies_are_refused_without_installing_a_hook() { + let (root, container) = container_config(""); + let path = root.join("guest/config"); + let mut network = HostNetwork::claim(&container).unwrap(); + let mut logger = Logger::new(Mode::Buffer); + for contents in [ + "lxc.net.0.type = phys\nlxc.net.0.link = eth0\n", + "lxc.net.0.type = veth\nlxc.net.0.link = lxcbr0\nlxc.net.0.veth.mode = router\n", + "lxc.net.0.type = veth\n", + ] { + fs::write(&path, contents).unwrap(); + assert!(network + .configure(&container, &ContainerPolicy::default(), false, &mut logger) + .is_err()); + assert_eq!(fs::read_to_string(&path).unwrap(), contents); + assert!(network.installed().unwrap().is_none()); + } + drop(network); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn no_interface_needs_no_host_firewall_or_startup_hook() { + let contents = "lxc.net.4.type = empty\n"; + let (root, container) = container_config(contents); + let mut network = HostNetwork::claim(&container).unwrap(); + network + .configure( + &container, + &ContainerPolicy::default(), + true, + &mut Logger::new(Mode::Buffer), + ) + .unwrap(); + + assert!(network.installed().unwrap().is_none()); + assert_eq!( + fs::read_to_string(root.join("guest/config")).unwrap(), + contents + ); + network.remove().unwrap(); + drop(network); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn dropping_the_author_preserves_the_script_and_original_user_hook() { + let contents = "# BEGIN MXC host network user-managed\nlxc.hook.version = 1\nlxc.net.2.type = veth\nlxc.net.2.link = lxcbr0\nlxc.net.2.script.up = /user/hook --argument\n# END MXC host network user-managed\n"; + let (root, container) = container_config(contents); + let mut network = HostNetwork::claim(&container).unwrap(); + network + .configure( + &container, + &ContainerPolicy::default(), + false, + &mut Logger::new(Mode::Buffer), + ) + .unwrap(); + let script = network.directory.join("up-2.sh"); + drop(network); + + let network = HostNetwork::claim(&container).unwrap(); + assert!(network.installed().unwrap().is_some()); + let configured = fs::read_to_string(root.join("guest/config")).unwrap(); + assert!(configured.starts_with(contents)); + assert!(fs::read_to_string(script) + .unwrap() + .contains("/user/hook --argument")); + assert_eq!( + without_owned_hooks(&configured, &network.installed().unwrap().unwrap().token) + .unwrap() + .trim_end(), + contents.trim_end() + ); + drop(network); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn a_second_author_cannot_claim_the_same_container() { + let (root, container) = container_config("lxc.net.0.type = empty\n"); + let owner = HostNetwork::claim(&container).unwrap(); + assert!(HostNetwork::claim(&container).is_err()); + drop(owner); + let next = HostNetwork::claim(&container).unwrap(); + drop(next); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn equivalent_storage_paths_cannot_claim_a_container_twice() { + let (root, container) = container_config("lxc.net.0.type = empty\n"); + let owner = HostNetwork::claim(&container).unwrap(); + for path in [ + format!("{}/", root.display()), + format!("{}/guest/..", root.display()), + ] { + let alias = LxcContainer::new("guest", Some(&path)); + assert!( + HostNetwork::claim(&alias).is_err(), + "equivalent storage path {path} acquired ownership of an already owned container" + ); + } + drop(owner); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn equivalent_storage_path_finds_retained_policy() { + let (root, container) = + container_config("lxc.net.0.type = veth\nlxc.net.0.link = lxcbr0\n"); + let mut network = HostNetwork::claim(&container).unwrap(); + network + .configure( + &container, + &ContainerPolicy::default(), + false, + &mut Logger::new(Mode::Buffer), + ) + .unwrap(); + let token = network.installed().unwrap().unwrap().token; + drop(network); + + let path = format!("{}/", root.display()); + let alias = LxcContainer::new("guest", Some(&path)); + let retained = HostNetwork::claim(&alias).unwrap(); + assert_eq!( + retained.installed().unwrap().map(|policy| policy.token), + Some(token), + "retained filtering was lost through equivalent storage path {path}" + ); + drop(retained); + fs::remove_dir_all(root).unwrap(); + } + + #[cfg(unix)] + #[test] + fn symlink_storage_path_cannot_claim_an_owned_container() { + let (root, container) = container_config("lxc.net.0.type = empty\n"); + let path = root.join("alias"); + std::os::unix::fs::symlink(&root, &path).unwrap(); + let alias = LxcContainer::new("guest", Some(path.to_str().unwrap())); + let owner = HostNetwork::claim(&container).unwrap(); + assert!( + HostNetwork::claim(&alias).is_err(), + "symlink storage path acquired ownership of an already owned container" + ); + drop(owner); + fs::remove_dir_all(root).unwrap(); + } + + #[cfg(target_os = "linux")] + #[test] + fn generated_hooks_are_valid_bash_with_quoted_paths_and_user_arguments() { + let script = startup_script( + "12345678901234567890123456789012", + 3, + "lxcbr0", + "/user/hook --argument", + Path::new("/policy path/it's-ready"), + "ip daddr 203.0.113.0/24 meta l4proto tcp tcp dport 80-90 drop\naccept\n", + ); + let mut child = Command::new("bash") + .args(["-n"]) + .stdin(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + child + .stdin + .take() + .unwrap() + .write_all(script.as_bytes()) + .unwrap(); + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + } +} diff --git a/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs b/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs index cb04ee555..20b2eb613 100644 --- a/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs +++ b/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs @@ -128,43 +128,6 @@ fn build_attach_args_with_env_control( args } -/// Whether MXC put firewall chains in the container's network namespace. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum ContainerFirewall { - Installed, - Absent, -} - -/// Drop Network capabilities for sandboxing purposes. -#[cfg(target_os = "linux")] -fn confine_network_capabilities(command: &mut std::process::Command) { - use std::os::unix::process::CommandExt; - - // `libc` does not export these; the values are from linux/capability.h. - const CAP_NET_ADMIN: libc::c_ulong = 12; - const CAP_NET_RAW: libc::c_ulong = 13; - - // SAFETY: `pre_exec` runs between fork and exec, where only - // async-signal-safe work is permitted. `prctl` is a bare syscall, and this - // closure allocates nothing and captures nothing. - unsafe { - command.pre_exec(|| { - // Drop CAP_NET_ADMIN and CAP_NET_RAW from the bounding set. - if libc::prctl(libc::PR_CAPBSET_DROP, CAP_NET_ADMIN, 0, 0, 0) != 0 { - return Err(std::io::Error::last_os_error()); - } - - if libc::prctl(libc::PR_CAPBSET_DROP, CAP_NET_RAW, 0, 0, 0) != 0 { - return Err(std::io::Error::last_os_error()); - } - - Ok(()) - }); - } -} - -/// A `/etc/hosts` helper emits one `mxc:` diagnostic line, so this is far above -/// any legitimate output. #[cfg(target_os = "linux")] const MAX_CAPTURED_BYTES: u64 = 8 * 1024; @@ -249,21 +212,22 @@ pub enum StartNetwork { } impl StartNetwork { - fn to_start_args(self, configured_interfaces: usize) -> Vec { + fn to_start_args(self) -> Vec { match self { StartNetwork::FromContainerConfig => Vec::new(), StartNetwork::NoInterface => { - // Index 0 is emitted even for a container that configures no - // interface at all: without an `lxc.net` entry LXC leaves the - // container in the host's network namespace. - let mut args = Vec::new(); - for index in 0..configured_interfaces.max(1) { - args.push("-s".to_string()); - args.push(format!("lxc.net.{index}.type=empty")); - args.push("-s".to_string()); - args.push(format!("lxc.net.{index}.flags=up")); - } - args + // Reset included interfaces before creating an isolated loopback namespace. + [ + "-s", + "lxc.net=", + "-s", + "lxc.net.0.type=empty", + "-s", + "lxc.net.0.flags=up", + ] + .into_iter() + .map(String::from) + .collect() } } } @@ -397,6 +361,21 @@ impl LxcContainer { } } + pub(crate) fn is_stopped(&self) -> Result { + let output = self + .lxc_command("lxc-info") + .args(["-s", "-H"]) + .output() + .map_err(|error| format!("Cannot inspect container state: {error}"))?; + if !output.status.success() { + return Err(format!( + "Cannot inspect container state: {}", + String::from_utf8_lossy(&output.stderr) + )); + } + Ok(String::from_utf8_lossy(&output.stdout).trim() == "STOPPED") + } + pub fn init_pid(&self) -> Option { let output = self.lxc_command("lxc-info").arg("-p").output().ok()?; if !output.status.success() { @@ -491,29 +470,56 @@ impl LxcContainer { out } - pub fn start(&self, network: StartNetwork) -> Result<(), String> { - let mut cmd = self.lxc_command("lxc-start"); - cmd.args(network.to_start_args(self.configured_interface_count())); - Self::run_tool(cmd) - } - - /// How many `lxc.net.N` interfaces the container's config declares. - fn configured_interface_count(&self) -> usize { - let Ok(config) = std::fs::read_to_string(self.config_file_path()) else { - return 0; - }; - Self::highest_interface_index(&config).map_or(0, |index| index + 1) + fn validate_unprivileged(&self) -> Result<(), String> { + let validation = (|| { + let pid = self.init_pid().ok_or_else(|| { + format!( + "Failed to get container '{}' init PID to inspect UID map", + self.name + ) + })?; + let mapping = + std::fs::read_to_string(format!("/proc/{pid}/uid_map")).map_err(|error| { + format!( + "Failed to read container '{}' UID map for init PID {pid}: {error}", + self.name + ) + })?; + let root_is_shifted = mapping.lines().any(|line| { + let columns = line + .split_whitespace() + .map(str::parse::) + .collect::, _>>(); + matches!( + columns.as_deref(), + Ok([0, outside, count]) if *outside != 0 && *count != 0 + ) + }); + if !root_is_shifted { + return Err(format!( + "Container '{}' has no unprivileged root UID mapping. MXC will not run the requested command.", + self.name + )); + } + Ok(()) + })(); + + if let Err(error) = validation { + return match self.stop_owned() { + Ok(()) => Err(error), + Err(stop_error) => Err(format!( + "{error}; failed to stop rejected container: {stop_error}" + )), + }; + } + Ok(()) } - fn highest_interface_index(config: &str) -> Option { - config - .lines() - .filter_map(|line| { - let key = line.split('=').next()?.trim(); - let index = key.strip_prefix("lxc.net.")?.split('.').next()?; - index.parse::().ok() - }) - .max() + pub fn start(&self, network: StartNetwork) -> Result<(), String> { + let mut cmd = self.lxc_command("lxc-start"); + cmd.args(network.to_start_args()); + Self::run_tool(cmd)?; + self.validate_unprivileged() } pub fn exec( @@ -523,12 +529,23 @@ impl LxcContainer { _timeout_ms: u32, ) -> Result<(i32, String, String), String> { // TODO: Implement timeout and working directory support. - let mut cmd = self.lxc_command("lxc-execute"); + self.start(StartNetwork::FromContainerConfig)?; + let mut cmd = self.lxc_command("lxc-attach"); cmd.args(["--", "/bin/sh", "-c", command]); let output = cmd .output() - .map_err(|e| format!("Failed to run lxc-execute: {}", e))?; + .map_err(|e| format!("Failed to run lxc-attach: {}", e)); + if let Err(stop_error) = self.stop() { + return Err(match output { + Ok(output) => format!( + "Command exited with code {}; failed to stop container: {stop_error}", + output.status.code().unwrap_or(-1) + ), + Err(error) => format!("{error}; failed to stop container: {stop_error}"), + }); + } + let output = output?; Ok(( output.status.code().unwrap_or(-1), @@ -547,7 +564,6 @@ impl LxcContainer { env: &[String], force_clear_env: bool, timeout: Option, - firewall: ContainerFirewall, ) -> Result<(i32, String, String), String> { use crate::mxc_pty::{run_with_pty, PtyOptions, PtyOutcome, Signal}; @@ -555,17 +571,7 @@ impl LxcContainer { // inner shell would ignore Ctrl-C. const UNBLOCK: &[Signal] = &[Signal::SIGHUP, Signal::SIGTERM, Signal::SIGINT]; - let mut cmd = self.lxc_command("lxc-attach"); - cmd.args(build_attach_args_with_env_control( - env, - working_directory, - command, - force_clear_env, - )); - - if firewall == ContainerFirewall::Installed { - confine_network_capabilities(&mut cmd); - } + let cmd = self.attach_command(command, working_directory, env, force_clear_env)?; let options = PtyOptions { unblock_signals: UNBLOCK, @@ -594,7 +600,6 @@ impl LxcContainer { _env: &[String], _force_clear_env: bool, _timeout: Option, - _firewall: ContainerFirewall, ) -> Result<(i32, String, String), String> { Err("LxcContainer::attach_run is only supported on Linux".to_string()) } @@ -609,29 +614,18 @@ impl LxcContainer { env: &[String], force_clear_env: bool, timeout: Option, - firewall: ContainerFirewall, ) -> Result<(i32, String, String), String> { use crate::mxc_common::interruptible_reader::wrap_pipe; use crate::mxc_common::sandbox_process::{wait_with_timeout, StreamCloser, WaitError}; use std::process::Stdio; - let mut cmd = self.lxc_command("lxc-attach"); - cmd.args(build_attach_args_with_env_control( - env, - working_directory, - command, - force_clear_env, - )); + let mut cmd = self.attach_command(command, working_directory, env, force_clear_env)?; cmd.stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); unblock_fatal_signals(&mut cmd); - if firewall == ContainerFirewall::Installed { - confine_network_capabilities(&mut cmd); - } - let mut child = cmd .spawn() .map_err(|e| format!("Failed to run lxc-attach: {}", e))?; @@ -691,7 +685,6 @@ impl LxcContainer { _env: &[String], _force_clear_env: bool, _timeout: Option, - _firewall: ContainerFirewall, ) -> Result<(i32, String, String), String> { Err("LxcContainer::attach_capture is only supported on Linux".to_string()) } @@ -708,12 +701,10 @@ impl LxcContainer { working_directory: &str, env: &[String], force_clear_env: bool, - firewall: ContainerFirewall, ) -> Result { use std::process::Stdio; - let mut cmd = - self.attach_command(command, working_directory, env, force_clear_env, firewall); + let mut cmd = self.attach_command(command, working_directory, env, force_clear_env)?; cmd.stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -730,15 +721,13 @@ impl LxcContainer { working_directory: &str, env: &[String], force_clear_env: bool, - firewall: ContainerFirewall, size: crate::mxc_pty::PtySize, ) -> Result<(std::process::Child, crate::mxc_pty::LivePty), String> { use crate::mxc_pty::Signal; const UNBLOCK: &[Signal] = &[Signal::SIGHUP, Signal::SIGTERM, Signal::SIGINT]; - let mut cmd = - self.attach_command(command, working_directory, env, force_clear_env, firewall); + let mut cmd = self.attach_command(command, working_directory, env, force_clear_env)?; let pty = crate::mxc_pty::LivePty::attach_forwarded(&mut cmd, size, UNBLOCK) .map_err(|error| format!("Failed to allocate lxc-attach PTY: {error}"))?; let mut child = cmd @@ -794,19 +783,12 @@ impl LxcContainer { working_directory: &str, env: &[String], force_clear_env: bool, - firewall: ContainerFirewall, - ) -> std::process::Command { + ) -> Result { let mut cmd = self.lxc_command("lxc-attach"); - cmd.args(build_attach_args_with_env_control( - env, - working_directory, - command, - force_clear_env, - )); - if firewall == ContainerFirewall::Installed { - confine_network_capabilities(&mut cmd); - } - cmd + let args = + build_attach_args_with_env_control(env, working_directory, command, force_clear_env); + cmd.args(args); + Ok(cmd) } /// Stub for the workspace-wide clippy lane that runs on Windows. @@ -817,13 +799,18 @@ impl LxcContainer { _working_directory: &str, _env: &[String], _force_clear_env: bool, - _firewall: ContainerFirewall, ) -> Result { Err("LxcContainer::attach_spawn is only supported on Linux".to_string()) } /// Stop the container by killing it, not by asking it to exit. pub fn stop(&self) -> Result<(), String> { + let mut network = super::host_network::HostNetwork::claim(self)?; + self.stop_owned()?; + network.remove() + } + + pub(crate) fn stop_owned(&self) -> Result<(), String> { Self::run_release_tool(self.stop_command()) } @@ -838,6 +825,12 @@ impl LxcContainer { } pub fn destroy(&self) -> Result<(), String> { + let mut network = super::host_network::HostNetwork::claim(self)?; + self.destroy_owned()?; + network.remove() + } + + pub(crate) fn destroy_owned(&self) -> Result<(), String> { let mut cmd = self.lxc_command("lxc-destroy"); cmd.arg("-f"); @@ -895,71 +888,24 @@ mod tests { #[test] fn a_run_with_no_interface_states_that_to_lxc_start() { assert_eq!( - StartNetwork::NoInterface.to_start_args(1), - ["-s", "lxc.net.0.type=empty", "-s", "lxc.net.0.flags=up"], - "lxc-start reads each config item from the -s that precedes it, \ - and loopback stays up for a workload that binds 127.0.0.1" - ); - } - - #[test] - fn every_configured_interface_is_emptied_not_just_the_first() { - assert_eq!( - StartNetwork::NoInterface.to_start_args(3), + StartNetwork::NoInterface.to_start_args(), [ "-s", - "lxc.net.0.type=empty", - "-s", - "lxc.net.0.flags=up", - "-s", - "lxc.net.1.type=empty", - "-s", - "lxc.net.1.flags=up", + "lxc.net=", "-s", - "lxc.net.2.type=empty", + "lxc.net.0.type=empty", "-s", - "lxc.net.2.flags=up", + "lxc.net.0.flags=up" ], - "an interface nobody names keeps its link, and a policy that permits \ - no network installs no chain to filter it" - ); - } - - #[test] - fn a_container_configuring_no_interface_still_gets_one_emptied() { - assert_eq!( - StartNetwork::NoInterface.to_start_args(0), - ["-s", "lxc.net.0.type=empty", "-s", "lxc.net.0.flags=up"], - "LXC leaves a container with no lxc.net entry in the host's network namespace" + "lxc-start reads each config item from the -s that precedes it, \ + and loopback stays up for a workload that binds 127.0.0.1" ); } - #[test] - fn the_interface_count_comes_from_the_highest_index_the_config_names() { - for (config, expected) in [ - ("", None), - ("lxc.net.0.type = veth\n", Some(0)), - ("lxc.net.0.type = veth\nlxc.net.1.type = veth\n", Some(1)), - ("lxc.net.4.type = veth\nlxc.net.1.type = veth\n", Some(4)), - (" lxc.net.2.link = lxcbr0\n", Some(2)), - // Keys that only look like an interface entry. - ("lxc.network.0.type = veth\n", None), - ("lxc.net.x.type = veth\n", None), - ] { - assert_eq!( - LxcContainer::highest_interface_index(config), - expected, - "config {config:?}" - ); - } - } - #[test] fn a_run_that_keeps_the_container_config_states_nothing() { assert!( - StartNetwork::FromContainerConfig - .to_start_args(2) - .is_empty(), + StartNetwork::FromContainerConfig.to_start_args().is_empty(), "the container's own config must be left to decide its interfaces" ); } @@ -1458,25 +1404,6 @@ mod tests { ); } - // The attach paths confine the workload only when chains are installed: - // dropping a capability is privileged, so applying it to every run costs an - // unprivileged caller the whole execution. - #[cfg(target_os = "linux")] - #[test] - fn confining_a_command_takes_a_privilege_an_unprivileged_caller_lacks() { - // SAFETY: `geteuid` is a thread-safe, side-effect-free libc call. - let running_as_root = unsafe { libc::geteuid() } == 0; - - let mut confined = std::process::Command::new("/bin/true"); - confine_network_capabilities(&mut confined); - - assert_eq!( - confined.status().is_ok(), - running_as_root, - "a confined command must spawn only for a caller holding CAP_SETPCAP" - ); - } - #[cfg(target_os = "linux")] #[test] fn an_unconfined_command_spawns_whoever_the_caller_is() { diff --git a/src/mxc-sdk/src/backends/lxc/common/lxc_runner.rs b/src/mxc-sdk/src/backends/lxc/common/lxc_runner.rs index 00061b7a8..9d8f801a5 100644 --- a/src/mxc-sdk/src/backends/lxc/common/lxc_runner.rs +++ b/src/mxc-sdk/src/backends/lxc/common/lxc_runner.rs @@ -34,12 +34,11 @@ use crate::mxc_pty::{LivePty, PtySize as UnixPtySize}; use crate::lxc_common::availability::LxcTooling; use crate::lxc_common::filesystem_mounts; +use crate::lxc_common::host_network::HostNetwork; use crate::lxc_common::loopback_relay::LoopbackRelay; -use crate::lxc_common::lxc_bindings::{ContainerFirewall, LxcContainer, StartNetwork}; +use crate::lxc_common::lxc_bindings::{LxcContainer, StartNetwork}; use crate::lxc_common::network_ingress::IngressManager; -use crate::lxc_common::network_iptables::{ - needs_network, plan_network, EgressHookPoint, NetworkIptablesManager, -}; +use crate::lxc_common::network_iptables::{plan_network, EgressHookPoint, NetworkIptablesManager}; use crate::lxc_common::signal_cleanup; const HOSTS_PIN_MARKER: &str = "#mxc-proxy-pin"; @@ -175,6 +174,7 @@ impl ContainerNameClaim { impl Drop for ContainerNameClaim { fn drop(&mut self) { + signal_cleanup::clear_active_registration(&self.name); lock_live_container_names().remove(&self.name); } } @@ -303,14 +303,25 @@ impl LxcScriptRunner { &self, container: &LxcContainer, container_created: bool, + network: &mut HostNetwork, logger: &mut Logger, - ) { + ) -> Vec { let release = self.release_kind(container_created); let result = match release { - ContainerRelease::Destroy => container.destroy(), - ContainerRelease::Stop => container.stop(), + ContainerRelease::Destroy => container.destroy_owned(), + ContainerRelease::Stop => container.stop_owned(), }; + let mut failures = Vec::new(); + match &result { + Ok(()) => { + if let Err(error) = network.remove() { + failures.push(error); + } + } + Err(error) => failures.push(format!("failed to {} container: {error}", release.verb())), + } Self::report_release_failure(release, result, logger); + failures } /// Release a sandbox whose workload never got a handle to own its cleanup, @@ -318,19 +329,34 @@ impl LxcScriptRunner { #[cfg_attr(not(target_os = "linux"), allow(dead_code))] fn abandon_prepared(&self, prepared: &mut PreparedSandbox, logger: &mut Logger) -> Vec { prepared.relay = None; + if let Some(network) = &mut prepared.host_network { + let release = self.release_kind(prepared.container_created); + let result = match release { + ContainerRelease::Destroy => prepared.container.destroy_owned(), + ContainerRelease::Stop if prepared.force_stopped => Ok(()), + ContainerRelease::Stop => prepared.container.stop_owned(), + }; + return match result { + Ok(()) => { + prepared.released = matches!(release, ContainerRelease::Destroy); + network.remove().err().into_iter().collect() + } + Err(error) => vec![format!("failed to {} container: {error}", release.verb())], + }; + } // Namespace-scoped chains come out before the container does. let mut failures = prepared.remove_in_container_state(self.cleanup_policy, logger); let release = self.release_kind(prepared.container_created); let result = match release { - ContainerRelease::Destroy => prepared.container.destroy(), + ContainerRelease::Destroy => prepared.container.destroy_owned(), // Already down, and `lxc-stop` against a container that is not // running reports a failure with nothing behind it. ContainerRelease::Stop if prepared.force_stopped => Ok(()), - ContainerRelease::Stop => prepared.container.stop(), + ContainerRelease::Stop => prepared.container.stop_owned(), }; match &result { Ok(()) => { @@ -345,6 +371,7 @@ impl LxcScriptRunner { failures } + #[cfg(test)] fn set_up_network_rules( &self, container_name: &str, @@ -368,6 +395,7 @@ impl LxcScriptRunner { Ok((fw_manager, ingress_manager)) } + #[cfg(test)] fn egress_apply_outcome(result: Result) -> Result<(), String> { match result { Ok(true) => Ok(()), @@ -376,6 +404,7 @@ impl LxcScriptRunner { } } + #[cfg(test)] fn ingress_apply_outcome(result: Result) -> Result<(), String> { match result { Ok(true) => Ok(()), @@ -384,6 +413,7 @@ impl LxcScriptRunner { } } + #[cfg(test)] fn enforce_network_readiness( &self, container_name: &str, @@ -475,9 +505,6 @@ impl LxcScriptRunner { } } - if self.destroy_on_exit { - signal_cleanup::set_active(&container_name); - } let _ = writeln!(logger, "Container name: {}", container_name); let _ = writeln!( logger, @@ -496,6 +523,14 @@ impl LxcScriptRunner { } let container = LxcContainer::new(&container_name, None); + let mut host_network = HostNetwork::claim(&container) + .map_err(|error| ScriptResponse::failed(FailurePhase::LaunchFailed, &error))?; + signal_cleanup::set_active_lifecycle( + &container_name, + self.destroy_on_exit, + self.cleanup_policy, + ); + signal_cleanup::set_active_host_network(host_network.clone()); let mut container_created = false; if !container.is_defined() { @@ -515,32 +550,45 @@ impl LxcScriptRunner { if let Err(e) = filesystem_mounts::configure_filesystem_mounts(&container, &request.policy, logger) { - if self.destroy_on_exit || container_created { - let _ = container.destroy(); - } + let failures = self.release_after_failure( + &container, + container_created, + &mut host_network, + logger, + ); return Err(ScriptResponse::failed( FailurePhase::LaunchFailed, - &format!("Failed to configure filesystem: {}", e), + &launch_failure_message(&format!("Failed to configure filesystem: {e}"), &failures), )); } // LXC reads the network section only at start. A container an earlier // run left running is still on that run's topology. - if container.is_running() { + let stopped = container_created + || container + .is_stopped() + .map_err(|error| ScriptResponse::failed(FailurePhase::LaunchFailed, &error))?; + if !stopped { let _ = writeln!( logger, "Container already running; stopping it so this run's network policy applies." ); - if let Err(e) = container.stop() { - if self.destroy_on_exit || container_created { - let _ = container.destroy(); - } + if let Err(e) = container.stop_owned() { + let failures = self.release_after_failure( + &container, + container_created, + &mut host_network, + logger, + ); return Err(ScriptResponse::failed( FailurePhase::LaunchFailed, - &format!( - "Failed to stop a container left running by an earlier run: {}. \ + &launch_failure_message( + &format!( + "Failed to stop a container left running by an earlier run: {}. \ Its network policy is the earlier run's, so the script was not run.", - e + e + ), + &failures, ), )); } @@ -558,50 +606,56 @@ impl LxcScriptRunner { StartNetwork::FromContainerConfig }; + if let Err(error) = + host_network.configure(&container, &request.policy, plan.omits_interface(), logger) + { + let failures = self.release_after_failure( + &container, + container_created, + &mut host_network, + logger, + ); + return Err(ScriptResponse::failed( + FailurePhase::LaunchFailed, + &launch_failure_message(&error, &failures), + )); + } + let _ = writeln!(logger, "Starting LXC container..."); if let Err(e) = container.start(network) { - if self.destroy_on_exit || container_created { - let _ = container.destroy(); - } + let reason = host_network.startup_failure(&e); + let failures = self.release_after_failure( + &container, + container_created, + &mut host_network, + logger, + ); return Err(ScriptResponse::failed( FailurePhase::LaunchFailed, - &format!("Failed to start container: {}", e), + &launch_failure_message(&format!("Failed to start container: {reason}"), &failures), )); } let _ = writeln!(logger, "Container started successfully."); - let needs_network = needs_network(&request.policy); - - if needs_network { - // Alpine DHCP leases can arrive at about nine seconds; thirty - // seconds leaves margin. - let timeout = Duration::from_secs(30); - if let Some(response) = self.enforce_network_readiness( - &container_name, + if let Err(error) = host_network.verify_started() { + let failures = self.release_after_failure( + &container, container_created, - timeout, + &mut host_network, logger, - Self::wait_for_network, - |release| match release { - ContainerRelease::Destroy => container.destroy(), - ContainerRelease::Stop => container.stop(), - }, - ) { - return Err(response); - } - } - - // The init PID names the container's network namespace. - let netns_pid = container.init_pid(); - let Some(hook_point) = egress_hook_point(netns_pid, plan.installs_firewall()) else { - self.release_after_failure(&container, container_created, logger); + ); return Err(ScriptResponse::failed( FailurePhase::PostLaunchFailed, - "Failed to discover the container init PID; cannot enter the container \ - network namespace to install its firewall. Aborting rather than \ - running without it.", + &launch_failure_message(&error, &failures), )); - }; + } + + if !plan.omits_interface() && NetworkIptablesManager::permits_ipv4_dhcp(&request.policy) { + // DHCP runs behind the installed policy; an absent lease does not forbid local commands. + let _ = Self::wait_for_network(&container_name, Duration::from_secs(30), logger); + } + + let netns_pid = container.init_pid(); if let Some(pid) = netns_pid { let _ = writeln!(logger, "Container init PID: {}", pid); @@ -610,64 +664,29 @@ impl LxcScriptRunner { } } - let setup = self.set_up_network_rules( - &container_name, - hook_point, - netns_pid, - &request.policy, - logger, - ); - - let (fw_manager, ingress_manager) = match setup { - Ok(managers) => managers, - Err(e) => { - self.release_after_failure(&container, container_created, logger); - return Err(ScriptResponse::failed(FailurePhase::PostLaunchFailed, &e)); - } - }; - - // figure out if a firewall needs to be installed. - let is_firewall_installed = if fw_manager.rules_applied() - || ingress_manager - .as_ref() - .is_some_and(|mgr| mgr.rules_applied()) - { - ContainerFirewall::Installed - } else { - ContainerFirewall::Absent - }; - - // Dropping CAP_NET_RAW also removed raw ICMP. Widen the container's - // ping_group_range so unprivileged ping still works. - if is_firewall_installed == ContainerFirewall::Installed { - if let Err(e) = fw_manager.allow_ping_sockets(logger) { - self.release_after_failure(&container, container_created, logger); - return Err(ScriptResponse::failed(FailurePhase::PostLaunchFailed, &e)); - } - } - // A reused container can carry a pin left by an older run. let stale_pin_error = Self::clear_stale_hosts_pin(container_created, |command| { container - .attach_capture( - command, - "/", - &[], - true, - Some(HOSTS_COMMAND_TIMEOUT), - is_firewall_installed, - ) + .attach_capture(command, "/", &[], true, Some(HOSTS_COMMAND_TIMEOUT)) .map_err(|e| e.to_string()) }); if let Err(reason) = stale_pin_error { - self.release_after_failure(&container, container_created, logger); + let failures = self.release_after_failure( + &container, + container_created, + &mut host_network, + logger, + ); return Err(ScriptResponse::failed( FailurePhase::PostLaunchFailed, - &format!( - "Failed to clear a stale network proxy pin from the container's \ + &launch_failure_message( + &format!( + "Failed to clear a stale network proxy pin from the container's \ /etc/hosts: {}. The script was not run, because it could have resolved \ the pinned hostname to an address this policy did not authorize.", - reason + reason + ), + &failures, ), )); } @@ -690,14 +709,22 @@ impl LxcScriptRunner { match started { Ok(relay) => Some(relay), Err(e) => { - self.release_after_failure(&container, container_created, logger); + let failures = self.release_after_failure( + &container, + container_created, + &mut host_network, + logger, + ); return Err(ScriptResponse::failed( FailurePhase::PostLaunchFailed, - &format!( + &launch_failure_message( + &format!( "LXC: cannot relay runtimeConfig.networkProxy into the container: \ {}. The script was not run.", e ), + &failures, + ), )); } } @@ -706,11 +733,11 @@ impl LxcScriptRunner { }; Ok(PreparedSandbox { - fw_manager, - ingress_manager, + fw_manager: NetworkIptablesManager::new(&container_name, EgressHookPoint::Unhooked), + ingress_manager: None, + host_network: Some(host_network), container, container_created, - firewall: is_firewall_installed, relay, released: false, force_stopped: false, @@ -739,10 +766,14 @@ impl LxcScriptRunner { &prepared.exec_env, true, prepared.timeout, - prepared.firewall, ); - let response = match result { + let stop_failure = if result.is_err() { + prepared.force_stop().err() + } else { + None + }; + let mut response = match result { Ok((exit_code, stdout, stderr)) => ScriptResponse { exit_code, standard_out: stdout, @@ -753,7 +784,18 @@ impl LxcScriptRunner { Err(e) => ScriptResponse::error(&format!("Execution failed: {}", e)), }; - prepared.tear_down(self.cleanup_policy, self.destroy_on_exit, logger); + let mut failures = prepared.tear_down(self.cleanup_policy, self.destroy_on_exit, logger); + if let Some(error) = stop_failure { + failures.push(format!( + "failed to stop container after execution failure: {error}" + )); + } + if !failures.is_empty() { + response.error_message = launch_failure_message(&response.error_message, &failures); + if response.exit_code == 0 { + response.exit_code = 1; + } + } response } @@ -827,13 +869,12 @@ impl LxcScriptRunner { struct PreparedSandbox { fw_manager: NetworkIptablesManager, ingress_manager: Option, + host_network: Option, container: LxcContainer, #[cfg_attr(not(target_os = "linux"), allow(dead_code))] container_created: bool, - firewall: ContainerFirewall, - /// Carries the runtime proxy's loopback port into the container, and holds /// the container's network namespace open until it is dropped. relay: Option, @@ -926,6 +967,28 @@ impl PreparedSandbox { ) -> Vec { let mut failures = Vec::new(); self.relay = None; + if let Some(network) = &mut self.host_network { + if destroy_on_exit && !self.released { + let _ = writeln!(logger, "Destroying container..."); + match self.container.destroy_owned() { + Ok(()) => self.released = true, + Err(error) => { + return vec![format!( + "failed to destroy container: {error}; host filtering retained" + )]; + } + } + } + if self.stop_failed && !self.released { + return vec!["the container could not be stopped; host filtering retained".into()]; + } + if cleanup_policy || self.released { + if let Err(error) = network.remove() { + failures.push(error); + } + } + return failures; + } // Destroying takes the network namespace down with it. Chains that // were never hooked live on the host and still have to come out below. @@ -937,7 +1000,7 @@ impl PreparedSandbox { if destroying { let _ = writeln!(logger, "Destroying container..."); - match self.container.destroy() { + match self.container.destroy_owned() { Ok(()) => { self.released = true; self.note_namespace_gone(); @@ -969,14 +1032,12 @@ impl PreparedSandbox { } } - /// Kill every process in the container, which is the only way to reach a - /// workload that lives in its PID namespace. #[cfg_attr(not(target_os = "linux"), allow(dead_code))] fn force_stop(&mut self) -> Result<(), String> { if self.force_stopped { return Ok(()); } - if let Err(e) = self.container.stop() { + if let Err(e) = self.container.stop_owned() { self.stop_failed = true; return Err(e); } @@ -1169,7 +1230,6 @@ impl LxcScriptRunner { &prepared.start_directory, &prepared.exec_env, true, - prepared.firewall, ) .map(|child| (child, None)), StdioMode::Pty(size) => prepared @@ -1179,7 +1239,6 @@ impl LxcScriptRunner { &prepared.start_directory, &prepared.exec_env, true, - prepared.firewall, UnixPtySize { rows: size.rows, cols: size.cols, @@ -1216,8 +1275,6 @@ impl LxcScriptRunner { (out, out_canceller, err, err_canceller) } (out_result, err_result) => { - // The workload is already running and no handle exists to own - // its cleanup, so it comes down here. if prepared.force_stop().is_err() { let _ = child.kill(); } @@ -1296,15 +1353,7 @@ struct LxcChild { #[cfg(target_os = "linux")] impl LxcChild { fn terminate(&mut self) -> std::io::Result<()> { - // Killed first so the reap, and the stream drains waiting on it, are - // never parked behind a stop that is slow or wedged. The workload is - // orphaned for as long as the stop takes, with its chains still up. let _ = self.child.kill(); - - // `lxc-attach` is a host process while the workload runs in the - // container's PID namespace under container init, so nothing aimed at - // the host process or its group reaches the workload; only stopping the - // container does. self.prepared.force_stop().map_err(std::io::Error::other) } } @@ -1481,7 +1530,7 @@ impl SandboxProcess for LxcSandboxProcess { if let Err(error) = self.kill_for_timeout() { let error = format!( "LXC: script timed out, and the container could not be stopped, so the \ - workload may still be running: {error}" + workload may still be running: {error}" ); self.inner.termination_error = Some(error.clone()); return Err(std::io::Error::other(error)); @@ -1502,8 +1551,6 @@ impl SandboxProcess for LxcSandboxProcess { } fn kill(&mut self) -> std::io::Result<()> { - // Only a clean teardown releases the container, so a failed one leaves - // this latch open and still reaches the stop below. if self.teardown_done { return Ok(()); } @@ -1571,9 +1618,6 @@ impl SandboxProcess for LxcSandboxProcess { match wait_with_timeout(&mut self.inner.child, remaining_timeout) { Ok(status) => Ok(status.code().unwrap_or(-1)), Err(WaitError::Timeout) => { - // Stopping the container releases the pipe write ends a - // backgrounded descendant would otherwise hold past the - // deadline, so the drains below can finish. if let Err(error) = self.kill_for_timeout() { let error = format!( "LXC: script timed out, and the container could not be stopped, so \ @@ -1634,6 +1678,7 @@ fn uuid_simple() -> String { /// Where egress rules attach, or `None` when a firewall is needed and the /// container's network namespace cannot be found. +#[cfg(test)] fn egress_hook_point(netns_pid: Option, installs_firewall: bool) -> Option { match netns_pid { Some(pid) => Some(EgressHookPoint::ContainerNetns(pid)), @@ -1907,6 +1952,57 @@ mod tests { ); } + #[test] + fn a_stopped_survivor_retains_its_preserved_host_policy() { + let fake = crate::lxc_common::network_iptables::test_firewall::install(); + let mut prepared = prepared_with_applied_rules("stopped-preserved-host"); + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("target") + .join("host-network-tests") + .join(uuid_simple()); + let config = root.join(prepared.container.name()).join("config"); + std::fs::create_dir_all(config.parent().unwrap()).unwrap(); + std::fs::write(&config, "lxc.net.0.type = veth\nlxc.net.0.link = lxcbr0\n").unwrap(); + prepared.container = + LxcContainer::new(prepared.container.name(), Some(root.to_str().unwrap())); + let mut network = HostNetwork::claim(&prepared.container).unwrap(); + network + .configure( + &prepared.container, + &ContainerPolicy::default(), + false, + &mut Logger::new(Mode::Buffer), + ) + .unwrap(); + let directory = std::fs::read_dir(root.join(".mxc-network")) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + std::fs::write(directory.join("up-0.ready"), b"ready").unwrap(); + network.verify_started().unwrap(); + let configured = std::fs::read_to_string(&config).unwrap(); + prepared.host_network = Some(network); + prepared.force_stopped = true; + let issued = fake.issued(); + + let failures = prepared.tear_down(false, false, &mut Logger::new(Mode::Buffer)); + let attempted = fake.issued(); + drop(prepared); + + assert!(failures.is_empty(), "{failures:?}"); + assert_eq!(std::fs::read_to_string(config).unwrap(), configured); + for file in ["policy.json", "up-0.sh", "up-0.ready"] { + assert!(directory.join(file).is_file(), "removed retained {file}"); + } + std::fs::remove_dir_all(root).unwrap(); + assert_eq!( + attempted, issued, + "stopping a survivor removed its preserved host policy" + ); + } + /// Install a hooked egress chain on a prepared sandbox, so the /// netns-scoped state has something to forget. Ingress is left alone: its /// manager drives `nsenter` through a runner the iptables fake does not @@ -2139,7 +2235,7 @@ mod tests { ingress_manager, container: LxcContainer::new(&name, Some("/var/lib/lxc")), container_created: false, - firewall: ContainerFirewall::Absent, + host_network: None, relay: None, released: false, force_stopped: false, diff --git a/src/mxc-sdk/src/backends/lxc/common/mod.rs b/src/mxc-sdk/src/backends/lxc/common/mod.rs index e2b9ac9fc..752d6ffe5 100644 --- a/src/mxc-sdk/src/backends/lxc/common/mod.rs +++ b/src/mxc-sdk/src/backends/lxc/common/mod.rs @@ -5,6 +5,7 @@ pub mod availability; #[cfg(target_os = "linux")] mod connection; pub mod filesystem_mounts; +pub(crate) mod host_network; pub(crate) mod loopback_relay; pub mod lxc_bindings; pub mod lxc_runner; diff --git a/src/mxc-sdk/src/backends/lxc/common/network_ingress.rs b/src/mxc-sdk/src/backends/lxc/common/network_ingress.rs index 23da80707..9be7c8463 100644 --- a/src/mxc-sdk/src/backends/lxc/common/network_ingress.rs +++ b/src/mxc-sdk/src/backends/lxc/common/network_ingress.rs @@ -50,7 +50,7 @@ const ICMPV6_NEIGHBOR_ADVERTISEMENT: &str = "136"; const ICMPV6_MULTICAST_LISTENER_REPORT_V2: &str = "143"; /// RFC 4890 permits dropping echo request, echo reply, and redirect at a firewall. -const ICMPV6_ALLOW_TYPES: [&str; 12] = [ +pub(crate) const ICMPV6_ALLOW_TYPES: [&str; 12] = [ ICMPV6_ROUTER_SOLICITATION, ICMPV6_ROUTER_ADVERTISEMENT, ICMPV6_NEIGHBOR_SOLICITATION, @@ -301,7 +301,7 @@ impl IngressManager { } } - fn permissive_inbound_field(policy: &ContainerPolicy) -> Option<&'static str> { + pub(crate) fn permissive_inbound_field(policy: &ContainerPolicy) -> Option<&'static str> { if let Some(ingress) = policy.network_ingress.as_ref() { if ingress.default == NetworkAction::Allow { return Some("network.ingress.default"); diff --git a/src/mxc-sdk/src/backends/lxc/common/network_iptables.rs b/src/mxc-sdk/src/backends/lxc/common/network_iptables.rs index 55729fffe..f9e938a31 100644 --- a/src/mxc-sdk/src/backends/lxc/common/network_iptables.rs +++ b/src/mxc-sdk/src/backends/lxc/common/network_iptables.rs @@ -49,6 +49,7 @@ pub(crate) fn plan_network(policy: &ContainerPolicy) -> NetworkPlan { } } +#[cfg(test)] pub(crate) fn needs_network(policy: &ContainerPolicy) -> bool { !plan_network(policy).omits_interface() } @@ -454,6 +455,111 @@ impl NetworkIptablesManager { Self::lower_egress(policy).map(|_| ()) } + pub(crate) fn host_egress_rules( + policy: &ContainerPolicy, + logger: &mut Logger, + ) -> Result { + let lowered = Self::build_policy_rules_logged("egress", policy, logger)?; + let mut output = String::new(); + for (family, rules) in [("ip", lowered.ipv4), ("ip6", lowered.ipv6)] { + for args in rules { + let mut expression = format!("{family} daddr {}", args[3]); + let mut protocol = ""; + for pair in args[4..].chunks_exact(2) { + match pair[0].as_str() { + "-p" => { + protocol = match pair[1].as_str() { + "icmpv6" => "ipv6-icmp", + other => other, + }; + expression.push_str(&format!(" meta l4proto {protocol}")); + } + "--dport" => expression + .push_str(&format!(" {protocol} dport {}", pair[1].replace(':', "-"))), + "-j" => { + if pair[1] == "ACCEPT" { + output.push_str(&Self::dhcp_request_tracking_rule( + &expression, + family, + protocol, + )); + expression.push_str(" accept"); + } else { + expression.push_str(" drop"); + } + } + other => { + return Err(format!("Unsupported lowered firewall argument {other}")) + } + } + } + output.push_str(&expression); + output.push('\n'); + } + } + if Self::effective_default_policy(policy) == NetworkAction::Allow { + for family in ["ip", "ip6"] { + output.push_str(&Self::dhcp_request_tracking_rule( + &format!("ether type {family}"), + family, + "", + )); + } + output.push_str("accept\n"); + } else { + output.push_str("drop\n"); + } + Ok(output) + } + + fn dhcp_request_tracking_rule(expression: &str, family: &str, protocol: &str) -> String { + if !protocol.is_empty() && protocol != "udp" { + return String::new(); + } + + // The transaction IDs follow four payload bytes in RFC 2131 §2 and one in RFC 8415 §8. + let (version, client, server, offset, width) = if family == "ip" { + (4, 68, 67, 96, 32) + } else { + (6, 546, 547, 72, 24) + }; + format!( + "{expression} udp sport {client} udp dport {server} update @dhcp{version}_transactions {{ @th,{offset},{width} timeout 30s }}\n" + ) + } + + pub(crate) fn permits_ipv4_dhcp(policy: &ContainerPolicy) -> bool { + let Ok(entries) = Self::lower_egress(policy) else { + return false; + }; + for entry in entries { + let Some((IpFamily::V4, block)) = network_blocks::resolve(&entry.destination) else { + continue; + }; + let std::net::IpAddr::V4(address) = block.address(IpFamily::V4) else { + continue; + }; + let mask = u32::MAX + .checked_shl(32 - u32::from(block.prefix())) + .unwrap_or(0); + if u32::from(address) & mask != mask { + continue; + } + let matches = match entry.matching { + RuleMatch::AnyTraffic => true, + RuleMatch::Transport { + protocol: TransportProtocol::Udp, + ports, + } => ports.is_none_or(|range| range.start <= 67 && range.end >= 67), + _ => false, + }; + if matches { + return entry.action == RuleAction::Allow; + } + } + Self::effective_default_policy(policy) == NetworkAction::Allow + } + fn lower_egress(policy: &ContainerPolicy) -> Result, String> { match policy.network_egress.as_ref() { Some(egress) => Self::lower_directional_egress(egress), @@ -1183,6 +1289,7 @@ impl NetworkIptablesManager { // The ownership record keeps signal-time cleanup from flushing a later // run's chain under the same container name. #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + #[cfg(test)] pub(crate) fn force_cleanup( container_name: &str, hook_point: EgressHookPoint, @@ -1288,7 +1395,7 @@ pub(crate) mod test_firewall { } } - pub(super) fn intercept(command: &str, args: &[&str]) -> Option> { + pub(crate) fn intercept(command: &str, args: &[&str]) -> Option> { STATE.with(|slot| { let mut slot = slot.borrow_mut(); let state = slot.as_mut()?; @@ -1339,6 +1446,148 @@ mod tests { } } + #[test] + fn host_rules_preserve_deny_precedence_exclusions_ports_and_both_families() { + let policy = ContainerPolicy { + network_egress: Some(NetworkEgressPolicy { + default: NetworkAction::Deny, + deny: vec![NetworkRule { + to: vec![NetworkPeer { + cidr: "192.0.2.0/24".parse().unwrap(), + except: Vec::new(), + }], + ports: vec![NetworkPort { + protocol: NetworkProtocol::Tcp, + port: Some(80), + end_port: Some(90), + }], + }], + allow: vec![NetworkRule { + to: vec![ + NetworkPeer { + cidr: "192.0.2.0/24".parse().unwrap(), + except: vec!["192.0.2.128/25".parse().unwrap()], + }, + NetworkPeer { + cidr: "2001:db8::/64".parse().unwrap(), + except: Vec::new(), + }, + ], + ports: Vec::new(), + }], + }), + ..Default::default() + }; + let rules = + NetworkIptablesManager::host_egress_rules(&policy, &mut Logger::new(Mode::Buffer)) + .unwrap(); + assert_eq!( + rules + .lines() + .filter(|rule| !rule.contains(" update @dhcp")) + .collect::>() + .join("\n"), + "ip daddr 192.0.2.0/24 meta l4proto tcp tcp dport 80-90 drop\n\ + ip daddr 192.0.2.0/25 accept\n\ + ip6 daddr 2001:db8::/64 accept\n\ + drop" + ); + } + + #[test] + fn host_rules_use_the_configured_allow_default() { + let rules = NetworkIptablesManager::host_egress_rules( + &filtered_policy(), + &mut Logger::new(Mode::Buffer), + ) + .unwrap(); + assert!(rules.ends_with("accept\n")); + } + + #[test] + fn permitted_dhcp_requests_are_tracked_after_the_configured_deny_rules() { + let mut policy = filtered_policy(); + policy + .network_egress + .as_mut() + .unwrap() + .deny + .push(NetworkRule { + to: vec![NetworkPeer { + cidr: "255.255.255.255/32".parse().unwrap(), + except: Vec::new(), + }], + ports: vec![NetworkPort { + protocol: NetworkProtocol::Udp, + port: Some(67), + end_port: None, + }], + }); + let rules = + NetworkIptablesManager::host_egress_rules(&policy, &mut Logger::new(Mode::Buffer)) + .unwrap(); + let deny = rules.find("udp dport 67 drop").unwrap(); + let tracking = rules + .find("udp sport 68 udp dport 67 update @dhcp4_transactions") + .expect("an allowed request must register its DHCP response transaction"); + assert!(deny < tracking, "{rules}"); + assert!( + rules.contains("udp sport 546 udp dport 547 update @dhcp6_transactions"), + "{rules}" + ); + assert!(rules.ends_with("accept\n")); + } + + #[test] + fn explicit_dhcp_deny_suppresses_the_lease_wait() { + let mut policy = filtered_policy(); + assert!(NetworkIptablesManager::permits_ipv4_dhcp(&policy)); + policy + .network_egress + .as_mut() + .unwrap() + .deny + .push(NetworkRule { + to: vec![NetworkPeer { + cidr: "255.255.255.255/32".parse().unwrap(), + except: Vec::new(), + }], + ports: vec![NetworkPort { + protocol: NetworkProtocol::Udp, + port: Some(67), + end_port: None, + }], + }); + assert!(!NetworkIptablesManager::permits_ipv4_dhcp(&policy)); + } + + #[test] + fn a_dhcp_allow_waits_only_if_its_destination_is_not_excluded() { + let mut policy = ContainerPolicy { + network_egress: Some(NetworkEgressPolicy { + default: NetworkAction::Deny, + allow: vec![NetworkRule { + to: vec![NetworkPeer { + cidr: "0.0.0.0/0".parse().unwrap(), + except: Vec::new(), + }], + ports: vec![NetworkPort { + protocol: NetworkProtocol::Any, + port: Some(67), + end_port: Some(68), + }], + }], + deny: Vec::new(), + }), + ..Default::default() + }; + assert!(NetworkIptablesManager::permits_ipv4_dhcp(&policy)); + policy.network_egress.as_mut().unwrap().allow[0].to[0] + .except + .push("255.255.255.255/32".parse().unwrap()); + assert!(!NetworkIptablesManager::permits_ipv4_dhcp(&policy)); + } + #[test] fn allowing_ping_opens_the_ping_group_range_in_the_container_netns() { let fake = super::test_firewall::install(); diff --git a/src/mxc-sdk/src/backends/lxc/common/signal_cleanup.rs b/src/mxc-sdk/src/backends/lxc/common/signal_cleanup.rs index e72364e2b..cc751d2d1 100644 --- a/src/mxc-sdk/src/backends/lxc/common/signal_cleanup.rs +++ b/src/mxc-sdk/src/backends/lxc/common/signal_cleanup.rs @@ -4,8 +4,8 @@ //! Process-level cleanup for fatal signals. //! //! SIGHUP, SIGTERM, and SIGINT can interrupt the runner before normal cleanup -//! runs. The watchdog waits for them through `sigwait`, destroys the active -//! container, and exits with the signal-style status code. +//! runs. The watchdog applies the active lifecycle and exits with the +//! signal-style status code. //! //! Only the executor binary installs it: one slot cannot describe the several //! sandboxes a library host can hold, and a library may not exit its caller's @@ -20,21 +20,19 @@ use std::thread; #[cfg(target_os = "linux")] use nix::sys::signal::{SigSet, Signal}; +use crate::lxc_common::host_network::HostNetwork; #[cfg(target_os = "linux")] use crate::lxc_common::lxc_bindings::LxcContainer; -#[cfg(target_os = "linux")] -use crate::lxc_common::network_ingress::IngressManager; use crate::lxc_common::network_iptables::CreatedResources; -#[cfg(target_os = "linux")] -use crate::lxc_common::network_iptables::{EgressHookPoint, NetworkIptablesManager}; -#[cfg(target_os = "linux")] -use crate::mxc_common::logger::{Logger, Mode}; #[derive(Default)] struct ActiveSandbox { name: Option, created: CreatedResources, netns_pid: Option, + host_network: Option, + destroy_on_exit: bool, + cleanup_policy: bool, } static ACTIVE_CONTAINER: OnceLock> = OnceLock::new(); @@ -64,6 +62,10 @@ fn watchdog_installed() -> bool { /// Registers the container the watchdog destroys. pub fn set_active(name: &str) { + set_active_lifecycle(name, true, true); +} + +pub(crate) fn set_active_lifecycle(name: &str, destroy_on_exit: bool, cleanup_policy: bool) { if !watchdog_installed() { return; } @@ -71,6 +73,26 @@ pub fn set_active(name: &str) { slot.name = Some(name.to_owned()); slot.created = CreatedResources::default(); slot.netns_pid = None; + slot.host_network = None; + slot.destroy_on_exit = destroy_on_exit; + slot.cleanup_policy = cleanup_policy; +} + +pub(crate) fn set_active_host_network(network: HostNetwork) { + if !watchdog_installed() { + return; + } + let mut slot = lock_slot(); + if slot.name.is_some() { + slot.host_network = Some(network); + } +} + +pub(crate) fn clear_active_registration(name: &str) { + let mut slot = lock_slot(); + if slot.name.as_deref() == Some(name) { + *slot = ActiveSandbox::default(); + } } pub fn set_active_pid(pid: u32) { @@ -148,17 +170,25 @@ fn run_watchdog(mask: SigSet) -> ! { let Ok(sig) = mask.wait() else { continue }; let active = std::mem::take(&mut *lock_slot()); if let Some(name) = active.name { - let mut buf_logger = Logger::new(Mode::Buffer); - if let Some(pid) = active.netns_pid { - NetworkIptablesManager::force_cleanup( - &name, - EgressHookPoint::ContainerNetns(pid), - active.created, - &mut buf_logger, - ); - IngressManager::force_cleanup(&name, pid, &mut buf_logger); + let result = if active.destroy_on_exit { + LxcContainer::new(&name, None).destroy_owned() + } else { + Ok(()) + }; + match result { + Ok(()) => { + if active.destroy_on_exit || active.cleanup_policy { + if let Some(mut network) = active.host_network { + if let Err(error) = network.remove() { + eprintln!("LXC signal cleanup: {error}"); + } + } + } + } + Err(error) => { + eprintln!("LXC signal cleanup: {error}; host filtering retained"); + } } - let _ = LxcContainer::new(&name, None).destroy(); } std::process::exit(128 + sig as i32); } diff --git a/src/mxc-sdk/tests/lxc_common_chain_name_script_drift_spec.rs b/src/mxc-sdk/tests/lxc_common_chain_name_script_drift_spec.rs index 7b4396b73..e8a6910ed 100644 --- a/src/mxc-sdk/tests/lxc_common_chain_name_script_drift_spec.rs +++ b/src/mxc-sdk/tests/lxc_common_chain_name_script_drift_spec.rs @@ -2,6 +2,7 @@ // Licensed under the MIT License. use mxc_sdk::lxc_common::network_iptables::chain_name_for; +use std::collections::BTreeSet; use std::fs; use std::path::PathBuf; @@ -14,15 +15,6 @@ fn scripts_dir() -> PathBuf { .join("scripts") } -const CHAIN_ASSERTING_SCRIPTS: &[&str] = &[ - "run_lxc_network_cidr_boundary_test.sh", - "run_lxc_network_deny_precedence_test.sh", - "run_lxc_network_dualstack_test.sh", - "run_lxc_network_enforcement_test.sh", - "run_lxc_network_invalid_cidr_test.sh", - "run_lxc_network_ipv6_cidr_test.sh", -]; - fn network_scripts() -> Vec<(String, String)> { let dir = scripts_dir(); let entries = @@ -169,26 +161,88 @@ fn the_shape_the_scripts_check_accepts_the_names_the_code_produces() { } #[test] -fn every_chain_asserting_script_derives_the_name_it_asserts_on() { - let scripts = network_scripts(); - - for expected in CHAIN_ASSERTING_SCRIPTS { - let (_, body) = scripts - .iter() - .find(|(name, _)| name == expected) - .unwrap_or_else(|| { - panic!( - "{expected} is listed as a chain-asserting script but is not in {}. \ - If it was renamed or removed, update CHAIN_ASSERTING_SCRIPTS.", - scripts_dir().display() - ) - }); - - assert!( - body.contains("mxc_chains") || body.contains("derive_chain_name"), - "{expected} asserts on MXC chains but never derives a chain name. \ - Without a derivation its assertions cannot be checking a real \ - chain. Use the mxc_chains snapshot or derive_chain_name." - ); +fn cidr_fixtures_retain_the_destinations_the_kernel_checks_expect() { + let cases: [(&str, &[&str], &[&str]); 3] = [ + ( + "lxc_network_cidr_boundary", + &[ + "0.0.0.0/0", + "::/0", + "140.82.112.0/20", + "140.82.112.5/32", + "2606:50c0::/32", + ], + &["198.51.100.42/32", "2001:db8::5/128"], + ), + ( + "lxc_network_ipv6_cidr", + &[ + "140.82.112.0/20", + "2606:50c0::/32", + "2606:50c0:8000::153/128", + ], + &["10.0.0.0/8", "2001:db8::/32", "fe80::1/128"], + ), + ( + "lxc_network_dualstack_cidr", + &[ + "127.0.0.1/32", + "1.1.1.1/32", + "2606:4700:4700::1111/128", + "8.8.8.8/32", + "2001:4860:4860::8888/128", + ], + &["10.0.0.0/8", "2001:db8::/32"], + ), + ]; + for (fixture, allow, deny) in cases { + let path = scripts_dir() + .parent() + .expect("scripts directory has no parent") + .join("configs") + .join(format!("{fixture}.json")); + let config: serde_json::Value = + serde_json::from_slice(&fs::read(&path).expect("could not read CIDR fixture")) + .expect("CIDR fixture is not valid JSON"); + let egress = &config["network"]["egress"]; + assert_eq!(egress["default"], "deny", "{fixture} changed its default"); + for (action, expected) in [("allow", allow), ("deny", deny)] { + let actual: BTreeSet<_> = egress[action] + .as_array() + .expect("CIDR rules are not an array") + .iter() + .flat_map(|rule| { + rule["to"] + .as_array() + .expect("CIDR destinations are not an array") + }) + .map(|target| target["cidr"].as_str().expect("CIDR is not a string")) + .collect(); + assert_eq!( + actual, + expected.iter().copied().collect::>(), + "{fixture} changed the {action} destinations checked in the host kernel" + ); + } } } + +#[cfg(target_os = "linux")] +#[test] +fn ordinary_raw_packet_controls_survive_unavailable_io_uring() { + let output = std::process::Command::new("python3") + .env("PYTHONDONTWRITEBYTECODE", "1") + .arg( + scripts_dir() + .join("lib") + .join("test_io_uring_egress_probe.py"), + ) + .output() + .expect("could not run the offline raw/packet probe regression"); + assert!( + output.status.success(), + "ordinary raw/packet checks disappeared when io_uring was unavailable:\n{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} diff --git a/src/mxc-sdk/tests/streaming_lxc.rs b/src/mxc-sdk/tests/streaming_lxc.rs index daf2311e1..1a96b1aa6 100644 --- a/src/mxc-sdk/tests/streaming_lxc.rs +++ b/src/mxc-sdk/tests/streaming_lxc.rs @@ -216,8 +216,8 @@ fn container_is_running(name: &str) -> bool { output.status.success() && String::from_utf8_lossy(&output.stdout).contains("RUNNING") } -/// An LXC streaming request (`/tmp` read-write, no network) with the given -/// command, container name, and timeout (ms; `0` == run until exit). +/// An LXC streaming request with no network and the given command, container +/// name, and timeout (ms; `0` == run until exit). /// /// Cases that end in `wait()` pass [`LIVE_TIMEOUT_MS`] rather than `0`: with no /// script timeout the backend waits on `lxc-attach` forever, so a wedged attach @@ -257,7 +257,7 @@ fn lxc_request_with_network( ) -> mxc_sdk::v1::ContainerRequest { ContainerRequest { filesystem: Some(FilesystemPolicy { - readwrite_paths: vec!["/tmp".to_string()], + readwrite_paths: vec![], readonly_paths: vec![], denied_paths: vec![], clear_policy_on_exit: None, @@ -643,8 +643,6 @@ fn streaming_lxc_tears_down_a_networked_container() { let _guard = exclusive(); let name = container_name("network"); - // Outbound access puts the container on the bridge and installs egress - // chains, so this is the case whose teardown has firewall rules to remove. let network = NetworkPolicy { egress: Some(NetworkEgressPolicy { default: Some(NetworkAction::Allow), @@ -663,7 +661,5 @@ fn streaming_lxc_tears_down_a_networked_container() { assert!(out.contains("NETWORKED"), "got: {out:?}"); assert_eq!(proc.wait().expect("wait"), WaitResult::Exited(0)); - // The chains live in the container's own network namespace, so destroying - // the container is what proves they are gone. assert_container_released(&name); } diff --git a/src/mxc-sdk/tests/wxc_e2e_tests_e2e_lxc_network_capability.rs b/src/mxc-sdk/tests/wxc_e2e_tests_e2e_lxc_network_capability.rs index dbdc8f560..b65e09e9a 100644 --- a/src/mxc-sdk/tests/wxc_e2e_tests_e2e_lxc_network_capability.rs +++ b/src/mxc-sdk/tests/wxc_e2e_tests_e2e_lxc_network_capability.rs @@ -6,162 +6,112 @@ #[path = "support/wxc_e2e_tests.rs"] mod wxc_e2e_tests; use serde_json::json; +use std::fs; +use std::process::Command; use wxc_e2e_tests::{has_lxc_host, has_platform_exec, run_platform_config_value}; const CAP_NET_ADMIN: u64 = 1 << 12; const CAP_NET_RAW: u64 = 1 << 13; -const NETWORK_CAPABILITIES: u64 = CAP_NET_ADMIN | CAP_NET_RAW; -/// Whether the LXC capability prerequisites are present. -fn ready() -> bool { - has_platform_exec() && has_lxc_host() +struct ContainerCleanup(String); + +impl Drop for ContainerCleanup { + fn drop(&mut self) { + let config = json!({ + "version": "1.0.0", + "containerId": &self.0, + "containment": "lxc", + "process": { "commandLine": "true" }, + "lifecycle": { "destroyOnExit": true, "preservePolicy": false }, + "lxc": { "distribution": "alpine", "release": "3.23" }, + "network": { "egress": { "default": "allow" } } + }); + let result = run_platform_config_value("lxc capability cleanup", &config, &[], None); + if !std::thread::panicking() { + assert_eq!( + result.code, + Some(0), + "MXC could not destroy the capability test container\n{}", + result.combined_output() + ); + } + } } -/// Read one capability mask out of the container's `/proc/self/status`. -fn capability_mask(status: &str, field: &str) -> u64 { - status +fn assert_network_capabilities(status: &str) { + let effective = status .lines() - .find(|line| line.starts_with(field)) + .find(|line| line.starts_with("CapEff:")) .and_then(|line| line.split_whitespace().nth(1)) .and_then(|mask| u64::from_str_radix(mask, 16).ok()) - .unwrap_or_else(|| panic!("the container reported no readable {field} line\n{status}")) -} - -#[test] -fn workload_holds_neither_network_capability() { - if !ready() { - return; - } - - let config = json!({ - "version": "0.9.0-alpha", - "containerId": "lxc-network-capability", - "containment": "lxc", - "process": { "commandLine": "sh -c \"cat /proc/self/status\"" }, - "lifecycle": { "destroyOnExit": true }, - "lxc": { "distribution": "alpine", "release": "3.23" }, - - // An allow rule puts the workload behind the firewall. - "network": { - "egress": { - "default": "deny", - "allow": [{ "to": [{ "cidr": "198.51.100.2/32" }] }] - } - } - }); - - let result = run_platform_config_value("lxc network capability", &config, &[], None); - let status = result.combined_output(); - - let held = capability_mask(&status, "CapEff:") | capability_mask(&status, "CapPrm:"); + .unwrap_or_else(|| panic!("no readable effective capability mask\n{status}")); assert_eq!( - held & NETWORK_CAPABILITIES, - 0, - "the workload still holds CAP_NET_ADMIN or CAP_NET_RAW\n{status}" + effective & (CAP_NET_ADMIN | CAP_NET_RAW), + CAP_NET_ADMIN | CAP_NET_RAW, + "a filtered guest lost CAP_NET_ADMIN or CAP_NET_RAW\n{status}" ); } #[test] -fn workload_cannot_regain_a_network_capability() { - if !ready() { +fn filtered_guests_retain_network_administration_across_reuse() { + if !has_platform_exec() || !has_lxc_host() { return; } - let config = json!({ - "version": "0.9.0-alpha", - "containerId": "lxc-capability-regain", - "containment": "lxc", - "process": { "commandLine": "sh -c \"cat /proc/self/status\"" }, - "lifecycle": { "destroyOnExit": true }, - "lxc": { "distribution": "alpine", "release": "3.23" }, - "network": { - "egress": { - "default": "deny", - "allow": [{ "to": [{ "cidr": "198.51.100.2/32" }] }] + let container = ContainerCleanup(format!("lxc-network-capability-{}", std::process::id())); + for command in [ + "sh -c 'touch /mxc-capability-reuse; cat /proc/self/status'", + "sh -c 'test -f /mxc-capability-reuse && cat /proc/self/status'", + ] { + let config = json!({ + "version": "1.0.0", + "containerId": &container.0, + "containment": "lxc", + "process": { "commandLine": command }, + "lifecycle": { "destroyOnExit": false, "preservePolicy": true }, + "lxc": { "distribution": "alpine", "release": "3.23" }, + "network": { + "egress": { + "default": "allow", + "deny": [{ "to": [{ "cidr": "198.51.100.2/32" }] }] + } } - } - }); - - let result = run_platform_config_value("lxc capability regain", &config, &[], None); - let status = result.combined_output(); - - // An exec can only grant a capability that is in one of these sets. - // https://man7.org/linux/man-pages/man7/capabilities.7.html - let regainable = capability_mask(&status, "CapBnd:") - | capability_mask(&status, "CapInh:") - | capability_mask(&status, "CapAmb:"); - - assert_eq!( - regainable & NETWORK_CAPABILITIES, - 0, - "an exec could hand the workload CAP_NET_ADMIN or CAP_NET_RAW back\n{status}" - ); -} - -#[test] -fn workload_can_still_ping() { - if !ready() { - return; + }); + let result = run_platform_config_value("lxc container capabilities", &config, &[], None); + assert_eq!( + result.code, + Some(0), + "MXC could not create or reuse the filtered container\n{}", + result.combined_output() + ); + assert_network_capabilities(&result.combined_output()); + + let init = Command::new("lxc-info") + .args(["-n", &container.0, "-p", "-H"]) + .output() + .expect("query the container init PID"); + assert!( + init.status.success(), + "could not query the running init: {init:?}" + ); + let pid: u32 = String::from_utf8_lossy(&init.stdout) + .trim() + .parse() + .expect("lxc-info must report a running init PID"); + let status = fs::read_to_string(format!("/proc/{pid}/status")) + .expect("read init status from the host"); + assert_network_capabilities(&status); + + let attachment = Command::new("lxc-attach") + .args(["-n", &container.0, "--", "cat", "/proc/self/status"]) + .output() + .expect("read an ordinary attachment's status"); + assert!( + attachment.status.success(), + "an ordinary attachment could not read its status\n{}", + String::from_utf8_lossy(&attachment.stderr) + ); + assert_network_capabilities(&String::from_utf8_lossy(&attachment.stdout)); } - - let config = json!({ - "version": "0.9.0-alpha", - "containerId": "lxc-ping-allowed", - "containment": "lxc", - "process": { "commandLine": "ping -c 1 127.0.0.1" }, - "lifecycle": { "destroyOnExit": true }, - "lxc": { "distribution": "ubuntu", "release": "noble" }, - "network": { - "egress": { - "default": "deny", - "allow": [{ "to": [{ "cidr": "198.51.100.2/32" }] }] - } - } - }); - - let result = run_platform_config_value("lxc ping allowed", &config, &[], None); - - assert_eq!( - result.code, - Some(0), - "the workload could not ping after the capability drop\n{}", - result.combined_output() - ); -} - -#[test] -fn workload_cannot_open_a_raw_or_packet_socket() { - if !ready() { - return; - } - - let probe = format!( - "python3 -c '{}'", - include_str!("support/raw_socket_probe.py") - ); - - // Use Ubuntu for python3. - let config = json!({ - "version": "0.9.0-alpha", - "containerId": "lxc-raw-socket-refusal", - "containment": "lxc", - "process": { "commandLine": probe }, - "lifecycle": { "destroyOnExit": true }, - "lxc": { "distribution": "ubuntu", "release": "noble" }, - "network": { - "egress": { - "default": "deny", - "allow": [{ "to": [{ "cidr": "198.51.100.2/32" }] }] - } - } - }); - - let result = run_platform_config_value("lxc raw socket refusal", &config, &[], None); - let output = result.combined_output(); - - assert!( - output.contains("raw EPERM") && output.contains("packet EPERM"), - "the workload opened a raw or packet socket the capability drop must refuse\n{output}" - ); } diff --git a/tests/scripts/lib/io_uring_egress_probe.py b/tests/scripts/lib/io_uring_egress_probe.py index cf12f9dc0..38d1d4236 100644 --- a/tests/scripts/lib/io_uring_egress_probe.py +++ b/tests/scripts/lib/io_uring_egress_probe.py @@ -27,6 +27,9 @@ def io_uring_socket(family, kind, protocol): if ring < 0: error = ctypes.get_errno() print(next((line.strip() for line in open("/proc/self/status") if line.startswith("Seccomp:")), "Seccomp: unknown"), flush=True) + if error == 1: + print("MXC_IO_URING_SETUP_FAILED errno=1") + raise SystemExit(0) raise SystemExit("MXC_IO_URING_SETUP_FAILED errno=%d" % error) _, sq_tail, _, _, _, _, sq_array = struct.unpack_from("7I", params, 40) sq = mmap.mmap(ring, sq_array + 4, offset=0) @@ -45,26 +48,23 @@ def io_uring_socket(family, kind, protocol): except OSError: return None -# A socket ping proves the peer answers and fills the neighbor entry the packet frame needs. -ping = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_ICMP) -ping.settimeout(3) -ping.sendto(echo(1), (PEER, 0)) -try: - ping.recv(64) -except OSError: - raise SystemExit("MXC_PEER_UNREACHABLE") +for create_socket in (socket.socket, io_uring_socket): + replies = [] -escaped = False - -raw = io_uring_socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_ICMP) -if raw: + raw = create_socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_ICMP) + if raw is None: + raise SystemExit("MXC_RAW_SOCKET_FAILED") raw.sendto(echo(2), (PEER, 0)) - escaped |= replied(raw, 0) + replies.append(replied(raw, 0)) -packet = io_uring_socket(socket.AF_PACKET, socket.SOCK_RAW, socket.htons(0x0800)) -if packet: + packet = create_socket(socket.AF_PACKET, socket.SOCK_RAW, socket.htons(0x0800)) + if packet is None: + raise SystemExit("MXC_PACKET_SOCKET_FAILED") route = next(line.split() for line in open("/proc/net/route") if line.split()[1] == "00000000") gateway = socket.inet_ntoa(struct.pack(""$run_log" || exit 1 + trap ' + status=$? + if [ -n "$run_pid" ]; then + kill -TERM "$run_pid" 2>/dev/null || true + wait "$run_pid" 2>/dev/null || true + fi + cat "$run_log" + rm -f "$run_config" "$run_log" + exit "$status" + ' EXIT + + # Hold the original probe until host tools have assigned its guest address. + container="$(python3 - "$config" "$run_config" <<'PY' +import json, shlex, sys +config = json.load(open(sys.argv[1])) +command = config["process"]["commandLine"] +gate = ( + "set -e; rm -f /mxc-static-network-ready; " + "echo MXC_STATIC_NETWORK_READY; " + "while [ ! -f /mxc-static-network-ready ]; do sleep 0.1; done; exec " +) +config["process"]["commandLine"] = "sh -c " + shlex.quote(gate + command) +with open(sys.argv[2], "w") as stream: + json.dump(config, stream) +print(config["containerId"]) +PY +)" || exit 1 + + "$LXC_EXEC" --debug "$run_config" >"$run_log" 2>&1 & + run_pid=$! + for ((attempt = 0; attempt < 600; attempt++)); do + grep -Fq MXC_STATIC_NETWORK_READY "$run_log" && break + kill -0 "$run_pid" 2>/dev/null || { echo "FAIL: the probe exited before address setup."; exit 1; } + sleep 0.5 + done + grep -Fq MXC_STATIC_NETWORK_READY "$run_log" \ + || { echo "FAIL: the probe never reached address setup."; exit 1; } + init_pid="$(lxc-info -n "$container" -p -H)" || exit 1 + + # Reserve an address in the bridge's subnet without requiring DHCP. + read -r host_ip guest_cidr < <( + ip -j -4 address show dev lxcbr0 | python3 -c ' +import ipaddress, json, sys +address = json.load(sys.stdin)[0]["addr_info"][0] +network = ipaddress.ip_network(f"{address['\''local'\'']}/{address['\''prefixlen'\'']}", strict=False) +print(address["local"], f"{network.broadcast_address - 2}/{network.prefixlen}") +' + ) || exit 1 + + # Enter init's network namespace using host-installed addressing tools. + nsenter -t "$init_pid" -n ip -4 address flush dev eth0 || exit 1 + nsenter -t "$init_pid" -n ip address replace "$guest_cidr" dev eth0 || exit 1 + nsenter -t "$init_pid" -n ip link set eth0 up || exit 1 + nsenter -t "$init_pid" -n ip route replace default via "$host_ip" || exit 1 + lxc-attach -n "$container" -- touch /mxc-static-network-ready || exit 1 + + if wait "$run_pid"; then + status=0 + else + status=$? + fi + run_pid="" + exit "$status" +) diff --git a/tests/scripts/lib/test_io_uring_egress_probe.py b/tests/scripts/lib/test_io_uring_egress_probe.py new file mode 100644 index 000000000..3af372745 --- /dev/null +++ b/tests/scripts/lib/test_io_uring_egress_probe.py @@ -0,0 +1,56 @@ +import contextlib +import ctypes +import io +from pathlib import Path +import runpy +import socket +import unittest +from unittest import mock + + +class RawPacketControls(unittest.TestCase): + def test_ordinary_traffic_is_checked_when_io_uring_is_unavailable(self): + probe = Path(__file__).with_name("io_uring_egress_probe.py") + files = { + "/proc/net/route": "eth0 00000000 0103000A\n", + "/proc/net/arp": "10.0.3.1 0x1 0x2 02:00:00:00:00:01 * eth0\n", + "/sys/class/net/eth0/address": "02:00:00:00:00:02\n", + "/proc/self/status": "Seccomp:\t2\n", + } + reply = bytes.fromhex("4500001c0000000040010000cb0071020a000302") + bytes(8) + for allowed in (True, False): + with self.subTest(allowed=allowed): + raw, packet, source = mock.Mock(), mock.Mock(), mock.Mock() + raw.recv.return_value = reply + packet.recv.return_value = bytes(14) + reply + if not allowed: + raw.recv.side_effect = TimeoutError + packet.recv.side_effect = TimeoutError + source.getsockname.return_value = ("10.0.3.2", 9) + libc = mock.Mock() + libc.syscall.return_value = -1 + output = io.StringIO() + code = 0 + with ( + mock.patch.object(ctypes, "CDLL", return_value=libc), + mock.patch.object(ctypes, "get_errno", return_value=1), + mock.patch.object(socket, "create_connection"), + mock.patch.object(socket, "socket", side_effect=[raw, packet, source]), + mock.patch("builtins.open", side_effect=lambda path: io.StringIO(files[path])), + mock.patch("sys.argv", [str(probe), "203.0.113.2"]), + contextlib.redirect_stdout(output), + ): + try: + runpy.run_path(str(probe), run_name="__main__") + except SystemExit as error: + code = error.code + self.assertEqual(code, 0, output.getvalue()) + self.assertIn( + "MXC_NET_ALLOWED" if allowed else "MXC_NET_BLOCKED", + output.getvalue(), + ) + self.assertIn("MXC_IO_URING_SETUP_FAILED errno=1", output.getvalue()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/scripts/run_lxc_all_tests.sh b/tests/scripts/run_lxc_all_tests.sh index 5922b9c22..f6073da46 100644 --- a/tests/scripts/run_lxc_all_tests.sh +++ b/tests/scripts/run_lxc_all_tests.sh @@ -82,6 +82,7 @@ echo " Run it manually with: sudo bash $SCRIPT_DIR/run_lxc_rootless_no_netw echo "" run_test "Basic LXC" "$SCRIPT_DIR/run_lxc_basic_test.sh" +run_test "LXC Unprivileged Admission" "$SCRIPT_DIR/run_lxc_unprivileged_test.sh" run_test "LXC Filesystem" "$SCRIPT_DIR/run_lxc_filesystem_test.sh" run_test "LXC Object Validation" "$SCRIPT_DIR/run_lxc_object_test.sh" run_test "LXC Most-Specific Path" "$SCRIPT_DIR/run_lxc_most_specific_test.sh" diff --git a/tests/scripts/run_lxc_filesystem_test.sh b/tests/scripts/run_lxc_filesystem_test.sh index 1e85142b7..b4c3c17d7 100644 --- a/tests/scripts/run_lxc_filesystem_test.sh +++ b/tests/scripts/run_lxc_filesystem_test.sh @@ -18,8 +18,14 @@ fi # Create test directories READONLY_DIR=$(mktemp -d) READWRITE_DIR=$(mktemp -d) +trap 'rm -rf "$READONLY_DIR" "$READWRITE_DIR"' EXIT echo "test content" > "$READONLY_DIR/test.txt" +# Give mapped container users access to the host fixtures. +chmod 755 "$READONLY_DIR" +chmod 644 "$READONLY_DIR/test.txt" +chmod 777 "$READWRITE_DIR" + echo "Running LXC filesystem test..." echo "Readonly dir: $READONLY_DIR" echo "Readwrite dir: $READWRITE_DIR" @@ -31,6 +37,4 @@ CONFIG=$(cat "$REPO_DIR/tests/configs/lxc_filesystem_test.json" | \ echo "$CONFIG" | "$LXC_EXEC" --config /dev/stdin -# Cleanup -rm -rf "$READONLY_DIR" "$READWRITE_DIR" echo "LXC filesystem test complete." diff --git a/tests/scripts/run_lxc_inbound_deny_test.sh b/tests/scripts/run_lxc_inbound_deny_test.sh index f53d9ff88..8accc92ab 100644 --- a/tests/scripts/run_lxc_inbound_deny_test.sh +++ b/tests/scripts/run_lxc_inbound_deny_test.sh @@ -1,236 +1,42 @@ #!/bin/bash -# LXC inbound (ingress) default-deny test -# -# The inbound chain is the one piece of the LXC firewall that does NOT live on -# the host: it is programmed inside the container's own network namespace via -# `nsenter -t -n`. That is exactly why it needs a live test. Every -# unit test in `network_ingress.rs` asserts planned argv against an injected -# runner, so none of them can catch the chain being programmed into the wrong -# namespace -- and a host-execution bug of precisely that shape has already -# happened once in this file's history. -# -# The directional ingress default-deny and unsupported allow postures are -# exercised; an unsupported allow must fail closed before running a workload. -# -# Assertions are on rule programming and namespace containment, not on whether -# a remote peer can reach the container: reachability depends on the host's -# bridge and uplink, but where the chain is installed does not. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" -LXC_EXEC="$REPO_DIR/src/target/release/lxc-exec" - -if [ ! -f "$LXC_EXEC" ]; then +LXC_EXEC="${LXC_EXEC:-$REPO_DIR/src/target/release/lxc-exec}" +if [ "$LXC_EXEC" = "$REPO_DIR/src/target/release/lxc-exec" ] && [ ! -f "$LXC_EXEC" ]; then LXC_EXEC="$REPO_DIR/src/target/debug/lxc-exec" fi -# An honest skip for a missing prerequisite: exit 77 so run_lxc_all_tests.sh -# records SKIPPED rather than PASS. A suite that could not run must not look green. -SKIP_EXIT=77 -skip() { - echo "SKIP: $1" - exit "$SKIP_EXIT" -} - -[ "$(id -u)" -eq 0 ] || skip "requires root for iptables/ip6tables and LXC." -command -v iptables >/dev/null 2>&1 || skip "iptables is not installed." -command -v ip6tables >/dev/null 2>&1 || skip "ip6tables is not installed." -# Presence is not usability, and the difference is load-bearing twice over. On -# a host without kernel IPv6 the binary is normally installed and every -# invocation still fails: under `set -o pipefail` the chain snapshots below -# would then abort this script before a single assertion ran, reporting FAIL on -# a host configuration the implementation explicitly supports. Probing here -# instead turns that into an honest SKIP. -# -# The whole test is skipped rather than just its IPv6 half. The container -# probe runs the same binary through `nsenter`, so it fails too, and the -# inbound install's IPv6 classification -- a first-class part of what this test -# covers -- then depends on whether the *container* namespace has an -# `if_inet6`, which the host cannot predict. Asserting the IPv4 half alone -# would be non-deterministic, and a flaky test is worse than a declared gap. -ip6tables -S >/dev/null 2>&1 || skip "ip6tables is installed but unusable here (no kernel IPv6 support), so the inbound IPv6 classification cannot be exercised." -command -v nsenter >/dev/null 2>&1 || skip "nsenter is not installed." -command -v lxc-create >/dev/null 2>&1 || skip "LXC (lxc-create) is not installed." -[ -f "$LXC_EXEC" ] || skip "lxc-exec binary not built; run build.sh first." - -DENY_CONFIG="$REPO_DIR/tests/configs/lxc_inbound_default_deny.json" -DIRECTIONAL_CONFIG="$REPO_DIR/tests/configs/lxc_inbound_permissive_ingress.json" -DENY_INGRESS_CONFIG="$REPO_DIR/tests/configs/lxc_inbound_deny_ingress.json" - -[ -f "$DENY_CONFIG" ] || skip "missing config $DENY_CONFIG." -[ -f "$DIRECTIONAL_CONFIG" ] || skip "missing config $DIRECTIONAL_CONFIG." - fail() { echo "FAIL: $1" exit 1 } -# List the inbound chains a tool currently holds *on the host*. The ingress -# chain uses the MXCI- prefix, distinct from the MXC- egress chain, so this -# matches only inbound chains and cannot be confused by egress state. Matching -# the prefix rather than a literal keeps this correct across naming changes, -# since the name is a digest of the container name. -mxci_chains() { - "$1" -S 2>/dev/null | sed -n 's/^-N \(MXCI-.*\)$/\1/p' | sort -} - -mxc_egress_chains() { - "$1" -S 2>/dev/null | sed -n 's/^-N \(MXC-.*\)$/\1/p' | sort -} - -# The core containment assertion. An inbound chain must never appear in the -# host's tables: if one does, the rules were programmed outside the container -# namespace and are filtering the host instead of the sandbox. Compared against -# a snapshot so leftovers from an earlier failed run are not blamed on this one. -assert_no_new_chains() { - local tool="$1" before="$2" lister="$3" what="$4" after="" leaked="" chain - # Captured before iterating rather than piped in from a process - # substitution, whose exit status is not the loop's. A failed enumeration - # would otherwise read as zero chains and pass while verifying nothing. - if ! after="$("$lister" "$tool")"; then - fail "could not enumerate $tool chains, so $what was not verified." - fi - while IFS= read -r chain; do - [ -n "$chain" ] || continue - grep -Fxq "$chain" <<<"$before" || leaked="$leaked $chain" - done <<<"$after" - if [ -n "$leaked" ]; then - fail "$tool: unexpected $what on the host:$leaked" - fi -} - -MXCI_BEFORE_V4="$(mxci_chains iptables)" -MXCI_BEFORE_V6="$(mxci_chains ip6tables)" -MXC_BEFORE_V4="$(mxc_egress_chains iptables)" -MXC_BEFORE_V6="$(mxc_egress_chains ip6tables)" - -# --------------------------------------------------------------------------- -# Case 1: explicit directional default-deny -# --------------------------------------------------------------------------- - -echo "Running LXC inbound default-deny test..." - -# The container command itself is trivial; a non-zero exit from an unrelated -# cause must not mask the firewall assertions below. -OUTPUT=$("$LXC_EXEC" --debug "$DENY_CONFIG" 2>&1 || true) -echo "$OUTPUT" - -if ! grep -Fq "Container init PID:" <<<"$OUTPUT"; then - fail "no container init PID was logged, so the inbound chain could not have been namespaced." -fi - -INBOUND_CHAIN="$(sed -n 's/^.*Creating inbound iptables chain: \([^ ]*\).*$/\1/p' <<<"$OUTPUT" | head -n 1)" -if [ -z "$INBOUND_CHAIN" ]; then - fail "the inbound chain was never created; default-deny was not applied." -fi - -# Shape and length are asserted rather than hard-coded, so a malformed or -# over-long name fails here instead of surfacing as an opaque iptables error. -if ! grep -Eq '^MXCI-([A-Za-z0-9_-]{1,6}-)?[a-z2-7]{16}$' <<<"$INBOUND_CHAIN"; then - fail "inbound chain name '$INBOUND_CHAIN' does not match the documented MXCI-- shape." -fi -if [ "${#INBOUND_CHAIN}" -gt 28 ]; then - fail "inbound chain name '$INBOUND_CHAIN' exceeds the 28-character iptables ceiling." -fi - -# The inbound chain must be distinct from the egress chain for the same -# container -- sharing a name would let either teardown destroy the other. -EGRESS_CHAIN="$(sed -n 's/^.*Programmed [a-z0-9]* rule: -A \([^ ]*\) .*$/\1/p' <<<"$OUTPUT" | head -n 1)" -if [ -n "$EGRESS_CHAIN" ] && [ "$EGRESS_CHAIN" = "$INBOUND_CHAIN" ]; then - fail "inbound and egress share chain name '$INBOUND_CHAIN'." -fi - -if ! grep -Fq "Inbound (network.ingress) policy: DROP new inbound connections (default-deny)" <<<"$OUTPUT"; then - fail "the inbound policy was not the default-deny decision." -fi - -# Fail-closed paths must not have been taken silently. -if grep -Fq "Inbound network policy error" <<<"$OUTPUT"; then - fail "inbound firewall setup reported an error on the default-deny path." -fi -if grep -Fq "Failed to apply inbound network firewall rules" <<<"$OUTPUT"; then - fail "inbound firewall rules were not applied." -fi - -# The whole point of this test: the chain lives in the container namespace, so -# it must be absent from the host's tables both during and after the run. -assert_no_new_chains iptables "$MXCI_BEFORE_V4" mxci_chains "inbound chain(s)" -assert_no_new_chains ip6tables "$MXCI_BEFORE_V6" mxci_chains "inbound chain(s)" - -# Egress cleanup must still hold; an inbound regression must not be masked by, -# or mask, a leaked egress chain. -assert_no_new_chains iptables "$MXC_BEFORE_V4" mxc_egress_chains "egress chain(s) left behind" -assert_no_new_chains ip6tables "$MXC_BEFORE_V6" mxc_egress_chains "egress chain(s) left behind" - -echo "PASS: inbound default-deny installed in the container namespace." - -# --------------------------------------------------------------------------- -# Case 2: directional permissive ingress must fail closed -# --------------------------------------------------------------------------- - -echo "Running LXC directional permissive-ingress refusal test..." - -DIRECTIONAL_OUTPUT=$("$LXC_EXEC" --debug "$DIRECTIONAL_CONFIG" 2>&1 || true) -echo "$DIRECTIONAL_OUTPUT" - -# The refusal has to name the field the operator wrote. A directional-contract author who never -# typed `allowLocalNetwork` cannot act on a message about it. -if ! grep -Fq "network.ingress.default" <<<"$DIRECTIONAL_OUTPUT"; then - fail "the refusal did not name network.ingress.default, so a directional-contract author is told to change a field their config does not contain." +BEFORE_V4="$(iptables-save | sed '/^#/d')" +BEFORE_V6="$(ip6tables-save | sed '/^#/d')" +BEFORE_NFT="$(nft -s list ruleset)" +for case in lxc_inbound_default_deny lxc_inbound_deny_ingress; do + OUTPUT=$("$LXC_EXEC" "$REPO_DIR/tests/configs/$case.json" 2>&1) \ + || fail "the supported default-deny workload failed: $OUTPUT" + echo "$OUTPUT" + case "$case" in + lxc_inbound_default_deny) MARKER="inbound-default-deny-ok" ;; + lxc_inbound_deny_ingress) MARKER="MXC_WORKLOAD_RAN" ;; + esac + grep -Fq "$MARKER" <<<"$OUTPUT" || fail "the supported default-deny workload never ran." +done + +if OUTPUT=$("$LXC_EXEC" "$REPO_DIR/tests/configs/lxc_inbound_permissive_ingress.json" 2>&1); then + fail "an unsupported permissive-ingress request was accepted: $OUTPUT" fi - -if ! grep -Fq "not yet implemented" <<<"$DIRECTIONAL_OUTPUT"; then - fail "network.ingress.default 'allow' did not report a not-yet-implemented refusal." -fi - -if grep -Fq "this-should-never-run" <<<"$DIRECTIONAL_OUTPUT"; then - fail "the workload executed despite an unenforceable inbound policy." -fi - -assert_no_new_chains iptables "$MXCI_BEFORE_V4" mxci_chains "inbound chain(s)" -assert_no_new_chains ip6tables "$MXCI_BEFORE_V6" mxci_chains "inbound chain(s)" -assert_no_new_chains iptables "$MXC_BEFORE_V4" mxc_egress_chains "egress chain(s) left behind" -assert_no_new_chains ip6tables "$MXC_BEFORE_V6" mxc_egress_chains "egress chain(s) left behind" - -echo "PASS: directional permissive ingress refused and rolled back." - -# --------------------------------------------------------------------------- -# Case 3: a directional deny ingress installs the chain and reports its own field -# --------------------------------------------------------------------------- - -echo "Running LXC directional deny-ingress enforcement test..." - -MXCI_BEFORE_V4="$(mxci_chains iptables)" -MXCI_BEFORE_V6="$(mxci_chains ip6tables)" -MXC_BEFORE_V4="$(mxc_egress_chains iptables)" -MXC_BEFORE_V6="$(mxc_egress_chains ip6tables)" - -DENY_INGRESS_OUTPUT=$("$LXC_EXEC" --debug "$DENY_INGRESS_CONFIG" 2>&1 || true) -echo "$DENY_INGRESS_OUTPUT" - -if ! grep -Fq "MXC_WORKLOAD_RAN" <<<"$DENY_INGRESS_OUTPUT"; then - fail "a directional deny ingress did not run the workload; the enforceable posture was refused." -fi - -# A directional-contract author has no allowLocalNetwork to change, so reporting one sends them -# looking for a field their schema does not define. -if ! grep -Fq "Inbound (network.ingress) policy: DROP new inbound connections (default-deny)" <<<"$DENY_INGRESS_OUTPUT"; then - fail "the inbound decision was not reported against network.ingress." -fi - -if grep -Fq "Inbound (allowLocalNetwork)" <<<"$DENY_INGRESS_OUTPUT"; then - fail "a directional run reported its inbound decision against allowLocalNetwork, a field its schema does not define." -fi - -if grep -Fq "Network policy requests no firewall" <<<"$DENY_INGRESS_OUTPUT"; then - fail "a stated directional ingress posture was treated as requesting no firewall, so nothing was enforced." -fi - -assert_no_new_chains iptables "$MXCI_BEFORE_V4" mxci_chains "inbound chain(s)" -assert_no_new_chains ip6tables "$MXCI_BEFORE_V6" mxci_chains "inbound chain(s)" -assert_no_new_chains iptables "$MXC_BEFORE_V4" mxc_egress_chains "egress chain(s) left behind" -assert_no_new_chains ip6tables "$MXC_BEFORE_V6" mxc_egress_chains "egress chain(s) left behind" - -echo "PASS: directional deny ingress enforced and reported against its own field." -echo "LXC inbound default-deny test passed." +echo "$OUTPUT" +grep -Fq network.ingress.default <<<"$OUTPUT" || fail "the refusal did not identify the requested ingress field." +grep -Fq "not yet implemented" <<<"$OUTPUT" || fail "the unsupported ingress request was not explicitly refused." +if grep -Fq this-should-never-run <<<"$OUTPUT"; then + fail "the workload ran despite the unsupported ingress request." +fi +[ "$(iptables-save | sed '/^#/d')" = "$BEFORE_V4" ] || fail "ingress validation leaked IPv4 host rules." +[ "$(ip6tables-save | sed '/^#/d')" = "$BEFORE_V6" ] || fail "ingress validation leaked IPv6 host rules." +[ "$(nft -s list ruleset)" = "$BEFORE_NFT" ] || fail "ingress validation leaked nftables host rules." +echo "PASS: supported ingress requests ran, and unsupported ingress was refused without host rule leaks." diff --git a/tests/scripts/run_lxc_inbound_traffic_test.sh b/tests/scripts/run_lxc_inbound_traffic_test.sh index b45c366f8..bf6ee2272 100644 --- a/tests/scripts/run_lxc_inbound_traffic_test.sh +++ b/tests/scripts/run_lxc_inbound_traffic_test.sh @@ -1,228 +1,134 @@ #!/bin/bash -# LXC inbound default-deny: traffic-level test -# -# Requires that an unsolicited inbound connection to the container fails and -# that the chain's DROP counters move. The other inbound test asserts that the -# chain is programmed correctly; nothing puts a packet in front of it, so a -# chain that drops nothing passes today. -# -# Covers the container once the policy is installed. The unfiltered interval -# before installation is a separate open defect, and the exposure there arrives -# over lxcbr0 in a sub-second window this shape cannot sample deterministically. - set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" -LXC_EXEC="$REPO_DIR/src/target/release/lxc-exec" - -if [ ! -f "$LXC_EXEC" ]; then +LXC_EXEC="${LXC_EXEC:-$REPO_DIR/src/target/release/lxc-exec}" +if [ "$LXC_EXEC" = "$REPO_DIR/src/target/release/lxc-exec" ] && [ ! -f "$LXC_EXEC" ]; then LXC_EXEC="$REPO_DIR/src/target/debug/lxc-exec" fi -# Exit 77 marks a missing prerequisite, which the suite runner counts as -# SKIPPED rather than PASS. -SKIP_EXIT=77 -skip() { - echo "SKIP: $1" - exit "$SKIP_EXIT" -} - fail() { echo "FAIL: $1" exit 1 } -[ "$(id -u)" -eq 0 ] || skip "requires root for iptables/ip6tables, LXC, and moving an interface between namespaces." -command -v iptables >/dev/null 2>&1 || skip "iptables is not installed." -command -v ip6tables >/dev/null 2>&1 || skip "ip6tables is not installed." -command -v nsenter >/dev/null 2>&1 || skip "nsenter is not installed." -command -v ip >/dev/null 2>&1 || skip "iproute2 (ip) is not installed." -command -v lxc-create >/dev/null 2>&1 || skip "LXC (lxc-create) is not installed." -[ -f "$LXC_EXEC" ] || skip "lxc-exec binary not built; run build.sh first." - -# A host without kernel IPv6 has the binary installed and fails every -# invocation, so probe it rather than trusting `command -v`. The whole test -# skips rather than its IPv6 half alone, because the in-container probe runs -# the same binary and fails the same way. -ip6tables -S >/dev/null 2>&1 || skip "ip6tables is installed but not usable on this host (no kernel IPv6?)." - -CONFIG="$REPO_DIR/tests/configs/lxc_inbound_traffic_deny.json" -[ -f "$CONFIG" ] || skip "missing config $CONFIG." - -# Must match `containerId` in the config: the runner uses that value verbatim as -# the LXC container name. -CONTAINER="CLI-LXC-Inbound-Traffic" - -VETH_HOST="mxcprobe0" -VETH_PEER="mxcprobe1" -PROBE_HOST_V4="10.77.78.1" -PROBE_NS_V4="10.77.78.2" -PROBE_HOST_V6="fd00:77:78::1" -PROBE_NS_V6="fd00:77:78::2" -PROBE_PORT=7777 - -RUN_LOG="$(mktemp)" +# shellcheck source=lib/lxc_peer_listener.sh +. "$SCRIPT_DIR/lib/lxc_peer_listener.sh" + +CONTAINER="mxc-inbound-traffic-$$" +CONFIG="$REPO_DIR/tests/configs/.lxc-inbound-$$.json" +RUN_LOG="$REPO_DIR/tests/configs/.lxc-inbound-$$.log" +LISTENER_V4_LOG="$REPO_DIR/tests/configs/.lxc-inbound-v4-$$.log" +LISTENER_V6_LOG="$REPO_DIR/tests/configs/.lxc-inbound-v6-$$.log" +HOST_V6="fd00:77:78::1" +GUEST_V6="fd00:77:78::2" +PORT=$((30000 + $$ % 10000)) RUN_PID="" -VETH_CREATED=0 - +LISTENER_V4_PID="" +LISTENER_V6_PID="" +HOST_V6_ADDED=false cleanup() { - if [ -n "$RUN_PID" ] && kill -0 "$RUN_PID" 2>/dev/null; then - - # SIGTERM rather than SIGKILL: the runner's signal handler is what tears - # down the chains and the container, and SIGKILL would strand both. + local status=$? + if [ -n "$RUN_PID" ]; then kill -TERM "$RUN_PID" 2>/dev/null || true wait "$RUN_PID" 2>/dev/null || true fi - if [ "$VETH_CREATED" = 1 ]; then - ip link del "$VETH_HOST" >/dev/null 2>&1 || true + for pid in "$LISTENER_V4_PID" "$LISTENER_V6_PID"; do + if [ -n "$pid" ]; then + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + fi + done + if lxc-info -n "$CONTAINER" >/dev/null 2>&1; then + python3 - "$CONFIG" <<'PY' +import json, sys +path = sys.argv[1] +config = json.load(open(path)) +config["process"]["commandLine"] = "true" +config["lifecycle"] = {"destroyOnExit": True, "preservePolicy": False} +with open(path, "w") as stream: + json.dump(config, stream) +PY + "$LXC_EXEC" "$CONFIG" >/dev/null 2>&1 || { echo "FAIL: MXC could not clean up the ingress fixture."; status=1; } + fi + if [ "$HOST_V6_ADDED" = true ]; then + ip -6 address del "$HOST_V6/64" dev lxcbr0 fi - lxc-destroy -n "$CONTAINER" -f >/dev/null 2>&1 || true - rm -f "$RUN_LOG" + rm -f "$CONFIG" "$RUN_LOG" "$LISTENER_V4_LOG" "$LISTENER_V6_LOG" + exit "$status" } trap cleanup EXIT -# Leftovers from an interrupted earlier run would make the setup below fail for -# a reason that has nothing to do with the code under test. -ip link del "$VETH_HOST" >/dev/null 2>&1 || true -lxc-destroy -n "$CONTAINER" -f >/dev/null 2>&1 || true - -# List the inbound chains a tool holds *on the host*. The MXCI- prefix is the -# ingress chain's, distinct from the MXC- egress chain, so egress state cannot -# be mistaken for a leak here. -mxci_chains() { - "$1" -S 2>/dev/null | sed -n 's/^-N \(MXCI-.*\)$/\1/p' | sort -} - -MXCI_BEFORE_V4="$(mxci_chains iptables)" -MXCI_BEFORE_V6="$(mxci_chains ip6tables)" - -echo "Running LXC inbound traffic test..." - -"$LXC_EXEC" --debug "$CONFIG" >"$RUN_LOG" 2>&1 & +# Address the existing managed interface rather than add an unfiltered one. +read -r GUEST_V4 PREFIX < <( + ip -j -4 address show dev lxcbr0 | python3 -c ' +import ipaddress, json, sys +address = json.load(sys.stdin)[0]["addr_info"][0] +network = ipaddress.ip_network(f"{address['\''local'\'']}/{address['\''prefixlen'\'']}", strict=False) +print(network.broadcast_address - 2, network.prefixlen) +') +[ -n "$GUEST_V4" ] && [ -n "$PREFIX" ] || fail "could not address the test guest." +ip -6 address add "$HOST_V6/64" dev lxcbr0 nodad +HOST_V6_ADDED=true +python3 - "$CONTAINER" >"$CONFIG" <<'PY' +import json, sys +print(json.dumps({ + "version": "1.0.0", + "containerId": sys.argv[1], + "containment": "lxc", + "process": {"commandLine": "sh -c 'echo MXC_INBOUND_READY; while [ ! -f /mxc-inbound-finish ]; do sleep 1; done'"}, + "lifecycle": {"destroyOnExit": False, "preservePolicy": False}, + "lxc": {"distribution": "alpine", "release": "3.23"}, + "network": {"egress": {"default": "allow"}, "ingress": {"default": "deny", "hostLoopback": "deny"}} +})) +PY +: >"$RUN_LOG" +"$LXC_EXEC" "$CONFIG" >"$RUN_LOG" 2>&1 & RUN_PID=$! - -# The workload is attached only after the chains are hooked, so its marker is -# what says the container is both live and filtered. -# -# The bound is generous because a cold host downloads the Alpine image first. -WAITED=0 -until grep -Fq "MXC_INBOUND_PROBE_READY" "$RUN_LOG" 2>/dev/null; do - if ! kill -0 "$RUN_PID" 2>/dev/null; then - echo "--- lxc-exec output ---" - cat "$RUN_LOG" - fail "lxc-exec exited before the workload ran; the container never reached the filtered state." - fi - if [ "$WAITED" -ge 600 ]; then - echo "--- lxc-exec output ---" - cat "$RUN_LOG" - fail "the workload marker never appeared within 300s." - fi +for ((attempt = 0; attempt < 600; attempt++)); do + grep -Fq MXC_INBOUND_READY "$RUN_LOG" && break + kill -0 "$RUN_PID" 2>/dev/null || { cat "$RUN_LOG"; fail "the ingress workload exited before readiness."; } sleep 0.5 - WAITED=$((WAITED + 1)) done - -INBOUND_CHAIN="$(sed -n 's/^.*Creating inbound iptables chain: \([^ ]*\).*$/\1/p' "$RUN_LOG" | head -n 1)" -[ -n "$INBOUND_CHAIN" ] || fail "no inbound chain was created; there is no default-deny to probe." - -NETNS_PID="$(sed -n 's/^.*Container init PID: \([0-9]*\).*$/\1/p' "$RUN_LOG" | head -n 1)" -[ -n "$NETNS_PID" ] || fail "no container init PID was logged; the container namespace cannot be entered." - -in_container_net() { - nsenter -t "$NETNS_PID" -n "$@" -} - -# Put the probe's own interface into the container's namespace. Addressing both -# ends by hand is what removes the dependency on lxcbr0 and on the container's -# DHCP lease. -ip link add "$VETH_HOST" type veth peer name "$VETH_PEER" \ - || fail "could not create the probe veth pair." -VETH_CREATED=1 -ip link set "$VETH_PEER" netns "$NETNS_PID" \ - || fail "could not move the probe interface into the container namespace." - -ip addr add "$PROBE_HOST_V4/24" dev "$VETH_HOST" -ip -6 addr add "$PROBE_HOST_V6/64" dev "$VETH_HOST" nodad -ip link set "$VETH_HOST" up - -in_container_net ip addr add "$PROBE_NS_V4/24" dev "$VETH_PEER" -in_container_net ip -6 addr add "$PROBE_NS_V6/64" dev "$VETH_PEER" nodad -in_container_net ip link set "$VETH_PEER" up - -# The first probe has to resolve the peer's address before it can send, and -# without a moment here that resolution eats into its four-second timeout. -sleep 2 - -# Total packets dropped by the chain, across every DROP rule in it. -x because a -# truncated '1K' would not compare. -# -# Summing rather than reading the NEW-state rule alone: where IPv6 conntrack is -# not loaded an inbound SYN is not classified NEW and falls through to the -# chain's terminal DROP instead. -drop_packets() { - in_container_net "$1" -L "$INBOUND_CHAIN" -n -v -x 2>/dev/null \ - | awk '$3 == "DROP" { total += $1 } END { print total + 0 }' -} - -assert_inbound_dropped() { - local tool="$1" target="$2" family="$3" before="" after="" rc=0 - - before="$(drop_packets "$tool")" - [ -n "$before" ] || fail "$family: could not read the DROP counters on $INBOUND_CHAIN." - - # Nothing listens on this port. INPUT is traversed before the socket lookup, - # so the verdict is the chain's either way. - timeout 4 bash -c "exec 3<>/dev/tcp/$target/$PROBE_PORT" >/dev/null 2>&1 && rc=0 || rc=$? - if [ "$rc" = 0 ]; then - fail "$family: an unsolicited inbound connection to the container succeeded; inbound default-deny is not enforcing." +grep -Fq MXC_INBOUND_READY "$RUN_LOG" || fail "the ingress workload never became ready." +INIT_PID="$(lxc-info -n "$CONTAINER" -p -H)" +nsenter -t "$INIT_PID" -n ip address replace "$GUEST_V4/$PREFIX" dev eth0 +nsenter -t "$INIT_PID" -n ip -6 address add "$GUEST_V6/64" dev eth0 nodad +nsenter -t "$INIT_PID" -n ip link set eth0 up +nsenter -t "$INIT_PID" -n iptables -F +nsenter -t "$INIT_PID" -n iptables -X +nsenter -t "$INIT_PID" -n iptables -P INPUT ACCEPT +nsenter -t "$INIT_PID" -n ip6tables -F +nsenter -t "$INIT_PID" -n ip6tables -X +nsenter -t "$INIT_PID" -n ip6tables -P INPUT ACCEPT + +nsenter -t "$INIT_PID" -n python3 -m http.server "$PORT" --bind "$GUEST_V4" >"$LISTENER_V4_LOG" 2>&1 & +LISTENER_V4_PID=$! +nsenter -t "$INIT_PID" -n python3 -m http.server "$PORT" --bind "$GUEST_V6" >"$LISTENER_V6_LOG" 2>&1 & +LISTENER_V6_PID=$! +for target in "$GUEST_V4" "$GUEST_V6"; do + nsenter -t "$INIT_PID" -n timeout 5 bash -c \ + "until (exec 3<>/dev/tcp/$target/$PORT) 2>/dev/null; do sleep 0.1; done" \ + || fail "the guest listener at $target:$PORT never became reachable locally." + if timeout 3 bash -c "exec 3<>/dev/tcp/$target/$PORT"; then + fail "unsolicited ingress to $target succeeded after the guest firewall was flushed." fi +done +peer_listener_running "$LISTENER_V4_PID" && peer_listener_running "$LISTENER_V6_PID" \ + || fail "a guest listener exited." - after="$(drop_packets "$tool")" - [ -n "$after" ] || fail "$family: could not re-read the DROP counters on $INBOUND_CHAIN." - if [ "$after" -le "$before" ]; then - fail "$family: the chain's DROP counters did not move ($before -> $after); the probe packets never reached the chain, so the connection failure proves nothing." - fi - - echo " $family inbound dropped by the chain: $before -> $after" -} - -# The address is passed bare, not bracketed: bash resolves the /dev/tcp host -# field with getaddrinfo, which rejects '[...]' as a name. -assert_inbound_dropped iptables "$PROBE_NS_V4" IPv4 -assert_inbound_dropped ip6tables "$PROBE_NS_V6" IPv6 - -echo "PASS: unsolicited inbound is dropped inside the container namespace." - -if ! in_container_net ip link show "$VETH_PEER" >/dev/null 2>&1; then - fail "the probe interface vanished from the container namespace." -fi - -# Let the workload's sleep finish so destroyOnExit tears the container down the -# way a normal run would, rather than leaving the signal path to do it. -ip link del "$VETH_HOST" >/dev/null 2>&1 || true -VETH_CREATED=0 +lxc-attach -n "$CONTAINER" -- touch /mxc-inbound-finish if ! wait "$RUN_PID"; then - echo "--- lxc-exec output ---" cat "$RUN_LOG" - fail "lxc-exec exited non-zero." + fail "the ingress workload failed." fi RUN_PID="" +[ "$(lxc-info -n "$CONTAINER" -s -H)" = RUNNING ] || fail "the ingress test container did not survive." -# The chain belongs in the container's namespace, so one on the host means the -# rules were programmed against the wrong namespace. -leaked_chains() { - local tool="$1" before="$2" after="" leaked="" chain - if ! after="$(mxci_chains "$tool")"; then - fail "could not enumerate $tool chains, so the host was not verified clean." - fi - while IFS= read -r chain; do - [ -n "$chain" ] || continue - grep -Fxq "$chain" <<<"$before" || leaked="$leaked $chain" - done <<<"$after" - [ -z "$leaked" ] || fail "$tool: inbound chain(s) left on the host:$leaked" -} - -leaked_chains iptables "$MXCI_BEFORE_V4" -leaked_chains ip6tables "$MXCI_BEFORE_V6" - -echo "PASS: LXC inbound traffic test." +# The same listeners must be reachable over the managed interface after teardown. +for target in "$GUEST_V4" "$GUEST_V6"; do + timeout 5 bash -c "until (exec 3<>/dev/tcp/$target/$PORT) 2>/dev/null; do sleep 0.1; done" \ + || fail "ingress to $target did not recover when preservePolicy=false removed the policy." +done +echo "PASS: IPv4 and IPv6 ingress survived guest firewall flushing and ended with policy teardown." diff --git a/tests/scripts/run_lxc_most_specific_test.sh b/tests/scripts/run_lxc_most_specific_test.sh index 7d4ef5d9e..d4d04501c 100644 --- a/tests/scripts/run_lxc_most_specific_test.sh +++ b/tests/scripts/run_lxc_most_specific_test.sh @@ -42,7 +42,9 @@ sudo rm -rf "$BASE" sudo mkdir -p "$BASE/data/secret_child" echo "CHILD_KEPT" | sudo tee "$BASE/data/secret_child/keep.txt" > /dev/null echo "PARENT_SECRET" | sudo tee "$BASE/data/sibling.txt" > /dev/null -sudo chown -R "$(id -u):$(id -g)" "$BASE/data/secret_child" + +# Give mapped container users access to the writable child. +sudo chmod 777 "$BASE/data/secret_child" echo "Running LXC most-specific test (denied parent, rw child)..." OUT=$("$LXC_EXEC" --experimental \ diff --git a/tests/scripts/run_lxc_network_cidr_boundary_test.sh b/tests/scripts/run_lxc_network_cidr_boundary_test.sh index 3de2d552e..745db8709 100644 --- a/tests/scripts/run_lxc_network_cidr_boundary_test.sh +++ b/tests/scripts/run_lxc_network_cidr_boundary_test.sh @@ -1,228 +1,99 @@ #!/bin/bash -# LXC CIDR boundary network filtering test -# -# Proves roadmap item 19 / AB#62830559 accepts boundary-valid CIDR -# destinations while using the default-block firewall path. The boundary values -# pinned here are IPv4/IPv6 /0, network-aligned /20 and /32 prefixes, -# and single-address IPv4 /32 and IPv6 /128 prefixes. -# -# This fixture asserts boundary prefixes are accepted and programmed, not -# effective reachability; the GA egress test covers deny precedence. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" -LXC_EXEC="$REPO_DIR/src/target/release/lxc-exec" - -if [ ! -f "$LXC_EXEC" ]; then +LXC_EXEC="${LXC_EXEC:-$REPO_DIR/src/target/release/lxc-exec}" +if [ "$LXC_EXEC" = "$REPO_DIR/src/target/release/lxc-exec" ] && [ ! -f "$LXC_EXEC" ]; then LXC_EXEC="$REPO_DIR/src/target/debug/lxc-exec" fi -# An honest skip for a missing prerequisite: exit 77 so run_lxc_all_tests.sh -# records SKIPPED rather than PASS. A suite that could not run must not look green. -SKIP_EXIT=77 -skip() { - echo "SKIP: $1" - exit "$SKIP_EXIT" -} - -[ "$(id -u)" -eq 0 ] || skip "requires root for iptables/ip6tables and LXC." -command -v iptables >/dev/null 2>&1 || skip "iptables is not installed." -command -v ip6tables >/dev/null 2>&1 || skip "ip6tables is not installed." -command -v lxc-create >/dev/null 2>&1 || skip "LXC (lxc-create) is not installed." -command -v python3 >/dev/null 2>&1 || skip "python3 is not installed." -[ -f "$LXC_EXEC" ] || skip "lxc-exec binary not built; run build.sh first." - -CONFIG="$REPO_DIR/tests/configs/lxc_network_cidr_boundary.json" -EXPECTED_ALLOWED_HOSTS=( - "0.0.0.0/0" - "::/0" - "140.82.112.0/20" - "140.82.112.5/32" - "2606:50c0::/32" -) -EXPECTED_BLOCKED_HOSTS=( - "198.51.100.42/32" - "2001:db8::5/128" -) - fail() { echo "FAIL: $1" exit 1 } -# shellcheck source=lib/chain_name.sh -. "$SCRIPT_DIR/lib/chain_name.sh" - -assert_programmed_rule() { - local table="$1" dest="$2" target="$3" - # The --debug log emits one line per destination rule actually generated, - # derived from the built rule args. Asserting it here fails if - # destination-rule emission is deleted while chain/default/hook logging is - # kept. This inspects the rule contents while the chain is being programmed - # rather than only checking post-run cleanup. - if ! grep -Fq "Programmed $table rule: -A $CHAIN_NAME -d $dest -j $target" <<<"$OUTPUT"; then - fail "expected $table rule for '$dest' -> $target was not programmed." - fi -} +CONFIG="$REPO_DIR/tests/configs/${1:-lxc_network_cidr_boundary}.json" +BEFORE_FILE="$REPO_DIR/tests/configs/.lxc-cidr-before-$$.json" +LIVE_FILE="$REPO_DIR/tests/configs/.lxc-cidr-live-$$.json" +BEFORE_NFT="$(nft -s list ruleset)" +nft -j list ruleset >"$BEFORE_FILE" +mapfile -t CONFIGS < <(python3 - "$CONFIG" "$$" <<'PY' +import base64, json, sys +config = json.load(open(sys.argv[1])) +assert config["network"]["egress"]["default"] == "deny" +config["containerId"] += "-" + sys.argv[2] +config["process"]["commandLine"] = "true" +for destroy, preserve in ((False, True), (True, False)): + config["lifecycle"] = {"destroyOnExit": destroy, "preservePolicy": preserve} + print(base64.b64encode(json.dumps(config).encode()).decode()) +PY +) +[ "${#CONFIGS[@]}" = 2 ] || fail "could not prepare the CIDR fixture." -# Compared against a snapshot taken before the run, so chains left behind by an -# earlier failed run are not blamed on this one. -assert_no_new_mxc_chains() { - local tool="$1" before="$2" after="" leaked="" chain - # Captured before iterating rather than piped in from a process - # substitution, whose exit status is not the loop's. A failed enumeration - # would otherwise read as zero chains and pass this assertion while - # verifying nothing. - if ! after="$(mxc_chains "$tool")"; then - fail "could not enumerate $tool chains, so cleanup was not verified." - fi - while IFS= read -r chain; do - [ -n "$chain" ] || continue - grep -Fxq "$chain" <<<"$before" || leaked="$leaked $chain" - done <<<"$after" - if [ -n "$leaked" ]; then - fail "$tool chain(s) left behind after lxc-exec completed:$leaked" - fi +cleanup() { + "$LXC_EXEC" --config-base64 "${CONFIGS[1]}" >/dev/null 2>&1 || true + rm -f "$BEFORE_FILE" "$LIVE_FILE" } - -assert_firewall_chain_cleaned_up() { - assert_no_new_mxc_chains iptables "$MXC_CHAINS_BEFORE_V4" - assert_no_new_mxc_chains ip6tables "$MXC_CHAINS_BEFORE_V6" +trap cleanup EXIT + +OUTPUT=$("$LXC_EXEC" --config-base64 "${CONFIGS[0]}" 2>&1) \ + || fail "the valid CIDR fixture was refused: $OUTPUT" +nft -j list ruleset >"$LIVE_FILE" + +python3 - "$CONFIG" "$BEFORE_FILE" "$LIVE_FILE" <<'PY' +import ipaddress, json, pathlib, sys +config = json.load(open(sys.argv[1])) +expected = { + "lxc_network_cidr_boundary": ( + {"0.0.0.0/0", "::/0", "140.82.112.0/20", "140.82.112.5/32", "2606:50c0::/32"}, + {"198.51.100.42/32", "2001:db8::5/128"} + ), + "lxc_network_ipv6_cidr": ( + {"140.82.112.0/20", "2606:50c0::/32", "2606:50c0:8000::153/128"}, + {"10.0.0.0/8", "2001:db8::/32", "fe80::1/128"} + ), + "lxc_network_dualstack_cidr": ( + {"127.0.0.1/32", "1.1.1.1/32", "2606:4700:4700::1111/128", "8.8.8.8/32", "2001:4860:4860::8888/128"}, + {"10.0.0.0/8", "2001:db8::/32"} + ) +}[pathlib.Path(sys.argv[1]).stem] +for action, destinations in zip(("allow", "deny"), expected): + actual = {target["cidr"] for rule in config["network"]["egress"][action] for target in rule["to"]} + assert actual == destinations, f"{action} destinations drifted in the CIDR fixture" +before = json.load(open(sys.argv[2]))["nftables"] +after = json.load(open(sys.argv[3]))["nftables"] +identifiers = { + (item["rule"]["family"], item["rule"]["table"], item["rule"]["chain"], item["rule"]["handle"]) + for item in before if "rule" in item } - -load_config_hosts() { - python3 - "$CONFIG" <<'PY' -import ipaddress -import json -import sys - -egress = json.load(open(sys.argv[1]))["network"]["egress"] -assert egress["default"] == "deny" -for label in ("allow", "deny"): - for rule in egress[label]: - for dest in rule["to"]: - ipaddress.ip_network(dest["cidr"], strict=True) - print(label + "\t" + dest["cidr"]) +rules = [ + item["rule"] for item in after if "rule" in item + and (item["rule"]["family"], item["rule"]["table"], item["rule"]["chain"], item["rule"]["handle"]) not in identifiers +] +for action, verdict in (("allow", "accept"), ("deny", "drop")): + for configured in config["network"]["egress"][action]: + for destination in configured["to"]: + expected = ipaddress.ip_network(destination["cidr"]) + protocol = "ip" if expected.version == 4 else "ip6" + found = False + for rule in rules: + if {verdict: None} not in rule["expr"]: + continue + for statement in rule["expr"]: + match = statement.get("match", {}) + if match.get("left") != {"payload": {"protocol": protocol, "field": "daddr"}} or match.get("op") != "==": + continue + value = match["right"] + if isinstance(value, dict) and "prefix" in value: + value = f"{value['prefix']['addr']}/{value['prefix']['len']}" + if ipaddress.ip_network(value) == expected: + found = True + assert found, f"host kernel has no {protocol} destination rule for {expected} -> {verdict}" PY -} - -contains_host() { - local needle="$1" - shift - local host - for host in "$@"; do - if [ "$host" = "$needle" ]; then - return 0 - fi - done - return 1 -} - -CONFIG_HOST_TEXT="$(load_config_hosts)" || fail "directional rules contain an invalid or non-network-aligned CIDR." -mapfile -t CONFIG_HOST_LINES <<<"$CONFIG_HOST_TEXT" -CONFIG_ALLOWED_HOSTS=() -CONFIG_BLOCKED_HOSTS=() -for line in "${CONFIG_HOST_LINES[@]}"; do - list="${line%%$'\t'*}" - host="${line#*$'\t'}" - case "$list" in - allow) CONFIG_ALLOWED_HOSTS+=("$host") ;; - deny) CONFIG_BLOCKED_HOSTS+=("$host") ;; - *) fail "unexpected host list '$list' in $CONFIG." ;; - esac -done - -if [ "${#CONFIG_ALLOWED_HOSTS[@]}" -ne "${#EXPECTED_ALLOWED_HOSTS[@]}" ]; then - fail "allowed host count ${#CONFIG_ALLOWED_HOSTS[@]} does not match expected count ${#EXPECTED_ALLOWED_HOSTS[@]}." -fi -if [ "${#CONFIG_BLOCKED_HOSTS[@]}" -ne "${#EXPECTED_BLOCKED_HOSTS[@]}" ]; then - fail "blocked host count ${#CONFIG_BLOCKED_HOSTS[@]} does not match expected count ${#EXPECTED_BLOCKED_HOSTS[@]}." -fi -for expected in "${EXPECTED_ALLOWED_HOSTS[@]}"; do - if ! contains_host "$expected" "${CONFIG_ALLOWED_HOSTS[@]}"; then - fail "expected allowed host '$expected' is missing from $CONFIG." - fi -done -for expected in "${EXPECTED_BLOCKED_HOSTS[@]}"; do - if ! contains_host "$expected" "${CONFIG_BLOCKED_HOSTS[@]}"; then - fail "expected blocked host '$expected' is missing from $CONFIG." - fi -done - -ALL_CONFIG_HOSTS=("${CONFIG_ALLOWED_HOSTS[@]}" "${CONFIG_BLOCKED_HOSTS[@]}") - -echo "Running LXC CIDR boundary network filtering test..." - -MXC_CHAINS_BEFORE_V4="$(mxc_chains iptables)" -MXC_CHAINS_BEFORE_V6="$(mxc_chains ip6tables)" - -set +e -OUTPUT=$("$LXC_EXEC" --debug "$CONFIG" 2>&1) -STATUS=$? -set -e -echo "$OUTPUT" - -derive_chain_name "$OUTPUT" - -# The container command is a local success command (see the fixture), so a -# non-zero status reflects a firewall-setup failure on boundary-valid prefixes -# rather than an unrelated network outage. -if [ "$STATUS" -ne 0 ]; then - fail "lxc-exec exited with status $STATUS for boundary-valid prefixes." -fi - -# Valid prefixes include both families' inclusive bounds, including /0. -for host in "${ALL_CONFIG_HOSTS[@]}"; do - if echo "$OUTPUT" | grep -Fq "Warning: could not resolve host '$host'"; then - fail "host '$host' was not resolved." - fi -done - -# Inspect the actual destination rules generated -- not merely the absence of an -# unresolved-host warning. Each allow entry must yield an ACCEPT rule and each -# block entry a DROP rule, in the correct family's table, so that deleting -# destination-rule emission fails this test even though chain/default/hook -# logging is unchanged. -assert_programmed_rule iptables "0.0.0.0/0" ACCEPT -assert_programmed_rule ip6tables "::/0" ACCEPT -assert_programmed_rule iptables "140.82.112.0/20" ACCEPT -assert_programmed_rule iptables "140.82.112.5/32" ACCEPT -assert_programmed_rule ip6tables "2606:50c0::/32" ACCEPT -assert_programmed_rule iptables "198.51.100.42/32" DROP -assert_programmed_rule ip6tables "2001:db8::5/128" DROP - -if ! echo "$OUTPUT" | grep -q "Default network policy: DROP"; then - fail "default-deny policy was not applied." -fi -if echo "$OUTPUT" | grep -q "Default network policy: ACCEPT"; then - fail "default-allow policy was applied unexpectedly." -fi - -# The v6 half is required by roadmap item 19 / AB#62830559; skipping it would be a dual-stack bypass. -if echo "$OUTPUT" | grep -q "IPv6 firewall rule(s) not applied"; then - fail "IPv6 rules were skipped; ip6tables is unusable on this host." -fi - -if ! echo "$OUTPUT" | grep -q "Creating iptables/ip6tables chain:"; then - fail "firewall chain creation was not logged." -fi - -if echo "$OUTPUT" | grep -qE "^(ip6?tables) .* failed:|Firewall setup failed:"; then - fail "iptables/ip6tables rejected a boundary-valid rule." -fi - -# The OUTPUT hook is what puts the chain on the container's egress path; a run -# that skipped it enforces nothing, so PASS must require it. Fail on the -# skipped-hook warning and require the positive install confirmation. -if echo "$OUTPUT" | grep -Fq "Skipping the OUTPUT hook"; then - fail "OUTPUT hook was skipped; the container network namespace was not found." -fi -if ! echo "$OUTPUT" | grep -Fq "OUTPUT hook installed"; then - fail "OUTPUT hook installation was not confirmed." -fi - -assert_firewall_chain_cleaned_up -echo "PASS: CIDR boundary entries were accepted and programmed under default deny." -echo "LXC CIDR boundary network filtering test complete." +"$LXC_EXEC" --config-base64 "${CONFIGS[1]}" || fail "the CIDR fixture could not be destroyed." +[ "$(nft -s list ruleset)" = "$BEFORE_NFT" ] || fail "CIDR fixture cleanup left host rules." +trap - EXIT +rm -f "$BEFORE_FILE" "$LIVE_FILE" +echo "PASS: configured IPv4 and IPv6 destinations and verdicts reached the host kernel." diff --git a/tests/scripts/run_lxc_network_deny_all_loopback_test.sh b/tests/scripts/run_lxc_network_deny_all_loopback_test.sh index 035d6f683..2b9abea8a 100644 --- a/tests/scripts/run_lxc_network_deny_all_loopback_test.sh +++ b/tests/scripts/run_lxc_network_deny_all_loopback_test.sh @@ -36,6 +36,9 @@ OMIT_CONFIG="$REPO_DIR/tests/configs/lxc_network_deny_all_loopback_omit.json" LXC_EXEC="$REPO_DIR/src/target/release/lxc-exec" [ -x "$LXC_EXEC" ] || LXC_EXEC="$REPO_DIR/src/target/debug/lxc-exec" +# shellcheck source=lib/lxc_static_network.sh +. "$SCRIPT_DIR/lib/lxc_static_network.sh" + SKIP_EXIT=77 skip() { @@ -124,7 +127,7 @@ run_config() { local label="$1" config="$2" status=0 echo "--- $label ---" set +e - CASE_OUTPUT=$("$LXC_EXEC" --debug "$config" 2>&1) + CASE_OUTPUT=$("${@:3}" "$config" 2>&1) status=$? set -e echo "$CASE_OUTPUT" @@ -150,7 +153,7 @@ assert_reported() { fi } -run_config "positive control: request permitting one destination" "$CTRL_CONFIG" +run_config "positive control: request permitting one destination" "$CTRL_CONFIG" run_with_static_guest_ipv4 assert_reported "loopback" "$LOOPBACK_OK" "$LOOPBACK_DEAD" assert_reported "egress" "$NET_ALLOWED" "$NET_BLOCKED" @@ -164,7 +167,7 @@ fi echo "Control reached the network and its own loopback." -run_config "case under test: request permitting nothing and naming no proxy" "$OMIT_CONFIG" +run_config "case under test: request permitting nothing and naming no proxy" "$OMIT_CONFIG" "$LXC_EXEC" --debug assert_reported "loopback" "$LOOPBACK_OK" "$LOOPBACK_DEAD" assert_reported "egress" "$NET_ALLOWED" "$NET_BLOCKED" diff --git a/tests/scripts/run_lxc_network_deny_precedence_test.sh b/tests/scripts/run_lxc_network_deny_precedence_test.sh index 2f4344258..b60a7bde4 100644 --- a/tests/scripts/run_lxc_network_deny_precedence_test.sh +++ b/tests/scripts/run_lxc_network_deny_precedence_test.sh @@ -1,28 +1,11 @@ #!/bin/bash -# LXC deny-precedence enforcement test -# -# A destination named in both allowedHosts and blockedHosts must be blocked. -# The chain is first-match-wins, so this is decided entirely by which list is -# emitted first -- there is no separate precedence pass to assert on. That -# makes it invisible to any test that only inspects rules individually, and it -# is why this assertion is behavioral rather than a log grep. -# -# Both configs name the same destination set, 0.0.0.0/0 and ::/0, so the rules -# are literal CIDRs rather than a hostname resolved once per list entry. A -# hostname would be resolved separately for the allow entry and the block -# entry, and round-robin DNS could hand back different addresses for the two, -# which would make the outcome depend on which address wget happened to pick. -# -# The control run is what makes the overlap run mean anything. Without it, a -# host with no working egress at all -- or a change that broke networking -# outright -- would produce the same blocked verdict and look like a pass. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" -LXC_EXEC="$REPO_DIR/src/target/release/lxc-exec" +LXC_EXEC="${LXC_EXEC:-$REPO_DIR/src/target/release/lxc-exec}" -if [ ! -f "$LXC_EXEC" ]; then +if [ "$LXC_EXEC" = "$REPO_DIR/src/target/release/lxc-exec" ] && [ ! -f "$LXC_EXEC" ]; then LXC_EXEC="$REPO_DIR/src/target/debug/lxc-exec" fi @@ -50,61 +33,12 @@ fail() { exit 1 } -# shellcheck source=lib/chain_name.sh -. "$SCRIPT_DIR/lib/chain_name.sh" # shellcheck source=lib/lxc_peer_listener.sh . "$SCRIPT_DIR/lib/lxc_peer_listener.sh" -# Compared against a snapshot taken before the run, so chains left behind by an -# earlier failed run are not blamed on this one. -assert_no_new_mxc_chains() { - local tool="$1" before="$2" after="" leaked="" chain - # Captured before iterating rather than piped in from a process - # substitution, whose exit status is not the loop's. A failed enumeration - # would otherwise read as zero chains and pass this assertion while - # verifying nothing. - if ! after="$(mxc_chains "$tool")"; then - fail "could not enumerate $tool chains, so cleanup was not verified." - fi - while IFS= read -r chain; do - [ -n "$chain" ] || continue - grep -Fxq "$chain" <<<"$before" || leaked="$leaked $chain" - done <<<"$after" - if [ -n "$leaked" ]; then - fail "$tool chain(s) left behind after lxc-exec completed:$leaked" - fi -} - -# The named chain must be gone, and the run must not have leaked any other -# MXC-owned chain either. The first check is specific to the container this -# case ran; the second catches a rename or a partial rollback that leaves a -# differently named chain behind. -assert_firewall_chain_cleaned_up() { - local chain="$1" - if iptables -S "$chain" >/dev/null 2>&1; then - fail "iptables chain '$chain' was left behind after lxc-exec completed." - fi - if ip6tables -S "$chain" >/dev/null 2>&1; then - fail "ip6tables chain '$chain' was left behind after lxc-exec completed." - fi - assert_no_new_mxc_chains iptables "$MXC_CHAINS_BEFORE_V4" - assert_no_new_mxc_chains ip6tables "$MXC_CHAINS_BEFORE_V6" -} - -assert_no_forward_reference() { - if iptables -S FORWARD 2>/dev/null | grep -Fq -- "$1"; then - fail "a FORWARD rule still references chain '$1' after teardown." - fi -} +# shellcheck source=lib/lxc_static_network.sh +. "$SCRIPT_DIR/lib/lxc_static_network.sh" -# A listener on the host or on the bridge gateway is delivered through INPUT -# and answers with no firewall in the path. The peer lives in its own network -# namespace behind a veth, reached only through the FORWARD hook the chain -# filters on. -# -# Both runs must aim here: the control run shows this exact address is -# reachable when only the allow list names it, which leaves the deny entry as -# the only thing that can account for the overlap run's blocked verdict. PEER_NETNS="mxc-denyprec-peer" PEER_HOST_VETH="mxcdph0" PEER_VETH="mxcdpp0" @@ -117,7 +51,7 @@ PEER_PREFIX="29" PEER_PORT="443" PEER_LISTENER_PID="" -PEER_LISTENER_LOG="$(mktemp)" +PEER_LISTENER_LOG="$REPO_DIR/tests/configs/.lxc-precedence-peer-$$.log" IP_FORWARD_WAS="" teardown_peer() { if [ -n "$PEER_LISTENER_PID" ]; then @@ -182,23 +116,26 @@ done echo "Running LXC deny-precedence enforcement test..." echo "--- control: destination allowed, nothing blocked ---" -MXC_CHAINS_BEFORE_V4="$(mxc_chains iptables)" -MXC_CHAINS_BEFORE_V6="$(mxc_chains ip6tables)" -CONTROL_OUTPUT=$("$LXC_EXEC" --debug "$CONTROL_CONFIG" 2>&1 || true) +BEFORE_V4="$(iptables-save | sed '/^#/d')" +BEFORE_V6="$(ip6tables-save | sed '/^#/d')" +BEFORE_NFT="$(nft -s list ruleset)" +if ! CONTROL_OUTPUT=$(run_with_static_guest_ipv4 "$CONTROL_CONFIG" 2>&1); then + fail "the control fixture failed: $CONTROL_OUTPUT" +fi echo "$CONTROL_OUTPUT" if ! echo "$CONTROL_OUTPUT" | grep -Fq "MXC_NET_ALLOWED"; then fail "the control destination was unreachable with an allow-everything policy, so this host cannot distinguish a deny-precedence failure from a broken network." fi -derive_chain_name "$CONTROL_OUTPUT" -assert_no_forward_reference "$CHAIN_NAME" -assert_firewall_chain_cleaned_up "$CHAIN_NAME" +[ "$(iptables-save | sed '/^#/d')" = "$BEFORE_V4" ] || fail "the control leaked IPv4 host rules." +[ "$(ip6tables-save | sed '/^#/d')" = "$BEFORE_V6" ] || fail "the control leaked IPv6 host rules." +[ "$(nft -s list ruleset)" = "$BEFORE_NFT" ] || fail "the control leaked nftables host rules." echo "--- overlap: same destination in both allowedHosts and blockedHosts ---" -MXC_CHAINS_BEFORE_V4="$(mxc_chains iptables)" -MXC_CHAINS_BEFORE_V6="$(mxc_chains ip6tables)" -OVERLAP_OUTPUT=$("$LXC_EXEC" --debug "$OVERLAP_CONFIG" 2>&1 || true) +if ! OVERLAP_OUTPUT=$(run_with_static_guest_ipv4 "$OVERLAP_CONFIG" 2>&1); then + fail "the overlap fixture failed: $OVERLAP_OUTPUT" +fi echo "$OVERLAP_OUTPUT" if echo "$OVERLAP_OUTPUT" | grep -Fq "MXC_NET_ALLOWED"; then @@ -208,9 +145,9 @@ if ! echo "$OVERLAP_OUTPUT" | grep -Fq "MXC_NET_BLOCKED"; then fail "the overlap case produced no verdict at all; the container command did not run." fi -derive_chain_name "$OVERLAP_OUTPUT" -assert_no_forward_reference "$CHAIN_NAME" -assert_firewall_chain_cleaned_up "$CHAIN_NAME" +[ "$(iptables-save | sed '/^#/d')" = "$BEFORE_V4" ] || fail "the deny-precedence run leaked IPv4 host rules." +[ "$(ip6tables-save | sed '/^#/d')" = "$BEFORE_V6" ] || fail "the deny-precedence run leaked IPv6 host rules." +[ "$(nft -s list ruleset)" = "$BEFORE_NFT" ] || fail "the deny-precedence run leaked nftables host rules." echo "PASS: a destination in both lists was blocked, and the same destination was reachable when only allowed." echo "LXC deny-precedence enforcement test complete." diff --git a/tests/scripts/run_lxc_network_dualstack_test.sh b/tests/scripts/run_lxc_network_dualstack_test.sh index 122efb23e..2fbbfae28 100644 --- a/tests/scripts/run_lxc_network_dualstack_test.sh +++ b/tests/scripts/run_lxc_network_dualstack_test.sh @@ -1,177 +1,5 @@ #!/bin/bash -# LXC mixed IPv4/IPv6 directional firewall rule programming test. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" -LXC_EXEC="$REPO_DIR/src/target/release/lxc-exec" - -if [ ! -f "$LXC_EXEC" ]; then - LXC_EXEC="$REPO_DIR/src/target/debug/lxc-exec" -fi - -# An honest skip for a missing prerequisite: exit 77 so run_lxc_all_tests.sh -# records SKIPPED rather than PASS. A suite that could not run must not look green. -SKIP_EXIT=77 -skip() { - echo "SKIP: $1" - exit "$SKIP_EXIT" -} - -[ "$(id -u)" -eq 0 ] || skip "requires root for iptables/ip6tables and LXC." -command -v iptables >/dev/null 2>&1 || skip "iptables is not installed." -command -v ip6tables >/dev/null 2>&1 || skip "ip6tables is not installed." -command -v lxc-create >/dev/null 2>&1 || skip "LXC (lxc-create) is not installed." -command -v python3 >/dev/null 2>&1 || skip "python3 is not installed." -[ -f "$LXC_EXEC" ] || skip "lxc-exec binary not built; run build.sh first." - -CONFIG="$REPO_DIR/tests/configs/lxc_network_dualstack_cidr.json" -EXPECTED_ALLOWED_HOST_COUNT=5 -EXPECTED_BLOCKED_HOST_COUNT=2 -EXPECTED_ALLOWED_HOSTS=( - "127.0.0.1/32" - "1.1.1.1/32" - "2606:4700:4700::1111/128" - "8.8.8.8/32" - "2001:4860:4860::8888/128" -) -EXPECTED_BLOCKED_HOSTS=( - "10.0.0.0/8" - "2001:db8::/32" -) - -fail() { - echo "FAIL: $1" - exit 1 -} - -# shellcheck source=lib/chain_name.sh -. "$SCRIPT_DIR/lib/chain_name.sh" - -assert_programmed_rule() { - local table="$1" dest="$2" target="$3" - # The --debug log emits one line per destination rule actually generated, - # derived from the built rule args. Asserting it here fails if - # destination-rule emission is deleted while chain/default/hook logging is - # kept. This inspects the rule contents while the chain is being programmed - # rather than only checking that an unresolved-host warning is absent. - if ! grep -Fq "Programmed $table rule: -A $CHAIN_NAME -d $dest -j $target" <<<"$OUTPUT"; then - fail "expected $table rule for '$dest' -> $target was not programmed." - fi -} - -# Compared against a snapshot taken before the run, so chains left behind by an -# earlier failed run are not blamed on this one. -assert_no_new_mxc_chains() { - local tool="$1" before="$2" after="" leaked="" chain - # Captured before iterating rather than piped in from a process - # substitution, whose exit status is not the loop's. A failed enumeration - # would otherwise read as zero chains and pass this assertion while - # verifying nothing. - if ! after="$(mxc_chains "$tool")"; then - fail "could not enumerate $tool chains, so cleanup was not verified." - fi - while IFS= read -r chain; do - [ -n "$chain" ] || continue - grep -Fxq "$chain" <<<"$before" || leaked="$leaked $chain" - done <<<"$after" - if [ -n "$leaked" ]; then - fail "$tool chain(s) left behind after lxc-exec completed:$leaked" - fi -} - -assert_firewall_chain_cleaned_up() { - assert_no_new_mxc_chains iptables "$MXC_CHAINS_BEFORE_V4" - assert_no_new_mxc_chains ip6tables "$MXC_CHAINS_BEFORE_V6" -} - -load_config_counts() { - python3 -c 'import json, sys; e=json.load(open(sys.argv[1]))["network"]["egress"]; assert e["default"] == "deny"; print(len(e["allow"]), len(e["deny"]))' "$CONFIG" -} - -load_config_hosts() { - python3 -c 'import json, sys; e=json.load(open(sys.argv[1]))["network"]["egress"]; [print(label + "\t" + dest["cidr"]) for label in ("allow", "deny") for rule in e[label] for dest in rule["to"]]' "$CONFIG" -} - -counts="$(load_config_counts)" || fail "could not read the directional rule counts." -read -r allowed_count blocked_count <<<"$counts" -if [ "$allowed_count" -ne "$EXPECTED_ALLOWED_HOST_COUNT" ]; then - fail "config allow rule count $allowed_count does not match expected count $EXPECTED_ALLOWED_HOST_COUNT." -fi -if [ "$blocked_count" -ne "$EXPECTED_BLOCKED_HOST_COUNT" ]; then - fail "config deny rule count $blocked_count does not match expected count $EXPECTED_BLOCKED_HOST_COUNT." -fi - -hosts="$(load_config_hosts)" || fail "could not read directional destinations." -mapfile -t CONFIG_HOSTS <<<"$hosts" -EXPECTED_HOSTS=("${EXPECTED_ALLOWED_HOSTS[@]}" "${EXPECTED_BLOCKED_HOSTS[@]}") -if [ "${#CONFIG_HOSTS[@]}" -ne "${#EXPECTED_HOSTS[@]}" ]; then - fail "destination count drifted." -fi -for i in "${!EXPECTED_HOSTS[@]}"; do - expected="${EXPECTED_HOSTS[$i]}" - if [ "$i" -lt "$EXPECTED_ALLOWED_HOST_COUNT" ]; then label=allow; else label=deny; fi - found=0 - for actual in "${CONFIG_HOSTS[@]}"; do - if [ "$actual" = "$label"$'\t'"$expected" ]; then - found=1 - break - fi - done - if [ "$found" -ne 1 ]; then - fail "expected host '$expected' is missing from $CONFIG." - fi -done - -echo "Running LXC mixed-family directional network filtering test..." - -# The container command may fail if the host has no outbound route; this test is -# only asserting firewall setup and destination-family handling. -MXC_CHAINS_BEFORE_V4="$(mxc_chains iptables)" -MXC_CHAINS_BEFORE_V6="$(mxc_chains ip6tables)" -OUTPUT=$("$LXC_EXEC" --debug "$CONFIG" 2>&1 || true) -echo "$OUTPUT" - -derive_chain_name "$OUTPUT" - -for host in "${EXPECTED_HOSTS[@]}"; do - if grep -Fq "Warning: could not resolve host '$host'" <<<"$OUTPUT"; then - fail "host '$host' was not resolved." - fi -done - -# Prove a positive IPv6 rule exists rather than only checking that no -# unresolved-host warning appeared. The IPv6 literal and the IPv6 CIDR are -# offline-safe and deterministic, so their v6 rules are always asserted. -assert_programmed_rule iptables "1.1.1.1/32" ACCEPT -assert_programmed_rule ip6tables "2606:4700:4700::1111/128" ACCEPT -assert_programmed_rule ip6tables "2001:4860:4860::8888/128" ACCEPT -assert_programmed_rule ip6tables "2001:db8::/32" DROP - -if ! grep -Fq "Creating iptables/ip6tables chain:" <<<"$OUTPUT"; then - fail "iptables/ip6tables chain creation was not logged." -fi - -if ! grep -Fq "Default network policy: DROP" <<<"$OUTPUT"; then - fail "default-deny policy was not applied." -fi - -# This warning means the IPv6 half was skipped. -if grep -Fq "IPv6 firewall rule(s) not applied" <<<"$OUTPUT"; then - fail "IPv6 rules were skipped; the dual-stack bypass is still open on this run." -fi - -# The OUTPUT hook is what puts the chain on the container's egress path; a run -# that skipped it enforces nothing, so PASS must require it. Fail on the -# skipped-hook warning and require the positive install confirmation. -if grep -Fq "Skipping the OUTPUT hook" <<<"$OUTPUT"; then - fail "OUTPUT hook was skipped; the container network namespace was not found." -fi -if ! grep -Fq "OUTPUT hook installed" <<<"$OUTPUT"; then - fail "OUTPUT hook installation was not confirmed." -fi - -assert_firewall_chain_cleaned_up - -echo "PASS: mixed-family directional destinations were programmed." -echo "LXC mixed-family network filtering test complete." +exec bash "$SCRIPT_DIR/run_lxc_network_cidr_boundary_test.sh" lxc_network_dualstack_cidr diff --git a/tests/scripts/run_lxc_network_enforcement_test.sh b/tests/scripts/run_lxc_network_enforcement_test.sh index 6774f92a0..0a9437277 100644 --- a/tests/scripts/run_lxc_network_enforcement_test.sh +++ b/tests/scripts/run_lxc_network_enforcement_test.sh @@ -1,25 +1,11 @@ #!/bin/bash -# LXC network policy enforcement test -# -# Every other network script asserts that the OUTPUT hook was *installed*. -# That is a log line, and a hook can install cleanly, name the right chain, -# and still match no packet -- which is exactly how a fully populated deny-all -# chain that filtered nothing once passed every script in this directory. -# -# This script asserts the guarantee itself rather than the log: a destination -# the policy does not allow must be unreachable from inside the container. -# -# Both directions are required, and the allow case is not decoration. A -# blocked-only assertion would also pass on a host with no working network at -# all, or on a change that broke egress outright, so it proves nothing on its -# own. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" -LXC_EXEC="$REPO_DIR/src/target/release/lxc-exec" +LXC_EXEC="${LXC_EXEC:-$REPO_DIR/src/target/release/lxc-exec}" -if [ ! -f "$LXC_EXEC" ]; then +if [ "$LXC_EXEC" = "$REPO_DIR/src/target/release/lxc-exec" ] && [ ! -f "$LXC_EXEC" ]; then LXC_EXEC="$REPO_DIR/src/target/debug/lxc-exec" fi @@ -46,64 +32,12 @@ fail() { exit 1 } -# shellcheck source=lib/chain_name.sh -. "$SCRIPT_DIR/lib/chain_name.sh" # shellcheck source=lib/lxc_peer_listener.sh . "$SCRIPT_DIR/lib/lxc_peer_listener.sh" -# Compared against a snapshot taken before the run, so chains left behind by an -# earlier failed run are not blamed on this one. -assert_no_new_mxc_chains() { - local tool="$1" before="$2" after="" leaked="" chain - # Captured before iterating rather than piped in from a process - # substitution, whose exit status is not the loop's. A failed enumeration - # would otherwise read as zero chains and pass this assertion while - # verifying nothing. - if ! after="$(mxc_chains "$tool")"; then - fail "could not enumerate $tool chains, so cleanup was not verified." - fi - while IFS= read -r chain; do - [ -n "$chain" ] || continue - grep -Fxq "$chain" <<<"$before" || leaked="$leaked $chain" - done <<<"$after" - if [ -n "$leaked" ]; then - fail "$tool chain(s) left behind after lxc-exec completed:$leaked" - fi -} - -# The named chain must be gone, and the run must not have leaked any other -# MXC-owned chain either. The first check is specific to the container this -# case ran; the second catches a rename or a partial rollback that leaves a -# differently named chain behind. -assert_firewall_chain_cleaned_up() { - local chain="$1" - if iptables -S "$chain" >/dev/null 2>&1; then - fail "iptables chain '$chain' was left behind after lxc-exec completed." - fi - if ip6tables -S "$chain" >/dev/null 2>&1; then - fail "ip6tables chain '$chain' was left behind after lxc-exec completed." - fi - assert_no_new_mxc_chains iptables "$MXC_CHAINS_BEFORE_V4" - assert_no_new_mxc_chains ip6tables "$MXC_CHAINS_BEFORE_V6" -} - -# A hook that references the chain but survives teardown leaves the next -# container's traffic running through a stale rule, so the reference count -# matters as much as the chain itself. -assert_no_forward_reference() { - if iptables -S FORWARD 2>/dev/null | grep -Fq -- "$1"; then - fail "a FORWARD rule still references chain '$1' after teardown." - fi -} +# shellcheck source=lib/lxc_static_network.sh +. "$SCRIPT_DIR/lib/lxc_static_network.sh" -# A listener on the host or on the bridge gateway is delivered through INPUT -# and answers with no firewall in the path. The peer lives in its own network -# namespace behind a veth, reached only through the FORWARD hook the chain -# filters on. -# -# Both cases must aim at this one address: the deny case shows a destination -# that is demonstrably reachable becoming unreachable under policy, rather than -# merely failing to reach something. PEER_NETNS="mxc-enforce-peer" PEER_HOST_VETH="mxcenh0" PEER_VETH="mxcenp0" @@ -115,7 +49,7 @@ PEER_IP="198.51.100.2" PEER_PREFIX="29" PEER_PORT="443" PEER_LISTENER_PID="" -PEER_LISTENER_LOG="$(mktemp)" +PEER_LISTENER_LOG="$REPO_DIR/tests/configs/.lxc-enforcement-peer-$$.log" IP_FORWARD_WAS="" teardown_peer() { if [ -n "$PEER_LISTENER_PID" ]; then @@ -190,9 +124,12 @@ echo "Running LXC network policy enforcement test..." # code, so a wrapper that swallows or rewrites the status cannot turn a # reachable destination into an apparent block. echo "--- deny case: default policy blocks, nothing allowed ---" -MXC_CHAINS_BEFORE_V4="$(mxc_chains iptables)" -MXC_CHAINS_BEFORE_V6="$(mxc_chains ip6tables)" -DENY_OUTPUT=$("$LXC_EXEC" --debug "$DENY_CONFIG" 2>&1 || true) +BEFORE_V4="$(iptables-save | sed '/^#/d')" +BEFORE_V6="$(ip6tables-save | sed '/^#/d')" +BEFORE_NFT="$(nft -s list ruleset)" +if ! DENY_OUTPUT=$("$LXC_EXEC" --debug "$DENY_CONFIG" 2>&1); then + fail "the deny fixture failed: $DENY_OUTPUT" +fi echo "$DENY_OUTPUT" if echo "$DENY_OUTPUT" | grep -Fq "MXC_NET_ALLOWED"; then @@ -208,16 +145,14 @@ if ! echo "$DENY_OUTPUT" | grep -Fq "MXC_LOOPBACK_OK"; then fail "the deny case returned no loopback verdict; the in-container listener did not run." fi -# A policy permitting nothing is withheld by giving the container no interface -# at all, which installs no chain and leaves no name to derive. Comparing the -# snapshots still catches anything left behind. -assert_no_new_mxc_chains iptables "$MXC_CHAINS_BEFORE_V4" -assert_no_new_mxc_chains ip6tables "$MXC_CHAINS_BEFORE_V6" +[ "$(iptables-save | sed '/^#/d')" = "$BEFORE_V4" ] || fail "the isolated run leaked IPv4 host rules." +[ "$(ip6tables-save | sed '/^#/d')" = "$BEFORE_V6" ] || fail "the isolated run leaked IPv6 host rules." +[ "$(nft -s list ruleset)" = "$BEFORE_NFT" ] || fail "the isolated run leaked nftables host rules." echo "--- allow case: same default, destination explicitly allowed ---" -MXC_CHAINS_BEFORE_V4="$(mxc_chains iptables)" -MXC_CHAINS_BEFORE_V6="$(mxc_chains ip6tables)" -ALLOW_OUTPUT=$("$LXC_EXEC" --debug "$ALLOW_CONFIG" 2>&1 || true) +if ! ALLOW_OUTPUT=$(run_with_static_guest_ipv4 "$ALLOW_CONFIG" 2>&1); then + fail "the allow fixture failed: $ALLOW_OUTPUT" +fi echo "$ALLOW_OUTPUT" if echo "$ALLOW_OUTPUT" | grep -Fq "MXC_NET_BLOCKED"; then @@ -227,9 +162,9 @@ if ! echo "$ALLOW_OUTPUT" | grep -Fq "MXC_NET_ALLOWED"; then fail "the allow case produced no verdict at all; the container command did not run." fi -derive_chain_name "$ALLOW_OUTPUT" -assert_no_forward_reference "$CHAIN_NAME" -assert_firewall_chain_cleaned_up "$CHAIN_NAME" +[ "$(iptables-save | sed '/^#/d')" = "$BEFORE_V4" ] || fail "the allowed run leaked IPv4 host rules." +[ "$(ip6tables-save | sed '/^#/d')" = "$BEFORE_V6" ] || fail "the allowed run leaked IPv6 host rules." +[ "$(nft -s list ruleset)" = "$BEFORE_NFT" ] || fail "the allowed run leaked nftables host rules." echo "PASS: a disallowed destination was blocked, an allowed destination was reachable," echo " and the blocked container still reached its own loopback." diff --git a/tests/scripts/run_lxc_network_ga_egress_test.sh b/tests/scripts/run_lxc_network_ga_egress_test.sh index 3ef76a9a2..5e05230dc 100644 --- a/tests/scripts/run_lxc_network_ga_egress_test.sh +++ b/tests/scripts/run_lxc_network_ga_egress_test.sh @@ -1,21 +1,11 @@ #!/bin/bash -# LXC egress enforcement test -# -# Asserts reachability rather than a log line: a chain can install cleanly, -# name the right chain, and still filter nothing. -# -# Every case probes a CI-controlled peer this script stands up in its own routed -# namespace, including the udp/53 cases, which query a resolver it hosts. -# -# A directional posture carries no port 53 exemption, unlike the legacy chain, -# which is what the two DNS cases pin. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" -LXC_EXEC="$REPO_DIR/src/target/release/lxc-exec" +LXC_EXEC="${LXC_EXEC:-$REPO_DIR/src/target/release/lxc-exec}" -if [ ! -f "$LXC_EXEC" ]; then +if [ "$LXC_EXEC" = "$REPO_DIR/src/target/release/lxc-exec" ] && [ ! -f "$LXC_EXEC" ]; then LXC_EXEC="$REPO_DIR/src/target/debug/lxc-exec" fi @@ -63,86 +53,29 @@ fail() { exit 1 } -# shellcheck source=lib/chain_name.sh -. "$SCRIPT_DIR/lib/chain_name.sh" # shellcheck source=lib/lxc_peer_listener.sh . "$SCRIPT_DIR/lib/lxc_peer_listener.sh" # shellcheck source=lib/lxc_dns_peer.sh . "$SCRIPT_DIR/lib/lxc_dns_peer.sh" -# The snapshot keeps chains left behind by an earlier failed run from being -# blamed on this one. -assert_no_new_mxc_chains() { - local tool="$1" before="$2" after="" leaked="" chain - if ! after="$(mxc_chains "$tool")"; then - fail "could not enumerate $tool chains, so cleanup was not verified." - fi - while IFS= read -r chain; do - [ -n "$chain" ] || continue - grep -Fxq "$chain" <<<"$before" || leaked="$leaked $chain" - done <<<"$after" - if [ -n "$leaked" ]; then - fail "$tool chain(s) left behind after lxc-exec completed:$leaked" - fi -} - -assert_firewall_chain_cleaned_up() { - local chain="$1" - if iptables -S "$chain" >/dev/null 2>&1; then - fail "iptables chain '$chain' was left behind after lxc-exec completed." - fi - if ip6tables -S "$chain" >/dev/null 2>&1; then - fail "ip6tables chain '$chain' was left behind after lxc-exec completed." - fi - assert_no_new_mxc_chains iptables "$MXC_CHAINS_BEFORE_V4" - assert_no_new_mxc_chains ip6tables "$MXC_CHAINS_BEFORE_V6" -} - -assert_no_forward_reference() { - if iptables -S FORWARD 2>/dev/null | grep -Fq -- "$1"; then - fail "a FORWARD rule still references chain '$1' after teardown." - fi -} - -# A backend reading `enforcementMode` alone skips the chain and still reports -# success, which is a silent unenforced run rather than a failure. -assert_enforcement_not_skipped() { - if echo "$1" | grep -Fq "requests no firewall; skipping iptables"; then - fail "the 0.9 config was treated as not using the firewall, so no rules were installed. The directional posture is not reaching the firewall gate." - fi -} +# shellcheck source=lib/lxc_static_network.sh +. "$SCRIPT_DIR/lib/lxc_static_network.sh" -# A policy that permits nothing is served by keeping the container off the -# network entirely, so no chain is created and there is none to inspect. -assert_no_chain_installed() { - if echo "$1" | grep -Eq "Creating iptables/ip6tables chain:|Programmed [a-z0-9]* rule:"; then - fail "a policy permitting nothing installed a firewall chain, so the container was put on the network and then filtered instead of being left off it." - fi -} - -# Each case names the posture its policy should reach. Reading the posture back -# out of the run instead would let a filtered policy that installed nothing pass -# as an isolated one, which is the silent-unenforced case worth catching. run_case() { - local label="$1" config="$2" posture="${3:-filtered}" output="" + local label="$1" config="$2" output="" echo "--- $label ---" - MXC_CHAINS_BEFORE_V4="$(mxc_chains iptables)" - MXC_CHAINS_BEFORE_V6="$(mxc_chains ip6tables)" - output=$("$LXC_EXEC" --debug "$config" 2>&1 || true) + BEFORE_V4="$(iptables-save | sed '/^#/d')" + BEFORE_V6="$(ip6tables-save | sed '/^#/d')" + BEFORE_NFT="$(nft -s list ruleset)" + if ! output=$("${@:3}" "$config" 2>&1); then + fail "$label failed: $output" + fi echo "$output" CASE_OUTPUT="$output" - if [ "$posture" = isolated ]; then - assert_no_chain_installed "$output" - assert_no_new_mxc_chains iptables "$MXC_CHAINS_BEFORE_V4" - assert_no_new_mxc_chains ip6tables "$MXC_CHAINS_BEFORE_V6" - return - fi - - assert_enforcement_not_skipped "$output" - derive_chain_name "$output" - assert_no_forward_reference "$CHAIN_NAME" - assert_firewall_chain_cleaned_up "$CHAIN_NAME" + [ "$(iptables-save | sed '/^#/d')" = "$BEFORE_V4" ] || fail "$label leaked IPv4 host rules." + [ "$(ip6tables-save | sed '/^#/d')" = "$BEFORE_V6" ] || fail "$label leaked IPv6 host rules." + [ "$(nft -s list ruleset)" = "$BEFORE_NFT" ] || fail "$label leaked nftables host rules." } assert_blocked() { @@ -179,10 +112,6 @@ assert_allowed() { fi } -# A listener on the host or on the bridge gateway is delivered through INPUT -# and answers with no firewall in the path. The peer lives in its own network -# namespace behind a veth, reached only through the FORWARD hook the chain -# filters on (-i / --physdev-in ). PEER_NETNS="mxc-ga-egress-peer" PEER_HOST_VETH="mxcgah0" PEER_VETH="mxcgap0" @@ -204,10 +133,10 @@ PEER_DNS_ANSWER="203.0.113.99" PEER_LISTENER_PID="" PEER_UDP_LISTENER_PID="" -PEER_LISTENER_LOG="$(mktemp)" -PEER_UDP_LISTENER_LOG="$(mktemp)" -PEER_DNS_LISTENER_LOG="$(mktemp)" -IO_URING_CONFIG="$(mktemp --suffix=.json)" +PEER_LISTENER_LOG="$REPO_DIR/tests/configs/.lxc-ga-peer-$$.log" +PEER_UDP_LISTENER_LOG="$REPO_DIR/tests/configs/.lxc-ga-udp-$$.log" +PEER_DNS_LISTENER_LOG="$REPO_DIR/tests/configs/.lxc-ga-dns-$$.log" +IO_URING_CONFIG="$REPO_DIR/tests/configs/.lxc-ga-io-uring-$$.json" IP_FORWARD_WAS="" teardown_peer() { if [ -n "$PEER_LISTENER_PID" ]; then @@ -361,103 +290,114 @@ done # An egress-only config is the shape a backend claiming only the two egress # bits would reject outright, which makes any verdict here a test of the # support declaration. -run_case "deny case: egress.default deny, no rules" "$DENY_CONFIG" isolated +run_case "deny case: egress.default deny, no rules" "$DENY_CONFIG" "$LXC_EXEC" --debug assert_blocked "egress succeeded under egress.default deny with no allow rules, so a container that should never have reached the network reached it." -run_case "allow case: same default, destination allowed on tcp/443" "$ALLOW_CONFIG" +run_case "allow case: same default, destination allowed on tcp/443" "$ALLOW_CONFIG" run_with_static_guest_ipv4 assert_allowed "an explicitly allowed destination was unreachable. The policy is over-blocking, so the deny case above proves nothing." -run_case "wrong-port case: same destination allowed on tcp/444" "$WRONG_PORT_CONFIG" +run_case "wrong-port case: same destination allowed on tcp/444" "$WRONG_PORT_CONFIG" run_with_static_guest_ipv4 assert_blocked "traffic to tcp/443 succeeded while the policy allowed only tcp/444. The port selector is being dropped, so the allow case above proves only that the destination matched." -run_case "dns-denied case: egress.default deny, DNS probe to the peer resolver" "$DNS_DENIED_CONFIG" isolated +run_case "dns-denied case: egress.default deny, DNS probe to the peer resolver" "$DNS_DENIED_CONFIG" "$LXC_EXEC" --debug assert_blocked "a DNS query succeeded under egress.default deny with no allow rules, which leaves this container a DNS-tunnel path out of a policy that permits nothing." -run_case "dns-allowed case: same probe, resolver allowed on udp/53" "$DNS_ALLOWED_CONFIG" +run_case "dns-allowed case: same probe, resolver allowed on udp/53" "$DNS_ALLOWED_CONFIG" run_with_static_guest_ipv4 assert_allowed "a DNS query to an explicitly allowed resolver was unreachable. DNS is over-blocked, so the dns-denied case above proves only that this container has no DNS at all." -run_case "deny-rule case: egress.default allow, one destination denied on udp/53" "$DENY_RULE_CONFIG" +run_case "deny-rule case: egress.default allow, one destination denied on udp/53" "$DENY_RULE_CONFIG" run_with_static_guest_ipv4 assert_blocked "a denied destination stayed reachable under egress.default allow. Entries from egress.deny are not reaching the chain, so a config written as allow-with-exceptions enforces nothing." -run_case "except case: allow the peer range except the resolver, probe the excluded address" "$EXCEPT_EXCLUDED_CONFIG" +run_case "except case: allow the peer range except the resolver, probe the excluded address" "$EXCEPT_EXCLUDED_CONFIG" run_with_static_guest_ipv4 assert_blocked "an address named in except was reachable through the rule that excludes it. The exclusion is being dropped, so the surrounding allow is wider than written." -run_case "except case: same policy, probe an address the exclusion does not cover" "$EXCEPT_SIBLING_CONFIG" +run_case "except case: same policy, probe an address the exclusion does not cover" "$EXCEPT_SIBLING_CONFIG" run_with_static_guest_ipv4 assert_allowed "an address inside the allowed range but outside except was unreachable. The exclusion is over-blocking, so the case above proves only that the whole rule failed to install." # The chain is first-match-wins, so a carve-out programmed as its own accept # rule would answer for the peer before the second rule's deny is reached. -run_case "shadow case: deny the peer's range except the peer, then deny the peer outright" "$EXCEPT_SHADOW_CONFIG" +run_case "shadow case: deny the peer's range except the peer, then deny the peer outright" "$EXCEPT_SHADOW_CONFIG" run_with_static_guest_ipv4 assert_blocked "a destination denied by its own rule was reachable because an earlier rule excluded it. An except carve-out is escaping the rule that declared it and accepting traffic a later deny names, which turns a deny into an allow." -run_case "shadow-control case: the same first rule with no second deny" "$EXCEPT_SHADOW_CONTROL_CONFIG" +run_case "shadow-control case: the same first rule with no second deny" "$EXCEPT_SHADOW_CONTROL_CONFIG" run_with_static_guest_ipv4 assert_allowed "an address excluded from a deny was unreachable under egress.default allow. The exclusion is not narrowing its own rule, so the shadow case above proves only that everything was blocked." -run_case "icmp case: egress.default deny, peer allowed on protocol icmp" "$ICMP_ALLOWED_CONFIG" +run_case "icmp case: egress.default deny, peer allowed on protocol icmp" "$ICMP_ALLOWED_CONFIG" run_with_static_guest_ipv4 assert_allowed "an ICMP echo to a peer allowed on protocol icmp was unreachable. Either the icmp selector never reached the chain, or the container cannot open a raw socket at all, which would make the icmp-denied case below pass without filtering anything." -run_case "icmp case: same icmp-only policy, probe tcp/443" "$ICMP_NO_TCP_CONFIG" +run_case "icmp case: same icmp-only policy, probe tcp/443" "$ICMP_NO_TCP_CONFIG" run_with_static_guest_ipv4 assert_blocked "tcp/443 succeeded while the policy allowed only icmp. The protocol selector is being dropped, so an icmp rule opens every transport to its destination." -run_case "icmp case: egress.default deny, peer allowed on tcp/443 only" "$ICMP_DENIED_CONFIG" +run_case "icmp case: egress.default deny, peer allowed on tcp/443 only" "$ICMP_DENIED_CONFIG" run_with_static_guest_ipv4 assert_blocked "an ICMP echo succeeded while the policy allowed only tcp/443. ICMP is not carrying the port selectors, so a policy written for one transport leaks another." -# The policy lets ICMP reach the peer, leaving the socket refusal as the only block. -# Ubuntu is used because Alpine ships no python3. +# Run the same raw and packet probes with ICMP permitted before denying it. python3 - "$SCRIPT_DIR/lib/io_uring_egress_probe.py" "$PEER_IP" "$PEER_CIDR" >"$IO_URING_CONFIG" <<'PY' -import json, sys +import json, shlex, sys probe, peer, cidr = sys.argv[1:] print(json.dumps({ "version": "1.0.0", "containerId": "CLI-LXC-GA-Egress-Io-Uring", "containment": "lxc", - "process": {"commandLine": "python3 -c '" + open(probe).read() + "' " + peer}, + "process": {"commandLine": "python3 -c " + shlex.quote(open(probe).read()) + " " + peer}, "lifecycle": {"destroyOnExit": True}, "lxc": {"distribution": "ubuntu", "release": "noble"}, - "network": {"egress": {"default": "deny", "allow": [{"to": [{"cidr": cidr}], "ports": [{"protocol": "icmp"}]}]}}, + "network": {"egress": {"default": "deny", "allow": [{"to": [{"cidr": cidr}], "ports": [{"protocol": "tcp", "port": 443}, {"protocol": "icmp"}]}]}}, })) PY -run_case "io_uring case: peer allowed on protocol icmp, echo sent from raw and packet sockets opened through io_uring" "$IO_URING_CONFIG" -# Hosts such as RHEL refuse io_uring_setup with EPERM, which leaves the workload -# no io_uring sockets to escape with. Any other setup error is a broken probe. -# The workload runs on a pty, so its lines end in \r\n. +run_case "raw/packet control: ICMP permitted through ordinary and io_uring sockets" "$IO_URING_CONFIG" run_with_static_guest_ipv4 +assert_allowed "the permitted raw and packet probes did not both receive replies; a denied probe would prove nothing." + +# Remove the pty carriage returns before matching the optional io_uring result. if grep -Fxq "MXC_IO_URING_SETUP_FAILED errno=1" <<<"${CASE_OUTPUT//$'\r'/}"; then log_io_uring_host_state - echo "io_uring case: the workload cannot open io_uring (EPERM), so there are no io_uring sockets to test." -else - assert_blocked "an echo from a raw or packet socket opened through io_uring reached the peer. io_uring hands the workload the sockets the capability drop takes away." + echo "SKIP: io_uring permitted-flow check unavailable (EPERM); ordinary raw/packet control passed." +fi + +python3 - "$IO_URING_CONFIG" <<'PY' +import json, sys +path = sys.argv[1] +config = json.load(open(path)) +config["network"]["egress"]["allow"][0]["ports"] = [{"protocol": "tcp", "port": 443}] +with open(path, "w") as stream: + json.dump(config, stream) +PY +run_case "raw/packet deny: ICMP denied through ordinary and io_uring sockets" "$IO_URING_CONFIG" run_with_static_guest_ipv4 +assert_blocked "a raw or packet socket bypassed the configured ICMP deny." +if grep -Fxq "MXC_IO_URING_SETUP_FAILED errno=1" <<<"${CASE_OUTPUT//$'\r'/}"; then + echo "SKIP: io_uring denied-flow check unavailable (EPERM); ordinary raw/packet deny passed." fi -run_case "port-range case: peer allowed on tcp/440-445, probe tcp/443" "$PORT_RANGE_INSIDE_CONFIG" +run_case "port-range case: peer allowed on tcp/440-445, probe tcp/443" "$PORT_RANGE_INSIDE_CONFIG" run_with_static_guest_ipv4 assert_allowed "a port inside the allowed range 440-445 was unreachable. endPort is not reaching the chain, so a range selector blocks the traffic it was written to permit." -run_case "port-range case: peer allowed on tcp/444-446, probe tcp/443" "$PORT_RANGE_OUTSIDE_CONFIG" +run_case "port-range case: peer allowed on tcp/444-446, probe tcp/443" "$PORT_RANGE_OUTSIDE_CONFIG" run_with_static_guest_ipv4 assert_blocked "tcp/443 succeeded while the allowed range started at 444. The range bounds are not enforced, so the case above proves only that some rule installed." -run_case "port-range case: peer allowed on tcp/438-442, probe tcp/443" "$PORT_RANGE_ABOVE_CONFIG" +run_case "port-range case: peer allowed on tcp/438-442, probe tcp/443" "$PORT_RANGE_ABOVE_CONFIG" run_with_static_guest_ipv4 assert_blocked "tcp/443 succeeded while the allowed range ended at 442. The range's upper bound is not reaching the chain, so a range opens every port above its start." -run_case "protocol-any case: peer allowed on protocol any, probe tcp/443" "$ANY_TCP_CONFIG" +run_case "protocol-any case: peer allowed on protocol any, probe tcp/443" "$ANY_TCP_CONFIG" run_with_static_guest_ipv4 assert_allowed "tcp/443 was unreachable under protocol any, so the any selector is not reaching the chain." -run_case "protocol-any case: same policy, probe icmp" "$ANY_ICMP_CONFIG" +run_case "protocol-any case: same policy, probe icmp" "$ANY_ICMP_CONFIG" run_with_static_guest_ipv4 assert_allowed "an ICMP echo was unreachable under protocol any. The any selector is being lowered to the transports alone, so it is narrower than written." -run_case "protocol-any case: same policy, probe udp/$PEER_UDP_PORT" "$ANY_UDP_UNSCOPED_CONFIG" +run_case "protocol-any case: same policy, probe udp/$PEER_UDP_PORT" "$ANY_UDP_UNSCOPED_CONFIG" run_with_static_guest_ipv4 assert_allowed "udp/$PEER_UDP_PORT was unreachable under protocol any carrying no port. An unscoped any is being lowered to a protocol list that omits udp, which the port-scoped cases below would not catch." -run_case "protocol-any case: peer allowed on any port 443, probe tcp/443" "$ANY_PORT_MATCH_CONFIG" +run_case "protocol-any case: peer allowed on any port 443, probe tcp/443" "$ANY_PORT_MATCH_CONFIG" run_with_static_guest_ipv4 assert_allowed "tcp/443 was unreachable while protocol any allowed port 443, so the tcp/udp fan-out is not reaching the chain." -run_case "protocol-any case: peer allowed on any port 444, probe tcp/443" "$ANY_WRONG_PORT_CONFIG" +run_case "protocol-any case: peer allowed on any port 444, probe tcp/443" "$ANY_WRONG_PORT_CONFIG" run_with_static_guest_ipv4 assert_blocked "tcp/443 succeeded while protocol any allowed only port 444. The fan-out drops the port selector, so any carrying a port opens every port." -run_case "protocol-any case: peer allowed on any port $PEER_UDP_PORT, probe udp/$PEER_UDP_PORT" "$ANY_UDP_CONFIG" +run_case "protocol-any case: peer allowed on any port $PEER_UDP_PORT, probe udp/$PEER_UDP_PORT" "$ANY_UDP_CONFIG" run_with_static_guest_ipv4 assert_allowed "udp/$PEER_UDP_PORT was unreachable while protocol any allowed that port. Only the TCP half of the fan-out is reaching the chain, so the tcp cases above prove nothing about udp." -run_case "protocol-any case: peer allowed on any port 8054, probe udp/$PEER_UDP_PORT" "$ANY_UDP_WRONG_PORT_CONFIG" +run_case "protocol-any case: peer allowed on any port 8054, probe udp/$PEER_UDP_PORT" "$ANY_UDP_WRONG_PORT_CONFIG" run_with_static_guest_ipv4 assert_blocked "udp/$PEER_UDP_PORT succeeded while protocol any allowed only port 8054. The UDP half of the fan-out ignores the port selector." -echo "PASS: egress rules filtered by destination, by port, by port range, by protocol, by resolver, by deny rule, and by exclusion, no exclusion answered for a destination a later rule denied, and no socket opened through io_uring reached the peer." +echo "PASS: egress rules filtered by destination, port, port range, protocol, resolver, deny precedence, and exclusion." echo "LXC egress enforcement test complete." diff --git a/tests/scripts/run_lxc_network_ipv6_cidr_test.sh b/tests/scripts/run_lxc_network_ipv6_cidr_test.sh index 6e82d3113..960d8d69f 100644 --- a/tests/scripts/run_lxc_network_ipv6_cidr_test.sh +++ b/tests/scripts/run_lxc_network_ipv6_cidr_test.sh @@ -1,175 +1,5 @@ #!/bin/bash -# LXC IPv6 + CIDR network filtering test -# -# Exercises tests/configs/lxc_network_ipv6_cidr.json, whose directional rules -# carry IPv4 and IPv6 CIDRs, including single-address prefixes. Assertions are on -# firewall setup rather than on whether the container reaches the network: -# reachability depends on the host's uplink, but rule programming does not. -# -# A misrouted address family is a hard failure, not a silent one -- -# `run_firewall_command` returns Err when iptables/ip6tables rejects a rule, so -# handing an IPv6 CIDR to iptables (or a v4 CIDR to ip6tables) aborts setup and -# is caught here. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" -LXC_EXEC="$REPO_DIR/src/target/release/lxc-exec" - -if [ ! -f "$LXC_EXEC" ]; then - LXC_EXEC="$REPO_DIR/src/target/debug/lxc-exec" -fi - -# An honest skip for a missing prerequisite: exit 77 so run_lxc_all_tests.sh -# records SKIPPED rather than PASS. A suite that could not run must not look green. -SKIP_EXIT=77 -skip() { - echo "SKIP: $1" - exit "$SKIP_EXIT" -} - -[ "$(id -u)" -eq 0 ] || skip "requires root for iptables/ip6tables and LXC." -command -v iptables >/dev/null 2>&1 || skip "iptables is not installed." -command -v ip6tables >/dev/null 2>&1 || skip "ip6tables is not installed." -command -v lxc-create >/dev/null 2>&1 || skip "LXC (lxc-create) is not installed." -command -v python3 >/dev/null 2>&1 || skip "python3 is not installed." -[ -f "$LXC_EXEC" ] || skip "lxc-exec binary not built; run build.sh first." - -CONFIG="$REPO_DIR/tests/configs/lxc_network_ipv6_cidr.json" -EXPECTED_HOSTS=( - "140.82.112.0/20" - "2606:50c0::/32" - "2606:50c0:8000::153/128" - "10.0.0.0/8" - "2001:db8::/32" - "fe80::1/128" -) - -fail() { - echo "FAIL: $1" - exit 1 -} - -# shellcheck source=lib/chain_name.sh -. "$SCRIPT_DIR/lib/chain_name.sh" - -assert_programmed_rule() { - local table="$1" dest="$2" target="$3" - # The --debug log emits one line per destination rule actually generated, - # derived from the built rule args. Asserting it here fails if - # destination-rule emission is deleted while chain/default/hook logging is - # kept -- the exact vacuity flagged in review. This inspects the rule - # contents while the chain is being programmed rather than only checking - # post-run cleanup. - if ! grep -Fq "Programmed $table rule: -A $CHAIN_NAME -d $dest -j $target" <<<"$OUTPUT"; then - fail "expected $table rule for '$dest' -> $target was not programmed." - fi -} - -# Compared against a snapshot taken before the run, so chains left behind by an -# earlier failed run are not blamed on this one. -assert_no_new_mxc_chains() { - local tool="$1" before="$2" after="" leaked="" chain - # Captured before iterating rather than piped in from a process - # substitution, whose exit status is not the loop's. A failed enumeration - # would otherwise read as zero chains and pass this assertion while - # verifying nothing. - if ! after="$(mxc_chains "$tool")"; then - fail "could not enumerate $tool chains, so cleanup was not verified." - fi - while IFS= read -r chain; do - [ -n "$chain" ] || continue - grep -Fxq "$chain" <<<"$before" || leaked="$leaked $chain" - done <<<"$after" - if [ -n "$leaked" ]; then - fail "$tool chain(s) left behind after lxc-exec completed:$leaked" - fi -} - -assert_firewall_chain_cleaned_up() { - assert_no_new_mxc_chains iptables "$MXC_CHAINS_BEFORE_V4" - assert_no_new_mxc_chains ip6tables "$MXC_CHAINS_BEFORE_V6" -} - -load_config_hosts() { - python3 -c 'import json, sys; e=json.load(open(sys.argv[1]))["network"]["egress"]; assert e["default"] == "deny"; [print(label + "\t" + dest["cidr"]) for label in ("allow", "deny") for rule in e[label] for dest in rule["to"]]' "$CONFIG" -} - -CONFIG_HOST_LINES="$(load_config_hosts)" || fail "could not read the directional rules." -mapfile -t CONFIG_HOSTS <<<"$CONFIG_HOST_LINES" -if [ "${#CONFIG_HOSTS[@]}" -ne "${#EXPECTED_HOSTS[@]}" ]; then - fail "config host count ${#CONFIG_HOSTS[@]} does not match expected count ${#EXPECTED_HOSTS[@]}." -fi -for i in "${!EXPECTED_HOSTS[@]}"; do - expected="${EXPECTED_HOSTS[$i]}" - if [ "$i" -lt 3 ]; then label=allow; else label=deny; fi - found=0 - for actual in "${CONFIG_HOSTS[@]}"; do - if [ "$actual" = "$label"$'\t'"$expected" ]; then - found=1 - break - fi - done - if [ "$found" -ne 1 ]; then - fail "expected host '$expected' is missing from $CONFIG." - fi -done - -echo "Running LXC IPv6/CIDR network filtering test..." - -# The container command may fail on a host with no outbound route; the firewall -# assertions below are what this test is about. -MXC_CHAINS_BEFORE_V4="$(mxc_chains iptables)" -MXC_CHAINS_BEFORE_V6="$(mxc_chains ip6tables)" -OUTPUT=$("$LXC_EXEC" --debug "$CONFIG" 2>&1 || true) -echo "$OUTPUT" - -derive_chain_name "$OUTPUT" - -# Every directional destination must survive validation and rule programming. -for host in "${EXPECTED_HOSTS[@]}"; do - if echo "$OUTPUT" | grep -Fq "Warning: could not resolve host '$host'"; then - fail "host '$host' was not resolved." - fi -done - -# Inspect the actual destination rules that were generated -- not merely the -# absence of an unresolved-host warning. Each allow entry must yield an ACCEPT -# rule and each block entry a DROP rule, in the correct family's table, so that -# deleting destination-rule emission fails this test. -assert_programmed_rule iptables "140.82.112.0/20" ACCEPT -assert_programmed_rule ip6tables "2606:50c0::/32" ACCEPT -assert_programmed_rule ip6tables "2606:50c0:8000::153/128" ACCEPT -assert_programmed_rule iptables "10.0.0.0/8" DROP -assert_programmed_rule ip6tables "2001:db8::/32" DROP -assert_programmed_rule ip6tables "fe80::1/128" DROP - -# A rejected rule aborts setup. -if echo "$OUTPUT" | grep -qE "^(ip6?tables) .* failed:|Firewall setup failed:"; then - fail "iptables/ip6tables rejected a rule." -fi - -if ! echo "$OUTPUT" | grep -q "Default network policy: DROP"; then - fail "default-deny policy was not applied." -fi - -# The v6 half is the point of the test: if ip6tables is unusable the v6 rules -# are skipped with a warning, which would make this a v4-only run. -if echo "$OUTPUT" | grep -q "IPv6 firewall rule(s) not applied"; then - fail "IPv6 rules were skipped; ip6tables is unusable on this host." -fi - -# The OUTPUT hook is what puts the chain on the container's egress path; a run -# that skipped it enforces nothing, so PASS must require it. Fail on the -# skipped-hook warning and require the positive install confirmation. -if echo "$OUTPUT" | grep -Fq "Skipping the OUTPUT hook"; then - fail "OUTPUT hook was skipped; the container network namespace was not found." -fi -if ! echo "$OUTPUT" | grep -Fq "OUTPUT hook installed"; then - fail "OUTPUT hook installation was not confirmed." -fi - -assert_firewall_chain_cleaned_up - -echo "PASS: IPv6 and CIDR entries were validated and programmed." -echo "LXC IPv6/CIDR network filtering test complete." +exec bash "$SCRIPT_DIR/run_lxc_network_cidr_boundary_test.sh" lxc_network_ipv6_cidr diff --git a/tests/scripts/run_lxc_network_no_network_test.sh b/tests/scripts/run_lxc_network_no_network_test.sh index 586523bb9..7ae6d687d 100644 --- a/tests/scripts/run_lxc_network_no_network_test.sh +++ b/tests/scripts/run_lxc_network_no_network_test.sh @@ -19,6 +19,9 @@ if [ ! -f "$LXC_EXEC" ]; then LXC_EXEC="$REPO_DIR/src/target/debug/lxc-exec" fi +# shellcheck source=lib/lxc_static_network.sh +. "$SCRIPT_DIR/lib/lxc_static_network.sh" + SKIP_EXIT=77 skip() { echo "SKIP: $1" @@ -78,7 +81,7 @@ run_config() { local label="$1" config="$2" status=0 echo "--- $label ---" set +e - CASE_OUTPUT=$("$LXC_EXEC" --debug "$config" 2>&1) + CASE_OUTPUT=$("${@:3}" "$config" 2>&1) status=$? set -e echo "$CASE_OUTPUT" @@ -107,10 +110,10 @@ assert_blocked() { echo "Running LXC omitted-network-section test..." -run_config "positive control: request with explicit egress allow to $PROBE_ADDRESS" "$ALLOW_CONFIG" +run_config "positive control: request with explicit egress allow to $PROBE_ADDRESS" "$ALLOW_CONFIG" run_with_static_guest_ipv4 assert_allowed "an explicitly allowed destination was unreachable on the positive control. The second run's blocked result would prove nothing, so this test fails rather than proceeding." -run_config "case under test: request with network section omitted entirely" "$OMIT_CONFIG" +run_config "case under test: request with network section omitted entirely" "$OMIT_CONFIG" "$LXC_EXEC" --debug assert_blocked "the workload reached $PROBE_ADDRESS under a request with no network section. The supported contract (docs/schema.md) states that omitted network policy selects directional deny defaults." echo "PASS: a request with the network section omitted cannot reach the network." diff --git a/tests/scripts/run_lxc_network_preserve_policy_test.sh b/tests/scripts/run_lxc_network_preserve_policy_test.sh index b7f60b819..e859898ab 100644 --- a/tests/scripts/run_lxc_network_preserve_policy_test.sh +++ b/tests/scripts/run_lxc_network_preserve_policy_test.sh @@ -1,101 +1,265 @@ #!/bin/bash -# LXC preserved-policy lifecycle test -# -# `preservePolicy` promises the network policy outlives the run. No other -# script asserts that promise, and asserting it needs the kernel rather than a -# log line: the runner can log that it skipped removal and have the rules -# removed anyway on its way out. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" -LXC_EXEC="$REPO_DIR/src/target/release/lxc-exec" - -if [ ! -f "$LXC_EXEC" ]; then +LXC_EXEC="${LXC_EXEC:-$REPO_DIR/src/target/release/lxc-exec}" +if [ "$LXC_EXEC" = "$REPO_DIR/src/target/release/lxc-exec" ] && [ ! -f "$LXC_EXEC" ]; then LXC_EXEC="$REPO_DIR/src/target/debug/lxc-exec" fi -CONFIG="$REPO_DIR/tests/configs/lxc_network_preserve_policy.json" -CONTAINER="CLI-LXC-Net-Preserve" - -SKIP_EXIT=77 -skip() { - echo "SKIP: $1" - exit "$SKIP_EXIT" -} - fail() { echo "FAIL: $1" exit 1 } -# shellcheck source=lib/chain_name.sh -. "$SCRIPT_DIR/lib/chain_name.sh" +# shellcheck source=lib/lxc_peer_listener.sh +. "$SCRIPT_DIR/lib/lxc_peer_listener.sh" -[ "$(id -u)" -eq 0 ] || skip "requires root for iptables/ip6tables and LXC." -command -v iptables >/dev/null 2>&1 || skip "iptables is not installed." -command -v ip6tables >/dev/null 2>&1 || skip "ip6tables is not installed." -command -v lxc-create >/dev/null 2>&1 || skip "LXC (lxc-create) is not installed." -[ -f "$LXC_EXEC" ] || skip "lxc-exec binary not built; run build.sh first." - -CHAIN_NAME="" - -# This test deliberately leaves a container behind, so it owns the container's -# removal on every exit path rather than the runner. The policy it preserves -# lives in that container's network namespace and goes with it. +CONTAINER="mxc-policy-lifecycle-$$" +CONFIG="$REPO_DIR/tests/configs/.lxc-policy-lifecycle-$$.json" +RUN_LOG="$REPO_DIR/tests/configs/.lxc-policy-lifecycle-$$.log" +LISTENER_LOG="$REPO_DIR/tests/configs/.lxc-policy-listener-$$.log" +CONTROL_LOG="$REPO_DIR/tests/configs/.lxc-policy-control-$$.log" +LISTENER_PID="" +CONTROL_PID="" +RUN_PID="" cleanup() { - lxc-destroy -n "$CONTAINER" -f >/dev/null 2>&1 || true + local status=$? + if [ -n "$RUN_PID" ]; then + kill -TERM "$RUN_PID" 2>/dev/null || true + wait "$RUN_PID" 2>/dev/null || true + fi + if [ -n "$LISTENER_PID" ]; then + kill "$LISTENER_PID" 2>/dev/null || true + wait "$LISTENER_PID" 2>/dev/null || true + fi + if [ -n "$CONTROL_PID" ]; then + kill "$CONTROL_PID" 2>/dev/null || true + wait "$CONTROL_PID" 2>/dev/null || true + fi + if lxc-info -n "$CONTAINER" >/dev/null 2>&1; then + python3 - "$CONFIG" <<'PY' +import json, sys +path = sys.argv[1] +config = json.load(open(path)) +config["process"]["commandLine"] = "true" +config["lifecycle"] = {"destroyOnExit": True, "preservePolicy": False} +with open(path, "w") as stream: + json.dump(config, stream) +PY + "$LXC_EXEC" "$CONFIG" >/dev/null 2>&1 || { echo "FAIL: MXC could not clean up the policy fixture."; status=1; } + fi + rm -f "$CONFIG" "$RUN_LOG" "$LISTENER_LOG" "$CONTROL_LOG" + exit "$status" } trap cleanup EXIT -# Guards against a stale container from an earlier failed run being mistaken -# for this run's work. -lxc-destroy -n "$CONTAINER" -f >/dev/null 2>&1 || true +# Reserve a static guest address in the bridge's subnet; DHCP is not exempted. +read -r HOST_IP GUEST_CIDR < <( + ip -j -4 address show dev lxcbr0 | python3 -c ' +import ipaddress, json, sys +address = json.load(sys.stdin)[0]["addr_info"][0] +network = ipaddress.ip_network(f"{address['\''local'\'']}/{address['\''prefixlen'\'']}", strict=False) +print(address["local"], f"{network.broadcast_address - 2}/{network.prefixlen}") +') +[ -n "$HOST_IP" ] && [ -n "$GUEST_CIDR" ] || fail "could not address the test guest." +DENIED_PORT=$((20000 + $$ % 10000)) +ALLOWED_PORT=$((DENIED_PORT + 1)) -echo "Running LXC preserved-policy test..." +python3 -m http.server "$DENIED_PORT" --bind "$HOST_IP" >"$LISTENER_LOG" 2>&1 & +LISTENER_PID=$! +python3 -m http.server "$ALLOWED_PORT" --bind "$HOST_IP" >"$CONTROL_LOG" 2>&1 & +CONTROL_PID=$! +await_peer_tcp "$HOST_IP" "$DENIED_PORT" || fail "the denied-port listener is unreachable." +await_peer_tcp "$HOST_IP" "$ALLOWED_PORT" || fail "the allowed-port listener is unreachable." +peer_listener_running "$LISTENER_PID" && peer_listener_running "$CONTROL_PID" \ + || fail "a test listener did not start." -OUTPUT=$("$LXC_EXEC" --debug "$CONFIG" 2>&1 || true) -echo "$OUTPUT" +for dhcp in allow deny; do + BEFORE_NFT="$(nft -s list ruleset)" + python3 - "$CONTAINER" "$dhcp" >"$CONFIG" <<'PY' +import json, sys +egress = {"default": "allow"} +if sys.argv[2] == "deny": + egress["deny"] = [{"ports": [{"protocol": "udp", "port": 67}]}] +print(json.dumps({ + "version": "1.0.0", + "containerId": sys.argv[1], + "containment": "lxc", + "process": {"commandLine": "true"}, + "lifecycle": {"destroyOnExit": False, "preservePolicy": True}, + "lxc": {"distribution": "alpine", "release": "3.23"}, + "network": {"egress": egress, "ingress": {"default": "deny"}} +})) +PY + if ! "$LXC_EXEC" "$CONFIG" >"$RUN_LOG" 2>&1; then + cat "$RUN_LOG" + fail "the fresh DHCP=$dhcp container did not start." + fi + INIT_PID="$(lxc-info -n "$CONTAINER" -p -H)" -if ! grep -Fq "MXC_PRESERVE_RAN" <<<"$OUTPUT"; then - fail "the container command did not run, so nothing below is being verified." -fi + # Stop an image-started client before clearing its lease. + lxc-attach -n "$CONTAINER" -- sh -c 'killall udhcpc; status=$?; [ "$status" -le 1 ]' \ + >>"$RUN_LOG" 2>&1 || { cat "$RUN_LOG"; fail "could not stop the image DHCP client."; } + nsenter -t "$INIT_PID" -n ip -4 address flush dev eth0 + if lxc-attach -n "$CONTAINER" -- udhcpc -i eth0 -n -q -t 3 -T 2 >>"$RUN_LOG" 2>&1; then + DHCP_EXIT=0 + else + DHCP_EXIT=$? + fi + LEASE="$(nsenter -t "$INIT_PID" -n ip -o -4 address show dev eth0 scope global)" + if [ "$dhcp" = allow ]; then + if [ "$DHCP_EXIT" -ne 0 ] || [ -z "$LEASE" ]; then + cat "$RUN_LOG" + fail "permitted DHCP did not acquire an IPv4 lease with ingress default deny." + fi + elif [ "$DHCP_EXIT" -ne 1 ] || [ -n "$LEASE" ]; then + cat "$RUN_LOG" + fail "explicitly denied DHCP must fail to lease and leave eth0 without IPv4; client exit=$DHCP_EXIT, address=$LEASE." + fi -derive_chain_name "$OUTPUT" + python3 - "$CONFIG" <<'PY' +import json, sys +path = sys.argv[1] +config = json.load(open(path)) +config["lifecycle"] = {"destroyOnExit": True, "preservePolicy": False} +with open(path, "w") as stream: + json.dump(config, stream) +PY + if ! "$LXC_EXEC" "$CONFIG" >"$RUN_LOG" 2>&1; then + cat "$RUN_LOG" + fail "MXC could not destroy the DHCP=$dhcp container." + fi + CONTAINERS="$(lxc-ls -1)" + grep -Fxq "$CONTAINER" <<<"$CONTAINERS" && fail "DHCP=$dhcp cleanup left the container defined." + [ "$(nft -s list ruleset)" = "$BEFORE_NFT" ] \ + || fail "DHCP=$dhcp left host filtering after destruction." + echo "PASS: DHCP=$dhcp acquired the expected IPv4 lease state." +done -# The positive control. Were the chain never installed, every assertion below -# would be asserting the absence of something that was never there. -if ! grep -Fq "OUTPUT hook installed" <<<"$OUTPUT"; then - fail "no OUTPUT hook was installed, so this run never had a policy to preserve." -fi +for preserve in false true; do + for destroy in false true; do + BEFORE_V4="$(iptables-save)" + BEFORE_V6="$(ip6tables-save)" + BEFORE_NFT="$(nft -s list ruleset)" + python3 - "$CONTAINER" "$preserve" "$destroy" "$HOST_IP" "$DENIED_PORT" >"$CONFIG" <<'PY' +import json, sys +container, preserve, destroy, host, port = sys.argv[1:] +print(json.dumps({ + "version": "1.0.0", + "containerId": container, + "containment": "lxc", + "process": {"commandLine": "sh -c 'cat /proc/self/status; echo MXC_POLICY_READY; while [ ! -f /mxc-policy-finish ]; do sleep 1; done'"}, + "lifecycle": {"preservePolicy": preserve == "true", "destroyOnExit": destroy == "true"}, + "lxc": {"distribution": "alpine", "release": "3.23"}, + "network": {"egress": { + "default": "allow", + "deny": [{"to": [{"cidr": host + "/32"}], "ports": [{"protocol": "tcp", "port": int(port)}]}] + }} +})) +PY + : >"$RUN_LOG" + "$LXC_EXEC" "$CONFIG" >"$RUN_LOG" 2>&1 & + RUN_PID=$! + for ((attempt = 0; attempt < 600; attempt++)); do + grep -Fq MXC_POLICY_READY "$RUN_LOG" && break + kill -0 "$RUN_PID" 2>/dev/null || { cat "$RUN_LOG"; fail "the policy workload exited before readiness."; } + sleep 0.5 + done + grep -Fq MXC_POLICY_READY "$RUN_LOG" || fail "the policy workload never became ready." + INIT_PID="$(lxc-info -n "$CONTAINER" -p -H)" + STARTUP_COMMAND="$(sed -n 's/^lxc\.net\.0\.script\.up[[:space:]]*=[[:space:]]*//p' "/var/lib/lxc/$CONTAINER/config")" + SCRIPT="$(python3 -c 'import shlex, sys; args = shlex.split(sys.argv[1]); print(args[-1] if args else "")' "$STARTUP_COMMAND")" + ATTACH_STATUS="$(lxc-attach -n "$CONTAINER" -- cat /proc/self/status)" + for status in "$(<"/proc/$INIT_PID/status")" "$ATTACH_STATUS" "$(<"$RUN_LOG")"; do + MASK="$(awk '$1 == "CapEff:" { print $2; exit }' <<<"$status" | tr -d '\r')" + [ -n "$MASK" ] || fail "init, attachment, or workload reported no effective capability mask." + (( (16#$MASK & 0x3000) == 0x3000 )) \ + || fail "init, attachment, or workload lost effective CAP_NET_ADMIN or CAP_NET_RAW." + done -if ! lxc-info -n "$CONTAINER" >/dev/null 2>&1; then - fail "destroyOnExit was false, but the container was destroyed." -fi + # Enter init's network namespace without depending on guest-installed tools. + nsenter -t "$INIT_PID" -n ip address replace "$GUEST_CIDR" dev eth0 + nsenter -t "$INIT_PID" -n ip link set eth0 up + nsenter -t "$INIT_PID" -n ip route replace default via "$HOST_IP" + nsenter -t "$INIT_PID" -n iptables -F + nsenter -t "$INIT_PID" -n iptables -X + nsenter -t "$INIT_PID" -n iptables -P OUTPUT ACCEPT + nsenter -t "$INIT_PID" -n ip6tables -F + nsenter -t "$INIT_PID" -n ip6tables -X + nsenter -t "$INIT_PID" -n ip6tables -P OUTPUT ACCEPT -# The preserved rules live in the container's network namespace, so reaching -# them means entering it. `lxc-info -p -H` prints the init PID alone. -INIT_PID="$(lxc-info -n "$CONTAINER" -p -H 2>/dev/null || true)" -if [ -z "$INIT_PID" ] || [ "$INIT_PID" = "-1" ]; then - fail "could not read the init PID of the preserved container." -fi + nsenter -t "$INIT_PID" -n timeout 3 bash -c "exec 3<>/dev/tcp/$HOST_IP/$ALLOWED_PORT" \ + || fail "the guest cannot reach the allowed listener; a denied connection would prove nothing." + if nsenter -t "$INIT_PID" -n timeout 3 bash -c "exec 3<>/dev/tcp/$HOST_IP/$DENIED_PORT"; then + fail "flushing the guest firewall bypassed the configured egress deny." + fi -in_ns() { - nsenter -t "$INIT_PID" -n "$@" -} + lxc-attach -n "$CONTAINER" -- touch /mxc-policy-finish + if ! wait "$RUN_PID"; then + cat "$RUN_LOG" + fail "the lifecycle run failed." + fi + RUN_PID="" -if ! in_ns iptables -S "$CHAIN_NAME" >/dev/null 2>&1; then - fail "preservePolicy was set, but iptables chain '$CHAIN_NAME' was removed." -fi + if [ "$destroy" = true ]; then + CONTAINERS="$(lxc-ls -1)" + grep -Fxq "$CONTAINER" <<<"$CONTAINERS" && fail "destroyOnExit left the test container defined." + else + [ "$(lxc-info -n "$CONTAINER" -s -H)" = RUNNING ] \ + || fail "destroyOnExit=false did not leave the container running." + INIT_PID="$(lxc-info -n "$CONTAINER" -p -H)" + nsenter -t "$INIT_PID" -n timeout 3 bash -c "exec 3<>/dev/tcp/$HOST_IP/$ALLOWED_PORT" \ + || fail "the surviving container lost its allowed route." + if nsenter -t "$INIT_PID" -n timeout 3 bash -c "exec 3<>/dev/tcp/$HOST_IP/$DENIED_PORT"; then + [ "$preserve" = false ] || fail "preservePolicy=true removed the surviving container's deny." + else + [ "$preserve" = true ] || fail "preservePolicy=false retained the surviving container's deny." + fi + if [ "$preserve" = true ] && [ -n "$SCRIPT" ]; then + [ -f "$SCRIPT" ] || fail "preservePolicy=true removed the surviving container's startup script." + fi + fi -if ! in_ns ip6tables -S "$CHAIN_NAME" >/dev/null 2>&1; then - fail "preservePolicy was set, but ip6tables chain '$CHAIN_NAME' was removed." -fi - -# A surviving chain that nothing dispatches to is not a preserved policy. -if ! in_ns iptables -S OUTPUT 2>/dev/null | grep -Fq -- "$CHAIN_NAME"; then - fail "chain '$CHAIN_NAME' survived but no OUTPUT rule reaches it, so the container is unfiltered." -fi + if [ "$preserve" = false ] || [ "$destroy" = true ]; then + # iptables-save includes timestamps; compare only kernel rules. + [ "$(iptables-save | sed '/^#/d')" = "$(sed '/^#/d' <<<"$BEFORE_V4")" ] \ + || fail "IPv4 host rules survived policy teardown." + [ "$(ip6tables-save | sed '/^#/d')" = "$(sed '/^#/d' <<<"$BEFORE_V6")" ] \ + || fail "IPv6 host rules survived policy teardown." + [ "$(nft -s list ruleset)" = "$BEFORE_NFT" ] \ + || fail "host nftables rules survived policy teardown." + [ -z "$SCRIPT" ] || [ ! -e "$SCRIPT" ] || fail "the retired startup script still exists." + if [ "$destroy" = false ] && [ -n "$SCRIPT" ]; then + if grep -Fq "$STARTUP_COMMAND" "/var/lib/lxc/$CONTAINER/config"; then + fail "preservePolicy=false left the retired startup hook registered." + fi + fi + fi -echo "PASS: the policy and its OUTPUT hook outlived the run." -echo "LXC preserved-policy test complete." + if [ "$destroy" = false ]; then + python3 - "$CONFIG" <<'PY' +import json, sys +path = sys.argv[1] +config = json.load(open(path)) +config["process"]["commandLine"] = "sh -c 'rm -f /mxc-policy-finish; echo MXC_POLICY_REUSED'" +config["lifecycle"] = {"destroyOnExit": True, "preservePolicy": False} +with open(path, "w") as stream: + json.dump(config, stream) +PY + OUTPUT=$("$LXC_EXEC" "$CONFIG") || fail "MXC could not reuse and destroy its surviving container." + grep -Fq MXC_POLICY_REUSED <<<"$OUTPUT" || fail "the reused workload did not run." + CONTAINERS="$(lxc-ls -1)" + grep -Fxq "$CONTAINER" <<<"$CONTAINERS" && fail "reuse cleanup left the container defined." + [ "$(iptables-save | sed '/^#/d')" = "$(sed '/^#/d' <<<"$BEFORE_V4")" ] \ + || fail "IPv4 host rules leaked after reuse cleanup." + [ "$(ip6tables-save | sed '/^#/d')" = "$(sed '/^#/d' <<<"$BEFORE_V6")" ] \ + || fail "IPv6 host rules leaked after reuse cleanup." + [ "$(nft -s list ruleset)" = "$BEFORE_NFT" ] \ + || fail "host nftables rules leaked after reuse cleanup." + [ -z "$SCRIPT" ] || [ ! -e "$SCRIPT" ] || fail "reuse cleanup left the old startup script." + fi + echo "PASS: preservePolicy=$preserve destroyOnExit=$destroy." + done +done diff --git a/tests/scripts/run_lxc_network_reuse_loosen_test.sh b/tests/scripts/run_lxc_network_reuse_loosen_test.sh index 777997449..be398caae 100644 --- a/tests/scripts/run_lxc_network_reuse_loosen_test.sh +++ b/tests/scripts/run_lxc_network_reuse_loosen_test.sh @@ -26,6 +26,9 @@ if [ ! -f "$LXC_EXEC" ]; then LXC_EXEC="$REPO_DIR/src/target/debug/lxc-exec" fi +# shellcheck source=lib/lxc_static_network.sh +. "$SCRIPT_DIR/lib/lxc_static_network.sh" + SKIP_EXIT=77 skip() { echo "SKIP: $1" @@ -129,7 +132,7 @@ run_config() { local label="$1" config="$2" status=0 echo "--- $label ---" set +e - CASE_OUTPUT=$("$LXC_EXEC" --debug "$config" 2>&1) + CASE_OUTPUT=$("${@:3}" "$config" 2>&1) status=$? set -e echo "$CASE_OUTPUT" @@ -161,12 +164,12 @@ echo "Running LXC reuse policy-loosening test..." # Run 1: positive control on a fresh container id. If this fails, the probe # destination is unreachable from this host right now, and run 3's result would # be meaningless. -run_config "run 1 (positive control): fresh container, explicit egress allow to $PROBE_ADDRESS" "$CTRL_CONFIG" +run_config "run 1 (positive control): fresh container, explicit egress allow to $PROBE_ADDRESS" "$CTRL_CONFIG" run_with_static_guest_ipv4 assert_allowed "an explicitly allowed destination was unreachable on the positive control. Run 3 would prove nothing without a working allow path, so this test fails rather than proceeding." # Run 2: setup run on the reuse container id, with no network section. The # container must survive so run 3 reuses the same instance. -run_config "run 2 (setup): reuse id, no network section -- workload must be blocked" "$DENY_CONFIG" +run_config "run 2 (setup): reuse id, no network section -- workload must be blocked" "$DENY_CONFIG" "$LXC_EXEC" --debug assert_blocked "the workload reached $PROBE_ADDRESS on a request with no network section. The container was not isolated as expected, so run 3 would not be testing the loosening scenario." if ! lxc-info -n "$REUSE_CONTAINER" >/dev/null 2>&1; then @@ -181,7 +184,7 @@ echo "--- container survived run 2 and is still running as PID $INIT_PID ---" # Run 3: the case under test. Same reuse id as run 2, now with an explicit # egress allow. The workload must reach the allowed destination. -run_config "run 3 (case under test): same reuse id, explicit egress allow to $PROBE_ADDRESS" "$ALLOW_CONFIG" +run_config "run 3 (case under test): same reuse id, explicit egress allow to $PROBE_ADDRESS" "$ALLOW_CONFIG" run_with_static_guest_ipv4 assert_allowed "a container reused under an explicit egress allow could not reach $PROBE_ADDRESS. The workload was blocked despite the policy explicitly permitting the destination on the second run." # Reapplying policy means restarting the container, so the init PID must have diff --git a/tests/scripts/run_lxc_network_reuse_tighten_test.sh b/tests/scripts/run_lxc_network_reuse_tighten_test.sh index 246d93b21..eaeda3477 100644 --- a/tests/scripts/run_lxc_network_reuse_tighten_test.sh +++ b/tests/scripts/run_lxc_network_reuse_tighten_test.sh @@ -27,6 +27,9 @@ if [ ! -f "$LXC_EXEC" ]; then LXC_EXEC="$REPO_DIR/src/target/debug/lxc-exec" fi +# shellcheck source=lib/lxc_static_network.sh +. "$SCRIPT_DIR/lib/lxc_static_network.sh" + # Exit 77 is what run_lxc_all_tests.sh records as SKIPPED rather than PASS. SKIP_EXIT=77 skip() { @@ -104,7 +107,7 @@ run_config() { local label="$1" config="$2" status=0 echo "--- $label ---" set +e - CASE_OUTPUT=$("$LXC_EXEC" --debug "$config" 2>&1) + CASE_OUTPUT=$("${@:3}" "$config" 2>&1) status=$? set -e echo "$CASE_OUTPUT" @@ -133,7 +136,7 @@ assert_blocked() { echo "Running LXC reuse policy-tightening test..." -run_config "first run: policy allows tcp/443 to $PROBE_ADDRESS, container survives" "$ALLOW_CONFIG" +run_config "first run: policy allows tcp/443 to $PROBE_ADDRESS, container survives" "$ALLOW_CONFIG" run_with_static_guest_ipv4 assert_allowed "an explicitly allowed destination was unreachable on the first run. The container never had the access this test is about taking away, so the second run below proves nothing." if ! lxc-info -n "$CONTAINER" >/dev/null 2>&1; then @@ -150,7 +153,7 @@ if [ -z "$INIT_PID" ] || [ "$INIT_PID" = "-1" ]; then fi echo "--- container survived the first run and is still running as PID $INIT_PID ---" -run_config "second run: same container, policy permits no network" "$DENY_CONFIG" +run_config "second run: same container, policy permits no network" "$DENY_CONFIG" "$LXC_EXEC" --debug assert_blocked "a surviving container reused under a policy that permits no network still reached $PROBE_ADDRESS. It is still on the first run's topology, so tightening the policy on a reused container does nothing and the workload keeps access the current policy never granted." # Reapplying policy means restarting the container, so the init PID must have diff --git a/tests/scripts/run_lxc_network_test.sh b/tests/scripts/run_lxc_network_test.sh index e718add04..f90065e5b 100644 --- a/tests/scripts/run_lxc_network_test.sh +++ b/tests/scripts/run_lxc_network_test.sh @@ -32,6 +32,9 @@ fail() { # shellcheck source=lib/lxc_peer_listener.sh . "$SCRIPT_DIR/lib/lxc_peer_listener.sh" +# shellcheck source=lib/lxc_static_network.sh +. "$SCRIPT_DIR/lib/lxc_static_network.sh" + [ "$(id -u)" -eq 0 ] || skip "requires root for iptables and LXC." command -v iptables >/dev/null 2>&1 || skip "iptables is not installed." command -v lxc-create >/dev/null 2>&1 || skip "LXC (lxc-create) is not installed." @@ -134,7 +137,7 @@ assert any({"cidr": sys.argv[2]} in rule["to"] for rule in egress["allow"]) assert any({"cidr": sys.argv[3]} in rule["to"] for rule in egress["deny"]) PY -OUTPUT=$("$LXC_EXEC" "$CONFIG" 2>&1) || fail "lxc-exec failed: $OUTPUT" +OUTPUT=$(run_with_static_guest_ipv4 "$CONFIG" 2>&1) || fail "lxc-exec failed: $OUTPUT" echo "$OUTPUT" grep -Fq "MXC_NET_ALLOWED" <<<"$OUTPUT" \ || fail "the explicitly allowed peer was unreachable or the workload never ran." diff --git a/tests/scripts/run_lxc_unprivileged_test.sh b/tests/scripts/run_lxc_unprivileged_test.sh new file mode 100644 index 000000000..57605eb9d --- /dev/null +++ b/tests/scripts/run_lxc_unprivileged_test.sh @@ -0,0 +1,94 @@ +#!/bin/bash +# A reused container with an identity UID map must not run a workload. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" +LXC_EXEC="$REPO_DIR/src/target/release/lxc-exec" +if [ ! -f "$LXC_EXEC" ]; then + LXC_EXEC="$REPO_DIR/src/target/debug/lxc-exec" +fi + +fail() { + echo "FAIL: $1" + exit 1 +} + +[ "$(id -u)" -eq 0 ] || fail "requires root for LXC." +[ -f "$LXC_EXEC" ] || fail "lxc-exec not built; run build.sh first." + +CONTAINER="CLI-LXC-Unprivileged-$$" +WORK_DIR="$REPO_DIR/tests/output/lxc-unprivileged-$$" +mkdir -p "$WORK_DIR" + +cleanup() { + local status=$? + lxc-destroy -n "$CONTAINER" -f || status=1 + rm -rf "$WORK_DIR" + exit "$status" +} +trap cleanup EXIT + +case "$(uname -m)" in + x86_64) ARCH=amd64 ;; + aarch64) ARCH=arm64 ;; + *) fail "unsupported LXC image architecture: $(uname -m)" ;; +esac + +# Override only this container's defaults, without using the host bridge. +printf 'lxc.net.0.type = empty\n' > "$WORK_DIR/default.conf" +lxc-create -n "$CONTAINER" -f "$WORK_DIR/default.conf" -t download -- \ + --dist alpine --release 3.23 --arch "$ARCH" + +# Keep the host user namespace despite the configured shifted IDs. +cat >> "/var/lib/lxc/$CONTAINER/config" <<'EOF' +lxc.namespace.keep = user +lxc.idmap = u 0 100000 65536 +lxc.idmap = g 0 100000 65536 +EOF + +lxc-start -n "$CONTAINER" -d +lxc-wait -n "$CONTAINER" -s RUNNING -t 30 +UID_MAP="$(lxc-attach -n "$CONTAINER" -- cat /proc/self/uid_map)" +read -r inside outside count extra <<< "$UID_MAP" +[ "$inside $outside $count" = "0 0 4294967295" ] && [ -z "$extra" ] \ + || fail "fixture did not retain the identity UID map: $UID_MAP" +METADATA="$(lxc-ls --fancy --fancy-format NAME,UNPRIVILEGED --filter "^$CONTAINER$")" +awk -v name="$CONTAINER" '$1 == name && $2 == "true" { found = 1 } END { exit !found }' \ + <<< "$METADATA" || fail "fixture metadata did not report unprivileged: $METADATA" +lxc-stop -n "$CONTAINER" -k + +# Make the container's cat report a shifted root mapping. +rm "/var/lib/lxc/$CONTAINER/rootfs/bin/cat" +cat > "/var/lib/lxc/$CONTAINER/rootfs/bin/cat" <<'EOF' +#!/bin/sh +echo '0 100000 65536' +EOF +chmod +x "/var/lib/lxc/$CONTAINER/rootfs/bin/cat" + +cat > "$WORK_DIR/request.json" <&1)" || status=$? +echo "$OUTPUT" +[ "$status" -ne 0 ] || fail "MXC admitted a container with an identity UID map." +grep -Fq 'has no unprivileged root UID mapping.' <<< "$OUTPUT" \ + || fail "MXC failed without reporting the UID-map admission rejection." +[ ! -e "/var/lib/lxc/$CONTAINER/rootfs/mxc-admission-workload-ran" ] \ + || fail "the rejected container ran its workload." +STATE="$(lxc-info -n "$CONTAINER" -s -H)" +[ "$STATE" = STOPPED ] \ + || fail "MXC left the rejected reused container $STATE with destroyOnExit false." + +echo "PASS: MXC refused the live identity UID map before workload execution and stopped the reused container."