Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions .github/workflows/SDK.Dotnet.Test.Job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,22 @@ jobs:
sudo DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::Retries=3 install -y \
lxc lxc-templates lxc-utils iptables debootstrap uidmap bridge-utils

- name: Configure unprivileged LXC
if: matrix.os_label == 'linux'
shell: bash
run: |
set -euo pipefail
for file in /etc/lxc/default.conf /etc/subuid /etc/subgid; do
sudo cp -a "$file" "$RUNNER_TEMP/mxc-lxc-$(basename "$file")"
done
sudo bash -c 'echo "root:100000:65536" >> /etc/subuid'
sudo bash -c 'echo "root:100000:65536" >> /etc/subgid'

# Remove existing mappings before adding the test-only UID/GID range.
sudo sed -i '/idmap/d' /etc/lxc/default.conf
sudo bash -c 'echo "lxc.idmap = u 0 100000 65536" >> /etc/lxc/default.conf'
sudo bash -c 'echo "lxc.idmap = g 0 100000 65536" >> /etc/lxc/default.conf'

# The policy these tests use permits no network, so the container starts
# with no interface and the runner's Docker-set FORWARD policy cannot
# reach them.
Expand Down Expand Up @@ -133,6 +149,18 @@ jobs:
fi
done

- name: Restore LXC configuration
if: always() && matrix.os_label == 'linux'
shell: bash
run: |
set -euo pipefail
for file in /etc/lxc/default.conf /etc/subuid /etc/subgid; do
backup="$RUNNER_TEMP/mxc-lxc-$(basename "$file")"
if [ -f "$backup" ]; then
sudo mv "$backup" "$file"
fi
done

# This regression requires Linux and a usable Bubblewrap. Install it in
# the Linux lane and make an unavailable backend fail instead of skip.
- name: Install Bubblewrap
Expand Down
35 changes: 33 additions & 2 deletions .github/workflows/SDK.Integration.Test.Job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,8 @@ jobs:
# installs a similar but narrower set and sets the same skip
# env var for the same reason (compounded by 1ES Hosted Pool
# egress filtering).
packages: bubblewrap lxc lxc-utils dnsmasq-base iptables bridge-utils
version: 1.4
packages: bubblewrap lxc lxc-utils uidmap dnsmasq-base iptables bridge-utils
version: 1.5

- name: Verify or repair Linux containment dependencies
if: matrix.os_label == 'linux'
Expand All @@ -123,6 +123,7 @@ jobs:
bubblewrap
lxc
lxc-utils
uidmap
dnsmasq-base
iptables
bridge-utils
Expand All @@ -132,6 +133,8 @@ jobs:
bwrap
lxc-start
lxc-create
newuidmap
newgidmap
dnsmasq
iptables
brctl
Expand Down Expand Up @@ -174,6 +177,22 @@ jobs:
command -v "$command"
done

- name: Configure unprivileged LXC
if: matrix.os_label == 'linux'
shell: bash
run: |
set -euo pipefail
for file in /etc/lxc/default.conf /etc/subuid /etc/subgid; do
sudo cp -a "$file" "$RUNNER_TEMP/mxc-lxc-$(basename "$file")"
done
sudo bash -c 'echo "root:100000:65536" >> /etc/subuid'
sudo bash -c 'echo "root:100000:65536" >> /etc/subgid'

# Remove existing mappings before adding the test-only UID/GID range.
sudo sed -i '/idmap/d' /etc/lxc/default.conf
sudo bash -c 'echo "lxc.idmap = u 0 100000 65536" >> /etc/lxc/default.conf'
sudo bash -c 'echo "lxc.idmap = g 0 100000 65536" >> /etc/lxc/default.conf'

- name: Start LXC services (cache restore skips postinst hooks)
if: matrix.os_label == 'linux'
shell: bash
Expand Down Expand Up @@ -217,3 +236,15 @@ jobs:
else
npm test
fi

- name: Restore LXC configuration
if: always() && matrix.os_label == 'linux'
shell: bash
run: |
set -euo pipefail
for file in /etc/lxc/default.conf /etc/subuid /etc/subgid; do
backup="$RUNNER_TEMP/mxc-lxc-$(basename "$file")"
if [ -f "$backup" ]; then
sudo mv "$backup" "$file"
fi
done
25 changes: 25 additions & 0 deletions .github/workflows/lxc-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,20 @@ jobs:
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
lxc lxc-templates lxc-utils iptables debootstrap uidmap bridge-utils

- name: Configure unprivileged LXC
run: |
set -euo pipefail
for file in /etc/lxc/default.conf /etc/subuid /etc/subgid; do
sudo cp -a "$file" "$RUNNER_TEMP/mxc-lxc-$(basename "$file")"
done
sudo bash -c 'echo "root:100000:65536" >> /etc/subuid'
sudo bash -c 'echo "root:100000:65536" >> /etc/subgid'

# Remove existing mappings before adding the test-only UID/GID range.
sudo sed -i '/idmap/d' /etc/lxc/default.conf
sudo bash -c 'echo "lxc.idmap = u 0 100000 65536" >> /etc/lxc/default.conf'
sudo bash -c 'echo "lxc.idmap = g 0 100000 65536" >> /etc/lxc/default.conf'

# GitHub-hosted runners ship Docker, and Docker sets the IPv4 FORWARD
# policy to DROP. That breaks these tests twice over.
#
Expand Down Expand Up @@ -123,6 +137,17 @@ jobs:
MXC_LXC_TESTS_REQUIRE_EXECUTION: "1"
run: sudo --preserve-env=MXC_LXC_TESTS_REQUIRE_EXECUTION,PATH,HOME "$(command -v cargo)" test -p mxc-sdk --test streaming_lxc --test sdk_helpers

- name: Restore LXC configuration
if: always()
run: |
set -euo pipefail
for file in /etc/lxc/default.conf /etc/subuid /etc/subgid; do
backup="$RUNNER_TEMP/mxc-lxc-$(basename "$file")"
if [ -f "$backup" ]; then
sudo mv "$backup" "$file"
fi
done

- name: Show leftover firewall state on failure
if: failure()
run: |
Expand Down
111 changes: 68 additions & 43 deletions docs/backends/lxc/lxc-backend.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,34 +18,43 @@ Creates an LXC container to provide:

- **Process isolation** via Linux namespaces (PID, mount, network, user)
- **Filesystem isolation** via bind mounts with read-only/read-write/denied enforcement
- **Network isolation** via iptables rules inside the container's own network namespace
- **Network isolation** via host-owned nftables bridge rules on the container's network interfaces

## Prerequisites

- Linux kernel >= 2.6.32, or >= 3.12 to run unprivileged
- LXC >= 5.0 installed (`liblxc-dev` for building, `lxc-utils` for runtime)
- Root privileges, or [unprivileged LXC](#running-without-root) for a policy
that asks for no network.
Filtering egress or ingress installs iptables rules in the container's
network namespace, which requires root.
- Linux kernel >= 3.12.
- LXC >= 5.0 installed (`liblxc-dev` for building, `lxc-utils` for runtime).
- Bash and nftables (`nft`) for filtered networking.
- Host kernel support for nftables bridge filtering and connection tracking
(`CONFIG_NF_TABLES_BRIDGE` and `CONFIG_NF_CONNTRACK_BRIDGE`).
Installing the nftables package alone does not provide missing kernel support.
- Root access to run MXC and to configure LXC.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This contradicts "Running without root" (line 57), and this PR also dropped its chmod o+x ~ step, which is still needed on 0750 Ubuntu homes (run_lxc_rootless_no_network_test.sh:31-32 still does it).
Please say root is needed to configure LXC and filter network traffic and restore the chmod o+x ~ step?

Filtering egress or ingress installs rules in the host's network namespace,
which requires root.
- Valid UID and GID root mappings to run LXC containers unprivileged. See [LXC Getting Started](https://linuxcontainers.org/lxc/getting-started/#create-unprivileged-containers-as-root-with-shared-uid-and-gid-ranges).

MXC requires LXC containers to run unprivileged. After starting the container,
MXC reads `/proc/<pid>/uid_map` for the container's init process directly on the
host before running the requested command. MXC accepts the container when a
readable range maps container root to a nonzero parent UID.

### Installation

**Debian/Ubuntu:**
```bash
sudo apt install lxc lxc-utils liblxc-dev
sudo apt install lxc lxc-utils liblxc-dev nftables
```

**Fedora/RHEL:**
```bash
sudo dnf install lxc lxc-templates dnsmasq lxc-devel
sudo dnf install lxc lxc-templates dnsmasq lxc-devel nftables
```
RHEL needs EPEL enabled first β€” see
[Red Hat Enterprise Linux setup](#red-hat-enterprise-linux-setup).

**Arch Linux:**
```bash
sudo pacman -S lxc
sudo pacman -S lxc nftables
```

### Running without root
Expand All @@ -55,8 +64,6 @@ for unprivileged LXC. Everything else needs root.

1. Follow LXC's
[Create Unprivileged Containers as a User](https://linuxcontainers.org/lxc/getting-started/#create-unprivileged-containers-as-a-user).
2. Let the container's root user reach its files under your home directory:
`chmod o+x ~`.


### Container networking
Expand Down Expand Up @@ -155,13 +162,40 @@ not reachable through `process.env`.

## Network Policy

Directional `network.egress` rules govern port 53 like any other destination;
there is no implicit DNS exemption.

`preservePolicy` leaves the inbound and outbound chains in place after the run.
The chains live in the container's network namespace, so they last only as long
as the container keeps running; stopping or destroying it takes them with it. A
partially installed chain from a failed run is torn down regardless.
MXC filters traffic on the host-side interfaces of the container, not inside
the container. For each bridge-attached veth interface, MXC generates a
host-owned Bash script and registers it through `lxc.net.<index>.script.up`.
Existing interface-up hooks run first. The generated hook installs inbound
and outbound nftables bridge rules before container init starts.
Both IPv4 and IPv6 traffic pass through these rules. Unsupported interface
topologies or unavailable host filtering reject startup.

MXC does not remove `CAP_NET_ADMIN` or `CAP_NET_RAW` from container processes.
Init, attached commands, and workloads can use the capabilities their container
provides without controlling the host-owned filters. Flushing the guest's
firewall does not remove those filters.

Directional `network.egress` rules govern DNS and DHCP like other traffic;
neither receives an implicit exemption. DHCP replies are admitted only for
requests allowed by the policy. Denying DHCP prevents DHCP address assignment;
other permitted traffic needs an address and route configured separately.

`preservePolicy` retains installed filtering, startup hooks, and generated
scripts after the run, including when the container is stopped. Restarting
the retained container reinstalls filtering before init starts. A subsequent
MXC run replaces retained filtering with the new request's policy.

| `preservePolicy` | `destroyOnExit` | After the run |
|---|---|---|
| `false` | `false` | Leave the container running; remove owned filtering, hooks, and scripts. |
| `true` | `false` | Leave the container running; retain owned filtering, hooks, and scripts. |
| `false` | `true` | Destroy the container, then remove owned host filtering and support files. |
| `true` | `true` | Destroy the container, then remove owned host filtering and support files. |

A timeout or `kill()` stops the whole container; preservation still retains
its installed policy. Failed destruction reports an error and retains the
surviving container's filtering and support files. An incompletely installed
policy is not preserved.

### Proxy

Expand Down Expand Up @@ -275,18 +309,20 @@ container init, where nothing aimed at the host `lxc-attach` process reaches it.
LXC reads a run's network section only when the container starts. Omit
`containerId` for a generated name. The claim is released when the handle drops.

**Teardown is owed on every terminal path,** including a drop without `wait`:
the network chains come down, and the container is released. A reused container
**Teardown is owed on every terminal path,** including a drop without `wait`,
and follows the [network policy lifecycle settings](#network-policy).
A reused container
also has any stale proxy address mapping in its hosts file cleared before its
new workload launches. MXC checks for marked lines inside the running
container using `lxc-attach` and removes them if present. A failed check
refuses to run the workload.
A teardown failure is reported through `Sandbox::warnings`, which a bare drop
leaves no handle to read.

**The container stays up for as long as the handle lives.** `lxc-exec` is
short-lived by comparison. Weigh that against the threat model before embedding
streaming in a long-lived service.
**An ordinary run keeps the container up while the handle lives.** A timeout
or `kill()` can stop it earlier. `lxc-exec` is short-lived by comparison.
Weigh that against the threat model before embedding streaming in a long-lived
service.

## Building

Expand All @@ -313,7 +349,7 @@ streaming in a long-lived service.

2. **Install LXC and the pieces it needs at runtime.**
```bash
sudo dnf install -y lxc lxc-templates dnsmasq iptables
sudo dnf install -y lxc lxc-templates dnsmasq iptables nftables
```

Add `lxc-devel` as well if you are building MXC against liblxc rather than
Expand Down Expand Up @@ -342,24 +378,13 @@ These failures carry the `backend_error` code and exit `-1`.

## Limitations

- **The workload can send only ICMP Echo.** A `protocol: "icmp"` rule lets every
ICMP type through the firewall, but the command MXC runs can originate only Echo
requests, the ones `ping` sends.
- **The network restrictions bind only the command MXC runs and its children.**
MXC takes away that command's ability to change the firewall or send raw
packets, but the container's init process (PID 1) keeps both. The command
runs as root inside the container, so it can ask init to start a new process,
for example with `systemd-run` on Ubuntu. That process is not a child of the
command, and it can change the firewall and send raw packets.
- **Network access stops after about an hour.** The stock `lxcbr0` hands out
DHCP leases that last one hour, and MXC's firewall blocks the DHCP request the
container sends to renew one. When the lease expires the container loses its
address and the workload can no longer reach the network. A run that finishes
within the hour is unaffected.
- **IPv6 egress is not supported.** An `egress` rule naming an IPv6 destination
installs and reports success, but no IPv6 traffic reaches that destination.
Stock `lxcbr0` gives the container no IPv6 address, so this surfaces only on a
host that provides one.
- **Cross-container reply isolation has an unfixed defect.** A container that
adopts another container's IP and Ethernet addresses can receive that
container's permitted reply despite its own policy denying the endpoint.
This was reproduced with UDP on a shared bridge. The host filter recognizes
tracked replies without verifying that the receiving interface belongs to
the container that originated the request. Host-owned filtering does not
yet provide the required reply isolation between those containers.
- **A container reaches the proxy only while a run lasts.** See [Proxy](#proxy).
- **No container lifecycle operations.** LXC implements `ScriptRunner` (create-and-run execution) and
`SandboxBackend` (streaming over pipes or a caller-controlled PTY), not
Expand Down
8 changes: 5 additions & 3 deletions scripts/ci/prepare-linux-host.sh
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,8 @@ load_conntrack_module() {
}

install_lxc() {
if command -v lxc-start >/dev/null 2>&1; then
if command -v lxc-start >/dev/null 2>&1 &&
command -v nft >/dev/null 2>&1; then
return
fi

Expand All @@ -135,14 +136,14 @@ install_lxc() {
local packages
if [[ "$package_manager" == "apt-get" ]]; then
apt_update
packages=(lxc dnsmasq-base iptables bridge-utils)
packages=(lxc dnsmasq-base iptables nftables bridge-utils)
# Debian dropped lxc-utils; Ubuntu still ships it.
if apt-cache show lxc-utils >/dev/null 2>&1; then
packages+=(lxc-utils)
fi
else
install_epel
packages=(lxc lxc-templates dnsmasq iptables)
packages=(lxc lxc-templates dnsmasq iptables nftables)
fi
install_packages "${packages[@]}"
}
Expand Down Expand Up @@ -302,6 +303,7 @@ case "$backend" in
lxc)
install_lxc
command -v lxc-start
command -v nft
sudo -n true
# Package installs may not activate the AppArmor profile, which
# lxc-start needs.
Expand Down
5 changes: 5 additions & 0 deletions sdk/node/tests/integration/linux-process-container.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,8 @@ describe(`Linux LXC Container (schema ${schemaVersion})`, {
it('should mount readwrite filesystem path', async () => {
tempDir = createTempDir('mxc-lxc-test');
fs.writeFileSync(path.join(tempDir, 'test.txt'), 'original');
fs.chmodSync(tempDir, 0o777);
fs.chmodSync(path.join(tempDir, 'test.txt'), 0o666);
const policy = { filesystem: { readwritePaths: [tempDir] } };
const script = `cat ${tempDir}/test.txt && echo 'overwritten' > ${tempDir}/test.txt && cat ${tempDir}/test.txt`;
const result = await runLxc(script, policy, `lxc-rw-${schemaVersion}`);
Expand All @@ -135,6 +137,8 @@ describe(`Linux LXC Container (schema ${schemaVersion})`, {
it('should mount readonly filesystem path', async () => {
tempDir = createTempDir('mxc-lxc-test');
fs.writeFileSync(path.join(tempDir, 'data.txt'), 'readonly content');
fs.chmodSync(tempDir, 0o755);
fs.chmodSync(path.join(tempDir, 'data.txt'), 0o644);
const policy = { filesystem: { readonlyPaths: [tempDir] } };
const result = await runLxc(
`cat ${tempDir}/data.txt && echo 'Read succeeded'`,
Expand All @@ -147,6 +151,7 @@ describe(`Linux LXC Container (schema ${schemaVersion})`, {

it('should download file to writable mount', { skip: lxcNetworkSkipReason }, async () => {
tempDir = createTempDir('mxc-lxc-test');
fs.chmodSync(tempDir, 0o777);
const policy = {
filesystem: { readwritePaths: [tempDir] },
network: outboundNetwork(),
Expand Down
Loading
Loading