silkd, sdk/python, ci: find on /proc, glibc headroom, keepalive, images base, guest speedups

.github/workflows/build-os-images.yml

@@ -27,6 +27,10 @@ on:
         description: sha-pinned silkd carrier ref (empty = unchanged)
         type: string
         default: ""
+      base:
+        description: commit the last green images run built (empty = build all)
+        type: string
+        default: ""
       no_cache:
         type: boolean
         default: false
@@ -63,7 +67,7 @@ jobs:
           CARRIER_CHANGED: ${{ inputs.boot_changed || inputs.silkd_changed }}
           MANUAL: ${{ github.event_name == 'workflow_dispatch' }}
           MANUAL_NO_CACHE: ${{ github.event.inputs.no-cache || inputs.no_cache }}
-          BEFORE: ${{ github.event.before }}
+          BASE: ${{ inputs.base }}
           SHA: ${{ github.sha }}
         run: |
           all_names() {
@@ -76,16 +80,16 @@ jobs:
           }
 
           # A changed carrier (boot/silkd) refolds into every image; a manual
-          # run, a first push (no BEFORE), or a change to how the images are
+          # run, no known base, or a change to how the images are
           # built rebuilds all. Otherwise only the git-changed image dirs
           # rebuild. The workflow diff is last so a manual run never runs git.
           wf=.github/workflows
-          if [[ "$CARRIER_CHANGED" == "true" || "$MANUAL" == "true" || "$BEFORE" == "0000000000000000000000000000000000000000" ]] ||
-             git diff --name-only "$BEFORE" "$SHA" -- "$wf/build-os-images.yml" "$wf/images.yml" | grep -q .; then
+          if [[ "$CARRIER_CHANGED" == "true" || "$MANUAL" == "true" || -z "$BASE" ]] ||
+             git diff --name-only "$BASE" "$SHA" -- "$wf/build-os-images.yml" "$wf/images.yml" | grep -q .; then
             all_names
             all_changed=""
           else
-            all_changed=$(git diff --name-only "$BEFORE" "$SHA" -- 'os-image/' ':!*.md' | sort -u)
+            all_changed=$(git diff --name-only "$BASE" "$SHA" -- 'os-image/' ':!*.md' | sort -u)
             declare -A names
             for f in $all_changed; do
               name=$(echo "$f" | cut -d'/' -f2)
@@ -319,8 +323,8 @@ jobs:
         id: epoch
         run: echo "epoch=$(git log -1 --format=%ct)" >> "$GITHUB_OUTPUT"
 
-      # SOURCE_DATE_EPOCH plus rewrite-timestamp clamp every file's mtime, so a
-      # rebuild of one commit writes the same layer.
+      # SOURCE_DATE_EPOCH plus rewrite-timestamp clamp every file's mtime; a rebuild writes
+      # the same layer only when the packages it installs (apt, unpinned pip) are unchanged.
       - name: Build and push
         uses: docker/build-push-action@v7.4.0
         env:

.github/workflows/images.yml

@@ -32,6 +32,7 @@ concurrency:
   cancel-in-progress: false
 
 permissions:
+  actions: read
   contents: read
   packages: write
 
@@ -42,6 +43,7 @@ jobs:
       boot: ${{ steps.diff.outputs.boot }}
       silkd: ${{ steps.diff.outputs.silkd }}
       osimage: ${{ steps.diff.outputs.osimage }}
+      base: ${{ steps.diff.outputs.base }}
     steps:
       - name: Checkout
         uses: actions/checkout@v7.0.1
@@ -52,14 +54,20 @@ jobs:
         id: diff
         env:
           EVT: ${{ github.event_name }}
-          BEFORE: ${{ github.event.before }}
           SHA: ${{ github.sha }}
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+          BRANCH: ${{ github.ref_name }}
         run: |
-          # Manual run or a first/force push with no base: build everything.
-          if [[ "$EVT" == "workflow_dispatch" || -z "$BEFORE" || "$BEFORE" == "0000000000000000000000000000000000000000" ]]; then
-            boot=true; silkd=true; osimage=true
+          # The base is the last green run's commit, not event.before: the queue cancels a pending push unbuilt.
+          base=""
+          if [[ "$EVT" != "workflow_dispatch" ]]; then
+            base=$(gh run list --workflow images.yml --branch "$BRANCH" --status success --limit 1 --json headSha --jq '.[0].headSha // ""')
+          fi
+          if [[ -z "$base" ]] || ! git cat-file -e "$base^{commit}" 2>/dev/null; then
+            base=""; boot=true; silkd=true; osimage=true
           else
-            changed() { git diff --name-only "$BEFORE" "$SHA" -- "$@" ':!*.md' | grep -q .; }
+            changed() { git diff --name-only "$base" "$SHA" -- "$@" ':!*.md' | grep -q .; }
             # Each area carries its own build workflow: changing how an image is
             # built must rebuild it, or the run goes green having built nothing.
             wf=.github/workflows
@@ -71,8 +79,9 @@ jobs:
             echo "boot=$boot"
             echo "silkd=$silkd"
             echo "osimage=$osimage"
+            echo "base=$base"
           } >> "$GITHUB_OUTPUT"
-          echo "Detected: boot=$boot silkd=$silkd osimage=$osimage"
+          echo "Detected: base=${base:-none} boot=$boot silkd=$silkd osimage=$osimage"
 
   boot:
     needs: detect
@@ -102,5 +111,6 @@ jobs:
       silkd_changed: ${{ needs.detect.outputs.silkd == 'true' }}
       boot_image: ${{ needs.boot.outputs.image }}
       silkd_image: ${{ needs.silkd.outputs.image }}
+      base: ${{ needs.detect.outputs.base }}
       no_cache: ${{ github.event.inputs.no-cache == 'true' }}
     secrets: inherit

.github/workflows/release.yml

@@ -17,6 +17,10 @@ jobs:
         uses: actions/checkout@v7.0.1
         with:
           fetch-depth: 0
+      - name: Check tag matches silkd version
+        run: |
+          crate_ver=$(sed -n 's/^version = "\(.*\)"/\1/p' silkd/Cargo.toml | head -1)
+          [ "$crate_ver" = "${GITHUB_REF_NAME#v}" ] || { echo "tag ${GITHUB_REF_NAME#v} != silkd/Cargo.toml $crate_ver" >&2; exit 1; }
       - name: Set up Go
         uses: actions/setup-go@v7.0.0
         with:

Makefile

@@ -60,7 +60,7 @@ sandboxd: ## build dist/sandboxd
 go-test: ## go test -race across the Go modules
 	for m in $(GO_MODULES); do (cd $$m && GOWORK=off go test -race ./...) || exit 1; done
 
-bench: ## claim-tier + data-plane benchmarks on this node (see docs/benchmarks.md)
+bench: ## claim-tier + data-plane benchmarks on this node; removes every sbx-* VM (see docs/benchmarks.md)
 	bash scripts/bench.sh
 
 go-lint: golangci-lint ## golangci-lint (run + fmt --diff) across the Go modules, GOOS linux+darwin

docs/benchmarks.md

@@ -70,6 +70,8 @@ unpooled template, then measures the data plane (`fs_stat` round-trips via
 in its table so new runs remain comparable with the dated log below. It
 prints a markdown table stamped with the host evidence: virtualization
 (`systemd-detect-virt`), CPU, kernel, cocoon version, image digest.
+On exit it removes every `sbx-*` VM on the cocoon root, a serving
+sandboxd's claims included, so run it only on a node that serves no claims.
 
 Knobs (environment variables): `WARM`/`WARM_N` (warm-pool depth and burst
 size — the burst must stay within the depth, or refill loses the race and

docs/deploy.md

@@ -634,6 +634,14 @@ cold boot + snapshot export, tens of seconds) and keeps each pool topped up
 with claim-ready clones.
 `GET /v1/info` shows `"golden": true` and `warm` at target when the node is
 ready to serve warm claims.
+A golden left on disk is adopted on start when its stamp matches the pool's
+image, lane, interception, `storage` and `warmup`. The stamp does not record
+the cocoon or Cloud Hypervisor version, so a golden from before an upgrade of
+either is adopted. If its clones then fail, sandboxd retires it once the pool's
+refills have failed 8 times running (a node-capacity failure does not count)
+and builds a new golden; until it is ready, claims for that pool that find no
+warm VM cold-boot. If failures continue with the new golden, it is retired again at most once every
+30 s.
 
 A minimal systemd unit (shipped as
 [`packaging/sandboxd.service`](https://github.com/cocoonstack/sandbox/blob/main/packaging/sandboxd.service)):
@@ -647,6 +655,7 @@ After=network-online.target
 [Service]
 ExecStart=/usr/local/bin/sandboxd -config /etc/sandboxd/config.json
 Restart=on-failure
+RestartSec=5s
 KillMode=process
 Environment=SANDBOXD_LOG_LEVEL=info
 

docs/sandboxd-api.md

@@ -409,7 +409,9 @@ timeouts return 504 and cancellation returns 408.
 A mounted writable attachment marks the image dirty before the guest can write.
 Live removal and release clear the marker only when the unmount observes and
 normally unmounts the exact writable whole-device filesystem and finds no
-residual device mounts. Live removal also needs a confirmed host detach. Both
+residual device mounts. These checks read silkd's own mount namespace, so a
+mount of the device in another namespace, or one moved there, is not seen.
+Live removal also needs a confirmed host detach. Both
 mount and superblock options must be writable; ext4's `emergency_ro` after a
 filesystem error counts as read-only. A missing device or pre-unmounted
 filesystem does not prove clean unmount. The marker remains on uncertainty and
@@ -1146,7 +1148,9 @@ connection to its silkd with the unread bytes replayed, so a client sees one
 silkd and no close. When the sandbox sleeps again meanwhile (a new hibernate or
 an archive), the connection's next read moves to a fresh reader of the new sleep;
 when it wakes, the connection moves to the woken sandbox, so reads never come
-from a stale sleep. Files
+from a stale sleep. Pipelining a request behind a read that has not yet sent
+its terminal frame is unsupported: when that request moves the connection, the
+reader closes and the rest of the read's reply is lost. Files
 and symlinks resolve inside the sandbox's own filesystem as of the guest `sync`
 that runs right before the last hibernate's pause; a write the guest made in
 the instant between that `sync` and the pause is visible only after a wake. A

docs/silkd.md

@@ -23,7 +23,9 @@ frames tagged `"type"` until a terminal one (`exit`, `done`, or `error`),
 then reads the next request on the same connection. `stdin`, `stdin_close`,
 `data` and `data_end` frames are input for the RPC in flight: after its
 terminal frame they are dropped, and a request frame sent while a streaming
-RPC still runs ends that RPC's input and is served after it. A client that
+RPC still runs ends that RPC's input and is served after it, except in the
+upload of `fs_write` or `fs_push`: there a request frame before `data_end`
+fails the upload with `bad_request` and is dropped. A client that
 closes after one RPC is served exactly as before; `info` reports `proto: 2`
 on a daemon that keeps the connection. Binary payloads ride base64 in `data`
 fields. Frames are capped at 8 MiB; requests carry `"v": 1` and unknown

packaging/sandboxd.service

@@ -6,6 +6,7 @@ After=network-online.target
 [Service]
 ExecStart=/usr/local/bin/sandboxd -config /etc/sandboxd/config.json
 Restart=on-failure
+RestartSec=5s
 KillMode=process
 Environment=SANDBOXD_LOG_LEVEL=info
 

scripts/archive-e2e.sh

@@ -2,8 +2,8 @@
 # Bare-metal acceptance for lifecycle auto-archive: a claim idles into
 # hibernation, then to a store checkpoint (its local VM dropped), and the next
 # access wakes it from the store with the same id and guest state intact.
-# Runs a dedicated sandboxd on port 7778 so it never collides with
-# sandboxd-e2e.sh (7777).
+# Runs a dedicated sandboxd on port 7778 (7777=e2e), but cleanup removes every
+# sbx-* VM on the cocoon root, so it cannot run beside another e2e or sandboxd.
 set -euo pipefail
 
 TEMPLATE=${TEMPLATE:-rt:24.04}

scripts/egress-e2e.sh

@@ -19,7 +19,7 @@ REACH=127.0.0.1
 BRIDGE_CFG=""
 NICADDR=""
 if [[ $NET == egress ]]; then
-  REACH=$(ip -4 addr show "$BRIDGE" 2>/dev/null | awk '/inet /{print $2}' | cut -d/ -f1 | head -1)
+  REACH=$(ip -4 addr show "$BRIDGE" 2>/dev/null | awk '/inet /{print $2}' | cut -d/ -f1 | head -1 || true)
   [[ -z $REACH ]] && { echo "bridge $BRIDGE has no IPv4 — egress lane needs a gateway"; exit 1; }
   BRIDGE_CFG="\"bridges\": [\"$BRIDGE\"],"
   NICADDR="${REACH%.*}.222/24" # static guest NIC (image does not DHCP), so a blocked direct egress is the nft lock

sdk/python/cocoonsandbox/conn.py

@@ -10,13 +10,15 @@
 import time
 import urllib.parse
 from collections.abc import Callable, Iterator
-from typing import Any, BinaryIO, Protocol, TypeVar
+from typing import Any, BinaryIO, Protocol, TypeVar, cast
 
 from .errors import APIError, ProtocolError, SandboxError, SandboxTimeout, SilkdError
 from .frames import MAX_FRAME, decode_response, encode_request
 
 KEEP_ALIVE_CONNS = 8
 _FD_SETSIZE = 1024
+_READ_BUFFER = 64 * 1024
+_TCP_KEEPALIVE = {"TCP_KEEPIDLE": 15, "TCP_KEEPALIVE": 15, "TCP_KEEPINTVL": 15, "TCP_KEEPCNT": 9}
 
 _CloseableT = TypeVar("_CloseableT", bound="_Closeable")
 
@@ -194,6 +196,10 @@ def dial_agent(
     except OSError as exc:
         raise _handshake_error(f"dial {addr}", exc) from exc
     sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+    sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
+    for opt, setting in _TCP_KEEPALIVE.items():
+        if hasattr(socket, opt):
+            sock.setsockopt(socket.IPPROTO_TCP, getattr(socket, opt), setting)
     request = (
         f"GET /v1/sandboxes/{sandbox_id}/agent HTTP/1.1\r\n"
         f"Host: {endpoint.netloc}\r\n"
@@ -206,7 +212,7 @@ def dial_agent(
     try:
         if endpoint.scheme != "https":
             sock.settimeout(None)
-            return Conn(sock, sock.makefile("rb"), request, timeout)
+            return Conn(sock, _buffered_reader(sock), request, timeout)
         context = ssl_context or ssl.create_default_context()
         sock = context.wrap_socket(sock, server_hostname=host, do_handshake_on_connect=False)
         sock.settimeout(remaining_timeout(timeout, deadline))
@@ -217,7 +223,7 @@ def dial_agent(
         # a TLS edge may be a proxy that drops bytes sent ahead of the 101
         sock.settimeout(remaining_timeout(timeout, deadline))
         sock.sendall(request)
-        reader = sock.makefile("rb")
+        reader = _buffered_reader(sock)
         _read_upgrade(reader, lambda: sock.settimeout(remaining_timeout(timeout, deadline)))
         remaining_timeout(timeout, deadline)
         sock.settimeout(None)
@@ -255,6 +261,10 @@ def is_quiet(sock: socket.socket | None) -> bool:
         return False
 
 
+def _buffered_reader(sock: socket.socket) -> BinaryIO:
+    return cast(BinaryIO, sock.makefile("rb", buffering=_READ_BUFFER))
+
+
 def _handshake_error(what: str, exc: OSError) -> SandboxError:
     if isinstance(exc, (TimeoutError, socket.timeout)):
         return SandboxTimeout(f"{what} timed out")

sdk/python/tests/test_hardening.py

@@ -37,6 +37,14 @@ def test_dial_agent_wraps_refused_connection(dead_addr):
         dial_agent(dead_addr, "sb_1", "tok", 0.5)
 
 
+def test_dial_agent_enables_tcp_keepalive(black_hole):
+    with dial_agent(black_hole, "sb_1", "tok", 0.5) as conn:
+        assert conn._sock.getsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE)
+        for idle in ("TCP_KEEPIDLE", "TCP_KEEPALIVE"):
+            if hasattr(socket, idle):
+                assert conn._sock.getsockopt(socket.IPPROTO_TCP, getattr(socket, idle)) == 15, idle
+
+
 def test_send_on_a_dead_socket_raises_a_typed_error():
     ours, theirs = socket.socketpair()
     theirs.close()

silkd/Dockerfile

@@ -7,7 +7,7 @@
 # /silkd-static is musl-static for guests without glibc, such as the android
 # flavor's non-systemd userspace. Built per-platform (amd64/arm64) on native
 # runners; TARGETARCH picks the musl target triple.
-FROM rust:1.99.1-slim@sha256:b8e062b1c3a80183df83d9ba187cadd2b5ab84bc78d453dbfeb2fccd5f982247 AS build
+FROM rust:1.99.1-slim-bookworm@sha256:24707e70a4f1e23d9a7b25202ec43126d28a52294b96aad5c670df1fd1f1d400 AS build
 ARG TARGETARCH
 WORKDIR /silkd
 # Before any rustup/cargo call: the pin, not the base image's default, is the