Stop leaking warm snapshots; sweep the self-hosted Macs' disks

.github/actions/arm64-host/action.yml

@@ -13,6 +13,21 @@ inputs:
 runs:
   using: composite
   steps:
+    # The self-hosted Mac and Linux hosts keep their disk between jobs:
+    # show what fills it, and remove what hluk and its tests left there.
+    - name: Disk usage and cleanup
+      if: runner.os != 'Windows'
+      shell: bash
+      run: bash dev/ci-disk.sh
+
+    # hluk's cache (pulled images, warm snapshots) in the job's temp
+    # directory, which the runner empties when the job ends, cancelled or
+    # not: a job leaves none of it behind.
+    - name: A job-scoped hluk cache
+      if: runner.os != 'Windows'
+      shell: bash
+      run: echo "HLUK_CACHE_DIR=$RUNNER_TEMP/hluk-cache" >> "$GITHUB_ENV"
+
     # Self-hosted runners come without a Rust toolchain: install the one
     # rust-toolchain.toml pins, as hyperlight's jobs do.
     - name: Rust toolchain version

.github/workflows/runner-disk.yml

@@ -0,0 +1,74 @@
+name: runner disk
+
+# Sweeps what hluk and this repo's jobs keep on the self-hosted Macs, the
+# only arm64 hosts that keep their disk between jobs (the Linux and
+# Windows ones are fresh machines each time): each runner is a user of
+# its own, with its own hluk cache and build directories, and a Mac's
+# runners share its disk with hyperlight's jobs.  One job per runner, each
+# holding its runner a while so the jobs spread over all of them; the last
+# job lists the runners none reached, to run this again for.  Other repos'
+# files are shown, never removed.
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - .github/workflows/runner-disk.yml
+      - dev/ci-disk.sh
+
+permissions:
+  contents: read
+  actions: read
+
+jobs:
+  sweep:
+    name: sweep (${{ matrix.slot }})
+    runs-on: [self-hosted, macos, arm64, hvf]
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        slot: [1, 2, 3, 4, 5, 6, 7, 8]
+    steps:
+      # Fetched, not checked out: a full disk has no room for a checkout.
+      - name: Sweep
+        # bash -eo pipefail: a failed download fails the job, which then
+        # does not count as a runner swept.
+        shell: bash
+        env:
+          SWEEP: "1"
+          SCRIPT: https://raw.githubusercontent.com/${{ github.repository }}/${{ github.event.pull_request.head.sha || github.sha }}/dev/ci-disk.sh
+        run: |
+          echo "runner: $RUNNER_NAME ($(whoami))"
+          curl -fsSL --retry 3 "$SCRIPT" | bash
+
+      # Held, so this runner takes no other sweep job and they all spread.
+      - name: Hold the runner
+        run: sleep 300
+
+  coverage:
+    needs: sweep
+    if: always()
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ github.token }}
+      RUNNERS: >-
+        Simons-Mac-mini-runner-1 Simons-Mac-mini-runner-2
+        Simons-Mac-mini-runner-3 Simons-Mac-mini-runner-4
+        azures-Mac-mini-runner-1 azures-Mac-mini-runner-2
+        azures-Mac-mini-runner-3 azures-Mac-mini-runner-4
+    steps:
+      - name: Runners swept
+        run: |
+          swept=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs?per_page=100" \
+            --jq '.jobs[] | select(.name | startswith("sweep")) | select(.conclusion == "success") | .runner_name' | sort -u)
+          echo "Swept:"; echo "$swept" | sed 's/^/  /'
+          missed=""
+          for r in $RUNNERS; do
+            echo "$swept" | grep -qx "$r" || missed="$missed $r"
+          done
+          if [ -n "$missed" ]; then
+            echo "::warning::Not swept, run this again:$missed"
+          else
+            echo "Every runner swept."
+          fi

CHANGELOG.md

@@ -22,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 
 ### Fixed
 
+- Every hluk upgrade left the warm snapshots of each guest in the cache for good: a snapshot another hluk build saved, which this one refuses, was pruned only once its rootfs was rebuilt. `hluk run` now removes those of the same rootfs when it saves its own.
 - `os.chmod` (and every `chmod`) on a mount failed with `EIO`: the host took the mode as unsigned and the kernel sends it signed, so the host call was refused before it ran. It now reaches the host file, permission bits only: a guest cannot set setuid, setgid or sticky.
 - DNS messages are read only as far as a resolver could be asked: a name longer than 253 characters, a label with a dot or a byte that is not printable ASCII, or an extended label type makes the message no well-formed query (refused) or answer (ignored). The label `a.b` read as the two labels `a` and `b`, and a forged answer could spell out a name kilobytes long through a compression loop.
 - A `sendto` on a connected TCP socket was judged by the address it named, though the data goes to the socket's peer: a guest connected to a resolver on port 53 could send anything there by naming an address the policy allows, past the check of its DNS questions. A `sendto` on a stream socket is a `send` to its peer now, checked as one.

dev/ci-disk.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+# Shows what fills a self-hosted runner's disk, and removes what hluk and
+# its tests leave there: these runners keep their disk between jobs.
+# .github/actions/arm64-host runs it before anything else, so it needs no
+# free space itself.  It only touches the runner user's own files.
+#   MIN_FREE_GB  below this much free space after the cleanup, also remove
+#                this checkout's target/ and cargo's download cache
+#                (default 20)
+#   SWEEP=1      remove all that hluk and this repo's jobs keep: the whole
+#                hluk cache, every hluk-* temp directory, and this repo's
+#                target/ directories (.github/workflows/runner-disk.yml)
+set -uo pipefail
+
+min_free_kb=$((${MIN_FREE_GB:-20} * 1024 * 1024))
+cache="${HLUK_CACHE_DIR:-${XDG_CACHE_HOME:-$HOME/.cache}/hluk}"
+tmp="${TMPDIR:-/tmp}"
+tmp="${tmp%/}"
+
+free_kb() { df -Pk "$HOME" | awk 'NR == 2 { print $4 }'; }
+
+# The paths given, largest first, in GB.
+sizes() {
+    for p in "$@"; do
+        [ -e "$p" ] && du -sk "$p" 2>/dev/null
+    done | sort -rn | awk '{ printf "%7.1f GB  %s\n", $1 / 1048576, $2 }'
+}
+
+before=$(free_kb)
+echo "== Disk"
+df -h "$HOME"
+echo "== Largest in $HOME"
+sizes "$HOME"/* "$HOME"/.[!.]* | head -15
+if [ -n "${RUNNER_WORKSPACE:-}" ]; then
+    echo "== Runner work directories"
+    sizes "$RUNNER_WORKSPACE"/../* "$RUNNER_WORKSPACE"/*/target | head -15
+fi
+echo "== hluk cache ($cache)"
+sizes "$cache"/* "$cache"/snapshots/* | head -15
+echo "== Largest in $tmp"
+sizes "$tmp"/* | head -10
+
+# Warm snapshots and pulled images nobody has saved or pulled for a few
+# days: an earlier hluk's or kernel's, which no run restores any more,
+# and the next run that wants one makes it again.
+echo "== Cleanup"
+if [ "${SWEEP:-}" = 1 ]; then
+    rm -rf "$cache" && echo "removed $cache"
+    find "$tmp" -mindepth 1 -maxdepth 1 -name 'hluk-*' -print -exec rm -rf {} +
+    if [ -n "${RUNNER_WORKSPACE:-}" ]; then
+        for t in "$RUNNER_WORKSPACE"/*/target; do
+            [ -d "$t" ] && rm -rf "$t" && echo "removed $t"
+        done
+    fi
+fi
+if [ -d "$cache/snapshots" ]; then
+    find "$cache/snapshots" -mindepth 1 -maxdepth 1 \( -name '*.part' -o -mtime +2 \) \
+        -print -exec rm -rf {} +
+fi
+if [ -d "$cache/rootfs" ]; then
+    find "$cache/rootfs" -type f -mtime +7 -print -delete
+fi
+# Test directories a killed test left behind (they are removed on drop).
+find "$tmp" -mindepth 1 -maxdepth 1 -name 'hluk-*' -mtime +0 -print -exec rm -rf {} +
+
+if [ "$(free_kb)" -lt "$min_free_kb" ] && [ -n "${GITHUB_WORKSPACE:-}" ]; then
+    echo "Under ${MIN_FREE_GB:-20} GB free: removing $GITHUB_WORKSPACE/target and cargo's download cache"
+    rm -rf "$GITHUB_WORKSPACE/target" "${CARGO_HOME:-$HOME/.cargo}/registry/cache"
+fi
+
+after=$(free_kb)
+awk -v b="$before" -v a="$after" 'BEGIN { printf "== Freed %.1f GB; %.1f GB free\n", (a - b) / 1048576, a / 1048576 }'

src/cli/rootfs.rs

@@ -223,19 +223,28 @@ pub fn find_snapshot(stamp: &str) -> Result<Option<PathBuf>, String> {
 /// alone (see `prune_superseded`).
 const ENV_VARIANTS_KEPT: usize = 4;
 
-/// Remove the snapshots taken from an earlier version of `stamp`'s rootfs
-/// file (same path, different size or mtime: it was rebuilt), other than
-/// `keep`.  Snapshots of the same file under other settings (memory,
-/// entry, `warm_exec`) are other guests and stay; of those that differ in
-/// their environment alone, the newest [`ENV_VARIANTS_KEPT`] do.  Returns
-/// how many went.
+/// Remove the snapshots of `stamp`'s rootfs that no run can restore any
+/// more, other than `keep`: those of an earlier version of the file (same
+/// path, different size or mtime: it was rebuilt), and those another hluk
+/// build saved (another `key`: its kernel or host functions, which this
+/// build refuses).  Without the second, every hluk upgrade left a
+/// snapshot of each guest behind for good.  Snapshots of the same file
+/// under other settings (memory, entry, `warm_exec`) are other guests and
+/// stay; of those that differ in their environment alone, the newest
+/// [`ENV_VARIANTS_KEPT`] do.  Returns how many went.
 pub fn prune_superseded(stamp: &str, keep: &Path) -> Result<usize, String> {
+    prune_among(list_snapshots()?, stamp, keep)
+}
+
+/// [`prune_superseded`] over `snapshots`.
+fn prune_among(snapshots: Vec<SnapshotEntry>, stamp: &str, keep: &Path) -> Result<usize, String> {
     let Some(rootfs) = stamp_field(stamp, "rootfs") else {
         return Ok(0);
     };
-    let same_file = |other: &str| {
-        stamp_field(other, "size") == stamp_field(stamp, "size")
-            && stamp_field(other, "mtime") == stamp_field(stamp, "mtime")
+    let restorable = |other: &str| {
+        ["key", "size", "mtime"]
+            .iter()
+            .all(|k| stamp_field(other, k) == stamp_field(stamp, k))
     };
     // Snapshots keyed by an environment (see `stamp`) differ in it alone:
     // keep the newest few, so runs alternating between a few values all
@@ -245,11 +254,11 @@ pub fn prune_superseded(stamp: &str, keep: &Path) -> Result<usize, String> {
     }
     let mut pruned = 0;
     let mut variants = Vec::new();
-    for entry in list_snapshots()? {
+    for entry in snapshots {
         if entry.dir == keep || stamp_field(&entry.stamp, "rootfs") != Some(rootfs) {
             continue;
         }
-        if !same_file(&entry.stamp) {
+        if !restorable(&entry.stamp) {
             fs::remove_dir_all(&entry.dir)
                 .map_err(|e| format!("cannot remove {}: {e}", entry.dir.display()))?;
             pruned += 1;
@@ -514,6 +523,41 @@ fn docker_error(e: std::io::Error) -> String {
 mod tests {
     use super::*;
 
+    /// Snapshots no run can restore go: a rebuilt rootfs's, and another
+    /// hluk build's; another guest of the same file, and another rootfs's,
+    /// stay.
+    #[test]
+    fn prune_drops_what_cannot_be_restored() {
+        let dir = tempfile::tempdir().unwrap();
+        let stamp = |key: &str, rootfs: &str, mtime: u32, scratch: u32| {
+            format!(
+                "key={key}\nrootfs={rootfs}\nsize=1\nmtime={mtime}\nscratch_mb={scratch}\nentry=\nwarm_exec=\"\"\n"
+            )
+        };
+        let entry = |name: &str, stamp: String| {
+            let d = dir.path().join(name);
+            std::fs::create_dir(&d).unwrap();
+            SnapshotEntry {
+                dir: d,
+                stamp,
+                bytes: 0,
+            }
+        };
+        let ours = stamp("k1", "/r.cpio", 2, 256);
+        let snapshots = vec![
+            entry("ours", ours.clone()),
+            entry("rebuilt", stamp("k1", "/r.cpio", 1, 256)),
+            entry("old-hluk", stamp("k0", "/r.cpio", 2, 256)),
+            entry("other-guest", stamp("k1", "/r.cpio", 2, 512)),
+            entry("other-rootfs", stamp("k0", "/s.cpio", 2, 256)),
+        ];
+        let keep = dir.path().join("ours");
+        assert_eq!(prune_among(snapshots, &ours, &keep), Ok(2));
+        let left = |name: &str| dir.path().join(name).exists();
+        assert!(left("ours") && left("other-guest") && left("other-rootfs"));
+        assert!(!left("rebuilt") && !left("old-hluk"));
+    }
+
     #[test]
     fn a_stamp_describes_itself() {
         let entry = SnapshotEntry {