From 530585ec101b16cac86060ae2bfcfb12eaa71e2b Mon Sep 17 00:00:00 2001 From: danbugs Date: Thu, 22 Oct 2026 00:26:58 +0000 Subject: [PATCH 1/3] rootfs: prune warm snapshots another hluk build saved A snapshot's stamp names the hluk build that saved it (key=), and another build refuses it, but pruning only removed the snapshots of a rebuilt rootfs: every hluk upgrade left each guest's warm snapshots in the cache for good, about 1 GB for the agent and python images. On the self-hosted Macs, whose runners keep their disk, every commit that changed the kernel added a set, until a Mac's disk was full. Prune those of the same rootfs too when saving one, and test what goes and what stays. Signed-off-by: danbugs --- CHANGELOG.md | 1 + src/cli/rootfs.rs | 66 +++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 56 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3adaacc2..8c258833 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Fixed +- Every hluk upgrade left the warm snapshots of each guest in the cache for good: a snapshot another hluk build saved, which this one refuses, was pruned only once its rootfs was rebuilt. `hluk run` now removes those of the same rootfs when it saves its own. - `os.chmod` (and every `chmod`) on a mount failed with `EIO`: the host took the mode as unsigned and the kernel sends it signed, so the host call was refused before it ran. It now reaches the host file, permission bits only: a guest cannot set setuid, setgid or sticky. - DNS messages are read only as far as a resolver could be asked: a name longer than 253 characters, a label with a dot or a byte that is not printable ASCII, or an extended label type makes the message no well-formed query (refused) or answer (ignored). The label `a.b` read as the two labels `a` and `b`, and a forged answer could spell out a name kilobytes long through a compression loop. - A `sendto` on a connected TCP socket was judged by the address it named, though the data goes to the socket's peer: a guest connected to a resolver on port 53 could send anything there by naming an address the policy allows, past the check of its DNS questions. A `sendto` on a stream socket is a `send` to its peer now, checked as one. diff --git a/src/cli/rootfs.rs b/src/cli/rootfs.rs index 57518c49..b39d70df 100644 --- a/src/cli/rootfs.rs +++ b/src/cli/rootfs.rs @@ -223,19 +223,28 @@ pub fn find_snapshot(stamp: &str) -> Result, String> { /// alone (see `prune_superseded`). const ENV_VARIANTS_KEPT: usize = 4; -/// Remove the snapshots taken from an earlier version of `stamp`'s rootfs -/// file (same path, different size or mtime: it was rebuilt), other than -/// `keep`. Snapshots of the same file under other settings (memory, -/// entry, `warm_exec`) are other guests and stay; of those that differ in -/// their environment alone, the newest [`ENV_VARIANTS_KEPT`] do. Returns -/// how many went. +/// Remove the snapshots of `stamp`'s rootfs that no run can restore any +/// more, other than `keep`: those of an earlier version of the file (same +/// path, different size or mtime: it was rebuilt), and those another hluk +/// build saved (another `key`: its kernel or host functions, which this +/// build refuses). Without the second, every hluk upgrade left a +/// snapshot of each guest behind for good. Snapshots of the same file +/// under other settings (memory, entry, `warm_exec`) are other guests and +/// stay; of those that differ in their environment alone, the newest +/// [`ENV_VARIANTS_KEPT`] do. Returns how many went. pub fn prune_superseded(stamp: &str, keep: &Path) -> Result { + prune_among(list_snapshots()?, stamp, keep) +} + +/// [`prune_superseded`] over `snapshots`. +fn prune_among(snapshots: Vec, stamp: &str, keep: &Path) -> Result { let Some(rootfs) = stamp_field(stamp, "rootfs") else { return Ok(0); }; - let same_file = |other: &str| { - stamp_field(other, "size") == stamp_field(stamp, "size") - && stamp_field(other, "mtime") == stamp_field(stamp, "mtime") + let restorable = |other: &str| { + ["key", "size", "mtime"] + .iter() + .all(|k| stamp_field(other, k) == stamp_field(stamp, k)) }; // Snapshots keyed by an environment (see `stamp`) differ in it alone: // keep the newest few, so runs alternating between a few values all @@ -245,11 +254,11 @@ pub fn prune_superseded(stamp: &str, keep: &Path) -> Result { } let mut pruned = 0; let mut variants = Vec::new(); - for entry in list_snapshots()? { + for entry in snapshots { if entry.dir == keep || stamp_field(&entry.stamp, "rootfs") != Some(rootfs) { continue; } - if !same_file(&entry.stamp) { + if !restorable(&entry.stamp) { fs::remove_dir_all(&entry.dir) .map_err(|e| format!("cannot remove {}: {e}", entry.dir.display()))?; pruned += 1; @@ -514,6 +523,41 @@ fn docker_error(e: std::io::Error) -> String { mod tests { use super::*; + /// Snapshots no run can restore go: a rebuilt rootfs's, and another + /// hluk build's; another guest of the same file, and another rootfs's, + /// stay. + #[test] + fn prune_drops_what_cannot_be_restored() { + let dir = tempfile::tempdir().unwrap(); + let stamp = |key: &str, rootfs: &str, mtime: u32, scratch: u32| { + format!( + "key={key}\nrootfs={rootfs}\nsize=1\nmtime={mtime}\nscratch_mb={scratch}\nentry=\nwarm_exec=\"\"\n" + ) + }; + let entry = |name: &str, stamp: String| { + let d = dir.path().join(name); + std::fs::create_dir(&d).unwrap(); + SnapshotEntry { + dir: d, + stamp, + bytes: 0, + } + }; + let ours = stamp("k1", "/r.cpio", 2, 256); + let snapshots = vec![ + entry("ours", ours.clone()), + entry("rebuilt", stamp("k1", "/r.cpio", 1, 256)), + entry("old-hluk", stamp("k0", "/r.cpio", 2, 256)), + entry("other-guest", stamp("k1", "/r.cpio", 2, 512)), + entry("other-rootfs", stamp("k0", "/s.cpio", 2, 256)), + ]; + let keep = dir.path().join("ours"); + assert_eq!(prune_among(snapshots, &ours, &keep), Ok(2)); + let left = |name: &str| dir.path().join(name).exists(); + assert!(left("ours") && left("other-guest") && left("other-rootfs")); + assert!(!left("rebuilt") && !left("old-hluk")); + } + #[test] fn a_stamp_describes_itself() { let entry = SnapshotEntry { From 6b78e25b880353723d9d30aa400f467c6566bda1 Mon Sep 17 00:00:00 2001 From: danbugs Date: Thu, 22 Oct 2026 00:26:58 +0000 Subject: [PATCH 2/3] ci: show what fills the self-hosted Macs' disks, trim what hluk leaves, and sweep every runner The Macs' runners keep their disk between jobs, and a Mac's runners share it with hyperlight's jobs; the Linux and Windows arm64 hosts are fresh machines each time. dev/ci-disk.sh shows what fills the disk and removes what hluk and its tests leave there: warm snapshots and pulled images untouched for days, hluk-test-* directories a killed test left, and, under 20 GB free, the checkout's target/ and cargo's download cache. The arm64-host action runs it first, before anything that needs room. runner-disk.yml (dispatched, or on a change to it or the script) sweeps each runner of both Macs: the whole hluk cache, every hluk-* temp directory and this repo's build directories, one job per runner, each holding its runner so they spread, and lists any runner none reached. Other repos' files are shown, not removed. Signed-off-by: danbugs --- .github/actions/arm64-host/action.yml | 7 +++ .github/workflows/runner-disk.yml | 74 +++++++++++++++++++++++++++ dev/ci-disk.sh | 71 +++++++++++++++++++++++++ 3 files changed, 152 insertions(+) create mode 100644 .github/workflows/runner-disk.yml create mode 100755 dev/ci-disk.sh diff --git a/.github/actions/arm64-host/action.yml b/.github/actions/arm64-host/action.yml index 36c61f2c..4cd6f6dd 100644 --- a/.github/actions/arm64-host/action.yml +++ b/.github/actions/arm64-host/action.yml @@ -13,6 +13,13 @@ inputs: runs: using: composite steps: + # The self-hosted Mac and Linux hosts keep their disk between jobs: + # show what fills it, and remove what hluk and its tests left there. + - name: Disk usage and cleanup + if: runner.os != 'Windows' + shell: bash + run: bash dev/ci-disk.sh + # Self-hosted runners come without a Rust toolchain: install the one # rust-toolchain.toml pins, as hyperlight's jobs do. - name: Rust toolchain version diff --git a/.github/workflows/runner-disk.yml b/.github/workflows/runner-disk.yml new file mode 100644 index 00000000..313f8e51 --- /dev/null +++ b/.github/workflows/runner-disk.yml @@ -0,0 +1,74 @@ +name: runner disk + +# Sweeps what hluk and this repo's jobs keep on the self-hosted Macs, the +# only arm64 hosts that keep their disk between jobs (the Linux and +# Windows ones are fresh machines each time): each runner is a user of +# its own, with its own hluk cache and build directories, and a Mac's +# runners share its disk with hyperlight's jobs. One job per runner, each +# holding its runner a while so the jobs spread over all of them; the last +# job lists the runners none reached, to run this again for. Other repos' +# files are shown, never removed. + +on: + workflow_dispatch: + pull_request: + paths: + - .github/workflows/runner-disk.yml + - dev/ci-disk.sh + +permissions: + contents: read + actions: read + +jobs: + sweep: + name: sweep (${{ matrix.slot }}) + runs-on: [self-hosted, macos, arm64, hvf] + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + slot: [1, 2, 3, 4, 5, 6, 7, 8] + steps: + # Fetched, not checked out: a full disk has no room for a checkout. + - name: Sweep + # bash -eo pipefail: a failed download fails the job, which then + # does not count as a runner swept. + shell: bash + env: + SWEEP: "1" + SCRIPT: https://raw.githubusercontent.com/${{ github.repository }}/${{ github.event.pull_request.head.sha || github.sha }}/dev/ci-disk.sh + run: | + echo "runner: $RUNNER_NAME ($(whoami))" + curl -fsSL --retry 3 "$SCRIPT" | bash + + # Held, so this runner takes no other sweep job and they all spread. + - name: Hold the runner + run: sleep 300 + + coverage: + needs: sweep + if: always() + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ github.token }} + RUNNERS: >- + Simons-Mac-mini-runner-1 Simons-Mac-mini-runner-2 + Simons-Mac-mini-runner-3 Simons-Mac-mini-runner-4 + azures-Mac-mini-runner-1 azures-Mac-mini-runner-2 + azures-Mac-mini-runner-3 azures-Mac-mini-runner-4 + steps: + - name: Runners swept + run: | + swept=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs?per_page=100" \ + --jq '.jobs[] | select(.name | startswith("sweep")) | select(.conclusion == "success") | .runner_name' | sort -u) + echo "Swept:"; echo "$swept" | sed 's/^/ /' + missed="" + for r in $RUNNERS; do + echo "$swept" | grep -qx "$r" || missed="$missed $r" + done + if [ -n "$missed" ]; then + echo "::warning::Not swept, run this again:$missed" + else + echo "Every runner swept." + fi diff --git a/dev/ci-disk.sh b/dev/ci-disk.sh new file mode 100755 index 00000000..cdaf1dfe --- /dev/null +++ b/dev/ci-disk.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# Shows what fills a self-hosted runner's disk, and removes what hluk and +# its tests leave there: these runners keep their disk between jobs. +# .github/actions/arm64-host runs it before anything else, so it needs no +# free space itself. It only touches the runner user's own files. +# MIN_FREE_GB below this much free space after the cleanup, also remove +# this checkout's target/ and cargo's download cache +# (default 20) +# SWEEP=1 remove all that hluk and this repo's jobs keep: the whole +# hluk cache, every hluk-* temp directory, and this repo's +# target/ directories (.github/workflows/runner-disk.yml) +set -uo pipefail + +min_free_kb=$((${MIN_FREE_GB:-20} * 1024 * 1024)) +cache="${HLUK_CACHE_DIR:-${XDG_CACHE_HOME:-$HOME/.cache}/hluk}" +tmp="${TMPDIR:-/tmp}" +tmp="${tmp%/}" + +free_kb() { df -Pk "$HOME" | awk 'NR == 2 { print $4 }'; } + +# The paths given, largest first, in GB. +sizes() { + for p in "$@"; do + [ -e "$p" ] && du -sk "$p" 2>/dev/null + done | sort -rn | awk '{ printf "%7.1f GB %s\n", $1 / 1048576, $2 }' +} + +before=$(free_kb) +echo "== Disk" +df -h "$HOME" +echo "== Largest in $HOME" +sizes "$HOME"/* "$HOME"/.[!.]* | head -15 +if [ -n "${RUNNER_WORKSPACE:-}" ]; then + echo "== Runner work directories" + sizes "$RUNNER_WORKSPACE"/../* "$RUNNER_WORKSPACE"/*/target | head -15 +fi +echo "== hluk cache ($cache)" +sizes "$cache"/* "$cache"/snapshots/* | head -15 +echo "== Largest in $tmp" +sizes "$tmp"/* | head -10 + +# Warm snapshots and pulled images nobody has saved or pulled for a few +# days: an earlier hluk's or kernel's, which no run restores any more, +# and the next run that wants one makes it again. +echo "== Cleanup" +if [ "${SWEEP:-}" = 1 ]; then + rm -rf "$cache" && echo "removed $cache" + find "$tmp" -mindepth 1 -maxdepth 1 -name 'hluk-*' -print -exec rm -rf {} + + if [ -n "${RUNNER_WORKSPACE:-}" ]; then + for t in "$RUNNER_WORKSPACE"/*/target; do + [ -d "$t" ] && rm -rf "$t" && echo "removed $t" + done + fi +fi +if [ -d "$cache/snapshots" ]; then + find "$cache/snapshots" -mindepth 1 -maxdepth 1 \( -name '*.part' -o -mtime +2 \) \ + -print -exec rm -rf {} + +fi +if [ -d "$cache/rootfs" ]; then + find "$cache/rootfs" -type f -mtime +7 -print -delete +fi +# Test directories a killed test left behind (they are removed on drop). +find "$tmp" -mindepth 1 -maxdepth 1 -name 'hluk-*' -mtime +0 -print -exec rm -rf {} + + +if [ "$(free_kb)" -lt "$min_free_kb" ] && [ -n "${GITHUB_WORKSPACE:-}" ]; then + echo "Under ${MIN_FREE_GB:-20} GB free: removing $GITHUB_WORKSPACE/target and cargo's download cache" + rm -rf "$GITHUB_WORKSPACE/target" "${CARGO_HOME:-$HOME/.cargo}/registry/cache" +fi + +after=$(free_kb) +awk -v b="$before" -v a="$after" 'BEGIN { printf "== Freed %.1f GB; %.1f GB free\n", (a - b) / 1048576, a / 1048576 }' From b11c33e8d180dd06ce5087a575a77100c029fef4 Mon Sep 17 00:00:00 2001 From: danbugs Date: Thu, 22 Oct 2026 00:28:24 +0000 Subject: [PATCH 3/3] arm64-host: a job-scoped hluk cache Point HLUK_CACHE_DIR at the job's temp directory on the self-hosted hosts, which the runner empties when the job ends, cancelled or not, so the images a job pulls and the warm snapshots it saves go with it, instead of staying in the runner user's home for later jobs, and hyperlight's, to find the disk full. Signed-off-by: danbugs --- .github/actions/arm64-host/action.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/actions/arm64-host/action.yml b/.github/actions/arm64-host/action.yml index 4cd6f6dd..e41bd819 100644 --- a/.github/actions/arm64-host/action.yml +++ b/.github/actions/arm64-host/action.yml @@ -20,6 +20,14 @@ runs: shell: bash run: bash dev/ci-disk.sh + # hluk's cache (pulled images, warm snapshots) in the job's temp + # directory, which the runner empties when the job ends, cancelled or + # not: a job leaves none of it behind. + - name: A job-scoped hluk cache + if: runner.os != 'Windows' + shell: bash + run: echo "HLUK_CACHE_DIR=$RUNNER_TEMP/hluk-cache" >> "$GITHUB_ENV" + # Self-hosted runners come without a Rust toolchain: install the one # rust-toolchain.toml pins, as hyperlight's jobs do. - name: Rust toolchain version