From a074e079962cfa298aa807c32026fc2b8dd37fe5 Mon Sep 17 00:00:00 2001 From: Luther Monson Date: Mon, 6 Jul 2026 23:48:39 -0700 Subject: [PATCH] fix(macos-vm): always run ephemerd's fresh runner, not a stale baked copy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit macOS VM jobs would register a runner, report "ready", but never get assigned their job — the VM booted, ran ~90s, exited 0, and the job sat queued forever, re-provisioning a 4GB VM every ~10 min. Native macOS jobs were unaffected, which was the tell. Root cause: the base image bakes a GHA runner into /Users/admin/actions- runner at provisioning time. ephemerd separately copies the *current* runner into /Library/ephemerd/runner over SSH, but the per-job setup script only copied it into the run dir when no run.sh already existed — so the baked runner always won. Once the pinned runner version was bumped (GitHub deprecated the old one; its broker returns 403 and refuses the JIT runner), every VM job silently ran the stale, deprecated runner. Fix: the setup script now always overwrites /Users/admin/actions-runner from the freshly-copied /Library/ephemerd/runner and kills any already- running runner (e.g. one auto-started from a baked LaunchDaemon) first, so the fresh runner owns the job's JIT config. The overwrite is gated on the fresh source existing so the baked fallback is never deleted when ephemerd provided nothing. Extracted the inline setup script to a testable const with a regression guard (TestMacOSRunnerSetupScript_*) asserting the always-refresh + kill behavior and that the old skip-if-exists guard never returns. Co-Authored-By: Claude Opus 4.6 --- pkg/vm/macosvm_darwin.go | 82 +++++++++++++++++++++-------------- pkg/vm/macosvm_darwin_test.go | 55 +++++++++++++++++++++++ 2 files changed, 104 insertions(+), 33 deletions(-) create mode 100644 pkg/vm/macosvm_darwin_test.go diff --git a/pkg/vm/macosvm_darwin.go b/pkg/vm/macosvm_darwin.go index 61fb95f..63e3dab 100644 --- a/pkg/vm/macosvm_darwin.go +++ b/pkg/vm/macosvm_darwin.go @@ -576,6 +576,51 @@ func (m *darwinMacOSVM) injectRunnerIntoClone(ctx context.Context) error { return nil } +// macOSRunnerSetupScript is the bash run over SSH to start the GHA runner in a +// per-job macOS VM. It takes one %s: the base64 JIT config. +// +// Key invariant: it ALWAYS refreshes /Users/admin/actions-runner from the +// freshly-copied /Library/ephemerd/runner and kills any already-running +// runner first. A runner baked into the base image at provisioning time goes +// stale when the pinned runner version is bumped — GitHub deprecates the old +// version, its broker refuses the JIT runner, so the runner registers and +// reports "ready" but is never assigned its job and the VM job loops forever. +// Native jobs are unaffected (they run the host runner directly). An earlier +// revision only copied when no run.sh already existed, which silently used the +// stale baked runner; never reintroduce that guard. +const macOSRunnerSetupScript = ` +# Firewall: block private networks EXCEPT the Vz NAT subnet +cat > /tmp/pf-ephemerd.conf << 'PFEOF' +pass quick to 192.168.64.0/24 +block out quick to 10.0.0.0/8 +block out quick to 172.16.0.0/12 +block out quick to 192.168.0.0/16 +block out quick to 169.254.0.0/16 +pass out all +PFEOF +sudo pfctl -f /tmp/pf-ephemerd.conf -e 2>/dev/null || true + +# Always run the runner ephemerd just provided, not a stale baked copy. +RUNNER_SRC="/Library/ephemerd/runner" +RUNNER_DIR="/Users/admin/actions-runner" +pkill -f Runner.Listener 2>/dev/null || true +if [ -f "$RUNNER_SRC/run.sh" ]; then + rm -rf "$RUNNER_DIR" + cp -R "$RUNNER_SRC" "$RUNNER_DIR" + chown -R admin:staff "$RUNNER_DIR" +fi +cd "$RUNNER_DIR" +./run.sh --jitconfig '%s' /tmp/runner.log 2>&1 & +RUNNER_PID=$! +disown $RUNNER_PID 2>/dev/null || true + +# Randomize password LAST +RAND_PASS=$(head -c 32 /dev/urandom | base64 | tr -d '/+=' | head -c 32) +dscl . -passwd /Users/admin admin "$RAND_PASS" 2>/dev/null || true + +echo "runner started (pid=$RUNNER_PID)" +` + // setupRunnerViaSSH connects to the VM using the Tart default credentials // (admin/admin) and starts the GitHub Actions runner. This is more reliable // than LaunchDaemons which may be blocked by SIP/SSV on modern macOS. @@ -693,39 +738,10 @@ func (m *darwinMacOSVM) setupRunnerViaSSH(ctx context.Context, ip string) error } // Start the runner + firewall in the background (fire and forget). - // Runner binary is pre-installed in the base image (inherited by clone). - // Only JIT config is per-job, passed inline. - setupScript := fmt.Sprintf(` -# Firewall: block private networks EXCEPT the Vz NAT subnet -cat > /tmp/pf-ephemerd.conf << 'PFEOF' -pass quick to 192.168.64.0/24 -block out quick to 10.0.0.0/8 -block out quick to 172.16.0.0/12 -block out quick to 192.168.0.0/16 -block out quick to 169.254.0.0/16 -pass out all -PFEOF -sudo pfctl -f /tmp/pf-ephemerd.conf -e 2>/dev/null || true - -# Runner is pre-installed at /Library/ephemerd/runner/ (Data volume). -# Copy to home dir so the runner has a writable work directory. -RUNNER_SRC="/Library/ephemerd/runner" -RUNNER_DIR="/Users/admin/actions-runner" -if [ ! -f "$RUNNER_DIR/run.sh" ] && [ -f "$RUNNER_SRC/run.sh" ]; then - cp -R "$RUNNER_SRC" "$RUNNER_DIR" - chown -R admin:staff "$RUNNER_DIR" -fi -cd "$RUNNER_DIR" -./run.sh --jitconfig '%s' /tmp/runner.log 2>&1 & -RUNNER_PID=$! -disown $RUNNER_PID 2>/dev/null || true - -# Randomize password LAST -RAND_PASS=$(head -c 32 /dev/urandom | base64 | tr -d '/+=' | head -c 32) -dscl . -passwd /Users/admin admin "$RAND_PASS" 2>/dev/null || true - -echo "runner started (pid=$RUNNER_PID)" -`, strings.TrimSpace(string(jitData))) + // The runner is copied fresh into the VM before this runs; see + // macOSRunnerSetupScript for why we always refresh it. Only the JIT + // config is per-job, passed inline. + setupScript := fmt.Sprintf(macOSRunnerSetupScript, strings.TrimSpace(string(jitData))) session, err := client.NewSession() if err != nil { diff --git a/pkg/vm/macosvm_darwin_test.go b/pkg/vm/macosvm_darwin_test.go new file mode 100644 index 0000000..e39d18d --- /dev/null +++ b/pkg/vm/macosvm_darwin_test.go @@ -0,0 +1,55 @@ +//go:build darwin + +package vm + +import ( + "fmt" + "strings" + "testing" +) + +// TestMacOSRunnerSetupScript_AlwaysRefreshesRunner guards the fix for the +// stale-baked-runner bug: a runner baked into the base image goes stale after +// the pinned version is bumped (GitHub deprecates it, the broker refuses the +// JIT runner, and the VM job loops forever "queued"). The setup script must +// always overwrite the run dir with the freshly-provided runner and kill any +// already-running one first — never fall back to a baked copy when a fresh +// one is available. +func TestMacOSRunnerSetupScript_AlwaysRefreshesRunner(t *testing.T) { + // Must kill any already-running (baked/LaunchDaemon) runner first. + if !strings.Contains(macOSRunnerSetupScript, "pkill -f Runner.Listener") { + t.Error("setup script must kill any pre-existing runner before starting the fresh one") + } + + // Must remove the (possibly stale) run dir and re-copy from the fresh src. + if !strings.Contains(macOSRunnerSetupScript, `rm -rf "$RUNNER_DIR"`) { + t.Error("setup script must remove the run dir so a stale baked runner can't be reused") + } + if !strings.Contains(macOSRunnerSetupScript, `cp -R "$RUNNER_SRC" "$RUNNER_DIR"`) { + t.Error("setup script must copy the freshly-provided runner into the run dir") + } + + // The old skip-if-exists guard must never come back — it silently ran the + // stale baked runner. + if strings.Contains(macOSRunnerSetupScript, `[ ! -f "$RUNNER_DIR/run.sh" ]`) { + t.Error("setup script must NOT skip the copy when a run.sh already exists (that reuses the stale baked runner)") + } + + // The overwrite must be gated on the fresh source actually existing, so we + // never delete the baked fallback when ephemerd provided nothing. + if !strings.Contains(macOSRunnerSetupScript, `if [ -f "$RUNNER_SRC/run.sh" ]; then`) { + t.Error("setup script must gate the overwrite on the fresh runner source existing") + } +} + +// TestMacOSRunnerSetupScript_TakesJITConfig confirms the template still has +// exactly one %s (the JIT config) so fmt.Sprintf wiring stays correct. +func TestMacOSRunnerSetupScript_TakesJITConfig(t *testing.T) { + if got := strings.Count(macOSRunnerSetupScript, "%s"); got != 1 { + t.Fatalf("setup script should contain exactly one %%s (JIT config), got %d", got) + } + out := fmt.Sprintf(macOSRunnerSetupScript, "JITDATA") + if !strings.Contains(out, "--jitconfig 'JITDATA'") { + t.Errorf("formatted script should embed the JIT config in the run.sh invocation:\n%s", out) + } +}