fix(macos-vm): always run ephemerd fresh runner, not a stale baked copy

pkg/vm/macosvm_darwin.go

@@ -576,6 +576,51 @@ func (m *darwinMacOSVM) injectRunnerIntoClone(ctx context.Context) error {
 	return nil
 }
 
+// macOSRunnerSetupScript is the bash run over SSH to start the GHA runner in a
+// per-job macOS VM. It takes one %s: the base64 JIT config.
+//
+// Key invariant: it ALWAYS refreshes /Users/admin/actions-runner from the
+// freshly-copied /Library/ephemerd/runner and kills any already-running
+// runner first. A runner baked into the base image at provisioning time goes
+// stale when the pinned runner version is bumped — GitHub deprecates the old
+// version, its broker refuses the JIT runner, so the runner registers and
+// reports "ready" but is never assigned its job and the VM job loops forever.
+// Native jobs are unaffected (they run the host runner directly). An earlier
+// revision only copied when no run.sh already existed, which silently used the
+// stale baked runner; never reintroduce that guard.
+const macOSRunnerSetupScript = `
+# Firewall: block private networks EXCEPT the Vz NAT subnet
+cat > /tmp/pf-ephemerd.conf << 'PFEOF'
+pass quick to 192.168.64.0/24
+block out quick to 10.0.0.0/8
+block out quick to 172.16.0.0/12
+block out quick to 192.168.0.0/16
+block out quick to 169.254.0.0/16
+pass out all
+PFEOF
+sudo pfctl -f /tmp/pf-ephemerd.conf -e 2>/dev/null || true
+
+# Always run the runner ephemerd just provided, not a stale baked copy.
+RUNNER_SRC="/Library/ephemerd/runner"
+RUNNER_DIR="/Users/admin/actions-runner"
+pkill -f Runner.Listener 2>/dev/null || true
+if [ -f "$RUNNER_SRC/run.sh" ]; then
+  rm -rf "$RUNNER_DIR"
+  cp -R "$RUNNER_SRC" "$RUNNER_DIR"
+  chown -R admin:staff "$RUNNER_DIR"
+fi
+cd "$RUNNER_DIR"
+./run.sh --jitconfig '%s' </dev/null >/tmp/runner.log 2>&1 &
+RUNNER_PID=$!
+disown $RUNNER_PID 2>/dev/null || true
+
+# Randomize password LAST
+RAND_PASS=$(head -c 32 /dev/urandom | base64 | tr -d '/+=' | head -c 32)
+dscl . -passwd /Users/admin admin "$RAND_PASS" 2>/dev/null || true
+
+echo "runner started (pid=$RUNNER_PID)"
+`
+
 // setupRunnerViaSSH connects to the VM using the Tart default credentials
 // (admin/admin) and starts the GitHub Actions runner. This is more reliable
 // than LaunchDaemons which may be blocked by SIP/SSV on modern macOS.
@@ -693,39 +738,10 @@ func (m *darwinMacOSVM) setupRunnerViaSSH(ctx context.Context, ip string) error
 	}
 
 	// Start the runner + firewall in the background (fire and forget).
-	// Runner binary is pre-installed in the base image (inherited by clone).
-	// Only JIT config is per-job, passed inline.
-	setupScript := fmt.Sprintf(`
-# Firewall: block private networks EXCEPT the Vz NAT subnet
-cat > /tmp/pf-ephemerd.conf << 'PFEOF'
-pass quick to 192.168.64.0/24
-block out quick to 10.0.0.0/8
-block out quick to 172.16.0.0/12
-block out quick to 192.168.0.0/16
-block out quick to 169.254.0.0/16
-pass out all
-PFEOF
-sudo pfctl -f /tmp/pf-ephemerd.conf -e 2>/dev/null || true
-
-# Runner is pre-installed at /Library/ephemerd/runner/ (Data volume).
-# Copy to home dir so the runner has a writable work directory.
-RUNNER_SRC="/Library/ephemerd/runner"
-RUNNER_DIR="/Users/admin/actions-runner"
-if [ ! -f "$RUNNER_DIR/run.sh" ] && [ -f "$RUNNER_SRC/run.sh" ]; then
-  cp -R "$RUNNER_SRC" "$RUNNER_DIR"
-  chown -R admin:staff "$RUNNER_DIR"
-fi
-cd "$RUNNER_DIR"
-./run.sh --jitconfig '%s' </dev/null >/tmp/runner.log 2>&1 &
-RUNNER_PID=$!
-disown $RUNNER_PID 2>/dev/null || true
-
-# Randomize password LAST
-RAND_PASS=$(head -c 32 /dev/urandom | base64 | tr -d '/+=' | head -c 32)
-dscl . -passwd /Users/admin admin "$RAND_PASS" 2>/dev/null || true
-
-echo "runner started (pid=$RUNNER_PID)"
-`, strings.TrimSpace(string(jitData)))
+	// The runner is copied fresh into the VM before this runs; see
+	// macOSRunnerSetupScript for why we always refresh it. Only the JIT
+	// config is per-job, passed inline.
+	setupScript := fmt.Sprintf(macOSRunnerSetupScript, strings.TrimSpace(string(jitData)))
 
 	session, err := client.NewSession()
 	if err != nil {

pkg/vm/macosvm_darwin_test.go

@@ -0,0 +1,55 @@
+//go:build darwin
+
+package vm
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+)
+
+// TestMacOSRunnerSetupScript_AlwaysRefreshesRunner guards the fix for the
+// stale-baked-runner bug: a runner baked into the base image goes stale after
+// the pinned version is bumped (GitHub deprecates it, the broker refuses the
+// JIT runner, and the VM job loops forever "queued"). The setup script must
+// always overwrite the run dir with the freshly-provided runner and kill any
+// already-running one first — never fall back to a baked copy when a fresh
+// one is available.
+func TestMacOSRunnerSetupScript_AlwaysRefreshesRunner(t *testing.T) {
+	// Must kill any already-running (baked/LaunchDaemon) runner first.
+	if !strings.Contains(macOSRunnerSetupScript, "pkill -f Runner.Listener") {
+		t.Error("setup script must kill any pre-existing runner before starting the fresh one")
+	}
+
+	// Must remove the (possibly stale) run dir and re-copy from the fresh src.
+	if !strings.Contains(macOSRunnerSetupScript, `rm -rf "$RUNNER_DIR"`) {
+		t.Error("setup script must remove the run dir so a stale baked runner can't be reused")
+	}
+	if !strings.Contains(macOSRunnerSetupScript, `cp -R "$RUNNER_SRC" "$RUNNER_DIR"`) {
+		t.Error("setup script must copy the freshly-provided runner into the run dir")
+	}
+
+	// The old skip-if-exists guard must never come back — it silently ran the
+	// stale baked runner.
+	if strings.Contains(macOSRunnerSetupScript, `[ ! -f "$RUNNER_DIR/run.sh" ]`) {
+		t.Error("setup script must NOT skip the copy when a run.sh already exists (that reuses the stale baked runner)")
+	}
+
+	// The overwrite must be gated on the fresh source actually existing, so we
+	// never delete the baked fallback when ephemerd provided nothing.
+	if !strings.Contains(macOSRunnerSetupScript, `if [ -f "$RUNNER_SRC/run.sh" ]; then`) {
+		t.Error("setup script must gate the overwrite on the fresh runner source existing")
+	}
+}
+
+// TestMacOSRunnerSetupScript_TakesJITConfig confirms the template still has
+// exactly one %s (the JIT config) so fmt.Sprintf wiring stays correct.
+func TestMacOSRunnerSetupScript_TakesJITConfig(t *testing.T) {
+	if got := strings.Count(macOSRunnerSetupScript, "%s"); got != 1 {
+		t.Fatalf("setup script should contain exactly one %%s (JIT config), got %d", got)
+	}
+	out := fmt.Sprintf(macOSRunnerSetupScript, "JITDATA")
+	if !strings.Contains(out, "--jitconfig 'JITDATA'") {
+		t.Errorf("formatted script should embed the JIT config in the run.sh invocation:\n%s", out)
+	}
+}