Integrate authenticated guest reconnect and guarded runner recovery

cmd/rainier/lifecycle.go

@@ -115,7 +115,7 @@ func eligibilityText(verdict, note string) string {
 // yet" from "never".
 func attachNote(s session) string {
 	switch s.State {
-	case "queued", "creating":
+	case "queued", "creating", "resuming":
 		return "waits for it to start"
 	case "suspended_warm", "suspended_cold":
 		return "resumes it first"
@@ -141,7 +141,7 @@ func stopNote(s session) string {
 		return "this build does not know this state; the server decides"
 	}
 	switch s.State {
-	case "queued", "creating":
+	case "queued", "creating", "resuming":
 		return "only a running session can be stopped"
 	case "suspended_warm", "suspended_cold":
 		return "already stopped"

cmd/rainier/sessionstate.go

@@ -77,7 +77,7 @@ const (
 // exited would be the collapse this file exists to prevent.
 func lifecycleOf(s session) string {
 	switch s.State {
-	case "queued", "creating":
+	case "queued", "creating", "resuming":
 		return lifecycleStarting
 	case "running":
 		// Regardless of child_exit_code. The sandbox is up.
@@ -188,7 +188,7 @@ const (
 // Everything else is refused by the endpoint.
 func canAttach(s session) string {
 	switch s.State {
-	case "running", "queued", "creating", "suspended_warm", "suspended_cold":
+	case "running", "queued", "creating", "resuming", "suspended_warm", "suspended_cold":
 		return eligibleYes
 	case "failed":
 		if s.Reachable {
@@ -216,7 +216,7 @@ func canStop(s session) string {
 	switch s.State {
 	case "running":
 		return eligibleYes
-	case "queued", "creating", "suspended_warm", "suspended_cold",
+	case "queued", "creating", "resuming", "suspended_warm", "suspended_cold",
 		"failed", "dead", "canceled", "destroyed":
 		return eligibleNo
 	default:
@@ -231,7 +231,7 @@ func canStop(s session) string {
 // a live container and deleting it is the cleanup.
 func canDelete(s session) string {
 	switch s.State {
-	case "running", "queued", "suspended_warm", "suspended_cold",
+	case "running", "queued", "resuming", "suspended_warm", "suspended_cold",
 		"failed", "dead", "canceled":
 		return eligibleYes
 	case "creating":

cmd/rainier/sessionstate_test.go

@@ -28,6 +28,7 @@ func TestThreeDimensions(t *testing.T) {
 			"creating", session{State: "creating", Reachable: true},
 			lifecycleStarting, processNone, connectionAvailable,
 		},
+		{"resuming", session{State: "resuming", Reachable: true}, lifecycleStarting, processNone, connectionAvailable},
 		{
 			"running with a live child", session{State: "running", Reachable: true},
 			lifecycleRunning, processRunning, connectionAvailable,
@@ -159,6 +160,7 @@ func TestActionEligibilityFollowsRawStates(t *testing.T) {
 			// it with a conflict because a dispatch may be in flight.
 			"creating", session{State: "creating"}, eligibleYes, eligibleNo, eligibleNo,
 		},
+		{"resuming", session{State: "resuming"}, eligibleYes, eligibleNo, eligibleYes},
 		{"suspended warm", session{State: "suspended_warm"}, eligibleYes, eligibleNo, eligibleYes},
 		{"suspended cold", session{State: "suspended_cold"}, eligibleYes, eligibleNo, eligibleYes},
 		{

cmd/rainier/wait.go

@@ -45,7 +45,7 @@ func initialAttachGuidance(ctx context.Context, cfg cli.Config, id string, since
 	} else {
 		row := resp.Session
 		switch row.State {
-		case "queued", "creating":
+		case "queued", "creating", "resuming":
 			explanation = "session is " + row.State + " and not ready to attach"
 			if row.QueueReason != "" {
 				// Queue text is server-owned prose. Redact known credentials before

cmd/runnerd/main.go

@@ -34,6 +34,8 @@ func main() {
 		"bearer token for the controld dial (required when --controld is set; or set RAINIER_RUNNER_TOKEN, which keeps it out of the process list)")
 	driverFlag := flag.String("driver", envDefault("RAINIER_RUNNER_DRIVER", "docker"),
 		"execution driver for session sandboxes: docker | microvm")
+	microvmGuestReconnect := flag.Bool("microvm-guest-reconnect", os.Getenv("RAINIER_MICROVM_GUEST_RECONNECT") == "1",
+		"opt in to authenticated recovery of surviving microVM guests after control connection or runner restart; requires a matching control plane and guest image")
 	kernelPath := flag.String("kernel", envDefault("RAINIER_KERNEL_PATH", ""),
 		"guest vmlinux kernel path for the microvm driver (required when --driver=microvm; the runner refuses to start without a readable one)")
 	rootfsPath := flag.String("rootfs", envDefault("RAINIER_ROOTFS_PATH", ""),
@@ -149,14 +151,15 @@ func main() {
 			log.Fatalf("--driver=microvm: %v", err)
 		}
 		mvm, err := driver.NewMicrovm(driver.MicrovmOpts{
-			Checkpoint:  ckpt,
-			KernelPath:  *kernelPath,
-			BaseRootfs:  *rootfsPath,
-			StateDir:    *microvmStateDir,
-			TotalSlots:  *slots,
-			VCPU:        *microvmVCPUs,
-			MemoryMiB:   *microvmMemoryMiB,
-			ImageSource: imageSource,
+			GuestReconnect: *microvmGuestReconnect,
+			Checkpoint:     ckpt,
+			KernelPath:     *kernelPath,
+			BaseRootfs:     *rootfsPath,
+			StateDir:       *microvmStateDir,
+			TotalSlots:     *slots,
+			VCPU:           *microvmVCPUs,
+			MemoryMiB:      *microvmMemoryMiB,
+			ImageSource:    imageSource,
 
 			SlotGuestCIDR:  *microvmGuestCIDR,
 			SlotUplinkCIDR: *microvmUplinkCIDR,

cmd/sessiond/bootlaunch.go

@@ -0,0 +1,31 @@
+package main
+
+import (
+	"context"
+	"slices"
+
+	"github.com/tokencanopy/rainier/internal/relay"
+	"github.com/tokencanopy/rainier/protocol/runner"
+)
+
+// guestBoot is the launch state consumed by main before preparing the boot chain.
+type guestBoot struct {
+	conn    relay.Conn
+	config  runner.BootConfig
+	argv    []string
+	failure *bootFailure
+}
+
+func bootGuest(ctx context.Context, dial dialSession, boots *bootstrapper, imageCommand []string) (guestBoot, error) {
+	conn, cfg, failure, err := bootOverVsock(ctx, dial, boots)
+	if err != nil {
+		return guestBoot{}, err
+	}
+	// The image command only supplies a fallback. A microVM's per-session
+	// command arrives on the authenticated boot channel, not in PID 1's argv.
+	command := imageCommand
+	if len(cfg.Cmd) > 0 {
+		command = cfg.Cmd
+	}
+	return guestBoot{conn: conn, config: cfg, argv: slices.Clone(command), failure: failure}, nil
+}

cmd/sessiond/bootlaunch_test.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"context"
+	"os/exec"
+	"testing"
+	"time"
+
+	"github.com/tokencanopy/rainier/protocol/runner"
+)
+
+func TestGuestBootLaunchesRequestedCommand(t *testing.T) {
+	for _, requested := range []bool{true, false} {
+		name := "image_fallback"
+		if requested {
+			name = "requested_command"
+		}
+		t.Run(name, func(t *testing.T) {
+			cleanEnv(t)
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			defer cancel()
+			dial, hosts := fakeTransport(t)
+			type result struct {
+				boot guestBoot
+				err  error
+			}
+			done := make(chan result, 1)
+			go func() {
+				boot, err := bootGuest(ctx, dial, &bootstrapper{}, []string{"/bin/sh", "-c", "printf image_default"})
+				done <- result{boot, err}
+			}()
+			host := <-hosts
+			cfg := runner.BootConfig{Protocol: runner.SessionBootstrapProtocolVersion, SessionID: "session_command_test"}
+			want := "image_default"
+			if requested {
+				// Shell-looking text must remain one literal argument, not be re-parsed.
+				want = "requested ; $(false) ' quoted"
+				cfg.Cmd = []string{"/bin/sh", "-c", `printf '%s' "$1"`, "command-test", want}
+			}
+			host.sendBootConfig(cfg)
+			got := <-done
+			if got.err != nil || got.boot.failure != nil {
+				t.Fatalf("guest boot failed: %v %v", got.err, got.boot.failure)
+			}
+			defer got.boot.conn.Close()
+			output, err := exec.CommandContext(ctx, got.boot.argv[0], got.boot.argv[1:]...).Output()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if string(output) != want {
+				t.Fatalf("launched output %q, want %q", output, want)
+			}
+		})
+	}
+}

cmd/sessiond/main.go

@@ -112,20 +112,21 @@ func main() {
 	if *transport == transportVsock {
 		dialer = vsockTransport()
 		preamble = func(ctx context.Context, c relay.Conn) error { return reBootstrap(ctx, c, boots) }
-		conn, cfg, failure, err := bootOverVsock(context.Background(), dialer, boots)
+		boot, err := bootGuest(context.Background(), dialer, boots, argv)
 		if err != nil {
 			// Deliberately fatal, and the same judgement prepareBoot's own
 			// failure gets: a guest that could not read its configuration
 			// does not know what session it is, what to run, or where its
 			// egress goes. Dying is what makes the runner notice.
 			log.Fatalf("microvm boot: %v", err)
 		}
-		firstConn, overVsock = conn, true
+		firstConn, overVsock = boot.conn, true
+		argv = boot.argv
 		if *sessionID == "" {
-			*sessionID = cfg.SessionID
+			*sessionID = boot.config.SessionID
 		}
-		if failure != nil {
-			secretsFailure = failure.Error()
+		if boot.failure != nil {
+			secretsFailure = boot.failure.Error()
 		}
 	} else if *transport != transportWebSocket {
 		log.Fatalf("unknown --transport %q (valid: %s, %s)", *transport, transportWebSocket, transportVsock)

cmd/sessiond/reconnect.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"crypto/ed25519"
 	"crypto/rand"
+	"crypto/sha256"
 	"encoding/base64"
 	"encoding/json"
 	"errors"
@@ -29,6 +30,7 @@ type guestReconnectIdentity struct {
 	key           ed25519.PrivateKey
 	enrolled      bool
 	epoch         uint64
+	launch        [32]byte
 }
 
 func (b *bootstrapper) enrollGuest(ctx context.Context, c relay.Conn, cfg runner.BootConfig) (map[string]string, error) {
@@ -38,7 +40,7 @@ func (b *bootstrapper) enrollGuest(ctx context.Context, c relay.Conn, cfg runner
 		return nil, errGuestReconnect
 	}
 	// A failed or unsupported opt-in is sticky: later dials cannot downgrade.
-	g := &guestReconnectIdentity{session: cfg.SessionID}
+	g := &guestReconnectIdentity{session: cfg.SessionID, launch: guestLaunchIdentity(cfg)}
 	b.guest = g
 	b.mu.Unlock()
 	if cfg.GuestReconnect != runner.GuestReconnectProtocol || !guestID(cfg.SessionID) {
@@ -106,7 +108,7 @@ func (b *bootstrapper) reconnectGuest(ctx context.Context, c relay.Conn) error {
 	delivery, cancel := context.WithTimeout(ctx, time.Duration(accepted.ExpiresInSec)*time.Second)
 	defer cancel()
 	cfg, err := readBootConfig(delivery, c)
-	if err != nil || cfg.SessionID != session || cfg.GuestReconnect != 1 || cfg.BootstrapToken != accepted.Token {
+	if err != nil || cfg.SessionID != session || cfg.GuestReconnect != 1 || cfg.BootstrapToken != accepted.Token || guestLaunchIdentity(cfg) != g.launch {
 		return errGuestReconnect
 	}
 	b.mu.Lock()
@@ -335,3 +337,22 @@ func guestReady(ctx context.Context, c relay.Conn, epoch uint64) error {
 	}
 	return nil
 }
+
+// The live boot retains only a digest of fields whose consumers run once.
+// Repositories, git attribution, setup, proxy policy and agent custody paths
+// cannot be refreshed merely by changing sessiond's environment. A changed
+// launch requires a new boot; ordinary environment values and secret names may
+// still refresh. The digest never reaches the host or durable storage.
+func guestLaunchIdentity(cfg runner.BootConfig) [32]byte {
+	cfg.BootstrapToken = ""
+	cfg.SecretNames = nil
+	launchEnv := map[string]string{}
+	for key, value := range cfg.Env {
+		if key == "RAINIER_AGENTS_B64" || strings.HasPrefix(value, "/rainier/agents/") {
+			launchEnv[key] = value
+		}
+	}
+	cfg.Env = launchEnv
+	body, _ := json.Marshal(cfg)
+	return sha256.Sum256(body)
+}

cmd/sessiond/reconnect_launch_test.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"github.com/tokencanopy/rainier/protocol/runner"
+	"testing"
+)
+
+func TestReconnectRejectsChangedBootArtifactsBeforeRedemption(t *testing.T) {
+	for _, field := range []string{"command", "repository", "git", "setup", "proxy", "agent-manifest"} {
+		t.Run(field, func(t *testing.T) {
+			b, ch := reconnectFixture(t)
+			guest, host, ctx := reconnectPair(t)
+			done := make(chan error, 1)
+			go func() { done <- reBootstrap(ctx, guest, b); guest.Close() }()
+			accepted := acceptProof(t, ctx, host, b, ch, 1)
+			cfg := runner.BootConfig{Protocol: 1, GuestReconnect: 1, SessionID: "session-test", BootstrapToken: accepted.Token}
+			switch field {
+			case "command":
+				cfg.Cmd = []string{"different_test"}
+			case "repository":
+				cfg.Repos = []runner.RepoSpec{{Owner: "example", Name: "changed"}}
+			case "git":
+				cfg.GitAuthorName = "Changed Test"
+			case "setup":
+				cfg.Setup = "echo changed_test"
+			case "proxy":
+				cfg.ProxyURL = "http://proxy.example.invalid:80"
+			case "agent-manifest":
+				cfg.Env = map[string]string{"RAINIER_AGENTS_B64": "changed_test"}
+			}
+			sendReconnectConfig(t, ctx, host, cfg)
+			if raw, err := host.Read(ctx); err == nil || len(raw) != 0 {
+				t.Fatal("changed launch artifacts reached token redemption")
+			}
+			if err := <-done; err != errGuestReconnect {
+				t.Fatal("changed launch artifacts accepted")
+			}
+		})
+	}
+}

cmd/sessiond/reconnect_test.go

@@ -75,7 +75,7 @@ func reconnectFixture(t *testing.T) (*bootstrapper, runner.GuestReconnectChallen
 	if err != nil {
 		t.Fatal(err)
 	}
-	return &bootstrapper{guest: &guestReconnectIdentity{session: "session-test", boot: "boot-test", key: key, enrolled: true}}, runner.GuestReconnectChallenge{Protocol: 1, SessionID: "session-test", BootEpoch: "boot-test", HostIncarnation: "host-test", AttemptID: "attempt-test", PlacementGeneration: 1, Challenge: base64.RawURLEncoding.EncodeToString(make([]byte, 32))}
+	return &bootstrapper{guest: &guestReconnectIdentity{session: "session-test", boot: "boot-test", key: key, enrolled: true, launch: guestLaunchIdentity(runner.BootConfig{Protocol: 1, GuestReconnect: 1, SessionID: "session-test"})}}, runner.GuestReconnectChallenge{Protocol: 1, SessionID: "session-test", BootEpoch: "boot-test", HostIncarnation: "host-test", AttemptID: "attempt-test", PlacementGeneration: 1, Challenge: base64.RawURLEncoding.EncodeToString(make([]byte, 32))}
 }
 func writeReconnect(t *testing.T, ctx context.Context, c relay.Conn, kind string, v any) {
 	t.Helper()

control/contract_test.go

@@ -185,6 +185,7 @@ func TestSessionStateVocabulary(t *testing.T) {
 	values := map[control.SessionState]string{
 		control.StateQueued:        "queued",
 		control.StateCreating:      "creating",
+		control.StateResuming:      "resuming",
 		control.StateRunning:       "running",
 		control.StateSuspendedWarm: "suspended_warm",
 		control.StateSuspendedCold: "suspended_cold",
@@ -200,7 +201,7 @@ func TestSessionStateVocabulary(t *testing.T) {
 	}
 
 	terminalStates := map[control.SessionState]bool{
-		control.StateQueued: false, control.StateCreating: false, control.StateRunning: false,
+		control.StateQueued: false, control.StateCreating: false, control.StateResuming: false, control.StateRunning: false,
 		control.StateSuspendedWarm: false, control.StateSuspendedCold: false,
 		control.StateCanceled: true, control.StateFailed: true,
 		control.StateDead: true, control.StateDestroyed: true,
@@ -213,7 +214,7 @@ func TestSessionStateVocabulary(t *testing.T) {
 
 	slotStates := map[control.SessionState]bool{
 		control.StateQueued:   false,
-		control.StateCreating: true, control.StateRunning: true,
+		control.StateCreating: true, control.StateResuming: true, control.StateRunning: true,
 		control.StateSuspendedWarm: true, control.StateSuspendedCold: false,
 		control.StateCanceled: false, control.StateFailed: false,
 		control.StateDead: false, control.StateDestroyed: false,
@@ -225,7 +226,7 @@ func TestSessionStateVocabulary(t *testing.T) {
 	}
 
 	wantOrder := []control.SessionState{
-		control.StateQueued, control.StateCreating, control.StateRunning,
+		control.StateQueued, control.StateCreating, control.StateResuming, control.StateRunning,
 		control.StateSuspendedWarm, control.StateSuspendedCold,
 	}
 	if len(control.NonTerminal) != len(wantOrder) {