Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cmd/rainier/lifecycle.go
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ func eligibilityText(verdict, note string) string {
// yet" from "never".
func attachNote(s session) string {
switch s.State {
case "queued", "creating":
case "queued", "creating", "resuming":
return "waits for it to start"
case "suspended_warm", "suspended_cold":
return "resumes it first"
Expand All @@ -141,7 +141,7 @@ func stopNote(s session) string {
return "this build does not know this state; the server decides"
}
switch s.State {
case "queued", "creating":
case "queued", "creating", "resuming":
return "only a running session can be stopped"
case "suspended_warm", "suspended_cold":
return "already stopped"
Expand Down
8 changes: 4 additions & 4 deletions cmd/rainier/sessionstate.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ const (
// exited would be the collapse this file exists to prevent.
func lifecycleOf(s session) string {
switch s.State {
case "queued", "creating":
case "queued", "creating", "resuming":
return lifecycleStarting
case "running":
// Regardless of child_exit_code. The sandbox is up.
Expand Down Expand Up @@ -188,7 +188,7 @@ const (
// Everything else is refused by the endpoint.
func canAttach(s session) string {
switch s.State {
case "running", "queued", "creating", "suspended_warm", "suspended_cold":
case "running", "queued", "creating", "resuming", "suspended_warm", "suspended_cold":
return eligibleYes
case "failed":
if s.Reachable {
Expand Down Expand Up @@ -216,7 +216,7 @@ func canStop(s session) string {
switch s.State {
case "running":
return eligibleYes
case "queued", "creating", "suspended_warm", "suspended_cold",
case "queued", "creating", "resuming", "suspended_warm", "suspended_cold",
"failed", "dead", "canceled", "destroyed":
return eligibleNo
default:
Expand All @@ -231,7 +231,7 @@ func canStop(s session) string {
// a live container and deleting it is the cleanup.
func canDelete(s session) string {
switch s.State {
case "running", "queued", "suspended_warm", "suspended_cold",
case "running", "queued", "resuming", "suspended_warm", "suspended_cold",
"failed", "dead", "canceled":
return eligibleYes
case "creating":
Expand Down
2 changes: 2 additions & 0 deletions cmd/rainier/sessionstate_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ func TestThreeDimensions(t *testing.T) {
"creating", session{State: "creating", Reachable: true},
lifecycleStarting, processNone, connectionAvailable,
},
{"resuming", session{State: "resuming", Reachable: true}, lifecycleStarting, processNone, connectionAvailable},
{
"running with a live child", session{State: "running", Reachable: true},
lifecycleRunning, processRunning, connectionAvailable,
Expand Down Expand Up @@ -159,6 +160,7 @@ func TestActionEligibilityFollowsRawStates(t *testing.T) {
// it with a conflict because a dispatch may be in flight.
"creating", session{State: "creating"}, eligibleYes, eligibleNo, eligibleNo,
},
{"resuming", session{State: "resuming"}, eligibleYes, eligibleNo, eligibleYes},
{"suspended warm", session{State: "suspended_warm"}, eligibleYes, eligibleNo, eligibleYes},
{"suspended cold", session{State: "suspended_cold"}, eligibleYes, eligibleNo, eligibleYes},
{
Expand Down
2 changes: 1 addition & 1 deletion cmd/rainier/wait.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func initialAttachGuidance(ctx context.Context, cfg cli.Config, id string, since
} else {
row := resp.Session
switch row.State {
case "queued", "creating":
case "queued", "creating", "resuming":
explanation = "session is " + row.State + " and not ready to attach"
if row.QueueReason != "" {
// Queue text is server-owned prose. Redact known credentials before
Expand Down
19 changes: 11 additions & 8 deletions cmd/runnerd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ func main() {
"bearer token for the controld dial (required when --controld is set; or set RAINIER_RUNNER_TOKEN, which keeps it out of the process list)")
driverFlag := flag.String("driver", envDefault("RAINIER_RUNNER_DRIVER", "docker"),
"execution driver for session sandboxes: docker | microvm")
microvmGuestReconnect := flag.Bool("microvm-guest-reconnect", os.Getenv("RAINIER_MICROVM_GUEST_RECONNECT") == "1",
"opt in to authenticated recovery of surviving microVM guests after control connection or runner restart; requires a matching control plane and guest image")
kernelPath := flag.String("kernel", envDefault("RAINIER_KERNEL_PATH", ""),
"guest vmlinux kernel path for the microvm driver (required when --driver=microvm; the runner refuses to start without a readable one)")
rootfsPath := flag.String("rootfs", envDefault("RAINIER_ROOTFS_PATH", ""),
Expand Down Expand Up @@ -149,14 +151,15 @@ func main() {
log.Fatalf("--driver=microvm: %v", err)
}
mvm, err := driver.NewMicrovm(driver.MicrovmOpts{
Checkpoint: ckpt,
KernelPath: *kernelPath,
BaseRootfs: *rootfsPath,
StateDir: *microvmStateDir,
TotalSlots: *slots,
VCPU: *microvmVCPUs,
MemoryMiB: *microvmMemoryMiB,
ImageSource: imageSource,
GuestReconnect: *microvmGuestReconnect,
Checkpoint: ckpt,
KernelPath: *kernelPath,
BaseRootfs: *rootfsPath,
StateDir: *microvmStateDir,
TotalSlots: *slots,
VCPU: *microvmVCPUs,
MemoryMiB: *microvmMemoryMiB,
ImageSource: imageSource,

SlotGuestCIDR: *microvmGuestCIDR,
SlotUplinkCIDR: *microvmUplinkCIDR,
Expand Down
31 changes: 31 additions & 0 deletions cmd/sessiond/bootlaunch.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package main

import (
"context"
"slices"

"github.com/tokencanopy/rainier/internal/relay"
"github.com/tokencanopy/rainier/protocol/runner"
)

// guestBoot is the launch state consumed by main before preparing the boot chain.
type guestBoot struct {
conn relay.Conn
config runner.BootConfig
argv []string
failure *bootFailure
}

func bootGuest(ctx context.Context, dial dialSession, boots *bootstrapper, imageCommand []string) (guestBoot, error) {
conn, cfg, failure, err := bootOverVsock(ctx, dial, boots)
if err != nil {
return guestBoot{}, err
}
// The image command only supplies a fallback. A microVM's per-session
// command arrives on the authenticated boot channel, not in PID 1's argv.
command := imageCommand
if len(cfg.Cmd) > 0 {
command = cfg.Cmd
}
return guestBoot{conn: conn, config: cfg, argv: slices.Clone(command), failure: failure}, nil
}
55 changes: 55 additions & 0 deletions cmd/sessiond/bootlaunch_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
package main

import (
"context"
"os/exec"
"testing"
"time"

"github.com/tokencanopy/rainier/protocol/runner"
)

func TestGuestBootLaunchesRequestedCommand(t *testing.T) {
for _, requested := range []bool{true, false} {
name := "image_fallback"
if requested {
name = "requested_command"
}
t.Run(name, func(t *testing.T) {
cleanEnv(t)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
dial, hosts := fakeTransport(t)
type result struct {
boot guestBoot
err error
}
done := make(chan result, 1)
go func() {
boot, err := bootGuest(ctx, dial, &bootstrapper{}, []string{"/bin/sh", "-c", "printf image_default"})
done <- result{boot, err}
}()
host := <-hosts
cfg := runner.BootConfig{Protocol: runner.SessionBootstrapProtocolVersion, SessionID: "session_command_test"}
want := "image_default"
if requested {
// Shell-looking text must remain one literal argument, not be re-parsed.
want = "requested ; $(false) ' quoted"
cfg.Cmd = []string{"/bin/sh", "-c", `printf '%s' "$1"`, "command-test", want}
}
host.sendBootConfig(cfg)
got := <-done
if got.err != nil || got.boot.failure != nil {
t.Fatalf("guest boot failed: %v %v", got.err, got.boot.failure)
}
defer got.boot.conn.Close()
output, err := exec.CommandContext(ctx, got.boot.argv[0], got.boot.argv[1:]...).Output()
if err != nil {
t.Fatal(err)
}
if string(output) != want {
t.Fatalf("launched output %q, want %q", output, want)
}
})
}
}
11 changes: 6 additions & 5 deletions cmd/sessiond/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -112,20 +112,21 @@ func main() {
if *transport == transportVsock {
dialer = vsockTransport()
preamble = func(ctx context.Context, c relay.Conn) error { return reBootstrap(ctx, c, boots) }
conn, cfg, failure, err := bootOverVsock(context.Background(), dialer, boots)
boot, err := bootGuest(context.Background(), dialer, boots, argv)
if err != nil {
// Deliberately fatal, and the same judgement prepareBoot's own
// failure gets: a guest that could not read its configuration
// does not know what session it is, what to run, or where its
// egress goes. Dying is what makes the runner notice.
log.Fatalf("microvm boot: %v", err)
}
firstConn, overVsock = conn, true
firstConn, overVsock = boot.conn, true
argv = boot.argv
if *sessionID == "" {
*sessionID = cfg.SessionID
*sessionID = boot.config.SessionID
}
if failure != nil {
secretsFailure = failure.Error()
if boot.failure != nil {
secretsFailure = boot.failure.Error()
}
} else if *transport != transportWebSocket {
log.Fatalf("unknown --transport %q (valid: %s, %s)", *transport, transportWebSocket, transportVsock)
Expand Down
49 changes: 46 additions & 3 deletions cmd/sessiond/reconnect.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"context"
"crypto/ed25519"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/json"
"errors"
Expand All @@ -29,6 +30,7 @@ type guestReconnectIdentity struct {
key ed25519.PrivateKey
enrolled bool
epoch uint64
launch [32]byte
}

func (b *bootstrapper) enrollGuest(ctx context.Context, c relay.Conn, cfg runner.BootConfig) (map[string]string, error) {
Expand All @@ -38,7 +40,7 @@ func (b *bootstrapper) enrollGuest(ctx context.Context, c relay.Conn, cfg runner
return nil, errGuestReconnect
}
// A failed or unsupported opt-in is sticky: later dials cannot downgrade.
g := &guestReconnectIdentity{session: cfg.SessionID}
g := &guestReconnectIdentity{session: cfg.SessionID, launch: guestLaunchIdentity(cfg)}
b.guest = g
b.mu.Unlock()
if cfg.GuestReconnect != runner.GuestReconnectProtocol || !guestID(cfg.SessionID) {
Expand Down Expand Up @@ -106,7 +108,7 @@ func (b *bootstrapper) reconnectGuest(ctx context.Context, c relay.Conn) error {
delivery, cancel := context.WithTimeout(ctx, time.Duration(accepted.ExpiresInSec)*time.Second)
defer cancel()
cfg, err := readBootConfig(delivery, c)
if err != nil || cfg.SessionID != session || cfg.GuestReconnect != 1 || cfg.BootstrapToken != accepted.Token {
if err != nil || cfg.SessionID != session || cfg.GuestReconnect != 1 || cfg.BootstrapToken != accepted.Token || guestLaunchIdentity(cfg) != g.launch {
return errGuestReconnect
}
b.mu.Lock()
Expand All @@ -127,7 +129,7 @@ func (b *bootstrapper) reconnectGuest(ctx context.Context, c relay.Conn) error {
if err = b.refreshConfiguration(delivery, cfg, env); err != nil {
return errGuestReconnect
}
return nil
return guestReady(delivery, c, accepted.Epoch)
}

func guestProof(ctx context.Context, c relay.Conn, session, boot string, key ed25519.PrivateKey, epoch uint64) (runner.GuestReconnectAcceptResponse, error) {
Expand Down Expand Up @@ -313,3 +315,44 @@ func (b *bootstrapper) bindExecEnvironment(execs *sandboxexec.Runner, extra []st
return ctx.Err()
}
}

// guestReady keeps terminal/exec and credential RPC offline after applying
// configuration until the host acknowledges this exact accepted epoch. A lost
// acknowledgment requires a new signed attempt, never replay of a spent token.
func guestReady(ctx context.Context, c relay.Conn, epoch uint64) error {
ctx, cancel := context.WithTimeout(ctx, guestHandshakeWait)
defer cancel()
ready := runner.GuestReconnectReady{Protocol: runner.GuestReconnectProtocol, Epoch: epoch}
body, _ := json.Marshal(ready)
if ctx.Err() != nil || relay.WriteGuestReconnectFrame(ctx, c, relay.KindGuestReconnectReady, body) != nil {
return errGuestReconnect
}
event, err := relay.ReadGuestReconnectFrame(ctx, c)
if err != nil || event.Kind != relay.KindGuestReconnectReadyAck {
return errGuestReconnect
}
acknowledgment, err := runner.DecodeGuestReconnectReady(event.Payload)
if err != nil || acknowledgment.Epoch != epoch || ctx.Err() != nil {
return errGuestReconnect
}
return nil
}

// The live boot retains only a digest of fields whose consumers run once.
// Repositories, git attribution, setup, proxy policy and agent custody paths
// cannot be refreshed merely by changing sessiond's environment. A changed
// launch requires a new boot; ordinary environment values and secret names may
// still refresh. The digest never reaches the host or durable storage.
func guestLaunchIdentity(cfg runner.BootConfig) [32]byte {
cfg.BootstrapToken = ""
cfg.SecretNames = nil
launchEnv := map[string]string{}
for key, value := range cfg.Env {
if key == "RAINIER_AGENTS_B64" || strings.HasPrefix(value, "/rainier/agents/") {
launchEnv[key] = value
}
}
cfg.Env = launchEnv
body, _ := json.Marshal(cfg)
return sha256.Sum256(body)
}
40 changes: 40 additions & 0 deletions cmd/sessiond/reconnect_launch_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package main

import (
"github.com/tokencanopy/rainier/protocol/runner"
"testing"
)

func TestReconnectRejectsChangedBootArtifactsBeforeRedemption(t *testing.T) {
for _, field := range []string{"command", "repository", "git", "setup", "proxy", "agent-manifest"} {
t.Run(field, func(t *testing.T) {
b, ch := reconnectFixture(t)
guest, host, ctx := reconnectPair(t)
done := make(chan error, 1)
go func() { done <- reBootstrap(ctx, guest, b); guest.Close() }()
accepted := acceptProof(t, ctx, host, b, ch, 1)
cfg := runner.BootConfig{Protocol: 1, GuestReconnect: 1, SessionID: "session-test", BootstrapToken: accepted.Token}
switch field {
case "command":
cfg.Cmd = []string{"different_test"}
case "repository":
cfg.Repos = []runner.RepoSpec{{Owner: "example", Name: "changed"}}
case "git":
cfg.GitAuthorName = "Changed Test"
case "setup":
cfg.Setup = "echo changed_test"
case "proxy":
cfg.ProxyURL = "http://proxy.example.invalid:80"
case "agent-manifest":
cfg.Env = map[string]string{"RAINIER_AGENTS_B64": "changed_test"}
}
sendReconnectConfig(t, ctx, host, cfg)
if raw, err := host.Read(ctx); err == nil || len(raw) != 0 {
t.Fatal("changed launch artifacts reached token redemption")
}
if err := <-done; err != errGuestReconnect {
t.Fatal("changed launch artifacts accepted")
}
})
}
}
Loading
Loading