runtime: prove isolation per backend; firecracker needs no krun

SECURITY.md

@@ -15,7 +15,10 @@ Two layers, checked separately.
 kernel. A guest kernel exploit takes over the guest, which is thrown away.
 `box build` proves a sandbox got its own kernel by comparing it with the
 kernel a plain container sees, and every `run`, `exec` and `up` re-checks that
-the runtime is still the one that passed.
+the runtime is still the one that passed. Each backend is proven on its own,
+against a fingerprint of only what it runs (krun and crun for podman and krun;
+the pinned Firecracker build and crun for firecracker), so a check passed on
+one backend never clears another.
 
 **The VMM's confinement.** libkrun's own documentation states that the guest
 and its VMM "pertain to the same security context": the VMM is a host process

internal/cli/commands.go

@@ -131,7 +131,7 @@ func verify(name string, s boxfile.Spec) error {
 		return err
 	}
 	// Prime the cache so runs on this same runtime skip the double boot.
-	runtime.MarkVerified(guest, baseline)
+	runtime.MarkVerified(s, guest, baseline)
 	fmt.Printf("isolated: sandbox %s, outside %s\n", guest, baseline)
 	return nil
 }

internal/runtime/doctor.go

@@ -8,6 +8,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	goruntime "runtime"
+	"sort"
 	"strings"
 
 	"box/internal/sandbox"
@@ -132,16 +133,25 @@ func Doctor() []Check {
 		add(Check{Name: "SDK socket", Warn: true, Detail: err.Error()})
 	}
 
-	if t, ok := loadToken(); ok {
-		id, _ := RuntimeIdentity()
-		if id == t.Identity {
-			add(Check{Name: "isolation", OK: true,
+	// Each backend's proof is its own; report every one that has been made.
+	cache := loadCache()
+	proven := make([]string, 0, len(cache.Backends))
+	for b := range cache.Backends {
+		proven = append(proven, b)
+	}
+	sort.Strings(proven)
+	for _, b := range proven {
+		t := cache.Backends[b]
+		name := "isolation (" + b + ")"
+		if id, _ := RuntimeIdentity(b); id == t.Identity {
+			add(Check{Name: name, OK: true,
 				Detail: fmt.Sprintf("verified %s: guest %s, outside %s", t.VerifiedAt, t.Guest, t.Baseline)})
 		} else {
-			add(Check{Name: "isolation", Warn: true,
+			add(Check{Name: name, Warn: true,
 				Detail: "the runtime changed since it was last verified; the next run re-checks it"})
 		}
-	} else {
+	}
+	if len(proven) == 0 {
 		add(Check{Name: "isolation", Warn: true, Detail: "not verified yet",
 			Fix: "box build <name>   proves a sandbox gets its own kernel"})
 	}

internal/runtime/report.go

@@ -163,7 +163,7 @@ func SecurityReport(name string, s boxfile.Spec) (Report, error) {
 		r.Network.Egress = "none"
 	}
 	r.Filesystem.Data = dataReport(name, s)
-	r.Host = hostReport()
+	r.Host = hostReport(b.Name())
 	r.Live = liveReport(name, b.Name(), r.VMM, r.Network.OwnNamespace)
 	r.Findings = findings(r, s)
 	return r, nil
@@ -371,7 +371,8 @@ func statInto(dr *DataReport, path string) {
 	dr.Mode = fmt.Sprintf("%o", fi.Mode().Perm())
 }
 
-func hostReport() HostReport {
+// hostReport describes the host, and the isolation proof for backend.
+func hostReport(backend string) HostReport {
 	h := HostReport{}
 	if f, err := os.OpenFile("/dev/kvm", os.O_RDWR, 0); err == nil {
 		f.Close()
@@ -380,9 +381,9 @@ func hostReport() HostReport {
 	if b, err := os.ReadFile("/proc/sys/kernel/osrelease"); err == nil {
 		h.Kernel = strings.TrimSpace(string(b))
 	}
-	if t, ok := loadToken(); ok {
+	if t, ok := loadToken(backend); ok {
 		h.GuestKernel, h.IsolationVerifiedAt = t.Guest, t.VerifiedAt
-		if id, err := RuntimeIdentity(); err == nil {
+		if id, err := RuntimeIdentity(backend); err == nil {
 			h.IsolationCurrent = t.Identity == id
 		}
 	}

internal/runtime/verify.go

@@ -45,15 +45,21 @@ func CheckIsolation(name string, s boxfile.Spec) (guest, baseline string, err er
 }
 
 // RuntimeIdentity fingerprints the parts of the toolchain that decide whether a
-// sandbox actually gets its own kernel. When any of them changes -- the krun
-// symlink is re-pointed, crun is upgraded, libkrun support is dropped, podman
-// changes -- the fingerprint changes and a cached verification no longer
-// applies. It is cheap (a couple of --version calls and a stat), so it can gate
-// every run without the cost of booting a VM.
-func RuntimeIdentity() (string, error) {
-	parts := []string{goruntime.GOOS + "/" + goruntime.GOARCH}
+// sandbox on backend actually gets its own kernel. When any of them changes --
+// the krun symlink is re-pointed, crun is upgraded, libkrun support is dropped,
+// the pinned Firecracker build changes, podman changes -- the fingerprint
+// changes and a cached verification no longer applies. It is cheap (a couple
+// of --version calls and a stat), so it can gate every run without the cost
+// of booting a VM.
+//
+// Each backend has its own: what proves the krun path says nothing about
+// Firecracker, and the reverse. It covers only what that backend runs, so a
+// host without krun can still use firecracker.
+func RuntimeIdentity(backend string) (string, error) {
+	parts := []string{goruntime.GOOS + "/" + goruntime.GOARCH, "backend:" + backend}
 
-	if goruntime.GOOS == "darwin" {
+	switch {
+	case goruntime.GOOS == "darwin":
 		// Isolation lives inside the podman machine VM, so its identity, not a
 		// host-side krun, is what matters.
 		out, err := exec.Command("podman", "machine", "inspect", "--format",
@@ -62,27 +68,25 @@ func RuntimeIdentity() (string, error) {
 			return "", fmt.Errorf("no podman machine to fingerprint: %w", err)
 		}
 		parts = append(parts, "machine:"+strings.TrimSpace(string(out)))
-	} else {
-		// The krun symlink and the crun binary it resolves to are the whole
-		// isolation path; capture where it points and the binary's identity so
-		// a re-pointed link or an upgraded crun both change the fingerprint.
-		krun, err := exec.LookPath("krun")
+	case backend == "firecracker":
+		// The VMM and guest kernel are pinned by hash and checked before every
+		// use (fcAssets), so the pins identify them; crun is what confines and
+		// runs the VMM.
+		parts = append(parts, "fc:"+fcVersion+":"+fcBinarySHA, "fckernel:"+fcKernelSHA, "fctools:"+fcToolsImage)
+		p, err := binIdentity("crun")
 		if err != nil {
-			return "", fmt.Errorf("no 'krun' on PATH")
+			return "", err
 		}
-		real, err := filepath.EvalSymlinks(krun)
+		parts = append(parts, p...)
+	default:
+		// The krun symlink and the crun binary it resolves to are the whole
+		// isolation path; the version string carries the +LIBKRUN tag, and
+		// losing it is exactly the degradation this guards against.
+		p, err := binIdentity("krun")
 		if err != nil {
-			real = krun
-		}
-		parts = append(parts, "krun:"+krun+"->"+real)
-		if fi, err := os.Stat(real); err == nil {
-			parts = append(parts, fmt.Sprintf("crun:%d:%d", fi.Size(), fi.ModTime().UnixNano()))
-		}
-		// The version string carries the +LIBKRUN tag; losing it is exactly the
-		// degradation this guards against.
-		if out, err := exec.Command("krun", "--version").Output(); err == nil {
-			parts = append(parts, "krunver:"+strings.TrimSpace(string(out)))
+			return "", err
 		}
+		parts = append(parts, p...)
 	}
 	if out, err := exec.Command("podman", "--version").Output(); err == nil {
 		parts = append(parts, "podman:"+strings.TrimSpace(string(out)))
@@ -91,6 +95,27 @@ func RuntimeIdentity() (string, error) {
 	return hex.EncodeToString(sum[:]), nil
 }
 
+// binIdentity is where bin resolves on PATH, through any symlink, and that
+// file's size, mtime and version: a re-pointed link or an upgrade changes it.
+func binIdentity(bin string) ([]string, error) {
+	path, err := exec.LookPath(bin)
+	if err != nil {
+		return nil, fmt.Errorf("no '%s' on PATH", bin)
+	}
+	real, err := filepath.EvalSymlinks(path)
+	if err != nil {
+		real = path
+	}
+	parts := []string{bin + ":" + path + "->" + real}
+	if fi, err := os.Stat(real); err == nil {
+		parts = append(parts, fmt.Sprintf("%s-file:%d:%d", bin, fi.Size(), fi.ModTime().UnixNano()))
+	}
+	if out, err := exec.Command(bin, "--version").Output(); err == nil {
+		parts = append(parts, bin+"-version:"+strings.TrimSpace(string(out)))
+	}
+	return parts, nil
+}
+
 // verifyToken is the cached result of a successful isolation check. A run
 // trusts it only while Identity still matches the current runtime.
 type verifyToken struct {
@@ -100,54 +125,82 @@ type verifyToken struct {
 	VerifiedAt string `json:"verified_at"`
 }
 
-func loadToken() (verifyToken, bool) {
+// verifyCache holds one token per backend. A check passed on one backend
+// never clears another: each boots its guest a different way.
+type verifyCache struct {
+	Backends map[string]verifyToken `json:"backends"`
+}
+
+// loadCache reads the cache. A missing, corrupt or older-format file (a
+// single token, from before tokens were per backend) reads as empty, which
+// costs one re-verify and never clears a run.
+func loadCache() verifyCache {
+	c := verifyCache{Backends: map[string]verifyToken{}}
 	p, err := sandbox.VerifyCachePath()
 	if err != nil {
-		return verifyToken{}, false
+		return c
 	}
 	b, err := os.ReadFile(p)
 	if err != nil {
-		return verifyToken{}, false
+		return c
 	}
-	var t verifyToken
-	if json.Unmarshal(b, &t) != nil {
-		return verifyToken{}, false
+	var got verifyCache
+	if json.Unmarshal(b, &got) != nil || got.Backends == nil {
+		return c
 	}
-	return t, true
+	return got
+}
+
+func loadToken(backend string) (verifyToken, bool) {
+	t, ok := loadCache().Backends[backend]
+	return t, ok && t.Identity != ""
 }
 
-// storeToken records a verification. It is best-effort: a cache-write failure
-// only costs a re-verify next time, so it must never fail a build or run.
-func storeToken(id, guest, baseline string) {
+// storeToken records a verification for backend. It is best-effort: a
+// cache-write failure only costs a re-verify next time, so it must never
+// fail a build or run.
+func storeToken(backend, id, guest, baseline string) {
 	p, err := sandbox.VerifyCachePath()
 	if err != nil {
 		return
 	}
 	if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
 		return
 	}
-	if b, err := json.Marshal(verifyToken{
+	c := loadCache()
+	c.Backends[backend] = verifyToken{
 		Identity:   id,
 		Guest:      guest,
 		Baseline:   baseline,
 		VerifiedAt: time.Now().UTC().Format(time.RFC3339),
-	}); err == nil {
-		os.WriteFile(p, b, 0o644)
+	}
+	if b, err := json.Marshal(c); err == nil {
+		// Beside and renamed, so a concurrent reader never sees half a file.
+		tmp := p + ".tmp"
+		if os.WriteFile(tmp, b, 0o644) == nil {
+			os.Rename(tmp, p)
+		}
 	}
 }
 
-// isolationCurrent reports whether the cached token already cleared runtime id.
-func isolationCurrent(id string) bool {
-	t, ok := loadToken()
+// isolationCurrent reports whether backend's cached token already cleared
+// runtime id.
+func isolationCurrent(backend, id string) bool {
+	t, ok := loadToken(backend)
 	return ok && t.Identity == id
 }
 
-// MarkVerified caches a successful isolation check for the current runtime, so
-// later runs can trust it without booting two more VMs. Callers that have just
-// proven isolation (build, the verify command) use it to prime the cache.
-func MarkVerified(guest, baseline string) {
-	if id, err := RuntimeIdentity(); err == nil {
-		storeToken(id, guest, baseline)
+// MarkVerified caches a successful isolation check of a sandbox with spec s,
+// for its backend and the current runtime, so later runs can trust it without
+// booting two more VMs. Callers that have just proven isolation (build, the
+// verify command) use it to prime the cache.
+func MarkVerified(s boxfile.Spec, guest, baseline string) {
+	b, err := backendFor(s)
+	if err != nil {
+		return
+	}
+	if id, err := RuntimeIdentity(b.Name()); err == nil {
+		storeToken(b.Name(), id, guest, baseline)
 	}
 }
 
@@ -158,17 +211,21 @@ func MarkVerified(guest, baseline string) {
 // the run rather than executing on a shared kernel. It returns fresh=true when
 // it had to re-verify, so the caller can explain the pause.
 func EnsureIsolated(name string, s boxfile.Spec) (fresh bool, err error) {
-	id, err := RuntimeIdentity()
+	b, err := backendFor(s)
+	if err != nil {
+		return false, err
+	}
+	id, err := RuntimeIdentity(b.Name())
 	if err != nil {
 		return false, err
 	}
-	if isolationCurrent(id) {
-		return false, nil // this exact runtime is already proven
+	if isolationCurrent(b.Name(), id) {
+		return false, nil // this exact runtime is already proven for this backend
 	}
 	guest, baseline, err := CheckIsolation(name, s)
 	if err != nil {
 		return true, err
 	}
-	storeToken(id, guest, baseline)
+	storeToken(b.Name(), id, guest, baseline)
 	return true, nil
 }