diff --git a/SECURITY.md b/SECURITY.md index fdebee8..de2d130 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -15,7 +15,10 @@ Two layers, checked separately. kernel. A guest kernel exploit takes over the guest, which is thrown away. `box build` proves a sandbox got its own kernel by comparing it with the kernel a plain container sees, and every `run`, `exec` and `up` re-checks that -the runtime is still the one that passed. +the runtime is still the one that passed. Each backend is proven on its own, +against a fingerprint of only what it runs (krun and crun for podman and krun; +the pinned Firecracker build and crun for firecracker), so a check passed on +one backend never clears another. **The VMM's confinement.** libkrun's own documentation states that the guest and its VMM "pertain to the same security context": the VMM is a host process diff --git a/internal/cli/commands.go b/internal/cli/commands.go index ff64768..ab890ea 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -131,7 +131,7 @@ func verify(name string, s boxfile.Spec) error { return err } // Prime the cache so runs on this same runtime skip the double boot. - runtime.MarkVerified(guest, baseline) + runtime.MarkVerified(s, guest, baseline) fmt.Printf("isolated: sandbox %s, outside %s\n", guest, baseline) return nil } diff --git a/internal/runtime/doctor.go b/internal/runtime/doctor.go index c79377d..394d686 100644 --- a/internal/runtime/doctor.go +++ b/internal/runtime/doctor.go @@ -8,6 +8,7 @@ import ( "os/exec" "path/filepath" goruntime "runtime" + "sort" "strings" "box/internal/sandbox" @@ -132,16 +133,25 @@ func Doctor() []Check { add(Check{Name: "SDK socket", Warn: true, Detail: err.Error()}) } - if t, ok := loadToken(); ok { - id, _ := RuntimeIdentity() - if id == t.Identity { - add(Check{Name: "isolation", OK: true, + // Each backend's proof is its own; report every one that has been made. + cache := loadCache() + proven := make([]string, 0, len(cache.Backends)) + for b := range cache.Backends { + proven = append(proven, b) + } + sort.Strings(proven) + for _, b := range proven { + t := cache.Backends[b] + name := "isolation (" + b + ")" + if id, _ := RuntimeIdentity(b); id == t.Identity { + add(Check{Name: name, OK: true, Detail: fmt.Sprintf("verified %s: guest %s, outside %s", t.VerifiedAt, t.Guest, t.Baseline)}) } else { - add(Check{Name: "isolation", Warn: true, + add(Check{Name: name, Warn: true, Detail: "the runtime changed since it was last verified; the next run re-checks it"}) } - } else { + } + if len(proven) == 0 { add(Check{Name: "isolation", Warn: true, Detail: "not verified yet", Fix: "box build proves a sandbox gets its own kernel"}) } diff --git a/internal/runtime/report.go b/internal/runtime/report.go index d508136..65bfc4d 100644 --- a/internal/runtime/report.go +++ b/internal/runtime/report.go @@ -163,7 +163,7 @@ func SecurityReport(name string, s boxfile.Spec) (Report, error) { r.Network.Egress = "none" } r.Filesystem.Data = dataReport(name, s) - r.Host = hostReport() + r.Host = hostReport(b.Name()) r.Live = liveReport(name, b.Name(), r.VMM, r.Network.OwnNamespace) r.Findings = findings(r, s) return r, nil @@ -371,7 +371,8 @@ func statInto(dr *DataReport, path string) { dr.Mode = fmt.Sprintf("%o", fi.Mode().Perm()) } -func hostReport() HostReport { +// hostReport describes the host, and the isolation proof for backend. +func hostReport(backend string) HostReport { h := HostReport{} if f, err := os.OpenFile("/dev/kvm", os.O_RDWR, 0); err == nil { f.Close() @@ -380,9 +381,9 @@ func hostReport() HostReport { if b, err := os.ReadFile("/proc/sys/kernel/osrelease"); err == nil { h.Kernel = strings.TrimSpace(string(b)) } - if t, ok := loadToken(); ok { + if t, ok := loadToken(backend); ok { h.GuestKernel, h.IsolationVerifiedAt = t.Guest, t.VerifiedAt - if id, err := RuntimeIdentity(); err == nil { + if id, err := RuntimeIdentity(backend); err == nil { h.IsolationCurrent = t.Identity == id } } diff --git a/internal/runtime/verify.go b/internal/runtime/verify.go index 0c38199..10e8bb0 100644 --- a/internal/runtime/verify.go +++ b/internal/runtime/verify.go @@ -45,15 +45,21 @@ func CheckIsolation(name string, s boxfile.Spec) (guest, baseline string, err er } // RuntimeIdentity fingerprints the parts of the toolchain that decide whether a -// sandbox actually gets its own kernel. When any of them changes -- the krun -// symlink is re-pointed, crun is upgraded, libkrun support is dropped, podman -// changes -- the fingerprint changes and a cached verification no longer -// applies. It is cheap (a couple of --version calls and a stat), so it can gate -// every run without the cost of booting a VM. -func RuntimeIdentity() (string, error) { - parts := []string{goruntime.GOOS + "/" + goruntime.GOARCH} +// sandbox on backend actually gets its own kernel. When any of them changes -- +// the krun symlink is re-pointed, crun is upgraded, libkrun support is dropped, +// the pinned Firecracker build changes, podman changes -- the fingerprint +// changes and a cached verification no longer applies. It is cheap (a couple +// of --version calls and a stat), so it can gate every run without the cost +// of booting a VM. +// +// Each backend has its own: what proves the krun path says nothing about +// Firecracker, and the reverse. It covers only what that backend runs, so a +// host without krun can still use firecracker. +func RuntimeIdentity(backend string) (string, error) { + parts := []string{goruntime.GOOS + "/" + goruntime.GOARCH, "backend:" + backend} - if goruntime.GOOS == "darwin" { + switch { + case goruntime.GOOS == "darwin": // Isolation lives inside the podman machine VM, so its identity, not a // host-side krun, is what matters. out, err := exec.Command("podman", "machine", "inspect", "--format", @@ -62,27 +68,25 @@ func RuntimeIdentity() (string, error) { return "", fmt.Errorf("no podman machine to fingerprint: %w", err) } parts = append(parts, "machine:"+strings.TrimSpace(string(out))) - } else { - // The krun symlink and the crun binary it resolves to are the whole - // isolation path; capture where it points and the binary's identity so - // a re-pointed link or an upgraded crun both change the fingerprint. - krun, err := exec.LookPath("krun") + case backend == "firecracker": + // The VMM and guest kernel are pinned by hash and checked before every + // use (fcAssets), so the pins identify them; crun is what confines and + // runs the VMM. + parts = append(parts, "fc:"+fcVersion+":"+fcBinarySHA, "fckernel:"+fcKernelSHA, "fctools:"+fcToolsImage) + p, err := binIdentity("crun") if err != nil { - return "", fmt.Errorf("no 'krun' on PATH") + return "", err } - real, err := filepath.EvalSymlinks(krun) + parts = append(parts, p...) + default: + // The krun symlink and the crun binary it resolves to are the whole + // isolation path; the version string carries the +LIBKRUN tag, and + // losing it is exactly the degradation this guards against. + p, err := binIdentity("krun") if err != nil { - real = krun - } - parts = append(parts, "krun:"+krun+"->"+real) - if fi, err := os.Stat(real); err == nil { - parts = append(parts, fmt.Sprintf("crun:%d:%d", fi.Size(), fi.ModTime().UnixNano())) - } - // The version string carries the +LIBKRUN tag; losing it is exactly the - // degradation this guards against. - if out, err := exec.Command("krun", "--version").Output(); err == nil { - parts = append(parts, "krunver:"+strings.TrimSpace(string(out))) + return "", err } + parts = append(parts, p...) } if out, err := exec.Command("podman", "--version").Output(); err == nil { parts = append(parts, "podman:"+strings.TrimSpace(string(out))) @@ -91,6 +95,27 @@ func RuntimeIdentity() (string, error) { return hex.EncodeToString(sum[:]), nil } +// binIdentity is where bin resolves on PATH, through any symlink, and that +// file's size, mtime and version: a re-pointed link or an upgrade changes it. +func binIdentity(bin string) ([]string, error) { + path, err := exec.LookPath(bin) + if err != nil { + return nil, fmt.Errorf("no '%s' on PATH", bin) + } + real, err := filepath.EvalSymlinks(path) + if err != nil { + real = path + } + parts := []string{bin + ":" + path + "->" + real} + if fi, err := os.Stat(real); err == nil { + parts = append(parts, fmt.Sprintf("%s-file:%d:%d", bin, fi.Size(), fi.ModTime().UnixNano())) + } + if out, err := exec.Command(bin, "--version").Output(); err == nil { + parts = append(parts, bin+"-version:"+strings.TrimSpace(string(out))) + } + return parts, nil +} + // verifyToken is the cached result of a successful isolation check. A run // trusts it only while Identity still matches the current runtime. type verifyToken struct { @@ -100,25 +125,41 @@ type verifyToken struct { VerifiedAt string `json:"verified_at"` } -func loadToken() (verifyToken, bool) { +// verifyCache holds one token per backend. A check passed on one backend +// never clears another: each boots its guest a different way. +type verifyCache struct { + Backends map[string]verifyToken `json:"backends"` +} + +// loadCache reads the cache. A missing, corrupt or older-format file (a +// single token, from before tokens were per backend) reads as empty, which +// costs one re-verify and never clears a run. +func loadCache() verifyCache { + c := verifyCache{Backends: map[string]verifyToken{}} p, err := sandbox.VerifyCachePath() if err != nil { - return verifyToken{}, false + return c } b, err := os.ReadFile(p) if err != nil { - return verifyToken{}, false + return c } - var t verifyToken - if json.Unmarshal(b, &t) != nil { - return verifyToken{}, false + var got verifyCache + if json.Unmarshal(b, &got) != nil || got.Backends == nil { + return c } - return t, true + return got +} + +func loadToken(backend string) (verifyToken, bool) { + t, ok := loadCache().Backends[backend] + return t, ok && t.Identity != "" } -// storeToken records a verification. It is best-effort: a cache-write failure -// only costs a re-verify next time, so it must never fail a build or run. -func storeToken(id, guest, baseline string) { +// storeToken records a verification for backend. It is best-effort: a +// cache-write failure only costs a re-verify next time, so it must never +// fail a build or run. +func storeToken(backend, id, guest, baseline string) { p, err := sandbox.VerifyCachePath() if err != nil { return @@ -126,28 +167,40 @@ func storeToken(id, guest, baseline string) { if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { return } - if b, err := json.Marshal(verifyToken{ + c := loadCache() + c.Backends[backend] = verifyToken{ Identity: id, Guest: guest, Baseline: baseline, VerifiedAt: time.Now().UTC().Format(time.RFC3339), - }); err == nil { - os.WriteFile(p, b, 0o644) + } + if b, err := json.Marshal(c); err == nil { + // Beside and renamed, so a concurrent reader never sees half a file. + tmp := p + ".tmp" + if os.WriteFile(tmp, b, 0o644) == nil { + os.Rename(tmp, p) + } } } -// isolationCurrent reports whether the cached token already cleared runtime id. -func isolationCurrent(id string) bool { - t, ok := loadToken() +// isolationCurrent reports whether backend's cached token already cleared +// runtime id. +func isolationCurrent(backend, id string) bool { + t, ok := loadToken(backend) return ok && t.Identity == id } -// MarkVerified caches a successful isolation check for the current runtime, so -// later runs can trust it without booting two more VMs. Callers that have just -// proven isolation (build, the verify command) use it to prime the cache. -func MarkVerified(guest, baseline string) { - if id, err := RuntimeIdentity(); err == nil { - storeToken(id, guest, baseline) +// MarkVerified caches a successful isolation check of a sandbox with spec s, +// for its backend and the current runtime, so later runs can trust it without +// booting two more VMs. Callers that have just proven isolation (build, the +// verify command) use it to prime the cache. +func MarkVerified(s boxfile.Spec, guest, baseline string) { + b, err := backendFor(s) + if err != nil { + return + } + if id, err := RuntimeIdentity(b.Name()); err == nil { + storeToken(b.Name(), id, guest, baseline) } } @@ -158,17 +211,21 @@ func MarkVerified(guest, baseline string) { // the run rather than executing on a shared kernel. It returns fresh=true when // it had to re-verify, so the caller can explain the pause. func EnsureIsolated(name string, s boxfile.Spec) (fresh bool, err error) { - id, err := RuntimeIdentity() + b, err := backendFor(s) + if err != nil { + return false, err + } + id, err := RuntimeIdentity(b.Name()) if err != nil { return false, err } - if isolationCurrent(id) { - return false, nil // this exact runtime is already proven + if isolationCurrent(b.Name(), id) { + return false, nil // this exact runtime is already proven for this backend } guest, baseline, err := CheckIsolation(name, s) if err != nil { return true, err } - storeToken(id, guest, baseline) + storeToken(b.Name(), id, guest, baseline) return true, nil } diff --git a/internal/runtime/verify_test.go b/internal/runtime/verify_test.go index 28e517d..7df2cf1 100644 --- a/internal/runtime/verify_test.go +++ b/internal/runtime/verify_test.go @@ -2,6 +2,9 @@ package runtime import ( "os" + "path/filepath" + goruntime "runtime" + "strings" "testing" "box/internal/sandbox" @@ -14,32 +17,49 @@ import ( func TestIsolationCache(t *testing.T) { t.Setenv("BOX_HOME", t.TempDir()) - if isolationCurrent("A") { + if isolationCurrent("podman", "A") { t.Fatal("empty cache must not read as verified") } - storeToken("A", "6.12.0", "6.18.0") - if !isolationCurrent("A") { + storeToken("podman", "A", "6.12.0", "6.18.0") + if !isolationCurrent("podman", "A") { t.Error("the stored identity should be current") } - if isolationCurrent("B") { + if isolationCurrent("podman", "B") { t.Error("a different runtime identity must not be trusted") } // A later verification of a new runtime replaces the old one. - storeToken("B", "6.12.0", "6.18.0") - if isolationCurrent("A") { + storeToken("podman", "B", "6.12.0", "6.18.0") + if isolationCurrent("podman", "A") { t.Error("the superseded identity must no longer be current") } - if !isolationCurrent("B") { + if !isolationCurrent("podman", "B") { t.Error("the newly stored identity should be current") } } +// A proof made on one backend says nothing about another: they boot guests +// through different toolchains, and one can be broken while the other works. +func TestIsolationProofIsPerBackend(t *testing.T) { + t.Setenv("BOX_HOME", t.TempDir()) + storeToken("firecracker", "same-id", "6.1.155", "6.17.0") + if isolationCurrent("podman", "same-id") || isolationCurrent("krun", "same-id") { + t.Fatal("a firecracker proof must not clear podman or krun") + } + storeToken("podman", "p-id", "6.12.0", "6.17.0") + if !isolationCurrent("firecracker", "same-id") { + t.Error("proving podman must not drop the firecracker proof") + } + if !isolationCurrent("podman", "p-id") { + t.Error("the podman proof should be current") + } +} + func TestTokenRoundTrip(t *testing.T) { t.Setenv("BOX_HOME", t.TempDir()) - storeToken("id-123", "guest-k", "base-k") - got, ok := loadToken() + storeToken("krun", "id-123", "guest-k", "base-k") + got, ok := loadToken("krun") if !ok { t.Fatal("token should load after store") } @@ -49,23 +69,98 @@ func TestTokenRoundTrip(t *testing.T) { if got.VerifiedAt == "" { t.Error("VerifiedAt should be stamped") } + if _, ok := loadToken("podman"); ok { + t.Error("another backend has no token") + } } -// A corrupt cache file must degrade to "not verified" (forcing a fresh check), -// never crash or falsely clear a run. -func TestCorruptTokenIsNotCurrent(t *testing.T) { - t.Setenv("BOX_HOME", t.TempDir()) - p, err := sandbox.VerifyCachePath() +// A corrupt cache, or one in the format from before tokens were per backend, +// must degrade to "not verified" (forcing a fresh check), never crash or +// falsely clear a run. +func TestUnreadableCacheIsNotCurrent(t *testing.T) { + for name, body := range map[string]string{ + "corrupt": "not json", + "old format": `{"identity":"A","guest":"6.12.0","baseline":"6.18.0","verified_at":"2026-01-01T00:00:00Z"}`, + } { + t.Run(name, func(t *testing.T) { + t.Setenv("BOX_HOME", t.TempDir()) + p, err := sandbox.VerifyCachePath() + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + if _, ok := loadToken("podman"); ok { + t.Error("an unreadable cache must not load") + } + if isolationCurrent("podman", "A") { + t.Error("an unreadable cache must not read as verified") + } + storeToken("podman", "A", "g", "b") + if !isolationCurrent("podman", "A") { + t.Error("a fresh proof should replace the unreadable cache") + } + }) + } +} + +// fakeBins puts executables answering --version with the given text on a +// PATH of their own, so the identity sees exactly these tools. +func fakeBins(t *testing.T, bins map[string]string) string { + t.Helper() + dir := t.TempDir() + for name, version := range bins { + script := "#!/bin/sh\necho '" + version + "'\n" + if err := os.WriteFile(filepath.Join(dir, name), []byte(script), 0o755); err != nil { + t.Fatal(err) + } + } + t.Setenv("PATH", dir) + return dir +} + +// A host with only the firecracker backend has no krun, and must not need +// it: firecracker never runs krun. +func TestFirecrackerIdentityNeedsNoKrun(t *testing.T) { + if goruntime.GOOS == "darwin" { + t.Skip("macOS fingerprints the podman machine") + } + fakeBins(t, map[string]string{"crun": "crun version 1.14", "podman": "podman version 4.9.3"}) + fc, err := RuntimeIdentity("firecracker") if err != nil { - t.Fatal(err) + t.Fatalf("firecracker identity without krun: %v", err) } - if err := os.WriteFile(p, []byte("not json"), 0o644); err != nil { - t.Fatal(err) + if _, err := RuntimeIdentity("podman"); err == nil || !strings.Contains(err.Error(), "krun") { + t.Errorf("podman needs krun, and should say so: %v", err) + } + if _, err := RuntimeIdentity("krun"); err == nil { + t.Error("krun needs krun on PATH") } - if _, ok := loadToken(); ok { - t.Error("a corrupt token must not load") + + // An upgraded crun is a different runtime for firecracker. + fakeBins(t, map[string]string{"crun": "crun version 1.15", "podman": "podman version 4.9.3"}) + if fc2, _ := RuntimeIdentity("firecracker"); fc2 == fc { + t.Error("a changed crun must change the firecracker identity") + } +} + +// With krun present, each backend still gets an identity of its own, so no +// cached proof can be read as another's. +func TestIdentitiesDifferPerBackend(t *testing.T) { + dir := fakeBins(t, map[string]string{"crun": "crun version 1.14 +LIBKRUN", "podman": "podman version 4.9.3"}) + if err := os.Symlink(filepath.Join(dir, "crun"), filepath.Join(dir, "krun")); err != nil { + t.Fatal(err) } - if isolationCurrent("anything") { - t.Error("a corrupt cache must not read as verified") + seen := map[string]string{} + for _, b := range []string{"podman", "krun", "firecracker"} { + id, err := RuntimeIdentity(b) + if err != nil { + t.Fatalf("%s: %v", b, err) + } + if other, dup := seen[id]; dup { + t.Errorf("%s and %s share an identity", b, other) + } + seen[id] = b } } diff --git a/internal/sandbox/sandbox.go b/internal/sandbox/sandbox.go index 0a6e029..2be1e44 100644 --- a/internal/sandbox/sandbox.go +++ b/internal/sandbox/sandbox.go @@ -114,9 +114,10 @@ func LogPath(name string) (string, error) { // someone else's box/ image with /data and the rw mounts attached. func ImageTag(name string) string { return "localhost/box/" + name + ":latest" } -// VerifyCachePath holds the last successful isolation check, keyed on the -// runtime's identity. It is not per-sandbox: isolation is a property of the -// podman + krun toolchain, so one cleared runtime clears it for every sandbox. +// VerifyCachePath holds the last successful isolation check of each backend, +// keyed on that backend's runtime identity. It is not per-sandbox: isolation +// is a property of the toolchain a backend boots guests with, so one cleared +// runtime clears that backend for every sandbox, and no other backend. func VerifyCachePath() (string, error) { h, err := Home() if err != nil {