data_size: cap /data, so an agent cannot fill the disk (#10)

README.md

@@ -40,6 +40,7 @@ backend: auto          # firecracker when it fits, else podman; or name one
 isolation: strict      # the VMM runs as a UID that is not yours
 network: bridge        # or none, for no network at all
 warm: 2                # keep VMs booted so runs start in ms
+data_size: 10G         # cap /data, so an agent cannot fill your disk
 packages: [python3, git]
 ```
 

SECURITY.md

@@ -28,6 +28,7 @@ as it can:
 - 6 capabilities, the ones virtiofs and low ports need; the other 5 podman
   grants by default are dropped
 - a pids limit and a memory limit (`ram_mib` + 256 MiB)
+- `/data` capped by `data_size` (see below), so an agent cannot fill your disk
 - only the directories the Boxfile declares, `/data` and any `mounts:`
 - with `isolation: strict`, a subordinate host UID of its own, which owns
   nothing of yours and nothing of any other sandbox's
@@ -46,6 +47,20 @@ rather than let two sandboxes share a UID. `box doctor` shows how many strict
 sandboxes your range holds. A destroyed sandbox's kept `/data` is handed back
 to you, so a later sandbox given the same slot does not own it.
 
+## /data's size
+
+`data_size` in the Boxfile caps `/data` (new Boxfiles say 10G). On
+firecracker `/data` is a disk of exactly that size, so the guest's own kernel
+refuses a write that does not fit. On podman and krun `/data` is a host
+directory shared over virtiofs, and a rootless process cannot put a quota on
+it, so box measures it: it refuses to start a sandbox that is over, and
+checks every 2 seconds while one runs -- a cold run, an `exec`, a warm run, or
+a process left running in an `up` sandbox -- stopping the VM when it goes
+over. A fast writer can overshoot by what it writes in those 2 seconds; the
+limit is what the sandbox must get back under, not a wall. A Boxfile without
+`data_size` has no limit on podman and krun (8G on firecracker), and
+`box inspect` warns about it. The escape suite fills `/data` and checks both.
+
 ## Checking it yourself
 
 `box inspect <name> --security-report` (or `=json`) states what a sandbox's

docs/docs.html

@@ -77,7 +77,7 @@ <h2 id="boxfile">Boxfile</h2>
     <p>One YAML file per sandbox at <code>~/.box/sandboxes/&lt;name&gt;/Boxfile</code>. Unknown keys are rejected, so a typo fails loudly.</p>
     <div class="table-wrap build"><table>
       <thead><tr><th>Field</th><th>Values</th><th>Default</th><th>Meaning</th></tr></thead>
-      <tbody><tr><td class="mono">base</td><td>image reference</td><td class="mono">alpine:latest</td><td>The image every VM starts from.</td></tr><tr><td class="mono">backend</td><td>auto, podman, krun, firecracker</td><td class="mono">podman (new Boxfiles: auto)</td><td>What boots the VM. auto picks firecracker when the sandbox is offline, has no host mounts, no warm pool and no forks, and the host can run it; podman otherwise. The pick is made once and shown by box inspect. See Architecture.</td></tr><tr><td class="mono">isolation</td><td>standard, strict</td><td class="mono">standard</td><td>strict runs the VMM as a subordinate UID of the sandbox's own.</td></tr><tr><td class="mono">cpus</td><td>1 to 16</td><td class="mono">2</td><td>vCPUs.</td></tr><tr><td class="mono">ram_mib</td><td>128 and up</td><td class="mono">2048</td><td>Guest RAM in MiB.</td></tr><tr><td class="mono">network</td><td>bridge, none</td><td class="mono">bridge</td><td>Internet access, or none at all.</td></tr><tr><td class="mono">readonly</td><td>bool</td><td class="mono">false</td><td>Read-only root; /tmp and /data stay writable.</td></tr><tr><td class="mono">timeout_seconds</td><td>int</td><td class="mono">0</td><td>Per-command limit; exits 124. 0 is unlimited.</td></tr><tr><td class="mono">warm</td><td>0 to 8</td><td class="mono">0</td><td>VMs kept booted for run (podman, krun).</td></tr><tr><td class="mono">warmup</td><td>shell lines</td><td class="mono"></td><td>Run once in each warm VM before it serves a run.</td></tr><tr><td class="mono">packages</td><td>names</td><td class="mono"></td><td>Installed with the image's package manager.</td></tr><tr><td class="mono">run</td><td>shell lines</td><td class="mono"></td><td>Extra build steps.</td></tr><tr><td class="mono">env</td><td>map</td><td class="mono"></td><td>Environment for every command.</td></tr><tr><td class="mono">mounts</td><td>host, guest, mode</td><td class="mono"></td><td>Host directories shared in (podman, krun); ro by default.</td></tr><tr><td class="mono">blueprint</td><td>users, write_files, runcmd</td><td class="mono"></td><td>Provisioning applied at build time.</td></tr><tr><td class="mono">pkgmgr</td><td>apk, apt, dnf</td><td class="mono">inferred</td><td>Override the package manager.</td></tr><tr><td class="mono">passt</td><td>bool</td><td class="mono">false</td><td>A real NIC with a default route (podman, krun).</td></tr><tr><td class="mono">seccomp</td><td>path</td><td class="mono"></td><td>A seccomp profile for the VMM.</td></tr></tbody>
+      <tbody><tr><td class="mono">base</td><td>image reference</td><td class="mono">alpine:latest</td><td>The image every VM starts from.</td></tr><tr><td class="mono">backend</td><td>auto, podman, krun, firecracker</td><td class="mono">podman (new Boxfiles: auto)</td><td>What boots the VM. auto picks firecracker when the sandbox is offline, has no host mounts, no warm pool and no forks, and the host can run it; podman otherwise. The pick is made once and shown by box inspect. See Architecture.</td></tr><tr><td class="mono">isolation</td><td>standard, strict</td><td class="mono">standard</td><td>strict runs the VMM as a subordinate UID of the sandbox's own.</td></tr><tr><td class="mono">cpus</td><td>1 to 16</td><td class="mono">2</td><td>vCPUs.</td></tr><tr><td class="mono">ram_mib</td><td>128 and up</td><td class="mono">2048</td><td>Guest RAM in MiB.</td></tr><tr><td class="mono">network</td><td>bridge, none</td><td class="mono">bridge</td><td>Internet access, or none at all.</td></tr><tr><td class="mono">readonly</td><td>bool</td><td class="mono">false</td><td>Read-only root; /tmp and /data stay writable.</td></tr><tr><td class="mono">timeout_seconds</td><td>int</td><td class="mono">0</td><td>Per-command limit; exits 124. 0 is unlimited.</td></tr><tr><td class="mono">warm</td><td>0 to 8</td><td class="mono">0</td><td>VMs kept booted for run (podman, krun).</td></tr><tr><td class="mono">data_size</td><td>e.g. 512M, 10G</td><td class="mono">none (8G on firecracker); new Boxfiles: 10G</td><td>Caps /data. Exact on firecracker, where /data is a disk of this size. On podman and krun box measures it: it will not start a sandbox that is over, and stops a running one within 2 s of going over.</td></tr><tr><td class="mono">warmup</td><td>shell lines</td><td class="mono"></td><td>Run once in each warm VM before it serves a run.</td></tr><tr><td class="mono">packages</td><td>names</td><td class="mono"></td><td>Installed with the image's package manager.</td></tr><tr><td class="mono">run</td><td>shell lines</td><td class="mono"></td><td>Extra build steps.</td></tr><tr><td class="mono">env</td><td>map</td><td class="mono"></td><td>Environment for every command.</td></tr><tr><td class="mono">mounts</td><td>host, guest, mode</td><td class="mono"></td><td>Host directories shared in (podman, krun); ro by default.</td></tr><tr><td class="mono">blueprint</td><td>users, write_files, runcmd</td><td class="mono"></td><td>Provisioning applied at build time.</td></tr><tr><td class="mono">pkgmgr</td><td>apk, apt, dnf</td><td class="mono">inferred</td><td>Override the package manager.</td></tr><tr><td class="mono">passt</td><td>bool</td><td class="mono">false</td><td>A real NIC with a default route (podman, krun).</td></tr><tr><td class="mono">seccomp</td><td>path</td><td class="mono"></td><td>A seccomp profile for the VMM.</td></tr></tbody>
     </table></div>
 
     <h2 id="cli">CLI</h2>

internal/boxfile/boxfile.go

@@ -46,6 +46,7 @@ type Spec struct {
 	ReadOnlyRootfs bool              `yaml:"readonly"`
 	TimeoutSeconds int               `yaml:"timeout_seconds"` // 0 = unlimited
 	Warm           int               `yaml:"warm"`            // VMs kept booted for run; 0 = boot per run
+	DataSize       string            `yaml:"data_size"`       // /data's limit, e.g. 10G; empty = none (8G disk on firecracker)
 	Isolation      string            `yaml:"isolation"`       // "standard" or "strict": VMM under its own host UID
 	Backend        string            `yaml:"backend"`         // what boots the VM: podman (default), krun, firecracker
 	Seccomp        string            `yaml:"seccomp"`         // profile path, filters the VMM
@@ -132,6 +133,41 @@ func FirecrackerFits(s Spec) error {
 	return nil
 }
 
+// MinDataSize is the smallest data_size: below it ext4 has little room left.
+const MinDataSize = 64 << 20
+
+// DataBytes is data_size in bytes, or 0 when the Boxfile sets none.
+// Sizes are 1024-based: 512M, 10G, 1T; a bare number is bytes.
+func (s Spec) DataBytes() (int64, error) {
+	v := strings.TrimSpace(s.DataSize)
+	if v == "" {
+		return 0, nil
+	}
+	mult := int64(1)
+	switch strings.ToUpper(v[len(v)-1:]) {
+	case "K":
+		mult = 1 << 10
+	case "M":
+		mult = 1 << 20
+	case "G":
+		mult = 1 << 30
+	case "T":
+		mult = 1 << 40
+	}
+	num := v
+	if mult > 1 {
+		num = v[:len(v)-1]
+	}
+	n, err := strconv.ParseInt(strings.TrimSpace(num), 10, 64)
+	if err != nil || n <= 0 || n > (1<<62)/mult {
+		return 0, fmt.Errorf("data_size must be a size like 512M, 10G or 1T, got %q", s.DataSize)
+	}
+	if n*mult < MinDataSize {
+		return 0, fmt.Errorf("data_size must be at least 64M, got %q", s.DataSize)
+	}
+	return n * mult, nil
+}
+
 // BackendNames lists the accepted backends for error messages.
 func BackendNames() string { return strings.Join(Backends, ", ") }
 
@@ -150,6 +186,7 @@ isolation: standard   # strict: VMM runs as a UID that is not yours (use for age
 backend: auto         # auto: firecracker when it fits (offline, no mounts, no warm pool), else podman | podman | krun | firecracker
 timeout_seconds: 0    # per-run wall clock limit, 0 = unlimited
 warm: 0               # VMs kept booted so run starts in ms (each holds its RAM)
+data_size: 10G        # /data's limit: exact on firecracker, checked every 2 s on podman
 
 # Tools baked into the image. Reset every run; keep work in /data.
 packages:
@@ -245,6 +282,9 @@ func (s Spec) validate() error {
 			return err
 		}
 	}
+	if _, err := s.DataBytes(); err != nil {
+		return err
+	}
 	if s.Isolation != "standard" && s.Isolation != "strict" {
 		return fmt.Errorf("isolation must be standard or strict, got %q", s.Isolation)
 	}

internal/boxfile/datasize_test.go

@@ -0,0 +1,20 @@
+package boxfile
+
+import "testing"
+
+func TestDataBytes(t *testing.T) {
+	for in, want := range map[string]int64{
+		"": 0, "64M": 64 << 20, "512M": 512 << 20, "10G": 10 << 30, "10g": 10 << 30, "1T": 1 << 40,
+		" 2G ": 2 << 30, "67108864": 64 << 20,
+	} {
+		got, err := Spec{DataSize: in}.DataBytes()
+		if err != nil || got != want {
+			t.Errorf("%q: %d, %v; want %d", in, got, err, want)
+		}
+	}
+	for _, bad := range []string{"10X", "G", "-1G", "0", "1M", "63M", "ten", "99999999999T"} {
+		if _, err := (Spec{DataSize: bad}).DataBytes(); err == nil {
+			t.Errorf("%q accepted", bad)
+		}
+	}
+}

internal/cli/inspect.go

@@ -97,6 +97,12 @@ func printReport(r runtime.Report) {
 		data += ", owner " + d.Owner + ", mode " + d.Mode
 	}
 	row("/data", data+"), kept between runs")
+	limit := d.Limit
+	if d.Used != "" {
+		limit += ", " + d.Used + " used"
+	}
+	row("/data limit", limit)
+	row("enforced", d.Enforcement)
 	if len(r.Filesystem.Mounts) == 0 {
 		row("host mounts", "none")
 	}

internal/cli/root.go

@@ -65,7 +65,7 @@ func newRoot() *cobra.Command {
 		dataCmd(),
 		envCmd(), logsCmd(), verifyCmd(), inspectCmd(), doctorCmd(),
 		renameCmd(), destroyCmd(), nukeCmd(),
-		agentCmd(), prepareCmd(), tendCmd(), overlayCmd(), krunCmd(), exportCmd(), netnsCmd(), fcCmd(), tarOutCmd(), tarInCmd(),
+		agentCmd(), prepareCmd(), tendCmd(), watchDataCmd(), overlayCmd(), krunCmd(), exportCmd(), netnsCmd(), fcCmd(), tarOutCmd(), tarInCmd(),
 	)
 	return root
 }

internal/cli/up.go

@@ -312,6 +312,17 @@ func netnsCmd() *cobra.Command {
 
 // tendCmd tops up a sandbox's warm pool. It is started detached by run and
 // build, never by hand, and reports failures to the sandbox's log.
+func watchDataCmd() *cobra.Command {
+	return &cobra.Command{
+		Use: "__watch-data <name>", Hidden: true,
+		Args:              cobra.ExactArgs(1),
+		ValidArgsFunction: completeNothing,
+		RunE: func(_ *cobra.Command, args []string) error {
+			return runtime.WatchData(args[0])
+		},
+	}
+}
+
 func tendCmd() *cobra.Command {
 	return &cobra.Command{
 		Use: "__tend <name>", Hidden: true,

internal/runtime/backend_firecracker.go

@@ -100,7 +100,7 @@ func (b fcBackend) Launch(name string, s boxfile.Spec, l Launch) (*exec.Cmd, err
 	if err != nil {
 		return nil, err
 	}
-	disk, err := fcDataDisk(name)
+	disk, err := fcDataDisk(name, fcDataBytes(s))
 	if err != nil {
 		return nil, err
 	}

internal/runtime/fc_assets.go

@@ -27,8 +27,8 @@ const (
 	fcKernelName  = "vmlinux-6.1.155"
 	fcKernelURL   = "https://s3.amazonaws.com/spec.ccfc.min/firecracker-ci/v1.15/x86_64/" + fcKernelName
 	fcKernelSHA   = "e20e46d0c36c55c0d1014eb20576171b3f3d922260d9f792017aeff53af3d4f2"
-	fcToolsImage  = "localhost/box/fc-tools:latest"
-	fcDataDiskGiB = 8 // sparse: only what /data actually holds is stored
+	fcToolsImage  = "localhost/box/fc-tools:2" // 2: adds resize2fs; a new tag so an old build is not reused
+	fcDataDiskGiB = 8                          // sparse: only what /data actually holds is stored
 )
 
 // fcDir holds Firecracker's assets and every image's disk and snapshot.
@@ -154,7 +154,7 @@ func fcTools() error {
 	}
 	cmd := exec.Command("podman", "build", "-q", "-t", fcToolsImage, "-f", "-", ".")
 	cmd.Dir = os.TempDir()
-	cmd.Stdin = strings.NewReader("FROM docker.io/library/alpine:latest\nRUN apk add --no-cache e2fsprogs tar\n")
+	cmd.Stdin = strings.NewReader("FROM docker.io/library/alpine:latest\nRUN apk add --no-cache e2fsprogs e2fsprogs-extra tar\n")
 	if out, err := cmd.CombinedOutput(); err != nil {
 		return fmt.Errorf("building %s: %s", fcToolsImage, strings.TrimSpace(string(out)))
 	}

internal/runtime/fc_image.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
 	"strings"
 
 	"box/internal/boxfile"
@@ -45,7 +46,7 @@ func fcImageDir(name string, s boxfile.Spec) (string, error) {
 		return "", err
 	}
 	return filepath.Join(dir, "images", name,
-		fmt.Sprintf("%s-a%s-s%d-r%d-c%d-%s", id[:12], agentHash, shift, s.RAMMiB, s.CPUs, roFlag(s))), nil
+		fmt.Sprintf("%s-a%s-s%d-r%d-c%d-d%d-%s", id[:12], agentHash, shift, s.RAMMiB, s.CPUs, fcDataBytes(s)>>20, roFlag(s))), nil
 }
 
 func roFlag(s boxfile.Spec) string {
@@ -100,9 +101,33 @@ func fcImage(name string, s boxfile.Spec) (string, error) {
 	if err := fcSnapshot(name, s, dir); err != nil {
 		return "", err
 	}
+	removeStaleImages(dir)
 	return dir, nil
 }
 
+// removeStaleImages deletes a sandbox's other images. Each is a rootfs and a
+// full memory snapshot (its RAM in size), keyed on everything that shapes it
+// -- the image, box's own build, isolation, RAM, CPUs, /data's size -- so
+// every change, a box upgrade included, makes a new one; without this the
+// old ones stayed forever. A VM still restored from one keeps its files
+// open, and Linux keeps them until it exits.
+func removeStaleImages(current string) {
+	parent := filepath.Dir(current)
+	entries, err := os.ReadDir(parent)
+	if err != nil {
+		return
+	}
+	var stale []string
+	for _, e := range entries {
+		if e.IsDir() && e.Name() != filepath.Base(current) {
+			stale = append(stale, filepath.Join(parent, e.Name()))
+		}
+	}
+	if len(stale) > 0 {
+		inPodmanNS(append([]string{"rm", "-rf", "--"}, stale...)...).Run()
+	}
+}
+
 // fcRootfs writes rootfs.ext4: the image's files exported by podman and
 // written into an ext4 filesystem by the helper image, with the agent binary
 // and the image's environment added under /.box.
@@ -203,13 +228,40 @@ func FirecrackerDataExists(name string) bool {
 	return err == nil
 }
 
-func fcDataDisk(name string) (string, error) {
+// fcDataBytes is the size of a sandbox's /data disk: data_size, or 8 GiB.
+// The disk is sparse, so only what /data holds takes space.
+func fcDataBytes(s boxfile.Spec) int64 {
+	if n, err := s.DataBytes(); err == nil && n > 0 {
+		return n
+	}
+	return int64(fcDataDiskGiB) << 30
+}
+
+// fcDataDisk returns the sandbox's /data disk, created at size the first
+// time. The disk is data_size's limit: the guest's kernel refuses a write
+// that does not fit. A disk smaller than size is grown, offline, keeping
+// its files; one larger is refused, since shrinking a filesystem in place
+// is how data is lost.
+func fcDataDisk(name string, size int64) (string, error) {
 	disk, err := fcDataDiskPath(name)
 	if err != nil {
 		return "", err
 	}
-	if _, err := os.Stat(disk); err == nil {
-		return disk, nil
+	if fi, err := os.Stat(disk); err == nil {
+		switch {
+		case fi.Size() == size:
+			return disk, nil
+		case fi.Size() > size:
+			return "", fmt.Errorf("%s's /data disk is %s, larger than its data_size %s, and a disk cannot shrink in place: "+
+				"box data export %s ./%s-data && box reset %s -y, then box data import %s ./%s-data",
+				name, humanBytes(fi.Size()), humanBytes(size), name, name, name, name, name)
+		}
+		// Never resize a disk a running VM has mounted.
+		if holder, err := os.ReadFile(diskLockPath(name)); err == nil && fcAlive(string(holder)) {
+			return "", fmt.Errorf("%s's /data disk is in use by %s; bring it down to grow /data to its new data_size",
+				name, strings.TrimSpace(string(holder)))
+		}
+		return disk, growDisk(disk, size)
 	}
 	if err := os.MkdirAll(filepath.Dir(disk), 0o700); err != nil {
 		return "", err
@@ -220,7 +272,7 @@ func fcDataDisk(name string) (string, error) {
 	tmp := name + ".ext4.partial"
 	cmd := exec.Command("podman", "run", "--rm", "--pull=never", "--network=none",
 		"-v", filepath.Dir(disk)+":/out", fcToolsImage,
-		"sh", "-c", fmt.Sprintf("truncate -s %dG /out/%s && mkfs.ext4 -q -F -L data /out/%s", fcDataDiskGiB, tmp, tmp))
+		"sh", "-c", fmt.Sprintf("truncate -s %d /out/%s && mkfs.ext4 -q -F -L data /out/%s", size, tmp, tmp))
 	if out, err := cmd.CombinedOutput(); err != nil {
 		return "", fmt.Errorf("creating %s's /data disk: %s", name, strings.TrimSpace(string(out)))
 	}
@@ -230,6 +282,36 @@ func fcDataDisk(name string) (string, error) {
 	return disk, nil
 }
 
+// growDisk enlarges an unmounted ext4 disk and its filesystem. A strict
+// sandbox's disk belongs to its UID; it is given back to you for the resize
+// and handed to the sandbox again at its next start.
+func growDisk(disk string, size int64) error {
+	if err := fcTools(); err != nil {
+		return err
+	}
+	if out, err := exec.Command("podman", "unshare", "chown", "0:0", disk).CombinedOutput(); err != nil {
+		return fmt.Errorf("growing %s: %s", disk, strings.TrimSpace(string(out)))
+	}
+	// Check for the tools first, and put the size back if the resize fails:
+	// a grow either happens whole or leaves the disk as it was.
+	cmd := exec.Command("podman", "run", "--rm", "--pull=never", "--network=none",
+		"-v", filepath.Dir(disk)+":/out", fcToolsImage, "sh", "-c", `
+d=/out/$1
+command -v resize2fs >/dev/null || { echo "resize2fs is missing from the tools image"; exit 1; }
+orig=$(stat -c %s "$d")
+truncate -s "$2" "$d" || exit 1
+e2fsck -fy "$d" >/dev/null 2>&1
+if [ $? -gt 1 ] || ! resize2fs "$d" >/dev/null 2>&1; then
+  truncate -s "$orig" "$d"
+  echo "the filesystem could not be grown; the disk is unchanged"
+  exit 1
+fi`, "sh", filepath.Base(disk), strconv.FormatInt(size, 10))
+	if out, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("growing %s to %s: %s", disk, humanBytes(size), strings.TrimSpace(string(out)))
+	}
+	return nil
+}
+
 // fcSnapshot boots the image once, waits for its agent, pauses it and
 // snapshots it into dir.
 func fcSnapshot(name string, s boxfile.Spec, dir string) error {
@@ -249,7 +331,7 @@ func fcSnapshot(name string, s boxfile.Spec, dir string) error {
 	// a restored copy is prepared.
 	placeholder := filepath.Join(dir, "placeholder.ext4")
 	if f, err := os.Create(placeholder); err == nil {
-		f.Truncate(int64(fcDataDiskGiB) << 30)
+		f.Truncate(fcDataBytes(s)) // the snapshot's disk must be the size every restore attaches
 		f.Close()
 	}
 	defer os.Remove(placeholder)