Disk billing gap

.gitignore

@@ -21,6 +21,11 @@ Thumbs.db
 # Cloudflare Worker local secrets (wrangler dev)
 .dev.vars
 .dev.vars.*
+# Wrangler local state — miniflare D1/KV/R2 caches, dev sqlite DBs, etc.
+# Written by `wrangler dev` and `wrangler d1 migrations list` against a
+# --remote target (which still initializes a local shadow). Never commit.
+.wrangler/
+**/.wrangler/
 
 # Per-developer dev-box lifecycle state (written by deploy/*/deploy-qemu-dev.sh)
 deploy/**/.qemu-dev-state-*
@@ -47,7 +52,13 @@ cloudflare-workers/api-edge/assets/
 # Compiled binaries
 bin/
 opensandbox-worker
+opensandbox-server
+# Repo-root Go build outputs — produced when someone runs `go build ./cmd/X/`
+# from the repo root without `-o`. Path-anchored so we don't accidentally
+# ignore legitimate `server` / `worker` / `oc` DIRECTORIES elsewhere.
 /worker
+/server
+/oc
 
 # Worker env (secrets)
 worker.env

cloudflare-workers/api-edge/migrations/0004_usage_samples_disk_mb.sql

@@ -0,0 +1,7 @@
+-- Add disk_mb to usage_samples so the autumn-meter can attribute disk-overage
+-- GB-seconds against the running-sandbox billing bucket. Cell emits it in the
+-- usage_tick payload, events-ingest lands it in this column, autumn_meter reads
+-- it in the per-bucket aggregation. Default 0 = "no disk signal for this tick",
+-- which the autumn-meter treats as no overage (matches "free 20GB is included"
+-- and prevents pre-migration rows from silently accruing overage).
+ALTER TABLE usage_samples ADD COLUMN disk_mb INTEGER NOT NULL DEFAULT 0;

cloudflare-workers/api-edge/schema-snapshots/current_schema.sql

@@ -243,7 +243,8 @@ CREATE TABLE usage_meter_events (
 CREATE TABLE usage_samples (
   id TEXT PRIMARY KEY, org_id TEXT NOT NULL, sandbox_id TEXT NOT NULL,
   memory_mb INTEGER NOT NULL, cpu_count INTEGER NOT NULL, interval_s INTEGER NOT NULL,
-  ts INTEGER NOT NULL, cell_id TEXT NOT NULL, rolled_up INTEGER NOT NULL DEFAULT 0);
+  ts INTEGER NOT NULL, cell_id TEXT NOT NULL, rolled_up INTEGER NOT NULL DEFAULT 0,
+  disk_mb INTEGER NOT NULL DEFAULT 0);
 
 CREATE TABLE usage_snapshots (
   org_id            TEXT NOT NULL,

cloudflare-workers/api-edge/src/autumn_meter.ts

@@ -38,6 +38,16 @@ const TIER_FEATURE_BY_MEMORY_MB: Record<number, string> = {
   65536: "compute_64gb",
 };
 
+// Disk-overage billing. Every GB above DISK_FREE_ALLOWANCE_MB (20 GB, the
+// baseline every sandbox ships with) is metered at $0.0000001/GB-second
+// (~$0.26/GB-month) — see pricing.md + internal/billing/pricing.go.
+// The Autumn feature is charged in whole GB-seconds; we ceil the fractional
+// remainder per bucket so sub-unit accrual across ticks isn't silently
+// dropped. The Autumn dashboard must have a `disk_overage_gb_seconds`
+// feature attached to the pro product for track() to land.
+const DISK_FREE_ALLOWANCE_MB = 20480;
+const DISK_OVERAGE_FEATURE = "disk_overage_gb_seconds";
+
 // Globally-unique, retry-stable key. Autumn dedupes on the bare key across all
 // customers, so it includes the org; keyed on bucket start (not wall-clock) so a
 // replay reuses it. Matches autumn.UsageIdempotencyKey on the (removed) cell.
@@ -122,8 +132,14 @@ async function meterOrg(env: AutumnEnv, org: AutumnOrgRow, nowSec: number): Prom
   return true;
 }
 
-// trackBucket aggregates usage_samples in [from, to) by memory tier and tracks
-// one usage event per tier to Autumn. Returns true if the balance is now <= 0.
+// trackBucket aggregates usage_samples in [from, to) by memory tier + a single
+// disk-overage GB-second total, tracks one usage event per dimension to Autumn,
+// and returns true if the resulting balance is <= 0.
+//
+// The two dimensions are independent Autumn features (compute_{tier}, and
+// disk_overage_gb_seconds) so a customer running an idle-but-large-disk
+// sandbox accrues disk cost separately from compute cost. Both use the same
+// per-org per-bucket idempotency key namespace so a retry deduplicates cleanly.
 async function trackBucket(env: AutumnEnv, orgID: string, fromSec: number, toSec: number): Promise<boolean> {
   const aggRes = await env.OPENCOMPUTER_DB.prepare(
     `SELECT memory_mb AS memory_mb, SUM(interval_s) AS secs
@@ -134,7 +150,22 @@ async function trackBucket(env: AutumnEnv, orgID: string, fromSec: number, toSec
     .bind(orgID, fromSec * 1000, toSec * 1000)
     .all<TierAgg>();
   const tiers = aggRes.results ?? [];
-  if (tiers.length === 0) return false;
+
+  // Disk overage: SUM((disk_mb - 20480) * interval_s / 1024) — bytes above the
+  // free allowance × seconds, converted to GB-seconds. Only rows with real
+  // overage contribute (the WHERE clause culls 0/default rows so idle sandboxes
+  // at baseline size don't produce empty aggregation work). Returned as a real
+  // (SQLite REAL), ceiled below for the whole-GB-second Autumn feature.
+  const diskRes = await env.OPENCOMPUTER_DB.prepare(
+    `SELECT COALESCE(SUM((disk_mb - ?1) * interval_s), 0) / 1024.0 AS gb_seconds
+       FROM usage_samples
+      WHERE org_id = ?2 AND ts >= ?3 AND ts < ?4 AND disk_mb > ?1`,
+  )
+    .bind(DISK_FREE_ALLOWANCE_MB, orgID, fromSec * 1000, toSec * 1000)
+    .first<{ gb_seconds: number | null }>();
+  const diskGBSeconds = Math.ceil(diskRes?.gb_seconds ?? 0);
+
+  if (tiers.length === 0 && diskGBSeconds <= 0) return false;
 
   let remaining: number | null = null;
   for (const t of tiers) {
@@ -151,6 +182,16 @@ async function trackBucket(env: AutumnEnv, orgID: string, fromSec: number, toSec
       idempotencyKey: usageIdempotencyKey(orgID, fromSec, feature),
     });
   }
+
+  if (diskGBSeconds > 0) {
+    remaining = await trackAutumnUsage(env, {
+      customerID: orgID,
+      featureID: DISK_OVERAGE_FEATURE,
+      value: diskGBSeconds,
+      idempotencyKey: usageIdempotencyKey(orgID, fromSec, DISK_OVERAGE_FEATURE),
+    });
+  }
+
   return remaining !== null && remaining <= 0;
 }
 

cloudflare-workers/api-edge/src/index.ts

@@ -458,7 +458,8 @@ async function enforceCreatePolicy(
   // is_halted projection — no CreditAccount DO. On a halt, self-heal (re-check
   // Autumn) so a just-topped-up user isn't stuck behind a lagging webhook. The
   // free-tier memory/CPU/disk ceilings below are skipped for autumn orgs (they
-  // pay per GB-second); only the per-org max_disk_mb cap applies.
+  // pay per GB-second and are metered per-bucket by autumn_meter — disk overage
+  // via the `disk_overage_gb_seconds` feature).
   if (org.billing_provider === "autumn") {
     if (org.is_halted === 1 && (await selfHealHalt(env, orgID))) {
       return json({ error: "credits exhausted — top up to resume" }, 402);
@@ -484,7 +485,9 @@ async function enforceCreatePolicy(
 
   // Free-tier ceilings: 4GB / 1 vCPU, 20GB disk. Legacy only — autumn (prepaid)
   // orgs pay per GB-second and are gated by balance/halt, so they may launch any
-  // size. Disk is still bounded for everyone by the per-org max_disk_mb check below.
+  // size up to the platform ceiling (enforced upstream). The per-org
+  // `max_disk_mb` column is no longer enforced now that disk overage is metered
+  // to Autumn end-to-end; the D1 column stays as dead legacy.
   if (org.billing_provider !== "autumn" && plan === "free") {
     if (sizes.memoryMB > 4096 || sizes.cpuCount > 1) {
       return json({ error: "upgrade to pro for larger instances" }, 402);
@@ -494,14 +497,6 @@ async function enforceCreatePolicy(
     }
   }
 
-  // Per-org disk ceiling (all plans). 0 in D1 means "use the 20GB default".
-  if (sizes.diskMB > 0) {
-    const maxDisk = org.max_disk_mb > 0 ? org.max_disk_mb : 20480;
-    if (sizes.diskMB > maxDisk) {
-      return json({ error: `disk size ${sizes.diskMB}MB exceeds org limit of ${maxDisk}MB` }, 403);
-    }
-  }
-
   // Concurrent-sandbox limit (all plans). Counts `running` only — hibernated
   // sandboxes live in S3 and don't consume worker capacity. The count spans
   // every cell via the global sandboxes_index, which is the whole reason it

cloudflare-workers/events-ingest/src/index.ts

@@ -429,8 +429,8 @@ export default {
     // cross-DO debit fan-out, it's a local D1 write, so it should share the
     // batch's retry-on-failure guarantee (503 → CP forwarder keeps the PEL).
     const usageSampleInsert = env.OPENCOMPUTER_DB.prepare(
-      `INSERT INTO usage_samples (id, org_id, sandbox_id, memory_mb, cpu_count, interval_s, ts, cell_id)
-       VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)
+      `INSERT INTO usage_samples (id, org_id, sandbox_id, memory_mb, cpu_count, interval_s, ts, cell_id, disk_mb)
+       VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)
        ON CONFLICT(id) DO NOTHING`,
     );
     // Worker-validation guard. Edge usage is counted per-tick (one usage_sample
@@ -483,6 +483,7 @@ export default {
           memory_mb?: number;
           cpu_count?: number;
           interval_s?: number;
+          disk_mb?: number;
         };
         return usageSampleInsert.bind(
           e.id,
@@ -493,6 +494,10 @@ export default {
           p.interval_s ?? 0,
           Date.parse(e.timestamp) || Date.now(),
           e.cell_id,
+          // disk_mb is 0 on pre-billing-cutover rows (older cells that don't
+          // stamp it yet). autumn_meter reads that as "no overage this tick",
+          // matching the free 20GB allowance and keeping backfill safe.
+          p.disk_mb ?? 0,
         );
       });
 

cmd/server/main.go

@@ -103,6 +103,7 @@ func main() {
 		CFAdminSecret:    cfg.CFAdminSecret,
 		CFEventSecret:    cfg.CFEventSecret,
 		RequireCapToken:  proBillingEdge,
+		MaxDiskMB:        cfg.MaxDiskMB,
 	}
 
 	// Initialize PostgreSQL if configured
@@ -269,6 +270,26 @@ func main() {
 				_ = fwd.Stop(stopCtx)
 			}()
 			log.Printf("opensandbox: CF event forwarder started (endpoint=%s cell=%s)", cfg.CFEventEndpoint, cfg.CellID)
+
+			// Hibernation-billing sweeper — periodically synthesizes usage_tick
+			// events for hibernated sandboxes with disk overage, so the edge
+			// autumn_meter bills them even though no worker's usage_ticker fires
+			// while the qcow2 sits in Tigris. Sends directly to events-ingest via
+			// cfClient (bypasses Redis — the payload is generated on the CP, not
+			// on a worker). Inert unless opts.Store is wired (needs the shared PG
+			// to enumerate hibernated sandboxes + their current disk envelope).
+			if opts.Store != nil {
+				sweeper := controlplane.NewHibernationBillingSweeper(opts.Store, cfClient, cfg.CellID, 5*time.Minute)
+				if sweeper != nil {
+					sweeper.Start(context.Background())
+					defer func() {
+						stopCtx, stopCancel := context.WithTimeout(context.Background(), 5*time.Second)
+						defer stopCancel()
+						_ = sweeper.Stop(stopCtx)
+					}()
+					log.Printf("opensandbox: hibernation-billing sweeper started (5 min interval, cell=%s)", cfg.CellID)
+				}
+			}
 		} else if cfg.Mode == "server" {
 			log.Printf("opensandbox: CF event forwarder NOT started (CFEventEndpoint/Secret/CellID unset)")
 		}

cmd/worker/main.go

@@ -899,7 +899,7 @@ func main() {
 	// Inert unless CellID is set (combined-mode dev without Redis stream
 	// would write events to /dev/null since there's no consumer).
 	if cfg.CellID != "" && mgr != nil {
-		usageTicker := worker.NewUsageTicker(mgr, sandboxDBMgr, 20*time.Second, 10)
+		usageTicker := worker.NewUsageTicker(mgr, sandboxDBMgr, store, 20*time.Second, 10)
 		if usageTicker != nil {
 			// Wire the ticker as the manager's lifecycle observer so scale,
 			// destroy, hibernate, and wake events flush accurate final-slice

internal/api/internal_sandbox.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"fmt"
 	"net/http"
 	"strings"
 
@@ -136,15 +137,20 @@ func (s *Server) internalCreateSandbox(c echo.Context) error {
 
 	// Physical disk bounds. These are hardware/safety limits, not org policy:
 	// diskMB=1 boots-fail opaquely, diskMB=10_000_000 would allocate 10TB per
-	// sandbox. Per-org disk ceilings (free-tier cap, custom max_disk_mb) are
-	// enforced at the edge against D1 — see enforceCreatePolicy in
-	// cloudflare-workers/api-edge/src/index.ts. The cell trusts the cap-token
-	// and only checks these physical bounds.
+	// sandbox. The free-tier disk ceiling is enforced at the edge against D1
+	// (see enforceCreatePolicy in cloudflare-workers/api-edge/src/index.ts).
+	// Paying orgs are billed per GB-second above the 20GB free allowance via
+	// the edge autumn_meter — no per-org admission cap. The cell trusts the
+	// cap-token and only checks these physical bounds.
 	if cfg.DiskMB < 20480 {
 		return c.JSON(http.StatusBadRequest, map[string]string{"error": "diskMB must be at least 20480 (20GB)"})
 	}
-	if cfg.DiskMB > 262144 {
-		return c.JSON(http.StatusBadRequest, map[string]string{"error": "diskMB cannot exceed 262144 (256GB)"})
+	maxDiskMB := s.maxDiskMB
+	if maxDiskMB <= 0 {
+		maxDiskMB = 262144 // default 256GB
+	}
+	if cfg.DiskMB > maxDiskMB {
+		return c.JSON(http.StatusBadRequest, map[string]string{"error": fmt.Sprintf("diskMB cannot exceed %d (%dGB)", maxDiskMB, maxDiskMB/1024)})
 	}
 
 	// Declarative image or named snapshot → resolve to a checkpoint and use

internal/api/router.go

@@ -51,6 +51,7 @@ type Server struct {
 	cfAdminSecret  string              // HMAC shared with CreditAccount DO for /admin/halt-org and /admin/resume-org; empty disables auth (dev only)
 	cfEventSecret  string              // HMAC shared with the api-edge Worker for /internal/secret-refresh and other edge-→cell push paths
 	cellID     string                  // this control plane's cell_id (for the cap-token cell check)
+	maxDiskMB  int                     // platform per-sandbox disk ceiling; 0 = default 262144 (256GB)
 	platformOrgID uuid.UUID            // owner of the shared catalog snapshots; anchors the public-snapshot fallback + gates publish (uuid.Nil = fallback disabled)
 	mode       string                  // "server", "worker", "combined"
 	workerID   string                  // this worker's ID
@@ -162,6 +163,13 @@ type ServerOpts struct {
 	SandboxAPIProxy *proxy.SandboxAPIProxy             // nil except in server mode (proxies data-plane to workers)
 	StripeClient    *billing.StripeClient              // nil if Stripe not configured
 	RedisClient     *redis.Client                     // nil if Redis not configured (for health checks)
+	// MaxDiskMB caps `diskMB` at create time (both /api/sandboxes and the
+	// edge-fanout /internal/sandboxes/create). Not a per-org policy — a
+	// platform sanity ceiling protecting worker-density + hibernate/wake
+	// latency at very large disk sizes. Zero = code default (262144 = 256GB).
+	// Bump via OPENSANDBOX_MAX_DISK_MB once timeouts + worker density have
+	// been validated for the new ceiling.
+	MaxDiskMB int
 }
 
 // NewServer creates a new API server with all routes configured.
@@ -192,6 +200,7 @@ func NewServer(mgr sandbox.Manager, ptyMgr *sandbox.PTYManager, apiKey string, o
 		s.cfAdminSecret = opts.CFAdminSecret
 		s.cfEventSecret = opts.CFEventSecret
 		s.cellID = opts.CellID
+		s.maxDiskMB = opts.MaxDiskMB
 		if opts.PlatformOrgID != "" {
 			if pid, err := uuid.Parse(opts.PlatformOrgID); err == nil {
 				s.platformOrgID = pid

internal/api/sandbox.go

@@ -126,8 +126,8 @@ func (s *Server) createSandbox(c echo.Context) error {
 			// Free-tier gate (legacy only): trial-credit + machine-size limit.
 			// Autumn orgs pay per GB-second and are gated at the edge
 			// (balance/halt), so neither the credit check nor the size ceiling
-			// applies — they may launch any size (disk still bounded by the
-			// per-org MaxDiskMB knob below).
+			// applies — they may launch any size up to the platform ceiling
+			// (currently 256GB for disk; enforced above).
 			if effPlan == "free" && effProvider != "autumn" {
 				if org.FreeCreditsRemainingCents <= 0 {
 					return c.JSON(http.StatusPaymentRequired, map[string]string{
@@ -158,26 +158,33 @@ func (s *Server) createSandbox(c echo.Context) error {
 			"error": "diskMB must be at least 20480 (20GB)",
 		})
 	}
-	if cfg.DiskMB > 262144 {
+	maxDiskMB := s.maxDiskMB
+	if maxDiskMB <= 0 {
+		maxDiskMB = 262144 // default 256GB
+	}
+	if cfg.DiskMB > maxDiskMB {
 		return c.JSON(http.StatusBadRequest, map[string]string{
-			"error": "diskMB cannot exceed 262144 (256GB)",
+			"error": fmt.Sprintf("diskMB cannot exceed %d (%dGB)", maxDiskMB, maxDiskMB/1024),
 		})
 	}
 	if org != nil {
+		// Free-tier ceiling: 20GB disk. Autumn orgs pay per GB-second (metered
+		// by the edge autumn_meter → billed via Autumn's `disk_overage_gb_seconds`
+		// feature) so they may launch anything up to the platform ceiling
+		// (256GB by default, tunable via OPENSANDBOX_MAX_DISK_MB — see
+		// checked above; only free users are gated here.
+		//
+		// NOTE: the per-org `max_disk_mb` column is no longer enforced. Prior to
+		// disk-billing being wired end-to-end it was a manual admission cap ("has
+		// this customer been approved for larger disks"). Now that disk overage is
+		// actually billed, any paying org may launch any allowed size and get an
+		// accurate bill. The column stays in the schema as dead legacy; a follow-up
+		// migration can drop it.
 		if effPlan == "free" && effProvider != "autumn" && cfg.DiskMB > 20480 {
 			return c.JSON(http.StatusPaymentRequired, map[string]string{
 				"error": "upgrade to pro for larger disk sizes",
 			})
 		}
-		maxDisk := org.MaxDiskMB
-		if maxDisk == 0 {
-			maxDisk = 20480
-		}
-		if cfg.DiskMB > maxDisk {
-			return c.JSON(http.StatusForbidden, map[string]string{
-				"error": fmt.Sprintf("disk size %dMB exceeds org limit of %dMB", cfg.DiskMB, maxDisk),
-			})
-		}
 	}
 
 	// Declarative image or named snapshot → resolve to checkpoint and use createFromCheckpoint flow
@@ -2474,7 +2481,12 @@ func (s *Server) wakeSandboxRemote(c echo.Context, sandboxID string, req types.W
 		}
 	}
 
-	grpcCtx, cancel := context.WithTimeout(c.Request().Context(), 60*time.Second)
+	// 5 min covers cross-worker wake at the platform's 256GB disk cap: the
+	// target worker chunk-downloads (16-way parallel, 64MB chunks — see
+	// storage/s3.go) + tar-extract. At the old 60s cap, any wake of a >30GB-used
+	// sandbox on a worker that didn't already hold the qcow2 timed out. Same-
+	// worker wake is unaffected (opens a local qcow2 in <1s).
+	grpcCtx, cancel := context.WithTimeout(c.Request().Context(), 5*time.Minute)
 	defer cancel()
 
 	grpcResp, err := grpcClient.WakeSandbox(grpcCtx, &pb.WakeSandboxRequest{

internal/config/config.go

@@ -108,6 +108,13 @@ type Config struct {
 	DefaultSandboxCPUs     int // default vCPUs per sandbox, default 1
 	DefaultSandboxDiskMB   int // default disk quota per sandbox (MB), 0 = no quota
 
+	// MaxDiskMB caps `diskMB` accepted by the create APIs (/api/sandboxes and
+	// the edge-fanout /internal/sandboxes/create). Platform sanity ceiling —
+	// protects worker density + hibernate/wake latency at very large disks.
+	// 0 = code default 262144 (256GB). Bump via OPENSANDBOX_MAX_DISK_MB once
+	// timeouts + worker density have been validated for the new ceiling.
+	MaxDiskMB int
+
 	// QEMU VM configuration (worker mode)
 	KernelPath string // Path to vmlinux kernel
 	ImagesDir  string // Path to base rootfs images
@@ -372,6 +379,7 @@ func Load() (*Config, error) {
 		DefaultSandboxMemoryMB: envOrDefaultInt("OPENSANDBOX_DEFAULT_SANDBOX_MEMORY_MB", 256),
 		DefaultSandboxCPUs:     envOrDefaultInt("OPENSANDBOX_DEFAULT_SANDBOX_CPUS", 1),
 		DefaultSandboxDiskMB:   envOrDefaultInt("OPENSANDBOX_DEFAULT_SANDBOX_DISK_MB", 0),
+		MaxDiskMB:              envOrDefaultInt("OPENSANDBOX_MAX_DISK_MB", 0),
 
 		KernelPath:     os.Getenv("OPENSANDBOX_KERNEL_PATH"),     // default derived from DataDir
 		ImagesDir:      os.Getenv("OPENSANDBOX_IMAGES_DIR"),      // default derived from DataDir

internal/config/keyvault.go

@@ -74,6 +74,11 @@ var kvMapping = map[string]string{
 	"server-cf-admin-secret":    "OPENSANDBOX_CF_ADMIN_SECRET",
 	"server-session-jwt-secret": "OPENSANDBOX_SESSION_JWT_SECRET",
 	"server-halt-list-url":      "OPENSANDBOX_HALT_LIST_URL",
+	// Platform per-sandbox disk ceiling enforced by the CP admission checks.
+	// Zero = code default 262144 (256GB). Raise via KV to allow larger sandboxes
+	// on this cell after validating hibernate/wake/migration timeouts at the
+	// new ceiling.
+	"server-max-disk-mb":        "OPENSANDBOX_MAX_DISK_MB",
 	// Pre-warmed sandbox pool (control-plane owned). Per-cell, per-worker target.
 	"server-pool-target":   "OPENSANDBOX_POOL_TARGET",   // pooled boxes per worker (default 10; 0 disables)
 	"server-pool-enabled":  "OPENSANDBOX_POOL_ENABLED",  // "0" to disable (default on)