review: event stream cap, billing order, replica interval, silkd session and pty, compat cuts

.golangci.yml

@@ -59,8 +59,6 @@ linters:
   exclusions:
     generated: strict
     paths:
-      - vendor
-      - mocks
       - dist
     rules:
       - path: _test\.go

Makefile

@@ -57,8 +57,8 @@ sandboxd: ## build dist/sandboxd
 	mkdir -p dist
 	cd sandboxd && GOWORK=off go build -ldflags "-X main.version=$(SANDBOXD_VERSION)" -o ../dist/sandboxd .
 
-go-test: ## go test -race across the Go modules
-	for m in $(GO_MODULES); do (cd $$m && GOWORK=off go test -race ./...) || exit 1; done
+go-test: ## go test -race and one pass of every benchmark across the Go modules
+	for m in $(GO_MODULES); do (cd $$m && GOWORK=off go test -race ./... && GOWORK=off go test -run '^$$' -bench . -benchtime 1x ./...) || exit 1; done
 
 bench: ## claim-tier + data-plane benchmarks on this node; removes every sbx-* VM (see docs/benchmarks.md)
 	bash scripts/bench.sh

README.md

@@ -108,7 +108,7 @@ sandboxd](docs/deploy.md) covers release binaries, the full config and systemd.
   base SDK
 - `shell.yml` — shellcheck over every tracked shell script
 - `images.yml` — the single image entry point: on a push touching
-  `boot/**`, `silkd/**`, `protocol/**`, or `os-image/**` it builds the
+  `boot/**`, `silkd/**`, or `os-image/**` it builds the
   changed carriers (via `build-boot.yml` / `build-silkd.yml`,
   `workflow_call`) then os-images, in order, so the chain is deterministic
 - `build-os-images.yml` — bakes base + flavors FROM the sha-pinned carriers

docs/cluster.md

@@ -481,7 +481,7 @@ checkpoint store and keeps each claim's registry row on the latest snapshot:
   restart, or after it wakes from an archive or an adoption; until a claim woken
   that way commits its first replica it has no row, and it is lost with its node. A hibernated, archived or failed claim,
   one whose tenant was removed, and one with volumes (capture refuses them) is
-  skipped; a replicated claim that gains volumes
+  skipped, and a skipped claim keeps its interval, so one that wakes past it is due at once; a replicated claim that gains volumes
   drops its row and replica. The claim's policy pool, not its own template key,
   picks the interval, so a claim cloned from a template promoted out of a
   replicating pool follows that pool. Rounds share the maintenance budget with

docs/deploy.md

@@ -425,7 +425,7 @@ generation sweep.
 | `<prefix><id>/labels.json` | Optional labels, removed with the record. |
 | `<prefix>manifests/<id>/<gen>.json` | One JSON manifest per published generation: `lineage` and, per file, `path`, `size` and the present `chunks` (`index`, `sha256`). `<gen>` is `export-<16 hex>`. |
 | `<prefix>chunks/<lineage>/<sha256>` | A 16 MiB chunk. A lineage is the dedup scope: it defaults to the record id, and every checkpoint and archive of one claim shares the claim's sandbox id. A chunk is shared by every record of its lineage that names the same digest, and only by those. |
-| `<prefix>lineages/<lineage>/<id>` | Zero-byte marker: record `<id>` published into `<lineage>`, written after the manifest and before `meta.json`. A publish into a shared lineage reads only the records these list, so a crash between manifest and marker only costs a missed dedup. It only drives dedup, never chunk liveness. |
+| `<prefix>lineages/<lineage>/<id>` | Zero-byte marker: record `<id>` published into another record's lineage `<lineage>`, written after the manifest and before `meta.json`; a record in its own lineage writes none. A publish into a shared lineage reads only the records these list, so a crash between manifest and marker only costs a missed dedup. It only drives dedup, never chunk liveness. |
 
 **Offline reclamation only.** Stop every sandboxd node using the prefix and
 wait for all in-flight publishes, deletes and fetches to finish. Keep the

docs/egress.md

@@ -45,8 +45,9 @@ supported*), no authentication (the socket path already carries the sandbox's
 identity), and a `DOMAINNAME` destination is resolved host-side, so the guest
 still needs no resolver. Each door serves at most 256 connections per sandbox
 at a time; later ones sit unserved in the socket backlog until one closes, and
-the proxy keeps at most 64 idle upstream connections per sandbox — 64 more for
-decrypted traffic when the pool intercepts — so a guest cannot turn its proxy
+the proxy keeps at most 64 idle upstream connections per sandbox and exit (the
+node's own and each upstream proxy the claim has used), as many again for
+decrypted traffic when the pool intercepts, so a guest cannot turn its proxy
 into a host descriptor sink.
 
 A SOCKS5 tunnel takes the decision an HTTP `CONNECT` to the same host takes,
@@ -191,8 +192,7 @@ stays denied). A claim with `"egress": false`
 takes no policy at all, whatever its pool's: no door is bound, so the guest's
 dial is refused. Which layers a claim has is settled when it
 is made and kept for its life: a `PUT /v1/pools` that adds or drops a pool
-without a config-file policy changes the claims made after it, never a live one (a record from before this
-rule resolves against the live pool set). Secrets are registered separately and
+without a config-file policy changes the claims made after it, never a live one. Secrets are registered separately and
 referenced by name — the value comes from the environment, never the config
 file: a claim's own `guest: false` env first, the node's second
 ([below](#claim-env-and-secrets)).
@@ -311,9 +311,9 @@ gateway key per user, while the guest holds a placeholder.
   never logged, listed or returned.
 
 The proxy reads a snapshot of the claim's env that each env write publishes,
-without a lock, once per request whose matched rule names a secret or whose
-host has a claim credential, and once per `CONNECT` an `"inject"` rule covers;
-other requests pay nothing.
+without a lock, once per intercepted request, once per request whose matched
+rule names a secret, and once per `CONNECT` an `"inject"` rule covers; other
+requests pay nothing.
 
 ### Claim credentials
 

docs/sandboxd-api.md

@@ -320,7 +320,7 @@ Malformed or conflicting metadata, including a saved path outside `volume_roots`
 prevents startup. The catalog is node-local with one sandboxd writer.
 
 Sandboxd does not create, check, or delete backings. Keep them outside
-`data_dir`: sandboxd clears its `readers`, `goldens`, `fork-*`,
+`data_dir`: sandboxd clears its `readers`, `goldens`, `moves`, `fork-*`,
 `checkpoint-staging` and `template-staging` entries.
 
 ### Volume access lists
@@ -804,7 +804,7 @@ A refused reload changes nothing.
 ## POST /v1/drain
 
 Auth: root only (tenant tokens get 403). Cordons the node for maintenance: claim/fork/branch answer
-429 `node draining` (on a cluster a non-volume claim tries a peer redirect first, and
+429 with an error ending `node draining` (on a cluster a non-volume claim tries a peer redirect first, and
 peers stop redirecting here within a tick, because this node gossips no pools),
 unclaimed warm VMs are destroyed, and live claims keep serving until release
 or TTL. Pool ownership is untouched — no pools.json write, no config change.
@@ -924,8 +924,9 @@ never the source sandbox's. 400 metadata over a bound, an unknown
 `on_expire`, or a bad `env` entry.
 
 Checkpoints are node-local (unless the store is shared — see
-[Configuration](deploy.md#configuration)). On a miss the node probes its peers
-and answers with a redirect shaped like a warm-miss
+[Configuration](deploy.md#configuration)). On a miss a node with a node-local
+store probes its peers (a shared s3 store skips the probe: every node already
+resolves every record) and answers with a redirect shaped like a warm-miss
 [`POST /v1/claim`](#post-v1claim), at most 3 addresses; retry the same body
 (+`no_redirect: true`) at each candidate. With nothing to redirect to (or
 `no_redirect` set) and `checkpoint_peer_heal` on, it pulls the record and serves
@@ -1139,8 +1140,9 @@ them in `secret` (`gh,claim:<entry>`, never a value). An egress event routed thr
 [upstream proxy](egress.md#upstream-proxies) carries `upstream` (its `host:port`), and with
 `egress_usage_bytes` on, `egress_bytes` events add the connection's payload `tx`/`rx` bytes. A sandbox records at most 10 denied or rate-limited egress decisions per second after a burst of 100, so `egress` events do not count every refusal. A volume claim also carries
 `volumes`, the applied catalog names, and — omitted when empty — `volumes_rw`,
-the subset of those names claimed `rw`, so billing can discriminate write
-access (mounts and host paths are not billing dimensions). The file rotates at
+the subset of those names claimed `rw`, on its `claim` event, so billing can
+discriminate write access at claim (mounts and host paths are not billing
+dimensions); a live attach or detach records no usage event. The file rotates at
 64 MiB keeping one `.1` backup, so a tailing collector never loses a window
 silently. A last line a crash cut short is ended with a newline when the node
 reopens the journal, so it never swallows the next event. A [live move](#post-v1sandboxesidmove)

docs/sdk-python.md

@@ -90,7 +90,7 @@ directly. To recover a handle when only `id` + `token` survived (say,
 across a process restart):
 
 ```python
-sb = client.lookup(id, token)   # probes the entry node and every mesh peer concurrently
+sb = client.lookup(id, token)   # lists the peers, then probes the entry node and every peer concurrently
 ```
 
 ## Claiming
@@ -107,7 +107,6 @@ sb = client.new("ghcr.io/cocoonstack/sandbox/rt:24.04",
 | `net` | `"none"`, `"egress"` | `"none"` | Cloud Hypervisor network shape: `none` disables the NIC and uses vsock-only I/O; `egress` attaches a bridge/CNI NIC |
 | `size` | `"small"`, `"medium"`, `"large"`, `"xlarge"`, `"2xlarge"` | `"small"` | resource tier: 1cpu/512M, 2cpu/1G, 4cpu/4G, 4cpu/8G, 8cpu/16G |
 | `volumes` | bare names or `{name, mount?, mode?, attach_only?}` mappings | `None` | attach and mount up to eight unique catalog dataset disks; an omitted mount defaults to `/volumes/<name>`; `mode` is `"ro"` (default) or `"rw"` — `"rw"` requires the catalog entry's `writable: true`; accepted by `Client.new` and `Template.new` |
-| `mount` | bool | `True` | mount every requested volume. `False` attaches the devices and leaves the mounting to the workload; sandboxd then rejects a mapping carrying `mount` with 400 |
 | `ttl_seconds` | int | server default 5m | sandbox TTL, server-capped at 24h. The node reaps the sandbox after the TTL even if the client vanishes |
 | `deadline` | `time.monotonic()` value | `None` | keyword-only wall clock over the whole claim, redirect targets included: every request is bounded by whatever is left and a spent deadline raises `TimeoutError`. Without it each redirect candidate gets a fresh `timeout`, so a cluster with unreachable peers can cost a multiple of it. Also on `Template.new` and `Checkpoint.new` |
 
@@ -238,8 +237,7 @@ handle parks at most 8 idle connections. A kept connection counts as live for
 `idle_hibernate_seconds` until it closes, so keep the window below that
 setting; `close` and `hibernate` drop it at once.
 Streams (`watch`, `open_pty`, `dial_port`, an LSP session) take a connection
-of their own, and a guest whose silkd predates the back-to-back protocol
-gets one connection per call as before.
+of their own.
 
 The handle's `sb.deadline` remains the value returned when it was created or last
 renewed, so after an archive/wake transition read the current one from
@@ -281,8 +279,6 @@ tick — prefer the handle right after promoting (see
 `tpl.content_digest` identifies the published export bytes. A caller pinning
 the mutable name can compare a claim's `template_digest` with its expected
 value and close/refuse a mismatch.
-Templates published by an older sandboxd have empty digests until they are
-re-promoted after the node is upgraded.
 
 ## Checkpoints — branching and time travel
 

docs/sdk.md

@@ -166,8 +166,7 @@ at most 8 idle connections. A kept connection counts as live for
 setting; `Close` and `Hibernate` drop it at once. Long-lived streams
 (`Watch`, `OpenPty`, `DialPort`, an LSP session) take a connection of their
 own. Reuse needs a Unix client (the liveness peek is a `MSG_PEEK`); on
-Windows, and against a guest whose silkd predates the back-to-back protocol,
-every call dials as before.
+Windows every call dials as before.
 
 An explicit scheme in an owner, redirect, peer, or `Attach` address wins;
 a bare address inherits the entry client's scheme. Trust settings are shared
@@ -425,8 +424,6 @@ replaces the template. `Template.ContentDigest` identifies the published
 export bytes; a claim from that exact generation carries the same value in
 `Sandbox.TemplateDigest`. A caller pinning a mutable template name can compare
 the claim's value with its expected digest and close/refuse a mismatch.
-Templates published by an older sandboxd have empty digests until they are
-re-promoted after the node is upgraded.
 
 **On the default local-disk backend templates live on one node**, and on a
 cluster the parent claim may have been redirected — the returned `Template`

docs/silkd.md

@@ -25,9 +25,8 @@ then reads the next request on the same connection. `stdin`, `stdin_close`,
 terminal frame they are dropped, and a request frame sent while a streaming
 RPC still runs ends that RPC's input and is served after it, except in the
 upload of `fs_write` or `fs_push`: there a request frame before `data_end`
-fails the upload with `bad_request` and is dropped. A client that
-closes after one RPC is served exactly as before; `info` reports `proto: 2`
-on a daemon that keeps the connection. Binary payloads ride base64 in `data`
+fails the upload with `bad_request` and is dropped. A client may close
+after any terminal frame. Binary payloads ride base64 in `data`
 fields. Frames are capped at 8 MiB; requests carry `"v": 1` and unknown
 fields are ignored (the forward-compatibility story).
 
@@ -59,7 +58,7 @@ suites — a frame only one side can parse fails CI.
 | git | `git_clone {url, path, branch?, depth?, auth?}` / `git_status {path}` / `git_add {path, files}` / `git_commit {path, message, author}` / `git_push {path, auth?}` / `git_pull {path, auth?}` / `git_branch {path, action, name?}` | `git_status_result` (porcelain v2), `git_commit_result{hash}`, `git_branches{current, branches}` for `git_branch {action: "list"}`; every other verb terminates with `done`. A status past about 1 MiB of entries carries `truncated: true` with the head of the list, so one frame never exceeds the cap. `auth` is injected as an in-memory header, never written to guest disk |
 | port | `port_forward {port}` | relays guest TCP 127.0.0.1:port over this connection: `ready` once connected, then `data` both ways. Each direction ends on its own: `data_end` half-closes the guest socket, `done` reports the guest closing its write side, and the RPC ends once both have. A client that only drops the connection leaves the relay open until the guest side closes or writes. Works on both lanes — the no-network lane's only way in |
 | lsp | `lsp_start {language, root?}` / `lsp_request {server_id}` / `lsp_stop {server_id}` | a broker for the language server a flavor image ships: `lsp_start` spawns the argv named in `/etc/silkd/lsp.d/<language>` (absent on the base image → `not_found` naming the flavor) and answers `lsp_started{server_id}`; `lsp_request` attaches the JSON-RPC byte stream (`ready`, then `data` both ways — silkd pipes bytes, it never parses LSP; `data_end` half-closes the server's stdin) and the stream ending reaps the server (v1 single-shot); `lsp_stop` kills it early, and a server nothing attaches within 5 minutes is reaped |
-| misc | `info` | `info{version, proto, uptime_secs, procs, sessions}` — sandboxd uses it for readiness and the SDKs use `proto` to negotiate connection reuse; there is no public SDK method. Distinct from the control plane's `GET /v1/info`, which reports node pools and claims |
+| misc | `info` | `info{version, uptime_secs, procs, sessions}` — sandboxd uses it for readiness; there is no public SDK method. Distinct from the control plane's `GET /v1/info`, which reports node pools and claims |
 
 ## Error kinds
 

e2e/cmd/rpcbench/main.go

@@ -179,11 +179,12 @@ func edgeTLS(addr, caCert string) (*tls.Config, error) {
 func dialAgent(ctx context.Context, addr, id, token string, tlsCfg *tls.Config) (net.Conn, error) {
 	host := strings.TrimPrefix(strings.TrimPrefix(addr, "https://"), "http://")
 	var d net.Dialer
-	raw, err := d.DialContext(ctx, "tcp", host)
+	dialed, err := d.DialContext(ctx, "tcp", host)
 	if err != nil {
 		return nil, err
 	}
-	stop := context.AfterFunc(ctx, func() { _ = raw.Close() })
+	raw := dialed
+	stop := context.AfterFunc(ctx, func() { _ = dialed.Close() })
 	defer stop()
 	scheme := "http"
 	if tlsCfg != nil {

e2e/cmd/smoke/main.go

@@ -319,7 +319,7 @@ func smokeHibernate(ctx context.Context, client *sandbox.Client, sb *sandbox.San
 	if err = wantHibernated(ctx, client, 1); err != nil {
 		return err
 	}
-	out, err := sess.Exec(ctx, "sh", "-c", "echo $HIBMARK") // first guest access wakes the VM
+	out, err := sess.Exec(ctx, "sh", "-c", "echo $HIBMARK")
 	if err != nil {
 		return fmt.Errorf("exec after hibernate: %w", err)
 	}

e2e/cmd/volumesmoke/main.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"net/http"
 	"os"
 	"path"
 	"path/filepath"
@@ -224,7 +225,7 @@ func runWritable(ctx context.Context, client *sandbox.Client, template, volume s
 		_ = busy.Close()
 		return errors.New("second writable claim succeeded while the first held the image")
 	}
-	if !strings.Contains(busyErr.Error(), "409") {
+	if apiErr, ok := errors.AsType[*sandbox.APIError](busyErr); !ok || apiErr.Status != http.StatusConflict {
 		return fmt.Errorf("second writable claim failed without 409: %w", busyErr)
 	}
 	if err = writer.Close(); err != nil {

mcp/server_test.go

@@ -21,7 +21,7 @@ import (
 )
 
 var infoFrame = func() string {
-	frame, err := wire.EncodeResponse(&wire.InfoResp{Version: "test", Proto: wire.KeepAliveProto})
+	frame, err := wire.EncodeResponse(&wire.InfoResp{Version: "test"})
 	if err != nil {
 		panic(err)
 	}

mcp/tools.go

@@ -431,7 +431,7 @@ func parse(raw jsontext.Value, v any) error {
 }
 
 func unmarshalInput(raw []byte, v any) error {
-	return json.Unmarshal(raw, v, json.MatchCaseInsensitiveNames(true), jsontext.AllowInvalidUTF8(true))
+	return json.Unmarshal(raw, v, jsontext.AllowInvalidUTF8(true))
 }
 
 func jsonText(v any) string {

protocol/wire/fixtures/v1/resp_info.json

@@ -1 +1 @@
-{"type":"info","version":"0.1.0","proto":2,"uptime_secs":12,"procs":1,"sessions":0}
+{"type":"info","version":"0.1.0","uptime_secs":12,"procs":1,"sessions":0}

protocol/wire/frame.go

@@ -15,8 +15,6 @@ import (
 const (
 	// ProtoVersion is stamped into every request as "v"; silkd ignores unknown fields.
 	ProtoVersion = 1
-	// KeepAliveProto is the InfoResp.Proto from which silkd serves RPCs back to back on one connection.
-	KeepAliveProto = 2
 	// MaxFrame mirrors silkd's frame cap.
 	MaxFrame = 8 << 20
 	// BulkChunk mirrors silkd's BULK_CHUNK, the payload of one bulk data frame.
@@ -449,7 +447,7 @@ type DataEnd struct{}
 
 func (DataEnd) Op() string { return "data_end" }
 
-// Started reports the spawned pid (synthetic when the OS pid is unknown).
+// Started reports the spawned pid.
 type Started struct {
 	PID uint32 `json:"pid"`
 }
@@ -470,7 +468,7 @@ type Stderr struct {
 
 func (Stderr) RespType() string { return "stderr" }
 
-// Exit is the terminal frame of a foreground exec; -1 means killed or unknown.
+// Exit is the terminal frame of a foreground exec; a signaled child reports 128+signal, -1 an unknown status.
 type Exit struct {
 	Code int32 `json:"code"`
 }
@@ -482,7 +480,7 @@ type Done struct{}
 
 func (Done) RespType() string { return "done" }
 
-// Ready acknowledges an armed watch (events after it are guaranteed captured) or a connected port_forward.
+// Ready acknowledges an armed watch (events after it are guaranteed captured), a connected port_forward or an attached lsp_request.
 type Ready struct{}
 
 func (Ready) RespType() string { return "ready" }
@@ -500,7 +498,6 @@ func (e *ErrorResp) Error() string { return e.Kind + ": " + e.Message }
 // InfoResp answers Info.
 type InfoResp struct {
 	Version    string `json:"version"`
-	Proto      uint32 `json:"proto"`
 	UptimeSecs uint64 `json:"uptime_secs"`
 	Procs      int    `json:"procs"`
 	Sessions   int    `json:"sessions"`