feat(tenant): manage CPU and NUMA pinning without container restarts

docs/commands.md

@@ -380,12 +380,26 @@ Long-lived tenant containers ([tenant.md](tenant.md)). Every subcommand except `
 | `--size` | required | Volume size, such as `500G` or `2T`. |
 | `--fs` | `ext4` | Volume filesystem: `ext4` (can shrink offline) or `xfs`. |
 | `--gpus` | none | Comma-separated GPU indices. |
+| `--cpus` | unmanaged | Host CPU list (`N` or `N-M`, comma-separated), not a CPU quota. |
+| `--mems` | unmanaged | NUMA memory-node list (`N` or `N-M`, comma-separated). |
 | `--account` | none | `NAME[:UID[:GID]]`, repeatable; UIDs default to 1000, 1001, ... |
 | `--sudo` | `false` | Give the accounts sudo inside the container. |
 | `--operator` | none | Host account allowed `sudo rootfiles tenant ops` on this tenant (repeatable). |
 | `--port` | none | Extra published port `HOST:CONTAINER[/udp]` (repeatable). |
 | `--bind` | none | Extra bind mount `SRC:DST[:ro\|rw]` outside the volume quota (repeatable). |
 
+### rootfiles tenant cpuset
+
+`rootfiles tenant cpuset TENANT --cpus 0-3 --mems 0`. Save CPU/NUMA pinning and update the existing running or stopped container in place, without restarting. Mutating.
+
+| Flag | Default | Purpose |
+|---|---|---|
+| `--cpus` | retain spec | Host CPU list (`N` or `N-M`, comma-separated); explicit empty makes the key unmanaged. |
+| `--mems` | retain spec | NUMA memory-node list; explicit empty makes the key unmanaged. |
+| `--clear` | `false` | Remove cpuset from the spec and reset both container keys to the full host lists. Exclusive with the list flags. |
+
+Absent or empty spec keys are unmanaged, so hand-set pins stay untouched. Other pending settings remain pending; run `rootfiles tenant apply TENANT` separately to recreate for them. A failed update restores the previous spec. On cgroup v2, changing memory nodes migrates existing pages and may take time for large processes. `tenant status` shows `update` for live drift and `status TENANT` shows spec and actual pins.
+
 ### rootfiles tenant gpu
 
 `rootfiles tenant gpu TENANT --gpus 0,1`. Change a tenant's GPUs and recreate the container. Mutating.

docs/specs/cli-runtime.md

@@ -41,6 +41,8 @@ Shared plumbing every native `rootfiles` command runs through: the cobra root an
 
 ## Behaviour
 
+- `tenant cpuset` is a native mutation: it uses the same root, Linux and global-lock preflight as the other tenant mutations.
+
 - Aliases resolve to one command object: `tenant reset` -> `passwd`, `tunnel install` -> `setup`. Reader and mutation annotations stay on that canonical command. Deprecated backup/user flags retain old invocations; `user passwd --suffix` remains accepted until the next minor release.
 - Root help includes Rocky/fleet and derives profile names from `config.AvailableProfiles()`. The audited commands carry examples; Use strings contain positional syntax only, checked against Args arity by `cli_surface_test.go`. VLAN help/prompt examples use `203.0.113.10/32`.
 

docs/specs/tenant.md

@@ -35,9 +35,9 @@ Package boundary
 
 Spec
 - One spec per tenant at `<home_base>/.rootfiles/tenants/<t>.yaml` (`SpecDir`), mode 0600, under the root-only metadata tree checked by `RequireTrustedMetadata`. `LoadSpec` decodes strictly (`KnownFields`) and rejects a file whose `name` differs from its file name.
-- Fields (`Spec` in `internal/tenant/spec.go`): `name`, `image`, `gpus` (GPU indices), `ssh_port`, `ports` (`HOST:CONTAINER[/tcp|udp]`), `volume.size` (min 64M), `volume.fs` (ext4 default, or xfs), `users` (name, uid/gid 1000-65533, sudo), `operators`, `extra_binds` (`SRC:DST[:ro|rw]`, absolute, not on `/data`, `/home`, `/tmp`, `/etc/tenant-system`).
+- Fields (`Spec` in `internal/tenant/spec.go`): `name`, `image`, `gpus` (GPU indices), `ssh_port`, `ports` (`HOST:CONTAINER[/tcp|udp]`), `volume.size` (min 64M), `volume.fs` (ext4 default, or xfs), `users` (name, uid/gid 1000-65533, sudo), `operators`, `extra_binds` (`SRC:DST[:ro|rw]`, absolute, not on `/data`, `/home`, `/tmp`, `/etc/tenant-system`), optional `cpuset.cpus` and `cpuset.mems` (comma-separated Linux indices `N` or ranges `N-M`, N <= M, no spaces). Empty or absent cpuset keys are unmanaged.
 - `Validate` reports every problem at once. Tenant names match `^[a-z][a-z0-9-]{0,30}$`, contain no `.` (so sshpiper route names `<tenant>.<user>` cannot collide) and never end in `-prev`.
-- `Spec.Digest` covers only settings a new container can change: name, image, sorted GPUs, SSH port, ports, extra binds. Users, operators and volume size converge without a recreate. Adding a field that needs a new container means adding it to the digest.
+- `Spec.Digest` covers only settings a new container can change: name, image, sorted GPUs, SSH port, ports, extra binds. Users, operators, volume size and cpuset converge without a recreate. Cpuset is excluded from the digest; absent or empty cpuset preserves the legacy digest, run arguments and converge steps byte for byte. Adding a field that needs a new container means adding it to the digest.
 
 Host layout (root `modules.tenant.root`, default `/raid/tenants`, must be a root-only path chain: `config.RootOnlyHomeBase` in `tenantEnv`)
 - `<root>/<t>/` is 0700 so host users cannot reach setuid files a tenant leaves in its volume.
@@ -47,6 +47,7 @@ Host layout (root `modules.tenant.root`, default `/raid/tenants`, must be a root
 
 Container (`Manager.RunArgs`)
 - Created with `docker run`, not compose: name `tenant-<t>`, hostname `<t>`, `--restart=always --init --ipc=host`, labels `rootfiles.tenant`, `rootfiles.tenant.spec=<digest>`, `rootfiles.tenant.gpus=<indices>`, `-p <ssh_port>:22`.
+- Managed cpuset keys add `--cpuset-cpus` / `--cpuset-mems` on create and recreate. Multiple tenants may share the same CPUs/nodes.
 - GPUs always attach with `--runtime=nvidia --gpus "device=<indices>"`. Evidence (verified 2026-10-20): runc plus `--gpus` loses device access on any `systemctl daemon-reload`, because its devices come only from the prestart hook; the nvidia runtime lists them in the OCI spec. The daemon `default-runtime` is not changed.
 - `/data`, `/home`, `/tmp` are `--mount type=bind` from `vol/data|home|tmp`; `system` is bound read-only at `/etc/tenant-system`. A bind fails on a missing source, so a container never starts on an unmounted volume and writes past the quota. Keep it a bind, never a named volume.
 - Entrypoint is `/bin/sh /etc/tenant-system/init.sh` (`InitScript`), the image-agnostic boot contract. The only image requirement is `openssh-server`. On every start it creates users and groups, restores password hashes and host keys, starts sshd (trusting `piper.pub` for every account when present), runs `/data/.tenant/autostart.sh` as the first user, then sleeps. `init.sh users` re-runs only the account step.
@@ -57,6 +58,16 @@ Passwords
 - `SyncShadow` copies usable hashes of spec users from the running container to `system/shadow`; a stopped container keeps the stored hashes. A direct bind of `/etc/shadow` is not used because `passwd` renames the file.
 - Reads from a tenant container go through `readInContainer`: `timeout 10 head -c 4194304`, inside the container so symlinks resolve against its root.
 
+Live settings (`Manager.UpdateCpuset`, `Manager.ChangeCpuset`)
+- Before Apply or Import changes anything, managed CPU and memory-node lists must be subsets of sysfs `cpu/online` and `node/has_memory`. Unreadable/invalid sources block managed changes. Unmanaged keys do not read sysfs.
+- Inspect reads `HostConfig.CpusetCpus` and `CpusetMems`. With a current digest, Apply updates only differing managed keys through one `docker update`, before starting a stopped container or syncing accounts. Equivalent list representations do not drift; unmanaged pins remain untouched.
+- `tenant cpuset TENANT --cpus LIST --mems LIST` saves the spec and updates the existing running or stopped container only. Omitted flags retain existing spec keys; explicit empty values make them unmanaged. The command never runs full converge, stop, rename, run or restart. A missing or mismatched digest label remains pending and the output points to `rootfiles tenant apply TENANT` for recreation.
+- `--clear` is exclusive with the two list flags. It removes the spec field and explicitly resets both live keys to the full host online CPU and memory-node lists.
+- A live-update failure restores the previous spec bytes, including comments, through Runner. Dry-run changes neither spec nor container.
+- `tenant status` reports `update` for live-only drift; detail shows both desired settings and actual container values, including unmanaged pins.
+- Import carries legacy cpuset values into the spec and new container. Before a recreate, unmanaged actual pins warn that they will be lost and name the command to adopt them.
+- On cgroup v2 a live memory-node change migrates existing pages and may take time. Older binaries strictly reject the new field; remove it before downgrading (the live pin remains).
+
 Recreate (`Manager.recreate`, on digest change)
 - Refuses while `tenant-<t>-prev` exists, so an earlier rollback copy is never discarded.
 - Syncs hashes, warns with the writable-layer size, stops, renames to `-prev` with `--restart=no`, runs the new container and verifies it; on failure removes it and restores the old one (name, restart policy, start). On success `-prev` is kept until the operator removes it.

docs/tenant.md

@@ -69,6 +69,8 @@ sudo rootfiles tenant create lab \
 | `--size` | Volume size, such as `500G` or `2T` |
 | `--fs` | Volume filesystem: `ext4` (default; can shrink offline) or `xfs` |
 | `--gpus` | Comma-separated GPU indices |
+| `--cpus` | Host CPU list such as `0-3,8-11` (pinning, not a CPU quota) |
+| `--mems` | NUMA memory-node list such as `0` or `0-1` |
 | `--account NAME[:UID[:GID]]` | Account to create (repeatable); UIDs default to 1000, 1001, ... and the GID to the UID |
 | `--sudo` | Give the accounts sudo inside the container |
 | `--operator` | Host account allowed `sudo rootfiles tenant ops` on this tenant (repeatable) |
@@ -142,3 +144,29 @@ Import reads the running container: image, published ports (22 becomes the SSH p
 - One host per command; `fleet` only ships the module config.
 - The writable layer has no size limit on ext4 hosts.
 - Not supported on Rocky Linux.
+
+## CPU and NUMA pinning
+
+```yaml
+cpuset:
+  cpus: "0-3,8-11"
+  mems: "0"
+```
+
+Either key may be set alone. Absent or empty spec keys are unmanaged: apply leaves any hand-set pin untouched. CPU indices must be online (`/sys/devices/system/cpu/online`); memory nodes must have memory (`/sys/devices/system/node/has_memory`). Invalid or unavailable values are refused before any container changes. Several tenants may share the same CPUs/nodes.
+
+```bash
+sudo rootfiles tenant cpuset lab --cpus 0-3 --mems 0
+sudo rootfiles tenant status lab
+sudo rootfiles tenant cpuset lab --clear
+```
+
+`tenant cpuset` saves the spec and uses `docker update` on the same running or stopped container, without a restart. Omitted flags retain their existing spec keys. An explicit `--cpus ""` or `--mems ""` makes that key unmanaged and leaves the live setting alone. `--clear` instead removes the field and resets both live settings to the host's full lists. A failed Docker update restores the previous spec. Dry-run writes nothing.
+
+If the image, ports or other container settings have pending changes (including a container without a digest label), they remain pending. The command says when `rootfiles tenant apply lab` is still needed; apply may recreate for those other changes. Cpuset alone never changes the recreate digest. Create and recreate use the spec's managed pinning. Before recreating a container with unmanaged hand-set pins, rootfiles warns; adopt the values with `tenant cpuset` first if you want to keep them. `tenant import` carries existing pins into the new spec.
+
+`tenant status` shows `update` for live-only cpuset drift and `drift` for settings requiring recreation. Detailed status shows the spec and container values, including unmanaged pins.
+
+On cgroup v2, changing memory nodes migrates existing memory pages to the new nodes. This can take noticeable time for a large process.
+
+Rollback: older binaries strictly reject specs containing `cpuset`. Before downgrading, remove the field from each spec; the container keeps its current pinning. The release notes for the version introducing this feature must carry this step.

internal/cli/runtime.go

@@ -37,7 +37,7 @@ var mutatingCommands = [][]string{
 	{"tunnel", "setup"}, {"tunnel", "restart"},
 	{"tunnel", "update"}, {"tunnel", "uninstall"},
 	{"tenant", "create"}, {"tenant", "apply"}, {"tenant", "user", "add"},
-	{"tenant", "passwd"}, {"tenant", "gpu"}, {"tenant", "image"},
+	{"tenant", "cpuset"}, {"tenant", "passwd"}, {"tenant", "gpu"}, {"tenant", "image"},
 	{"tenant", "port"}, {"tenant", "volume", "resize"}, {"tenant", "restart"},
 	{"tenant", "import"}, {"tenant", "ops"},
 }

internal/cli/runtime_test.go

@@ -19,7 +19,7 @@ import (
 
 func TestMarkMutating(t *testing.T) {
 	root := NewRootCmd("test", "abc")
-	for _, path := range [][]string{{"apply"}, {"user", "add"}, {"gpu", "assign"}, {"tunnel", "setup"}} {
+	for _, path := range [][]string{{"apply"}, {"user", "add"}, {"gpu", "assign"}, {"tunnel", "setup"}, {"tenant", "cpuset"}} {
 		c, _, err := root.Find(path)
 		if err != nil || c.Annotations[annotMutates] != "true" {
 			t.Errorf("%v should be marked mutating", path)

internal/cli/tenant.go

@@ -27,7 +27,7 @@ host keys survive image changes. Specs live in <home_base>/.rootfiles/tenants.`,
 	cmd.AddCommand(
 		newTenantListCmd(), newTenantStatusCmd(), newTenantCreateCmd(), newTenantApplyCmd(),
 		newTenantUserCmd(), newTenantPasswdCmd(),
-		newTenantGPUCmd(), newTenantImageCmd(), newTenantPortCmd(), newTenantVolumeCmd(),
+		newTenantGPUCmd(), newTenantCpusetCmd(), newTenantImageCmd(), newTenantPortCmd(), newTenantVolumeCmd(),
 		newTenantRestartCmd(), newTenantLogsCmd(), newTenantImportCmd(), newTenantOpsCmd(),
 		newTenantSyncCmd(),
 	)
@@ -163,6 +163,11 @@ func newTenantCreateCmd() *cobra.Command {
 			s.Ports, _ = cmd.Flags().GetStringArray("port")
 			s.Operators, _ = cmd.Flags().GetStringArray("operator")
 			s.ExtraBinds, _ = cmd.Flags().GetStringArray("bind")
+			cpus, _ := cmd.Flags().GetString("cpus")
+			mems, _ := cmd.Flags().GetString("mems")
+			if cpus != "" || mems != "" {
+				s.Cpuset = &tenant.Cpuset{CPUs: cpus, Mems: mems}
+			}
 			if g, _ := cmd.Flags().GetString("gpus"); g != "" {
 				gpus, err := parseGPUList(g)
 				if err != nil {
@@ -210,6 +215,8 @@ func newTenantCreateCmd() *cobra.Command {
 	f.String("size", "", "Volume size, e.g. 500G or 2T")
 	f.String("fs", "ext4", "Volume filesystem: ext4 (can shrink offline) or xfs")
 	f.String("gpus", "", "Comma-separated GPU indices")
+	f.String("cpus", "", "Host CPU list (N or N-M, comma-separated; not a CPU quota)")
+	f.String("mems", "", "NUMA memory-node list (N or N-M, comma-separated)")
 	f.StringArray("account", nil, "Account NAME[:UID[:GID]] (repeatable; uid defaults to 1000, 1001, ...)")
 	f.Bool("sudo", false, "Give the accounts sudo inside the container")
 	f.StringArray("port", nil, "Extra published port HOST:CONTAINER[/udp] (repeatable)")
@@ -376,6 +383,56 @@ func newTenantGPUCmd() *cobra.Command {
 	return cmd
 }
 
+func newTenantCpusetCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "cpuset TENANT",
+		Short: "Save CPU/NUMA pinning and update the existing container without restarting",
+		Long: `Changes only CPU/NUMA pinning, leaving other pending settings for tenant apply.
+Empty or absent spec keys are unmanaged; omitted flags retain existing spec keys.
+--clear removes the spec field and resets
+both container settings to the host's full lists. On cgroup v2, changing memory
+nodes migrates existing memory pages and can take time for large processes.`,
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cpus, _ := cmd.Flags().GetString("cpus")
+			mems, _ := cmd.Flags().GetString("mems")
+			clear, _ := cmd.Flags().GetBool("clear")
+			if clear && (cmd.Flags().Changed("cpus") || cmd.Flags().Changed("mems")) {
+				return fmt.Errorf("--clear cannot be combined with --cpus or --mems; use --clear alone to reset both settings")
+			}
+			if !clear && !cmd.Flags().Changed("cpus") && !cmd.Flags().Changed("mems") {
+				return fmt.Errorf("no cpuset change requested; pass --cpus LIST, --mems LIST or --clear")
+			}
+			return tenantDo(cmd, args[0], "cpuset", func(ctx context.Context, _ *module.RunContext, m *tenant.Manager) error {
+				s, err := m.Load(args[0])
+				if err != nil {
+					return err
+				}
+				if clear {
+					s.Cpuset = nil
+				} else {
+					pin := tenant.Cpuset{}
+					if s.Cpuset != nil {
+						pin = *s.Cpuset
+					}
+					if cmd.Flags().Changed("cpus") {
+						pin.CPUs = cpus
+					}
+					if cmd.Flags().Changed("mems") {
+						pin.Mems = mems
+					}
+					s.Cpuset = &pin
+				}
+				return m.ChangeCpuset(ctx, s, clear)
+			})
+		},
+	}
+	cmd.Flags().String("cpus", "", "Host CPU list (N or N-M, comma-separated; empty makes it unmanaged)")
+	cmd.Flags().String("mems", "", "NUMA memory-node list (N or N-M, comma-separated; empty makes it unmanaged)")
+	cmd.Flags().Bool("clear", false, "Remove cpuset from the spec and reset the container to the full host lists")
+	return cmd
+}
+
 func newTenantImageCmd() *cobra.Command {
 	return &cobra.Command{
 		Use:   "image TENANT IMAGE",

internal/cli/tenant_cpuset_test.go

@@ -0,0 +1,32 @@
+package cli
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestTenantCpusetFlagErrorsBeforeHostAccess(t *testing.T) {
+	for _, args := range [][]string{nil, {"--clear", "--cpus", "0"}, {"--clear", "--mems", ""}} {
+		cmd := newTenantCpusetCmd()
+		if err := cmd.ParseFlags(args); err != nil {
+			t.Fatal(err)
+		}
+		err := cmd.RunE(cmd, []string{"lab"})
+		if err == nil || (!strings.Contains(err.Error(), "pass --cpus") && !strings.Contains(err.Error(), "use --clear alone")) {
+			t.Fatalf("%v: %v", args, err)
+		}
+	}
+}
+
+func TestTenantCpusetCreateAndCommandFlags(t *testing.T) {
+	create := newTenantCreateCmd()
+	cpuset := newTenantCpusetCmd()
+	for _, flag := range []string{"cpus", "mems"} {
+		if create.Flags().Lookup(flag) == nil || cpuset.Flags().Lookup(flag) == nil {
+			t.Fatalf("missing %s", flag)
+		}
+	}
+	if !strings.Contains(cpuset.Long, "migrates existing memory pages") {
+		t.Fatal("help omits memory migration")
+	}
+}