[#957] Lifecycle-aware cleanup of agent backend temp dirs

docs/install-vps.md

@@ -362,3 +362,12 @@ chmod 600 ~/.quadwork/.env
 14. Configure nginx reverse proxy + SSL
 15. Add HTTP basic auth
 16. Verify reboot survival: `sudo reboot`, then check `pm2 list`
+
+## Note: /tmp quotas and Claude temp
+
+Some VPS images mount `/tmp` with a per-user quota (`usrquota`). Claude Code
+accumulates temp under `/tmp/claude-{uid}`; if the quota fills up, every
+Claude bash command starts failing silently with exit 1 (see
+[troubleshooting](troubleshooting.md#every-claude-bash-command-fails-silently-exit-1-no-output)).
+QuadWork sweeps stale entries automatically (hourly + on agent teardown,
+72h age) — configurable via `temp_cleanup` in `~/.quadwork/config.json`.

docs/troubleshooting.md

@@ -226,3 +226,29 @@ existing network controls (127.0.0.1 bind + SSH tunnel, tailnet ACL, or the
 nginx Basic Auth from the VPS guide). The token + Origin allowlist are
 defence-in-depth against browser-based cross-origin attacks, not a substitute
 for network-level access control.
+
+## Every Claude bash command fails silently (exit 1, no output)
+
+**Symptom:** every bash command a `claude` agent runs — even `true` — returns
+exit 1 with empty stdout/stderr. The Read tool still works, a plain shell on
+the host works, and `codex` agents are unaffected. Easily mistaken for a
+bwrap/AppArmor/sandbox failure.
+
+**Cause:** Claude Code keeps its temp under `/tmp/claude-{uid}` and never
+cleans it up. On hosts where `/tmp` is mounted with a per-user quota
+(`usrquota`), that dir grows until the quota is exhausted — after which Claude
+can't write the temp files it needs before executing any command (#957).
+
+**Check:** `du -sh /tmp/claude-$(id -u)` and try `dd if=/dev/zero
+of=/tmp/probe bs=1M count=10` — a "Disk quota exceeded" error confirms it.
+
+**Fix:** QuadWork sweeps stale backend temp automatically (hourly, at boot,
+and on agent teardown; entries older than 72h). If you hit the quota *before*
+a sweep (e.g. the server was down), clear it manually:
+`find /tmp/claude-$(id -u)/* -maxdepth 0 -mmin +60 -exec rm -rf {} +`
+
+Tune or disable via `~/.quadwork/config.json`:
+
+```json
+{ "temp_cleanup": { "enabled": true, "max_age_hours": 72 } }
+```

server/index.js

@@ -12,6 +12,7 @@ const fileChat = require("./file-chat");
 const { dispatchToAgentPTY, cleanupSession: cleanupPtyDispatcher } = require("./pty-dispatcher");
 const { runAcMigration } = require("./migrate-ac");
 const selfHeal = require("./self-heal");
+const tempCleanup = require("./temp-cleanup"); // #957: stale backend-temp sweep
 const { injectModeForCommand } = require("../src/lib/injectMode.js");
 const telegramBridge = require("./bridges/telegram"); // #972: stop on shutdown
 const discordBridge = require("./bridges/discord");   // #972: stop on shutdown
@@ -789,6 +790,37 @@ async function stopAgentSession(key, { clearSelfHeal = false } = {}) {
   session.exitedUnexpectedly = false;
   const [projectId, agentId] = key.split("/");
   if (projectId && agentId) stopMcpProxy(projectId, agentId);
+  // #957: teardown is a known-safe moment to sweep stale backend temp. It's
+  // deferred (setImmediate) so it never blocks the stop path, and stale-only,
+  // so it never touches files a still-live agent on the shared /tmp/claude-{uid}
+  // is using. This only reads a session's OWN temp when that session is gone.
+  setImmediate(backendTempSweepTick);
+}
+
+// #957: sweep stale backend temp entries (/tmp/claude-{uid}, stray gemini
+// crash dumps). On hosts where /tmp has a per-user quota, unbounded Claude
+// temp eventually exhausts it and every Claude bash call fails silently with
+// exit 1 — see issue #957 for the full post-mortem. Runs on agent teardown and
+// on an hourly timer; opt-out / age via config.json
+// `temp_cleanup: { enabled, max_age_hours }`. The reentrancy guard stops a
+// teardown-triggered sweep from overlapping the periodic one.
+let _tempSweepRunning = false;
+function backendTempSweepTick() {
+  if (_tempSweepRunning) return;
+  _tempSweepRunning = true;
+  try {
+    const settings = tempCleanup.cleanupSettings(readConfig());
+    if (!settings.enabled) return;
+    const r = tempCleanup.sweepBackendTemp({ maxAgeHours: settings.maxAgeHours });
+    if (r.removed.length > 0) {
+      console.log(`[temp-cleanup] removed ${r.removed.length} stale entr${r.removed.length === 1 ? "y" : "ies"} (kept ${r.kept})`);
+    }
+    for (const e of r.errors) console.error(`[temp-cleanup] ${e}`);
+  } catch (err) {
+    console.error(`[temp-cleanup] sweep failed: ${err.message}`);
+  } finally {
+    _tempSweepRunning = false;
+  }
 }
 
 app.get("/api/agents", (_req, res) => {
@@ -2029,6 +2061,16 @@ if (!process.env.QUADWORK_SKIP_LISTEN) {
   _reseedRetryHandle = setInterval(reseedRetryTick, RESEED_RETRY_INTERVAL_MS);
 }
 
+// #957: hourly stale-temp sweep, plus one at boot — a server that was down for
+// days should reclaim quota immediately, not an hour after start. The handle is
+// captured so shutdown() (#972) clears the timer on Ctrl+C / full reset.
+const TEMP_SWEEP_INTERVAL_MS = 60 * 60 * 1000;
+let _tempSweepHandle = null;
+if (!process.env.QUADWORK_SKIP_LISTEN) {
+  setImmediate(backendTempSweepTick);
+  _tempSweepHandle = setInterval(backendTempSweepTick, TEMP_SWEEP_INTERVAL_MS);
+}
+
 // #422 / quadwork#310: auto-continue after loop guard.
 //
 // Per opted-in project, poll AC's /api/status every 10s. When we see
@@ -2272,6 +2314,7 @@ function shutdown() {
   // Polling + watchdog timers.
   if (_autoStopHandle) { clearInterval(_autoStopHandle); _autoStopHandle = null; }
   if (_reseedRetryHandle) { clearInterval(_reseedRetryHandle); _reseedRetryHandle = null; }
+  if (_tempSweepHandle) { clearInterval(_tempSweepHandle); _tempSweepHandle = null; } // #957
   if (_watchdogHandle) { clearInterval(_watchdogHandle); _watchdogHandle = null; }
 
   // Trigger schedulers (clears each trigger's interval + duration timers).

server/temp-cleanup.js

@@ -0,0 +1,130 @@
+"use strict";
+
+// #957: lifecycle-aware cleanup of agent backend temp dirs.
+//
+// Claude Code overrides TMPDIR to /tmp/claude-{uid} and never cleans it up.
+// On Linux hosts where /tmp carries a per-user quota (usrquota), that dir
+// accumulates until the quota is exhausted — at which point EVERY Claude bash
+// call fails silently (exit 1, no output) because Claude can't write its
+// temp/sandbox-setup files before exec. Gemini similarly strands
+// /tmp/gemini-client-error-*.json crash dumps. Codex keeps its state under
+// ~/.codex (not /tmp) so it needs no sweep here.
+//
+// Design constraints (the #957 "never delete a live session's temp" invariant):
+// - /tmp/claude-{uid} is SHARED by every claude agent of this user, so a
+//   teardown of one agent must never delete another live agent's files.
+//   Per-entry ownership isn't resolvable at the dir level, so deletion is
+//   strictly age-based ("stale entries only") — the same sweep is therefore
+//   safe from any call site, teardown OR timer.
+// - Age uses max(mtime, atime, ctime) like systemd-tmpfiles, so anything a
+//   live session still touches stays "fresh" and is spared.
+// - Never throws: a cleanup bug must not break agent teardown or the server
+//   boot path. Errors are collected and reported in the result.
+// - Cross-platform no-op where it doesn't apply: Windows has no
+//   process.getuid → the claude dir is skipped; the gemini glob simply
+//   matches nothing where those files don't exist.
+
+const fs = require("fs");
+const os = require("os");
+const path = require("path");
+
+const DEFAULT_MAX_AGE_HOURS = 72;
+
+// Newest of mtime/atime/ctime — "last time anything observed this entry".
+function newestTimeMs(st) {
+  return Math.max(st.mtimeMs || 0, st.atimeMs || 0, st.ctimeMs || 0);
+}
+
+// Remove `entry` (file or dir) if its newest timestamp is older than the
+// cutoff. Returns "removed" | "kept" | "error".
+function removeIfStale(entry, cutoffMs, result) {
+  let st;
+  try {
+    st = fs.lstatSync(entry);
+  } catch {
+    return "kept"; // raced away — already gone
+  }
+  if (newestTimeMs(st) >= cutoffMs) {
+    result.kept += 1;
+    return "kept";
+  }
+  try {
+    fs.rmSync(entry, { recursive: true, force: true });
+    result.removed.push(entry);
+    return "removed";
+  } catch (err) {
+    result.errors.push(`${entry}: ${err.message}`);
+    return "error";
+  }
+}
+
+// Sweep stale backend temp entries. Options (all injectable for tests):
+//   maxAgeHours - entries with no mtime/atime/ctime newer than this are removed
+//   tmpRoot     - temp root (default os.tmpdir())
+//   uid         - numeric uid for the claude-{uid} dir (default process.getuid)
+//   now         - epoch ms (default Date.now())
+// Returns { removed: string[], kept: number, errors: string[] }.
+function sweepBackendTemp(opts = {}) {
+  const result = { removed: [], kept: 0, errors: [] };
+  try {
+    const maxAgeHours = Number.isFinite(opts.maxAgeHours) && opts.maxAgeHours > 0
+      ? opts.maxAgeHours
+      : DEFAULT_MAX_AGE_HOURS;
+    const tmpRoot = opts.tmpRoot || os.tmpdir();
+    const now = typeof opts.now === "number" ? opts.now : Date.now();
+    const cutoffMs = now - maxAgeHours * 60 * 60 * 1000;
+
+    // Claude: entries directly under /tmp/claude-{uid}. The dir itself is
+    // kept (claude recreates it anyway; deleting it while a session spawns
+    // would be a race for no benefit).
+    const uid = typeof opts.uid === "number"
+      ? opts.uid
+      : (typeof process.getuid === "function" ? process.getuid() : null);
+    if (uid !== null) {
+      const claudeDir = path.join(tmpRoot, `claude-${uid}`);
+      let entries = [];
+      try {
+        entries = fs.readdirSync(claudeDir);
+      } catch {
+        entries = []; // no dir → nothing to do
+      }
+      for (const name of entries) {
+        removeIfStale(path.join(claudeDir, name), cutoffMs, result);
+      }
+    }
+
+    // Gemini: stray crash dumps written directly to the temp root.
+    let rootEntries = [];
+    try {
+      rootEntries = fs.readdirSync(tmpRoot);
+    } catch {
+      rootEntries = [];
+    }
+    for (const name of rootEntries) {
+      if (name.startsWith("gemini-client-error-") && name.endsWith(".json")) {
+        removeIfStale(path.join(tmpRoot, name), cutoffMs, result);
+      }
+    }
+  } catch (err) {
+    result.errors.push(String(err && err.message ? err.message : err));
+  }
+  return result;
+}
+
+// Resolve {enabled, maxAgeHours} from config.json's optional top-level
+// `temp_cleanup` block. Defaults: enabled, 72h. `enabled: false` opts out.
+function cleanupSettings(cfg) {
+  const tc = (cfg && cfg.temp_cleanup) || {};
+  return {
+    enabled: tc.enabled !== false,
+    maxAgeHours: Number.isFinite(tc.max_age_hours) && tc.max_age_hours > 0
+      ? tc.max_age_hours
+      : DEFAULT_MAX_AGE_HOURS,
+  };
+}
+
+module.exports = {
+  sweepBackendTemp,
+  cleanupSettings,
+  DEFAULT_MAX_AGE_HOURS,
+};

server/temp-cleanup.test.js

@@ -0,0 +1,103 @@
+"use strict";
+
+// #957: unit tests for the stale backend-temp sweep. All filesystem work
+// happens inside a throwaway fixture dir; uid/now/tmpRoot are injected so
+// nothing ever touches the real /tmp/claude-{uid}. Plain node:assert script —
+// auto-discovered by the #836 runner. Run with `node server/temp-cleanup.test.js`.
+
+const assert = require("node:assert/strict");
+const fs = require("fs");
+const os = require("os");
+const path = require("path");
+const { sweepBackendTemp, cleanupSettings, DEFAULT_MAX_AGE_HOURS } = require("./temp-cleanup");
+
+const HOUR = 60 * 60 * 1000;
+const NOW = 1_800_000_000_000; // fixed epoch for determinism (no Date.now())
+
+function makeFixture() {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), "qw-957-tempclean-"));
+  const claudeDir = path.join(root, "claude-1234");
+  fs.mkdirSync(claudeDir);
+  return { root, claudeDir };
+}
+
+// Create a file or dir and stamp its atime/mtime to `ageHours` before NOW.
+function touch(p, ageHours, { dir = false } = {}) {
+  if (dir) {
+    fs.mkdirSync(p, { recursive: true });
+    fs.writeFileSync(path.join(p, "inner.txt"), "x");
+  } else {
+    fs.writeFileSync(p, "x");
+  }
+  const t = new Date(NOW - ageHours * HOUR);
+  fs.utimesSync(p, t, t);
+}
+
+// ── stale vs a live session's fresh entries under claude-{uid} ────────────
+{
+  const { root, claudeDir } = makeFixture();
+  touch(path.join(claudeDir, "old-file"), 100);
+  touch(path.join(claudeDir, "old-dir"), 100, { dir: true });
+  // inner.txt written above bumps the dir mtime to real-now; restamp old.
+  fs.utimesSync(path.join(claudeDir, "old-dir"), new Date(NOW - 100 * HOUR), new Date(NOW - 100 * HOUR));
+  touch(path.join(claudeDir, "live-session-file"), 1); // a live agent just wrote this
+
+  const r = sweepBackendTemp({ tmpRoot: root, uid: 1234, now: NOW, maxAgeHours: 72 });
+
+  assert.ok(!fs.existsSync(path.join(claudeDir, "old-file")), "stale file under claude-{uid} removed");
+  assert.ok(!fs.existsSync(path.join(claudeDir, "old-dir")), "stale directory removed recursively");
+  assert.ok(fs.existsSync(path.join(claudeDir, "live-session-file")), "live session's fresh file PRESERVED");
+  assert.ok(fs.existsSync(claudeDir), "the claude-{uid} dir itself is kept");
+  assert.equal(r.removed.length, 2, "removed count = 2");
+  assert.equal(r.kept, 1, "kept count = 1 (the live file)");
+  assert.equal(r.errors.length, 0, "no errors");
+  fs.rmSync(root, { recursive: true, force: true });
+}
+
+// ── gemini crash dumps at the temp root ──────────────────────────────────
+{
+  const { root } = makeFixture();
+  const staleDump = path.join(root, "gemini-client-error-Turn.run-2026-06-03T00-28-55-388Z.json");
+  touch(staleDump, 100);
+  touch(path.join(root, "gemini-client-error-fresh.json"), 1);
+  touch(path.join(root, "unrelated-old-file.json"), 100);
+
+  const r = sweepBackendTemp({ tmpRoot: root, uid: 1234, now: NOW, maxAgeHours: 72 });
+
+  assert.ok(!fs.existsSync(staleDump), "stale gemini crash dump removed");
+  assert.ok(fs.existsSync(path.join(root, "gemini-client-error-fresh.json")), "fresh gemini dump spared");
+  assert.ok(fs.existsSync(path.join(root, "unrelated-old-file.json")), "non-gemini root file NEVER touched, even when old");
+  assert.equal(r.removed.length, 1, "only the stale gemini dump counts as removed");
+  fs.rmSync(root, { recursive: true, force: true });
+}
+
+// ── missing claude dir / null uid (Windows) → graceful no-op ─────────────
+{
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), "qw-957-tempclean-"));
+  const r1 = sweepBackendTemp({ tmpRoot: root, uid: 9999, now: NOW });
+  assert.equal(r1.removed.length, 0, "missing claude-{uid} dir → nothing removed");
+  assert.equal(r1.errors.length, 0, "missing claude-{uid} dir → no error");
+
+  const r2 = sweepBackendTemp({ tmpRoot: root, uid: null, now: NOW });
+  assert.equal(r2.removed.length, 0, "uid null → claude sweep skipped");
+  assert.equal(r2.errors.length, 0, "uid null → no error");
+  fs.rmSync(root, { recursive: true, force: true });
+}
+
+// ── never throws, even on a bogus tmpRoot ────────────────────────────────
+{
+  const r = sweepBackendTemp({ tmpRoot: "/nonexistent/qw-nope", uid: 1, now: NOW });
+  assert.ok(Array.isArray(r.removed) && r.removed.length === 0, "bogus tmpRoot → no-op, no throw");
+}
+
+// ── cleanupSettings: defaults + opt-out + custom/invalid age + null cfg ───
+{
+  assert.equal(cleanupSettings({}).enabled, true, "default: enabled");
+  assert.equal(cleanupSettings({}).maxAgeHours, DEFAULT_MAX_AGE_HOURS, `default: ${DEFAULT_MAX_AGE_HOURS}h`);
+  assert.equal(cleanupSettings({ temp_cleanup: { enabled: false } }).enabled, false, "enabled:false opts out");
+  assert.equal(cleanupSettings({ temp_cleanup: { max_age_hours: 24 } }).maxAgeHours, 24, "custom max_age_hours honored");
+  assert.equal(cleanupSettings({ temp_cleanup: { max_age_hours: -5 } }).maxAgeHours, DEFAULT_MAX_AGE_HOURS, "invalid age → default");
+  assert.equal(cleanupSettings(null).enabled, true, "null config → defaults, no throw");
+}
+
+console.log("temp-cleanup.test.js: all assertions passed");