diff --git a/docs/install-vps.md b/docs/install-vps.md index 41080fd..9ed6157 100644 --- a/docs/install-vps.md +++ b/docs/install-vps.md @@ -362,3 +362,12 @@ chmod 600 ~/.quadwork/.env 14. Configure nginx reverse proxy + SSL 15. Add HTTP basic auth 16. Verify reboot survival: `sudo reboot`, then check `pm2 list` + +## Note: /tmp quotas and Claude temp + +Some VPS images mount `/tmp` with a per-user quota (`usrquota`). Claude Code +accumulates temp under `/tmp/claude-{uid}`; if the quota fills up, every +Claude bash command starts failing silently with exit 1 (see +[troubleshooting](troubleshooting.md#every-claude-bash-command-fails-silently-exit-1-no-output)). +QuadWork sweeps stale entries automatically (hourly + on agent teardown, +72h age) — configurable via `temp_cleanup` in `~/.quadwork/config.json`. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 5ec355b..ba15d5f 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -226,3 +226,29 @@ existing network controls (127.0.0.1 bind + SSH tunnel, tailnet ACL, or the nginx Basic Auth from the VPS guide). The token + Origin allowlist are defence-in-depth against browser-based cross-origin attacks, not a substitute for network-level access control. + +## Every Claude bash command fails silently (exit 1, no output) + +**Symptom:** every bash command a `claude` agent runs — even `true` — returns +exit 1 with empty stdout/stderr. The Read tool still works, a plain shell on +the host works, and `codex` agents are unaffected. Easily mistaken for a +bwrap/AppArmor/sandbox failure. + +**Cause:** Claude Code keeps its temp under `/tmp/claude-{uid}` and never +cleans it up. On hosts where `/tmp` is mounted with a per-user quota +(`usrquota`), that dir grows until the quota is exhausted — after which Claude +can't write the temp files it needs before executing any command (#957). + +**Check:** `du -sh /tmp/claude-$(id -u)` and try `dd if=/dev/zero +of=/tmp/probe bs=1M count=10` — a "Disk quota exceeded" error confirms it. + +**Fix:** QuadWork sweeps stale backend temp automatically (hourly, at boot, +and on agent teardown; entries older than 72h). If you hit the quota *before* +a sweep (e.g. the server was down), clear it manually: +`find /tmp/claude-$(id -u)/* -maxdepth 0 -mmin +60 -exec rm -rf {} +` + +Tune or disable via `~/.quadwork/config.json`: + +```json +{ "temp_cleanup": { "enabled": true, "max_age_hours": 72 } } +``` diff --git a/server/index.js b/server/index.js index 58ae4de..8022b70 100644 --- a/server/index.js +++ b/server/index.js @@ -12,6 +12,7 @@ const fileChat = require("./file-chat"); const { dispatchToAgentPTY, cleanupSession: cleanupPtyDispatcher } = require("./pty-dispatcher"); const { runAcMigration } = require("./migrate-ac"); const selfHeal = require("./self-heal"); +const tempCleanup = require("./temp-cleanup"); // #957: stale backend-temp sweep const { injectModeForCommand } = require("../src/lib/injectMode.js"); const telegramBridge = require("./bridges/telegram"); // #972: stop on shutdown const discordBridge = require("./bridges/discord"); // #972: stop on shutdown @@ -789,6 +790,37 @@ async function stopAgentSession(key, { clearSelfHeal = false } = {}) { session.exitedUnexpectedly = false; const [projectId, agentId] = key.split("/"); if (projectId && agentId) stopMcpProxy(projectId, agentId); + // #957: teardown is a known-safe moment to sweep stale backend temp. It's + // deferred (setImmediate) so it never blocks the stop path, and stale-only, + // so it never touches files a still-live agent on the shared /tmp/claude-{uid} + // is using. This only reads a session's OWN temp when that session is gone. + setImmediate(backendTempSweepTick); +} + +// #957: sweep stale backend temp entries (/tmp/claude-{uid}, stray gemini +// crash dumps). On hosts where /tmp has a per-user quota, unbounded Claude +// temp eventually exhausts it and every Claude bash call fails silently with +// exit 1 — see issue #957 for the full post-mortem. Runs on agent teardown and +// on an hourly timer; opt-out / age via config.json +// `temp_cleanup: { enabled, max_age_hours }`. The reentrancy guard stops a +// teardown-triggered sweep from overlapping the periodic one. +let _tempSweepRunning = false; +function backendTempSweepTick() { + if (_tempSweepRunning) return; + _tempSweepRunning = true; + try { + const settings = tempCleanup.cleanupSettings(readConfig()); + if (!settings.enabled) return; + const r = tempCleanup.sweepBackendTemp({ maxAgeHours: settings.maxAgeHours }); + if (r.removed.length > 0) { + console.log(`[temp-cleanup] removed ${r.removed.length} stale entr${r.removed.length === 1 ? "y" : "ies"} (kept ${r.kept})`); + } + for (const e of r.errors) console.error(`[temp-cleanup] ${e}`); + } catch (err) { + console.error(`[temp-cleanup] sweep failed: ${err.message}`); + } finally { + _tempSweepRunning = false; + } } app.get("/api/agents", (_req, res) => { @@ -2029,6 +2061,16 @@ if (!process.env.QUADWORK_SKIP_LISTEN) { _reseedRetryHandle = setInterval(reseedRetryTick, RESEED_RETRY_INTERVAL_MS); } +// #957: hourly stale-temp sweep, plus one at boot — a server that was down for +// days should reclaim quota immediately, not an hour after start. The handle is +// captured so shutdown() (#972) clears the timer on Ctrl+C / full reset. +const TEMP_SWEEP_INTERVAL_MS = 60 * 60 * 1000; +let _tempSweepHandle = null; +if (!process.env.QUADWORK_SKIP_LISTEN) { + setImmediate(backendTempSweepTick); + _tempSweepHandle = setInterval(backendTempSweepTick, TEMP_SWEEP_INTERVAL_MS); +} + // #422 / quadwork#310: auto-continue after loop guard. // // Per opted-in project, poll AC's /api/status every 10s. When we see @@ -2272,6 +2314,7 @@ function shutdown() { // Polling + watchdog timers. if (_autoStopHandle) { clearInterval(_autoStopHandle); _autoStopHandle = null; } if (_reseedRetryHandle) { clearInterval(_reseedRetryHandle); _reseedRetryHandle = null; } + if (_tempSweepHandle) { clearInterval(_tempSweepHandle); _tempSweepHandle = null; } // #957 if (_watchdogHandle) { clearInterval(_watchdogHandle); _watchdogHandle = null; } // Trigger schedulers (clears each trigger's interval + duration timers). diff --git a/server/temp-cleanup.js b/server/temp-cleanup.js new file mode 100644 index 0000000..c432ff9 --- /dev/null +++ b/server/temp-cleanup.js @@ -0,0 +1,130 @@ +"use strict"; + +// #957: lifecycle-aware cleanup of agent backend temp dirs. +// +// Claude Code overrides TMPDIR to /tmp/claude-{uid} and never cleans it up. +// On Linux hosts where /tmp carries a per-user quota (usrquota), that dir +// accumulates until the quota is exhausted — at which point EVERY Claude bash +// call fails silently (exit 1, no output) because Claude can't write its +// temp/sandbox-setup files before exec. Gemini similarly strands +// /tmp/gemini-client-error-*.json crash dumps. Codex keeps its state under +// ~/.codex (not /tmp) so it needs no sweep here. +// +// Design constraints (the #957 "never delete a live session's temp" invariant): +// - /tmp/claude-{uid} is SHARED by every claude agent of this user, so a +// teardown of one agent must never delete another live agent's files. +// Per-entry ownership isn't resolvable at the dir level, so deletion is +// strictly age-based ("stale entries only") — the same sweep is therefore +// safe from any call site, teardown OR timer. +// - Age uses max(mtime, atime, ctime) like systemd-tmpfiles, so anything a +// live session still touches stays "fresh" and is spared. +// - Never throws: a cleanup bug must not break agent teardown or the server +// boot path. Errors are collected and reported in the result. +// - Cross-platform no-op where it doesn't apply: Windows has no +// process.getuid → the claude dir is skipped; the gemini glob simply +// matches nothing where those files don't exist. + +const fs = require("fs"); +const os = require("os"); +const path = require("path"); + +const DEFAULT_MAX_AGE_HOURS = 72; + +// Newest of mtime/atime/ctime — "last time anything observed this entry". +function newestTimeMs(st) { + return Math.max(st.mtimeMs || 0, st.atimeMs || 0, st.ctimeMs || 0); +} + +// Remove `entry` (file or dir) if its newest timestamp is older than the +// cutoff. Returns "removed" | "kept" | "error". +function removeIfStale(entry, cutoffMs, result) { + let st; + try { + st = fs.lstatSync(entry); + } catch { + return "kept"; // raced away — already gone + } + if (newestTimeMs(st) >= cutoffMs) { + result.kept += 1; + return "kept"; + } + try { + fs.rmSync(entry, { recursive: true, force: true }); + result.removed.push(entry); + return "removed"; + } catch (err) { + result.errors.push(`${entry}: ${err.message}`); + return "error"; + } +} + +// Sweep stale backend temp entries. Options (all injectable for tests): +// maxAgeHours - entries with no mtime/atime/ctime newer than this are removed +// tmpRoot - temp root (default os.tmpdir()) +// uid - numeric uid for the claude-{uid} dir (default process.getuid) +// now - epoch ms (default Date.now()) +// Returns { removed: string[], kept: number, errors: string[] }. +function sweepBackendTemp(opts = {}) { + const result = { removed: [], kept: 0, errors: [] }; + try { + const maxAgeHours = Number.isFinite(opts.maxAgeHours) && opts.maxAgeHours > 0 + ? opts.maxAgeHours + : DEFAULT_MAX_AGE_HOURS; + const tmpRoot = opts.tmpRoot || os.tmpdir(); + const now = typeof opts.now === "number" ? opts.now : Date.now(); + const cutoffMs = now - maxAgeHours * 60 * 60 * 1000; + + // Claude: entries directly under /tmp/claude-{uid}. The dir itself is + // kept (claude recreates it anyway; deleting it while a session spawns + // would be a race for no benefit). + const uid = typeof opts.uid === "number" + ? opts.uid + : (typeof process.getuid === "function" ? process.getuid() : null); + if (uid !== null) { + const claudeDir = path.join(tmpRoot, `claude-${uid}`); + let entries = []; + try { + entries = fs.readdirSync(claudeDir); + } catch { + entries = []; // no dir → nothing to do + } + for (const name of entries) { + removeIfStale(path.join(claudeDir, name), cutoffMs, result); + } + } + + // Gemini: stray crash dumps written directly to the temp root. + let rootEntries = []; + try { + rootEntries = fs.readdirSync(tmpRoot); + } catch { + rootEntries = []; + } + for (const name of rootEntries) { + if (name.startsWith("gemini-client-error-") && name.endsWith(".json")) { + removeIfStale(path.join(tmpRoot, name), cutoffMs, result); + } + } + } catch (err) { + result.errors.push(String(err && err.message ? err.message : err)); + } + return result; +} + +// Resolve {enabled, maxAgeHours} from config.json's optional top-level +// `temp_cleanup` block. Defaults: enabled, 72h. `enabled: false` opts out. +function cleanupSettings(cfg) { + const tc = (cfg && cfg.temp_cleanup) || {}; + return { + enabled: tc.enabled !== false, + maxAgeHours: Number.isFinite(tc.max_age_hours) && tc.max_age_hours > 0 + ? tc.max_age_hours + : DEFAULT_MAX_AGE_HOURS, + }; +} + +module.exports = { + sweepBackendTemp, + cleanupSettings, + DEFAULT_MAX_AGE_HOURS, +}; diff --git a/server/temp-cleanup.test.js b/server/temp-cleanup.test.js new file mode 100644 index 0000000..07deed9 --- /dev/null +++ b/server/temp-cleanup.test.js @@ -0,0 +1,103 @@ +"use strict"; + +// #957: unit tests for the stale backend-temp sweep. All filesystem work +// happens inside a throwaway fixture dir; uid/now/tmpRoot are injected so +// nothing ever touches the real /tmp/claude-{uid}. Plain node:assert script — +// auto-discovered by the #836 runner. Run with `node server/temp-cleanup.test.js`. + +const assert = require("node:assert/strict"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); +const { sweepBackendTemp, cleanupSettings, DEFAULT_MAX_AGE_HOURS } = require("./temp-cleanup"); + +const HOUR = 60 * 60 * 1000; +const NOW = 1_800_000_000_000; // fixed epoch for determinism (no Date.now()) + +function makeFixture() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "qw-957-tempclean-")); + const claudeDir = path.join(root, "claude-1234"); + fs.mkdirSync(claudeDir); + return { root, claudeDir }; +} + +// Create a file or dir and stamp its atime/mtime to `ageHours` before NOW. +function touch(p, ageHours, { dir = false } = {}) { + if (dir) { + fs.mkdirSync(p, { recursive: true }); + fs.writeFileSync(path.join(p, "inner.txt"), "x"); + } else { + fs.writeFileSync(p, "x"); + } + const t = new Date(NOW - ageHours * HOUR); + fs.utimesSync(p, t, t); +} + +// ── stale vs a live session's fresh entries under claude-{uid} ──────────── +{ + const { root, claudeDir } = makeFixture(); + touch(path.join(claudeDir, "old-file"), 100); + touch(path.join(claudeDir, "old-dir"), 100, { dir: true }); + // inner.txt written above bumps the dir mtime to real-now; restamp old. + fs.utimesSync(path.join(claudeDir, "old-dir"), new Date(NOW - 100 * HOUR), new Date(NOW - 100 * HOUR)); + touch(path.join(claudeDir, "live-session-file"), 1); // a live agent just wrote this + + const r = sweepBackendTemp({ tmpRoot: root, uid: 1234, now: NOW, maxAgeHours: 72 }); + + assert.ok(!fs.existsSync(path.join(claudeDir, "old-file")), "stale file under claude-{uid} removed"); + assert.ok(!fs.existsSync(path.join(claudeDir, "old-dir")), "stale directory removed recursively"); + assert.ok(fs.existsSync(path.join(claudeDir, "live-session-file")), "live session's fresh file PRESERVED"); + assert.ok(fs.existsSync(claudeDir), "the claude-{uid} dir itself is kept"); + assert.equal(r.removed.length, 2, "removed count = 2"); + assert.equal(r.kept, 1, "kept count = 1 (the live file)"); + assert.equal(r.errors.length, 0, "no errors"); + fs.rmSync(root, { recursive: true, force: true }); +} + +// ── gemini crash dumps at the temp root ────────────────────────────────── +{ + const { root } = makeFixture(); + const staleDump = path.join(root, "gemini-client-error-Turn.run-2026-06-03T00-28-55-388Z.json"); + touch(staleDump, 100); + touch(path.join(root, "gemini-client-error-fresh.json"), 1); + touch(path.join(root, "unrelated-old-file.json"), 100); + + const r = sweepBackendTemp({ tmpRoot: root, uid: 1234, now: NOW, maxAgeHours: 72 }); + + assert.ok(!fs.existsSync(staleDump), "stale gemini crash dump removed"); + assert.ok(fs.existsSync(path.join(root, "gemini-client-error-fresh.json")), "fresh gemini dump spared"); + assert.ok(fs.existsSync(path.join(root, "unrelated-old-file.json")), "non-gemini root file NEVER touched, even when old"); + assert.equal(r.removed.length, 1, "only the stale gemini dump counts as removed"); + fs.rmSync(root, { recursive: true, force: true }); +} + +// ── missing claude dir / null uid (Windows) → graceful no-op ───────────── +{ + const root = fs.mkdtempSync(path.join(os.tmpdir(), "qw-957-tempclean-")); + const r1 = sweepBackendTemp({ tmpRoot: root, uid: 9999, now: NOW }); + assert.equal(r1.removed.length, 0, "missing claude-{uid} dir → nothing removed"); + assert.equal(r1.errors.length, 0, "missing claude-{uid} dir → no error"); + + const r2 = sweepBackendTemp({ tmpRoot: root, uid: null, now: NOW }); + assert.equal(r2.removed.length, 0, "uid null → claude sweep skipped"); + assert.equal(r2.errors.length, 0, "uid null → no error"); + fs.rmSync(root, { recursive: true, force: true }); +} + +// ── never throws, even on a bogus tmpRoot ──────────────────────────────── +{ + const r = sweepBackendTemp({ tmpRoot: "/nonexistent/qw-nope", uid: 1, now: NOW }); + assert.ok(Array.isArray(r.removed) && r.removed.length === 0, "bogus tmpRoot → no-op, no throw"); +} + +// ── cleanupSettings: defaults + opt-out + custom/invalid age + null cfg ─── +{ + assert.equal(cleanupSettings({}).enabled, true, "default: enabled"); + assert.equal(cleanupSettings({}).maxAgeHours, DEFAULT_MAX_AGE_HOURS, `default: ${DEFAULT_MAX_AGE_HOURS}h`); + assert.equal(cleanupSettings({ temp_cleanup: { enabled: false } }).enabled, false, "enabled:false opts out"); + assert.equal(cleanupSettings({ temp_cleanup: { max_age_hours: 24 } }).maxAgeHours, 24, "custom max_age_hours honored"); + assert.equal(cleanupSettings({ temp_cleanup: { max_age_hours: -5 } }).maxAgeHours, DEFAULT_MAX_AGE_HOURS, "invalid age → default"); + assert.equal(cleanupSettings(null).enabled, true, "null config → defaults, no throw"); +} + +console.log("temp-cleanup.test.js: all assertions passed");