feat: tmux-backed session persistence — terminals survive server restarts

CODEBASE_DOCUMENTATION.md

@@ -85,6 +85,12 @@ server/utils/processUtils.js       - Shared spawn/env hardening helpers
 └─ Cross-platform behavior: non-Windows platforms pass through unchanged so Linux/macOS launch behavior stays stable
 server/utils/nodePtyCompat.js      - Runtime compatibility shim for the bundled `node-pty` Windows ConPTY loader
 └─ Windows PTY guard: wraps stale ConPTY calls in memory (`startProcess`, `connect`, `resize`, `clear`, `kill`) via `loadNativeModule` when available or direct `conpty.node` patching when package internals differ, so packaged installs survive read-only app-resource layouts and mixed node-pty variants
+server/utils/tmuxSessionBackend.js - tmux-backed session persistence (terminals survive app-server restarts)
+├─ Model: the orchestrator's pty is only a tmux CLIENT; the real shell/agent runs in a pane under the tmux server on a dedicated per-instance socket (`agent-workspace-<port>`), so nodemon reloads/updates/crashes detach instead of killing sessions, and `new-session -A` re-adopts them on the next createSession()
+├─ Env hygiene: scrubs `CLAUDECODE`/`CLAUDE_CODE_ENTRYPOINT`/`TMUX` at the tmux-server choke point so nested-session guards never trip; socket options make panes behave like plain terminals (status off, prefix None, mouse off, window-size latest)
+├─ Lifecycle: explicit close/terminate/workspace-teardown kills the tmux session (never leaks detached panes); server shutdown detaches only; tree-kills + process limits target the PANE pid, not the client
+├─ Fallback: Windows/tmux-less installs fail closed to direct node-pty spawning (`ORCHESTRATOR_SESSION_PERSISTENCE=0` or `config.sessions.persistence.enabled=false` to disable); survives server restarts only — reboots still rely on transcript resume
+└─ Observability: `GET /api/sessions/persistence` reports managed vs orphaned tmux sessions; adopted sessions backfill their buffer from `capture-pane` so the log endpoint/scrollback preload shows pre-restart history
 server/utils/pathUtils.js          - Shared slash-normalization + data-directory compatibility helpers for repo/worktree labels
 └─ Legacy migration: renames `~/.orchestrator` when possible, otherwise merges richer legacy state into `~/.agent-workspace` with conflict backups before falling back to the old directory
 server/tokenCounter.js             - Token usage tracking (if applicable)

client/app.js

@@ -34,6 +34,15 @@ class ClaudeOrchestrator {
   constructor() {
     this.sessions = new Map();
     this.activeView = [];
+    // Per-worktree button config lookups log "using defaults" for every
+    // worktree without a custom .orchestrator-config.json — normal, and noisy
+    // with many worktrees. Opt in via localStorage 'debug-worktree-config'.
+    this.debugWorktreeConfig = false;
+    try {
+      this.debugWorktreeConfig = window?.localStorage?.getItem('debug-worktree-config') === 'true';
+    } catch {
+      // ignore
+    }
     this.visibleTerminals = new Set(); // Track which terminals are visible
     // Second-layer filter applied after per-worktree visibility toggles:
     // 'all' | 'claude' | 'server'
@@ -4348,15 +4357,15 @@ class ClaudeOrchestrator {
     }
 
     if (!repositoryType) {
-      console.log(`No repositoryType found for session ${sessionId}, using defaults`);
+      if (this.debugWorktreeConfig) console.debug(`No repositoryType found for session ${sessionId}, using defaults`);
       return this.getDefaultButtons(terminalType, sessionId);
     }
 
     // Get worktree-specific cascaded config (pre-fetched)
     const cascadedConfig = this.worktreeConfigs.get(sessionId);
-    console.log(`Looking up worktree config for ${sessionId} (type: ${repositoryType}):`, cascadedConfig);
+    if (this.debugWorktreeConfig) console.debug(`Looking up worktree config for ${sessionId} (type: ${repositoryType}):`, cascadedConfig);
     if (!cascadedConfig || !cascadedConfig.buttons) {
-      console.log(`No worktree config or buttons found for ${sessionId}, using defaults`);
+      if (this.debugWorktreeConfig) console.debug(`No worktree config or buttons found for ${sessionId}, using defaults`);
       return this.getDefaultButtons(terminalType, sessionId);
     }
 

client/terminal.js

@@ -846,11 +846,30 @@ class TerminalManager {
               return;
             }
 
-            this.warnFit(
+            // 0x0 means the element simply isn't measurable right now — an
+            // off-screen terminal in the scrollable grid, or one whose fonts/
+            // layout haven't settled. That's expected with many worktrees and
+            // resolves on the next show/resize, so keep it at debug level.
+            // A small-but-nonzero proposal is a real fit problem worth warning.
+            const unmeasurable = proposedCols === 0 && proposedRows === 0;
+            const logFit = unmeasurable ? this.debugFit.bind(this) : this.warnFit.bind(this);
+            logFit(
               sessionId,
               'proposed-still-too-small',
               `Terminal ${sessionId} proposed fit still too small after 5 retries (${proposedCols}x${proposedRows}; min ${minStableCols}x${minStableRows}); skipping fit`
             );
+            // Off-screen terminals still deserve a delayed retry so they fit
+            // correctly once scrolled into view without a resize event.
+            if (unmeasurable) {
+              if (!this.delayedFitTimers) this.delayedFitTimers = new Map();
+              if (!this.delayedFitTimers.has(sessionId)) {
+                const t = setTimeout(() => {
+                  this.delayedFitTimers.delete(sessionId);
+                  this.fitTerminal(sessionId, 0);
+                }, 1200);
+                this.delayedFitTimers.set(sessionId, t);
+              }
+            }
             this.fitTimers.delete(sessionId);
             return;
           }

config.json

@@ -12,7 +12,11 @@
     "claudeTimeoutMs": 0,
     "serverTimeoutMs": 43200000,
     "maxBufferSize": 1000000,
-    "maxProcessesPerSession": 50
+    "maxProcessesPerSession": 50,
+    "persistence": {
+      "enabled": true,
+      "socketName": ""
+    }
   },
   "logging": {
     "level": "info"

server/commanderService.js

@@ -537,7 +537,13 @@ class CommanderService {
       return false;
     }
 
-    // Use pty.write directly since sendInput may not exist
+    // Route through the single input choke point so this shares the same
+    // handling as browser keystrokes: device-report stripping under tmux,
+    // PowerShell CRLF normalization, and activity/status bookkeeping. Falls
+    // back to a direct write only if writeToSession is somehow unavailable.
+    if (typeof this.sessionManager.writeToSession === 'function') {
+      return this.sessionManager.writeToSession(sessionId, input);
+    }
     if (session.pty) {
       session.pty.write(input);
       return true;

server/index.js

@@ -2776,6 +2776,17 @@ app.get('/api/audit/export', requirePolicyAction('audit_export'), proOnly, async
   }
 });
 
+// Session-persistence observability: whether terminals run inside tmux (and
+// survive server restarts), which live tmux sessions are managed vs orphaned.
+app.get('/api/sessions/persistence', (req, res) => {
+  try {
+    return res.json({ ok: true, ...sessionManager.getPersistenceStatus() });
+  } catch (error) {
+    logger.error('Failed to resolve session persistence status', { error: error.message, stack: error.stack });
+    return res.status(500).json({ ok: false, error: 'Failed to resolve session persistence status' });
+  }
+});
+
 app.get('/api/sessions/:sessionId/log', (req, res) => {
   try {
     const sessionId = String(req.params.sessionId || '').trim();

server/sessionManager.js

@@ -17,6 +17,7 @@ const {
 } = require('./utils/shellCommand');
 const { augmentProcessEnv, buildPowerShellArgs } = require('./utils/processUtils');
 const { loadNodePty } = require('./utils/nodePtyCompat');
+const { TmuxSessionBackend, stripDeviceReports } = require('./utils/tmuxSessionBackend');
 
 const logger = winston.createLogger({
   level: process.env.LOG_LEVEL || 'info',
@@ -116,6 +117,28 @@ class SessionManager extends EventEmitter {
     this.conversationSnapshotTtlMs = parseInt(process.env.CONVERSATION_SNAPSHOT_TTL_MS || '5000');
     this.conversationSnapshotCache = { timestamp: 0, files: null };
 
+    // Session persistence: terminals live inside tmux sessions on a dedicated
+    // per-instance socket, so they survive app-server restarts and are
+    // re-adopted on the next createSession() for the same id (issue #1025).
+    // ORCHESTRATOR_SESSION_PERSISTENCE=0 disables; unavailable tmux (Windows,
+    // minimal installs) falls back to direct node-pty spawning automatically.
+    const persistenceConfig = this.config.sessions?.persistence || {};
+    const persistenceEnvOverride = String(process.env.ORCHESTRATOR_SESSION_PERSISTENCE || '').trim();
+    const persistenceWanted = persistenceEnvOverride
+      ? persistenceEnvOverride !== '0'
+      : persistenceConfig.enabled !== false;
+    const instancePort = process.env.ORCHESTRATOR_PORT || this.config.server?.port || 'default';
+    this.sessionPersistence = new TmuxSessionBackend({
+      socketName: persistenceConfig.socketName || `agent-workspace-${instancePort}`,
+      logger
+    });
+    this.sessionPersistenceEnabled = persistenceWanted && this.sessionPersistence.isAvailable();
+    logger.info('Session persistence', {
+      enabled: this.sessionPersistenceEnabled,
+      wanted: persistenceWanted,
+      socket: this.sessionPersistence.socketName
+    });
+
     // Worktrees will be built when workspace is set
     this.worktrees = [];
   }
@@ -824,17 +847,47 @@ class SessionManager extends EventEmitter {
 
       const effectiveEnv = augmentProcessEnv(env);
 
-      const ptyProcess = pty.spawn(
-        config.command,
-        config.args,
-        this.buildPtyOptions(config, effectiveEnv)
-      );
+      // Persistence path: spawn a tmux CLIENT instead of the shell directly.
+      // The pane (real shell/agent) lives under the tmux server and survives
+      // app-server restarts; `new-session -A` re-attaches to a surviving
+      // session, which is how sessions are adopted after a restart.
+      let spawnCommand = config.command;
+      let spawnArgs = config.args;
+      let persistence = null;
+      if (this.sessionPersistenceEnabled) {
+        // A leaked TMUX var would make the client refuse to start ("nested").
+        delete effectiveEnv.TMUX;
+        delete effectiveEnv.TMUX_PANE;
+        this.sessionPersistence.ensureConfigured();
+        const adopted = this.sessionPersistence.hasSession(sessionId);
+        const spec = this.sessionPersistence.buildSpawnCommand({
+          sessionId,
+          command: config.command,
+          args: config.args,
+          cwd: config.cwd
+        });
+        spawnCommand = spec.command;
+        spawnArgs = spec.args;
+        persistence = { backend: 'tmux', name: spec.name, adopted };
+        if (adopted) {
+          logger.info('Adopting surviving persistent session', { sessionId });
+        }
+      }
+
+      const ptyOptions = this.buildPtyOptions(config, effectiveEnv);
+      if (persistence) {
+        // The outer client terminal must advertise 256-color support or tmux
+        // degrades every pane's rendering.
+        ptyOptions.name = 'xterm-256color';
+      }
+      const ptyProcess = pty.spawn(spawnCommand, spawnArgs, ptyOptions);
 
       const initialCwd = config.cwd || process.cwd();
-      
+
       const session = {
         id: sessionId,
         pty: ptyProcess,
+        persistence,
         type: config.type,
         worktreeId: config.worktreeId,
         repositoryName: config.repositoryName,  // For mixed-repo workspaces
@@ -857,7 +910,21 @@ class SessionManager extends EventEmitter {
         autoStarted: false,  // Track if auto-start has been triggered
         claudeLaunchState: null
       };
-
+
+      // Adopted sessions re-attach mid-flight, so the fresh client only sees a
+      // screen redraw. Backfill the buffer from tmux scrollback so the log
+      // endpoint / client history preload can show what happened before the
+      // restart. Mark it delivered: history is served via the log endpoint,
+      // not re-streamed as live output.
+      if (persistence?.adopted) {
+        const history = this.sessionPersistence.capturePane(sessionId, 2000);
+        if (history) {
+          session.buffer = history.endsWith('\n') ? history : `${history}\n`;
+          session.deliveredBufferLength = session.buffer.length;
+        }
+      }
+
+
       // Set up inactivity timer (respect per-type timeout; 0 disables)
       const effectiveTimeout = this.getSessionTimeout(session);
       if (effectiveTimeout > 0) {
@@ -1778,8 +1845,13 @@ class SessionManager extends EventEmitter {
 
     try {
       let payload = data;
-      // PowerShell terminals need CRLF to reliably execute commands written programmatically.
       if (typeof payload === 'string') {
+        // Under tmux, drop echoed device-attribute reports so they can't land
+        // on the active pane as "1;2c0;276;0c" junk (see stripDeviceReports).
+        if (this.sessionPersistenceEnabled && session.persistence) {
+          payload = stripDeviceReports(payload);
+        }
+        // PowerShell terminals need CRLF to reliably execute commands written programmatically.
         const shellKind = this.getShellKindForSession(sessionId);
         if (shellKind === 'powershell') {
           payload = payload.replace(/\r?\n/g, '\r\n');
@@ -2472,10 +2544,68 @@ class SessionManager extends EventEmitter {
     return true;
   }
 
+  // Pid owning the session's REAL process tree. For persistent sessions the
+  // pane process is a child of the tmux server, not of the client pty the
+  // orchestrator holds — tree-kills and child counting must target the pane.
+  getSessionProcessPid(session) {
+    if (session?.persistence?.backend === 'tmux' && this.sessionPersistenceEnabled) {
+      const panePid = this.sessionPersistence.panePid(session.id);
+      if (panePid) return panePid;
+    }
+    const pid = Number(session?.pty?.pid);
+    return Number.isFinite(pid) && pid > 0 ? pid : null;
+  }
+
+  // Explicit destruction of a persistent session. Killing only the client pty
+  // would DETACH the pane, leaking it as an orphaned tmux session — surviving
+  // is only for server restarts, never for user-initiated close/terminate.
+  destroyPersistentSession(session) {
+    if (session?.persistence?.backend !== 'tmux') return;
+    try {
+      this.sessionPersistence.killSession(session.id);
+    } catch (error) {
+      logger.warn('Failed to kill persistent tmux session', { sessionId: session.id, error: error.message });
+    }
+  }
+
+  // Observability for the persistence layer: what tmux knows vs what the
+  // orchestrator manages. "Orphaned" sessions survived a restart but were not
+  // re-adopted (e.g. their worktree was removed from the workspace meanwhile);
+  // they can be inspected via `tmux -L <socket> attach -t <name>` or killed.
+  getPersistenceStatus() {
+    const status = {
+      enabled: !!this.sessionPersistenceEnabled,
+      backend: 'tmux',
+      socketName: this.sessionPersistence?.socketName || null,
+      managed: [],
+      orphaned: []
+    };
+    if (!status.enabled) return status;
+
+    const known = new Map();
+    const collect = (map) => {
+      for (const session of map.values()) {
+        if (session?.persistence?.name) known.set(session.persistence.name, session.id);
+      }
+    };
+    collect(this.sessions);
+    for (const map of this.workspaceSessionMaps.values()) collect(map);
+
+    const live = this.sessionPersistence.listSessionNames();
+    for (const name of live) {
+      if (known.has(name)) {
+        status.managed.push({ name, sessionId: known.get(name) });
+      } else {
+        status.orphaned.push({ name });
+      }
+    }
+    return status;
+  }
+
   checkProcessLimit(session) {
     if (!session.pty || !session.pty.pid) return;
 
-    const pid = Number(session.pty.pid);
+    const pid = this.getSessionProcessPid(session);
     if (!Number.isFinite(pid) || pid <= 0) return;
 
     const { spawn } = require('child_process');
@@ -2600,7 +2730,11 @@ class SessionManager extends EventEmitter {
       session.pendingStatusTimer = null;
     }
 
-    const ptyPid = Number(session?.pty?.pid);
+    // Resolve the real process-tree pid BEFORE tearing anything down: for
+    // persistent sessions it comes from a tmux query that fails once the
+    // session is killed.
+    const processPid = this.getSessionProcessPid(session);
+    this.destroyPersistentSession(session);
 
     // Kill the PTY process if it exists
     if (session.pty) {
@@ -2616,7 +2750,7 @@ class SessionManager extends EventEmitter {
 
     // Best-effort process tree cleanup to avoid orphaned agent subprocesses
     // after terminals are closed/removed.
-    this.bestEffortKillProcessTree(ptyPid, { sessionId: sid });
+    this.bestEffortKillProcessTree(processPid, { sessionId: sid });
 
     // Remove from sessions map
     sessionMap.delete(sid);
@@ -3209,6 +3343,9 @@ class SessionManager extends EventEmitter {
     // Kill all PTY processes
     for (const [sessionId, session] of this.sessions) {
       try {
+        // Destructive path (workspace teardown): persistent panes must die
+        // with their sessions, not linger detached on the tmux socket.
+        this.destroyPersistentSession(session);
         if (session.pty) {
           session.pty.kill();
           logger.debug(`Killed session: ${sessionId}`);