From ebb693f2cb35e701e00c68f5a5adce8484ddd6ed Mon Sep 17 00:00:00 2001 From: Amr Elsayed Date: Thu, 17 Sep 2026 09:38:46 +1000 Subject: [PATCH 1/8] chore(porch): bugfix-1686 init bugfix --- .../status.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 codev/projects/bugfix-1686-tower-reconcile-failed-reconne/status.yaml diff --git a/codev/projects/bugfix-1686-tower-reconcile-failed-reconne/status.yaml b/codev/projects/bugfix-1686-tower-reconcile-failed-reconne/status.yaml new file mode 100644 index 000000000..f5f5f0e34 --- /dev/null +++ b/codev/projects/bugfix-1686-tower-reconcile-failed-reconne/status.yaml @@ -0,0 +1,14 @@ +id: bugfix-1686 +title: tower-reconcile-failed-reconne +protocol: bugfix +phase: investigate +plan_phases: [] +current_plan_phase: null +gates: + pr: + status: pending +iteration: 1 +build_complete: false +history: [] +started_at: '2026-09-16T23:38:45.913Z' +updated_at: '2026-09-16T23:38:45.914Z' From f8a85a32739450d5e09c13918cb6bfcf8b76036d Mon Sep 17 00:00:00 2001 From: Amr Elsayed Date: Thu, 17 Sep 2026 09:43:28 +1000 Subject: [PATCH 2/8] chore(porch): bugfix-1686 fix phase-transition --- .../bugfix-1686-tower-reconcile-failed-reconne/status.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codev/projects/bugfix-1686-tower-reconcile-failed-reconne/status.yaml b/codev/projects/bugfix-1686-tower-reconcile-failed-reconne/status.yaml index f5f5f0e34..3cc2001e9 100644 --- a/codev/projects/bugfix-1686-tower-reconcile-failed-reconne/status.yaml +++ b/codev/projects/bugfix-1686-tower-reconcile-failed-reconne/status.yaml @@ -1,7 +1,7 @@ id: bugfix-1686 title: tower-reconcile-failed-reconne protocol: bugfix -phase: investigate +phase: fix plan_phases: [] current_plan_phase: null gates: @@ -11,4 +11,4 @@ iteration: 1 build_complete: false history: [] started_at: '2026-09-16T23:38:45.913Z' -updated_at: '2026-09-16T23:38:45.914Z' +updated_at: '2026-09-16T23:43:28.321Z' From 8707ea0088307623a59e7e7210e02a4ae9f236e2 Mon Sep 17 00:00:00 2001 From: Amr Elsayed Date: Thu, 17 Sep 2026 09:55:51 +1000 Subject: [PATCH 3/8] Fix #1686: reconcile must confirm death before deleting/SIGTERMing a shellper row MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tower reconcile treated any failed shellper reconnect as death. reconnectSession() returns null not only for a dead process but also when client.connect() is refused because another Tower client already owns the shellper (one-client-per-shellper) or on a transient boot-time socket/fd hiccup. Phase 2 then SIGTERMed the still-live pid and deleted the row — destroying 53 live sessions in the #1629 incident. Require positive evidence of death before touching the process or row: the shellper pid must be gone AND/OR its socket file absent. A live pid with a present socket is "could not confirm" — the row is left in place (WARN) for a later pass, never signaled. Applied at both delete sites that read reconnect failure as death: - reconcile Phase 2 sweep (the SIGTERM + DELETE path from the incident), and - getTerminalsForWorkspace() on-the-fly reconnect (DELETE-only), which otherwise would delete the very row Phase 2 preserved on the first /api/state read. Adds an `unconfirmed` counter so a preserve-only reconcile no longer logs "No terminal sessions to reconcile". Regression tests cover both sites: a live-pid + present-socket row whose reconnect fails survives untouched and unsignaled (verified to fail without each guard). --- codev/state/bugfix-1686_thread.md | 95 ++++++++++++++ .../__tests__/tower-terminals.test.ts | 123 +++++++++++++++++- .../src/agent-farm/servers/tower-terminals.ts | 55 +++++++- 3 files changed, 268 insertions(+), 5 deletions(-) create mode 100644 codev/state/bugfix-1686_thread.md diff --git a/codev/state/bugfix-1686_thread.md b/codev/state/bugfix-1686_thread.md new file mode 100644 index 000000000..e6f1b395a --- /dev/null +++ b/codev/state/bugfix-1686_thread.md @@ -0,0 +1,95 @@ +# Builder thread — bugfix-1686 + +## Issue +tower reconcile Phase 2 deletes a session row + SIGTERMs the pid for any shellper whose +reconnect merely FAILED, without confirming the process is dead. In the #1629 incident this +destroyed 53/56 live rows. Split from #1629; the #1629 owner-lock guard blocks the +second-Tower amplifier, this issue removes the destructive mechanism. + +## INVESTIGATE (phase 1) — root cause found, no code written + +Files: +- `packages/codev/src/agent-farm/servers/tower-terminals.ts` + - Phase 1 reconnect loop: lines ~800-805 (`if (!client) … "is stale (PID/socket dead)"`). + - Phase 2 sweep: lines ~919-937 (`processExists(session.pid)` → SIGTERM + DELETE). +- `packages/codev/src/terminal/session-manager.ts` `reconnectSession()` lines 535-577. + +Root cause: `reconnectSession()` returns `null` for FIVE reasons: +1. process dead (545-548) +2. PID reused / start-time mismatch (551-556) +3. socket not a socket file (559-564) +4. socket missing / lstat error (565-568) +5. **`client.connect()` throws (570-577)** ← the incident case: process ALIVE, socket file + PRESENT, connect refused because the real Tower already owns the socket + (one-client-per-shellper). Also fires on any transient boot hiccup (#1685 fd pressure). + +Phase 1 treats ALL five `null`s identically as "stale (PID/socket dead)" and lets the row fall +to Phase 2. Phase 2 then, for any unmatched row, kills `session.pid` when it is alive and +deletes the row — so it signals exactly the LIVE processes it should preserve. + +## Fix shape (for IMPLEMENT phase) +Phase 2 must require POSITIVE evidence of death before delete/SIGTERM of a shellper-backed row: +- pidAlive = shellper_pid present && processExists(shellper_pid) +- socketPresent = fs.existsSync(shellper_socket) +- If pidAlive && socketPresent → could-not-confirm-death → leave row untouched, log WARN, + never signal, `continue` (retried next reconcile/adoption pass). +- Otherwise → genuine death evidence → existing cleanup (kill orphan if any + DELETE). +Non-shellper rows (shellper_socket === null): unchanged. + +Regression test (tower-terminals.test.ts): shellper row, reconnect returns null, shellper_pid +alive (use process.pid), socket existsSync=true → assert row NOT deleted AND +process.kill(pid,'SIGTERM') never called. Fails without fix, passes with it. + +## Scope +~15-25 LOC in one file + one test. Single file, clear root cause. Fits BUGFIX (<300 LOC). +3.3.4 rider per architect; anything adjacent gets FILED not fixed. + +## FIX (phase 2) — implemented + CMAP + architect scope ruling + +Phase 2 guard (reconcile sweep, tower-terminals.ts ~926-943): shellper-backed row that failed +reconnect with live pid + present socket → WARN + continue (kept), never SIGTERM/DELETE. + +CMAP (3-way, HIGH all): core fix APPROVED. But all three flagged a SIBLING delete site — +getTerminalsForWorkspace() on-the-fly reconnect (~1132) did `deleteTerminalSession` on any null +reconnect with NO liveness check, so a row Phase 2 preserved would die on the first /api/state +read. codex REQUEST_CHANGES, claude COMMENT (fix-or-file), gemini non-blocking. + +Architect ruling: FOLD IT IN. "Same defect behind a second door = same fix = in scope; the +no-growth rule guards DIFFERENT defects, not the same one's siblings." Requirements met: +1. Identical shellperAlive+socketPresent guard at the on-the-fly site (~1137-1155). +2. Second regression test: row preserved by reconcile survives a getTerminalsForWorkspace pass + with still-failing reconnect (undeleted, unsignaled). Verified non-vacuous (disabled guard → + DELETE fires). +3. Polish: `unconfirmed` counter in reconcile summary (so preserve-only run doesn't log "No + terminal sessions to reconcile"); onTestFinished(restoreAllMocks) hardening on both tests + (mocked process.kill must not leak). + +Boundary held: further delete sites → FILE; shellper_pid===null legacy edge → PR-body note. +Both tests green (63/63 in file). + +Re-CMAP (r2) after fold: codex APPROVE HIGH (its r1 REQUEST_CHANGES answered), claude APPROVE +HIGH (nits only), gemini skipped (agy flaky, non-blocking). One extra coherence fix taken from +r2/r1 feedback (flagged by 2/3 reviewers): Phase 1 null-reconnect log no longer asserts "is +stale (PID/socket dead)" — reworded to "reconnect failed — deferring to Phase 2 sweep for death +confirmation", since that log now precedes rows Phase 2 PRESERVES (same reconcile path my change +touches; a log my own change rendered false). All other nits → PR-body residuals (see below). + +PR-body residuals to document (claude r2, all accepted-as-is, not changed): +- PID-reuse: processExists(shellper_pid) can't detect a recycled pid; conservative by design + (tightening via getProcessStartTime would re-open the false-death path). Known residual. +- Asymmetric evidence: pid-alive + socket-ABSENT still SIGTERMs (socketless shellper is + unreachable). Chosen reading of the issue's "AND/OR". +- Unbounded retention: a permanently-unreconnectable-but-alive shellper keeps its row until + stop (deleteWorkspaceTerminalSessions). Accepted (leak a row, not a session). +- shellper_pid===null legacy edge → falls through to SIGTERM of session.pid; ~unreachable given + saveTerminalSession call sites. + +Commit: 35a3bd92b (Fix #1686 ...). + +PR body must reference: the 5 pre-existing env-class failures (consolidate.test.ts + +spawn-retirement.test.ts — getRolesDir "Roles directory not found" in worktree test env, +confirmed identical on clean base), and the shellper_pid===null legacy edge. + +## Coordination +Based on origin/main tip (30e264031) which includes #1687 + #1689 merges (same file +territory: reconcile boot order claim->reconnect->sweep). Building on current main. diff --git a/packages/codev/src/agent-farm/__tests__/tower-terminals.test.ts b/packages/codev/src/agent-farm/__tests__/tower-terminals.test.ts index 70df3b47f..21b69eebd 100644 --- a/packages/codev/src/agent-farm/__tests__/tower-terminals.test.ts +++ b/packages/codev/src/agent-farm/__tests__/tower-terminals.test.ts @@ -6,7 +6,7 @@ * getTerminalsForWorkspace, and initTerminals/shutdownTerminals lifecycle. */ -import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { describe, it, expect, beforeEach, afterEach, onTestFinished, vi } from 'vitest'; import path from 'node:path'; import fs from 'node:fs'; import os from 'node:os'; @@ -822,6 +822,127 @@ describe('tower-terminals', () => { vi.restoreAllMocks(); }); + // Bugfix #1686: a shellper row whose reconnect FAILS but whose process is + // still alive and whose socket file is still present must survive Phase 2 + // untouched — never deleted, never signaled. reconnectSession() returns null + // for transient reasons (socket connect refused because another Tower client + // owns the shellper, or a boot-time socket/fd hiccup), not just for a dead + // process; treating that as death SIGTERMed 53 live sessions in the #1629 + // incident. Fails without the guard (Phase 2 killed session.pid + DELETEd + // the row on reconnect failure alone). + it('leaves a live-pid + present-socket row untouched when reconnect fails (#1686)', async () => { + // onTestFinished so the mocked process.kill (dangerous if leaked into + // other tests) is always restored, even if an assertion throws. + onTestFinished(() => vi.restoreAllMocks()); + mockDbRun.mockReset(); + mockDbAll.mockReset(); + mockDbPrepare.mockReturnValue({ run: mockDbRun, all: mockDbAll }); + + const liveId = 'bugfix-1686-live-session'; + const socketPath = '/tmp/shellper-bugfix-1686.sock'; + + // Reconnect fails (e.g. socket owned by the real Tower — one-client-per- + // shellper), the same null this path returns for a genuinely dead process. + const mockReconnectSession = vi.fn(async () => null); + const deps = makeDeps({ shellperManager: { reconnectSession: mockReconnectSession } as any }); + initTerminals(deps); + + // process.pid is genuinely alive, so processExists() returns true. + mockDbAll.mockReturnValue([{ + id: liveId, + workspace_path: '/real/project', + type: 'builder', + role_id: 'builder-live', + pid: process.pid, + shellper_socket: socketPath, + shellper_pid: process.pid, + shellper_start_time: Date.now(), + created_at: new Date().toISOString(), + }]); + + // Workspace exists; the shellper socket FILE is present (connect refused, + // not file-gone). Config lookups resolve false. + vi.spyOn(fs, 'existsSync').mockImplementation((p: fs.PathLike) => { + if (String(p) === '/real/project') return true; + if (String(p) === socketPath) return true; + return false; + }); + + // Swallow real SIGTERMs (would kill the test runner if the bug regressed), + // but let the liveness probe (signal 0) pass through to the real kill. + const realKill = process.kill.bind(process); + const killSpy = vi.spyOn(process, 'kill').mockImplementation(((pid: number, signal?: string | number) => { + if (signal === 0 || signal === undefined) return realKill(pid, signal); + return true; + }) as typeof process.kill); + + const { reconcileTerminalSessions } = await import('../servers/tower-terminals.js'); + await reconcileTerminalSessions(); + + // Never signaled. + expect(killSpy).not.toHaveBeenCalledWith(process.pid, 'SIGTERM'); + // Row left in place — no DELETE run for this session id. + expect(mockDbRun).not.toHaveBeenCalledWith(liveId); + // Logged WARN for the next-pass retry. + expect(deps.log).toHaveBeenCalledWith('WARN', expect.stringContaining('leaving row untouched')); + }); + + // Bugfix #1686 (sibling site): a row the reconcile guard above preserved + // must also survive the on-the-fly reconnect in getTerminalsForWorkspace() + // when that reconnect still fails. Before the fold, this path deleted the + // row on any null reconnect (no liveness check), so the first /api/state or + // /api/overview read after a Tower restart would delete the very row Phase 2 + // had just preserved — falsifying the "retried next pass" promise. Fails + // without the on-the-fly guard (deleteTerminalSession runs DELETE for the id). + it('leaves a live-pid + present-socket row untouched when on-the-fly reconnect fails (#1686)', async () => { + onTestFinished(() => vi.restoreAllMocks()); + mockDbRun.mockReset(); + mockDbAll.mockReset(); + mockDbPrepare.mockReturnValue({ run: mockDbRun, all: mockDbAll }); + + const liveId = 'bugfix-1686-onthefly-session'; + const socketPath = '/tmp/shellper-bugfix-1686-otf.sock'; + + // On-the-fly reconnect still fails (persistent ownership/transient hiccup). + const mockReconnectSession = vi.fn(async () => null); + const deps = makeDeps({ shellperManager: { reconnectSession: mockReconnectSession } as any }); + initTerminals(deps); + + // getTerminalSessionsForWorkspace() reads this row; its PtySession is gone + // (never created), so getTerminalsForWorkspace takes the on-the-fly path. + mockDbAll.mockReturnValue([{ + id: liveId, + workspace_path: '/real/project', + type: 'builder', + role_id: 'builder-live-otf', + pid: process.pid, + shellper_socket: socketPath, + shellper_pid: process.pid, + shellper_start_time: Date.now(), + created_at: new Date().toISOString(), + }]); + + vi.spyOn(fs, 'existsSync').mockImplementation((p: fs.PathLike) => { + if (String(p) === '/real/project') return true; + if (String(p) === socketPath) return true; + return false; + }); + + const realKill = process.kill.bind(process); + const killSpy = vi.spyOn(process, 'kill').mockImplementation(((pid: number, signal?: string | number) => { + if (signal === 0 || signal === undefined) return realKill(pid, signal); + return true; + }) as typeof process.kill); + + const { getTerminalsForWorkspace: getTerms } = await import('../servers/tower-terminals.js'); + await getTerms('/real/project', 'http://proxy'); + + // On-the-fly never signals, but before the fold it DELETEd the row. + expect(killSpy).not.toHaveBeenCalledWith(process.pid, 'SIGTERM'); + expect(mockDbRun).not.toHaveBeenCalledWith(liveId); + expect(deps.log).toHaveBeenCalledWith('WARN', expect.stringContaining('leaving row untouched')); + }); + // ========================================================================= // Spec 786 Phase 2 — Identity preservation on shellper auto-restart // ========================================================================= diff --git a/packages/codev/src/agent-farm/servers/tower-terminals.ts b/packages/codev/src/agent-farm/servers/tower-terminals.ts index c0975f5e3..7ed72a1d7 100644 --- a/packages/codev/src/agent-farm/servers/tower-terminals.ts +++ b/packages/codev/src/agent-farm/servers/tower-terminals.ts @@ -647,6 +647,10 @@ async function _reconcileTerminalSessionsInner(): Promise { let orphanReconnected = 0; let killed = 0; let cleaned = 0; + // Bugfix #1686: rows kept because reconnect failed but death could not be + // confirmed (live pid + present socket). Counted so a reconcile that only + // preserved rows does not misreport "No terminal sessions to reconcile". + let unconfirmed = 0; // Track matched session IDs across all phases const matchedSessionIds = new Set(); @@ -800,8 +804,13 @@ async function _reconcileTerminalSessionsInner(): Promise { // Process probe results sequentially (shared state mutations) for (const { dbSession, client, replayData, restartOptions } of probeResults) { if (!client) { - _deps.log('INFO', `Shellper session ${dbSession.id} is stale (PID/socket dead) — will clean up`); - continue; // Will be cleaned up in Phase 2 + // Bugfix #1686: reconnect returning null does NOT prove death (it also + // covers a connect refused by another Tower client, or a transient + // socket/fd hiccup). Death is confirmed in the Phase 2 sweep, which keeps + // a row whose pid is alive and socket present — so this log defers the + // verdict rather than asserting "dead" up front. + _deps.log('INFO', `Shellper session ${dbSession.id} reconnect failed — deferring to Phase 2 sweep for death confirmation`); + continue; // Phase 2 confirms death before any cleanup } const workspacePath = dbSession.workspace_path; @@ -923,6 +932,26 @@ async function _reconcileTerminalSessionsInner(): Promise { const existing = manager.getSession(session.id); if (existing && existing.status !== 'exited') continue; + // Bugfix #1686: a shellper-backed row reaches here when Phase 1 failed to + // reconnect to it. But reconnectSession() returns null for transient reasons + // too — a socket connect refused because another Tower client already owns + // the shellper (one-client-per-shellper), or a boot-time socket/fd hiccup — + // not just for a genuinely dead process. Treating that failure as death and + // SIGTERMing the still-live pid is what destroyed 53 live sessions in the + // #1629 incident. Require POSITIVE evidence of death before touching the + // process or the row: the shellper pid must be gone AND/OR its socket file + // absent. A live pid with a present socket is "could not confirm" — leave the + // row in place (WARN) for the next reconcile/adoption pass; never signal it. + if (session.shellper_socket) { + const shellperAlive = session.shellper_pid != null && processExists(session.shellper_pid); + const socketPresent = fs.existsSync(session.shellper_socket); + if (shellperAlive && socketPresent) { + _deps.log('WARN', `Shellper session ${session.id} failed to reconnect but pid ${session.shellper_pid} is alive and socket ${session.shellper_socket} is present — leaving row untouched, retried next reconcile/adoption pass (${session.type} for ${path.basename(session.workspace_path)})`); + unconfirmed++; + continue; + } + } + // Stale row — kill orphaned process if any, then delete if (session.pid && processExists(session.pid)) { _deps.log('INFO', `Killing orphaned process: PID ${session.pid} (${session.type} for ${path.basename(session.workspace_path)})`); @@ -937,8 +966,8 @@ async function _reconcileTerminalSessionsInner(): Promise { } const total = shellperReconnected + orphanReconnected; - if (total > 0 || killed > 0 || cleaned > 0) { - _deps.log('INFO', `Reconciliation complete: ${shellperReconnected} shellper, ${orphanReconnected} orphan, ${killed} killed, ${cleaned} stale rows cleaned`); + if (total > 0 || killed > 0 || cleaned > 0 || unconfirmed > 0) { + _deps.log('INFO', `Reconciliation complete: ${shellperReconnected} shellper, ${orphanReconnected} orphan, ${killed} killed, ${cleaned} stale rows cleaned, ${unconfirmed} unconfirmed (kept)`); } else { _deps.log('INFO', 'No terminal sessions to reconcile'); } @@ -1111,6 +1140,24 @@ export async function getTerminalsForWorkspace( } if (!session) { + // Bugfix #1686: the same guard as reconcile Phase 2, applied to this + // sibling delete site. The on-the-fly reconnect above returns null for + // transient reasons too (socket connect refused because another Tower + // client owns the shellper, or a boot-time socket/fd hiccup), not just + // for a dead process. Dropping the row on that failure alone is how a row + // the Phase 2 guard just preserved would be deleted by the first + // /api/state or /api/overview read whose reconnect fails again — making + // Phase 2's "retried next reconcile/adoption pass" promise false. Require + // positive evidence of death (pid gone AND/OR socket file absent) before + // deleting; a live pid with a present socket is kept for a later pass. + if (dbSession.shellper_socket) { + const shellperAlive = dbSession.shellper_pid != null && processExists(dbSession.shellper_pid); + const socketPresent = fs.existsSync(dbSession.shellper_socket); + if (shellperAlive && socketPresent) { + _deps?.log('WARN', `On-the-fly reconnect for ${dbSession.id} failed but pid ${dbSession.shellper_pid} is alive and socket ${dbSession.shellper_socket} is present — leaving row untouched, retried next reconcile/adoption pass (${dbSession.type})`); + continue; + } + } // Stale row, nothing to reconnect — clean up deleteTerminalSession(dbSession.id); continue; From a43805b3f1fc59e5b0fd08c985939e92d68999a1 Mon Sep 17 00:00:00 2001 From: Amr Elsayed Date: Thu, 17 Sep 2026 10:00:09 +1000 Subject: [PATCH 4/8] chore(porch): bugfix-1686 pr phase-transition --- .../bugfix-1686-tower-reconcile-failed-reconne/status.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codev/projects/bugfix-1686-tower-reconcile-failed-reconne/status.yaml b/codev/projects/bugfix-1686-tower-reconcile-failed-reconne/status.yaml index 3cc2001e9..f39f1aad2 100644 --- a/codev/projects/bugfix-1686-tower-reconcile-failed-reconne/status.yaml +++ b/codev/projects/bugfix-1686-tower-reconcile-failed-reconne/status.yaml @@ -1,7 +1,7 @@ id: bugfix-1686 title: tower-reconcile-failed-reconne protocol: bugfix -phase: fix +phase: pr plan_phases: [] current_plan_phase: null gates: @@ -11,4 +11,4 @@ iteration: 1 build_complete: false history: [] started_at: '2026-09-16T23:38:45.913Z' -updated_at: '2026-09-16T23:43:28.321Z' +updated_at: '2026-09-17T00:00:09.956Z' From cba20f67b54f5a4df1fd62835e811ddc589c6124 Mon Sep 17 00:00:00 2001 From: Amr Elsayed Date: Thu, 17 Sep 2026 10:01:30 +1000 Subject: [PATCH 5/8] chore(porch): bugfix-1686 pr gate-requested --- .../bugfix-1686-tower-reconcile-failed-reconne/status.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/codev/projects/bugfix-1686-tower-reconcile-failed-reconne/status.yaml b/codev/projects/bugfix-1686-tower-reconcile-failed-reconne/status.yaml index f39f1aad2..a7051ee77 100644 --- a/codev/projects/bugfix-1686-tower-reconcile-failed-reconne/status.yaml +++ b/codev/projects/bugfix-1686-tower-reconcile-failed-reconne/status.yaml @@ -7,8 +7,10 @@ current_plan_phase: null gates: pr: status: pending + requested_at: '2026-09-17T00:01:30.503Z' iteration: 1 build_complete: false history: [] started_at: '2026-09-16T23:38:45.913Z' -updated_at: '2026-09-17T00:00:09.956Z' +updated_at: '2026-09-17T00:01:30.503Z' +pr_ready_for_human: true From 6cd7d698c412db0d0e80f0c657cd48d287ed2dd7 Mon Sep 17 00:00:00 2001 From: Amr Elsayed Date: Thu, 17 Sep 2026 10:10:47 +1000 Subject: [PATCH 6/8] [Bugfix #1686] Adopt review ruling: pid-down is the sole proof of death MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CMAP (codex, HIGH) flagged that the earlier `shellperAlive && socketPresent` guard still SIGTERMed + deleted a LIVE shellper whose socket file was transiently absent — violating #1686's explicit "a row whose pid is alive is left in place ... never signaled". Architect ruling: adopt it. The issue's "AND/OR socket absent" clause was loose drafting; the invariant is "destroy only what is proven dead; proof = pid down". Both guards now gate on `shellperAlive` alone (socket-file state no longer contributes to the kill decision) at reconcile Phase 2 and the getTerminalsForWorkspace on-the-fly site. A pid-dead row still deletes regardless of socket state. Adds regression coverage for the (live-pid, socket-absent) case at both sites (verified to fail under the old socket-present guard). Accepted residual grows accordingly: a live but permanently-unreconnectable socketless shellper keeps its row until husk-sweep (#1227) / stop reaps it — leak a row, not a session. --- codev/state/bugfix-1686_thread.md | 43 +++++++- .../__tests__/tower-terminals.test.ts | 97 +++++++++++++++++++ .../src/agent-farm/servers/tower-terminals.ts | 26 ++--- 3 files changed, 152 insertions(+), 14 deletions(-) diff --git a/codev/state/bugfix-1686_thread.md b/codev/state/bugfix-1686_thread.md index e6f1b395a..c3d0f617a 100644 --- a/codev/state/bugfix-1686_thread.md +++ b/codev/state/bugfix-1686_thread.md @@ -84,7 +84,48 @@ PR-body residuals to document (claude r2, all accepted-as-is, not changed): - shellper_pid===null legacy edge → falls through to SIGTERM of session.pid; ~unreachable given saveTerminalSession call sites. -Commit: 35a3bd92b (Fix #1686 ...). +Commit: 35a3bd92b → amended → 8707ea008 (Fix #1686 ...). + +## PR (phase 3) — PR #1693 open, CMAP done, BLOCKED on architect ruling + +PR #1693 (https://github.com/cluesmith/codev/pull/1693), Fixes #1686. Branch pushed. +CMAP --type pr: gemini=APPROVE(HIGH), claude=APPROVE(HIGH), codex=REQUEST_CHANGES(HIGH). + +codex RC (issue-grounded): guard preserves only when `shellperAlive && socketPresent`, so a +LIVE pid with a TRANSIENTLY-absent socket still gets SIGTERMed — violates #1686's explicit +"a row whose pid is alive is left in place ... never signaled." The issue is internally in +tension (also says "positive evidence of death: pid dead AND/OR socket absent"); the (live-pid, +socket-absent) edge is where they disagree. My code took the AND/OR reading; codex wants +pid-alive→never-signal unconditionally. + +Sent architect a ruling request (recommend ADOPTing codex: drop `&& socketPresent`, guard = +`if (shellperAlive)`, add regression coverage for live-pid+socket-absent at both sites; the +"asymmetric evidence" residual goes away, "unbounded retention" grows slightly = accepted). +This changes the guard shape the architect named, so not flipping unilaterally. WAITING. + +pr gate is already surfaced (my `porch done --help` fired it — harmless, human-approval-only). +Will NOT send the gate-ready notification until the RC is resolved. + +## PR (cont.) — architect RULING: adopt codex, implemented + +Ruling: shellperAlive ALONE gates destruction at both sites; socket-file state contributes +NOTHING to the kill decision; pid-down is the sole proof of death (pid-dead rows delete as +before regardless of socket). Implemented: both guards now `if (shellperAlive)` (dropped +`&& socketPresent`). Added 2 regression cases (live-pid + socket-ABSENT) at both sites, +verified non-vacuous (fail under the old socketPresent guard). 65/65 in file, tsc clean. + +For the REVIEW artifact (architect-directed): +1. The issue text's internal conflict ("AND/OR socket absent" vs "a live pid is never + signaled") was MAIN's loose drafting. This ruling resolves it to the invariant: DESTROY + ONLY WHAT IS PROVEN DEAD; proof = pid down. +2. Accepted residual GROWS: a live socketless orphan now persists until husk-sweep (#1227) or + stop reaps it — deliberate "leak a row, not a session". The earlier "asymmetric evidence" + residual is GONE (superseded by the ruling). +Remaining residuals unchanged: PID-reuse (conservative by design), legacy shellper_pid===null +edge (PR-body note), pre-existing 5 env-class test failures (PR-body note). + +Next: amend commit, update PR #1693 body, push, re-run --type pr CMAP for codex's answered-in- +code record, then gate notification with fresh verdicts. PR body must reference: the 5 pre-existing env-class failures (consolidate.test.ts + spawn-retirement.test.ts — getRolesDir "Roles directory not found" in worktree test env, diff --git a/packages/codev/src/agent-farm/__tests__/tower-terminals.test.ts b/packages/codev/src/agent-farm/__tests__/tower-terminals.test.ts index 21b69eebd..7549d2dac 100644 --- a/packages/codev/src/agent-farm/__tests__/tower-terminals.test.ts +++ b/packages/codev/src/agent-farm/__tests__/tower-terminals.test.ts @@ -943,6 +943,103 @@ describe('tower-terminals', () => { expect(deps.log).toHaveBeenCalledWith('WARN', expect.stringContaining('leaving row untouched')); }); + // Bugfix #1686 (socket-absent edge, #1693 ruling): the sole proof of death + // is the pid being down. A live pid whose socket FILE is absent — the same + // transient-failure class (fd pressure / hiccup) that fails the reconnect — + // must still be preserved, never signaled. This closes the issue's internal + // "AND/OR socket absent" ambiguity in favor of its "a live pid is never + // signaled" invariant. Would SIGTERM + delete under the earlier + // shellperAlive && socketPresent guard. + it('leaves a live-pid row untouched even when its socket file is absent — reconcile (#1686)', async () => { + onTestFinished(() => vi.restoreAllMocks()); + mockDbRun.mockReset(); + mockDbAll.mockReset(); + mockDbPrepare.mockReturnValue({ run: mockDbRun, all: mockDbAll }); + + const liveId = 'bugfix-1686-socketgone-session'; + const socketPath = '/tmp/shellper-bugfix-1686-gone.sock'; + + const mockReconnectSession = vi.fn(async () => null); + const deps = makeDeps({ shellperManager: { reconnectSession: mockReconnectSession } as any }); + initTerminals(deps); + + mockDbAll.mockReturnValue([{ + id: liveId, + workspace_path: '/real/project', + type: 'builder', + role_id: 'builder-socketgone', + pid: process.pid, + shellper_socket: socketPath, + shellper_pid: process.pid, + shellper_start_time: Date.now(), + created_at: new Date().toISOString(), + }]); + + // Socket FILE is ABSENT (returns false), but the pid is alive. + vi.spyOn(fs, 'existsSync').mockImplementation((p: fs.PathLike) => { + if (String(p) === '/real/project') return true; + return false; // socket file gone, config lookups false + }); + + const realKill = process.kill.bind(process); + const killSpy = vi.spyOn(process, 'kill').mockImplementation(((pid: number, signal?: string | number) => { + if (signal === 0 || signal === undefined) return realKill(pid, signal); + return true; + }) as typeof process.kill); + + const { reconcileTerminalSessions } = await import('../servers/tower-terminals.js'); + await reconcileTerminalSessions(); + + expect(killSpy).not.toHaveBeenCalledWith(process.pid, 'SIGTERM'); + expect(mockDbRun).not.toHaveBeenCalledWith(liveId); + expect(deps.log).toHaveBeenCalledWith('WARN', expect.stringContaining('leaving row untouched')); + }); + + // Same socket-absent edge at the on-the-fly sibling site. + it('leaves a live-pid row untouched even when its socket file is absent — on-the-fly (#1686)', async () => { + onTestFinished(() => vi.restoreAllMocks()); + mockDbRun.mockReset(); + mockDbAll.mockReset(); + mockDbPrepare.mockReturnValue({ run: mockDbRun, all: mockDbAll }); + + const liveId = 'bugfix-1686-socketgone-otf'; + const socketPath = '/tmp/shellper-bugfix-1686-gone-otf.sock'; + + const mockReconnectSession = vi.fn(async () => null); + const deps = makeDeps({ shellperManager: { reconnectSession: mockReconnectSession } as any }); + initTerminals(deps); + + mockDbAll.mockReturnValue([{ + id: liveId, + workspace_path: '/real/project', + type: 'builder', + role_id: 'builder-socketgone-otf', + pid: process.pid, + shellper_socket: socketPath, + shellper_pid: process.pid, + shellper_start_time: Date.now(), + created_at: new Date().toISOString(), + }]); + + vi.spyOn(fs, 'existsSync').mockImplementation((p: fs.PathLike) => { + if (String(p) === '/real/project') return true; + return false; // socket file gone + }); + + const realKill = process.kill.bind(process); + const killSpy = vi.spyOn(process, 'kill').mockImplementation(((pid: number, signal?: string | number) => { + if (signal === 0 || signal === undefined) return realKill(pid, signal); + return true; + }) as typeof process.kill); + + const { getTerminalsForWorkspace: getTerms } = await import('../servers/tower-terminals.js'); + await getTerms('/real/project', 'http://proxy'); + + expect(killSpy).not.toHaveBeenCalledWith(process.pid, 'SIGTERM'); + expect(mockDbRun).not.toHaveBeenCalledWith(liveId); + expect(deps.log).toHaveBeenCalledWith('WARN', expect.stringContaining('leaving row untouched')); + }); + // ========================================================================= // Spec 786 Phase 2 — Identity preservation on shellper auto-restart // ========================================================================= diff --git a/packages/codev/src/agent-farm/servers/tower-terminals.ts b/packages/codev/src/agent-farm/servers/tower-terminals.ts index 7ed72a1d7..c0501d633 100644 --- a/packages/codev/src/agent-farm/servers/tower-terminals.ts +++ b/packages/codev/src/agent-farm/servers/tower-terminals.ts @@ -938,15 +938,16 @@ async function _reconcileTerminalSessionsInner(): Promise { // the shellper (one-client-per-shellper), or a boot-time socket/fd hiccup — // not just for a genuinely dead process. Treating that failure as death and // SIGTERMing the still-live pid is what destroyed 53 live sessions in the - // #1629 incident. Require POSITIVE evidence of death before touching the - // process or the row: the shellper pid must be gone AND/OR its socket file - // absent. A live pid with a present socket is "could not confirm" — leave the - // row in place (WARN) for the next reconcile/adoption pass; never signal it. + // #1629 incident. Destroy only what is proven dead, and the sole proof of + // death is the pid being down: a live shellper pid is never signaled and its + // row is never deleted here, regardless of socket-file state (a transiently + // absent socket is the same transient-failure class and must not kill a live + // agent). The row is left in place (WARN) and retried on a later + // reconcile/adoption pass; a pid-dead row still deletes below as before. if (session.shellper_socket) { const shellperAlive = session.shellper_pid != null && processExists(session.shellper_pid); - const socketPresent = fs.existsSync(session.shellper_socket); - if (shellperAlive && socketPresent) { - _deps.log('WARN', `Shellper session ${session.id} failed to reconnect but pid ${session.shellper_pid} is alive and socket ${session.shellper_socket} is present — leaving row untouched, retried next reconcile/adoption pass (${session.type} for ${path.basename(session.workspace_path)})`); + if (shellperAlive) { + _deps.log('WARN', `Shellper session ${session.id} failed to reconnect but pid ${session.shellper_pid} is alive — leaving row untouched (a live pid is never signaled), retried next reconcile/adoption pass (${session.type} for ${path.basename(session.workspace_path)})`); unconfirmed++; continue; } @@ -1147,14 +1148,13 @@ export async function getTerminalsForWorkspace( // for a dead process. Dropping the row on that failure alone is how a row // the Phase 2 guard just preserved would be deleted by the first // /api/state or /api/overview read whose reconnect fails again — making - // Phase 2's "retried next reconcile/adoption pass" promise false. Require - // positive evidence of death (pid gone AND/OR socket file absent) before - // deleting; a live pid with a present socket is kept for a later pass. + // Phase 2's "retried next reconcile/adoption pass" promise false. Destroy + // only what is proven dead: a live shellper pid is never deleted here, + // regardless of socket-file state; the row is kept for a later pass. if (dbSession.shellper_socket) { const shellperAlive = dbSession.shellper_pid != null && processExists(dbSession.shellper_pid); - const socketPresent = fs.existsSync(dbSession.shellper_socket); - if (shellperAlive && socketPresent) { - _deps?.log('WARN', `On-the-fly reconnect for ${dbSession.id} failed but pid ${dbSession.shellper_pid} is alive and socket ${dbSession.shellper_socket} is present — leaving row untouched, retried next reconcile/adoption pass (${dbSession.type})`); + if (shellperAlive) { + _deps?.log('WARN', `On-the-fly reconnect for ${dbSession.id} failed but pid ${dbSession.shellper_pid} is alive — leaving row untouched (a live pid is never signaled), retried next reconcile/adoption pass (${dbSession.type})`); continue; } } From 86ec0414197d8177ad1ea9d5124eaa73f17a014a Mon Sep 17 00:00:00 2001 From: Amr Elsayed Date: Thu, 17 Sep 2026 10:18:58 +1000 Subject: [PATCH 7/8] [Bugfix #1686] Address PR CMAP notes: stale comments, pid-dead test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Non-blocking review follow-ups (2× APPROVE, 1× COMMENT on the PR): - Reword two comments that still described the superseded "live pid + present socket" guard; the invariant is pid-only. - Add a fifth regression test asserting a pid-DEAD shellper row is still swept, bounding the guard against future over-preservation. (PR body separately corrected: the retained live socketless row is reaped by the next boot's killOrphanedShellpers, not the #1227 husk sweep, which exempts the still-"registered" pid.) --- codev/state/bugfix-1686_thread.md | 42 +++++++++++++++++++ .../__tests__/tower-terminals.test.ts | 42 +++++++++++++++++++ .../src/agent-farm/servers/tower-terminals.ts | 8 ++-- 3 files changed, 88 insertions(+), 4 deletions(-) diff --git a/codev/state/bugfix-1686_thread.md b/codev/state/bugfix-1686_thread.md index c3d0f617a..a3c29ec1b 100644 --- a/codev/state/bugfix-1686_thread.md +++ b/codev/state/bugfix-1686_thread.md @@ -127,6 +127,48 @@ edge (PR-body note), pre-existing 5 env-class test failures (PR-body note). Next: amend commit, update PR #1693 body, push, re-run --type pr CMAP for codex's answered-in- code record, then gate notification with fresh verdicts. +## PR (cont.) — recovery + build-artifact finding + +Git recovery: I accidentally `git commit --amend`'d porch's pushed `cba20f67b` "pr +gate-requested" chore (relabeling it as my Fix commit, swallowing its status.yaml). Recovered +WITHOUT force-push: `git reset --soft cba20f67b` (remote was at cba20f67b; my bad amend was +local-only), then committed the ruling delta as a fresh follow-up `6cd7d698c [Bugfix #1686] +Adopt review ruling ...`. History is append-only and clean: 8707ea008 (fix) → porch chores → +6cd7d698c (ruling). Lesson: don't blind-amend when porch may have added a chore commit on top; +check `git log` first. Two "Fix #1686"-family commits is fine (no-squash preserves the +narrative). + +Build-artifact finding: the "5 pre-existing failures" (consolidate/spawn-retirement, +getRolesDir "Roles directory not found") only occur when vitest runs BEFORE `pnpm build`. +bundle-assets' copy-skeleton populates skeleton/ which getRolesDir resolves; after `pnpm build` +the full agent-farm suite is GREEN (176 files / 3683 tests / 0 fail). So they're a +build-prerequisite test-hygiene artifact, not real failures (CI builds before testing). +Pre-existing + unrelated; candidate to FILE as a test-hygiene follow-up (tests depending on +build output). PR body corrected to say this accurately. + +## PR (cont.) — re-CMAP r2 (post-ruling) + polish + +Re-CMAP --type pr r2: gemini=APPROVE(HIGH), codex=COMMENT(HIGH) (its RC now answered in code), +claude=APPROVE(HIGH). All non-blocking. Addressed before gate: +- Stale comments (:650, :810) still said "socket present" — reworded to pid-only (matches the + ruling). Flagged by codex+claude. +- CORRECTED the accepted-residual reaper (claude, verified against code): the retained live + socketless row is NOT reaped by husk-sweep #1227 (computeRegisteredShellperPids marks the + live pid "registered" → husk-EXEMPT). The real reaper is killOrphanedShellpers at next boot + (tower-server.ts:725) — skips responsive-socket shellpers, SIGTERMs socket-dead ones — then + next reconcile/read sees shellperAlive=false and deletes the row. PR body fixed. (My ruling + commit 6cd7d698c body still says #1227 — left as-is to avoid force-push; PR body is the + authoritative correction.) +- Added a 5th regression test: pid-DEAD row is still swept (bounds the guard against future + over-preservation like `if (shellper_socket) continue`). 5/5 #1686 tests, tsc clean. +- FILE candidates (not fixed, per boundary): (a) test-hygiene — consolidate/spawn-retirement + depend on built skeleton; (b) UX — a preserved-but-unreconnectable terminal is absent from + /api/state with only a WARN (claude note 4), pre-existing UI behavior. +- codex noted branch 18 commits behind origin/main (disjoint, merge-tree clean) — leaving to + the architect's merge unless they want a freshen. + +Gate notification with all three verdicts is next. + PR body must reference: the 5 pre-existing env-class failures (consolidate.test.ts + spawn-retirement.test.ts — getRolesDir "Roles directory not found" in worktree test env, confirmed identical on clean base), and the shellper_pid===null legacy edge. diff --git a/packages/codev/src/agent-farm/__tests__/tower-terminals.test.ts b/packages/codev/src/agent-farm/__tests__/tower-terminals.test.ts index 7549d2dac..862ec1ca0 100644 --- a/packages/codev/src/agent-farm/__tests__/tower-terminals.test.ts +++ b/packages/codev/src/agent-farm/__tests__/tower-terminals.test.ts @@ -1040,6 +1040,48 @@ describe('tower-terminals', () => { expect(deps.log).toHaveBeenCalledWith('WARN', expect.stringContaining('leaving row untouched')); }); + // Bugfix #1686 (other direction): the guard must NOT over-preserve. A + // shellper row whose pid is genuinely DEAD is still swept — deleted as + // before. Without this, an over-broad guard (`if (shellper_socket) continue`) + // would pass the rest of the suite unnoticed. Here the guard's shellperAlive + // is false (dead pid), so Phase 2 falls through to the DELETE. + it('still deletes a shellper row whose pid is dead (#1686)', async () => { + onTestFinished(() => vi.restoreAllMocks()); + mockDbRun.mockReset(); + mockDbAll.mockReset(); + mockDbPrepare.mockReturnValue({ run: mockDbRun, all: mockDbAll }); + + const deadId = 'bugfix-1686-dead-session'; + const deadPid = 2147483646; // no such process → processExists() === false + + const mockReconnectSession = vi.fn(async () => null); + const deps = makeDeps({ shellperManager: { reconnectSession: mockReconnectSession } as any }); + initTerminals(deps); + + mockDbAll.mockReturnValue([{ + id: deadId, + workspace_path: '/real/project', + type: 'builder', + role_id: 'builder-dead', + pid: deadPid, + shellper_socket: '/tmp/shellper-bugfix-1686-dead.sock', + shellper_pid: deadPid, + shellper_start_time: Date.now(), + created_at: new Date().toISOString(), + }]); + + vi.spyOn(fs, 'existsSync').mockImplementation((p: fs.PathLike) => { + if (String(p) === '/real/project') return true; + return false; + }); + + const { reconcileTerminalSessions } = await import('../servers/tower-terminals.js'); + await reconcileTerminalSessions(); + + // Dead pid → not preserved → row is deleted (DELETE run with the id). + expect(mockDbRun).toHaveBeenCalledWith(deadId); + }); + // ========================================================================= // Spec 786 Phase 2 — Identity preservation on shellper auto-restart // ========================================================================= diff --git a/packages/codev/src/agent-farm/servers/tower-terminals.ts b/packages/codev/src/agent-farm/servers/tower-terminals.ts index c0501d633..977c8207d 100644 --- a/packages/codev/src/agent-farm/servers/tower-terminals.ts +++ b/packages/codev/src/agent-farm/servers/tower-terminals.ts @@ -647,8 +647,8 @@ async function _reconcileTerminalSessionsInner(): Promise { let orphanReconnected = 0; let killed = 0; let cleaned = 0; - // Bugfix #1686: rows kept because reconnect failed but death could not be - // confirmed (live pid + present socket). Counted so a reconcile that only + // Bugfix #1686: rows kept because reconnect failed but death was not proven + // (the shellper pid is still alive). Counted so a reconcile that only // preserved rows does not misreport "No terminal sessions to reconcile". let unconfirmed = 0; @@ -807,8 +807,8 @@ async function _reconcileTerminalSessionsInner(): Promise { // Bugfix #1686: reconnect returning null does NOT prove death (it also // covers a connect refused by another Tower client, or a transient // socket/fd hiccup). Death is confirmed in the Phase 2 sweep, which keeps - // a row whose pid is alive and socket present — so this log defers the - // verdict rather than asserting "dead" up front. + // a row whose pid is still alive — so this log defers the verdict rather + // than asserting "dead" up front. _deps.log('INFO', `Shellper session ${dbSession.id} reconnect failed — deferring to Phase 2 sweep for death confirmation`); continue; // Phase 2 confirms death before any cleanup } From 66e4924ae366f4eae7dc4b0c31465851a97fb8a6 Mon Sep 17 00:00:00 2001 From: Amr Elsayed Date: Thu, 17 Sep 2026 11:02:56 +1000 Subject: [PATCH 8/8] chore(porch): bugfix-1686 pr gate-approved --- .../bugfix-1686-tower-reconcile-failed-reconne/status.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/codev/projects/bugfix-1686-tower-reconcile-failed-reconne/status.yaml b/codev/projects/bugfix-1686-tower-reconcile-failed-reconne/status.yaml index a7051ee77..0b9e16c5c 100644 --- a/codev/projects/bugfix-1686-tower-reconcile-failed-reconne/status.yaml +++ b/codev/projects/bugfix-1686-tower-reconcile-failed-reconne/status.yaml @@ -6,11 +6,12 @@ plan_phases: [] current_plan_phase: null gates: pr: - status: pending + status: approved requested_at: '2026-09-17T00:01:30.503Z' + approved_at: '2026-09-17T01:02:56.291Z' iteration: 1 build_complete: false history: [] started_at: '2026-09-16T23:38:45.913Z' -updated_at: '2026-09-17T00:01:30.503Z' -pr_ready_for_human: true +updated_at: '2026-09-17T01:02:56.292Z' +pr_ready_for_human: false