Keep killed git processes from leaving locks that block self-updates

.gitignore

@@ -55,6 +55,10 @@ slack-reactor/package-lock.json
 # untracked, the deployed fleet's self_update.py sees a perpetually-dirty tree,
 # does `git stash push -u`, and loops a 24h-delay cycle forever.
 .worktrees/
+# Claude Code's isolated agent worktrees, and the self-update marker a test run
+# inside one writes to its grandparent: the same perpetually-dirty tree.
+.claude/worktrees/
+.claude/.assistant/
 
 # Runtime state dir when an install keeps it inside the repo instead of ~
 # (metrics.jsonl, observer-summaries/, heartbeat.json — never committed).

CHANGELOG.md

@@ -40,6 +40,16 @@ The version is carried in `pyproject.toml` and `src/assistant/__init__.py`
   and reminders to finish older pending work before starting another task.
 
 ### Fixed
+- Keep a killed git from blocking self-updates. The pulse's timeout used to
+  SIGKILL a context check mid-`git status`, leaving `.git/index.lock` behind;
+  on 2026-09-28 that blocked every self-update, and seven repos on this machine
+  held such locks. Timeouts now send SIGTERM first (git removes its locks) and
+  still SIGKILL the group after 3 seconds, the background `git status` and
+  `git log` calls take no index lock at all, and self-update removes an
+  index.lock older than 10 minutes when no process has it open and no git
+  process is working in the repo. Each removal is recorded in the ledger.
+- Ignore Claude Code's `.claude/worktrees/` and the stray `.claude/.assistant/`
+  marker, which made the tree look dirty and set off daily auto-stashes.
 - Ping about an idle workspace only when it needs you: the agent's last lines
   ask you something, a permission or plan prompt is up, or the session stopped
   on an API error. Claude's idle alert fires about a minute after every turn,

README.md

@@ -305,6 +305,7 @@ These are structural, not just conventions — violating them will cause real pr
 ## Gotchas
 
 - **Self-update refuses a dirty or ahead tree.** `self_update.py` does `git pull --ff-only` only. A dirty tree is surfaced, never steamrolled.
+- **Self-update clears a stale git lock.** A `.git/index.lock` older than 10 minutes is removed before the update when `lsof` shows no process has it open and no git process working in the repo (a `git commit` waiting on its editor holds the lock without keeping the file open). A young or live lock is left alone, and every removal is ledgered.
 - **NO_INGEST_GUARD:** if the last send to a workspace returned `transcript_size_delta=0` (cmux sent OK but no Claude process was reading), the orchestrator skips the next resend. This breaks the cleanup-resend-loop class of bug structurally.
 - **The single-process daemon (`src/assistant/`) is opt-in.** The legacy pulse LaunchAgent keeps running until you explicitly switch over.
 - **mem0ai requires Python 3.12.** It lives in `.venv-mem0`; `ensure_venv()` transparently re-execs tools into that interpreter.

bin/build-ws-context.py

@@ -614,15 +614,15 @@ def cwd_state(cwd: str | None) -> tuple[bool, bool]:
         return False, False
     try:
         r = subprocess.run(
-            ["git", "-C", cwd, "status", "--porcelain"],
+            ["git", "--no-optional-locks", "-C", cwd, "status", "--porcelain"],
             capture_output=True, text=True, timeout=5,
         )
         dirty = bool(r.stdout.strip()) if r.returncode == 0 else False
     except Exception:
         dirty = False
     try:
         r = subprocess.run(
-            ["git", "-C", cwd, "log", "@{u}..", "--oneline"],
+            ["git", "--no-optional-locks", "-C", cwd, "log", "@{u}..", "--oneline"],
             capture_output=True, text=True, timeout=5,
         )
         unpushed = bool(r.stdout.strip()) if r.returncode == 0 else False

bin/comms_lib.py

@@ -241,6 +241,8 @@ def _clip(text: str, limit: int) -> str:
 _ACTION_PHRASES: dict[str, tuple[str, str]] = {
     "ready_for_merge": ("asked a workspace to merge its PR", "ask a workspace to merge its PR"),
     "self-update": ("updated Assistant to the latest code", "update Assistant to the latest code"),
+    "self-update-lock-cleared": ("cleared a stale git lock that was blocking my updates",
+                                 "clear a stale git lock that was blocking my updates"),
     "self-update-syntax-fail": ("updated Assistant to the latest code",
                                 "update Assistant to the latest code"),
     "strategist-context": ("started researching a decision that's waiting on you",

bin/pulse.py

@@ -295,6 +295,10 @@ def load_bedrock_env() -> dict:
 _BEDROCK_ENV = load_bedrock_env()
 
 
+# How long a timed-out child's process group gets after SIGTERM before SIGKILL.
+KILL_GRACE_SEC = 3
+
+
 def run(cmd: list[str], *, input_text: str | None = None,
         timeout: int = 30, env: dict | None = None,
         merge_bedrock: bool = False) -> tuple[int, str, str]:
@@ -331,6 +335,18 @@ def run(cmd: list[str], *, input_text: str | None = None,
         out, err = proc.communicate(input=input_text, timeout=timeout)
         return proc.returncode, out, err
     except subprocess.TimeoutExpired:
+        # SIGTERM first: git removes its lock files on SIGTERM, but a SIGKILL
+        # mid-`git status` left .git/index.lock behind in seven repos and
+        # blocked every later git write there (2026-09-28).
+        try:
+            os.killpg(proc.pid, signal.SIGTERM)
+        except (ProcessLookupError, PermissionError, OSError):
+            pass
+        try:
+            proc.communicate(timeout=KILL_GRACE_SEC)
+        except Exception:  # noqa: BLE001 — a timeout here just means SIGKILL next
+            pass
+        # SIGKILL the group regardless, so nothing that ignored SIGTERM survives.
         try:
             os.killpg(proc.pid, signal.SIGKILL)
         except (ProcessLookupError, PermissionError, OSError):
@@ -455,6 +471,21 @@ def self_update_pulse(pulse_idx: int) -> None:
     if result is None:
         return  # throttled — nothing to report
 
+    if result.get("cleared_stale_lock"):
+        # Something left a git lock behind again; make it visible, since the
+        # cleanup would otherwise hide a new cause.
+        append_ledger({
+            "ts": utc_iso(),
+            "epoch": utc_ts(),
+            "pulse_idx": pulse_idx,
+            "key": f"self-update-lock-cleared-p{pulse_idx}",
+            "kind": "self-update-lock-cleared",
+            "ws_ref": "(launchd)",
+            "outcome": "verified",
+            "evidence": result["cleared_stale_lock"][:300],
+        })
+        log.info("self-update: %s", result["cleared_stale_lock"])
+
     reason = result.get("skipped_reason")
     changed = result.get("changed")
     # Silent path: attempted, nothing to do, no problem — or a refused commit

bin/self_update.py

@@ -42,6 +42,9 @@
 from __future__ import annotations
 
 import json
+import os
+import shutil
+import signal
 import subprocess
 import sys
 import time
@@ -81,20 +84,149 @@
 REJECT_REMIND_SEC = 86400  # 1 day
 
 
+# How long a timed-out git gets after SIGTERM to remove its own lock files
+# before it's killed outright.
+KILL_GRACE_SEC = 3
+
+# git leaves `.git/index.lock` behind when it dies mid-write, and every later git
+# write in the repo then fails. A lock no process has open, older than this, is
+# stale (2026-09-28: one left by a killed `git status` blocked every self-update).
+STALE_LOCK_SEC = 600
+
+
 def _git(repo: Path, *args: str, timeout: int = 90) -> tuple[int, str, str]:
-    """Run a git command in `repo`. Returns (rc, stdout, stderr); never raises."""
+    """Run a git command in `repo`. Returns (rc, stdout, stderr); never raises.
+
+    `--no-optional-locks` keeps read-only commands like `status` from taking
+    the index lock at all. On a timeout git gets SIGTERM first — it removes
+    its lock files on SIGTERM, but a SIGKILL leaves them behind."""
     try:
-        p = subprocess.run(
-            ["git", "-C", str(repo), *args],
-            capture_output=True, text=True, timeout=timeout,
+        proc = subprocess.Popen(
+            ["git", "--no-optional-locks", "-C", str(repo), *args],
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
+            start_new_session=True,
         )
-        return p.returncode, p.stdout.strip(), p.stderr.strip()
+    except Exception as e:  # noqa: BLE001
+        return -1, "", str(e)
+    try:
+        out, err = proc.communicate(timeout=timeout)
+        return proc.returncode, out.strip(), err.strip()
     except subprocess.TimeoutExpired:
+        _terminate_group(proc)
         return -1, "", f"git {' '.join(args)} timed out after {timeout}s"
-    except Exception as e:  # noqa: BLE001
+    except Exception as e:  # noqa: BLE001 — e.g. undecodable output; never raise
+        _terminate_group(proc)
         return -1, "", str(e)
 
 
+def _terminate_group(proc: subprocess.Popen, grace: float = KILL_GRACE_SEC) -> None:
+    """SIGTERM a child's process group, give it `grace` seconds, then SIGKILL
+    the group regardless, so nothing that ignored SIGTERM survives. Every wait
+    is bounded."""
+    try:
+        os.killpg(proc.pid, signal.SIGTERM)
+    except (ProcessLookupError, PermissionError):
+        pass
+    try:
+        proc.communicate(timeout=grace)
+    except Exception:  # noqa: BLE001 — a timeout here just means SIGKILL next
+        pass
+    try:
+        os.killpg(proc.pid, signal.SIGKILL)
+    except (ProcessLookupError, PermissionError):
+        proc.kill()
+    try:
+        proc.communicate(timeout=5)
+    except Exception:  # noqa: BLE001 — reaping is best-effort after SIGKILL
+        pass
+
+
+def _lsof(*args: str) -> subprocess.CompletedProcess | None:
+    lsof = shutil.which("lsof") or "/usr/sbin/lsof"
+    try:
+        return subprocess.run([lsof, "-w", *args], capture_output=True, text=True, timeout=15)
+    except (OSError, subprocess.TimeoutExpired):
+        return None
+
+
+def _lsof_found_nothing(p: subprocess.CompletedProcess | None) -> bool:
+    """lsof exits 1 with no output when nothing matches; anything else means
+    something matched or lsof couldn't tell."""
+    return p is not None and p.returncode == 1 and not p.stdout.strip() and not p.stderr.strip()
+
+
+def _in_this_worktree(cwd: str, toplevel: str) -> bool:
+    """True if `cwd` is `toplevel` or inside it without crossing into a nested
+    repo or worktree (a folder with its own `.git`), which has its own index."""
+    top = Path(toplevel)
+    path = Path(cwd)
+    if path != top and top not in path.parents:
+        return False
+    while path != top:
+        if (path / ".git").exists():
+            return False
+        path = path.parent
+    return True
+
+
+def git_cwd_in(lsof_fields: str, toplevel: str) -> bool:
+    """True if `lsof -Fcn` output lists a git process (`git`, or a `git-*`
+    helper — not look-alikes such as `gitstatusd`) working in `toplevel`."""
+    command = ""
+    for line in lsof_fields.splitlines():
+        if line.startswith("c"):
+            command = line[1:]
+        elif line.startswith("n") and (command == "git" or command.startswith("git-")):
+            if _in_this_worktree(os.path.realpath(line[1:]), toplevel):
+                return True
+    return False
+
+
+def _lock_is_held(lock: Path, toplevel: str) -> bool:
+    """True if the lock may be live, or if that can't be ruled out.
+
+    Two checks, because git doesn't always keep the lock file open: `git commit
+    -a` or `-p` writes index.lock, closes it, and keeps holding the lock while
+    hooks and the editor run. So a lock is live if any process has it open, or
+    if any git process is working in the repo (git runs from the top folder)."""
+    if not _lsof_found_nothing(_lsof("-t", "--", str(lock))):
+        return True
+    p = _lsof("-a", "-c", "git", "-d", "cwd", "-Fcn")
+    if p is None or p.returncode not in (0, 1) or p.stderr.strip():
+        return True
+    return git_cwd_in(p.stdout, toplevel)
+
+
+def clear_stale_index_lock(repo: Path, *, now: float | None = None,
+                           max_age: float = STALE_LOCK_SEC,
+                           is_held=_lock_is_held) -> str | None:
+    """Remove the repo's index.lock if it's stale: older than `max_age`, with
+    no process holding it and no git working in the repo. Returns what was
+    removed, or None. A lock that's young, live, replaced while it was being
+    checked, or can't be checked is left alone."""
+    now = time.time() if now is None else now
+    rc, git_dir, _ = _git(repo, "rev-parse", "--absolute-git-dir")
+    rc2, toplevel, _ = _git(repo, "rev-parse", "--show-toplevel")
+    if rc != 0 or rc2 != 0 or not git_dir or not toplevel:
+        return None
+    lock = Path(git_dir) / "index.lock"
+    try:
+        before = lock.stat()
+    except OSError:
+        return None
+    age = now - before.st_mtime
+    if age < max_age or is_held(lock, os.path.realpath(toplevel)):
+        return None
+    try:
+        after = lock.stat()
+        if (after.st_ino, after.st_mtime_ns) != (before.st_ino, before.st_mtime_ns):
+            return None
+        lock.unlink()
+    except OSError:
+        return None
+    return f"removed a stale {lock} from {int(age // 60)} min ago that no git process held"
+
+
 def _stash_dirty(repo: Path, label: str) -> tuple[bool, str]:
     """Stash the dirty tree (tracked + untracked) under a labeled message.
 
@@ -288,6 +420,11 @@ def _log(msg: str) -> None:
     result: dict = {"attempted": True, "changed": False, "installed": False,
                     "skipped_reason": None}
 
+    cleared = clear_stale_index_lock(repo, now=now)
+    if cleared:
+        result["cleared_stale_lock"] = cleared
+        _log(cleared)
+
     rb = resolve_remote_branch(repo)
     if rb is None:
         result["skipped_reason"] = "no-remote"

src/assistant/slack.py

@@ -62,6 +62,8 @@ def _clip(text: str, limit: int) -> str:
 _ACTION_PHRASES: dict[str, tuple[str, str]] = {
     "ready_for_merge": ("asked a workspace to merge its PR", "ask a workspace to merge its PR"),
     "self-update": ("updated Assistant to the latest code", "update Assistant to the latest code"),
+    "self-update-lock-cleared": ("cleared a stale git lock that was blocking my updates",
+                                 "clear a stale git lock that was blocking my updates"),
     "self-update-syntax-fail": ("updated Assistant to the latest code",
                                 "update Assistant to the latest code"),
     "strategist-context": ("started researching a decision that's waiting on you",

tests/test_build_ws_context_in_process.py

@@ -323,6 +323,24 @@ def test_returns_false_false_when_cwd_missing(self):
         d, u = self.mod.cwd_state("/no/such/dir-x")
         self.assertEqual((d, u), (False, False))
 
+    def test_git_checks_never_rewrite_the_index(self):
+        """cwd_state runs under the pulse's timeout kill; a plain `git status`
+        rewrites the index under index.lock, and a kill at that moment leaves
+        the lock behind. Mutation probe: drop `--no-optional-locks` and the
+        index mtime changes."""
+        repo = self._tmp / "repo-locks"
+        repo.mkdir()
+        subprocess.run(["git", "init", "-q"], cwd=str(repo), check=True)
+        (repo / "f.txt").write_text("a")
+        subprocess.run(["git", "add", "f.txt"], cwd=str(repo), check=True)
+        subprocess.run(["git", "-c", "user.email=t@t", "-c", "user.name=t", "commit", "-qm", "i"],
+                       cwd=str(repo), check=True)
+        os.utime(repo / "f.txt", (1, 1))
+        index = repo / ".git" / "index"
+        before = index.stat().st_mtime_ns
+        self.mod.cwd_state(str(repo))
+        self.assertEqual(index.stat().st_mtime_ns, before)
+
     def test_clean_repo(self):
         # Init a real git repo + empty commit so @{u} doesn't error
         # (subprocess just returns rc != 0 when no upstream — we treat

tests/test_pulse.py

@@ -11,11 +11,13 @@
 import io
 import json
 import os
+import signal
 import subprocess
 import sys
 import textwrap
 import time
 import unittest
+import unittest.mock
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from unittest import mock
@@ -645,6 +647,56 @@ def test_timeout_with_pipe_holding_grandchild_returns_promptly(self):
             os.kill(gpid, 9)  # clean up before failing
             self.fail(f"grandchild {gpid} survived the group kill")
 
+    def test_timeout_sends_sigterm_first_so_children_clean_up(self):
+        """A SIGKILL mid-`git status` left .git/index.lock behind in seven
+        repos. The timeout now sends SIGTERM first, which git handles by
+        removing its locks. Mutation probe: go straight to SIGKILL and the
+        marker file is never written."""
+        marker = Path(self._tmp_obj.name) / "cleaned"
+        rc, _, err = self.mod.run(
+            ["/bin/sh", "-c", f'trap "echo yes > {marker}; exit 0" TERM; while :; do sleep 0.1; done'],
+            timeout=1)
+        self.assertEqual(rc, 124)
+        self.assertEqual(marker.read_text().strip(), "yes")
+
+    def test_group_members_that_ignore_sigterm_die_even_after_the_child_exits(self):
+        pid_file = Path(self._tmp_obj.name) / "stubborn.pid"
+        script = (f'(trap "" TERM; exec sleep 30) >/dev/null 2>&1 & echo $! > {pid_file}; '
+                  'trap "exit 0" TERM; while :; do sleep 0.1; done')
+        rc, _, _ = self.mod.run(["/bin/sh", "-c", script], timeout=1)
+        self.assertEqual(rc, 124)
+        gpid = int(pid_file.read_text().strip())
+        import time as _time
+        for _ in range(50):
+            try:
+                os.kill(gpid, 0)
+            except ProcessLookupError:
+                break
+            _time.sleep(0.1)
+        else:
+            os.kill(gpid, 9)
+            self.fail("a group member that ignored SIGTERM survived")
+
+    def test_timeout_kills_the_child_even_if_sigterm_cant_be_sent(self):
+        real_killpg = os.killpg
+
+        def no_term(pid, sig):
+            if sig == signal.SIGTERM:
+                raise ProcessLookupError
+            real_killpg(pid, sig)
+
+        with unittest.mock.patch.object(self.mod.os, "killpg", no_term):
+            rc, _, err = self.mod.run([sys.executable, "-c", "import time; time.sleep(30)"],
+                                      timeout=1)
+        self.assertEqual(rc, 124)
+
+    def test_timeout_still_kills_a_child_that_ignores_sigterm(self):
+        import time as _time
+        t0 = _time.time()
+        rc, _, err = self.mod.run(["/bin/sh", "-c", 'trap "" TERM; sleep 30'], timeout=1)
+        self.assertEqual(rc, 124)
+        self.assertLess(_time.time() - t0, 1 + self.mod.KILL_GRACE_SEC + 5)
+
     def test_input_text_still_reaches_stdin(self):
         # The Popen rewrite must preserve the input_text contract.
         rc, out, _ = self.mod.run(