From 1939785d21c5105859698435834293b7bb56c9cc Mon Sep 17 00:00:00 2001 From: Kun Chen <3233006+kunchenguid@users.noreply.github.com> Date: Tue, 4 Aug 2026 12:31:31 -0700 Subject: [PATCH 01/12] fix(bin): bound remote SSH dead-peer detection (#1699) * fix(remote): arm SSH dead-peer detection in fm-on.sh A vanished remote host mid-poll (a reboot, a dropped link) left ssh blocked indefinitely on a half-open TCP connection, because fm-on.sh's ssh invocation had no ServerAliveInterval/ServerAliveCountMax. This wedged the remote-reply ferry: fm-procevent.sh's runner blocked inside the ssh child and never reached its own no-result -> claim-release -> reconcile re-arm self-healing path, which otherwise already handles a nonzero exit with empty output correctly. Recovery required a manual retire and re-arm. Arm ServerAliveInterval=15 and ServerAliveCountMax=3 by default (bounded ~45s detection window), both overridable via FM_SSH_ALIVE_INTERVAL and FM_SSH_ALIVE_COUNT_MAX. This is a transport- level fix in fm-on.sh, so it covers every remote command routed through it, not just the reply ferry. The remote sshd answers keepalive probes independently of whatever the remote command is doing, so a legitimately long-but-alive command (a 55s poll, a clone, the doctor) is never falsely killed - only a truly vanished peer trips it, turning that case into a bounded, detectable ssh failure (exit 255) instead of an indefinite hang. Extends tests/fm-on.test.sh with a behavioral regression asserting a bounded, positive ServerAliveInterval/ServerAliveCountMax on the real ssh argv captured through the FM_SSH_BIN process seam, plus coverage that both are env-overridable. * no-mistakes(document): Document SSH dead-peer detection ownership --- bin/fm-on.sh | 16 ++++++++++++++++ docs/remote-secondmates.md | 2 +- tests/fm-on.test.sh | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/bin/fm-on.sh b/bin/fm-on.sh index 8af8ced3e3..5e24f2cef1 100755 --- a/bin/fm-on.sh +++ b/bin/fm-on.sh @@ -21,6 +21,14 @@ # This command explicitly disables agent forwarding, forwarding setup, and # configured SendEnv patterns. The remote entrypoint executes the selected # command under an empty environment with only its fixed runtime values. +# +# ServerAliveInterval/ServerAliveCountMax arm dead-peer detection so a vanished +# peer (a reboot, a dropped link) becomes a bounded ssh failure (exit 255) +# instead of an indefinite hang on a half-open TCP connection. The remote +# sshd answers keepalive probes independently of whatever the remote command +# is doing, so a legitimately long-but-alive remote command is never falsely +# killed. FM_SSH_ALIVE_INTERVAL and FM_SSH_ALIVE_COUNT_MAX override the +# defaults; the worst-case detection window is roughly interval * count. set -eu SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -88,9 +96,17 @@ ROOT_B64=$(printf '%s' "$ROOT" | encode_base64) HOME_B64=$(printf '%s' "$HOME_PATH" | encode_base64) ARGV_B64=$(printf '%s\0' "$COMMAND" "$@" | encode_base64) SSH_BIN=${FM_SSH_BIN:-ssh} +ALIVE_INTERVAL=${FM_SSH_ALIVE_INTERVAL:-15} +ALIVE_COUNT_MAX=${FM_SSH_ALIVE_COUNT_MAX:-3} +case "$ALIVE_INTERVAL" in ''|*[!0-9]*) die "FM_SSH_ALIVE_INTERVAL must be a positive integer: $ALIVE_INTERVAL" ;; esac +case "$ALIVE_COUNT_MAX" in ''|*[!0-9]*) die "FM_SSH_ALIVE_COUNT_MAX must be a positive integer: $ALIVE_COUNT_MAX" ;; esac +[ "$ALIVE_INTERVAL" -gt 0 ] || die "FM_SSH_ALIVE_INTERVAL must be a positive integer: $ALIVE_INTERVAL" +[ "$ALIVE_COUNT_MAX" -gt 0 ] || die "FM_SSH_ALIVE_COUNT_MAX must be a positive integer: $ALIVE_COUNT_MAX" "$SSH_BIN" \ -o ForwardAgent=no \ -o ClearAllForwardings=yes \ -o 'SendEnv=-*' \ + -o "ServerAliveInterval=$ALIVE_INTERVAL" \ + -o "ServerAliveCountMax=$ALIVE_COUNT_MAX" \ -- "$HOST" fm-remote-entrypoint.sh "$PROTOCOL" "$ROOT_B64" "$HOME_B64" "$ARGV_B64" diff --git a/docs/remote-secondmates.md b/docs/remote-secondmates.md index fcd5e83874..eebe083f68 100644 --- a/docs/remote-secondmates.md +++ b/docs/remote-secondmates.md @@ -15,7 +15,7 @@ Local second mates are unaffected and keep their ordinary backend and session se Configure an SSH alias in the primary account's normal OpenSSH configuration. Use ordinary public-key authentication, strict host-key verification, and a dedicated remote account where practical. Do not enable agent forwarding for Firstmate. -`fm-on.sh` also disables agent forwarding, forwarding setup, and configured `SendEnv` patterns on every call. +`fm-on.sh` also disables agent forwarding, forwarding setup, and configured `SendEnv` patterns on every call, and arms bounded SSH dead-peer detection so a vanished host (a reboot, a dropped link) fails within a bounded window instead of hanging indefinitely; its [script header](../bin/fm-on.sh) owns the keepalive defaults and environment overrides. Clone Firstmate on the remote host at an absolute code-root path. Expose that clone's fixed entrypoint on the account's non-interactive SSH `PATH`, for example: diff --git a/tests/fm-on.test.sh b/tests/fm-on.test.sh index 0a951f4ae1..178c3df828 100755 --- a/tests/fm-on.test.sh +++ b/tests/fm-on.test.sh @@ -141,6 +141,44 @@ assert_grep 'stderr: separate' "$TMP_ROOT/stderr" "remote stderr was not preserv assert_absent /tmp/fm-on-injected "shell-looking argv was interpreted" pass "fm-on preserves argv, stdin, stdout, stderr, and exit status without shell interpretation" +# A vanished remote peer must become a bounded ssh failure instead of an +# indefinite hang on a half-open TCP connection, so the existing no-result -> +# reconcile re-arm recovery can self-heal without manual intervention. Assert +# this on the real ssh argv the FM_SSH_BIN process seam captured, never on +# fm-on.sh source text. +LAST_SSH_ARGV=$(tail -n 1 "$SSH_LOG") +DEFAULT_INTERVAL=$(printf '%s\n' "$LAST_SSH_ARGV" | grep -oE 'ServerAliveInterval=[0-9]+' | cut -d= -f2) +DEFAULT_COUNT=$(printf '%s\n' "$LAST_SSH_ARGV" | grep -oE 'ServerAliveCountMax=[0-9]+' | cut -d= -f2) +[ -n "$DEFAULT_INTERVAL" ] || fail "the ssh transport did not arm ServerAliveInterval dead-peer detection" +[ -n "$DEFAULT_COUNT" ] || fail "the ssh transport did not arm ServerAliveCountMax dead-peer detection" +[ "$DEFAULT_INTERVAL" -gt 0 ] || fail "ServerAliveInterval was not a positive interval (got $DEFAULT_INTERVAL)" +[ "$DEFAULT_COUNT" -gt 0 ] || fail "ServerAliveCountMax was not a positive count (got $DEFAULT_COUNT)" +DEFAULT_WINDOW=$((DEFAULT_INTERVAL * DEFAULT_COUNT)) +[ "$DEFAULT_WINDOW" -le 120 ] \ + || fail "the default dead-peer detection window is not bounded to a sane ceiling (got ${DEFAULT_WINDOW}s = ${DEFAULT_INTERVAL}s x $DEFAULT_COUNT)" +pass "fm-on arms a bounded SSH dead-peer detection window by default (${DEFAULT_INTERVAL}s x $DEFAULT_COUNT = ${DEFAULT_WINDOW}s)" + +: > "$SSH_LOG" +FM_SSH_ALIVE_INTERVAL=7 FM_SSH_ALIVE_COUNT_MAX=2 fm_on ios fm-probe-two.sh >/dev/null +OVERRIDE_ARGV=$(tail -n 1 "$SSH_LOG") +assert_contains "$OVERRIDE_ARGV" 'ServerAliveInterval=7' "FM_SSH_ALIVE_INTERVAL override was not honored on the ssh transport" +assert_contains "$OVERRIDE_ARGV" 'ServerAliveCountMax=2' "FM_SSH_ALIVE_COUNT_MAX override was not honored on the ssh transport" +pass "fm-on's dead-peer detection window is env-overridable" + +SSH_CALLS_BEFORE_INVALID=$(cat "$SSH_COUNT") +set +e +INVALID_INTERVAL_OUT=$(FM_SSH_ALIVE_INTERVAL=0 fm_on ios fm-probe-two.sh 2>&1) +INVALID_INTERVAL_RC=$? +INVALID_COUNT_OUT=$(FM_SSH_ALIVE_COUNT_MAX=not-a-number fm_on ios fm-probe-two.sh 2>&1) +INVALID_COUNT_RC=$? +set -e +[ "$INVALID_INTERVAL_RC" -eq 1 ] || fail "a zero FM_SSH_ALIVE_INTERVAL was accepted (got exit $INVALID_INTERVAL_RC)" +[ "$INVALID_COUNT_RC" -eq 1 ] || fail "a non-integer FM_SSH_ALIVE_COUNT_MAX was accepted (got exit $INVALID_COUNT_RC)" +assert_contains "$INVALID_INTERVAL_OUT" 'FM_SSH_ALIVE_INTERVAL must be a positive integer' "invalid interval did not explain its constraint" +assert_contains "$INVALID_COUNT_OUT" 'FM_SSH_ALIVE_COUNT_MAX must be a positive integer' "invalid count did not explain its constraint" +[ "$(cat "$SSH_COUNT")" -eq "$SSH_CALLS_BEFORE_INVALID" ] || fail "invalid keepalive configuration launched ssh" +pass "fm-on rejects invalid dead-peer settings before launching ssh" + out=$(TOP_SECRET='must-not-cross' fm_on remote-mac fm-probe-two.sh) assert_contains "$out" "home=$REMOTE_HOME" "remote FM_HOME was not explicit" assert_contains "$out" "root=$REMOTE_ROOT" "remote root was not explicit" From 4a9979a33494a64d5e076b1241a476e1510faba0 Mon Sep 17 00:00:00 2001 From: Kun Chen <3233006+kunchenguid@users.noreply.github.com> Date: Tue, 4 Aug 2026 13:38:39 -0700 Subject: [PATCH 02/12] fix: report stale AXI tools during bootstrap (#1701) * feat(bootstrap): gate stale axi CLIs at the floors firstmate actually uses Add gh-axi 0.1.29 floor so bare --squash PR merges stop failing quietly on older builds. Raise tasks-axi to FM_TASKS_AXI_MIN=0.2.2 (multi-id mv) while keeping feature probes. Keep quota-axi at 0.1.16 after verifying schema 3 and per-model availability already ship there; runway remains optional. * no-mistakes(document): Clarify AXI compatibility documentation ownership --- .agents/skills/bootstrap-diagnostics/SKILL.md | 3 +- bin/fm-bootstrap.sh | 9 +- bin/fm-quota-axi-lib.sh | 4 + bin/fm-tasks-axi-lib.sh | 32 +++-- docs/configuration.md | 7 +- tests/fm-bootstrap.test.sh | 115 ++++++++++++++++-- tests/fm-secondmate-harness.test.sh | 30 ++++- tests/fm-secondmate-liveness.test.sh | 17 ++- tests/fm-secondmate-sync.test.sh | 13 +- tests/fm-session-start.test.sh | 11 +- tests/fm-shared-captain-inheritance.test.sh | 45 ++++++- tests/fm-startup-memory-budget.test.sh | 9 +- tests/fm-teardown.test.sh | 2 +- tests/fm-x-mode.test.sh | 11 +- 14 files changed, 269 insertions(+), 39 deletions(-) diff --git a/.agents/skills/bootstrap-diagnostics/SKILL.md b/.agents/skills/bootstrap-diagnostics/SKILL.md index daa0c4e421..0169ced269 100644 --- a/.agents/skills/bootstrap-diagnostics/SKILL.md +++ b/.agents/skills/bootstrap-diagnostics/SKILL.md @@ -19,7 +19,8 @@ When any diagnostic needs captain attention, report the plain consequence and re - `MISSING: (install: )` - list the missing tools to the captain with a one-line purpose each plus the printed install commands, wait for consent (one approval may cover the list), then run `bin/fm-bootstrap.sh install `. For `treehouse`, this also covers an installed version whose `treehouse get` lacks `--lease`; treat it as an upgrade request. For `no-mistakes`, this also covers an installed version older than 1.31.2, because crewmate validation briefs delegate gate mechanics to no-mistakes' version-matched guidance. - For `tasks-axi`, this also covers an installed build that fails the compatibility probe (`docs/configuration.md` "Backlog backend" owns the definition); `config/backlog-backend=manual` only suppresses the verbose `BOOTSTRAP_INFO: tasks-axi available` fact, not this missing-tool report. + For `gh-axi`, this also covers an installed version below the bootstrap-owned floor; treat it as an upgrade request so non-interactive PR merges keep a working bare `--squash` shorthand. + For `tasks-axi`, this also covers an installed build that fails the compatibility probe (`bin/fm-tasks-axi-lib.sh` owns the definition); `config/backlog-backend=manual` only suppresses the verbose `BOOTSTRAP_INFO: tasks-axi available` fact, not this missing-tool report. For `quota-axi`, bootstrap requires it because firstmate reads its current output directly before resolving every crew-dispatch profile array; without it, report the missing requirement and do not choose around an unexamined candidate. - `MISSING_MANUAL: (instructions: )` - tell the captain why the tool is required and give them the printed instructions URL, but do not pass the tool to `bin/fm-bootstrap.sh install`; wait for the captain to complete the manual installation, then rerun session start to confirm the dependency is present. - `BACKEND_INVALID: (known: )` - the resolved runtime backend has no verified dependency or lifecycle contract, so do not dispatch work until the invalid `FM_BACKEND` or `config/backend` value is corrected to one of the listed backends. diff --git a/bin/fm-bootstrap.sh b/bin/fm-bootstrap.sh index d3d3bb0789..9b6bc031bb 100755 --- a/bin/fm-bootstrap.sh +++ b/bin/fm-bootstrap.sh @@ -52,8 +52,11 @@ # "treehouse get --lease" support. # no-mistakes is also MISSING when its installed version is older than # 1.31.2. +# gh-axi is also MISSING when its installed version is older than +# 0.1.29, the first release whose bare --squash shorthand works for +# firstmate's non-interactive PR merge path. # tasks-axi and quota-axi are required bootstrap tools (same class as -# lavish-axi). tasks-axi is also version and feature gated (0.1.1+ +# lavish-axi). tasks-axi is also version and feature gated (0.2.2+ # with update --archive-body and mv [...]); an installed but # incompatible build reports MISSING like no-mistakes. A compatible # tasks-axi default backend is silent. quota-axi is required for the @@ -690,6 +693,7 @@ if ! BACKEND_TOOLS=$(fm_backend_required_tools "$BACKEND"); then fi TOOLS="$BACKEND_TOOLS $COMMON_TOOLS" NO_MISTAKES_MIN=1.31.2 +GH_AXI_MIN=0.1.29 treehouse_supports_lease() { treehouse get --help 2>&1 | grep -Eq '(^|[^[:alnum:]_-])--lease([^[:alnum:]_-]|$)' @@ -1029,6 +1033,9 @@ fi if command -v no-mistakes >/dev/null 2>&1 && ! tool_version_at_least no-mistakes "$NO_MISTAKES_MIN"; then echo "MISSING: no-mistakes (install: $(install_cmd no-mistakes))" fi +if command -v gh-axi >/dev/null 2>&1 && ! tool_version_at_least gh-axi "$GH_AXI_MIN"; then + echo "MISSING: gh-axi (install: $(install_cmd gh-axi))" +fi if command -v quota-axi >/dev/null 2>&1 && ! fm_quota_axi_compatible; then echo "MISSING: quota-axi (install: $(install_cmd quota-axi))" fi diff --git a/bin/fm-quota-axi-lib.sh b/bin/fm-quota-axi-lib.sh index 441c9ce2c9..98bb30c117 100644 --- a/bin/fm-quota-axi-lib.sh +++ b/bin/fm-quota-axi-lib.sh @@ -7,6 +7,10 @@ # those fields a dispatch candidate cannot be checked against the authentication # surface it actually uses, which is how one harness's expired CLI token used to # produce a captain-facing sign-out claim for a candidate that never read it. +# 0.1.16 already emits schemaVersion 3 with per-model effectiveAvailability; +# 0.1.17 only adds optional runway under that same schema. quota-array-dispatch +# treats absent runway or pace as disclosed uncertainty, so the floor stays +# 0.1.16 rather than tracking the latest additive field. # # This file is the single owner of that version number. bin/fm-bootstrap.sh # turns a failing check into the operator-facing MISSING diagnostic, which is diff --git a/bin/fm-tasks-axi-lib.sh b/bin/fm-tasks-axi-lib.sh index 54d02fcc9e..0fe9f4ad09 100644 --- a/bin/fm-tasks-axi-lib.sh +++ b/bin/fm-tasks-axi-lib.sh @@ -2,14 +2,24 @@ # Shared tasks-axi backend selection and compatibility probe for bootstrap, # teardown, and secondmate backlog handoff. # Usage: . bin/fm-tasks-axi-lib.sh -# Compatible means tasks-axi --version reports 0.1.1 or newer, +# +# Compatible means tasks-axi --version reports FM_TASKS_AXI_MIN or newer, # `tasks-axi update --help` exposes --archive-body for recoverable note rewrites, # and `tasks-axi mv --help` exposes [...] for atomic multi-ID moves required # by secondmate handoffs (introduced in tasks-axi 0.2.2). +# 0.2.2 is the floor because multi-ID mv is the true minimum firstmate uses; +# earlier builds could pass a 0.1.1 version check and still fail handoff. +# Feature probes stay as defense in depth for stripped or forked builds that +# advertise a current version without those flags. # `config/backlog-backend=manual` opts out of tasks-axi for routine firstmate # backlog mutations, but validated secondmate handoffs always use `tasks-axi mv`. # Absent or any other value keeps the default tasks-axi backend path, falling # back to manual mutation when the tool is not compatible. +# +# This file is the single owner of FM_TASKS_AXI_MIN. bin/fm-bootstrap.sh turns a +# failing check into the operator-facing MISSING diagnostic. + +FM_TASKS_AXI_MIN=0.2.2 fm_tasks_axi_version_parts() { local output @@ -21,17 +31,19 @@ fm_tasks_axi_version_parts() { } fm_tasks_axi_compatible() { - local parts major minor patch rest + local parts major minor patch extra + local min_major min_minor min_patch min_extra parts=$(fm_tasks_axi_version_parts) || return 1 [ -n "$parts" ] || return 1 - major=${parts%% *} - rest=${parts#* } - minor=${rest%% *} - patch=${rest##* } - - if [ "$major" -gt 0 ] || - { [ "$major" -eq 0 ] && [ "$minor" -gt 1 ]; } || - { [ "$major" -eq 0 ] && [ "$minor" -eq 1 ] && [ "$patch" -ge 1 ]; }; then + IFS=' ' read -r major minor patch extra <<< "$parts" + # An unparseable version is incompatible, never assumed current, so a + # development or vendored build cannot pass a floor it was never checked against. + [ -n "$major" ] && [ -n "$minor" ] && [ -n "$patch" ] && [ -z "$extra" ] || return 1 + IFS='.' read -r min_major min_minor min_patch min_extra <<< "$FM_TASKS_AXI_MIN" + [ -n "$min_major" ] && [ -n "$min_minor" ] && [ -n "$min_patch" ] && [ -z "$min_extra" ] || return 1 + if [ "$major" -gt "$min_major" ] || + { [ "$major" -eq "$min_major" ] && [ "$minor" -gt "$min_minor" ]; } || + { [ "$major" -eq "$min_major" ] && [ "$minor" -eq "$min_minor" ] && [ "$patch" -ge "$min_patch" ]; }; then fm_tasks_axi_update_has_archive_body && fm_tasks_axi_mv_has_multi_id return $? fi diff --git a/docs/configuration.md b/docs/configuration.md index 462b77038d..02bcb12858 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -39,8 +39,7 @@ Secondmate handoffs are separate and unconditional: `fm-backlog-handoff.sh` keep It moves in-scope `## Queued` items only and refuses `## In flight` and historical `## Done` records, which stay with their home for pruning or archiving. Handoff item bodies must use at least two leading spaces, and the helper refuses a selected item with a single-space or tab-indented continuation rather than risk orphaning it. Because bootstrap requires `tasks-axi` on `PATH` on every profile, that delegation works fleet-wide, and the `config/backlog-backend=manual` knob governs firstmate's own hand-editing of its backlog, not this validated helper. -Compatible means the shared bootstrap probe accepts `tasks-axi --version` as 0.1.1 or newer, `tasks-axi update --help` exposes `--archive-body`, and `tasks-axi mv --help` exposes `[...]` for the atomic multi-ID move introduced in 0.2.2 and required by handoff delegation. -That sentence is the single owner of the tasks-axi compatibility definition; every other document points here instead of restating the version gates. +Compatible means the installed build passes the shared version and feature probe owned by [`bin/fm-tasks-axi-lib.sh`](../bin/fm-tasks-axi-lib.sh), including the atomic multi-ID move required by handoff delegation. Bootstrap requires compatible `tasks-axi` on every profile; see "Toolchain" below for missing-tool reporting and silent default-backend behavior. Set the local, gitignored `config/backlog-backend` file to `manual` to force manual backlog editing and suppress the verbose `BOOTSTRAP_INFO: tasks-axi available` fact, not missing-tool reporting. Absent or `tasks-axi` selects the default tasks-axi backend. @@ -284,7 +283,8 @@ Secondmate homes inherit this file from the primary, so a secondmate's own crewm On session start the first mate detects what its required toolchain is missing or too old and lists each problem with either an exact install command or manual instructions. It installs automatically supported tools only after you say go; manual-only tools remain for you to install from the printed instructions. Required tools come in two parts: a universal toolchain every home needs regardless of backend, and a per-backend delta that follows the runtime backend actually resolved for this home. -The universal toolchain is node, git, gh with GitHub auth via `gh auth login`, no-mistakes v1.31.2 or newer, gh-axi, chrome-devtools-axi, lavish-axi, compatible tasks-axi per "Backlog backend" above, and quota-axi v0.1.16 or newer. +The universal toolchain is node, git, gh with GitHub auth via `gh auth login`, no-mistakes v1.31.2 or newer, compatible gh-axi, chrome-devtools-axi, lavish-axi, compatible tasks-axi per "Backlog backend" above, and compatible quota-axi. +The exact gh-axi floor is owned inline by [`bin/fm-bootstrap.sh`](../bin/fm-bootstrap.sh), while [`bin/fm-tasks-axi-lib.sh`](../bin/fm-tasks-axi-lib.sh) and [`bin/fm-quota-axi-lib.sh`](../bin/fm-quota-axi-lib.sh) own their tools' compatibility floors and rationale. This section is the single owner of that universal toolchain list; backend guides' prerequisites point here and add only their backend-specific tools. In that list, no-mistakes runs the validation pipeline, gh-axi, chrome-devtools-axi, and lavish-axi cover GitHub, browser, and rich-review operations, and tasks-axi plus quota-axi back backlog mutations and quota-aware array dispatch. The per-backend delta is required only for the backend resolved from `FM_BACKEND`, then `config/backend`, then runtime auto-detection, then default `tmux`, so a home is never told to install a tool an inactive backend or feature would need. @@ -297,6 +297,7 @@ When `config/crew-dispatch.json` exists, bootstrap also requires `jq` for dispat When X mode is opted in, bootstrap also requires `curl` and `jq` before arming the relay poll shim. `tasks-axi` and `quota-axi` are required bootstrap tools in every profile, the same class as `lavish-axi`. An absent or incompatible `tasks-axi` reports `MISSING: tasks-axi (install: npm install -g tasks-axi)`; when `config/backlog-backend` is not `manual` and compatible `tasks-axi` is on `PATH`, bootstrap stays silent and firstmate uses its verbs for routine backlog mutations, otherwise it hand-edits `data/backlog.md` until installation is approved and completed. +An absent or incompatible `gh-axi` reports `MISSING: gh-axi (install: npm install -g gh-axi && gh-axi setup hooks)`. An absent or too-old `quota-axi` reports `MISSING: quota-axi (install: npm install -g quota-axi)`; firstmate cannot resolve a profile array without a compatible binary. That floor exists because it is the first build reporting per-credential auth sources, without which a candidate cannot be judged against the authentication surface it actually uses. Bootstrap also reports a `TANGLE:` line when `FM_ROOT` is on a named non-default branch; follow the printed checkout remediation rather than treating it as an installable tool problem. diff --git a/tests/fm-bootstrap.test.sh b/tests/fm-bootstrap.test.sh index 6c72c0bbc4..5d721ae055 100755 --- a/tests/fm-bootstrap.test.sh +++ b/tests/fm-bootstrap.test.sh @@ -4,14 +4,15 @@ # Bootstrap prints one block or line per actionable problem, optional verbose # BOOTSTRAP_INFO fact, or completed bootstrap no-action fact and is silent when # all is well. firstmate consumes the exact 'MISSING: treehouse (install: ...)', -# 'MISSING: tasks-axi (install: ...)', 'MISSING: quota-axi (install: ...)', and +# 'MISSING: tasks-axi (install: ...)', 'MISSING: quota-axi (install: ...)', +# 'MISSING: gh-axi (install: ...)', and # 'BOOTSTRAP_INFO: ...' lines, so those contracts are pinned verbatim. The cases # are table-driven over the inputs that vary: whether `treehouse get --help` # advertises --lease, which (if any) tasks-axi version is on PATH, whether # tasks-axi update advertises --archive-body, whether its mv help advertises # multi-ID moves, whether quota-axi is on PATH, -# whether the local backend config opts out of tasks-axi backlog mutations, and -# which no-mistakes version is on PATH. +# whether the local backend config opts out of tasks-axi backlog mutations, +# which no-mistakes version is on PATH, and which gh-axi version is on PATH. # Dedicated fleet-sync cases pin the computed bootstrap timeout, explicit # override, blank-env defaulting, partial-output relay, and pre-launch timeout # scan. @@ -38,7 +39,16 @@ unset TMUX TMUX_PANE HERDR_ENV HERDR_PANE_ID HERDR_SESSION HERDR_SOCKET_PATH \ make_fake_toolchain() { local dir=$1 fakebin fakebin=$(fm_fakebin "$dir") - fm_fake_exit0 "$fakebin" tmux node gh-axi chrome-devtools-axi lavish-axi + fm_fake_exit0 "$fakebin" tmux node chrome-devtools-axi lavish-axi + cat > "$fakebin/gh-axi" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = --version ]; then + printf '%s\n' "${FM_FAKE_GH_AXI_VERSION:-0.1.29}" + exit 0 +fi +exit 0 +SH + chmod +x "$fakebin/gh-axi" cat > "$fakebin/gh" <<'SH' #!/usr/bin/env bash if [ "${1:-}" = auth ] && [ "${2:-}" = status ]; then @@ -69,7 +79,7 @@ fi exit 0 SH chmod +x "$fakebin/no-mistakes" - add_tasks_axi "$fakebin" "0.1.1" + add_tasks_axi "$fakebin" "0.2.2" add_quota_axi "$fakebin" printf '%s\n' "$fakebin" } @@ -281,16 +291,16 @@ test_bootstrap_reporting() { ;; esac done <<'ROWS' -treehouse --lease support is accepted silently^1^0.1.1^1^manual^empty^^ -treehouse without --lease reports an upgrade, gh auth is fine^0^0.1.1^1^-^grep^MISSING: treehouse (install: curl -fsSL https://kunchenguid.github.io/treehouse/install.sh | sh)^NEEDS_GH_AUTH -compatible tasks-axi is silent by default^1^0.1.1^1^-^empty^^ +treehouse --lease support is accepted silently^1^0.2.2^1^manual^empty^^ +treehouse without --lease reports an upgrade, gh auth is fine^0^0.2.2^1^-^grep^MISSING: treehouse (install: curl -fsSL https://kunchenguid.github.io/treehouse/install.sh | sh)^NEEDS_GH_AUTH +compatible tasks-axi is silent by default^1^0.2.2^1^-^empty^^ missing tasks-axi is required by default^1^-^1^-^exact^MISSING: tasks-axi (install: npm install -g tasks-axi)^ incompatible tasks-axi is required by default^1^0.1.0^1^-^exact^MISSING: tasks-axi (install: npm install -g tasks-axi)^ -tasks-axi without archive-body is required by default^1^0.1.2:noarchive^1^-^exact^MISSING: tasks-axi (install: npm install -g tasks-axi)^ +tasks-axi without archive-body is required by default^1^0.2.2:noarchive^1^-^exact^MISSING: tasks-axi (install: npm install -g tasks-axi)^ tasks-axi without multi-id mv is required by default^1^0.2.2:nomulti^1^-^exact^MISSING: tasks-axi (install: npm install -g tasks-axi)^ -missing quota-axi is required by default^1^0.1.1^0^manual^exact^MISSING: quota-axi (install: npm install -g quota-axi)^ +missing quota-axi is required by default^1^0.2.2^0^manual^exact^MISSING: quota-axi (install: npm install -g quota-axi)^ manual backlog backend still requires missing tasks-axi^1^-^1^manual^exact^MISSING: tasks-axi (install: npm install -g tasks-axi)^ -manual backlog backend suppresses tasks-axi availability^1^0.1.1^1^manual^empty^^ +manual backlog backend suppresses tasks-axi availability^1^0.2.2^1^manual^empty^^ ROWS pass "bootstrap reports treehouse lease + tasks-axi/quota-axi bootstrap contracts" } @@ -307,7 +317,6 @@ test_no_mistakes_min_version() { mkdir -p "$case_dir/home/config" printf '%s\n' manual > "$case_dir/home/config/backlog-backend" fakebin=$(make_fake_toolchain "$case_dir") - add_tasks_axi "$fakebin" "0.1.1" out=$(PATH="$fakebin:$BASE_PATH" FM_HOME="$case_dir/home" FM_ROOT_OVERRIDE="$case_dir/home" \ FM_FAKE_TREEHOUSE_LEASE_HELP=1 FM_FAKE_NO_MISTAKES_VERSION="$version" "$ROOT/bin/fm-bootstrap.sh") case "$mode" in @@ -326,6 +335,85 @@ ROWS pass "bootstrap enforces no-mistakes minimum version" } +test_gh_axi_min_version() { + local label version mode case_dir fakebin out missing n + missing='MISSING: gh-axi (install: npm install -g gh-axi && gh-axi setup hooks)' + n=0 + while IFS='^' read -r label version mode; do + [ -n "$label" ] || continue + n=$((n + 1)) + case_dir="$TMP_ROOT/gh-axi-$n" + mkdir -p "$case_dir/home/config" + printf '%s\n' manual > "$case_dir/home/config/backlog-backend" + fakebin=$(make_fake_toolchain "$case_dir") + out=$(PATH="$fakebin:$BASE_PATH" FM_HOME="$case_dir/home" FM_ROOT_OVERRIDE="$case_dir/home" \ + FM_FAKE_TREEHOUSE_LEASE_HELP=1 FM_FAKE_GH_AXI_VERSION="$version" "$ROOT/bin/fm-bootstrap.sh") + case "$mode" in + empty) + [ -z "$out" ] || fail "$label: expected silence, got: $out" ;; + missing) + [ "$out" = "$missing" ] || fail "$label: expected '$missing', got: $out" ;; + esac + done <<'ROWS' +minimum gh-axi version is accepted^0.1.29^empty +newer gh-axi patch is accepted^0.1.30^empty +newer gh-axi minor is accepted^0.2.0^empty +newer gh-axi major is accepted^1.0.0^empty +older gh-axi patch reports an upgrade^0.1.19^missing +much older gh-axi minor reports an upgrade^0.0.9^missing +unparseable gh-axi version reports an upgrade^gh-axi development build^missing +ROWS + pass "bootstrap enforces gh-axi minimum version" +} + +test_tasks_axi_min_version() { + local label version mode case_dir fakebin out missing n archive_body multi_id + missing='MISSING: tasks-axi (install: npm install -g tasks-axi)' + n=0 + while IFS='^' read -r label version mode; do + [ -n "$label" ] || continue + n=$((n + 1)) + case_dir="$TMP_ROOT/tasks-axi-$n" + mkdir -p "$case_dir/home/config" + printf '%s\n' manual > "$case_dir/home/config/backlog-backend" + fakebin=$(make_fake_toolchain "$case_dir") + archive_body=yes + multi_id=yes + case "$version" in + *:noarchive) + archive_body=no + version=${version%:noarchive} + ;; + esac + case "$version" in + *:nomulti) + multi_id=no + version=${version%:nomulti} + ;; + esac + add_tasks_axi "$fakebin" "$version" "$archive_body" "$multi_id" + out=$(PATH="$fakebin:$BASE_PATH" FM_HOME="$case_dir/home" FM_ROOT_OVERRIDE="$case_dir/home" \ + FM_FAKE_TREEHOUSE_LEASE_HELP=1 "$ROOT/bin/fm-bootstrap.sh") + case "$mode" in + empty) + [ -z "$out" ] || fail "$label: expected silence, got: $out" ;; + missing) + [ "$out" = "$missing" ] || fail "$label: expected '$missing', got: $out" ;; + esac + done <<'ROWS' +minimum tasks-axi version is accepted^0.2.2^empty +newer tasks-axi patch is accepted^0.2.3^empty +newer tasks-axi minor is accepted^0.3.0^empty +newer tasks-axi major is accepted^1.0.0^empty +older tasks-axi with features reports an upgrade^0.1.1^missing +pre-multi-id tasks-axi reports an upgrade^0.2.1^missing +unparseable tasks-axi version reports an upgrade^tasks-axi development build^missing +tasks-axi at floor without archive-body reports an upgrade^0.2.2:noarchive^missing +tasks-axi at floor without multi-id reports an upgrade^0.2.2:nomulti^missing +ROWS + pass "bootstrap enforces tasks-axi minimum version" +} + # 0.1.16 is the first quota-axi that reports per-credential auth sources and Grok # state.authStatus. Before it, a dispatch candidate could not be scoped to its own # authentication surface, which is exactly how one harness's expired CLI token @@ -342,7 +430,6 @@ test_quota_axi_min_version() { mkdir -p "$case_dir/home/config" printf '%s\n' manual > "$case_dir/home/config/backlog-backend" fakebin=$(make_fake_toolchain "$case_dir") - add_tasks_axi "$fakebin" "0.1.1" out=$(PATH="$fakebin:$BASE_PATH" FM_HOME="$case_dir/home" FM_ROOT_OVERRIDE="$case_dir/home" \ FM_FAKE_TREEHOUSE_LEASE_HELP=1 FM_FAKE_QUOTA_AXI_VERSION="$version" "$ROOT/bin/fm-bootstrap.sh") case "$mode" in @@ -834,6 +921,8 @@ ROWS test_bootstrap_reporting test_no_mistakes_min_version +test_gh_axi_min_version +test_tasks_axi_min_version test_quota_axi_min_version test_git_is_required_with_supported_install_instruction test_orca_backend_gates_orca_tool_only_when_selected diff --git a/tests/fm-secondmate-harness.test.sh b/tests/fm-secondmate-harness.test.sh index 1220ad8c3f..e65109c2c4 100755 --- a/tests/fm-secondmate-harness.test.sh +++ b/tests/fm-secondmate-harness.test.sh @@ -938,7 +938,16 @@ make_fake_toolchain() { local dir=$1 fakebin fakebin="$dir/fakebin" mkdir -p "$fakebin" - fm_fake_exit0 "$fakebin" node gh-axi chrome-devtools-axi lavish-axi + fm_fake_exit0 "$fakebin" node chrome-devtools-axi lavish-axi + cat > "$fakebin/gh-axi" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = --version ]; then + printf '%s\n' '0.1.29' + exit 0 +fi +exit 0 +SH + chmod +x "$fakebin/gh-axi" # tmux fake supports fm-send's composer-verified submit path and optional # FM_FAKE_TMUX_LOG / FM_FAKE_TMUX_FAIL_LITERAL for reread-nudge assertions. cat > "$fakebin/tmux" <<'SH' @@ -985,6 +994,25 @@ fi exit 0 SH chmod +x "$fakebin/no-mistakes" + cat > "$fakebin/tasks-axi" <<'SH' +#!/usr/bin/env bash +case "${1:-} ${2:-}" in + "--version ") printf '%s\n' '0.2.2' ;; + "update --help") printf '%s\n' 'usage: tasks-axi update [flags]' ' --archive-body' ;; + "mv --help") printf '%s\n' 'usage: tasks-axi mv [...] --to ' ;; +esac +exit 0 +SH + chmod +x "$fakebin/tasks-axi" + cat > "$fakebin/quota-axi" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = --version ]; then + printf '%s\n' '0.1.16' + exit 0 +fi +exit 0 +SH + chmod +x "$fakebin/quota-axi" printf '%s\n' "$fakebin" } diff --git a/tests/fm-secondmate-liveness.test.sh b/tests/fm-secondmate-liveness.test.sh index ed35663896..e9c5bc3fec 100755 --- a/tests/fm-secondmate-liveness.test.sh +++ b/tests/fm-secondmate-liveness.test.sh @@ -206,7 +206,16 @@ test_agent_state_dispatcher_and_compatibility() { make_toolchain() { local dir=$1 fakebin fakebin=$(fm_fakebin "$dir") - fm_fake_exit0 "$fakebin" node gh-axi chrome-devtools-axi lavish-axi pi-signed + fm_fake_exit0 "$fakebin" node chrome-devtools-axi lavish-axi pi-signed + cat > "$fakebin/gh-axi" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = --version ]; then + printf '%s\n' '0.1.29' + exit 0 +fi +exit 0 +SH + chmod +x "$fakebin/gh-axi" cat > "$fakebin/gh" <<'SH' #!/usr/bin/env bash exit 0 @@ -232,7 +241,7 @@ SH cat > "$fakebin/tasks-axi" <<'SH' #!/usr/bin/env bash case "${1:-} ${2:-}" in - "--version ") printf '%s\n' '0.1.1' ;; + "--version ") printf '%s\n' '0.2.2' ;; "update --help") printf '%s\n' 'usage: tasks-axi update [flags]' ' --archive-body' ;; "mv --help") printf '%s\n' 'usage: tasks-axi mv [...] --to ' ;; esac @@ -241,6 +250,10 @@ SH chmod +x "$fakebin/tasks-axi" cat > "$fakebin/quota-axi" <<'SH' #!/usr/bin/env bash +if [ "${1:-}" = --version ]; then + printf '%s\n' '0.1.16' + exit 0 +fi exit 0 SH chmod +x "$fakebin/quota-axi" diff --git a/tests/fm-secondmate-sync.test.sh b/tests/fm-secondmate-sync.test.sh index d67350a1e7..203ae1fb8e 100755 --- a/tests/fm-secondmate-sync.test.sh +++ b/tests/fm-secondmate-sync.test.sh @@ -291,7 +291,16 @@ make_fake_toolchain() { local dir=$1 fakebin fakebin="$dir/fakebin" mkdir -p "$fakebin" - fm_fake_exit0 "$fakebin" node gh-axi chrome-devtools-axi lavish-axi + fm_fake_exit0 "$fakebin" node chrome-devtools-axi lavish-axi + cat > "$fakebin/gh-axi" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = --version ]; then + printf '%s\n' '0.1.29' + exit 0 +fi +exit 0 +SH + chmod +x "$fakebin/gh-axi" cat > "$fakebin/tmux" <<'SH' #!/usr/bin/env bash if [ -n "${FM_FAKE_TMUX_LOG:-}" ]; then @@ -338,7 +347,7 @@ SH cat > "$fakebin/tasks-axi" <<'SH' #!/usr/bin/env bash case "${1:-} ${2:-}" in - "--version ") printf '%s\n' '0.1.1' ;; + "--version ") printf '%s\n' '0.2.2' ;; "update --help") printf '%s\n' 'usage: tasks-axi update [flags]' ' --archive-body' ;; "mv --help") printf '%s\n' 'usage: tasks-axi mv [...] --to ' ;; esac diff --git a/tests/fm-session-start.test.sh b/tests/fm-session-start.test.sh index 6af5c2f663..1d5eb9e6d6 100755 --- a/tests/fm-session-start.test.sh +++ b/tests/fm-session-start.test.sh @@ -59,7 +59,16 @@ new_world() { # test deliberately breaks one. Mirrors fm-bootstrap.test.sh's fixture. make_fake_toolchain() { local fakebin=$1 - fm_fake_exit0 "$fakebin" tmux node gh-axi chrome-devtools-axi lavish-axi + fm_fake_exit0 "$fakebin" tmux node chrome-devtools-axi lavish-axi + cat > "$fakebin/gh-axi" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = --version ]; then + printf '%s\n' '0.1.29' + exit 0 +fi +exit 0 +SH + chmod +x "$fakebin/gh-axi" cat > "$fakebin/gh" <<'SH' #!/usr/bin/env bash exit 0 diff --git a/tests/fm-shared-captain-inheritance.test.sh b/tests/fm-shared-captain-inheritance.test.sh index 88534b41be..bf45987030 100755 --- a/tests/fm-shared-captain-inheritance.test.sh +++ b/tests/fm-shared-captain-inheritance.test.sh @@ -216,6 +216,46 @@ SH printf '%s\n' "$fakebin" } +# Version-aware stubs so bootstrap's tool floors stay quiet in fixture PATH. +add_bootstrap_compatible_tools() { + local fakebin=$1 + fm_fake_exit0 "$fakebin" node chrome-devtools-axi lavish-axi gh treehouse + cat > "$fakebin/gh-axi" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = --version ]; then + printf '%s\n' '0.1.29' + exit 0 +fi +exit 0 +SH + cat > "$fakebin/no-mistakes" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = --version ]; then + printf '%s\n' 'no-mistakes version v1.31.2 (fake)' + exit 0 +fi +exit 0 +SH + cat > "$fakebin/tasks-axi" <<'SH' +#!/usr/bin/env bash +case "${1:-} ${2:-}" in + "--version ") printf '%s\n' '0.2.2' ;; + "update --help") printf '%s\n' 'usage: tasks-axi update [flags]' ' --archive-body' ;; + "mv --help") printf '%s\n' 'usage: tasks-axi mv [...] --to ' ;; +esac +exit 0 +SH + cat > "$fakebin/quota-axi" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = --version ]; then + printf '%s\n' '0.1.16' + exit 0 +fi +exit 0 +SH + chmod +x "$fakebin/gh-axi" "$fakebin/no-mistakes" "$fakebin/tasks-axi" "$fakebin/quota-axi" +} + new_git_world() { local name=$1 w root home c1 w="$TMP_ROOT/$name" @@ -286,7 +326,7 @@ EOF printf -- '- sm - fixture secondmate (home: %s; scope: fixture; projects: sample; added 2026-07-16)\n' "$sm" \ > "$data_override/secondmates.md" fakebin=$(make_fake_spawn_toolchain "$w") - fm_fake_exit0 "$fakebin" node gh-axi chrome-devtools-axi lavish-axi gh treehouse no-mistakes tasks-axi quota-axi + add_bootstrap_compatible_tools "$fakebin" out=$(PATH="$fakebin:$BASE_PATH" FM_HOME="$home" FM_ROOT_OVERRIDE="$root" \ FM_DATA_OVERRIDE="$data_override" \ @@ -336,7 +376,8 @@ test_session_start_digest_labels_shared_file_and_read_once_rule() { $rec EOF fakebin=$(make_fake_spawn_toolchain "$w") - fm_fake_exit0 "$fakebin" node gh-axi chrome-devtools-axi lavish-axi gh treehouse no-mistakes tasks-axi quota-axi pgrep + add_bootstrap_compatible_tools "$fakebin" + fm_fake_exit0 "$fakebin" pgrep out=$(PATH="$fakebin:$BASE_PATH" FM_HOME="$home" FM_ROOT_OVERRIDE="$root" \ "$ROOT/bin/fm-session-start.sh") diff --git a/tests/fm-startup-memory-budget.test.sh b/tests/fm-startup-memory-budget.test.sh index 59eeb0649b..6cb87a42d5 100755 --- a/tests/fm-startup-memory-budget.test.sh +++ b/tests/fm-startup-memory-budget.test.sh @@ -15,7 +15,14 @@ CONFIG_PUSH="$ROOT/bin/fm-config-push.sh" make_fake_toolchain() { local dir=$1 fakebin fakebin=$(fm_fakebin "$dir") - fm_fake_exit0 "$fakebin" node gh-axi chrome-devtools-axi lavish-axi quota-axi + fm_fake_exit0 "$fakebin" node chrome-devtools-axi lavish-axi + cat > "$fakebin/gh-axi" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = --version ]; then + printf '%s\n' '0.1.29' +fi +exit 0 +SH cat > "$fakebin/quota-axi" <<'SH' #!/usr/bin/env bash if [ "${1:-}" = --version ]; then diff --git a/tests/fm-teardown.test.sh b/tests/fm-teardown.test.sh index 23ef6b8af2..478ca5e442 100755 --- a/tests/fm-teardown.test.sh +++ b/tests/fm-teardown.test.sh @@ -133,7 +133,7 @@ add_compatible_tasks_axi() { cat > "$case_dir/fakebin/tasks-axi" <<'SH' #!/usr/bin/env bash if [ "${1:-}" = --version ]; then - printf '%s\n' '0.1.1' + printf '%s\n' '0.2.2' exit 0 fi if [ "${1:-}" = update ] && [ "${2:-}" = --help ]; then diff --git a/tests/fm-x-mode.test.sh b/tests/fm-x-mode.test.sh index 24cdb60393..ea47a08c26 100755 --- a/tests/fm-x-mode.test.sh +++ b/tests/fm-x-mode.test.sh @@ -721,7 +721,16 @@ test_bootstrap_reports_missing_x_dependency() { local home fakebin out tool tool_path home="$TMP_ROOT/boot-missing-x"; mkdir -p "$home" fakebin=$(fm_fakebin "$home") - fm_fake_exit0 "$fakebin" tmux node no-mistakes gh-axi chrome-devtools-axi lavish-axi curl + fm_fake_exit0 "$fakebin" tmux node no-mistakes chrome-devtools-axi lavish-axi curl + cat > "$fakebin/gh-axi" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = --version ]; then + printf '%s\n' '0.1.29' + exit 0 +fi +exit 0 +SH + chmod +x "$fakebin/gh-axi" for tool in dirname grep tail; do tool_path=$(command -v "$tool") || fail "test host must provide $tool" ln -s "$tool_path" "$fakebin/$tool" From d0461e4b489c518eb744430742af62d73e2a16d0 Mon Sep 17 00:00:00 2001 From: Christopher McKay <101884182+karotkriss@users.noreply.github.com> Date: Tue, 4 Aug 2026 19:14:27 -0400 Subject: [PATCH 03/12] fix: prevent false watcher-down alarms in Claude sessions (#1661) * fix(guard): stop false watcher-down alarm mid-turn under Claude auto-arm bin/fm-guard.sh derived its watcher-health verdict from fm_watcher_healthy, which requires a live watcher process holding the home lock. Under the Claude Stop-hook auto-arm supervision model the watcher is armed at each turn end and exits on its wake, so it runs only between turns. Every guarded command run mid-turn therefore found no live watcher and printed the "WATCHER DOWN - SUPERVISION IS OFF" banner even though supervision was healthy. Because the episode key was derived from the beacon mtime (which the between-turns watcher advances every poll), the full banner re-printed on essentially every command, and the message always blamed a "fresh beacon" that was in fact fresh. Make the pull guard's health check model-aware via a new fm_watcher_supervision_verdict in bin/fm-wake-lib.sh: - Under the auto-arm model a beacon fresh within FM_GUARD_GRACE is healthy even with no live watcher process; only a beacon stale beyond grace (or absent) is a genuine lapse and alarms. - Under every persistent-watcher harness (codex foreground checkpoint, opencode/pi/grok background arm, tmux, unknown) a live identity-matched watcher with a fresh beacon is still required, unchanged. The banner now names the true failing condition, a missing live watcher process versus a genuinely stale beacon, instead of always blaming the beacon, and the once-per-episode dedup keys on that condition rather than the beacon mtime so a genuine lapse announces once and does not re-print each turn. The turn-end guard keeps the strict fm_watcher_healthy check because it fires at the turn boundary, where the auto-arm brings a fresh watcher up and it cooperates with that arm. fm_watcher_healthy itself is unchanged, so the arm layer's start/attach/replace decisions are unaffected. Tests in tests/fm-guard-stale-banner.test.sh cover the auto-arm healthy fresh-beacon-without-a-watcher case, the auto-arm stale-beacon alarm and its stable episode, the true-reason banner wording, and the reason-keyed episode surviving a beacon mtime change; existing persistent-model cases are pinned to that model. * no-mistakes(review): Pin secondmate supervision model to launched harness * no-mistakes(document): Align watcher documentation with model-aware supervision health --- bin/fm-guard.sh | 56 +++++++++------- bin/fm-spawn.sh | 6 +- bin/fm-supervision-lib.sh | 9 ++- bin/fm-wake-drain.sh | 16 ++--- bin/fm-wake-lib.sh | 76 ++++++++++++++++++++++ docs/architecture.md | 6 +- docs/arm-pretool-check.md | 2 +- docs/scripts.md | 4 +- docs/sessionstart-nudge.md | 2 +- docs/supervision-protocols/claude.md | 2 +- docs/turnend-guard.md | 18 +++--- docs/verification/supervision.md | 16 +++++ tests/fm-guard-stale-banner.test.sh | 97 +++++++++++++++++++++++++++- tests/fm-secondmate-harness.test.sh | 36 +++++++++++ 14 files changed, 292 insertions(+), 54 deletions(-) diff --git a/bin/fm-guard.sh b/bin/fm-guard.sh index 5698d376ab..24151de92e 100755 --- a/bin/fm-guard.sh +++ b/bin/fm-guard.sh @@ -6,12 +6,18 @@ # non-default branch, because that means firstmate-on-itself work landed in the # primary instead of an isolated worktree. # Then, if a task is in flight (a state/.meta exists) or X-mode relay -# polling is active (state/x-watch.check.sh exists) and no identity-matched -# watcher has a liveness beacon (state/.last-watcher-beat, touched every poll -# cycle) fresh within FM_GUARD_GRACE seconds, prints a loud, clearly delimited -# banner so the agent cannot skim past it in the tool output of whatever it was -# doing - the one channel every harness has. The full banner is emitted once per -# distinct staleness episode in this FM_HOME (keyed to beacon mtime or absence); +# polling is active (state/x-watch.check.sh exists) and supervision is not +# healthy, prints a loud, clearly delimited banner so the agent cannot skim past +# it in the tool output of whatever it was doing - the one channel every harness +# has. Supervision health is MODEL-AWARE (fm_watcher_supervision_verdict in +# bin/fm-wake-lib.sh): under the Claude Stop auto-arm model the watcher runs only +# between turns, so mid-turn a fresh beacon with no live watcher is healthy and +# only a stale beacon (beyond FM_GUARD_GRACE) is a genuine lapse; under every +# persistent-watcher harness a live identity-matched watcher with a fresh beacon +# is required. The banner names the true failing condition (a missing live +# watcher process vs a genuinely stale beacon). The full banner is emitted once +# per distinct down-episode in this FM_HOME (keyed to the failing condition, not +# the beacon mtime, which a healthy between-turns watcher advances every poll); # later guarded commands in the same episode print a one-line reminder instead. # Episode state lives only under state/.guard-watcher-stale-banner (volatile, # bounded). Independent alarms (queued wakes, worktree tangle) are never @@ -43,18 +49,14 @@ STALE_BANNER_MARKER="$STATE/.guard-watcher-stale-banner" # shellcheck source=bin/fm-supervision-lib.sh . "$SCRIPT_DIR/fm-supervision-lib.sh" -# Deterministic episode key from beacon state: same continuous stale beacon -# (or continuous absence) shares a key; a recovered-then-restale beacon gets a -# new mtime and therefore a new episode. +# Deterministic episode key from the qualitative down-state (the failing +# condition), NOT the beacon mtime: under the auto-arm model a healthy +# between-turns watcher advances that mtime every poll, which made the "same +# episode" key change every turn and re-print the full banner. Keying on the +# failing condition keeps one continuous down-episode stable, while positive +# recovery clears the marker (below) and re-arms the next episode. fm_guard_stale_episode_key() { - local state=$1 beat m - beat="$state/.last-watcher-beat" - if [ -e "$beat" ]; then - m=$(fm_sup_stat_mtime "$beat") - printf 'beat:%s\n' "${m:-unknown}" - else - printf 'beat:absent\n' - fi + printf '%s\n' "$1" } # Claim the full banner for this episode. Exit 0 = print full banner (this call @@ -150,10 +152,9 @@ in_flight=$FM_SUP_IN_FLIGHT sources=$FM_SUP_SOURCES needed=$FM_SUP_NEEDED beacon_desc=$FM_SUP_BEACON_DESC -watcher_healthy=false -if fm_watcher_healthy "$STATE" "$WATCH" "$GRACE" "$FM_HOME"; then - watcher_healthy=true -fi +fm_watcher_supervision_verdict "$STATE" "$WATCH" "$GRACE" "$FM_HOME" +watcher_healthy=$FM_WATCHER_VERDICT_OK +watcher_down_reason=$FM_WATCHER_VERDICT_REASON if [ "$needed" = false ]; then # Leave the unhealthy state (nothing riding on the watcher): clear so a later # work or X-mode need + stale combination is a fresh episode even if the @@ -168,7 +169,7 @@ fi # bordered banner FIRST so it reads as an alarm, not a buried stderr line. Later # calls in the same episode get a one-line reminder only. if [ "$watcher_healthy" = false ]; then - episode_key=$(fm_guard_stale_episode_key "$STATE") + episode_key=$(fm_guard_stale_episode_key "$watcher_down_reason") episode_key=${episode_key%$'\n'} print_full_banner=0 if [ "$READ_ONLY" -eq 1 ]; then @@ -193,12 +194,17 @@ if [ "$watcher_healthy" = false ]; then { printf '●%s\n' "$rule" printf '● WATCHER DOWN - SUPERVISION IS OFF\n' + if [ "$watcher_down_reason" = no-watcher ]; then + watcher_cause=$(printf 'no live watcher process holds this home lock (last beat: %s)' "$beacon_desc") + else + watcher_cause=$(printf 'no watcher has a fresh beacon (last beat: %s, grace %ss)' "$beacon_desc" "$GRACE") + fi if [ "$in_flight" -gt 0 ]; then - printf '● %s task(s) in flight, but no watcher has a fresh beacon (last beat: %s, grace %ss).\n' "$in_flight" "$beacon_desc" "$GRACE" + printf '● %s task(s) in flight, but %s.\n' "$in_flight" "$watcher_cause" elif [ "$sources" -gt 0 ]; then - printf '● %s process-event source(s) registered, but no watcher has a fresh beacon (last beat: %s, grace %ss).\n' "$sources" "$beacon_desc" "$GRACE" + printf '● %s process-event source(s) registered, but %s.\n' "$sources" "$watcher_cause" else - printf '● X-mode relay polling needs supervision, but no watcher has a fresh beacon (last beat: %s, grace %ss).\n' "$beacon_desc" "$GRACE" + printf '● X-mode relay polling needs supervision, but %s.\n' "$watcher_cause" fi if [ "$READ_ONLY" -eq 1 ]; then printf '● This read-only session should report the lapse, not repair it.\n' diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index f1ad34bdd9..273a134425 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -2105,6 +2105,10 @@ fi if [ "$KIND" = secondmate ]; then sq_home=$(shell_quote "$PROJ_ABS") sq_primary_home=$(shell_quote "$FM_HOME") + case "$HARNESS" in + claude) supervision_model=autoarm ;; + *) supervision_model=persistent ;; + esac # Deliver the primary's EFFECTIVE trace-context decision as a normalized on/off # literal (never the raw FM_TRACE_CONTEXT string) so a FM_TRACE_CONTEXT override # on the primary reaches the secondmate's OWN workers, not just the copied @@ -2112,7 +2116,7 @@ if [ "$KIND" = secondmate ]; then # not enable them across the launch boundary (bin/fm-trace-context-lib.sh header). # Reuse the single frozen decision from the carrier resolution above so the # injected carrier and this on/off snapshot are guaranteed to agree. - LAUNCH="FM_ROOT_OVERRIDE= FM_STATE_OVERRIDE= FM_DATA_OVERRIDE= FM_PROJECTS_OVERRIDE= FM_CONFIG_OVERRIDE= FM_PUBLIC_FOLLOWUP_PRIMARY_HOME=$sq_primary_home FM_HOME=$sq_home FM_TRACE_CONTEXT=$SPAWN_TRACE_EFFECTIVE $LAUNCH" + LAUNCH="FM_ROOT_OVERRIDE= FM_STATE_OVERRIDE= FM_DATA_OVERRIDE= FM_PROJECTS_OVERRIDE= FM_CONFIG_OVERRIDE= FM_PUBLIC_FOLLOWUP_PRIMARY_HOME=$sq_primary_home FM_HOME=$sq_home FM_TRACE_CONTEXT=$SPAWN_TRACE_EFFECTIVE FM_SUPERVISION_MODEL=$supervision_model $LAUNCH" fi # Export GOTMPDIR into the crewmate's pane shell so the agent and every child # process (go build, go test, ...) inherit it. Sent before the launch command so diff --git a/bin/fm-supervision-lib.sh b/bin/fm-supervision-lib.sh index dc223d344c..252d0c93c2 100644 --- a/bin/fm-supervision-lib.sh +++ b/bin/fm-supervision-lib.sh @@ -6,9 +6,12 @@ # work (a state/.meta exists) or an X-mode relay poll # (state/x-watch.check.sh), and whether its watcher has a fresh liveness beacon # (state/.last-watcher-beat, touched every poll cycle, within the grace window). -# bin/fm-guard.sh and bin/fm-turnend-guard.sh use fm_watcher_healthy from -# bin/fm-wake-lib.sh for their warning and block decisions, so a fresh leftover -# beacon never counts as a live watcher. The status fields here retain the +# bin/fm-turnend-guard.sh uses the PID-strict fm_watcher_healthy from +# bin/fm-wake-lib.sh for its block decision. bin/fm-guard.sh uses the model-aware +# fm_watcher_supervision_verdict (also in bin/fm-wake-lib.sh): under the Claude +# Stop auto-arm model, where the watcher only runs between turns, a fresh beacon +# with no live watcher is healthy; under persistent-watcher harnesses a live +# identity-matched watcher is still required. The status fields here retain the # beacon-age details used in their messages. # Portable mtime; Linux stat lacks -f, macOS stat lacks -c. diff --git a/bin/fm-wake-drain.sh b/bin/fm-wake-drain.sh index c3bf7335c0..f5fd160cda 100755 --- a/bin/fm-wake-drain.sh +++ b/bin/fm-wake-drain.sh @@ -12,16 +12,16 @@ DRAIN_LOCK_HELD=false RAW_ROWS= # Defense in depth for the supervision chain: this script runs at the top of -# every wake-handling and recovery turn, so assert watcher liveness here too. A +# every wake-handling and recovery turn, so assert supervision health here too. A # lapsed supervision chain then surfaces on a plain drain-and-handle turn, not # only when a guarded supervision script (fm-peek/fm-send/...) happens to run. -# Reuse fm-guard.sh's existing graced, beacon-based alarm (FM_GUARD_GRACE) - do -# not duplicate the beacon math. Because the watcher touches its beacon every -# poll cycle, a normal fire leaves a recent beacon well inside grace and stays -# silent; only a genuine stale-beyond-grace lapse with work in flight warns. Call -# after the queue is emptied so guard never re-prints its own queued-wakes notice -# for the records this run just drained, and never let a guard hiccup change the -# drain's exit status. +# Reuse fm-guard.sh's model-aware alarm and FM_GUARD_GRACE instead of duplicating +# its supervision verdict. Under Claude's between-turns auto-arm model, a normal +# fire leaves a recent beacon well inside grace and stays silent mid-turn. Under +# persistent-watcher models, the guard also requires the live identity-matched +# watcher. Call after the queue is emptied so guard never re-prints its own +# queued-wakes notice for the records this run just drained, and never let a +# guard hiccup change the drain's exit status. assert_watcher_liveness() { "$SCRIPT_DIR/fm-guard.sh" || true } diff --git a/bin/fm-wake-lib.sh b/bin/fm-wake-lib.sh index 0aeac11414..3af1642b31 100755 --- a/bin/fm-wake-lib.sh +++ b/bin/fm-wake-lib.sh @@ -115,6 +115,82 @@ fm_watcher_healthy() { return 0 } +# fm_watcher_healthy above is the PID-STRICT primitive: true only when a live, +# identity-matched watcher PROCESS holds this home's lock with a fresh beacon. The +# arm layer (bin/fm-watch-arm.sh, bin/fm-claude-stop-autoarm.sh) needs exactly +# that - it decides whether to start, attach to, or replace a real watcher +# process, so a leftover beacon must never satisfy it. bin/fm-turnend-guard.sh +# also keeps this strict check because it fires at the turn boundary where the +# auto-arm brings a fresh watcher up. The pull warning (bin/fm-guard.sh) fires +# mid-turn, where the auto-arm model runs no watcher at all, so it wants a +# different, model-aware question: + +# fm_supervision_model +# Print the supervision model of this home's PRIMARY harness: +# autoarm Claude Stop-hook auto-arm: the watcher is armed at each turn end +# and exits on its wake, so it runs only BETWEEN turns. Mid-turn a +# fresh beacon with no live watcher process is the healthy state. +# persistent every other harness (codex foreground checkpoint, opencode/pi/grok +# background arm, tmux, unknown): the watcher runs as a tracked live +# process, so a live identity-matched pid is the real liveness signal. +# FM_SUPERVISION_MODEL overrides detection (tests, and callers that already know +# the harness). Otherwise bin/fm-harness.sh is the single detection owner, so this +# stays consistent with the harness-specific repair line the guards already emit. +fm_supervision_model() { + local harness + case "${FM_SUPERVISION_MODEL:-}" in + autoarm|persistent) printf '%s\n' "$FM_SUPERVISION_MODEL"; return 0 ;; + esac + harness=$("$FM_WAKE_LIB_DIR/fm-harness.sh" 2>/dev/null || printf unknown) + case "$harness" in + claude) printf 'autoarm\n' ;; + *) printf 'persistent\n' ;; + esac +} + +# fm_watcher_supervision_verdict [grace] [home] +# Model-aware "is supervision healthy right now" verdict for the pull warning +# guard (bin/fm-guard.sh), NOT the arm layer or the turn-end guard. Sets: +# FM_WATCHER_VERDICT_OK true when supervision is healthy for this model +# FM_WATCHER_VERDICT_REASON when not ok, the true failing condition: +# no-watcher - a live watcher process is the real +# signal for this model but none holds +# the lock (the beacon is still fresh) +# stale-beacon - the beacon is stale beyond grace or +# absent (a genuine supervision lapse) +# autoarm: a fresh beacon within grace is healthy even with no live watcher, +# because the watcher only runs between turns; only a stale beacon is a lapse. +# persistent: require a live identity-matched watcher with a fresh beacon +# (fm_watcher_healthy); a fresh leftover beacon with no live watcher is still down. +# shellcheck disable=SC2034 # Read by callers after the function returns. +FM_WATCHER_VERDICT_OK=false +# shellcheck disable=SC2034 # Read by callers after the function returns. +FM_WATCHER_VERDICT_REASON=stale-beacon +fm_watcher_supervision_verdict() { + local state=$1 watch=$2 grace=${3:-${FM_GUARD_GRACE:-300}} home=${4:-$FM_HOME} + local beat age fresh=false + FM_WATCHER_VERDICT_OK=false + FM_WATCHER_VERDICT_REASON=stale-beacon + beat="$state/.last-watcher-beat" + age=$(fm_path_age "$beat") + case "$age" in + ''|*[!0-9]*) ;; + *) [ "$age" -lt "$grace" ] && fresh=true ;; + esac + if [ "$(fm_supervision_model)" = autoarm ]; then + [ "$fresh" = true ] && FM_WATCHER_VERDICT_OK=true + return 0 + fi + if fm_watcher_healthy "$state" "$watch" "$grace" "$home"; then + # shellcheck disable=SC2034 # Read by callers after the function returns. + FM_WATCHER_VERDICT_OK=true + elif [ "$fresh" = true ]; then + # shellcheck disable=SC2034 # Read by callers after the function returns. + FM_WATCHER_VERDICT_REASON=no-watcher + fi + return 0 +} + fm_lock_clean_known_files() { local lockdir=$1 rm -f \ diff --git a/docs/architecture.md b/docs/architecture.md index cc1d6e1a95..cc8f182452 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -65,9 +65,9 @@ It suppresses failed-looking closes when the same identity-matched watcher is he [`watcher-continuity.md`](watcher-continuity.md) owns Claude's residual active-turn coverage and watcher-status command-gating boundary. The existing turn-end guard remains the final backstop for all five harness-engine protocols, with pi-signed sharing Pi's protocol and the `--claude` mode cooperating with the auto-arm claim. Its `--restart` mode signals only the watcher recorded in the current home's `state/.watch.lock`, so restarting one home cannot kill sibling secondmate watchers. -A pull-based guard (`bin/fm-guard.sh`) warns through supervision tool output if the primary checkout is tangled, if work, process-event sources, or X-mode relay polling needs supervision without a healthy identity-matched watcher, or if queued wakes are waiting to be drained. -The drain script calls that guard after emptying the queue, which avoids repeating the queued-wakes warning for records it just consumed while still warning on stale watcher liveness. -It leads with a prominent bordered tangle banner, while `bin/fm-guard.sh` owns the stale-watcher banner/reminder policy so repeated guarded commands stay noisy without reprinting the full watcher-down banner in the same episode. +A pull-based guard (`bin/fm-guard.sh`) warns through supervision tool output if the primary checkout is tangled, if work, process-event sources, or X-mode relay polling has an unhealthy model-aware supervision verdict, or if queued wakes are waiting to be drained. +The drain script calls that guard after emptying the queue, which avoids repeating the queued-wakes warning for records it just consumed while still warning on unhealthy supervision. +It leads with a prominent bordered tangle banner, while `bin/fm-guard.sh` owns the watcher-down banner and reminder policy so repeated guarded commands stay noisy without reprinting the full banner in the same episode. On every verified primary harness, tracked hook integration gives the primary session a push-based backstop: when work, a process-event source, or X-mode relay polling needs supervision and no identity-matched watcher lock with a fresh beacon is live, direct Stop hooks block and passive turn-end hooks force one bounded follow-up. The guard covers the main primary and genuinely marked secondmate homes, exempts child crewmate/scout worktrees, is loop-safe per harness, and is documented in [turnend-guard.md](turnend-guard.md). diff --git a/docs/arm-pretool-check.md b/docs/arm-pretool-check.md index f4747e0abd..a07084d25f 100644 --- a/docs/arm-pretool-check.md +++ b/docs/arm-pretool-check.md @@ -13,7 +13,7 @@ A shell background operator, pipeline, redirection, wrapper, or unrelated comman The seatbelt rejects those command shapes before execution. This policy is not a post-arm liveness guarantee. -`bin/fm-guard.sh`, `bin/fm-turnend-guard.sh`, the watcher lock, and the watcher beacon still prove whether supervision is healthy after an allowed call. +`bin/fm-guard.sh` and `bin/fm-turnend-guard.sh` apply their respective post-arm supervision predicates to the watcher lock and beacon after an allowed call. The classifier never executes, sources, evaluates, or expands any part of the submitted command. It tokenizes the bytes and classifies lexical execution positions only. diff --git a/docs/scripts.md b/docs/scripts.md index 0935b3e5d9..f0960b9389 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -31,7 +31,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-test-run.sh` | Behavior-test runner: selection, portable lanes, proven-isolated `--jobs`, coverage guard, timing/JSON | | `fm-test-isolation-proof.sh` | Concurrent isolation proof and proven-isolated candidate set owner | | `fm-ensure-agents-md.sh` | Ensure a project's real `AGENTS.md`, its `CLAUDE.md` symlink, and the canonical self-governance section | -| `fm-guard.sh` | Warn on primary-checkout tangles, pending queued wakes, and stale watcher liveness | +| `fm-guard.sh` | Warn on primary-checkout tangles, pending queued wakes, and unhealthy supervision | | `fm-primary-scope-lib.sh` | Shared marker-or-plain-checkout primary-home predicate for tracked hooks | | `fm-session-lock-lib.sh` | Shared session-lock harness identity (ancestry walk and holder liveness) for fm-lock.sh and the Claude Stop auto-arm | | `fm-claude-stop-autoarm.sh` | Claude Stop `asyncRewake` hook owning tokenless watcher continuity with single-flight exit-2 rewake (docs/watcher-continuity.md) | @@ -80,7 +80,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-tasks-axi-lib.sh` | Shared backlog-backend selector and `tasks-axi` compatibility probe | | `fm-quota-axi-lib.sh` | Shared `quota-axi` compatibility floor for the bootstrap diagnostic | | `fm-vendor-auth-probe.sh`| Run one hard-bounded, non-destructive authentication probe of a named vendor CLI and report the fact | -| `fm-wake-drain.sh` | Atomically drain queued watcher wakes, emit bounded best-effort status-event annotations, then assert watcher liveness | +| `fm-wake-drain.sh` | Atomically drain queued watcher wakes, emit bounded best-effort status-event annotations, then assert supervision health | | `fm-wake-lib.sh` | Shared durable wake queue, portable locks, and watcher identity/health helpers | | `fm-classify-lib.sh` | Shared captain-relevant and declared-external-wait wake classification vocabulary | | `fm-send.sh` | Send one verified literal line or supported key through the target's recorded backend | diff --git a/docs/sessionstart-nudge.md b/docs/sessionstart-nudge.md index c39c814925..5ea54bea2e 100644 --- a/docs/sessionstart-nudge.md +++ b/docs/sessionstart-nudge.md @@ -10,7 +10,7 @@ The Ahoy skill owns the rule that this marked operational input is never a capta `bin/fm-sessionstart-nudge.sh` is the single command every harness adapter invokes. It sources `bin/fm-gate-refuse-lib.sh` and stays silent for a no-mistakes gate agent identified by `NO_MISTAKES_GATE` or a `.no-mistakes/repos/*.git` git-common-dir. It shares `bin/fm-primary-scope-lib.sh` with `bin/fm-turnend-guard.sh`, so the hooks use one primary-detection owner. -The Shared Predicate section of [`turnend-guard.md`](turnend-guard.md#shared-predicate) owns marker validation, plain-checkout detection, and required Firstmate-shaped paths. +The Guard Predicates section of [`turnend-guard.md`](turnend-guard.md#guard-predicates) owns marker validation, plain-checkout detection, and required Firstmate-shaped paths. Before printing, the wrapper reads `state/.lock` and walks at most eight parents from its own pid in its own separate, hard-coded loop, independent of `bin/fm-lock.sh`'s ancestry walk (`fm_harness_ancestry_pid()` in `bin/fm-session-lock-lib.sh`, which now walks up to sixteen parents and can extend past a claude-named match to a still-more-ancestral one) and of Pi's `lockOwnership()`. If the lock names a live pid in that ancestry, session start already ran in this harness session and the wrapper stays silent. diff --git a/docs/supervision-protocols/claude.md b/docs/supervision-protocols/claude.md index 2b32be24d7..041c0580ca 100644 --- a/docs/supervision-protocols/claude.md +++ b/docs/supervision-protocols/claude.md @@ -17,7 +17,7 @@ When this session owns supervision and away mode is not active: No PreToolUse hook denies fleet commands based on watcher status. [`watcher-continuity.md`](../watcher-continuity.md) owns the exact session-lock recovery boundary. 8. The turn-end guard (`bin/fm-turnend-guard.sh --claude`) remains the final backstop. - It uses the same live-watcher and fresh-beacon predicate as the pull guard. + It requires the PID-strict live-watcher and fresh-beacon predicate at the Stop boundary, while the mid-turn pull guard accepts a fresh beacon without a live process under Claude's between-turns auto-arm model. It allows the stop when a watcher is healthy or the role-verified auto-arm owns recovery, while fresh failure epochs advance the bounded one-time attended fail-open progression described in [`turnend-guard.md`](../turnend-guard.md). 9. Waiting on the hook-owned cycle is silent: do not send idle progress while the watcher is parked. diff --git a/docs/turnend-guard.md b/docs/turnend-guard.md index 63d13be70f..6e4ce53483 100644 --- a/docs/turnend-guard.md +++ b/docs/turnend-guard.md @@ -13,11 +13,11 @@ Do not infer this guard's scope, loop safety, or compatibility tradeoffs for tho `bin/fm-guard.sh` is a pull-based warning that runs only when another supervision command invokes it. The turn-end guard closes the remaining gap at the primary's own turn boundary. -When work, a process-event source, or X-mode relay polling needs supervision and no identity-matched watcher has a fresh beacon, the harness integration must either block the turn end or force one bounded follow-up that uses the recovery instruction from the emitted session-start protocol. -Both guards require the same live lock, process identity, home/path binding, and fresh-beacon predicate. +When work, a process-event source, or X-mode relay polling needs supervision at that boundary and no identity-matched watcher has a fresh beacon, the harness integration must either block the turn end or force one bounded follow-up that uses the recovery instruction from the emitted session-start protocol. +The mid-turn pull warning uses the model-aware supervision verdict described below, while the turn-end guard keeps the PID-strict watcher predicate. The guard remains a backstop; [`watcher-continuity.md`](watcher-continuity.md) owns normal continuity. -## Shared predicate +## Guard predicates The guard first calls the shared primary scope. A secondmate home runs its own primary Firstmate session, so a genuine `.fm-secondmate-home` marker includes it whether the home is a linked worktree or plain clone. @@ -30,10 +30,12 @@ For an in-scope primary, the guard counts in-flight work from `state/*.meta`. Registered `state/procevent/*.source` records also require supervision even though they have no task metadata. The default cross-harness mode exits silently with no supervision need. Every mode treats `state/x-watch.check.sh` as supervision need, so X-mode relay polling remains guarded without an in-flight task. -Otherwise it calls `fm_watcher_healthy [grace-seconds] [home]` from `bin/fm-wake-lib.sh`, the same identity-matched lock and fresh-beacon check used by `bin/fm-watch-arm.sh`. -`bin/fm-guard.sh` uses that same check rather than treating the status helper's fresh-beacon field as sufficient. -A stale beacon blocks even when a watcher pid is live. -A fresh leftover beacon blocks when the lock is missing, dead, or identity-mismatched. +Otherwise it calls `fm_watcher_healthy [grace-seconds] [home]` from `bin/fm-wake-lib.sh`, the same PID-strict identity-matched lock and fresh-beacon check used by `bin/fm-watch-arm.sh`: a stale beacon blocks even when a watcher pid is live, and a fresh leftover beacon blocks when the lock is missing, dead, or identity-mismatched. +The turn-end guard needs that strict check because it fires at the turn boundary, where the auto-arm is bringing a fresh watcher up for the upcoming idle period, and it cooperates with that arm rather than trusting a beacon left by the cycle that just ended. +`bin/fm-guard.sh`, the pull warning, instead uses the model-aware `fm_watcher_supervision_verdict` from the same library, because it fires mid-turn when the auto-arm model runs no watcher at all. +Under the Claude Stop auto-arm model a beacon fresh within grace is healthy even with no live watcher process, and only a beacon stale beyond grace (or absent) alarms. +Under every persistent-watcher harness a live identity-matched watcher with a fresh beacon is still required, so the pull guard keeps the same strict semantics there. +Its banner names the true failing condition, either a missing live watcher process or a genuinely stale beacon with its real age, and keys the once-per-episode dedup on that condition rather than the beacon mtime. `FM_STATE_OVERRIDE` wins over `FM_HOME/state`, and `FM_HOME` wins over repository-root `state/`. `FM_GUARD_GRACE` controls beacon freshness and defaults to 300 seconds. @@ -104,7 +106,7 @@ That warning uses `bin/fm-supervision-instructions.sh --repair-line`, so it alwa ## Regression coverage `tests/fm-turnend-guard.test.sh` covers the predicate, main and secondmate primary scope, child-worktree exclusion, `FM_HOME` and `FM_STATE_OVERRIDE` precedence, the live-lock and fresh-beacon guard predicate, the cooperative `--claude` claim wait, monotonic failed-epoch progression, bounded attended fail-open, post-alarm continuation suppression, positive recovery reset, Pi logical-run latching, missing-`jq` behavior, all five primary registrations, Grok native and legacy selection, typed field precedence, malformed input, and exactly-one-path safety. -`tests/fm-guard-stale-banner.test.sh` covers the matching pull-guard predicate, including the fresh-leftover-beacon negative control. +`tests/fm-guard-stale-banner.test.sh` covers the pull-guard predicate, including the persistent-model fresh-leftover-beacon negative control, the auto-arm model's healthy fresh-beacon-without-a-watcher case and its stale-beacon alarm, the true-reason banner wording, and the reason-keyed episode dedup surviving a beacon mtime change. `tests/fm-kimi-harness.test.sh` covers the separate Kimi crew hook's format preservation, idempotence, refusal cases, token guard, spawn registration, and teardown cleanup. `tests/fm-supervision-instructions.test.sh` covers recovery-line ownership and pi-signed's identity-preserving reuse of Pi's protocol. `FM_PI_LIVE_E2E=1 tests/fm-pi-primary-live-e2e.test.sh` is the opt-in isolated Pi path. diff --git a/docs/verification/supervision.md b/docs/verification/supervision.md index a9ed234fb5..8a64d3a0c5 100644 --- a/docs/verification/supervision.md +++ b/docs/verification/supervision.md @@ -171,6 +171,22 @@ fm-doc-audience-check: ok surfaces=61 local_links=174 FM_TEST_SUMMARY total=4 failed=0 skipped_gate=0 duration_ms=102585 ``` +The model-aware pull-guard predicate correction (`bin/fm-guard.sh` no longer reports a false watcher-down mid-turn under the Claude Stop auto-arm model, where the watcher runs only between turns) was verified on 2026-08-04 with the installed ShellCheck 0.11.0 and the same isolated behavior suites. + +```sh +bin/fm-lint.sh +bin/fm-doc-audience-check.sh +bin/fm-test-run.sh tests/fm-claude-stop-autoarm.test.sh tests/fm-guard-stale-banner.test.sh tests/fm-turnend-guard.test.sh tests/fm-supervision-instructions.test.sh +``` + +Observed output: + +```text +fm-lint.sh: ShellCheck 0.11.0 (pinned 0.11.0) +fm-doc-audience-check: ok surfaces=64 local_links=188 +FM_TEST_SUMMARY total=4 failed=0 skipped_gate=0 duration_ms=80078 +``` + The broader relevant regression pass was rerun on 2026-08-02 without live-home or daemon mutation. ```sh diff --git a/tests/fm-guard-stale-banner.test.sh b/tests/fm-guard-stale-banner.test.sh index 54091035dc..0dbe8c499e 100755 --- a/tests/fm-guard-stale-banner.test.sh +++ b/tests/fm-guard-stale-banner.test.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Regression tests for fm-guard's stale-watcher banner deduplication. +# Regression tests for fm-guard's watcher-down banner deduplication. # # The first stale command in one FM_HOME must print the full actionable watcher # banner. @@ -41,11 +41,15 @@ record_live_watcher() { printf '%s\n' "$identity" > "$home/state/.watch.lock/pid-identity" } +# These cases exercise the persistent-watcher model (a live pid is the real +# liveness signal), so pin the model rather than letting the host test runner's +# ambient harness ancestry pick it. run_guard_case() { local dir=$1 FM_ROOT_OVERRIDE="$(case_root "$dir")" \ FM_HOME="$(case_home "$dir")" \ FM_GUARD_GRACE=999 \ + FM_SUPERVISION_MODEL=persistent \ "$ROOT/bin/fm-guard.sh" 2>&1 } @@ -54,10 +58,22 @@ run_guard_case_read_only() { FM_ROOT_OVERRIDE="$(case_root "$dir")" \ FM_HOME="$(case_home "$dir")" \ FM_GUARD_GRACE=999 \ + FM_SUPERVISION_MODEL=persistent \ FM_GUARD_READ_ONLY=1 \ "$ROOT/bin/fm-guard.sh" 2>&1 } +# The Claude Stop auto-arm model: the watcher runs only between turns, so a fresh +# beacon with no live watcher process is the healthy mid-turn state. +run_guard_case_autoarm() { + local dir=$1 + FM_ROOT_OVERRIDE="$(case_root "$dir")" \ + FM_HOME="$(case_home "$dir")" \ + FM_GUARD_GRACE=999 \ + FM_SUPERVISION_MODEL=autoarm \ + "$ROOT/bin/fm-guard.sh" 2>&1 +} + count_text() { local haystack=$1 needle=$2 awk -v needle="$needle" 'index($0, needle) { c++ } END { print c + 0 }' < a genuine supervision lapse even under the auto-arm model. + out=$(run_guard_case_autoarm "$dir") + [ "$(count_text "$out" "WATCHER DOWN - SUPERVISION IS OFF")" -eq 1 ] \ + || fail "auto-arm model with an absent/stale beacon must alarm: $out" + assert_contains "$out" "no watcher has a fresh beacon" \ + "auto-arm stale-beacon banner must name the stale-beacon reason" + pass "fm-guard stale banner: auto-arm stale beacon alarms with the true reason" +} + +test_autoarm_stale_episode_is_stable() { + local dir out1 out2 + dir=$(make_guard_case autoarm-stable-episode) + out1=$(run_guard_case_autoarm "$dir") + out2=$(run_guard_case_autoarm "$dir") + [ "$(count_text "$out1" "WATCHER DOWN - SUPERVISION IS OFF")" -eq 1 ] \ + || fail "first auto-arm stale call did not print the full banner: $out1" + [ "$(count_text "$out2" "WATCHER DOWN - SUPERVISION IS OFF")" -eq 0 ] \ + || fail "auto-arm stale episode re-printed the full banner instead of deduping: $out2" + assert_contains "$out2" "full banner already printed this episode" \ + "second auto-arm stale call did not print the concise reminder" + pass "fm-guard stale banner: auto-arm stale episode stays one episode across calls" +} + +test_persistent_no_watcher_banner_names_missing_process() { + local dir out + dir=$(make_guard_case persistent-no-watcher-reason) + # A fresh beacon with no live watcher under the persistent model: the real + # failing condition is the missing process, not a stale beacon. + touch "$(case_home "$dir")/state/.last-watcher-beat" + out=$(run_guard_case "$dir") + assert_contains "$out" "no live watcher process holds this home lock" \ + "persistent no-watcher banner must name the missing watcher process" + assert_not_contains "$out" "no watcher has a fresh beacon" \ + "persistent no-watcher banner must not blame the fresh beacon" + pass "fm-guard stale banner: persistent no-watcher banner names the true reason" +} + +test_persistent_no_watcher_episode_survives_beacon_touch() { + local dir home out1 out2 + dir=$(make_guard_case persistent-no-watcher-episode) + home=$(case_home "$dir") + touch "$home/state/.last-watcher-beat" + out1=$(run_guard_case "$dir") + [ "$(count_text "$out1" "WATCHER DOWN - SUPERVISION IS OFF")" -eq 1 ] \ + || fail "first persistent no-watcher call did not print the full banner: $out1" + # The beacon mtime advancing with NO live watcher must not split the continuous + # down-episode. The old beacon-mtime episode key re-printed the full banner + # here; the reason-based key keeps it a single episode. Separate the touches by + # a second so the mtime genuinely changes at whole-second stat granularity. + sleep 1 + touch "$home/state/.last-watcher-beat" + out2=$(run_guard_case "$dir") + [ "$(count_text "$out2" "WATCHER DOWN - SUPERVISION IS OFF")" -eq 0 ] \ + || fail "advancing the beacon mtime with no live watcher re-printed the banner: $out2" + assert_contains "$out2" "full banner already printed this episode" \ + "same no-watcher episode did not print the concise reminder after a beacon touch" + pass "fm-guard stale banner: a no-watcher episode survives a beacon mtime change" +} + test_first_stale_call_prints_full_banner test_repeated_same_episode_prints_reminder_only +test_autoarm_fresh_beacon_without_watcher_is_healthy +test_autoarm_stale_beacon_alarms_with_correct_reason +test_autoarm_stale_episode_is_stable +test_persistent_no_watcher_banner_names_missing_process +test_persistent_no_watcher_episode_survives_beacon_touch test_fresh_beacon_without_live_watcher_stays_alarm test_x_mode_without_live_watcher_stays_alarm test_healthy_recovery_rearms_next_stale_episode diff --git a/tests/fm-secondmate-harness.test.sh b/tests/fm-secondmate-harness.test.sh index e65109c2c4..a39943706c 100755 --- a/tests/fm-secondmate-harness.test.sh +++ b/tests/fm-secondmate-harness.test.sh @@ -829,6 +829,41 @@ test_spawn_explicit_harness_uses_explicit_profile_axes() { pass "C8 spawn: an explicit --harness still honors explicit model/effort flags" } +test_spawned_secondmate_uses_its_harness_supervision_model() { + local harness expected w sm launchlog launch fakebin out + for harness in codex claude; do + w="$TMP_ROOT/spawn-supervision-model-$harness" + sm="$w/sm" + launchlog="$w/launch.log" + mkdir -p "$w/home/config" + printf '%s\n' "$harness" > "$w/home/config/secondmate-harness" + make_seeded_home "$sm" sm + spawn_secondmate_capture "$w" sm "$sm" "$launchlog" >/dev/null 2>&1 + fm_write_meta "$sm/state/task.meta" "window=firstmate:fm-task" "kind=ship" + touch "$sm/state/.last-watcher-beat" + fakebin="$w/tmux-sm/fakebin" + cat > "$fakebin/$harness" <&1) + case "$harness" in + codex) + expected='WATCHER DOWN - SUPERVISION IS OFF' + assert_contains "$out" "$expected" \ + "Codex secondmate inherited Claude auto-arm despite its persistent watcher model" + ;; + claude) + [ -z "$out" ] \ + || fail "Claude secondmate with a fresh beacon should use auto-arm supervision, got: $out" + ;; + esac + done + pass "C9 spawn: secondmate launch pins supervision to its own harness" +} + # The harness fallback chain (secondmate-harness -> crew-harness -> own) still # resolves correctly with no model/effort tokens anywhere in the chain, and a # crew/scout (non-secondmate) launch is entirely unaffected by this feature: no @@ -2384,6 +2419,7 @@ test_spawn_explicit_model_overrides_secondmate_harness_token test_spawn_explicit_effort_overrides_secondmate_harness_token test_spawn_explicit_harness_does_not_inherit_secondmate_harness_tokens test_spawn_explicit_harness_uses_explicit_profile_axes +test_spawned_secondmate_uses_its_harness_supervision_model test_spawn_fallback_chain_and_crew_scout_unaffected test_bootstrap_sweep_propagates_and_reconverges test_bootstrap_sweep_propagates_when_tracked_current From 1cd97c06bbb56764d8bf801830685f57bd84eb09 Mon Sep 17 00:00:00 2001 From: Kun Chen <3233006+kunchenguid@users.noreply.github.com> Date: Tue, 4 Aug 2026 16:29:37 -0700 Subject: [PATCH 04/12] test: prevent fixture temporary directory leaks (#1704) * fix(tests): stop fixture-tempdir helper from self-deleting under command substitution fm_test_tmproot is almost always called as `TMP_ROOT=$(fm_test_tmproot prefix)`, which forks a subshell to capture its stdout. The old implementation set its EXIT cleanup trap inside that call, so the trap fired - and deleted the fixture root - the instant the subshell exited, before the real caller's own EXIT trap was ever installed. Every test using the documented call pattern leaked its fixture root on every run; two suites had already independently discovered and worked around this with ad-hoc mktemp calls. Registration now goes through a $$-keyed registry file instead of in-process state, since $$ resolves to the invoking shell's PID even inside the subshell. The real cleanup trap is armed once at source time (always the real caller, never a subshell) for EXIT, INT, and TERM. A best-effort orphan sweep on next source reaps marked fixture roots old enough to be from a killed prior run. Simplifies the two existing ad-hoc workarounds (fm-procevent.test.sh, wake-helpers.sh) back onto the shared helper now that it works correctly. * no-mistakes(review): Preserve live fixtures during orphan reaping * no-mistakes(review): Harden fixture ownership against PID reuse * no-mistakes(review): Secure cleanup registry against path precreation * no-mistakes(review): Make fixture registration transactional * no-mistakes(document): Documentation already matches fixture cleanup behavior * no-mistakes: apply CI fixes * no-mistakes: apply CI fixes --- tests/fm-procevent.test.sh | 3 - tests/fm-test-fixture-cleanup.test.sh | 151 ++++++++++++++++++++++++++ tests/lib.sh | 82 ++++++++++++-- tests/wake-helpers.sh | 8 +- 4 files changed, 227 insertions(+), 17 deletions(-) create mode 100755 tests/fm-test-fixture-cleanup.test.sh diff --git a/tests/fm-procevent.test.sh b/tests/fm-procevent.test.sh index 3f63bc53fb..7514216347 100755 --- a/tests/fm-procevent.test.sh +++ b/tests/fm-procevent.test.sh @@ -18,9 +18,6 @@ set -u ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) TMP_ROOT=$(fm_test_tmproot fm-procevent-tests) -# fm_test_tmproot runs inside a command substitution, whose EXIT trap removes the -# directory it just registered, so recreate it before writing anything into it. -mkdir -p "$TMP_ROOT" export FM_PROCEVENT_CLAIM_ROOT="$TMP_ROOT/claims" BLOCKER="$TMP_ROOT/blocker.sh" diff --git a/tests/fm-test-fixture-cleanup.test.sh b/tests/fm-test-fixture-cleanup.test.sh new file mode 100755 index 0000000000..7561f2109f --- /dev/null +++ b/tests/fm-test-fixture-cleanup.test.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# Behavior tests for tests/lib.sh's shared fixture-tempdir helper +# (fm_test_tmproot / fm_test_cleanup / fm_test_reap_orphans). +# +# The near-universal call pattern across this suite is +# `TMP_ROOT=$(fm_test_tmproot prefix)`, which forks a subshell to capture the +# function's stdout. These tests spawn real, separate bash processes that use +# that exact pattern and assert the fixture root is actually gone once the +# owning process's guarded teardown has run - on a normal exit and on a +# terminating signal - plus that a stale marked fixture from a killed prior +# run gets reaped on the next source. Nothing here inspects tests/lib.sh's +# source text; it only observes filesystem state around the real helper. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +LIB="$ROOT/tests/lib.sh" + +test_fixture_root_gone_after_normal_exit() { + local child_out child_dir + child_out=$(bash -c ' + # shellcheck source=tests/lib.sh + . "'"$LIB"'" + d=$(fm_test_tmproot fm-test-cleanup-exit) + printf "%s\n" "$d" + if [ -d "$d" ]; then printf "mid:present\n"; else printf "mid:missing\n"; fi + ') + child_dir=$(printf '%s\n' "$child_out" | sed -n '1p') + assert_contains "$child_out" "mid:present" \ + "the fixture root was not present while its owning process was still alive" + assert_absent "$child_dir" \ + "fm_test_tmproot's fixture root survived its owning process's normal exit" + pass "fm_test_tmproot cleans up its fixture root on normal exit" +} + +test_fixture_root_gone_after_sigterm() { + local harness dirfile child_dir pid tries + harness=$(fm_test_tmproot fm-test-cleanup-sigterm-harness) + dirfile="$harness/child-dir" + bash -c ' + # shellcheck source=tests/lib.sh + . "'"$LIB"'" + d=$(fm_test_tmproot fm-test-cleanup-term) + printf "%s\n" "$d" > "'"$dirfile"'" + while :; do sleep 0.1; done + ' & + pid=$! + tries=0 + while [ "$tries" -lt 100 ]; do + [ -s "$dirfile" ] && break + sleep 0.05 + tries=$((tries + 1)) + done + [ -s "$dirfile" ] || fail "the child never published its fixture root before the wait timed out" + child_dir=$(cat "$dirfile") + assert_present "$child_dir" "the child's fixture root did not exist before it was signaled" + kill -TERM "$pid" + wait "$pid" 2>/dev/null + assert_absent "$child_dir" \ + "fm_test_tmproot's fixture root survived SIGTERM to its owning process" + pass "fm_test_tmproot cleans up its fixture root on SIGTERM" +} + +test_cleanup_registry_resists_precreation() { + local harness shared_tmp victim + harness=$(fm_test_tmproot fm-test-cleanup-registry-harness) + shared_tmp="$harness/shared-tmp" + victim="$harness/victim" + mkdir -p "$shared_tmp" "$victim" + + TMPDIR="$shared_tmp" bash -c ' + printf "%s\n" "$1" > "$TMPDIR/.fm-test-cleanup.$$" + . "$2" + ' _ "$victim" "$LIB" + + assert_present "$victim" \ + "a precreated predictable cleanup registry injected an arbitrary deletion target" + pass "the cleanup registry cannot be injected through path precreation" +} + +test_fixture_registration_failure_rolls_back_root() { + local harness failure_tmp registry_dir output leaked_root + harness=$(fm_test_tmproot fm-test-cleanup-registration-harness) + failure_tmp="$harness/tmp" + registry_dir="$harness/registry-dir" + mkdir -p "$failure_tmp" "$registry_dir" + + if output=$(TMPDIR="$failure_tmp" FM_TEST_CLEANUP_REGISTRY="$registry_dir" \ + fm_test_tmproot fm-test-cleanup-registration-failure 2>/dev/null); then + fail "fm_test_tmproot succeeded after its cleanup registry rejected registration" + fi + [ -z "$output" ] || fail "fm_test_tmproot published an unregistered fixture root" + for leaked_root in "$failure_tmp"/fm-test-cleanup-registration-failure.*; do + [ ! -e "$leaked_root" ] || fail "fm_test_tmproot leaked a root after registration failed" + done + pass "failed fixture registration rolls back the new root" +} + +test_orphan_sweep_respects_fixture_ownership() { + local harness dirfile active_dir stale_dir fresh_dir pid tries + harness=$(fm_test_tmproot fm-test-cleanup-orphan-harness) + dirfile="$harness/active-dir" + bash -c ' + # shellcheck source=tests/lib.sh + . "'"$LIB"'" + d=$(fm_test_tmproot fm-test-cleanup-active) + printf "%s\n" "$d" > "'"$dirfile"'" + while :; do sleep 0.1; done + ' & + pid=$! + tries=0 + while [ "$tries" -lt 100 ]; do + [ -s "$dirfile" ] && break + sleep 0.05 + tries=$((tries + 1)) + done + [ -s "$dirfile" ] || fail "the active child never published its fixture root before the wait timed out" + active_dir=$(cat "$dirfile") + touch -t 202001010000 "$active_dir/.fm-test-fixture" + + stale_dir=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-cleanup-stale.XXXXXX") + printf '%s\n%s\n' "$$" reused-process-identity > "$stale_dir/.fm-test-fixture" + touch -t 202001010000 "$stale_dir/.fm-test-fixture" + fresh_dir=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-cleanup-fresh.XXXXXX") + : > "$fresh_dir/.fm-test-fixture" + + bash -c ' + # shellcheck source=tests/lib.sh + . "'"$LIB"'" + ' + + assert_absent "$stale_dir" \ + "a stale fixture root whose PID was reused by another process was not reaped" + assert_present "$active_dir" \ + "the orphan reaper removed an old fixture root whose owning process was still alive" + assert_present "$fresh_dir" \ + "the orphan reaper removed a fresh marked fixture root it does not own yet" + kill -TERM "$pid" + wait "$pid" 2>/dev/null + assert_absent "$active_dir" \ + "the active fixture root survived its owning process's teardown" + rm -rf "$fresh_dir" + pass "the orphan sweep reaps only old fixtures without a live owner" +} + +test_fixture_root_gone_after_normal_exit +test_fixture_root_gone_after_sigterm +test_cleanup_registry_resists_precreation +test_fixture_registration_failure_rolls_back_root +test_orphan_sweep_respects_fixture_ownership diff --git a/tests/lib.sh b/tests/lib.sh index ee3b1d1476..1118c6deb8 100644 --- a/tests/lib.sh +++ b/tests/lib.sh @@ -53,29 +53,97 @@ pass() { # --- self-cleaning temp root ------------------------------------------------ # # fm_test_tmproot echoes a fresh temp dir and registers it for removal -# on EXIT. The first call installs the cleanup trap. A test file that needs -# extra teardown (e.g. killing a daemon) should define its own EXIT trap and -# call fm_test_cleanup from inside it so registered dirs are still removed. +# on EXIT/INT/TERM. A test file that needs extra teardown (e.g. killing a +# daemon) should define its own EXIT trap and call fm_test_cleanup from inside +# it so registered dirs are still removed. +# +# The call site is almost always `TMP_ROOT=$(fm_test_tmproot prefix)`, which +# forks a subshell to capture stdout. Anything that function does to the +# current shell's state - an array append, a trap - dies with that subshell +# and never reaches the real caller, so registration cannot go through +# in-process state. `$$` is the one thing bash keeps stable across that +# boundary (it always resolves to the invoking shell's PID, not the +# subshell's - see `man bash` on `$$`), so fm_test_tmproot records the +# directory in a `$$`-keyed registry file instead, and the trap that reaps +# that file is armed once, here, at source time - which always runs in the +# real caller, never a subshell. FM_TEST_CLEANUP_DIRS=() +FM_TEST_CLEANUP_REGISTRY=$(mktemp "${TMPDIR:-/tmp}/.fm-test-cleanup.$$.XXXXXX") || return 1 + +fm_test_pid_identity() { + local pid=$1 + FM_STATE_OVERRIDE="${TMPDIR:-/tmp}" bash -c \ + '. "$1"; fm_pid_identity "$2"' _ "$ROOT/bin/fm-wake-lib.sh" "$pid" +} + +FM_TEST_OWNER_IDENTITY=$(fm_test_pid_identity "$$") || { + rm -f "$FM_TEST_CLEANUP_REGISTRY" + return 1 +} fm_test_cleanup() { local d for d in "${FM_TEST_CLEANUP_DIRS[@]:-}"; do [ -n "$d" ] && rm -rf "$d" done + if [ -f "$FM_TEST_CLEANUP_REGISTRY" ]; then + while IFS= read -r d; do + [ -n "$d" ] && rm -rf "$d" + done < "$FM_TEST_CLEANUP_REGISTRY" + rm -f "$FM_TEST_CLEANUP_REGISTRY" + fi } fm_test_tmproot() { local prefix=${1:-fm-test} root - root=$(mktemp -d "${TMPDIR:-/tmp}/${prefix}.XXXXXX") - if [ "${#FM_TEST_CLEANUP_DIRS[@]}" -eq 0 ]; then - trap fm_test_cleanup EXIT + root=$(mktemp -d "${TMPDIR:-/tmp}/${prefix}.XXXXXX") || return 1 + if ! printf '%s\n%s\n' "$$" "$FM_TEST_OWNER_IDENTITY" > "$root/.fm-test-fixture" || + ! printf '%s\n' "$root" >> "$FM_TEST_CLEANUP_REGISTRY"; then + rm -rf "$root" + return 1 fi - FM_TEST_CLEANUP_DIRS+=("$root") printf '%s\n' "$root" } +trap fm_test_cleanup EXIT +trap 'fm_test_cleanup; exit 130' INT +trap 'fm_test_cleanup; exit 143' TERM + +# fm_test_reap_orphans: best-effort sweep for fixture roots left behind by a +# prior run that was killed hard enough to skip the traps above (e.g. a +# SIGKILL timeout). Only removes directories carrying the .fm-test-fixture +# marker fm_test_tmproot writes, so it never touches unrelated fm-* tmp dirs +# from real (non-test) firstmate commands. The marker identifies the owning +# shell across PID reuse, so the same live owner always wins over the age +# fallback for dead or unowned roots. +FM_TEST_ORPHAN_MAX_AGE_SECONDS=${FM_TEST_ORPHAN_MAX_AGE_SECONDS:-3600} + +fm_test_reap_orphans() { + local marker dir mtime now owner_pid owner_identity current_identity + now=$(date +%s) + for marker in "${TMPDIR:-/tmp}"/fm-*/.fm-test-fixture; do + [ -e "$marker" ] || continue + owner_pid=$(sed -n '1p' "$marker" 2>/dev/null) || owner_pid= + owner_identity=$(sed -n '2,$p' "$marker" 2>/dev/null) || owner_identity= + case "$owner_pid" in + '' | *[!0-9]*) ;; + *) + current_identity=$(fm_test_pid_identity "$owner_pid" 2>/dev/null) || current_identity= + if [ -n "$owner_identity" ] && [ "$current_identity" = "$owner_identity" ]; then + continue + fi + ;; + esac + mtime=$(stat -c %Y "$marker" 2>/dev/null || stat -f %m "$marker" 2>/dev/null) || continue + [ $((now - mtime)) -ge "$FM_TEST_ORPHAN_MAX_AGE_SECONDS" ] || continue + dir=$(dirname "$marker") + rm -rf "$dir" + done +} + +fm_test_reap_orphans + # --- fakebin / PATH shims --------------------------------------------------- # # fm_fakebin creates /fakebin and echoes it; prepend it to PATH to diff --git a/tests/wake-helpers.sh b/tests/wake-helpers.sh index dd0277c1d8..5964598c76 100644 --- a/tests/wake-helpers.sh +++ b/tests/wake-helpers.sh @@ -32,13 +32,7 @@ fi # that channel, to exercise graceful degradation. Suites that do not source this # harness still cannot fire a real notification: the daemon defaults the seam to # "discard" whenever it is sourced (its library-mode guard). -# Create the recorder dir with mktemp directly (not fm_test_tmproot, whose -# first call installs an EXIT trap that, invoked inside a command-substitution -# subshell, would delete the dir on subshell exit). Register it for the same -# cleanup and install the trap in THIS shell if it is the first registration. -_fm_wedge_rec_dir=$(mktemp -d "${TMPDIR:-/tmp}/fm-wedge-rec.XXXXXX") -if [ "${#FM_TEST_CLEANUP_DIRS[@]}" -eq 0 ]; then trap fm_test_cleanup EXIT; fi -FM_TEST_CLEANUP_DIRS+=("$_fm_wedge_rec_dir") +_fm_wedge_rec_dir=$(fm_test_tmproot fm-wedge-rec) cat > "$_fm_wedge_rec_dir/rec" <<'REC' #!/usr/bin/env bash printf '%s\t%s\n' "${1:-}" "${2:-}" >> "${FM_WEDGE_ALARM_LOG:-/dev/null}" From 3089a5790f339e44db6e995b57451102d2aa803c Mon Sep 17 00:00:00 2001 From: Kun Chen <3233006+kunchenguid@users.noreply.github.com> Date: Tue, 4 Aug 2026 16:47:06 -0700 Subject: [PATCH 05/12] feat(herdr): enable presentation spaces by default (#1708) * feat(herdr): default presentation spaces on with an explicit opt-out Herdr's disposable one-task presentation workspace was opt-in through the presence of local config/herdr-presentation-spaces. It is now on by default, and a home opts out by writing "off" into that same file. Values are read with the whole-file whitespace-stripped convention the other scalar config items already use, plus case folding. An absent file, an empty file, and "on" all resolve on; only "off" opts out; an unrecognized value warns and keeps the default rather than failing a spawn over a purely visual setting. The empty file is exactly the historical opt-in form, so every home that had already enabled the projection stays enabled with no migration step, and no previously enabled home can be turned off by the flip. Because absence now means on at both ends, secondmate inheritance needs no item-specific convergence: mirroring an absent primary file converges a secondmate to the same default-on rather than turning its projection off, and only an explicit primary opt-out propagates the opt-out. The gate itself moves into fm_backend_herdr_presentation_enabled in the Herdr adapter so the semantics have one owner that regressions can exercise directly. * no-mistakes(document): Document Herdr default-on presentation safety --------- Co-authored-by: kunchenguid --- AGENTS.md | 4 +- bin/backends/herdr.sh | 39 +++++++++- bin/fm-config-inherit-lib.sh | 7 +- bin/fm-spawn.sh | 6 +- docs/architecture.md | 2 +- docs/configuration.md | 4 +- docs/herdr-backend.md | 16 ++-- docs/verification/runtime-backends.md | 21 ++++- tests/fm-backend-autodetect-smoke.test.sh | 3 + ...ckend-herdr-launcher-workspace-e2e.test.sh | 10 ++- .../fm-backend-herdr-presentation-e2e.test.sh | 77 +++++++++++++------ ...ckend-herdr-workspace-per-home-e2e.test.sh | 4 + tests/fm-backend-herdr.test.sh | 69 ++++++++++++++++- tests/fm-secondmate-harness.test.sh | 56 +++++++++++++- 14 files changed, 271 insertions(+), 47 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9f90089778..d47731baed 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -71,7 +71,7 @@ config/backlog-backend backlog backend override; LOCAL, gitignored; absent or " config/backend runtime session-provider backend override for new tasks; LOCAL, gitignored; absent = falls through to runtime auto-detection (the runtime firstmate itself is executing inside), then tmux; tmux is the verified reference backend (docs/tmux-backend.md), while herdr, zellij, orca, and cmux are experimental spawn backends (docs/herdr-backend.md, docs/zellij-backend.md, docs/orca-backend.md, docs/cmux-backend.md) - herdr and cmux can also be selected by runtime auto-detection, zellij and orca never are (always explicit), and codex-app is not accepted; see docs/codex-app-backend.md; inherited by secondmate homes under the primary-authoritative contract in secondmate-provisioning config/calm Pi Calm presentation preference; LOCAL, gitignored, and not inherited; see docs/configuration.md "Pi Calm preference" config/startup-memory-budget primary-authoritative per-home startup-memory budget; LOCAL, gitignored, materialized as 7,500 estimated tokens by locked primary bootstrap and inherited into secondmate homes; see docs/configuration.md "Startup memory budget" -config/herdr-presentation-spaces optional presence flag for Herdr's default-off disposable single-task visual projection; LOCAL, gitignored; inherited by secondmate homes; see docs/herdr-backend.md "Optional presentation spaces" +config/herdr-presentation-spaces optional "off" opt-out from Herdr's default-on disposable single-task visual projection; LOCAL, gitignored; inherited by secondmate homes; see docs/herdr-backend.md "Presentation spaces" config/trace-context optional presence flag enabling default-off native W3C trace-context propagation to spawned agents; LOCAL, gitignored; inherited by secondmate homes; see docs/configuration.md "Trace context propagation" and docs/trace-context.md config/cmux-socket-password optional cmux control-socket password; LOCAL, gitignored; read fresh on every cmux CLI call and passed through without ever overriding an operator's own ambient CMUX_SOCKET_PASSWORD when absent (docs/cmux-backend.md "Setup") config/wedge-alarm optional away-mode wedge-alarm active-alert directives; LOCAL, gitignored; absent means auto (macOS Notification Center when available); see docs/wedge-alarm.md @@ -92,7 +92,7 @@ state/ volatile runtime signals; gitignored .grok-turnend-token firstmate-owned grok hook registry token for the task; removed by teardown .kimi-turnend-token firstmate-owned Kimi hook registry token for the task; removed by teardown .meta written by fm-spawn: window=, endpoint_task_id=, worktree=, project=, harness=, model=, effort=, kind=, mode=, yolo=, tasktmp=; an optional traceparent= only when trace context is enabled (docs/configuration.md "Trace context propagation"); kind=secondmate also records home= and projects=, plus remote_host=/remote_root=/remote_backend=/remote_herdr_session=/remote_target= for a remote route; a non-default runtime backend records further backend-specific fields (docs/configuration.md "Runtime backend"; bin/fm-backend.sh, section 8); fm-pr-check, including through fm-pr-merge, records one canonical pr= and the forge's pr_head= when available (GitHub pull requests and GitLab merge requests; docs/gitlab-merge-watch.md); fm-x-link appends x_request=, x_request_ts=, x_followups=, and optional x_platform=/x_reply_max_chars= for an X-mode-originated task (section 14) - .herdr-presentation quarantinable attempt and restart-binding journal for Herdr's optional visual projection; never task or endpoint authority; see docs/herdr-backend.md "Optional presentation spaces" + .herdr-presentation quarantinable attempt and restart-binding journal for Herdr's optional visual projection; never task or endpoint authority; see docs/herdr-backend.md "Presentation spaces" .check.sh authenticated slow poll; the watcher dispatches validated PR data and the byte-identified X shim through trusted repository scripts, runs registered custom checks from hash-validated private snapshots, and rejects every other state check without execution .check-trust private content binding created by fm-check-register.sh for an intentional custom check .pr-poll private validated data sidecar for the byte-static PR merge poll diff --git a/bin/backends/herdr.sh b/bin/backends/herdr.sh index cb677be0cf..a84c71a3bd 100644 --- a/bin/backends/herdr.sh +++ b/bin/backends/herdr.sh @@ -15,8 +15,9 @@ # herdr-verification-p2.md "Task container shape", refined by # docs/herdr-backend.md "Default task container shape"): ONE herdr workspace PER # FIRSTMATE HOME (the primary, and each secondmate, gets its own), ONE herdr TAB -# per task inside its home's workspace. An optional, default-off presentation -# flag creates a disposable workspace for a clean fresh task instead. That +# per task inside its home's workspace. The default-on presentation projection +# creates a disposable workspace for a clean fresh task instead unless the home +# opts out. That # workspace is a non-authoritative visual projection containing only the normal # task pane. Its random token and mutable label never authorize lookup, # adoption, reuse, closure, deletion, task ownership, or endpoint selection. @@ -108,8 +109,8 @@ FM_BACKEND_HERDR_ESCALATED_PREFIX=".herdr-escalated-" # at a seeded secondmate home's root, containing exactly that secondmate's id. # The primary firstmate home never carries this marker. FM_BACKEND_HERDR_SECONDMATE_MARKER=".fm-secondmate-home" -# The default-off presentation projection is intentionally separate from the -# authoritative task endpoint record. +# The presentation projection is intentionally separate from the authoritative +# task endpoint record. # A per-task journal lives under state/ as .herdr-presentation. # Version 1 records only the attempted projection's random correlator. # Version 2 additionally binds the successful projection's exact home, @@ -118,6 +119,36 @@ FM_BACKEND_HERDR_SECONDMATE_MARKER=".fm-secondmate-home" # No send, capture, Treehouse, or general task-ownership path reads it. FM_BACKEND_HERDR_PRESENTATION_JOURNAL_SUFFIX=".herdr-presentation" +# The config item a home writes to opt OUT of the projection. +FM_BACKEND_HERDR_PRESENTATION_CONFIG="herdr-presentation-spaces" + +# fm_backend_herdr_presentation_enabled : true when this home's +# children should be projected into disposable one-task workspaces +# (docs/herdr-backend.md "Presentation spaces" owns the full contract). +# Projection is ON by default, so an absent config file enables it; a home opts +# out by writing "off". Values are read with the whole-file whitespace-stripped +# convention the other scalar config items already use (config/backlog-backend, +# config/crew-harness), plus case folding. An empty file is the historical +# presence-based opt-in form and still means on, so no home that had the +# projection enabled can be turned off by the default flip. An unrecognized +# value warns and keeps the default rather than failing a spawn over a purely +# visual setting, so a typo is visible instead of silently disabling anything. +fm_backend_herdr_presentation_enabled() { # + local config_dir=${1:-} file value + [ -n "$config_dir" ] || return 0 + file="$config_dir/$FM_BACKEND_HERDR_PRESENTATION_CONFIG" + [ -f "$file" ] || return 0 + value=$(tr -d '[:space:]' < "$file" 2>/dev/null | tr '[:upper:]' '[:lower:]') || value="" + case "$value" in + off) return 1 ;; + ''|on) return 0 ;; + *) + echo "warning: $file: unrecognized value \"$value\"; herdr presentation spaces stay on (write \"off\" to opt out)" >&2 + return 0 + ;; + esac +} + # fm_backend_herdr_workspace_label: the per-firstmate-HOME herdr workspace # label (docs/herdr-backend.md "Default task container shape"). The PRIMARY home (no # secondmate marker) resolves to the constant "firstmate", byte-identical to diff --git a/bin/fm-config-inherit-lib.sh b/bin/fm-config-inherit-lib.sh index 554fc993bb..374ee5078d 100644 --- a/bin/fm-config-inherit-lib.sh +++ b/bin/fm-config-inherit-lib.sh @@ -8,8 +8,11 @@ # hand-edit backlog files too, primary config/backend pins that home's local # runtime-backend default for future spawns, primary config/startup-memory-budget # bounds that home's startup-memory curation, and primary -# config/herdr-presentation-spaces enables the same default-off Herdr presentation -# projection, and primary +# config/herdr-presentation-spaces carries the same Herdr presentation-projection +# choice - that item is default-ON, so an absent primary file and an absent +# destination file both mean on and the generic absence mirror below already +# converges a secondmate to the primary's default rather than turning it off; +# only an explicit primary "off" propagates an opt-out, and primary # config/trace-context is copied at the launch convergence point as part of the # default-off W3C trace-context setup, while live convergence leaves it unchanged. # The primary passes its frozen home-session decision into a newly launched diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index 273a134425..088079d025 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -51,8 +51,8 @@ # outside herdr has no workspace to inherit and uses this home's own labeled # workspace, which must then match exactly one. --secondmate is the deliberate # exception: it stands up that secondmate home's own workspace. -# Herdr additionally supports a default-off presentation-only layout when the -# local config/herdr-presentation-spaces flag exists. A clean fresh task first +# Herdr additionally uses a default-on presentation-only layout unless the +# local config/herdr-presentation-spaces file says off. A clean fresh task first # writes state/.herdr-presentation atomically, then creates a disposable # workspace containing only the ordinary task pane. A successful clean create # upgrades its attempt journal with exact home, session, workspace, tab, pane, @@ -1408,7 +1408,7 @@ case "$BACKEND" in fi HERDR_PRESENTATION_JOURNAL=$(fm_backend_herdr_projection_journal_path "$STATE" "$ID") HERDR_PROJECTED=0 - if [ "$KIND" != secondmate ] && [ -f "$CONFIG/herdr-presentation-spaces" ]; then + if [ "$KIND" != secondmate ] && fm_backend_herdr_presentation_enabled "$CONFIG"; then HERDR_SES=$(fm_backend_herdr_session) HERDR_PARENT_LABEL=$(FM_HOME="$HERDR_LABEL_HOME" fm_backend_herdr_workspace_label) if [ -e "$HERDR_PRESENTATION_JOURNAL" ] || [ -L "$HERDR_PRESENTATION_JOURNAL" ]; then diff --git a/docs/architecture.md b/docs/architecture.md index cc8f182452..335a7b3ae2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -122,7 +122,7 @@ For capable Herdr sessions, the same watcher replaces its terminal sleep with a The deeper session-start agent-process liveness probe is separate from that busy-state poll: tmux and Herdr have verified classifiers for secondmate recovery, Zellij remains unverified, and Orca and cmux do not support secondmate spawns. Herdr is experimental and can be selected explicitly or by runtime auto-detection: Treehouse remains its worktree provider, [`herdr-backend.md`](herdr-backend.md) owns current setup and safety limits, and [`verification/runtime-backends.md`](verification/runtime-backends.md#herdr) owns active empirical evidence. Herdr uses one tab per task; [Watching and task containers](herdr-backend.md#watching-and-task-containers) owns launcher-bound workspace placement, the label-only fallback, and recovery scope. -Its optional default-off presentation projection may place one clean new task in a disposable workspace without changing endpoint authority or lifecycle ownership; [Optional presentation spaces](herdr-backend.md#optional-presentation-spaces) owns that conditional design and its narrow home-local restored-shell cleanup at locked session start. +Its default-on presentation projection may place one clean new task in a disposable workspace without changing endpoint authority or lifecycle ownership; [Presentation spaces](herdr-backend.md#presentation-spaces) owns that conditional design and its narrow home-local restored-shell cleanup at locked session start. Zellij is experimental and selected only explicitly: Treehouse remains its worktree provider, [`zellij-backend.md`](zellij-backend.md) owns current setup and limits, and [`verification/runtime-backends.md`](verification/runtime-backends.md#zellij) owns active empirical evidence. Zellij's container shape is simpler than herdr's: one shared `firstmate` session, one tab per task, with no per-home workspace split; visible tab titles are scoped by the active home label plus a short hash of the resolved `FM_ROOT` path. Orca is experimental and selected only explicitly: Orca owns both worktree and terminal lifecycle, records `orca_worktree_id=` and `terminal=`, and removes worktrees through `orca worktree rm` only after the usual firstmate teardown checks pass. diff --git a/docs/configuration.md b/docs/configuration.md index 02bcb12858..d93e14e609 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -82,8 +82,8 @@ Missing, empty, duplicate, malformed, backend-inconsistent, or task-mismatched e Legacy tmux metadata remains cleanup-compatible when its exact window name is `fm-`; opaque non-tmux endpoints require their recorded `endpoint_task_id=` binding. `FM_HOME` determines Herdr's home label: the primary home uses `firstmate`, and a secondmate home marked by `.fm-secondmate-home` uses `2ndmate-`. [`herdr-backend.md`](herdr-backend.md#watching-and-task-containers) owns launcher-bound workspace placement, the label-only fallback, collision handling, and recovery behavior. -The optional local `config/herdr-presentation-spaces` presence flag instead enables Herdr's default-off disposable single-task visual projection; [Optional presentation spaces](herdr-backend.md#optional-presentation-spaces) owns its behavior, safety limits, recovery contract, and narrow locked session-start cleanup of exact restored idle-shell children. -The flag is default-off and inherited into secondmate homes under the primary-authoritative contract owned by [`secondmate-provisioning`](../.agents/skills/secondmate-provisioning/SKILL.md). +The local `config/herdr-presentation-spaces` file instead opts a home out of Herdr's default-on disposable single-task visual projection; [Presentation spaces](herdr-backend.md#presentation-spaces) owns its accepted values, default, migration, behavior, safety limits, recovery contract, and narrow locked session-start cleanup of exact restored idle-shell children. +The setting is inherited into secondmate homes under the primary-authoritative contract owned by [`secondmate-provisioning`](../.agents/skills/secondmate-provisioning/SKILL.md). For normal herdr operations, `HERDR_SESSION` selects the named session, but destructive test cleanup must not rely on `HERDR_SESSION` alone. Use the explicit guarded cleanup path described in [`docs/herdr-backend.md`](herdr-backend.md) instead of `herdr server stop`. For normal zellij operations, `FM_ZELLIJ_SESSION` selects the named session and defaults to `firstmate`. diff --git a/docs/herdr-backend.md b/docs/herdr-backend.md index 9834678df0..27ebd7250d 100644 --- a/docs/herdr-backend.md +++ b/docs/herdr-backend.md @@ -1,7 +1,7 @@ # Herdr runtime backend Herdr is an experimental agent-native terminal backend with native per-pane agent state and push events. -Firstmate requires Herdr protocol 14 or newer; versions 0.7.1, 0.7.3, 0.7.4, and 0.7.5 are verified, with protocol-16 features enabled only when available. +Firstmate requires Herdr protocol 14 or newer; broad backend verification covers versions 0.7.1, 0.7.3, 0.7.4, and 0.7.5, while the presentation-projection suite is additionally verified on 0.8.0 protocol 19 and protocol-16 features remain gated by availability. Herdr provides the terminal session while Treehouse continues to provide task worktrees. [`configuration.md`](configuration.md#runtime-backend-configbackend--fm_backend) owns shared backend selection and metadata semantics. @@ -66,12 +66,16 @@ Existing task operations use recorded endpoint ids and do not move a live task w The per-home workspace is reused while it has task tabs. Closing its last tab can remove the workspace, and the next spawn recreates it. -## Optional presentation spaces +## Presentation spaces -Create local gitignored `config/herdr-presentation-spaces` to request a disposable one-task workspace for each new crewmate or scout. -The setting is inherited into secondmate homes through the normal configuration-convergence owner. +Each new crewmate or scout is placed in a disposable one-task workspace by default. +A home opts out by writing `off` into local gitignored `config/herdr-presentation-spaces`. +An absent file, an empty file, and the value `on` all keep the projection enabled, values are compared with whitespace stripped and case ignored, and an unrecognized value warns and keeps the projection enabled rather than failing a spawn over a purely visual setting. +The empty file is the historical presence-based opt-in form, so every home that had already enabled the projection stays enabled with no migration step, and no previously enabled home can be turned off by the default. +A home that never created the file gains the projection at its next Herdr spawn; that flip is deliberate, and it reaches only the Herdr backend because no other runtime backend has a projection path. +The setting is inherited into secondmate homes through the normal configuration-convergence owner, and the default needs no special convergence: the primary's absent file and the secondmate's absent file both mean on, so leaving the default converges a secondmate to the same default rather than turning it off, and only an explicit primary `off` propagates the opt-out. A secondmate agent itself always stays in its ordinary parent workspace; only children launched by that home are eligible. -An absent or unconverged setting keeps the flat default. +An unconverged opt-out keeps the default projection in that home until convergence. Presentation is a best-effort visual projection, never task ownership or lifecycle authority. Only a fresh task with neither metadata nor an existing presentation journal is eligible for projected creation. @@ -141,6 +145,8 @@ A malformed or missing title or token, duplicate token, zero or multiple journal Operational compromises: - Grouping is best-effort; only an exact same-identity version 2 binding survives a Herdr restart in place. +- A failed journal publication or projected workspace create stops that spawn instead of falling back flat, so a Herdr create failure surfaces as a spawn failure in every Herdr home rather than only in homes that opted in; every earlier degradation on the fresh projected-create path (no session server, contended presentation lock, absent or ambiguous parent) still warns and continues flat. +- Recovery of an existing presentation journal deliberately refuses the spawn when the shared presentation lock is contended rather than falling back flat, and default-on makes that refusal reachable in any Herdr home. - Existing layouts are not force-renamed or rearranged. - Missing or ambiguous restart bindings fall back to the ordinary home workspace while the old projection remains untouched. - Crashes, lost responses, failed exact-pane cleanup, or human renames can leave quarantined spaces; session start removes only the exact home-local, uniquely journal-correlated, childless idle-shell shape above. diff --git a/docs/verification/runtime-backends.md b/docs/verification/runtime-backends.md index 544c8526fd..6c6cb18268 100644 --- a/docs/verification/runtime-backends.md +++ b/docs/verification/runtime-backends.md @@ -166,7 +166,7 @@ Claude, Codex, OpenCode, Pi, pi-signed, Grok, and Kimi share that backend cleanu ## Herdr The compatibility floor is protocol 14. -The latest active verification uses Herdr 0.7.5 protocol 17 on macOS aarch64, with earlier 0.7.5 protocol-16, 0.7.4, protocol-14, and 0.7.3 evidence retained where they define current behavior or fallbacks. +The presentation-projection suite's latest active verification uses Herdr 0.8.0 protocol 19 on macOS aarch64, every other section's latest uses Herdr 0.7.5 protocol 17 on macOS aarch64, and earlier 0.7.5 protocol-16, 0.7.4, protocol-14, and 0.7.3 evidence is retained where it defines current behavior or fallbacks. Protocol 17 keeps every protocol-16 feature gate satisfied; the event and workspace-move floors remain 16. Core read-only probes: @@ -325,6 +325,25 @@ ok - real Herdr lab: missing, renamed, and duplicate tokens trigger zero destruc ok - real Herdr lab validation completed on Herdr 0.7.5 with the default-session tripwire intact ``` +The projection suite ran again on 2026-08-04 against Herdr 0.8.0 protocol 19 for the default-on flip, where an absent `config/herdr-presentation-spaces` enables the projection and only the value `off` opts out: + +```sh +HERDR_LAB_HELPER=bin/fm-herdr-lab.sh \ + tests/fm-backend-herdr-presentation-e2e.test.sh +``` + +Observed default and opt-out guarantees: + +```text +ok - real Herdr lab: an opted-out spawn retains the Stage 1 Herdr command sequence with zero ordering calls +ok - real Herdr lab: a home that configured nothing is projected by default +ok - real Herdr lab: the primary presentation setting inherits into real secondmate homes +ok - real Herdr lab validation completed on Herdr 0.8.0 with the default-session tripwire intact +``` + +The projected spawn in that run used the historical empty opt-in file, so a home that had already enabled the projection keeps it without any migration step. +One concurrent cross-home recovery case refused under contention on a loaded machine and passed on an immediate rerun; recovery-path presentation lock contention is a deliberate hard refusal rather than a flat fallback, which default-on now makes reachable from any Herdr home. + The restored-shell session-start cleanup ran on 2026-07-24 against Herdr 0.7.5 protocol 17: ```sh diff --git a/tests/fm-backend-autodetect-smoke.test.sh b/tests/fm-backend-autodetect-smoke.test.sh index 54bdf85484..ef3ab7c2ed 100755 --- a/tests/fm-backend-autodetect-smoke.test.sh +++ b/tests/fm-backend-autodetect-smoke.test.sh @@ -87,6 +87,9 @@ trap on_exit EXIT STATE="$TMP_ROOT/state"; DATA="$TMP_ROOT/data"; CONFIG="$TMP_ROOT/config" mkdir -p "$STATE" "$DATA/$ID" "$CONFIG" +# Backend auto-detection is what is under test here, so opt out of the default-on +# presentation projection and keep the assertions on the flat per-home workspace. +printf 'off\n' > "$CONFIG/herdr-presentation-spaces" printf 'trivial autodetect-smoke brief: nothing to do.\n' > "$DATA/$ID/brief.md" PROJ="$TMP_ROOT/scratch-project" diff --git a/tests/fm-backend-herdr-launcher-workspace-e2e.test.sh b/tests/fm-backend-herdr-launcher-workspace-e2e.test.sh index 3cb8b49d0d..1fb79f1f0e 100755 --- a/tests/fm-backend-herdr-launcher-workspace-e2e.test.sh +++ b/tests/fm-backend-herdr-launcher-workspace-e2e.test.sh @@ -153,11 +153,15 @@ LAB_SOCKET=$(lab session list --json 2>/dev/null \ # --- scratch world ---------------------------------------------------------- +# Presentation spaces are on by default, so every home that asserts the FLAT +# layout below opts out explicitly rather than depending on that default. PRIMARY_HOME="$TMP_ROOT/primary-home" mkdir -p "$PRIMARY_HOME/state" "$PRIMARY_HOME/config" +printf 'off\n' > "$PRIMARY_HOME/config/herdr-presentation-spaces" SM_ID="lwsm1" SM_HOME="$TMP_ROOT/secondmate-home" mkdir -p "$SM_HOME/state" "$SM_HOME/config" "$SM_HOME/projects" "$SM_HOME/bin" "$SM_HOME/data" +printf 'off\n' > "$SM_HOME/config/herdr-presentation-spaces" printf '# scratch secondmate home AGENTS.md placeholder\n' > "$SM_HOME/AGENTS.md" printf '%s\n' "$SM_ID" > "$SM_HOME/.fm-secondmate-home" printf 'trivial e2e secondmate charter: nothing to do.\n' > "$SM_HOME/data/charter.md" @@ -165,12 +169,14 @@ printf 'trivial e2e secondmate charter: nothing to do.\n' > "$SM_HOME/data/chart SM2_ID="lwsm2" SM2_HOME="$TMP_ROOT/secondmate-home-2" mkdir -p "$SM2_HOME/state" "$SM2_HOME/config" "$SM2_HOME/projects" "$SM2_HOME/bin" "$SM2_HOME/data" +printf 'off\n' > "$SM2_HOME/config/herdr-presentation-spaces" printf '# scratch secondmate home AGENTS.md placeholder\n' > "$SM2_HOME/AGENTS.md" printf '%s\n' "$SM2_ID" > "$SM2_HOME/.fm-secondmate-home" printf 'trivial e2e secondmate charter: nothing to do.\n' > "$SM2_HOME/data/charter.md" -# A third primary-shaped home with presentation spaces ON, so the flat-path -# homes above stay flag-free and each layout is asserted in isolation. +# A third primary-shaped home that keeps presentation spaces ON through the +# historical empty opt-in file, so the default-on migration is exercised against +# real Herdr while the opted-out homes above assert the flat layout in isolation. PRES_HOME="$TMP_ROOT/presentation-home" mkdir -p "$PRES_HOME/state" "$PRES_HOME/config" : > "$PRES_HOME/config/herdr-presentation-spaces" diff --git a/tests/fm-backend-herdr-presentation-e2e.test.sh b/tests/fm-backend-herdr-presentation-e2e.test.sh index d168f3deae..0a02a40000 100755 --- a/tests/fm-backend-herdr-presentation-e2e.test.sh +++ b/tests/fm-backend-herdr-presentation-e2e.test.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -# Isolated real-Herdr E2E coverage for the default-off disposable single-task -# presentation projection and its best-effort owning-parent ordering across -# primary and secondmate homes. +# Isolated real-Herdr E2E coverage for the default-on disposable single-task +# presentation projection, its explicit opt-out, and its best-effort +# owning-parent ordering across primary and secondmate homes. # The test drives the real spawn and teardown scripts, a real Treehouse pool, # and the guarded named-session lab helper. set -u @@ -455,8 +455,11 @@ mkdir -p "$HOME_DIR/state" "$HOME_DIR/config" \ "$HOME_DIR/data/order-fail" "$HOME_DIR/data/fm-hibit-resume-r1" \ "$HOME_DIR/data/wheelhouse-healing-r1" mkdir -p "$HOME_DIR/data/active-seeded" "$HOME_DIR/data/abort-a" "$HOME_DIR/data/abort-b" \ - "$HOME_DIR/data/lock-contended" + "$HOME_DIR/data/lock-contended" "$HOME_DIR/data/default-on" touch "$HOME_DIR/state/.last-watcher-beat" +# Presentation spaces are on by default, so the flat baseline below opts out +# explicitly; the projected cases each restate the setting they exercise. +printf 'off\n' > "$HOME_DIR/config/herdr-presentation-spaces" printf 'Projection anchor fixture.\n' > "$HOME_DIR/data/anchor/brief.md" printf 'Projection E2E fixture.\n' > "$HOME_DIR/data/shape/brief.md" printf 'Projection ordering fixture A.\n' > "$HOME_DIR/data/order-a/brief.md" @@ -468,38 +471,66 @@ printf 'Projection active seeded fixture.\n' > "$HOME_DIR/data/active-seeded/bri printf 'Projection abort fixture A.\n' > "$HOME_DIR/data/abort-a/brief.md" printf 'Projection abort fixture B.\n' > "$HOME_DIR/data/abort-b/brief.md" printf 'Projection lock contention fixture.\n' > "$HOME_DIR/data/lock-contended/brief.md" +printf 'Projection default-on fixture.\n' > "$HOME_DIR/data/default-on/brief.md" make_project "$PROJECT_DIR" # Keep one ordinary primary task live so the durable firstmate workspace is # first and remains present while disposable workers are projected around it. spawn_task anchor "$HOME_DIR" "$PROJECT_DIR" > "$TMP_ROOT/anchor.out" 2> "$TMP_ROOT/anchor.err" \ - || fail "flag-off anchor spawn failed: $(cat "$TMP_ROOT/anchor.err")" + || fail "opted-out anchor spawn failed: $(cat "$TMP_ROOT/anchor.err")" ANCHOR_META="$HOME_DIR/state/anchor.meta" remember_meta_worktree "$ANCHOR_META" >/dev/null FIRSTMATE_WSID=$(grep '^herdr_workspace_id=' "$ANCHOR_META" | cut -d= -f2-) [ -n "$FIRSTMATE_WSID" ] || fail "anchor metadata did not record the firstmate workspace" -# The same task id and project run once with the flag absent and once with it -# present, so Treehouse commands and metadata can be compared directly. +# The same task id and project run once opted out and once projected, so +# Treehouse commands and metadata can be compared directly. : > "$TREEHOUSE_CALL_LOG" OFF_HERDR_START=$(log_line_count) OFF_MOVE_START=$(wc -l < "$MOVE_CALL_LOG" | tr -d '[:space:]') spawn_task shape "$HOME_DIR" "$PROJECT_DIR" > "$TMP_ROOT/off.out" 2> "$TMP_ROOT/off.err" \ - || fail "flag-off spawn failed: $(cat "$TMP_ROOT/off.err")" + || fail "opted-out spawn failed: $(cat "$TMP_ROOT/off.err")" OFF_HERDR_END=$(log_line_count) OFF_META="$TMP_ROOT/off.meta" cp "$HOME_DIR/state/shape.meta" "$OFF_META" OFF_WT=$(remember_meta_worktree "$OFF_META") cp "$TREEHOUSE_CALL_LOG" "$TMP_ROOT/off-treehouse.log" [ "$(wc -l < "$MOVE_CALL_LOG" | tr -d '[:space:]')" = "$OFF_MOVE_START" ] \ - || fail "flag-off spawn invoked the presentation-only workspace mover" + || fail "opted-out spawn invoked the presentation-only workspace mover" OFF_HERDR_CALLS=$(sed -n "$((OFF_HERDR_START + 1)),${OFF_HERDR_END}p" "$HERDR_CALL_LOG") if printf '%s\n' "$OFF_HERDR_CALLS" | grep -E $'^(api\tschema|session\tlist)' >/dev/null 2>&1; then - fail "flag-off spawn added presentation-ordering capability or socket calls" + fail "opted-out spawn added presentation-ordering capability or socket calls" fi -pass "real Herdr lab: flag-off spawn retains the Stage 1 Herdr command sequence with zero ordering calls" +pass "real Herdr lab: an opted-out spawn retains the Stage 1 Herdr command sequence with zero ordering calls" teardown_task shape "$HOME_DIR" > "$TMP_ROOT/off-teardown.out" 2> "$TMP_ROOT/off-teardown.err" \ - || fail "flag-off teardown failed: $(cat "$TMP_ROOT/off-teardown.err")" + || fail "opted-out teardown failed: $(cat "$TMP_ROOT/off-teardown.err")" + +# A home that configured nothing at all must be projected: this is the default, +# and the only difference from the opted-out spawn above is the removed file. +rm -f "$HOME_DIR/config/herdr-presentation-spaces" +spawn_task default-on "$HOME_DIR" "$PROJECT_DIR" > "$TMP_ROOT/default-on.out" 2> "$TMP_ROOT/default-on.err" \ + || fail "default-on spawn failed: $(cat "$TMP_ROOT/default-on.err")" +DEFAULT_ON_META="$HOME_DIR/state/default-on.meta" +remember_meta_worktree "$DEFAULT_ON_META" >/dev/null +DEFAULT_ON_JOURNAL="$HOME_DIR/state/default-on.herdr-presentation" +[ -f "$DEFAULT_ON_JOURNAL" ] \ + || fail "an unconfigured home did not publish a presentation journal by default" +DEFAULT_ON_TOKEN=$(grep '^projection_id=' "$DEFAULT_ON_JOURNAL" | cut -d= -f2-) +DEFAULT_ON_WSID=$(grep '^herdr_workspace_id=' "$DEFAULT_ON_META" | cut -d= -f2-) +[ -n "$DEFAULT_ON_WSID" ] && [ "$DEFAULT_ON_WSID" != "$FIRSTMATE_WSID" ] \ + || fail "an unconfigured home reused the flat firstmate workspace instead of projecting" +DEFAULT_ON_LABEL=$(lab workspace get "$DEFAULT_ON_WSID" | jq -r '.result.workspace.label // empty') +[ "$DEFAULT_ON_LABEL" = "└ default-on · p:$DEFAULT_ON_TOKEN" ] \ + || fail "default-on projection used an unexpected workspace label: $DEFAULT_ON_LABEL" +pass "real Herdr lab: a home that configured nothing is projected by default" +teardown_task default-on "$HOME_DIR" > "$TMP_ROOT/default-on-teardown.out" 2> "$TMP_ROOT/default-on-teardown.err" \ + || fail "default-on teardown failed: $(cat "$TMP_ROOT/default-on-teardown.err")" +if lab workspace get "$DEFAULT_ON_WSID" >/dev/null 2>&1; then + fail "default-on teardown left its disposable workspace behind" +fi +# The ordering scenarios below read the whole move log cumulatively against the +# projected workspaces that are still live, so this retired one starts them clean. +: > "$MOVE_CALL_LOG" SECOND_ONE_OUT=$(lab workspace create --cwd "$PROJECT_DIR" --label 2ndmate-alpha --no-focus) \ || fail "could not create the first secondmate presentation fixture" @@ -516,6 +547,8 @@ CAPTAIN_FOCUS="$SECOND_TWO_WSID/$SECOND_TWO_TAB" assert_focus_is "$CAPTAIN_FOCUS" "focused secondmate fixture" : > "$TREEHOUSE_CALL_LOG" +# The historical presence-based opt-in was an empty file; it must still project, +# so no home that had already enabled the projection is turned off by the default. : > "$HOME_DIR/config/herdr-presentation-spaces" SHAPE_FOCUS_AUDIT_START=$(focus_audit_line_count) spawn_task shape "$HOME_DIR" "$PROJECT_DIR" > "$TMP_ROOT/on.out" 2> "$TMP_ROOT/on.err" \ @@ -526,7 +559,7 @@ ON_META="$TMP_ROOT/on.meta" cp "$HOME_DIR/state/shape.meta" "$ON_META" ON_WT=$(remember_meta_worktree "$ON_META") cmp -s "$TMP_ROOT/off-treehouse.log" "$TREEHOUSE_CALL_LOG" \ - || fail "Treehouse command sequence changed between flag-off and projected spawns" + || fail "Treehouse command sequence changed between opted-out and projected spawns" JOURNAL="$HOME_DIR/state/shape.herdr-presentation" [ -f "$JOURNAL" ] || fail "projected spawn did not publish its presentation journal" TOKEN=$(grep '^projection_id=' "$JOURNAL" | cut -d= -f2-) @@ -659,7 +692,7 @@ PROJECTION_ORDER_START=$(log_line_count) normalize_meta "$OFF_META" > "$TMP_ROOT/off.meta.normalized" normalize_meta "$ON_META" > "$TMP_ROOT/on.meta.normalized" cmp -s "$TMP_ROOT/off.meta.normalized" "$TMP_ROOT/on.meta.normalized" \ - || fail "metadata changed beyond Herdr container IDs between flag-off and projected paths" + || fail "metadata changed beyond Herdr container IDs between opted-out and projected paths" # Two real concurrent primary spawns share the bounded presentation-order lock. # Their final relative order must match Herdr's actual serialized create order, @@ -875,18 +908,18 @@ mkdir -p "$SECOND_HOME_A/bin" printf '# Firstmate secondmate fixture\n' > "$SECOND_HOME_A/AGENTS.md" printf 'Secondmate alpha charter.\n' > "$SECOND_HOME_A/data/charter.md" -# Primary flag only; real inheritance must push presence into both secondmate homes. +# Primary setting only; real inheritance must push it into both secondmate homes. [ -f "$HOME_DIR/config/herdr-presentation-spaces" ] \ - || fail "primary presentation flag disappeared before multi-home inheritance" + || fail "primary presentation setting disappeared before multi-home inheritance" [ ! -e "$SECOND_HOME_A/config/herdr-presentation-spaces" ] \ - || fail "secondmate A unexpectedly had the presentation flag before inheritance" + || fail "secondmate A unexpectedly had a local presentation setting before inheritance" [ ! -e "$SECOND_HOME_B/config/herdr-presentation-spaces" ] \ - || fail "secondmate B unexpectedly had the presentation flag before inheritance" + || fail "secondmate B unexpectedly had a local presentation setting before inheritance" SECOND_SPAWN_LOG_START=$(log_line_count) spawn_secondmate_task alpha "$SECOND_HOME_A" > "$TMP_ROOT/alpha.out" 2> "$TMP_ROOT/alpha.err" \ || fail "secondmate alpha spawn failed: $(cat "$TMP_ROOT/alpha.err")" [ -f "$SECOND_HOME_A/config/herdr-presentation-spaces" ] \ - || fail "secondmate spawn did not inherit the presentation flag" + || fail "secondmate spawn did not inherit the presentation setting" [ ! -e "$HOME_DIR/state/alpha.herdr-presentation" ] \ || fail "secondmate spawn published a presentation journal" SECOND_META="$HOME_DIR/state/alpha.meta" @@ -909,10 +942,10 @@ propagate_inheritable_config "$HOME_DIR/config" "$SECOND_HOME_A/config" \ propagate_inheritable_config "$HOME_DIR/config" "$SECOND_HOME_B/config" \ || fail "inheritance into secondmate B failed" [ -f "$SECOND_HOME_A/config/herdr-presentation-spaces" ] \ - || fail "primary presentation flag did not reach secondmate A" + || fail "primary presentation setting did not reach secondmate A" [ -f "$SECOND_HOME_B/config/herdr-presentation-spaces" ] \ - || fail "primary presentation flag did not reach secondmate B" -pass "real Herdr lab: primary presentation opt-in inherits into real secondmate homes" + || fail "primary presentation setting did not reach secondmate B" +pass "real Herdr lab: the primary presentation setting inherits into real secondmate homes" # Keep the pre-existing 2ndmate-alpha/bravo workspaces as owning parents and captain focus. assert_focus_is "$CAPTAIN_FOCUS" "multi-home captain focus" diff --git a/tests/fm-backend-herdr-workspace-per-home-e2e.test.sh b/tests/fm-backend-herdr-workspace-per-home-e2e.test.sh index 1cb2f1f5a8..f857ebc694 100755 --- a/tests/fm-backend-herdr-workspace-per-home-e2e.test.sh +++ b/tests/fm-backend-herdr-workspace-per-home-e2e.test.sh @@ -84,12 +84,16 @@ fm_backend_source herdr || fail "fm_backend_source herdr failed" # --- scratch world: a primary-shaped home, a secondmate-shaped home, two projects --- +# This test asserts the per-home FLAT workspace shape, so both homes opt out of +# the default-on presentation projection rather than depending on that default. PRIMARY_HOME="$TMP_ROOT/primary-home" mkdir -p "$PRIMARY_HOME/state" "$PRIMARY_HOME/data/cm1" "$PRIMARY_HOME/config" +printf 'off\n' > "$PRIMARY_HOME/config/herdr-presentation-spaces" printf 'trivial e2e primary crewmate brief: nothing to do.\n' > "$PRIMARY_HOME/data/cm1/brief.md" SM_HOME="$TMP_ROOT/secondmate-home" mkdir -p "$SM_HOME/state" "$SM_HOME/data/cm2" "$SM_HOME/config" "$SM_HOME/projects" "$SM_HOME/bin" +printf 'off\n' > "$SM_HOME/config/herdr-presentation-spaces" printf '# scratch secondmate home AGENTS.md placeholder\n' > "$SM_HOME/AGENTS.md" printf 'e2esm1\n' > "$SM_HOME/.fm-secondmate-home" printf 'trivial e2e secondmate charter: nothing to do.\n' > "$SM_HOME/data/charter.md" diff --git a/tests/fm-backend-herdr.test.sh b/tests/fm-backend-herdr.test.sh index 16166ac283..16356cc2ad 100755 --- a/tests/fm-backend-herdr.test.sh +++ b/tests/fm-backend-herdr.test.sh @@ -831,7 +831,70 @@ test_create_task_creates_with_no_focus_flag() { pass "fm_backend_herdr_create_task: tab create passes --no-focus" } -# --- default-off disposable presentation projection ------------------------ +# --- default-on disposable presentation projection -------------------------- + +# fm_backend_herdr_presentation_enabled is the one gate bin/fm-spawn.sh consults +# before projecting a crewmate or scout, so these cases pin the default-on +# contract and its explicit opt-out at that interface. +presentation_enabled_verdict() { # -> "on"/"off" on stdout, warnings on stderr + bash -c ' + . "$0/bin/backends/herdr.sh" + if fm_backend_herdr_presentation_enabled "$1"; then printf "on\n"; else printf "off\n"; fi + ' "$ROOT" "$1" +} + +test_presentation_defaults_on_without_config() { + local dir config verdict + dir="$TMP_ROOT/presentation-default-on"; config="$dir/config"; mkdir -p "$config" + verdict=$(presentation_enabled_verdict "$config" 2>/dev/null) + [ "$verdict" = on ] || fail "an absent presentation config must resolve on, got '$verdict'" + verdict=$(presentation_enabled_verdict "$dir/missing-config-dir" 2>/dev/null) + [ "$verdict" = on ] || fail "a missing config dir must resolve on, got '$verdict'" + pass "herdr presentation: a home that set nothing gets the projection by default" +} + +test_presentation_legacy_opt_in_file_still_resolves_on() { + local dir config verdict stderr + dir="$TMP_ROOT/presentation-legacy-opt-in"; config="$dir/config"; mkdir -p "$config" + stderr="$dir/legacy.err" + # The historical opt-in was a bare `touch` of the file, so an empty file must + # keep meaning on - and must not warn, or every migrated home warns on every spawn. + : > "$config/herdr-presentation-spaces" + verdict=$(presentation_enabled_verdict "$config" 2>"$stderr") + [ "$verdict" = on ] || fail "a legacy empty opt-in file must resolve on, got '$verdict'" + [ ! -s "$stderr" ] || fail "a legacy empty opt-in file must not warn: $(cat "$stderr")" + printf '\n \n' > "$config/herdr-presentation-spaces" + verdict=$(presentation_enabled_verdict "$config" 2>"$stderr") + [ "$verdict" = on ] || fail "a whitespace-only opt-in file must resolve on, got '$verdict'" + [ ! -s "$stderr" ] || fail "a whitespace-only opt-in file must not warn: $(cat "$stderr")" + printf 'on\n' > "$config/herdr-presentation-spaces" + verdict=$(presentation_enabled_verdict "$config" 2>/dev/null) + [ "$verdict" = on ] || fail "an explicit on must resolve on, got '$verdict'" + pass "herdr presentation: an already-enabled home keeps the projection with no migration step" +} + +test_presentation_explicit_off_opts_out() { + local dir config verdict value + dir="$TMP_ROOT/presentation-opt-out"; config="$dir/config"; mkdir -p "$config" + for value in 'off' 'off +' ' off ' 'OFF' 'Off'; do + printf '%s' "$value" > "$config/herdr-presentation-spaces" + verdict=$(presentation_enabled_verdict "$config" 2>/dev/null) + [ "$verdict" = off ] || fail "the opt-out value '$value' must resolve off, got '$verdict'" + done + pass "herdr presentation: an explicit off opts the home out" +} + +test_presentation_unrecognized_value_warns_and_keeps_default() { + local dir config verdict stderr + dir="$TMP_ROOT/presentation-unrecognized"; config="$dir/config"; mkdir -p "$config" + stderr="$dir/unrecognized.err" + printf 'disabled\n' > "$config/herdr-presentation-spaces" + verdict=$(presentation_enabled_verdict "$config" 2>"$stderr") + [ "$verdict" = on ] || fail "an unrecognized value must keep the default on, got '$verdict'" + [ -s "$stderr" ] || fail "an unrecognized value must warn so a typo is visible" + pass "herdr presentation: an unrecognized value warns and keeps the default instead of failing a spawn" +} test_projection_journal_is_atomic_and_uses_128_bit_token() { local dir state out token parsed status @@ -3898,6 +3961,10 @@ test_create_task_refuses_when_agent_state_ambiguous test_create_task_husk_replacement_creates_before_closing test_create_task_creates_and_parses_ids test_create_task_creates_with_no_focus_flag +test_presentation_defaults_on_without_config +test_presentation_legacy_opt_in_file_still_resolves_on +test_presentation_explicit_off_opts_out +test_presentation_unrecognized_value_warns_and_keeps_default test_projection_journal_is_atomic_and_uses_128_bit_token test_projection_journal_v2_binds_and_advances_exact_endpoint test_projection_create_uses_exact_response_ids_and_leaves_one_task_pane diff --git a/tests/fm-secondmate-harness.test.sh b/tests/fm-secondmate-harness.test.sh index a39943706c..40cee35532 100755 --- a/tests/fm-secondmate-harness.test.sh +++ b/tests/fm-secondmate-harness.test.sh @@ -18,8 +18,10 @@ # config/startup-memory-budget, and config/trace-context - # down into each secondmate home's config/, so the secondmate's OWN crewmates, # dispatch profiles, backlog backend, runtime-backend default, Herdr -# presentation opt-in, startup-memory budget, and trace context inherit the -# primary's settings. +# presentation choice, startup-memory budget, and trace context inherit the +# primary's settings. config/herdr-presentation-spaces is default-ON, so an +# absent primary file and an absent destination file both mean on and the +# generic absence mirror already converges that item correctly. # It is primary-authoritative # (re-pushed at secondmate spawn, on the bootstrap secondmate sweep, and by # config push). @@ -1334,6 +1336,55 @@ test_backend_inheritance_present_and_absent() { pass "B12b backend inheritance: present values and primary absence converge exactly" } +# config/herdr-presentation-spaces is default-ON, so this item's convergence is +# asserted through the verdict the spawn gate actually reads in the destination +# home, not through file presence alone: mirroring the primary's absence must +# converge a secondmate to the same default rather than turning its projection off. +sm_presentation_verdict() { # -> on|off + bash -c ' + . "$0/bin/backends/herdr.sh" + if fm_backend_herdr_presentation_enabled "$1"; then printf "on\n"; else printf "off\n"; fi + ' "$ROOT" "$1" 2>/dev/null +} + +test_presentation_inheritance_default_on_and_opt_out() { + local w head out err status verdict + w=$(new_world presentation-inherit) + head=$(git -C "$w/main" rev-parse HEAD) + add_sm_worktree "$w" sm "$head" + err="$w/presentation-inherit.err" + + out=$(run_config_push "$w" 2>"$err"); status=$? + expect_code 0 "$status" "presentation default push should succeed" + [ -e "$w/sm/config/herdr-presentation-spaces" ] \ + && fail "primary default must not write an opt-out downstream" + verdict=$(sm_presentation_verdict "$w/sm/config") + [ "$verdict" = on ] || fail "primary default left the secondmate projection $verdict" + + mkdir -p "$w/sm/config" + printf 'off\n' > "$w/sm/config/herdr-presentation-spaces" + out=$(run_config_push "$w" 2>"$err"); status=$? + expect_code 0 "$status" "presentation reconverge push should succeed" + assert_contains "$out" "herdr-presentation-spaces: pushed - mirrored primary absence" \ + "a local secondmate opt-out should reconverge on the primary default" + verdict=$(sm_presentation_verdict "$w/sm/config") + [ "$verdict" = on ] || fail "primary default did not reconverge a locally opted-out secondmate ($verdict)" + + printf 'off\n' > "$w/home/config/herdr-presentation-spaces" + out=$(run_config_push "$w" 2>"$err"); status=$? + expect_code 0 "$status" "presentation opt-out push should succeed" + assert_contains "$out" "herdr-presentation-spaces: pushed" "explicit opt-out should report pushed" + verdict=$(sm_presentation_verdict "$w/sm/config") + [ "$verdict" = off ] || fail "explicit primary opt-out left the secondmate projection $verdict" + + : > "$w/home/config/herdr-presentation-spaces" + out=$(run_config_push "$w" 2>"$err"); status=$? + expect_code 0 "$status" "presentation legacy opt-in push should succeed" + verdict=$(sm_presentation_verdict "$w/sm/config") + [ "$verdict" = on ] || fail "a legacy primary opt-in file left the secondmate projection $verdict" + pass "B12c presentation inheritance: the primary default converges on, and only an explicit opt-out propagates off" +} + test_bootstrap_sweep_surfaces_config_propagation_failure() { local w c1 out fail_line w=$(new_world boot-prop-fail) @@ -2426,6 +2477,7 @@ test_bootstrap_sweep_propagates_when_tracked_current test_bootstrap_sweep_defers_dispatch_on_stale_unignored_home test_bootstrap_sweep_materializes_and_inherits_memory_default test_backend_inheritance_present_and_absent +test_presentation_inheritance_default_on_and_opt_out test_bootstrap_sweep_surfaces_config_propagation_failure test_bootstrap_rereads_after_partial_propagation test_config_push_propagates_reports_without_ff_or_nudge From bb352e7b70c054d5f7ca86d8cf2fd7032f2ecc43 Mon Sep 17 00:00:00 2001 From: Kun Chen <3233006+kunchenguid@users.noreply.github.com> Date: Tue, 4 Aug 2026 17:13:44 -0700 Subject: [PATCH 06/12] fix(bin): surface fleet-wide open decisions on every wake drain (#1711) * fix: surface consolidated open decisions on every wake-drain A needs-decision or blocked event buried under later, unrelated status appends was only ever shown via the last-line wake annotation, so a still-open captain decision could go silently missed even though status_open_decisions (fm-classify-lib.sh) already folds the whole status stream correctly and fleet-snapshot/bearings already reuse it. Wire that same fold into bin/fm-wake-drain.sh: a new fleet-wide scan_open_decisions wrapper scans every state/.status, and fm-wake-drain.sh prints a separate, bounded OPEN DECISIONS section on every drain (including the empty-queue fast path), so session-start and every wake-handling turn surface it for free without duplicating the open/resolved fold itself. Heartbeat wakes drain through the same script, so this covers that surface too. Also tighten status_open_decisions' file guard to skip an unreadable status file instead of leaking a bash redirection error, now that a fleet-wide directory scan can reach files a single targeted read would not. * no-mistakes(review): Prevent status symlinks leaking open decisions * fix: drop unbounded perl subprocess from status symlink guard The review step's own symlink-safety auto-fix (O_NOFOLLOW read via a perl subprocess) forked one perl process per status file scanned by the new fleet-wide open-decisions scan, with no cap - inflating fm-wake-drain.sh's total external-read cost from 8 (the existing annotation read_cap) to 18 in the enrichment-caps regression test. The plain [ -L "$f" ] check already rejects any status file that is itself a symlink before any read happens, which is exactly what the new regression test exercises and is the same defense level the sibling scan_captain_relevant_statuses/last_status_line already rely on elsewhere in this file (no O_NOFOLLOW). Drop the subprocess-based nofollow read and keep the cheap builtin guard. * no-mistakes(document): Document actionable fleet-wide open decision drains --- AGENTS.md | 2 + bin/fm-classify-lib.sh | 32 ++++- bin/fm-session-start.sh | 6 +- bin/fm-wake-drain.sh | 53 +++++++ docs/architecture.md | 1 + docs/scripts.md | 4 +- docs/supervision-protocols/claude.md | 2 +- docs/supervision-protocols/grok.md | 2 +- tests/fm-wake-drain-open-decisions.test.sh | 154 +++++++++++++++++++++ 9 files changed, 249 insertions(+), 7 deletions(-) create mode 100755 tests/fm-wake-drain-open-decisions.test.sh diff --git a/AGENTS.md b/AGENTS.md index d47731baed..f0af22e686 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -145,6 +145,7 @@ A lock-refused session must not spawn, steer, merge, drain the wake queue, repai Home-local stale Herdr projection cleanup and the six bootstrap MUTATING sweeps - non-executing legacy PR-check migration, fleet sync, secondmate convergence, secondmate liveness, pending remote handoff retry, and X-mode artifact writes - run only when this session actually holds the lock from step 1. The secondmate liveness sweep deterministically accounts for every registered secondmate: it relaunches only from the recovery-grade `dead` or `missing` states, preserves ambiguous, unreadable, or unreachable remote targets, and reports skipped or failed guarantees as `SECONDMATE_LIVENESS:` lines (`bin/fm-bootstrap.sh`; `bin/fm-backend.sh`'s `fm_backend_agent_state`; `docs/remote-secondmates.md`). 3. **Wake queue** - when locked, drains the durable wake queue and prints the raw records prominently as this turn's first work queue; a bounded, clearly labeled historical status-event annotation may follow a valid `signal` record but never replaces it or current-state reconciliation, and a lapsed watcher chain still surfaces here via the same guard alarm. + Every locked drain also prints a bounded fleet-wide `OPEN DECISIONS` section when durable decision records remain open, including when the queue itself is empty; reconcile those entries before continuing. When the lock could not be acquired and verified, the queue is left untouched because no session mutation is authorized, and the guard's tangle/watcher-liveness alarms still print in read-only advisory mode without drain, supervision repair, or checkout repair commands. 4. **Context digest** - the full contents of `data/projects.md`, `data/secondmates.md`, `data/captain.md`, `data/captain-shared.md`, and `data/learnings.md`, each clearly delimited. A file that does not exist prints an explicit `ABSENT` marker, never confused with an empty-but-present file: absence is meaningful (`captain.md` absent means use the firstmate repo's built-in defaults, `projects.md` absent means rebuild it from the clones under `projects/`, etc.). @@ -365,6 +366,7 @@ No turn ends blind while work is under way, including turns described as holding At the start of every wake-handling turn, drain the durable wake queue before peeking, reading beyond the reason line, steering, or starting work. Session start is the only exception because its one-shot digest already drained while locked or deliberately left the queue untouched in lock-refused read-only mode. +Treat any `OPEN DECISIONS` section from the drain as actionable reconciliation input even when no wake record was queued. A status line is a wake event, not current state; use `bin/fm-crew-state.sh` when current state matters, especially before re-escalating an old decision, blocker, or pause. A declared `paused:` event means a bounded external wait expected to clear on its own, while `blocked:` means firstmate action is needed. diff --git a/bin/fm-classify-lib.sh b/bin/fm-classify-lib.sh index d80840f6a1..8ad7e6813b 100755 --- a/bin/fm-classify-lib.sh +++ b/bin/fm-classify-lib.sh @@ -207,9 +207,15 @@ EOF # the file, no globals beyond the optional FM_CLASSIFY_RESOLVE_VERB override. This # is the durable open-set the fleet snapshot and any point-in-time consumer must use # instead of trusting the last status line. +# The scan_open_decisions wrapper below enumerates a whole directory rather than +# a single caller-chosen path, so a status file that is itself a symlink (e.g. +# escaping the state directory) is rejected outright with a plain [ -L ] check +# before any read - a cheap builtin, unlike fm_wake_latest_event's O_NOFOLLOW +# subprocess read, which exists for that function's much narrower payload-driven +# path resolution rather than this directory-local glob. status_open_decisions() { # local f=$1 line verb key note resolve held open='' stripped - [ -f "$f" ] || return 0 + [ -f "$f" ] && [ -r "$f" ] && [ ! -L "$f" ] || return 0 resolve=${FM_CLASSIFY_RESOLVE_VERB:-$FM_CLASSIFY_RESOLVE_VERB_DEFAULT} held=${FM_CLASSIFY_CAPTAIN_HELD_VERB:-$FM_CLASSIFY_CAPTAIN_HELD_VERB_DEFAULT} while IFS= read -r line || [ -n "$line" ]; do @@ -233,6 +239,30 @@ status_open_decisions() { # printf '%s' "$open" } +# Fleet-wide wrapper around status_open_decisions: scans every task's status +# log under and prefixes each still-open decision with its owning task +# id, so a per-wake or per-session surface can print the consolidated open set +# without re-walking the fold itself. A thin directory scan only - the fold +# above remains the ONE place the open/resolved semantics are decided. Prints +# one "\t\t\t" line per open decision, in glob (task id) +# order; prints nothing when none are open. +scan_open_decisions() { # + local state=$1 f task open line + for f in "$state"/*.status; do + [ -e "$f" ] || continue + task=$(basename "$f"); task="${task%.status}" + open=$(status_open_decisions "$f") || continue + [ -n "$open" ] || continue + while IFS= read -r line; do + [ -n "$line" ] || continue + printf '%s\t%s\n' "$task" "$line" + done < "$FM_WAKE_QUEUE" + fm_lock_release "$FM_WAKE_QUEUE_LOCK" + DRAIN_LOCK_HELD=false + (print_open_decisions_section) || true assert_watcher_liveness exit 0 fi @@ -75,5 +127,6 @@ DRAIN_LOCK_HELD=false # Raw output and queue deletion are authoritative. Everything below is # best-effort and cannot restore, duplicate, hide, or fail the consumed rows. (fm_wake_print_annotations "$RAW_ROWS") || true +(print_open_decisions_section) || true assert_watcher_liveness exit 0 diff --git a/docs/architecture.md b/docs/architecture.md index 335a7b3ae2..4d249606ce 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -29,6 +29,7 @@ After each drain, `fm-wake-drain.sh` runs the same liveness guard as the supervi Routine watcher polling, supervision no-ops, elapsed waiting time, and absorbed benign wakes stay silent. A declared external wait trades that silence for one bounded recheck per pause window, so a forgotten pause cannot remain invisible indefinitely. Crew status files are append-only wake-event logs, not current-state fields. +Because of that, a per-wake read of only the latest line can bury an earlier still-open `needs-decision`/`blocked` under later unrelated appends; `fm-wake-drain.sh` prints a separate, fleet-wide OPEN DECISIONS section on every drain (including the empty-queue path session-start relies on), built from `fm-classify-lib.sh`'s `status_open_decisions` fold so the buried decision keeps surfacing until it is explicitly resolved. `bin/fm-crew-state.sh ` is the cheap current-state read for an actionable heartbeat review: it attributes a no-mistakes run, active or terminal, only when it matches the crew's branch and current code identity, then keeps that run-step authoritative even if the pane has closed. The script header owns the exact run-head ancestry rules. During no-mistakes' `ci` monitor phase, it also reads the ci step log tail because `axi status` reports both "still waiting on checks" and "checks green, waiting on merge" as `ci,running`. diff --git a/docs/scripts.md b/docs/scripts.md index f0960b9389..b2caef631b 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -80,9 +80,9 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-tasks-axi-lib.sh` | Shared backlog-backend selector and `tasks-axi` compatibility probe | | `fm-quota-axi-lib.sh` | Shared `quota-axi` compatibility floor for the bootstrap diagnostic | | `fm-vendor-auth-probe.sh`| Run one hard-bounded, non-destructive authentication probe of a named vendor CLI and report the fact | -| `fm-wake-drain.sh` | Atomically drain queued watcher wakes, emit bounded best-effort status-event annotations, then assert supervision health | +| `fm-wake-drain.sh` | Atomically drain queued watcher wakes, emit bounded best-effort status-event annotations and a fleet-wide OPEN DECISIONS section, then assert supervision health | | `fm-wake-lib.sh` | Shared durable wake queue, portable locks, and watcher identity/health helpers | -| `fm-classify-lib.sh` | Shared captain-relevant and declared-external-wait wake classification vocabulary | +| `fm-classify-lib.sh` | Shared wake-classification vocabulary and durable keyed-decision folds and scans | | `fm-send.sh` | Send one verified literal line or supported key through the target's recorded backend | | `fm-busy-lib.sh` | Single owner of the semantic busy-state contract: verdicts, source attribution, and per-harness sources | | `fm-busy-event.sh` | The only writer of a task's semantic busy-state record; arms an incarnation and applies lifecycle events | diff --git a/docs/supervision-protocols/claude.md b/docs/supervision-protocols/claude.md index 041c0580ca..049e53b693 100644 --- a/docs/supervision-protocols/claude.md +++ b/docs/supervision-protocols/claude.md @@ -7,7 +7,7 @@ When this session owns supervision and away mode is not active: An actionable close wakes you through the hook's exit-2 rewake, delivered as a `Stop hook feedback` message. 3. On a `Stop hook feedback` wake (`signal:`, `stale:`, `check:`, or `heartbeat`), run `bin/fm-wake-drain.sh` first and handle the wake. Do not run `bin/fm-watch-arm.sh` after an ordinary wake; the next turn end re-arms automatically when supervision is still needed. - Do not invent a wake from an attach-status line alone; drain and act only on real wake records or a real watcher reason line. + Do not invent a wake from an attach-status line alone; drain and act only on real wake records, the drain's `OPEN DECISIONS` entries, or a real watcher reason line. 4. On the one `Stop hook feedback` automatic-mechanism failure notice (`firstmate watcher auto-arm FAILED ...`), drain, inspect the automatic mechanism failure, and do not turn the notice into a repeating manual-arm loop. 5. If the Stop hook does not claim the home or reports an exhausted failure, inspect its registration and watcher startup path before ending blind. Keep the Stop-owned automatic mechanism as the only Claude arm owner. diff --git a/docs/supervision-protocols/grok.md b/docs/supervision-protocols/grok.md index 22444b2bd7..6e6ea5c857 100644 --- a/docs/supervision-protocols/grok.md +++ b/docs/supervision-protocols/grok.md @@ -26,7 +26,7 @@ When you see a background-task-completed system reminder for the arm: 3. Handle `signal`, `stale`, `check`, or `heartbeat` using the harness-neutral contract in `AGENTS.md`. 4. Ordinary wake: re-arm the next cycle with the same background `bin/fm-watch-arm.sh` call if work remains in flight or X mode still needs polling. 5. Do not invent a wake from an attach-status line alone. - Drain the queue and act only on real wake records or a real watcher reason line. + Drain the queue and act only on real wake records, the drain's `OPEN DECISIONS` entries, or a real watcher reason line. Re-arm attaches to an existing healthy cycle when one is already present and follows its verified successor chain. See [`watcher-continuity.md`](../watcher-continuity.md) for the arm-layer successor and clean-close failure contract. diff --git a/tests/fm-wake-drain-open-decisions.test.sh b/tests/fm-wake-drain-open-decisions.test.sh new file mode 100755 index 0000000000..695e1d43c4 --- /dev/null +++ b/tests/fm-wake-drain-open-decisions.test.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash +# tests/fm-wake-drain-open-decisions.test.sh - behavior tests for the OPEN +# DECISIONS section bin/fm-wake-drain.sh prints on every drain (including the +# empty-queue fast path). The section is pure wiring around +# fm-classify-lib.sh's status_open_decisions fold (the ONE authoritative +# open/resolved statement); these tests exercise the real drain script over +# crafted status logs and assert on its printed output, not on the fold's own +# source text. +set -u + +# shellcheck source=tests/wake-helpers.sh +. "$(dirname "${BASH_SOURCE[0]}")/wake-helpers.sh" + +DRAIN="$ROOT/bin/fm-wake-drain.sh" + +TMP_ROOT=$(fm_test_tmproot fm-wake-drain-open-decisions-tests) + +test_buried_decision_still_surfaces() { + local dir state out + dir=$(make_case buried) + state="$dir/state" + out="$dir/drain.out" + # The needs-decision line sits under later routine and unrelated-key lines, + # exactly the burial scenario the fix targets: last-line-only reads would + # show "resolved [key=other]" and hide the still-open api-shape decision. + printf 'needs-decision [key=api-shape]: pick REST or RPC\n' > "$state/task1.status" + printf 'working: continuing other work\n' >> "$state/task1.status" + printf 'resolved [key=other]: unrelated decision closed\n' >> "$state/task1.status" + + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" || fail "drain failed on a buried decision" + + grep -F 'OPEN DECISIONS' "$out" >/dev/null || fail "buried decision produced no OPEN DECISIONS section" + grep -F 'task1' "$out" | grep -F '[key=api-shape]' | grep -F 'pick REST or RPC' >/dev/null \ + || fail "buried needs-decision was not surfaced with its task, key, and note" + pass "a needs-decision buried under later routine/other-key lines still reports as open" +} + +test_explicit_resolution_closes_it() { + local dir state out + dir=$(make_case resolved) + state="$dir/state" + out="$dir/drain.out" + printf 'needs-decision [key=api-shape]: pick REST or RPC\n' > "$state/task2.status" + printf 'resolved [key=api-shape]: went with REST\n' >> "$state/task2.status" + printf 'done: shipped\n' >> "$state/task2.status" + + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" || fail "drain failed after an explicit resolution" + + if grep -F 'OPEN DECISIONS' "$out" >/dev/null; then + fail "an explicitly resolved decision still printed as open: $(cat "$out")" + fi + pass "an explicit resolved [key=X] closes the keyed decision" +} + +test_later_unrelated_terminal_line_does_not_close_it() { + local dir state out + dir=$(make_case unrelated-terminal) + state="$dir/state" + out="$dir/drain.out" + # A later done: with no matching [key=...] token opens/closes only the + # "default" key; it must never clear the still-open api-shape decision. + printf 'needs-decision [key=api-shape]: pick REST or RPC\n' > "$state/task3.status" + printf 'done: unrelated later milestone\n' >> "$state/task3.status" + + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" || fail "drain failed after an unrelated terminal line" + + grep -F 'task3' "$out" | grep -F '[key=api-shape]' | grep -F 'pick REST or RPC' >/dev/null \ + || fail "a later unrelated terminal line incorrectly cleared the open decision" + pass "a later unrelated terminal line never clears an open decision" +} + +test_no_open_decisions_prints_nothing() { + local dir state out + dir=$(make_case none-open) + state="$dir/state" + out="$dir/drain.out" + printf 'working: on it\n' > "$state/task4.status" + printf 'done: shipped clean\n' > "$state/task5.status" + + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" || fail "drain failed with no open decisions" + + if grep -F 'OPEN DECISIONS' "$out" >/dev/null; then + fail "the empty case printed an OPEN DECISIONS section: $(cat "$out")" + fi + [ ! -s "$out" ] || fail "the empty case with no queued wakes was not silent: $(cat "$out")" + pass "no open decisions across the fleet prints nothing" +} + +test_open_decision_surfaces_even_with_an_unrelated_queued_wake() { + local dir state out + dir=$(make_case fleet-wide) + state="$dir/state" + out="$dir/drain.out" + # task6 has a buried, still-open decision but generates NO new queue record + # this turn; task7 is what actually wakes the drain. The fleet-wide scan + # must still catch task6's decision alongside task7's own raw row. + printf 'needs-decision [key=migration]: pick the rollout plan\n' > "$state/task6.status" + printf 'working: continuing\n' >> "$state/task6.status" + printf 'blocked: waiting on credentials\n' > "$state/task7.status" + append_wake "$state" signal task7.status "blocked: waiting on credentials" \ + || fail "queueing the unrelated wake failed" + + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" || fail "drain failed with a mixed fleet" + + grep "$(printf '\tsignal\ttask7.status\t')" "$out" >/dev/null || fail "task7's own raw row is missing" + grep -F 'task6' "$out" | grep -F '[key=migration]' >/dev/null \ + || fail "task6's buried decision was not surfaced even though only task7 queued a wake" + pass "the open-decision section is fleet-wide, not scoped to this drain's own queued records" +} + +test_buried_decision_surfaces_on_the_empty_queue_fast_path() { + local dir state out + dir=$(make_case empty-queue-fast-path) + state="$dir/state" + out="$dir/drain.out" + # No wake is queued at all (the empty-queue exit), but the decision is still + # open on disk - session-start relies on exactly this path. + printf 'needs-decision [key=api-shape]: pick REST or RPC\n' > "$state/task8.status" + printf 'working: continuing\n' >> "$state/task8.status" + + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" || fail "empty-queue drain failed" + + grep -F 'task8' "$out" | grep -F '[key=api-shape]' >/dev/null \ + || fail "the empty-queue fast path did not surface a still-open decision" + pass "a buried open decision surfaces even when the wake queue itself is empty" +} + +test_status_symlink_is_not_followed() { + local dir state out + dir=$(make_case status-symlink) + state="$dir/state" + out="$dir/drain.out" + mkdir -p "$dir/outside" + printf 'needs-decision [key=local]: keep this visible\n' > "$state/local.status" + printf 'needs-decision [key=foreign]: do not expose this\n' > "$dir/outside/foreign.status" + ln -s ../outside/foreign.status "$state/linked.status" + + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" || fail "drain failed with a symlinked status file" + + grep -F 'local [key=local] needs-decision: keep this visible' "$out" >/dev/null \ + || fail "the valid local decision did not surface alongside a rejected status symlink" + if grep -F 'do not expose this' "$out" >/dev/null; then + fail "the fleet scan followed a status symlink outside the state directory" + fi + pass "the fleet-wide decision scan does not follow status symlinks" +} + +test_buried_decision_still_surfaces +test_explicit_resolution_closes_it +test_later_unrelated_terminal_line_does_not_close_it +test_no_open_decisions_prints_nothing +test_open_decision_surfaces_even_with_an_unrelated_queued_wake +test_buried_decision_surfaces_on_the_empty_queue_fast_path +test_status_symlink_is_not_followed From 7ef26c48cb9bcda17e5a3ec4c7cf22edb9d17328 Mon Sep 17 00:00:00 2001 From: Kun Chen <3233006+kunchenguid@users.noreply.github.com> Date: Tue, 4 Aug 2026 17:16:03 -0700 Subject: [PATCH 07/12] fix(bin): abort parked runs and reap leaked processes before teardown (#1710) * fix(bin): abort orphaned no-mistakes runs and reap leaked processes at teardown Teardown could remove a task's worker while its no-mistakes pipeline run was still parked at a gate, leaving an orphaned run holding a fleet slot indefinitely (observed 2026-08-03: runs parked 7h39m and parked at a post-CI approval gate). It could also leave backgrounded/disowned descendant processes rooted under the worktree or tasktmp surviving reparented to init (observed: two `go test` binaries pinning CPU for hours with no live task meta to attribute them to). Add two coupled pre-teardown steps, both scoped to this task's exact branch/head or worktree/tasktmp so they can never touch another task's run or processes: - conclude_task_no_mistakes_run aborts a run parked at a gate via `no-mistakes axi abort`, cd'd into the exact worktree so the daemon resolves the run itself rather than teardown naming a --run id. - reap_task_worktree_processes sweeps for processes whose cwd is under the worktree or tasktmp (via `lsof -a -d cwd`) and TERM/KILLs them. Both run before any worktree return, branch delete, or backend kill, and are idempotent on a retried teardown. The branch+head attribution logic is factored out of bin/fm-crew-state.sh into the new shared bin/fm-nm-run-lib.sh so both scripts use the same ownership contract. * no-mistakes(review): Fail closed on incomplete teardown cleanup * no-mistakes(review): Bind teardown cleanup to verified run and process identities * no-mistakes(review): Require confirmed aborts and convergent identity-safe process reaping * no-mistakes(review): Handle process exits during teardown identity checks * no-mistakes(review): Restore teardown library in hermetic gotmp fixtures * no-mistakes(document): Document teardown run attribution and timeout * no-mistakes(lint): Rename shell variable conflicting with done keyword * no-mistakes: apply CI fixes --- bin/fm-crew-state.sh | 68 +--- bin/fm-nm-run-lib.sh | 73 ++++ bin/fm-teardown.sh | 363 ++++++++++++++++++++ bin/fm-test-run.sh | 7 + docs/configuration.md | 3 +- docs/scripts.md | 1 + tests/fm-backend.test.sh | 2 +- tests/fm-gotmp.test.sh | 2 + tests/fm-teardown.test.sh | 680 +++++++++++++++++++++++++++++++++++++- 9 files changed, 1137 insertions(+), 62 deletions(-) create mode 100644 bin/fm-nm-run-lib.sh diff --git a/bin/fm-crew-state.sh b/bin/fm-crew-state.sh index 30fc7b7236..2cb290373c 100755 --- a/bin/fm-crew-state.sh +++ b/bin/fm-crew-state.sh @@ -64,6 +64,8 @@ STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" . "$SCRIPT_DIR/fm-classify-lib.sh" # shellcheck source=bin/fm-busy-lib.sh . "$SCRIPT_DIR/fm-busy-lib.sh" +# shellcheck source=bin/fm-nm-run-lib.sh +. "$SCRIPT_DIR/fm-nm-run-lib.sh" ID=${1:-} [ -n "$ID" ] || { echo "usage: fm-crew-state.sh " >&2; exit 2; } @@ -167,41 +169,20 @@ crew_busy_verdict() { # } # --- no-mistakes run lookup (authoritative when a run matches this branch) -- +# trim, strip_quotes, the bounded nm_run call, nm_field's TOON parse, and the +# branch+head attribution rule below are thin wrappers over the ONE owner in +# bin/fm-nm-run-lib.sh, shared with fm-teardown.sh's pre-teardown run abort. -trim() { - local s=${1:-} - s="${s#"${s%%[![:space:]]*}"}" - s="${s%"${s##*[![:space:]]}"}" - printf '%s' "$s" -} -strip_quotes() { - local s - s=$(trim "${1:-}") - case "$s" in - \"*\") s=${s#\"}; s=${s%\"} ;; - esac - trim "$s" -} - -# Bounded no-mistakes call in the worktree; stdout only, never fails the script. -HAVE_TIMEOUT=none -if command -v timeout >/dev/null 2>&1; then HAVE_TIMEOUT=timeout -elif command -v gtimeout >/dev/null 2>&1; then HAVE_TIMEOUT=gtimeout -elif command -v perl >/dev/null 2>&1; then HAVE_TIMEOUT=perl -fi +trim() { fm_nm_trim "$@"; } +strip_quotes() { fm_nm_strip_quotes "$@"; } nm_run() { # - case "$HAVE_TIMEOUT" in - timeout) ( cd "$WT" && timeout "$NM_TIMEOUT" no-mistakes "$@" ) 2>/dev/null || true ;; - gtimeout) ( cd "$WT" && gtimeout "$NM_TIMEOUT" no-mistakes "$@" ) 2>/dev/null || true ;; - perl) ( cd "$WT" && perl -e 'my $t = shift; my $pid = fork; die "fork failed" unless defined $pid; if (!$pid) { setpgrp(0, 0); exec @ARGV } local $SIG{ALRM} = sub { kill "TERM", -$pid; select undef, undef, undef, 0.2; kill "KILL", -$pid; exit 124 }; alarm $t; waitpid $pid, 0; exit($? >> 8)' "$NM_TIMEOUT" no-mistakes "$@" ) 2>/dev/null || true ;; - *) true ;; - esac + fm_nm_run "$WT" "$NM_TIMEOUT" "$@" } # Scalar value of a TOON key in the captured run output ($RUN_OUT). RUN_OUT="" nm_field() { # - printf '%s\n' "$RUN_OUT" | sed -n "s/^[[:space:]]*$1:[[:space:]]*\(.*\)/\1/p" | head -1 + fm_nm_field "$RUN_OUT" "$1" } # Finding count from a findings[N]{...} table header; empty when none. nm_findings_count() { @@ -385,40 +366,19 @@ nm_runs_status_for_branch() { # CREW_BRANCH=$(git -C "$WT" symbolic-ref --quiet --short HEAD 2>/dev/null || true) # 0 if the active axi-status run's head field matches this worktree's code -# identity. Branch match is a precondition (caller). Rules: -# - missing/empty head field: cannot bind; reject the run -# - equal commits (short or full SHA): match -# - worktree HEAD is an ancestor of run head: match (pipeline fix commits on -# the same history advanced the run tip) -# - run head is a strict ancestor of worktree HEAD: no match (local work -# advanced outside the run) -# - diverged / run head not in this worktree: no match (rewritten branch tip) +# identity. Branch match is a precondition (caller). Rule owned by +# fm_nm_head_matches_worktree in bin/fm-nm-run-lib.sh. nm_run_head_matches_worktree() { - local run_head local_full run_full + local run_head run_head=$(strip_quotes "$(nm_field head)") - [ -n "$run_head" ] || return 1 - local_full=$(git -C "$WT" rev-parse HEAD 2>/dev/null) || return 1 - run_full=$(git -C "$WT" rev-parse --verify "${run_head}^{commit}" 2>/dev/null) || return 1 - [ "$run_full" = "$local_full" ] && return 0 - if git -C "$WT" merge-base --is-ancestor "$local_full" "$run_full" 2>/dev/null; then - return 0 - fi - return 1 + fm_nm_head_matches_worktree "$WT" "$run_head" } # Coarse runs-list rows are " ...". 0 if the short # sha for this branch row matches the worktree head under the same rules as # nm_run_head_matches_worktree (equal, or local is ancestor of run tip). nm_coarse_head_matches_worktree() { # - local run_head=$1 local_full run_full - [ -n "$run_head" ] || return 1 - local_full=$(git -C "$WT" rev-parse HEAD 2>/dev/null) || return 1 - run_full=$(git -C "$WT" rev-parse --verify "${run_head}^{commit}" 2>/dev/null) || return 1 - [ "$run_full" = "$local_full" ] && return 0 - if git -C "$WT" merge-base --is-ancestor "$local_full" "$run_full" 2>/dev/null; then - return 0 - fi - return 1 + fm_nm_head_matches_worktree "$WT" "$1" } HAVE_RUN=0 diff --git a/bin/fm-nm-run-lib.sh b/bin/fm-nm-run-lib.sh new file mode 100644 index 0000000000..7c210c23f5 --- /dev/null +++ b/bin/fm-nm-run-lib.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Shared no-mistakes axi run attribution primitives. +# +# ONE owner for the branch+code-identity matching rule that decides whether a +# no-mistakes run belongs to a given worktree, used by fm-crew-state.sh +# (read-only current-state reporting) and fm-teardown.sh (pre-teardown run +# abort, see its "Fix 1" header comment). Getting this wrong in either +# direction is unsafe: a false negative hides a genuinely parked run, and a +# false positive lets teardown act on a run it does not own. +# +# Bounded call to `no-mistakes "$@"` in dir $1, timeout $2 seconds. The bounded +# form preserves stdout, stderr, and exit status; the checked form discards +# stderr, while fm_nm_run keeps the fail-open query contract for read-only callers. +fm_nm_run_bounded() { # + local dir=$1 timeout_secs=$2 have_timeout=none + shift 2 + if command -v timeout >/dev/null 2>&1; then have_timeout=timeout + elif command -v gtimeout >/dev/null 2>&1; then have_timeout=gtimeout + elif command -v perl >/dev/null 2>&1; then have_timeout=perl + fi + case "$have_timeout" in + timeout) ( cd "$dir" && timeout "$timeout_secs" no-mistakes "$@" ) ;; + gtimeout) ( cd "$dir" && gtimeout "$timeout_secs" no-mistakes "$@" ) ;; + perl) ( cd "$dir" && perl -e 'my $t = shift; my $pid = fork; die "fork failed" unless defined $pid; if (!$pid) { setpgrp(0, 0); exec @ARGV } local $SIG{ALRM} = sub { kill "TERM", -$pid; select undef, undef, undef, 0.2; kill "KILL", -$pid; exit 124 }; alarm $t; waitpid $pid, 0; exit($? >> 8)' "$timeout_secs" no-mistakes "$@" ) ;; + *) return 1 ;; + esac +} + +fm_nm_run_checked() { # + fm_nm_run_bounded "$@" 2>/dev/null +} + +fm_nm_run() { # + fm_nm_run_checked "$@" || true +} + +fm_nm_trim() { + local s=${1:-} + s="${s#"${s%%[![:space:]]*}"}" + s="${s%"${s##*[![:space:]]}"}" + printf '%s' "$s" +} + +fm_nm_strip_quotes() { + local s + s=$(fm_nm_trim "${1:-}") + case "$s" in + \"*\") s=${s#\"}; s=${s%\"} ;; + esac + fm_nm_trim "$s" +} + +# Scalar value of a TOON key in captured `axi status` output $1. +fm_nm_field() { # + printf '%s\n' "$1" | sed -n "s/^[[:space:]]*$2:[[:space:]]*\(.*\)/\1/p" | head -1 +} + +# 0 if run head $2 matches worktree $1's code identity, per the same rule +# everywhere this attribution is needed: +# - missing/empty head: cannot bind; reject +# - equal commits (short or full SHA): match +# - worktree HEAD is an ancestor of run head: match (pipeline fix commits on +# the same history advanced the run tip past local HEAD) +# - run head is a strict ancestor of worktree HEAD, or diverged: no match +# (local work advanced outside the run, or the branch tip was rewritten) +fm_nm_head_matches_worktree() { # + local wt=$1 run_head=$2 local_full run_full + [ -n "$run_head" ] || return 1 + local_full=$(git -C "$wt" rev-parse HEAD 2>/dev/null) || return 1 + run_full=$(git -C "$wt" rev-parse --verify "${run_head}^{commit}" 2>/dev/null) || return 1 + [ "$run_full" = "$local_full" ] && return 0 + git -C "$wt" merge-base --is-ancestor "$local_full" "$run_full" 2>/dev/null +} diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index 14c8987153..8b2d080815 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -86,6 +86,41 @@ # is present; teardown clears only a provably stale lock, then re-runs the safety # checks before any destructive return. Teardown output notes every wait, retry, and # removal so the operator can see what happened. +# +# Pre-teardown cleanup sequence (runs once every landed/discard-work safety +# refusal above has already passed, and BEFORE any worktree return, branch +# delete, or backend kill below - a still-active run or a leaked process may +# own live work in that worktree): +# Fix 1 - conclude the task's own no-mistakes run. A ship task's worktree can +# be torn down while its no-mistakes pipeline run is still PARKED at a gate +# (awaiting_approval/fix_review/any awaiting_agent field), with no worker +# left to ever answer it - the run then sits there holding a fleet slot +# indefinitely (observed 2026-08-03: runs parked 7h39m and parked at a +# post-CI approval gate after the worker was already cleaned up). A run +# with an autonomous step still under way (running/fixing/ci) is left +# alone: no-mistakes drives those against its own gate-repo clone, not the +# crew's worktree, so they are not orphaned by removing the worktree. +# conclude_task_no_mistakes_run attributes the active-or-most-recent run to +# THIS task only when its branch AND code identity (bin/fm-nm-run-lib.sh's +# fm_nm_head_matches_worktree, the same rule bin/fm-crew-state.sh uses) both +# match this worktree, then runs `no-mistakes axi abort --run ` for +# that verified run instance. A run already terminal +# (an outcome is set) or not parked at a gate is left untouched. Idempotent: +# an already-aborted run reads back terminal and is skipped on retry. +# Fix 2 - reap leaked descendant processes. A backgrounded/disowned process +# started under the worktree (or its per-task tasktmp) does not receive the +# SIGHUP/SIGTERM that closing the backend pane sends to its own foreground +# process group, so it survives reparented to init (observed 2026-08-03: +# two `go test` binaries, deadlines blown past by ~100x, pinning CPU for +# hours with no live task meta to attribute them to once teardown had +# already removed it). reap_task_worktree_processes finds every process +# whose CURRENT WORKING DIRECTORY is this task's own worktree or tasktmp +# root via `lsof -a -d cwd` (cheap: bounded by process count, not by +# walking the worktree's file tree) and sends TERM, then KILL after a short +# grace period to any survivor whose process identity still matches. Both +# roots are unique per task and never +# shared, so this can never reach another task's or the primary's +# processes. Idempotent: nothing left to find is a silent no-op. set -eu SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -112,6 +147,8 @@ SUB_HOME_MARKER=".fm-secondmate-home" . "$SCRIPT_DIR/fm-secondmate-registry-lib.sh" # shellcheck source=bin/fm-wake-lib.sh . "$SCRIPT_DIR/fm-wake-lib.sh" +# shellcheck source=bin/fm-nm-run-lib.sh +. "$SCRIPT_DIR/fm-nm-run-lib.sh" if [ "$#" -lt 1 ] || ! fm_task_id_path_safe "$1"; then echo "error: invalid teardown request" >&2 exit 2 @@ -1019,6 +1056,320 @@ validate_worktree_teardown_safety() { fi } +# Fix 1 (see script header): does the active-or-most-recent no-mistakes run in +# worktree $1 belong to THIS task, and is it parked at a gate awaiting an agent +# that is about to be removed? Prints nothing; returns 0 only on a genuine +# match so the caller knows it is safe to abort - never a guess. +NM_TEARDOWN_TIMEOUT=${FM_TEARDOWN_NM_TIMEOUT:-10} +case "$NM_TEARDOWN_TIMEOUT" in ''|*[!0-9]*) NM_TEARDOWN_TIMEOUT=10 ;; esac +TASK_RUN_ID= +task_status_is_own_parked_run() { # + local wt=$1 out=$2 branch run_id run_branch run_head status outcome awaiting has_gate + TASK_RUN_ID= + branch=$(git -C "$wt" symbolic-ref --quiet --short HEAD 2>/dev/null) || return 1 + [ -n "$branch" ] || return 1 + [ -n "$out" ] || return 1 + run_id=$(fm_nm_strip_quotes "$(fm_nm_field "$out" id)") + [ -n "$run_id" ] || return 1 + run_branch=$(fm_nm_strip_quotes "$(fm_nm_field "$out" branch)") + [ -n "$run_branch" ] && [ "$run_branch" = "$branch" ] || return 1 + run_head=$(fm_nm_strip_quotes "$(fm_nm_field "$out" head)") + fm_nm_head_matches_worktree "$wt" "$run_head" || return 1 + outcome=$(fm_nm_strip_quotes "$(fm_nm_field "$out" outcome)") + [ -z "$outcome" ] || return 1 + status=$(fm_nm_strip_quotes "$(fm_nm_field "$out" status)") + awaiting=$(printf '%s\n' "$out" | grep -E '^[[:space:]]*awaiting_agent:' | head -1 || true) + has_gate=$(printf '%s\n' "$out" | grep -Eq '^[[:space:]]*gate:[[:space:]]*' && echo 1 || echo 0) + case "$status" in + awaiting_approval|fix_review) TASK_RUN_ID=$run_id; return 0 ;; + esac + if [ -n "$awaiting" ] || [ "$has_gate" = 1 ]; then + TASK_RUN_ID=$run_id + return 0 + fi + return 1 +} + +task_run_is_own_parked_run() { # + local wt=$1 out + # Accepted best-effort residual: query failures stay fail-open because making + # no-mistakes availability a prerequisite would block ship tasks with no run. + out=$(fm_nm_run "$wt" "$NM_TEARDOWN_TIMEOUT" axi status) + task_status_is_own_parked_run "$wt" "$out" +} + +task_status_is_terminal_run() { # + local out=$1 expected_id=$2 run_id outcome + run_id=$(fm_nm_strip_quotes "$(fm_nm_field "$out" id)") + [ "$run_id" = "$expected_id" ] || return 1 + outcome=$(fm_nm_strip_quotes "$(fm_nm_field "$out" outcome)") + case "$outcome" in + cancelled|failed|passed|checks-passed) return 0 ;; + esac + return 1 +} + +task_status_is_run_not_found() { # + local actual expected + actual=$(fm_nm_trim "$1") + expected=$(printf 'error: "run \\"%s\\" not found"' "$2") + [ "$actual" = "$expected" ] +} + +# Abort THIS task's own parked no-mistakes run before the worker that would +# have answered its gate is removed, so no run is left orphaned holding a +# fleet slot. Only KIND=ship drives a no-mistakes validation of its own +# worktree (scouts and secondmates never do, mirroring bin/fm-crew-state.sh); +# a run not attributed to this exact branch+head is left completely alone. +conclude_task_no_mistakes_run() { # + local wt=$1 out run_id + [ "$KIND" = ship ] || return 0 + [ -d "$wt" ] || return 0 + command -v no-mistakes >/dev/null 2>&1 || return 0 + task_run_is_own_parked_run "$wt" || return 0 + run_id=$TASK_RUN_ID + echo "teardown: no-mistakes run for $ID is parked at a gate; aborting before the worker is removed" >&2 + # Accepted best-effort residual: abort supports run-id targeting but no atomic + # live-state condition; fully closing the resume race needs upstream compare-and-cancel. + fm_nm_run_checked "$wt" "$NM_TEARDOWN_TIMEOUT" axi abort --run "$run_id" >/dev/null 2>&1 || true + if out=$(fm_nm_run_bounded "$wt" "$NM_TEARDOWN_TIMEOUT" axi status --run "$run_id" 2>&1); then + task_status_is_terminal_run "$out" "$run_id" && return 0 + elif task_status_is_run_not_found "$out" "$run_id"; then + return 0 + fi + echo "REFUSED: no-mistakes run for $ID is still parked after axi abort; confirm it stopped (no-mistakes axi status) or abort it manually (no-mistakes axi abort --run ) before retrying teardown." >&2 + return 1 +} + +# Fix 2 (see script header): pids of every process whose CURRENT WORKING +# DIRECTORY is exactly $1 or under it, from one bounded system-wide `lsof -a +# -d cwd` scan (never the recursive +D file-tree walk, which lsof itself +# documents as slow). Never $$ (this script's own pid). Empty output when +# nothing matches; failure means the scan could not establish a safe result. +pids_with_cwd_under() { # + local dir=$1 out pid path line + [ -n "$dir" ] && [ -d "$dir" ] || return 0 + dir=$(cd "$dir" && pwd -P) || return 1 + out=$(lsof -a -d cwd -Fpn 2>/dev/null) || return 1 + [ -n "$out" ] || return 0 + pid= + while IFS= read -r line; do + case "$line" in + p*) + pid=${line#p} + case "$pid" in ''|*[!0-9]*) return 1 ;; esac + ;; + fcwd) [ -n "$pid" ] || return 1 ;; + n*) + [ -n "$pid" ] || return 1 + path=${line#n} + case "$path" in + "$dir"|"$dir"/*) + [ -n "$pid" ] && [ "$pid" != "$$" ] && printf '%s\n' "$pid" + ;; + esac + ;; + '') ;; + *) return 1 ;; + esac + done < + local pid=$1 proc_root stat_line starttime value + local -a stat_fields + proc_root=${FM_PROC_ROOT_OVERRIDE:-/proc} + if [ -r "$proc_root/$pid/stat" ]; then + stat_line=$(cat "$proc_root/$pid/stat" 2>/dev/null) || return 1 + read -r -a stat_fields <<< "${stat_line##*)}" + [ "${#stat_fields[@]}" -ge 20 ] || return 1 + starttime=${stat_fields[19]} + case "$starttime" in ''|*[!0-9]*) return 1 ;; esac + printf 'starttime=%s\n' "$starttime" + return 0 + fi + value=$(LC_ALL=C ps -p "$pid" -o lstart= 2>/dev/null) || return 1 + value=$(fm_nm_trim "$value") + [ -n "$value" ] || return 1 + case "$value" in *$'\n'*|*$'\r'*) return 1 ;; esac + printf 'lstart=%s\n' "$value" +} + +task_process_identity_matches() { # + local current + current=$(task_process_identity "$1") || return 1 + [ "$current" = "$2" ] +} + +task_pid_list_contains() { # + printf '%s\n' "$1" | grep -Fxq "$2" +} + +task_pids_under_roots() { # ... + TASK_PIDS= + TASK_PIDS_FAILED_DIR= + local dir dir_pids pids="" + for dir in "$@"; do + [ -n "$dir" ] || continue + if ! dir_pids=$(pids_with_cwd_under "$dir"); then + TASK_PIDS_FAILED_DIR=$dir + return 1 + fi + pids="$pids +$dir_pids" + done + TASK_PIDS=$(printf '%s\n' "$pids" | grep -E '^[0-9]+$' | sort -un || true) +} + +reap_task_backend_process_group() { #