diff --git a/bin/session-stats.sh b/bin/session-stats.sh index 5e098fa..879a2e3 100755 --- a/bin/session-stats.sh +++ b/bin/session-stats.sh @@ -93,16 +93,6 @@ jq -R -s \ | select(type == "string") | try (sub("\\.[0-9]+Z$"; "Z") | fromdateiso8601) catch empty ] as $timestamps - | [ - $lines[] - # Marker counting is top-level-session-only (advisor-cabinet spec): - # sidechain (subagent) assistant text never contributes markers. - | select(.type == "assistant" and .isSidechain != true and (.message.content | type) == "array") - | .message.content[] - | select(.type == "text" and (.text | type) == "string") - | .text - ] - | join("\n") as $assistant_text # Skill INVOCATION and AUTHORING, counted here and not by the model. A skill runs as a Skill # tool call (input.skill) or as a slash command, which Claude Code records as a user line # carrying /name. Authoring is a Write or Edit whose path is a @@ -153,22 +143,8 @@ jq -R -s \ skills_invoked_count: ($skills_invoked | length), skills_invoked_counts: ($skills_invoked | group_by(.) | map({key: .[0], value: length}) | from_entries), skills_authored: ($skills_authored | unique | sort), - compliance_markers: ( - # Line-start counting (2026-07-20, advisor-cabinet spec): a marker counts - # only when it BEGINS a line of assistant text and is outside a ``` fence, - # so quoted reports and rule-file examples cannot inflate the counts. - reduce ($assistant_text | split("\n"))[] as $l ( - {fence: false, rb: 0, fp: 0, dl: 0, dok: 0}; - if ($l | test("^\\s*```")) then .fence = (.fence | not) - elif .fence then . - elif ($l | startswith("RETRY-BUDGET:")) then .rb += 1 - elif ($l | startswith("FETCH-PIVOT:")) then .fp += 1 - elif ($l | startswith("DELEGATED:")) then .dl += 1 - elif ($l | startswith("DIRECT-OK:")) then .dok += 1 - else . end - ) - | {"RETRY-BUDGET": .rb, "FETCH-PIVOT": .fp, "DELEGATED": .dl, "DIRECT-OK": .dok} - ), + # compliance_markers retired 2026-10-30 (omp: 2026-08-08): agents follow the retry + # and delegation rules without typing the marker line, so the count measured silence. transcript_bytes: $transcript_bytes, transcript_mtime: $transcript_mtime, isSidechain: (any($lines[]?; .isSidechain == true)), diff --git a/docs/plans/2026-10-03-omp-adapter.md b/docs/plans/2026-10-03-omp-adapter.md index d569622..e52aacb 100644 --- a/docs/plans/2026-10-03-omp-adapter.md +++ b/docs/plans/2026-10-03-omp-adapter.md @@ -142,7 +142,7 @@ Status: **in cc** already present; **PR n** ported by that PR (the plan's number | `9db799d` | stop L2 memory writes; broaden the skill-coverage walk for OMP (#3) | ported in part. Memory: n/a, cc routes pins to Mnemopi (#68), and L2 now cannot write at all (https://github.com/STRML/cc-autodream/pull/98). Skill walk: ported as the adapter-built `skills-inventory.txt` (https://github.com/STRML/cc-autodream/pull/96). The exit-non-zero-on-a-delivered-nothing-night fix (`abda904`) is in https://github.com/STRML/cc-autodream/pull/98. The L1 wall-time log and the self-locating install dir (`4f31240`, `8d739b7`) are in https://github.com/STRML/cc-autodream/pull/91 and https://github.com/STRML/cc-autodream/pull/85 | | `790d1b6` | README rewrite for the OMP port | n/a: it documents omp-autodream as a product; the unified README is written at the rename, and the facts it carried live in `adapters/omp/facts.md` and CLAUDE.md | | `6c41c46` | port cc-autodream to OMP | PR 1 and PR 4 | -| `0dffef5` | retire `compliance_markers` telemetry | ported for omp, not for claude. cc `session-stats.sh` still computes `compliance_markers` for claude, whose rule files still emit the markers; `adapters/omp/stats.sh` never did, and `adapters/omp/triage.md` tells omp workers not to emit it (https://github.com/STRML/cc-autodream/pull/97) | +| `0dffef5` | retire `compliance_markers` telemetry | ported for omp first; claude followed on 2026-10-30 (https://github.com/STRML/autodream/pull/170), when cc `session-stats.sh` stopped computing it; `adapters/omp/stats.sh` never did, and `adapters/omp/triage.md` tells omp workers not to emit it (https://github.com/STRML/cc-autodream/pull/97) | Not a commit but part of the work: omp-autodream #16 (advisor sidecar schema, open, being merged into omp-autodream first) lands in PR 4, and its `SESSION_TRIAGE.md` and `PROMPT.md` text lands in PR 5. diff --git a/prompts/PROMPT.md b/prompts/PROMPT.md index c566994..ce1875b 100644 --- a/prompts/PROMPT.md +++ b/prompts/PROMPT.md @@ -79,7 +79,7 @@ For each, ordered by (count × max-severity) descending, cap at 10: - **A defect in source code** is checkable, so check it. Re-find the *described* defect in the current file rather than re-reading the line number the transcript cited: those numbers are stale by construction and the same report proves it, citing `install.sh:10` for a `--dir` problem whose fix sits at `install.sh:32` and `src/proxy.py:455` for a bind whose fix sits at `src/proxy.py:754`. Keying on the cited line reads a shifted defect as fixed and a fixed line as live, which trades today's over-reporting for silent under-reporting. Read the region around the defect, or the whole file when it is small. A finding that survives keeps its evidence. One that does not becomes "already fixed" citing the **current `file:line` that shows the fix** — never a commit sha, which you have no way to look up and must not invent. - **A claim about this host or its tooling** (which CLI is installed, which review seat works) needs splitting before you can act on it. Where the pin rests on a **named file** whose presence *is* the claim, that part is checkable: Glob or Read it and say what you found. `reviewer-seats-on-this-host.md` is the worked example — it argues codex is absent because the shim "forwards to `/Applications/cmux.app/Contents/Resources/bin/cmux-codex-wrapper`, which is missing", and that path is stable, so a single Glob settles whether the pin's premise still holds. (On 2026-08-04 it did not; the wrapper was there and the pin was four days stale.) Where the claim is genuinely **run-only** — the same pin says "Do not diagnose this by probing PATH — probe by running `codex --version`" — you cannot settle it, because you have no way to execute anything. Say "pin dated ``, not verifiable without running the tool" and raise it as an open question. Do not let a file's existence stand in for a tool's behavior: that inference is what made the pin wrong in the first place. Never describe a probe you did not perform. -- **A purely behavioral finding** (`compliance_failure`, `tool_loop`, a missed marker) has no artifact to re-check. Its transcript evidence stands as-is and this gate does not apply. +- **A purely behavioral finding** (`compliance_failure`, `tool_loop`) has no artifact to re-check. Its transcript evidence stands as-is and this gate does not apply. Two limits on the above. Bound the read: the region around the defect, or the whole file when it is small, rather than pulling large files into a context that already holds every findings JSON. And note that you read the *working tree*, which can sit behind origin, so a defect you still find may already be fixed on a branch you cannot see. That is a far better basis than a `MEMORY.md` that merely stayed quiet, but it is not proof. @@ -198,7 +198,7 @@ Still read the field, for the one thing it proved good at: **compliance detectio ## How to start 1. Read the findings directory's `*.json` files (use Glob then Read). -2. Build an in-memory aggregate: group findings by category, count, sort by (count × severity). Exclude `*.stats.json` sidecars from the findings aggregate. Also sum each session's `compliance_markers` counts (`RETRY-BUDGET`, `FETCH-PIVOT`, `DELEGATED`, `DIRECT-OK`) when present — a finding missing the newer keys defaults them to 0 (older findings predate them). RETRY-BUDGET/FETCH-PIVOT measure the retry-budget rules working as designed. (One-time discontinuity: these two switched from substring to line-start counting on 2026-07-20, so totals compared across that date may shift once — don't flag that shift as drift.) DELEGATED/DIRECT-OK measure advisor-mode delegation compliance (`~/.claude/docs/guardrails/advisor-mode.md`; it lived at `~/.claude/rules/advisor-mode.md` and was auto-loaded until 2026-07-24, so a drop in DELEGATED right after that date may reflect the move to lazy loading rather than rule erosion): report the day's totals and flag drift (a sustained fall in DELEGATED across heavy sessions suggests the rule is eroding). Note: `DELEGATED` counts dispatches, not workers — a swarm is ONE marker; per-child health lives in manager reports and is out of marker-telemetry scope. When reporting a `tool_loop` pattern, split marker-present sessions (rule fired — healthy) from marker-absent ones (the actual non-compliance) and state both counts. Also collect the facet fields when present: `outcome` for the Activity-snapshot outcomes line, `underlying_goal` for Per-project notes, and the `instructions_given` lists for the compliance check (see Memory updates). Ignore `satisfaction_signals` entirely — retired 2026-07-28. +2. Build an in-memory aggregate: group findings by category, count, sort by (count × severity). Exclude `*.stats.json` sidecars from the findings aggregate. Ignore `compliance_markers` entirely: retired 2026-10-30 (omp 2026-08-08) because agents follow the retry and delegation rules without typing the `RETRY-BUDGET:` / `FETCH-PIVOT:` / `DELEGATED:` / `DIRECT-OK:` lines, so the counts only measured silence. Do NOT report a compliance-markers line in the Activity snapshot, and do NOT split `tool_loop` sessions by marker presence: judge retry discipline from the transcript evidence alone. Older findings still carry the field; leave it unread. A cached `tool_loop` finding whose `what` cites a missing marker ("no RETRY-BUDGET marker") was scored under the retired rule: re-judge its severity from its evidence quote, and rank a loop the agent broke out of on its own as `low`. Also collect the facet fields when present: `outcome` for the Activity-snapshot outcomes line, `underlying_goal` for Per-project notes, and the `instructions_given` lists for the compliance check (see Memory updates). Ignore `satisfaction_signals` entirely — retired 2026-07-28. **Advisor sidecars.** A finding with `"is_advisor": true` came from an OMP advisor sidecar: a reviewer model tailing a primary session, paired 1:1 with a separately triaged parent transcript in the same session dir. Count these as sessions, but **exclude their `turn_count` from the reported session-turn total** and say so in the activity snapshot (advisors carried 73% of the 2026-08-19 turn count and 0% of its tool calls; an unqualified total is meaningless). Their tool-behavior findings are suppressed at L1, so any `sandbox_friction`, `tool_loop` or `missed_skill` that still arrives from one is an L1 rule violation: file it under Triage failures rather than ranking it as a pattern. Content findings from advisors rank normally, but attribute them to the parent session's project, and never let an advisor finding and its parent's finding both count toward the same pattern's session count. When no finding carries `is_advisor`, ignore this paragraph. 3. Read `/skills-inventory.txt` (authoritative active skills) and `/adapter-facts.md`; reconcile built-in skills from your session skill surface. 4. Read `/changelog-window.md` (Upstream changes) and `/run-stats.txt` (Autodream self-audit) if present. diff --git a/prompts/SESSION_TRIAGE.md b/prompts/SESSION_TRIAGE.md index fbed230..7920821 100644 --- a/prompts/SESSION_TRIAGE.md +++ b/prompts/SESSION_TRIAGE.md @@ -22,7 +22,7 @@ Then: 3. **Write the JSON** with the Write tool to the literal output path from line 2 — exactly one JSON object, no prose around it. 4. Print `done` and exit. No commentary. -When present, a **Precomputed session stats** block follows this document at the end of the prompt. Its `turn_count`, `tool_call_count`, `tools_used`, `models_used`, and `compliance_markers` fields are authoritative: copy them verbatim into the output JSON. Do not derive or recount those fields from the transcript. If the block is absent, derive them from the transcript as before. +When present, a **Precomputed session stats** block follows this document at the end of the prompt. Its `turn_count`, `tool_call_count`, `tools_used`, and `models_used` fields are authoritative: copy them verbatim into the output JSON. Do not derive or recount those fields from the transcript. If the block is absent, derive them from the transcript as before. ## What to look for @@ -36,7 +36,7 @@ Quote 1-3 sentences of evidence for every finding. Don't synthesize, don't infer | `wrong_skill` | Claude invoked a skill that didn't fit; user corrected ("no use X instead"). | | `sandbox_friction` | `Operation not permitted`, `dangerouslyDisableSandbox: true` retries, `/tmp` writes failing, permission prompts denied. | | `memory_miss` | User says "I told you", "we established", "remember", "the same as last time"; Claude re-discovers a workaround that was used in a previous session. | -| `tool_loop` | Same command retried ≥3 times with minor variants (>2 close-but-different curl/grep/find variants in <10 turns). **Marker check:** the retry-budget rules require the agent to emit a literal `RETRY-BUDGET: ...` (or `FETCH-PIVOT: ...`) line in its response text when it stops/pivots. A loop that ends WITH such a marker means the rule worked — severity `low`, and say the marker was present. A loop with NO marker is the real finding — note "no RETRY-BUDGET marker" in `what`. | +| `tool_loop` | Same command retried ≥3 times with minor variants (>2 close-but-different curl/grep/find variants in <10 turns). Judge severity on the transcript alone: a loop the agent breaks out of on its own is `low`; one that keeps going or ends the session is the real finding. | | `permission_prompt` | Commands the user repeatedly allowed or repeatedly denied that should be in `.claude/settings.json` allowlist/denylist. | | `fabricated_id` | Claude quoted a SHA, PR number, line number, function name, or version that wasn't from a just-run command. See the HARD RULE above: tools like `StructuredOutput` that succeed but aren't in `skill_listing` are harness-provided — never flag them. | | `stop_projection` | "you must be tired", "let's pick this back up", "we should stop", any variant. | @@ -61,14 +61,10 @@ For a trivial session (a handful of turns, no substantive work), emit the facets ## Output schema Write EXACTLY this shape to `OUTPUT_PATH`. JSON only, no markdown fence, no prose. -`compliance_markers` holds counts of `RETRY-BUDGET:`, `FETCH-PIVOT:`, `DELEGATED:`, -and `DIRECT-OK:` marker lines in assistant text (0 when absent — most sessions). -Counting rule (all four markers, same rule): count only lines that BEGIN with the -marker string, in the top-level session's assistant text. Do NOT count markers that -appear inside quoted or fenced blocks, markers quoted from rule files, or markers -inside subagent transcripts — a main-loop message quoting a returned report does not -count. (This line-start rule replaced substring counting for RETRY-BUDGET/FETCH-PIVOT -on 2026-07-20; historical comparisons may shift once.) +Do NOT emit a `compliance_markers` field: it was retired on 2026-10-30. Agents follow +the retry and delegation rules without typing the `RETRY-BUDGET:` / `FETCH-PIVOT:` / +`DELEGATED:` / `DIRECT-OK:` lines, so the count only measured silence. Older findings +carry the field; ignore it there too. ```json { @@ -81,7 +77,6 @@ on 2026-07-20; historical comparisons may shift once.) "skills_invoked": ["schedule", "python-env-management"], "models_used": ["claude-opus-4-7"], "notable_initiatives": ["one-line summary of the main thing the user worked on"], - "compliance_markers": {"RETRY-BUDGET": 0, "FETCH-PIVOT": 0, "DELEGATED": 0, "DIRECT-OK": 0}, "underlying_goal": "one line of user intent, or null if it duplicates notable_initiatives", "outcome": "fully_achieved|mostly_achieved|partially_achieved|not_achieved|unclear_from_transcript", "satisfaction_signals": {"happy": 0, "satisfied": 0, "dissatisfied": 0, "frustrated": 0}, diff --git a/tests/run-all.sh b/tests/run-all.sh index 08c1fd3..b816e45 100755 --- a/tests/run-all.sh +++ b/tests/run-all.sh @@ -915,7 +915,7 @@ test_session_stats(){ '{"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"b","content":"result"}]}}' > "$fixture" "$REPO/bin/session-stats.sh" "$fixture" "$out" assert_eq "$(jq -r 'keys | sort | join(",")' "$out")" \ - "compliance_markers,duration_minutes,isSidechain,models_used,skills_authored,skills_invoked,skills_invoked_count,skills_invoked_counts,tool_call_count,tools_used,transcript_bytes,transcript_mtime,turn_count,user_message_count,user_turn_timestamps" \ + "duration_minutes,isSidechain,models_used,skills_authored,skills_invoked,skills_invoked_count,skills_invoked_counts,tool_call_count,tools_used,transcript_bytes,transcript_mtime,turn_count,user_message_count,user_turn_timestamps" \ "stats output has exactly the specified fields" assert_eq "$(jq -r '.user_turn_timestamps | length' "$out")" "0" "no timestamped user turns in this fixture -> empty user_turn_timestamps" assert_eq "$(jq -r .user_message_count "$out")" "1" "tool_result carriers are excluded from user message count" @@ -941,11 +941,9 @@ test_session_stats(){ '{"type":"assistant","message":{"content":[{"type":"text","text":"DELEGATED: scout — find X\nthe report said DELEGATED: quoted mid-line\n```\nDELEGATED: fenced example\nDIRECT-OK: fenced example\n```\nDIRECT-OK: tiny-edit — one-line fix"}]}}' \ '{"type":"assistant","isSidechain":true,"message":{"content":[{"type":"text","text":"DELEGATED: harvester — sidechain worker line"}]}}' > "$fixture" "$REPO/bin/session-stats.sh" "$fixture" "$out" - assert_eq "$(jq -r '.compliance_markers["RETRY-BUDGET"]' "$out")" "1" "only assistant text counts RETRY-BUDGET" - assert_eq "$(jq -r '.compliance_markers["DELEGATED"]' "$out")" "1" "DELEGATED counts line-start only (mid-line, fenced, sidechain excluded)" - assert_eq "$(jq -r '.compliance_markers["DIRECT-OK"]' "$out")" "1" "DIRECT-OK counts line-start only (fenced excluded)" - assert_eq "$(jq -r '.compliance_markers | keys | sort | join(",")' "$out")" \ - "DELEGATED,DIRECT-OK,FETCH-PIVOT,RETRY-BUDGET" "compliance_markers carries all four keys" + # compliance_markers retired for claude 2026-10-30 (omp: 2026-08-08): agents follow + # the retry rules without typing the line, so the count only measured silence. + assert_eq "$(jq -r 'has("compliance_markers")' "$out")" "false" "marker lines produce no compliance_markers field" fixture="$root/text-image.jsonl"; out="$root/text-image.stats.json" printf '%s\n' \