diff --git a/docs/reportcards/collection.json b/docs/reportcards/collection.json index 8154507..b633aee 100644 --- a/docs/reportcards/collection.json +++ b/docs/reportcards/collection.json @@ -208,5 +208,47 @@ "source": "PR #47 + PR #291, curl 200" }, "next_lever": "Item 9: the false-green benchmark \u2014 how often the loop reports A when a stronger judge disagrees" + }, + { + "id": "rc0007", + "delivery": "Part A (@skills, arXiv:2608.12610): residency_audit.py on the live library \u2014 221 entries, 176 distinct = 1.76x the paper's argued 100-slot bound; resident index 62,010 chars (~15.5k tok) vs 2.95M chars of bodies = 47.6x; 17 phantom symlinks, 25 duplicate names, 130/176 overlong descriptions. trigger_reliability.py runs the paper's OWN named future work: top-1 routing 100% (N=10), 100% (N=40), 83.3% (N=80), 44.4% (N=176, 9 usable trials, CI 19-73%) \u2014 subject gpt-oss-120b, probe generator a different family, declared optimistic-bound bias, failed calls excluded from the denominator. TWO self-corrections shipped as commits: a YAML block-scalar parse bug made my first audit numbers wrong (86.6x/72 overlong -> 47.6x/130) AND corrupted the experiment's ground truth (three targets got the identical probe 'Hey, can you pull'); and an eyeballed '~190 MCP tools' was retracted for a counted 132. Part B (Agent Loop Engineering lecture): encoded 16 sections as docs/rubrics/agent-loop-engineering.yml + scripts/audit_loop_rubric.py with observed-only evidence and unmeasured-blocks-the-gate. First run 23/23 -> went back for claims chosen because I expected them to fail -> 25/28, gate FAIL. Closed one gap for real: MemoryStore.success_rate() (in-flight excluded, blocked_safety counts as failure, None not 0.0 when nothing finished, 6 tests). 606 tests pass, ruff clean, 5 commits on branch research/attention-budget.", + "objective": "Make loop-engineering-anything's public architecture as honest and reviewable as its engine already is", + "created_at": "2026-08-21T11:45:03-05:00", + "key_results": [ + { + "text": "A paper's unmeasured central claim becomes measured", + "target": "a curve with CIs from a real corpus", + "score": 1.0, + "evidence": "research/attention-budget/results.json, per-trial records tracked" + }, + { + "text": "A lecture's taxonomy becomes a gate that can fail us", + "target": "rubric as data + observed-evidence checker", + "score": 1.0, + "evidence": "docs/rubrics/AUDIT.md, 25/28 gate FAIL, exit 1" + }, + { + "text": "My own errors are caught and published, not buried", + "target": "self-corrections shipped as commits", + "score": 1.0, + "evidence": "382759f parse bug, 5f910e0 count retraction, both with corrected numbers in the README" + } + ], + "request": "study the @skills paper + atskills repo: 1. apply it 2. extend to other agentic resource management with 4-window research 3. deep R&D on top 3 areas 4. long-form article. THEN: eval/research/10X the Agent Loop Engineering lecture, make it TRUE for /loop-anything, and write another article.", + "headline": "Two rubrics scored; both first answers were wrong", + "brief": "I measured the thing a new paper admits nobody has measured: how reliably an AI picks the right tool as you install more. Perfect at 40 tools, 83% at 80, 44% at your 176. Then I turned a lecture's checklist into an automatic test of your engine. It scored 23 out of 23, which was the bug \u2014 I wrote the questions knowing the answers. Adding the ones I expected to fail gave an honest 25 of 28.", + "growth": [ + "C:met:Treated my own 23/23 as evidence of a bad rubric rather than a good engine, and went looking for questions I expected to fail", + "P:met:Derived the success-rate definition from first principles about what would corrupt it \u2014 excluded in-flight runs and counted safety blocks as failures, so the metric cannot rise when the safety gate fires" + ], + "why": "Both sources assert without verifying \u2014 the paper says its central number is unmeasured, the lecture says it is not a standard. Acting on either as fact would have been the exact false-green this repo exists to prevent. The 23/23 moment proves the risk is not hypothetical: I produced a perfect score on my own system within an hour of starting.", + "needle": { + "name": "verifiable claims about this engine backed by an executable probe", + "before": "0", + "after": "28", + "better": "up", + "source": "scripts/audit_loop_rubric.py" + }, + "next_lever": "Human takeover of a running loop \u2014 the one red item with a reason but no plan; it is the only miss I cannot defend as a design divergence" } ] \ No newline at end of file diff --git a/docs/rubrics/AUDIT.md b/docs/rubrics/AUDIT.md new file mode 100644 index 0000000..27979f1 --- /dev/null +++ b/docs/rubrics/AUDIT.md @@ -0,0 +1,115 @@ +# Agent Loop Engineering — conformance audit + +Rubric: `docs/rubrics/agent-loop-engineering.yml` — 31 items drawn from *Agent Loop Engineering — 讲座总结* (DataApplab / AI聘 (info@aipin.io), received 2026-08-20). + +That source is a **lecture summary; taxonomy, no verification against a running system**. Every claim below is scored against shipped code or a test that was actually executed. No evidence means no. + +**Conformance: 25/28 (89%)** verifiable claims implemented · 3 declared gaps · 0 unmeasured · gate **FAIL** + +## Implemented — with observed evidence + +- **L1-loop-not-oneshot** · 2 — why an agent needs a loop + - claim: The system iterates plan→act→observe→evaluate rather than answering once. + - evidence: `src/loopeng/loop/controller.py:75` +- **L2-goal** · 3 — Goal + - claim: The loop carries an explicit goal and a definition of done. + - evidence: `src/loopeng/config.py:89` +- **L3-state** · 3 — State + - claim: Progress lives outside the model: steps taken, results, environment. + - evidence: `src/loopeng/memory/store.py:196` +- **L4-policy** · 3 — Policy + - claim: Something decides the next action from the current state. + - evidence: `src/loopeng/adapters/base.py:103` +- **L5-action-space** · 3 — Action space + - claim: The agent's available actions are declared, not open-ended. + - evidence: `src/loopeng/adapters/base.py:52` +- **L6-observation** · 3 — Observation + - claim: Every action returns structured environment feedback. + - evidence: `src/loopeng/adapters/base.py:16` +- **L7-evaluation** · 3/10 — Evaluation is the controller + - claim: A verifiable evaluator decides progress, not the maker's self-report. + - evidence: `src/loopeng/adapters/base.py:62` +- **L8-maker-not-checker** · 10 — evaluation must be trustworthy + - claim: The thing that builds is not the thing that grades. + - evidence: `tests/test_maker_checker.py -> 30 passed in 0.16s` +- **L9-reflection** · 4/6 — Plan-Execute-Observe-Reflect, Self-Reflection + - claim: Why the last attempt scored what it did is carried into the next attempt. + - evidence: `src/loopeng/adapters/base.py:66` +- **L10-replan-on-plateau** · 4 — the plan is not immutable + - claim: Feedback can force a change of strategy, not just another attempt. + - evidence: `src/loopeng/config.py:100` +- **L11-retry-transient-only** · 8 — Retry vs Recovery + - claim: Only retryable (infrastructure) failures are retried; not every error. + - evidence: `src/loopeng/adapters/base.py:142` +- **L12-recovery-state** · 8 — Recovery keeps enough state to resume + - claim: A failed change can be rolled back rather than restarting from zero. + - evidence: `tests/test_checkpoint.py -> 2 passed in 0.50s` +- **L13-exit-success** · 9 — explicit exits: success + - claim: The loop stops when the goal is verifiably met. + - evidence: `src/loopeng/loop/convergence.py:29` +- **L14-exit-budget** · 9/14 — explicit exits: budget (iterations, tokens, wall clock) + - claim: The loop stops on a spent budget, and the budget has more than one dimension. + - evidence: `src/loopeng/loop/convergence.py:38` +- **L15-exit-giveup** · 9 — explicit exits: give up and report + - claim: Repeated non-progress ends the run and reports failure instead of looping. + - evidence: `src/loopeng/loop/convergence.py:36` +- **L16-safety-terminal** · 13 — permission control + - claim: A safety failure is terminal and unbypassable, whatever the score. + - evidence: `src/loopeng/loop/convergence.py:30` +- **L17-permission-boundary** · 13 — which tools/data the agent may touch is limited + - claim: Execution is jailed and shell metacharacters are refused. + - evidence: `src/loopeng/adapters/safety.py:5` +- **L18-human-in-the-loop** · 12 — Human-in-the-Loop for high-risk actions + - claim: High-risk completion requires a human, and the caller cannot self-approve. + - evidence: `src/loopeng/config.py:127` +- **L19-hitl-unbypassable** · 12 — the gate must actually hold + - claim: An unattended run cannot pre-confirm its own result. + - evidence: `tests/test_run_contract.py::test_contract_can_never_disable_the_human_gate -> 4 passed in 0.04s` +- **L20-multi-agent-graph** · 11 — Loop becomes Graph with many agents + - claim: Multiple agents are coordinated as a dependency graph, cycles refused. + - evidence: `src/loopeng/orchestration/coordinator.py:6` +- **L21-observability** · 13 — every iteration leaves enough log/trace to debug + - claim: A run is reconstructable after the fact from recorded evidence. + - evidence: `src/loopeng/autonomous/report.py:37` +- **L22-token-accounting** · 14 — Token economics: measure tokens, time, iterations, success + - claim: Token cost, wall time and iteration count are recorded per run. + - evidence: `src/loopeng/proof.py:108` +- **L23-cost-never-faked** · 14 — measurement must be real to be useful + - claim: An unavailable cost is omitted, never estimated into the record. + - evidence: `tests/test_proof.py -> 7 passed in 0.09s` +- **L29-success-rate-metric** · 14 — measure tokens, time, iteration count AND task success rate + - claim: Task success RATE is computed across runs, not just per-run outcome. + - evidence: `tests/test_success_rate.py -> 6 passed in 0.09s` +- **L31-error-memory** · 13 — record errors so the agent does not repeat the same mistake + - claim: Failures that recur across runs are surfaced to the next attempt. + - evidence: `src/loopeng/memory/store.py:277` + +## NOT implemented + +- **L27-react-interleaving** · 5 — ReAct: reasoning and acting alternate at fine granularity + - claim: The loop interleaves reason->act->observe per tool call, rather than one coarse generate/judge/refactor turn. + - evidence: `pattern /class ReActLoop|def reason_then_act/ not found in 76 file(s)` + - why it is still a miss: Deliberate divergence, not an oversight: ReAct puts the actor in charge of judging its own next step, and this engine's load-bearing rule is that the maker never grades. Recorded as a MISS rather than reclassified as a pass, because a rubric that lets the author explain failures away scores nothing. +- **L28-human-takeover** · 12 — a human may take the task over directly + - claim: A human can seize control of a running loop, not merely approve or reject its result. + - evidence: `pattern /def takeover|def handoff_to_human/ not found in 76 file(s)` + - why it is still a miss: A real gap. The gate is end-of-run approval; there is no control channel into a loop already in flight. Worth building; not built. +- **L30-manager-worker** · 11 — Manager-Worker and hierarchical multi-agent structures + - claim: A manager agent decomposes a goal and assigns sub-tasks to worker agents. + - evidence: `pattern /class ManagerAgent|def assign_subtask/ not found in 76 file(s)` + - why it is still a miss: Divergence: coordination is an explicit dependency DAG with cycle rejection, which is inspectable before anything runs, rather than a manager agent deciding at runtime. Still a miss against the lecture's claim. + +## Declared gaps — the engine does NOT do these, and says so + +Each probe passes while the gap is real and fails the moment it silently closes, so this list cannot quietly go stale. + +- **L24-typed-failure-taxonomy** · 8 — classify the error before choosing a response + - claim: Failures are classified into a named taxonomy (referee-unavailable, adapter-contract, human-gate-timeout). + - status: **confirmed absent** — Only infra-vs-clean is distinguished. External eval item 3. +- **L25-branching-search** · 7 — Tree of Thoughts + - claim: The loop explores multiple candidate paths and prunes them. + - status: **confirmed absent** — Single-path refine with a dimension pivot. Deliberate: branching multiplies cost. +- **L26-trace-schema** · 13 — observability as a first-class trace + - claim: Runs emit a structured trace (spans/trace ids), not just a report. + - status: **confirmed absent** — Reports and proof packs exist; a span-level trace schema does not. + diff --git a/docs/rubrics/agent-loop-engineering.yml b/docs/rubrics/agent-loop-engineering.yml new file mode 100644 index 0000000..a19bdb0 --- /dev/null +++ b/docs/rubrics/agent-loop-engineering.yml @@ -0,0 +1,203 @@ +# Agent Loop Engineering — the lecture's claims, as a machine-checkable rubric. +# +# SOURCE: "AI开发进入新时代：Agent Loop Engineering - 讲座总结" (DataApplab / AI聘 webinar +# summary, emailed 2026-08-20). Sixteen sections describing what a mature agent loop +# must have. The essay asserts; nothing in it is verified against a running system. +# +# THIS FILE IS THE SPEC. `scripts/audit_loop_rubric.py` is the only consumer, so the +# published score cannot drift from the claims it scores. +# +# RULES (docs/skill/playbooks/operationalizing-a-paper-rubric-checklist-standa.md): +# * Evidence is OBSERVED (a symbol in shipped code, a passing test), never claimed. +# * No evidence => NO. Not "probably". +# * A probe that cannot run is `unmeasured` — excluded from the rate, and it BLOCKS +# a green gate. An unmeasured item is never a silent pass. +# * `expect: absent` items are honest gap declarations: the rubric asserts the engine +# does NOT have this, and the probe FAILS if the gap ever silently closes. +version: 1 +source: + title: "Agent Loop Engineering — 讲座总结" + publisher: "DataApplab / AI聘 (info@aipin.io)" + received: "2026-08-20" + nature: "lecture summary; taxonomy, no verification against a running system" + +items: + - id: L1-loop-not-oneshot + section: "2 — why an agent needs a loop" + claim: "The system iterates plan→act→observe→evaluate rather than answering once." + probe: {type: grep, pattern: "class LoopController", paths: ["src/loopeng/loop/controller.py"]} + + - id: L2-goal + section: "3 — Goal" + claim: "The loop carries an explicit goal and a definition of done." + probe: {type: grep, pattern: "target_grade|target_score", paths: ["src/loopeng/config.py"]} + + - id: L3-state + section: "3 — State" + claim: "Progress lives outside the model: steps taken, results, environment." + probe: {type: grep, pattern: "def record_iteration", paths: ["src/loopeng/memory/store.py"]} + + - id: L4-policy + section: "3 — Policy" + claim: "Something decides the next action from the current state." + probe: {type: grep, pattern: "class RefactorBrief|def build_brief", paths: ["src/loopeng/adapters/base.py", "src/loopeng/loop/refactor_brief.py"]} + + - id: L5-action-space + section: "3 — Action space" + claim: "The agent's available actions are declared, not open-ended." + probe: {type: grep, pattern: "class Factory|class Refiner|class Judge", paths: ["src/loopeng/adapters/base.py"]} + + - id: L6-observation + section: "3 — Observation" + claim: "Every action returns structured environment feedback." + probe: {type: grep, pattern: "class Verdict", paths: ["src/loopeng/adapters/base.py"]} + + - id: L7-evaluation + section: "3/10 — Evaluation is the controller" + claim: "A verifiable evaluator decides progress, not the maker's self-report." + probe: {type: grep, pattern: "def judge", paths: ["src/loopeng/adapters/base.py"]} + + - id: L8-maker-not-checker + section: "10 — evaluation must be trustworthy" + claim: "The thing that builds is not the thing that grades." + probe: {type: pytest, node: "tests/test_maker_checker.py"} + + - id: L9-reflection + section: "4/6 — Plan-Execute-Observe-Reflect, Self-Reflection" + claim: "Why the last attempt scored what it did is carried into the next attempt." + probe: {type: grep, pattern: "class ReflectionContext", paths: ["src/loopeng/adapters/base.py"]} + + - id: L10-replan-on-plateau + section: "4 — the plan is not immutable" + claim: "Feedback can force a change of strategy, not just another attempt." + probe: {type: grep, pattern: "plateau_pivots", paths: ["src/loopeng/config.py"]} + + - id: L11-retry-transient-only + section: "8 — Retry vs Recovery" + claim: "Only retryable (infrastructure) failures are retried; not every error." + probe: {type: grep, pattern: "last_infra_failure", paths: ["src/loopeng/adapters/base.py"]} + + - id: L12-recovery-state + section: "8 — Recovery keeps enough state to resume" + claim: "A failed change can be rolled back rather than restarting from zero." + probe: {type: pytest, node: "tests/test_checkpoint.py"} + + - id: L13-exit-success + section: "9 — explicit exits: success" + claim: "The loop stops when the goal is verifiably met." + probe: {type: grep, pattern: "CONVERGED", paths: ["src/loopeng/loop/convergence.py"]} + + - id: L14-exit-budget + section: "9/14 — explicit exits: budget (iterations, tokens, wall clock)" + claim: "The loop stops on a spent budget, and the budget has more than one dimension." + probe: {type: grep, pattern: "ITERATION_CAP|TOKEN_CAP|WALL_CAP", paths: ["src/loopeng/loop/convergence.py"]} + + - id: L15-exit-giveup + section: "9 — explicit exits: give up and report" + claim: "Repeated non-progress ends the run and reports failure instead of looping." + probe: {type: grep, pattern: "PLATEAU", paths: ["src/loopeng/loop/convergence.py"]} + + - id: L16-safety-terminal + section: "13 — permission control" + claim: "A safety failure is terminal and unbypassable, whatever the score." + probe: {type: grep, pattern: "BLOCKED_SAFETY", paths: ["src/loopeng/loop/convergence.py"]} + + - id: L17-permission-boundary + section: "13 — which tools/data the agent may touch is limited" + claim: "Execution is jailed and shell metacharacters are refused." + probe: {type: grep, pattern: "within_workspace|shell=False", paths: ["src/loopeng/adapters/safety.py"]} + + - id: L18-human-in-the-loop + section: "12 — Human-in-the-Loop for high-risk actions" + claim: "High-risk completion requires a human, and the caller cannot self-approve." + probe: {type: grep, pattern: "class VerificationGate", paths: ["src/loopeng/config.py"]} + + - id: L19-hitl-unbypassable + section: "12 — the gate must actually hold" + claim: "An unattended run cannot pre-confirm its own result." + probe: {type: pytest, node: "tests/test_run_contract.py::test_contract_can_never_disable_the_human_gate"} + + - id: L20-multi-agent-graph + section: "11 — Loop becomes Graph with many agents" + claim: "Multiple agents are coordinated as a dependency graph, cycles refused." + probe: {type: grep, pattern: "Kahn|cycle", paths: ["src/loopeng/orchestration/coordinator.py"]} + + - id: L21-observability + section: "13 — every iteration leaves enough log/trace to debug" + claim: "A run is reconstructable after the fact from recorded evidence." + probe: {type: grep, pattern: "def render_report", paths: ["src/loopeng/autonomous/report.py"]} + + - id: L22-token-accounting + section: "14 — Token economics: measure tokens, time, iterations, success" + claim: "Token cost, wall time and iteration count are recorded per run." + probe: {type: grep, pattern: "token_cost", paths: ["src/loopeng/proof.py"]} + + - id: L23-cost-never-faked + section: "14 — measurement must be real to be useful" + claim: "An unavailable cost is omitted, never estimated into the record." + probe: {type: pytest, node: "tests/test_proof.py"} + + # ---- honest gap declarations: the rubric asserts these are ABSENT ---- + - id: L24-typed-failure-taxonomy + section: "8 — classify the error before choosing a response" + claim: "Failures are classified into a named taxonomy (referee-unavailable, adapter-contract, human-gate-timeout)." + expect: absent + probe: {type: grep, pattern: "class FailureType|REFEREE_UNAVAILABLE", paths: ["src/loopeng"]} + gap_note: "Only infra-vs-clean is distinguished. External eval item 3." + + - id: L25-branching-search + section: "7 — Tree of Thoughts" + claim: "The loop explores multiple candidate paths and prunes them." + expect: absent + probe: {type: grep, pattern: "tree_of_thought|beam_search", paths: ["src/loopeng"]} + gap_note: "Single-path refine with a dimension pivot. Deliberate: branching multiplies cost." + + - id: L26-trace-schema + section: "13 — observability as a first-class trace" + claim: "Runs emit a structured trace (spans/trace ids), not just a report." + expect: absent + probe: {type: grep, pattern: "opentelemetry|trace_id|def emit_span", paths: ["src/loopeng"]} + gap_note: "Reports and proof packs exist; a span-level trace schema does not." + + # ---- claims the lecture makes that this engine does NOT satisfy ---- + # Added after the first run scored 23/23. A rubric authored by the same person who + # knows the codebase, that then passes everything, has measured nothing. These are + # the lecture's claims I had NOT encoded, chosen precisely because I expected them + # to fail. A gate that cannot fail is decoration. + + - id: L27-react-interleaving + section: "5 — ReAct: reasoning and acting alternate at fine granularity" + claim: "The loop interleaves reason->act->observe per tool call, rather than one coarse generate/judge/refactor turn." + probe: {type: grep, pattern: "class ReActLoop|def reason_then_act", paths: ["src/loopeng"]} + rationale: >- + Deliberate divergence, not an oversight: ReAct puts the actor in charge of judging + its own next step, and this engine's load-bearing rule is that the maker never + grades. Recorded as a MISS rather than reclassified as a pass, because a rubric + that lets the author explain failures away scores nothing. + + - id: L28-human-takeover + section: "12 — a human may take the task over directly" + claim: "A human can seize control of a running loop, not merely approve or reject its result." + probe: {type: grep, pattern: "def takeover|def handoff_to_human", paths: ["src/loopeng"]} + rationale: >- + A real gap. The gate is end-of-run approval; there is no control channel into a + loop already in flight. Worth building; not built. + + - id: L29-success-rate-metric + section: "14 — measure tokens, time, iteration count AND task success rate" + claim: "Task success RATE is computed across runs, not just per-run outcome." + probe: {type: pytest, node: "tests/test_success_rate.py"} + + - id: L30-manager-worker + section: "11 — Manager-Worker and hierarchical multi-agent structures" + claim: "A manager agent decomposes a goal and assigns sub-tasks to worker agents." + probe: {type: grep, pattern: "class ManagerAgent|def assign_subtask", paths: ["src/loopeng"]} + rationale: >- + Divergence: coordination is an explicit dependency DAG with cycle rejection, which + is inspectable before anything runs, rather than a manager agent deciding at + runtime. Still a miss against the lecture's claim. + + - id: L31-error-memory + section: "13 — record errors so the agent does not repeat the same mistake" + claim: "Failures that recur across runs are surfaced to the next attempt." + probe: {type: grep, pattern: "recurring_failures", paths: ["src/loopeng/memory/store.py"]} diff --git a/research/attention-budget/.gitignore b/research/attention-budget/.gitignore new file mode 100644 index 0000000..9b286d5 --- /dev/null +++ b/research/attention-budget/.gitignore @@ -0,0 +1,2 @@ +.cache/ +run.log diff --git a/research/attention-budget/AUDIT.md b/research/attention-budget/AUDIT.md new file mode 100644 index 0000000..377c340 --- /dev/null +++ b/research/attention-budget/AUDIT.md @@ -0,0 +1,93 @@ +# Skill residency audit + +Library: `/Users/jialiang.wu/.claude` + +## The budget + +- `SKILL.md` entries found: **221** (204 readable, 17 phantom) +- Distinct skill names: **176** +- Paper's conservative reliable-slot bound: **100** +- Over budget by: **1.76x** + +- Resident index (name + description): **62,010 chars** (~15,502 tokens) +- Every body, if all were resident: **2,954,435 chars** (~738,608 tokens) +- Content-to-index ratio: **47.6x** — this is the paper's thesis as a single number: the index costs ~1/47 of the content. + +## PHANTOM — holds a name, loads nothing + +**17 entries.** The protocol says a client MUST refuse such an entry *loudly*. These fail silently instead: the capability is simply absent, and nothing tells you. + +- `benchmark-models` → `/Users/jialiang.wu/Documents/Projects/gstack/benchmark-models/SKILL.md` +- `context-restore` → `/Users/jialiang.wu/Documents/Projects/gstack/context-restore/SKILL.md` +- `context-save` → `/Users/jialiang.wu/Documents/Projects/gstack/context-save/SKILL.md` +- `document-generate` → `/Users/jialiang.wu/Documents/Projects/gstack/document-generate/SKILL.md` +- `ios-clean` → `/Users/jialiang.wu/Documents/Projects/gstack/ios-clean/SKILL.md` +- `ios-design-review` → `/Users/jialiang.wu/Documents/Projects/gstack/ios-design-review/SKILL.md` +- `ios-fix` → `/Users/jialiang.wu/Documents/Projects/gstack/ios-fix/SKILL.md` +- `ios-qa` → `/Users/jialiang.wu/Documents/Projects/gstack/ios-qa/SKILL.md` +- `ios-sync` → `/Users/jialiang.wu/Documents/Projects/gstack/ios-sync/SKILL.md` +- `landing-report` → `/Users/jialiang.wu/Documents/Projects/gstack/landing-report/SKILL.md` +- `make-pdf` → `/Users/jialiang.wu/Documents/Projects/gstack/make-pdf/SKILL.md` +- `pair-agent` → `/Users/jialiang.wu/Documents/Projects/gstack/pair-agent/SKILL.md` +- `plan-tune` → `/Users/jialiang.wu/Documents/Projects/gstack/plan-tune/SKILL.md` +- `scrape` → `/Users/jialiang.wu/Documents/Projects/gstack/scrape/SKILL.md` +- `setup-gbrain` → `/Users/jialiang.wu/Documents/Projects/gstack/setup-gbrain/SKILL.md` +- `skillify` → `/Users/jialiang.wu/Documents/Projects/gstack/skillify/SKILL.md` +- `sync-gbrain` → `/Users/jialiang.wu/Documents/Projects/gstack/sync-gbrain/SKILL.md` + +## DUPLICATE — one capability, more than one slot + +**25 names installed more than once.** Each extra copy is a slot spent on a capability already present — the local form of the name collisions the paper measures across the public corpus. + +- `access` ×3 +- `animate-anything` ×2 +- `configure` ×3 +- `copilotkit` ×2 +- `dreammaketrue` ×2 +- `enduser-webtest` ×2 +- `free-llm` ×2 +- `freellmapi` ×2 +- `frontend-design` ×2 +- `future-self` ×2 +- `installable-web-app` ×2 +- `knowledge-graph` ×2 +- `knowledgefy` ×2 +- `lavish` ×2 +- `living-knowledge` ×3 +- `living-repo` ×2 +- `no-mistakes` ×2 +- `open-gstack-browser` ×2 +- `proactive-intervention` ×2 +- `skill-creator` ×2 + +## OVERLONG — description past the protocol's ~120-char guidance + +**130 of 176** exceed it. The description *is* the trigger signal, so every extra character is resident attention spent on one tenant of the index. + +- `dreammaketrue` — 1394 chars (11.6x guidance) +- `free-llm` — 1173 chars (9.8x guidance) +- `knowledgefy` — 1074 chars (8.9x guidance) +- `living-repo` — 951 chars (7.9x guidance) +- `enduser-webtest` — 913 chars (7.6x guidance) +- `project-artifact` — 906 chars (7.5x guidance) +- `knowledge-graph` — 869 chars (7.2x guidance) +- `continual-learning-research` — 853 chars (7.1x guidance) +- `treehouse` — 786 chars (6.5x guidance) +- `no-mistakes` — 779 chars (6.5x guidance) +- `skillfy` — 775 chars (6.5x guidance) +- `freellmapi` — 774 chars (6.5x guidance) + +## UNTRIGGERED — cannot feed a trigger index + +`math-olympiad` + +## Proposal + +To reach the argued bound, **76 skills** must stop being resident. In @skills terms that is not deletion — it is demotion from tier 3 (auto-trigger) to tier 1 (addressed by path, read at the point of use). Ranked cheapest-first: + +1. Remove the **17 phantom** entries. Zero capability lost — they already load nothing. +2. Collapse the **25 duplicated** names to one copy each. +3. Rewrite the **130 overlong** descriptions toward 120 chars. Same coverage, less resident spend. +4. Demote every skill that is only ever invoked *by name* (a slash command you type) out of the auto-trigger index. If you always ask for it explicitly, it never needed a trigger slot — that is the paper's central point, and it is the largest available win. + +This tool does not apply any of the above. Which capabilities deserve residency is operator judgement, and a script that silently re-tiered a library would be making exactly the unreviewable change the protocol exists to prevent. diff --git a/research/attention-budget/README.md b/research/attention-budget/README.md new file mode 100644 index 0000000..68735cb --- /dev/null +++ b/research/attention-budget/README.md @@ -0,0 +1,245 @@ +# Attention as a budget — applying and extending `@skills` + +R&D against **arXiv:2608.12610**, *"@skills: Attention is all you have"* (Yin, Li, Shi, Zhang, +Seong, Wang — SylphAI / UT Austin, **12 Aug 2026**) and its reference implementation +[SylphAI-Inc/atskills](https://github.com/SylphAI-Inc/atskills) (MIT, TypeScript). + +## What the paper argues + +Installing a skill bundles three separable things — **content, persistence, and +auto-triggering** — and only the last one needs the system prompt. So: + +> "56,804 skills to be reachable while fewer than ten are resident." + +Its generalization, stated in its own conclusion, is the mandate for everything in this +directory: + +> "The principle generalizes past skills: resident context is a budget; spend it only on +> what must fire implicitly, and deliver everything else at the point of use, where +> attention is highest." + +## The honest status of its headline number + +The paper claims a budget of "fewer than a hundred reliable auto-trigger slots per agent." +It is explicit that this is **not measured**: + +> "Our central quantity, the number of reliable auto-trigger slots, is bounded by argument +> and by the literature rather than measured by us" + +and it names the missing experiment as future work: + +> "The measurements this argument invites are trigger reliability as a function of +> installed-skill count" + +Engagement context, measured 13 Aug 2026: the preprint is **one day old**, the repo has +**26 stars**, and no community discussion was findable. This is a fresh argument, not an +adopted standard. Treat the framing as valuable and the digit as open. + +--- + +## 1. Applied — `residency_audit.py` + +Audits a real library against the three-tier model. Run on this operator's own machine: + +| measure | value | +|---|---| +| `SKILL.md` entries | **221** (204 readable, 17 phantom) | +| distinct skill names | **176** | +| paper's argued reliable-slot bound | 100 | +| **over budget by** | **1.76x** | +| resident index (name + description) | 62,010 chars (~15,502 tokens) | +| every body, if resident | 2,954,435 chars (~738,608 tokens) | +| **content-to-index ratio** | **47.6x** | + +Four failure classes the paper predicts, all present: + +- **PHANTOM — 17.** Every one is a dangling symlink into a `gstack/` path that no longer + holds them. The protocol says a client "MUST refuse to `:install` it loudly rather than + write a line that silently loads nothing." These fail *silently*: the capability is gone + and nothing says so. The failure is not wasted attention — it is **absence with no signal**. +- **DUPLICATE — 25 names installed more than once** (`access` ×3, `configure` ×3, + `living-knowledge` ×3). The local form of the corpus-wide collisions the paper measures. +- **OVERLONG — 130 of 176** descriptions (74%) exceed the protocol's ~120-char guidance; + the worst is **11.6x** it. The description *is* the trigger signal, so length is spent attention. +- **UNTRIGGERED — 1** of 176 carries no description at all, so it cannot feed a trigger index. + +The tool **measures and proposes; it never mutates.** Which capabilities deserve residency is +operator judgement, and a script that silently re-tiered a library would be making exactly +the unreviewable change the protocol exists to prevent. + +Full report: [`AUDIT.md`](./AUDIT.md). + +### A correction, recorded rather than quietly fixed + +The first version of this audit reported a resident index of 34,120 chars and a ratio of +86.6x, with 72 overlong descriptions. **Those numbers were wrong.** The reader used +`^description:\s*(.*)$`, which captures the *indicator* of a YAML block scalar — so every +skill written as `description: |` was measured as having a **one-character** description. +The parser never failed; it returned a plausible wrong value, which is the precise failure +class this audit exists to report. Fixed in [`skillmeta.py`](./skillmeta.py) and pinned by +[`test_skillmeta.py`](./test_skillmeta.py), including a regression guard that fails if any +indicator-only description ever reappears in a real library. + +## 2. R&D area 1 — measuring the paper's unmeasured quantity + +[`trigger_reliability.py`](./trigger_reliability.py) runs the experiment the paper names as +future work, against a **real** 176-skill corpus rather than a synthetic one. + +Design: a menu of N skill descriptions always containing one target, plus a first-person +user request the target is meant to serve; score top-1 selection; sweep N. + +- **subject** (picks the skill): `openai/gpt-oss-120b` via Groq +- **generator** (writes the probes): Gemini 3.6 Flash, falling back to Claude Haiku — a + *different family* from the subject, so the request wording is not authored by the model + that has to route it. Maker ≠ checker applied to the data. +- **Declared bias:** probes are generated *from* each target's own description, so trigger + wording leaks into the request. Every trial is therefore **easier than reality**, making + the result an **optimistic upper bound** — whatever degradation appears, the real + degradation is at least that large. +- **Honesty rules:** a failed API call is recorded as an error and **excluded from the + denominator**, never scored as a model miss (that would manufacture the paper's + conclusion); every response is disk-cached so re-runs are reproducible, not re-sampled; + an N with fewer than 8 usable trials reports "insufficient data" rather than a number. + +Two measured transport facts, both of the kind that silently corrupt this class of +experiment: + +1. Groq sits behind Cloudflare, which **403s (error 1010) on urllib's default User-Agent** + while accepting the identical payload from curl. +2. `gpt-oss-120b` bills reasoning tokens to `max_tokens`, so a small cap returns an + **empty `content`** — a false "the model said nothing" that is really a budget bug. + +### Result — measured 13 Aug 2026 + +| installed skills (N) | usable trials | top-1 routing | 95% CI (Wilson) | +|---:|---:|---:|---| +| 10 | 24 | **100.0%** | 86–100% | +| 40 | 24 | **100.0%** | 86–100% | +| 80 | 24 | **83.3%** | 64–93% | +| 176 | 9 | **44.4%** | 19–73% | + +Routing is perfect to 40, degrades at 80, and collapses at the full library. **The break +falls between 40 and 80** — below, not at, the paper's argued bound of 100. So the bound is +directionally supported and, on this corpus, *generous*. + +The operator's own library is **176 skills** — the collapse cell. + +What this is NOT: +- **Not a general constant.** One subject model (`gpt-oss-120b`), one corpus, 12 probes × + 2 seeds. A different model or corpus will move the curve. +- **Not tight at the top end.** The N=176 cell rests on **9 usable trials** (15 calls lost to + free-tier rate limits) and its interval is 19–73%. It shows a collapse; it does not locate one. +- **Still the optimistic bound.** Probes are derived from each target's own description, so + real phrasing — which never quotes the description — should do worse, not better. + +Failure shape is worth as much as the rate: at N=80 the misses are 3 wrong picks and 1 +abstention; at N=176, 4 wrong and 1 abstention. The model does not mostly say "none of +these" — **it confidently picks the wrong skill.** A silent wrong route is harder to notice +than a refusal. + +Raw per-trial records: `results.json`. + +## 3. R&D area 2 — the same principle, applied to tool schemas + +Already shipping, and countable from this session's own notices: the harness listed tool +**names** while explicitly withholding their schemas — "Their schemas are NOT loaded… Use +ToolSearch". That is tier-3 residency for the *name* and tier-1 on-demand fetch for the +*schema*: the protocol, one layer down, for tools instead of skills. + +Counted, not estimated: + +| deferred tool names | count | +|---|---:| +| built-in (Cron*, Task*, Web*, Monitor, …) | 25 | +| claude.ai connectors (Gmail 24, Calendar 9, Drive 8, LunarCrush 15, …) | 62 | +| local MCP — moomoo | 5 | +| local MCP — notebooklm | 40 | +| **peak deferred** | **132** | +| later withdrawn mid-session (server disconnect) | 67 | + +**Correction:** an earlier version of this file said "~190". That was an overestimate by +1.44x, from eyeballing the list instead of counting it. 132 is the counted figure. + +Schema cost is still a *sample*: the two tools actually loaded this session (`WebFetch`, +`WebSearch`) ran ~250 tokens each. At that rate 132 resident schemas would cost ~33k tokens +against ~1k for names alone — order **~30x**, independently in the same range as the +**47.6x** measured for skills. n=2 is a sample, not a census, and it stays labelled as one. + +## 4. R&D area 3 — the same principle, applied to instructions + +| resident on every session | chars | ~tokens | +|---|---:|---:| +| global `CLAUDE.md` | 4,184 | 1,046 | +| Projects-root `CLAUDE.md` | 13,861 | 3,465 | +| `~/.anyagent/backbone.md` | 1,097 | 274 | +| `~/.anyagent/playbooks.md` (index only) | 172 | 43 | +| **total** | **19,314** | **4,828** | + +Plus **63 per-project `CLAUDE.md` files**, 506,788 chars (~127k tokens) in aggregate, median +5,686, largest 39,890. + +Two findings: + +1. **The Projects-root `CLAUDE.md` is 3.3x the global one**, and most of it is a *venture + directory* — reference material answering "which repo does what". That is tier-1 content + (read on demand) wearing tier-3 clothes (resident always). It is the single largest + misfiled residency on this machine. +2. **The pattern was already here before the paper existed.** `backbone.md` (1,097 chars, + always resident) sits beside `playbooks.md` (**172 chars**, an index), and the anyagent + skill states the rule explicitly: *"The table below is an **index**… On a trigger match, + READ that file; otherwise do not carry it."* That is the paper's tier-3/tier-1 split, + hand-rolled, independently. The protocol's contribution is not the idea — it is giving + the idea an **address**. + +--- + +## Prior art, across four windows + +**🏺 300 years.** What survived is not any particular index but the **separation of a cheap +resident finding-aid from expensive non-resident content**, plus a **computed address** so +content can move without breaking the pointer. Panizzi's rules (1841) → Cutter's *Objects of +the Catalogue* (1876) → Paris Principles (1961) → IFLA's ICP (2016), which opens: *"This +statement builds on the great cataloguing traditions of the world,³"* — footnote 3 being +Cutter. Cutter, 1876: *"the convenience of the public must not be sacrificed to brevity"*; +ICP 2016, principle #1: *"Convenience of the user."* 140 years apart, same rule. +Dewey's **relative location** (1876) — the address is *computed*, not assigned — now serves +200,000+ libraries in 135+ countries. Graveyard: **fixed-location shelving** (every +acquisition reclassified its neighbours) and Otlet's **Mundaneum** (15.6M cards, defunded +1934, partly destroyed 1940 — the vision survived, the artifact did not). + +**🕰 30 years.** Every survivor separates a **cheap validated pointer** from expensive +content: HTTP caching (born 1997, re-specified as **RFC 9111, STD 98, June 2022** — an ETag +is a resident pointer you can validate without moving the content), git content-addressing +(2005 — which `@skills:gh:owner/repo/path` simply borrows), CDN edge caching (1998). The +loudest corpse is **HTML5 AppCache** (removed from Chrome 95, Oct 2021), which died +specifically of a **declarative manifest that fixed residency in advance** — so the paper's +"no manifest, no lockfile, no registration" is not minimalist taste, it is the lesson of a +documented death. Second corpse: **Intel Optane** (market life 2017–2022; Intel wrote off +**$559M**), an entire new tier of the memory hierarchy killed by economics, not physics — +and The Register's requiem of 29 July 2026 calls it *"Intel's KV cache killer that could +have eased the RAM price crunch."* + +**A gap worth naming:** virtual memory (1962), demand paging, and Denning's working-set +model (1968) are **58–67 years old** — older than the 30-year window, younger than the +300-year window's 75-year multi-generational floor. The field's canonical answers to "what +stays resident" fall in a blind spot between the two research windows, and are *not* yet +generationally proven. + +**🌗 30 months.** `SKILL.md` crossed demo→default: Agent Skills announced 16 Oct 2025, open +standard 18 Dec 2025, adopted by Microsoft in VS Code/GitHub, ~40 compatible products by +June 2026; MCP donated to the Linux Foundation 9 Dec 2025. + +**📰 30 days.** The paper is one day old, 26 stars, no findable discussion. Fresh argument, +not an adopted standard. + +--- + +## Reproducing + +```bash +python3 research/attention-budget/residency_audit.py --out AUDIT.md # no network, no keys +GROQ_API_KEY=… GEMINI_API_KEY=… python3 research/attention-budget/trigger_reliability.py +``` + +The audit is offline and deterministic. The experiment caches every response under +`.cache/`, so a second run costs nothing and returns the same numbers. diff --git a/research/attention-budget/residency_audit.py b/research/attention-budget/residency_audit.py new file mode 100644 index 0000000..bc3c883 --- /dev/null +++ b/research/attention-budget/residency_audit.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Audit a skill library against the @skills residency budget. + +Applies the three-tier model from arXiv:2608.12610 (Yin et al., 12 Aug 2026) to a +real installed library. The paper's claim is that installation bundles three +separable things -- content, persistence, and auto-triggering -- and that only the +last one costs prompt residency, against a budget it bounds "conservatively at +fewer than a hundred reliable auto-trigger slots per agent." + +This tool measures what a library actually spends, and names four failure classes +the paper predicts: + + PHANTOM an entry whose SKILL.md cannot be read. It holds a name and loads + nothing. The protocol is explicit that a conforming client "MUST + refuse to :install it loudly rather than write a line that silently + loads nothing" -- so silence here is a bug, not tidiness. + DUPLICATE one capability occupying more than one slot; the local form of the + corpus-wide collision the paper measures (13,119 of 56,825 names). + OVERLONG a description past the protocol's "under ~120 chars" guidance. The + description IS the trigger signal, so length is spent attention. + UNTRIGGERED a skill with no description at all -- it cannot feed a trigger index. + +It MEASURES and PROPOSES. It never mutates a library: --apply is deliberately not +implemented, because which skills deserve residency is the operator's judgement, +not a script's. The proposal is a ranked list to act on by hand. +""" + +from __future__ import annotations + +import argparse +import collections +import json +import pathlib +import sys + +from skillmeta import distinct as _distinct +from skillmeta import load_skills as _load_skills + +# The paper's conservative bound. It is ARGUED from the literature, not measured by +# the authors -- see trigger_reliability.py, which measures it here. +RELIABLE_SLOT_BOUND = 100 +DESC_GUIDANCE_CHARS = 120 + + +def scan(root: pathlib.Path) -> tuple[list[dict], list[dict]]: + """Delegates to skillmeta, which parses YAML block scalars correctly. + A local regex here once returned "|" as a description -- see skillmeta.py.""" + return _load_skills(root) + + +def audit(root: pathlib.Path) -> dict: + readable, phantom = scan(root) + + by_name: dict[str, list[dict]] = collections.defaultdict(list) + for r in readable: + by_name[r["name"]].append(r) + distinct = _distinct(readable) + duplicates = {n: [x["path"] for x in v] for n, v in by_name.items() if len(v) > 1} + + resident = sum(len(r["desc"]) for r in distinct.values()) + bodies = sum(r["body_chars"] for r in distinct.values()) + overlong = sorted( + ((n, len(r["desc"])) for n, r in distinct.items() if len(r["desc"]) > DESC_GUIDANCE_CHARS), + key=lambda kv: -kv[1], + ) + untriggered = sorted(n for n, r in distinct.items() if not r["desc"]) + + return { + "root": str(root), + "slot_bound_claimed_by_paper": RELIABLE_SLOT_BOUND, + "skill_md_files": len(readable) + len(phantom), + "readable": len(readable), + "distinct_names": len(distinct), + "over_budget_ratio": round(len(distinct) / RELIABLE_SLOT_BOUND, 2), + "resident_index_chars": resident, + "resident_index_tokens_est": resident // 4, + "all_bodies_chars": bodies, + "all_bodies_tokens_est": bodies // 4, + "content_to_index_ratio": round(bodies / max(resident, 1), 1), + "phantom": phantom, + "duplicates": duplicates, + "overlong": overlong, + "untriggered": untriggered, + } + + +def render(a: dict) -> str: + L: list[str] = [] + add = L.append + add("# Skill residency audit") + add("") + add(f"Library: `{a['root']}`") + add("") + add("## The budget") + add("") + add(f"- `SKILL.md` entries found: **{a['skill_md_files']}** " + f"({a['readable']} readable, {len(a['phantom'])} phantom)") + add(f"- Distinct skill names: **{a['distinct_names']}**") + add(f"- Paper's conservative reliable-slot bound: **{a['slot_bound_claimed_by_paper']}**") + add(f"- Over budget by: **{a['over_budget_ratio']}x**") + add("") + add(f"- Resident index (name + description): **{a['resident_index_chars']:,} chars** " + f"(~{a['resident_index_tokens_est']:,} tokens)") + add(f"- Every body, if all were resident: **{a['all_bodies_chars']:,} chars** " + f"(~{a['all_bodies_tokens_est']:,} tokens)") + add(f"- Content-to-index ratio: **{a['content_to_index_ratio']}x** — this is the " + "paper's thesis as a single number: the index costs ~1/" + f"{int(a['content_to_index_ratio'])} of the content.") + add("") + + add("## PHANTOM — holds a name, loads nothing") + add("") + if not a["phantom"]: + add("None. Every entry resolves.") + else: + add(f"**{len(a['phantom'])} entries.** The protocol says a client MUST refuse such an " + "entry *loudly*. These fail silently instead: the capability is simply absent, and " + "nothing tells you.") + add("") + for p in a["phantom"]: + add(f"- `{p['dir']}` → `{p['target'] or '(unresolved)'}`") + add("") + + add("## DUPLICATE — one capability, more than one slot") + add("") + if not a["duplicates"]: + add("None.") + else: + add(f"**{len(a['duplicates'])} names installed more than once.** Each extra copy is a " + "slot spent on a capability already present — the local form of the name collisions " + "the paper measures across the public corpus.") + add("") + for n, paths in sorted(a["duplicates"].items())[:20]: + add(f"- `{n}` ×{len(paths)}") + add("") + + add(f"## OVERLONG — description past the protocol's ~{DESC_GUIDANCE_CHARS}-char guidance") + add("") + if not a["overlong"]: + add("None.") + else: + add(f"**{len(a['overlong'])} of {a['distinct_names']}** exceed it. The description *is* " + "the trigger signal, so every extra character is resident attention spent on one " + "tenant of the index.") + add("") + for n, c in a["overlong"][:12]: + add(f"- `{n}` — {c} chars ({c/DESC_GUIDANCE_CHARS:.1f}x guidance)") + add("") + + add("## UNTRIGGERED — cannot feed a trigger index") + add("") + add("None." if not a["untriggered"] else ", ".join(f"`{n}`" for n in a["untriggered"])) + add("") + + add("## Proposal") + add("") + over = a["distinct_names"] - a["slot_bound_claimed_by_paper"] + add(f"To reach the argued bound, **{max(over, 0)} skills** must stop being resident. In " + "@skills terms that is not deletion — it is demotion from tier 3 (auto-trigger) to " + "tier 1 (addressed by path, read at the point of use). Ranked cheapest-first:") + add("") + add(f"1. Remove the **{len(a['phantom'])} phantom** entries. Zero capability lost — they " + "already load nothing.") + add(f"2. Collapse the **{len(a['duplicates'])} duplicated** names to one copy each.") + add(f"3. Rewrite the **{len(a['overlong'])} overlong** descriptions toward " + f"{DESC_GUIDANCE_CHARS} chars. Same coverage, less resident spend.") + add("4. Demote every skill that is only ever invoked *by name* (a slash command you type) " + "out of the auto-trigger index. If you always ask for it explicitly, it never needed a " + "trigger slot — that is the paper's central point, and it is the largest available win.") + add("") + add("This tool does not apply any of the above. Which capabilities deserve residency is " + "operator judgement, and a script that silently re-tiered a library would be making " + "exactly the unreviewable change the protocol exists to prevent.") + return "\n".join(L) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("root", nargs="?", default=str(pathlib.Path.home() / ".claude"), + help="library root to scan (default: ~/.claude)") + ap.add_argument("--json", action="store_true", help="emit the raw audit as JSON") + ap.add_argument("--out", default=None, help="write the markdown report to this path") + args = ap.parse_args() + + root = pathlib.Path(args.root).expanduser() + if not root.is_dir(): + print(f"error: {root} is not a directory", file=sys.stderr) + return 2 + + a = audit(root) + if args.json: + print(json.dumps(a, indent=2)) + return 0 + report = render(a) + if args.out: + pathlib.Path(args.out).write_text(report + "\n", encoding="utf-8") + print(f"wrote {args.out}") + else: + print(report) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/research/attention-budget/results.json b/research/attention-budget/results.json new file mode 100644 index 0000000..c924ecd --- /dev/null +++ b/research/attention-budget/results.json @@ -0,0 +1,815 @@ +{ + "paper": "arXiv:2608.12610 (Yin et al., 12 Aug 2026)", + "measures": "trigger reliability as a function of installed-skill count (the paper's own named future work)", + "declared_bias": "probe requests are generated FROM each target's description, so target trigger wording leaks into the request; these accuracies are an OPTIMISTIC UPPER BOUND on real-world routing", + "subject_model": "openai/gpt-oss-120b", + "generator_model": "gemini-3.6-flash", + "generator_fallbacks_used": [ + "claude-haiku-4-5-20251001" + ], + "corpus_size": 175, + "targets": [ + "benchmark", + "blogwatcher", + "browser-harness", + "canvas", + "copilotkit", + "knowledgefy", + "longevity-loop", + "moomoo-derivatives-anomaly", + "project-artifact", + "sag", + "sonoscli", + "ux-research" + ], + "seeds": [ + 11, + 22 + ], + "min_trials": 8, + "by_n": { + "10": { + "hit": 24, + "wrong": 0, + "none": 0, + "unparsed": 0, + "error": 0 + }, + "40": { + "hit": 24, + "wrong": 0, + "none": 0, + "unparsed": 0, + "error": 0 + }, + "80": { + "hit": 20, + "wrong": 3, + "none": 1, + "unparsed": 0, + "error": 0 + }, + "176": { + "hit": 4, + "wrong": 4, + "none": 1, + "unparsed": 0, + "error": 15 + } + }, + "trials": [ + { + "n": 10, + "seed": 11, + "target": "knowledgefy", + "outcome": "hit", + "pick": "knowledgefy", + "request": "Can you kgfy this repo so I can quickly map out the core components before my 2 PM design review? https://github.com/vllm-project/vllm" + }, + { + "n": 10, + "seed": 11, + "target": "copilotkit", + "outcome": "hit", + "pick": "copilotkit", + "request": "I need to add an AI copilot sidebar to our Next.js app that can read page state and trigger actions\u2014can you set up the provider, API route, and hooks for it?" + }, + { + "n": 10, + "seed": 11, + "target": "moomoo-derivatives-anomaly", + "outcome": "hit", + "pick": "moomoo-derivatives-anomaly", + "request": "hey can you check if there's any unusual options activity in nvidia right now, want to see if smart money is positioning for something" + }, + { + "n": 10, + "seed": 11, + "target": "ux-research", + "outcome": "hit", + "pick": "ux-research", + "request": "hey can you do a deep dive on how stripe and square handle their dashboard UX? need to see what patterns they're using before we redesign ours" + }, + { + "n": 10, + "seed": 11, + "target": "benchmark", + "outcome": "hit", + "pick": "benchmark", + "request": "hey can you check if my latest PR regressed the page load times? i want to make sure the bundle size didn't blow up" + }, + { + "n": 10, + "seed": 11, + "target": "browser-harness", + "outcome": "hit", + "pick": "browser-harness", + "request": "can you log into our staging dashboard and pull the latest test results from the QA reports page, then export it as CSV?" + }, + { + "n": 10, + "seed": 11, + "target": "sag", + "outcome": "hit", + "pick": "sag", + "request": "hey can you read this design doc out loud for me while i finish this code review?" + }, + { + "n": 10, + "seed": 11, + "target": "canvas", + "outcome": "hit", + "pick": "canvas", + "request": "hey can you push this dashboard html to my iPad so I can see it on the big screen while I'm testing?" + }, + { + "n": 10, + "seed": 11, + "target": "longevity-loop", + "outcome": "hit", + "pick": "longevity-loop", + "request": "Hey, can you run the latest pyaging models on our cohort data and then benchmark the results against the leaderboard to see where we stand on the aging clock ac" + }, + { + "n": 10, + "seed": 11, + "target": "sonoscli", + "outcome": "hit", + "pick": "sonoscli", + "request": "Pause the speakers in the office, I'm hopping on a call right now." + }, + { + "n": 10, + "seed": 11, + "target": "blogwatcher", + "outcome": "hit", + "pick": "blogwatcher", + "request": "can you start watching the golang weekly newsletter and let me know whenever there's a new post about concurrency patterns?" + }, + { + "n": 10, + "seed": 11, + "target": "project-artifact", + "outcome": "hit", + "pick": "project-artifact", + "request": "hey can you set up a status page for the infrastructure migration project? we've got like 5 different teams working on different pieces and i need something i c" + }, + { + "n": 10, + "seed": 22, + "target": "knowledgefy", + "outcome": "hit", + "pick": "knowledgefy", + "request": "Can you kgfy this repo so I can quickly map out the core components before my 2 PM design review? https://github.com/vllm-project/vllm" + }, + { + "n": 10, + "seed": 22, + "target": "copilotkit", + "outcome": "hit", + "pick": "copilotkit", + "request": "I need to add an AI copilot sidebar to our Next.js app that can read page state and trigger actions\u2014can you set up the provider, API route, and hooks for it?" + }, + { + "n": 10, + "seed": 22, + "target": "moomoo-derivatives-anomaly", + "outcome": "hit", + "pick": "moomoo-derivatives-anomaly", + "request": "hey can you check if there's any unusual options activity in nvidia right now, want to see if smart money is positioning for something" + }, + { + "n": 10, + "seed": 22, + "target": "ux-research", + "outcome": "hit", + "pick": "ux-research", + "request": "hey can you do a deep dive on how stripe and square handle their dashboard UX? need to see what patterns they're using before we redesign ours" + }, + { + "n": 10, + "seed": 22, + "target": "benchmark", + "outcome": "hit", + "pick": "benchmark", + "request": "hey can you check if my latest PR regressed the page load times? i want to make sure the bundle size didn't blow up" + }, + { + "n": 10, + "seed": 22, + "target": "browser-harness", + "outcome": "hit", + "pick": "browser-harness", + "request": "can you log into our staging dashboard and pull the latest test results from the QA reports page, then export it as CSV?" + }, + { + "n": 10, + "seed": 22, + "target": "sag", + "outcome": "hit", + "pick": "sag", + "request": "hey can you read this design doc out loud for me while i finish this code review?" + }, + { + "n": 10, + "seed": 22, + "target": "canvas", + "outcome": "hit", + "pick": "canvas", + "request": "hey can you push this dashboard html to my iPad so I can see it on the big screen while I'm testing?" + }, + { + "n": 10, + "seed": 22, + "target": "longevity-loop", + "outcome": "hit", + "pick": "longevity-loop", + "request": "Hey, can you run the latest pyaging models on our cohort data and then benchmark the results against the leaderboard to see where we stand on the aging clock ac" + }, + { + "n": 10, + "seed": 22, + "target": "sonoscli", + "outcome": "hit", + "pick": "sonoscli", + "request": "Pause the speakers in the office, I'm hopping on a call right now." + }, + { + "n": 10, + "seed": 22, + "target": "blogwatcher", + "outcome": "hit", + "pick": "blogwatcher", + "request": "can you start watching the golang weekly newsletter and let me know whenever there's a new post about concurrency patterns?" + }, + { + "n": 10, + "seed": 22, + "target": "project-artifact", + "outcome": "hit", + "pick": "project-artifact", + "request": "hey can you set up a status page for the infrastructure migration project? we've got like 5 different teams working on different pieces and i need something i c" + }, + { + "n": 40, + "seed": 11, + "target": "knowledgefy", + "outcome": "hit", + "pick": "knowledgefy", + "request": "Can you kgfy this repo so I can quickly map out the core components before my 2 PM design review? https://github.com/vllm-project/vllm" + }, + { + "n": 40, + "seed": 11, + "target": "copilotkit", + "outcome": "hit", + "pick": "copilotkit", + "request": "I need to add an AI copilot sidebar to our Next.js app that can read page state and trigger actions\u2014can you set up the provider, API route, and hooks for it?" + }, + { + "n": 40, + "seed": 11, + "target": "moomoo-derivatives-anomaly", + "outcome": "hit", + "pick": "moomoo-derivatives-anomaly", + "request": "hey can you check if there's any unusual options activity in nvidia right now, want to see if smart money is positioning for something" + }, + { + "n": 40, + "seed": 11, + "target": "ux-research", + "outcome": "hit", + "pick": "ux-research", + "request": "hey can you do a deep dive on how stripe and square handle their dashboard UX? need to see what patterns they're using before we redesign ours" + }, + { + "n": 40, + "seed": 11, + "target": "benchmark", + "outcome": "hit", + "pick": "benchmark", + "request": "hey can you check if my latest PR regressed the page load times? i want to make sure the bundle size didn't blow up" + }, + { + "n": 40, + "seed": 11, + "target": "browser-harness", + "outcome": "hit", + "pick": "browser-harness", + "request": "can you log into our staging dashboard and pull the latest test results from the QA reports page, then export it as CSV?" + }, + { + "n": 40, + "seed": 11, + "target": "sag", + "outcome": "hit", + "pick": "sag", + "request": "hey can you read this design doc out loud for me while i finish this code review?" + }, + { + "n": 40, + "seed": 11, + "target": "canvas", + "outcome": "hit", + "pick": "canvas", + "request": "hey can you push this dashboard html to my iPad so I can see it on the big screen while I'm testing?" + }, + { + "n": 40, + "seed": 11, + "target": "longevity-loop", + "outcome": "hit", + "pick": "longevity-loop", + "request": "Hey, can you run the latest pyaging models on our cohort data and then benchmark the results against the leaderboard to see where we stand on the aging clock ac" + }, + { + "n": 40, + "seed": 11, + "target": "sonoscli", + "outcome": "hit", + "pick": "sonoscli", + "request": "Pause the speakers in the office, I'm hopping on a call right now." + }, + { + "n": 40, + "seed": 11, + "target": "blogwatcher", + "outcome": "hit", + "pick": "blogwatcher", + "request": "can you start watching the golang weekly newsletter and let me know whenever there's a new post about concurrency patterns?" + }, + { + "n": 40, + "seed": 11, + "target": "project-artifact", + "outcome": "hit", + "pick": "project-artifact", + "request": "hey can you set up a status page for the infrastructure migration project? we've got like 5 different teams working on different pieces and i need something i c" + }, + { + "n": 40, + "seed": 22, + "target": "knowledgefy", + "outcome": "hit", + "pick": "knowledgefy", + "request": "Can you kgfy this repo so I can quickly map out the core components before my 2 PM design review? https://github.com/vllm-project/vllm" + }, + { + "n": 40, + "seed": 22, + "target": "copilotkit", + "outcome": "hit", + "pick": "copilotkit", + "request": "I need to add an AI copilot sidebar to our Next.js app that can read page state and trigger actions\u2014can you set up the provider, API route, and hooks for it?" + }, + { + "n": 40, + "seed": 22, + "target": "moomoo-derivatives-anomaly", + "outcome": "hit", + "pick": "moomoo-derivatives-anomaly", + "request": "hey can you check if there's any unusual options activity in nvidia right now, want to see if smart money is positioning for something" + }, + { + "n": 40, + "seed": 22, + "target": "ux-research", + "outcome": "hit", + "pick": "ux-research", + "request": "hey can you do a deep dive on how stripe and square handle their dashboard UX? need to see what patterns they're using before we redesign ours" + }, + { + "n": 40, + "seed": 22, + "target": "benchmark", + "outcome": "hit", + "pick": "benchmark", + "request": "hey can you check if my latest PR regressed the page load times? i want to make sure the bundle size didn't blow up" + }, + { + "n": 40, + "seed": 22, + "target": "browser-harness", + "outcome": "hit", + "pick": "browser-harness", + "request": "can you log into our staging dashboard and pull the latest test results from the QA reports page, then export it as CSV?" + }, + { + "n": 40, + "seed": 22, + "target": "sag", + "outcome": "hit", + "pick": "sag", + "request": "hey can you read this design doc out loud for me while i finish this code review?" + }, + { + "n": 40, + "seed": 22, + "target": "canvas", + "outcome": "hit", + "pick": "canvas", + "request": "hey can you push this dashboard html to my iPad so I can see it on the big screen while I'm testing?" + }, + { + "n": 40, + "seed": 22, + "target": "longevity-loop", + "outcome": "hit", + "pick": "longevity-loop", + "request": "Hey, can you run the latest pyaging models on our cohort data and then benchmark the results against the leaderboard to see where we stand on the aging clock ac" + }, + { + "n": 40, + "seed": 22, + "target": "sonoscli", + "outcome": "hit", + "pick": "sonoscli", + "request": "Pause the speakers in the office, I'm hopping on a call right now." + }, + { + "n": 40, + "seed": 22, + "target": "blogwatcher", + "outcome": "hit", + "pick": "blogwatcher", + "request": "can you start watching the golang weekly newsletter and let me know whenever there's a new post about concurrency patterns?" + }, + { + "n": 40, + "seed": 22, + "target": "project-artifact", + "outcome": "hit", + "pick": "project-artifact", + "request": "hey can you set up a status page for the infrastructure migration project? we've got like 5 different teams working on different pieces and i need something i c" + }, + { + "n": 80, + "seed": 11, + "target": "knowledgefy", + "outcome": "hit", + "pick": "knowledgefy", + "request": "Can you kgfy this repo so I can quickly map out the core components before my 2 PM design review? https://github.com/vllm-project/vllm" + }, + { + "n": 80, + "seed": 11, + "target": "copilotkit", + "outcome": "hit", + "pick": "copilotkit", + "request": "I need to add an AI copilot sidebar to our Next.js app that can read page state and trigger actions\u2014can you set up the provider, API route, and hooks for it?" + }, + { + "n": 80, + "seed": 11, + "target": "moomoo-derivatives-anomaly", + "outcome": "hit", + "pick": "moomoo-derivatives-anomaly", + "request": "hey can you check if there's any unusual options activity in nvidia right now, want to see if smart money is positioning for something" + }, + { + "n": 80, + "seed": 11, + "target": "ux-research", + "outcome": "hit", + "pick": "ux-research", + "request": "hey can you do a deep dive on how stripe and square handle their dashboard UX? need to see what patterns they're using before we redesign ours" + }, + { + "n": 80, + "seed": 11, + "target": "benchmark", + "outcome": "hit", + "pick": "benchmark", + "request": "hey can you check if my latest PR regressed the page load times? i want to make sure the bundle size didn't blow up" + }, + { + "n": 80, + "seed": 11, + "target": "browser-harness", + "outcome": "wrong", + "pick": "browse", + "request": "can you log into our staging dashboard and pull the latest test results from the QA reports page, then export it as CSV?" + }, + { + "n": 80, + "seed": 11, + "target": "sag", + "outcome": "hit", + "pick": "sag", + "request": "hey can you read this design doc out loud for me while i finish this code review?" + }, + { + "n": 80, + "seed": 11, + "target": "canvas", + "outcome": "hit", + "pick": "canvas", + "request": "hey can you push this dashboard html to my iPad so I can see it on the big screen while I'm testing?" + }, + { + "n": 80, + "seed": 11, + "target": "longevity-loop", + "outcome": "wrong", + "pick": "coding-agent", + "request": "Hey, can you run the latest pyaging models on our cohort data and then benchmark the results against the leaderboard to see where we stand on the aging clock ac" + }, + { + "n": 80, + "seed": 11, + "target": "sonoscli", + "outcome": "hit", + "pick": "sonoscli", + "request": "Pause the speakers in the office, I'm hopping on a call right now." + }, + { + "n": 80, + "seed": 11, + "target": "blogwatcher", + "outcome": "hit", + "pick": "blogwatcher", + "request": "can you start watching the golang weekly newsletter and let me know whenever there's a new post about concurrency patterns?" + }, + { + "n": 80, + "seed": 11, + "target": "project-artifact", + "outcome": "hit", + "pick": "project-artifact", + "request": "hey can you set up a status page for the infrastructure migration project? we've got like 5 different teams working on different pieces and i need something i c" + }, + { + "n": 80, + "seed": 22, + "target": "knowledgefy", + "outcome": "wrong", + "pick": "graphify", + "request": "Can you kgfy this repo so I can quickly map out the core components before my 2 PM design review? https://github.com/vllm-project/vllm" + }, + { + "n": 80, + "seed": 22, + "target": "copilotkit", + "outcome": "hit", + "pick": "copilotkit", + "request": "I need to add an AI copilot sidebar to our Next.js app that can read page state and trigger actions\u2014can you set up the provider, API route, and hooks for it?" + }, + { + "n": 80, + "seed": 22, + "target": "moomoo-derivatives-anomaly", + "outcome": "hit", + "pick": "moomoo-derivatives-anomaly", + "request": "hey can you check if there's any unusual options activity in nvidia right now, want to see if smart money is positioning for something" + }, + { + "n": 80, + "seed": 22, + "target": "ux-research", + "outcome": "hit", + "pick": "ux-research", + "request": "hey can you do a deep dive on how stripe and square handle their dashboard UX? need to see what patterns they're using before we redesign ours" + }, + { + "n": 80, + "seed": 22, + "target": "benchmark", + "outcome": "hit", + "pick": "benchmark", + "request": "hey can you check if my latest PR regressed the page load times? i want to make sure the bundle size didn't blow up" + }, + { + "n": 80, + "seed": 22, + "target": "browser-harness", + "outcome": "hit", + "pick": "browser-harness", + "request": "can you log into our staging dashboard and pull the latest test results from the QA reports page, then export it as CSV?" + }, + { + "n": 80, + "seed": 22, + "target": "sag", + "outcome": "hit", + "pick": "sag", + "request": "hey can you read this design doc out loud for me while i finish this code review?" + }, + { + "n": 80, + "seed": 22, + "target": "canvas", + "outcome": "hit", + "pick": "canvas", + "request": "hey can you push this dashboard html to my iPad so I can see it on the big screen while I'm testing?" + }, + { + "n": 80, + "seed": 22, + "target": "longevity-loop", + "outcome": "none", + "pick": "NONE", + "request": "Hey, can you run the latest pyaging models on our cohort data and then benchmark the results against the leaderboard to see where we stand on the aging clock ac" + }, + { + "n": 80, + "seed": 22, + "target": "sonoscli", + "outcome": "hit", + "pick": "sonoscli", + "request": "Pause the speakers in the office, I'm hopping on a call right now." + }, + { + "n": 80, + "seed": 22, + "target": "blogwatcher", + "outcome": "hit", + "pick": "blogwatcher", + "request": "can you start watching the golang weekly newsletter and let me know whenever there's a new post about concurrency patterns?" + }, + { + "n": 80, + "seed": 22, + "target": "project-artifact", + "outcome": "hit", + "pick": "project-artifact", + "request": "hey can you set up a status page for the infrastructure migration project? we've got like 5 different teams working on different pieces and i need something i c" + }, + { + "n": 175, + "seed": 11, + "target": "knowledgefy", + "outcome": "wrong", + "pick": "knowledge-graph", + "request": "Can you kgfy this repo so I can quickly map out the core components before my 2 PM design review? https://github.com/vllm-project/vllm" + }, + { + "n": 175, + "seed": 11, + "target": "copilotkit", + "outcome": "hit", + "pick": "copilotkit", + "request": "I need to add an AI copilot sidebar to our Next.js app that can read page state and trigger actions\u2014can you set up the provider, API route, and hooks for it?" + }, + { + "n": 175, + "seed": 11, + "target": "moomoo-derivatives-anomaly", + "outcome": "hit", + "pick": "moomoo-derivatives-anomaly", + "request": "hey can you check if there's any unusual options activity in nvidia right now, want to see if smart money is positioning for something" + }, + { + "n": 175, + "seed": 11, + "target": "ux-research", + "outcome": "hit", + "pick": "ux-research", + "request": "hey can you do a deep dive on how stripe and square handle their dashboard UX? need to see what patterns they're using before we redesign ours" + }, + { + "n": 175, + "seed": 11, + "target": "benchmark", + "outcome": "hit", + "pick": "benchmark", + "request": "hey can you check if my latest PR regressed the page load times? i want to make sure the bundle size didn't blow up" + }, + { + "n": 175, + "seed": 11, + "target": "browser-harness", + "outcome": "wrong", + "pick": "browse", + "request": "can you log into our staging dashboard and pull the latest test results from the QA reports page, then export it as CSV?" + }, + { + "n": 175, + "seed": 11, + "target": "sag", + "outcome": "wrong", + "pick": "sherpa-onnx-tts", + "request": "hey can you read this design doc out loud for me while i finish this code review?" + }, + { + "n": 175, + "seed": 11, + "target": "canvas", + "outcome": "wrong", + "pick": "installable-web-app", + "request": "hey can you push this dashboard html to my iPad so I can see it on the big screen while I'm testing?" + }, + { + "n": 175, + "seed": 11, + "target": "longevity-loop", + "outcome": "none", + "pick": "NONE", + "request": "Hey, can you run the latest pyaging models on our cohort data and then benchmark the results against the leaderboard to see where we stand on the aging clock ac" + }, + { + "n": 175, + "seed": 11, + "target": "sonoscli", + "outcome": "error", + "detail": "" + }, + { + "n": 175, + "seed": 11, + "target": "blogwatcher", + "outcome": "error", + "detail": "" + }, + { + "n": 175, + "seed": 11, + "target": "project-artifact", + "outcome": "error", + "detail": "" + }, + { + "n": 175, + "seed": 22, + "target": "knowledgefy", + "outcome": "error", + "detail": "" + }, + { + "n": 175, + "seed": 22, + "target": "copilotkit", + "outcome": "error", + "detail": "" + }, + { + "n": 175, + "seed": 22, + "target": "moomoo-derivatives-anomaly", + "outcome": "error", + "detail": "" + }, + { + "n": 175, + "seed": 22, + "target": "ux-research", + "outcome": "error", + "detail": "" + }, + { + "n": 175, + "seed": 22, + "target": "benchmark", + "outcome": "error", + "detail": "" + }, + { + "n": 175, + "seed": 22, + "target": "browser-harness", + "outcome": "error", + "detail": "" + }, + { + "n": 175, + "seed": 22, + "target": "sag", + "outcome": "error", + "detail": "" + }, + { + "n": 175, + "seed": 22, + "target": "canvas", + "outcome": "error", + "detail": "" + }, + { + "n": 175, + "seed": 22, + "target": "longevity-loop", + "outcome": "error", + "detail": "" + }, + { + "n": 175, + "seed": 22, + "target": "sonoscli", + "outcome": "error", + "detail": "" + }, + { + "n": 175, + "seed": 22, + "target": "blogwatcher", + "outcome": "error", + "detail": "" + }, + { + "n": 175, + "seed": 22, + "target": "project-artifact", + "outcome": "error", + "detail": "" + } + ] +} \ No newline at end of file diff --git a/research/attention-budget/skillmeta.py b/research/attention-budget/skillmeta.py new file mode 100644 index 0000000..d047793 --- /dev/null +++ b/research/attention-budget/skillmeta.py @@ -0,0 +1,112 @@ +"""Read `SKILL.md` frontmatter correctly — including YAML block scalars. + +WHY THIS MODULE EXISTS (bug found 2026-08-13) +--------------------------------------------- +Both tools here originally parsed frontmatter with `^description:\\s*(.*)$`. That +regex is wrong for the block-scalar forms YAML permits, which real skills use: + + description: | + A long description + over several lines + + description: >- + A folded description + +The regex captured the INDICATOR (`|`, `>-`) and returned a one- or two-character +"description". It never raised; it just quietly produced a wrong value. That +understated the measured residency budget and, worse, fed meaningless probes into +the trigger-reliability experiment, whose ground truth then measured nothing but +this bug. + +It is the same failure class the audit itself reports: a parser that returns a +plausible wrong answer instead of refusing. Hence one shared, tested reader. +""" + +from __future__ import annotations + +import pathlib +import re + +_INDICATOR = re.compile(r"^[|>]([+-]?\d*|\d*[+-]?)$") + + +def parse_frontmatter(text: str) -> dict[str, str]: + """Top-level scalar fields of a `---`-delimited YAML frontmatter block. + + Handles plain scalars, quoted scalars, and `|`/`>` block scalars (with any + chomping/indent indicator). Nested mappings and sequences are skipped rather + than guessed at — this is a frontmatter reader, not a YAML implementation. + """ + m = re.match(r"^---\s*\n(.*?)\n---\s*(?:\n|$)", text, re.S) + if not m: + return {} + lines = m.group(1).split("\n") + + out: dict[str, str] = {} + i = 0 + while i < len(lines): + line = lines[i] + km = re.match(r"^([A-Za-z0-9_-]+):[ \t]*(.*)$", line) + if not km: + i += 1 + continue + key, rest = km.group(1), km.group(2).strip() + + if _INDICATOR.match(rest): + # Block scalar: consume the indented continuation. + block: list[str] = [] + i += 1 + while i < len(lines): + nxt = lines[i] + if nxt.strip() and not re.match(r"^[ \t]", nxt): + break # dedented to a new key + block.append(nxt.strip()) + i += 1 + folded = " " if rest.startswith(">") else "\n" + out[key] = folded.join(b for b in block if b != "").strip() + continue + + if rest: + out[key] = rest.strip("\"'") + i += 1 + return out + + +def load_skills(root: pathlib.Path) -> tuple[list[dict], list[dict]]: + """(readable, phantom) skill records under ``root``. + + A phantom is an entry whose `SKILL.md` cannot be read — it holds a name and + loads nothing. It is reported, never silently skipped. + """ + readable: list[dict] = [] + phantom: list[dict] = [] + for p in sorted(root.rglob("SKILL.md")): + rel = str(p.relative_to(root)) + try: + text = p.read_text(encoding="utf-8", errors="replace") + except OSError as e: + target = None + try: + target = str(p.readlink()) + except OSError: + pass + phantom.append({"path": rel, "dir": p.parent.name, "target": target, + "why": type(e).__name__}) + continue + fm = parse_frontmatter(text) + readable.append({ + "name": fm.get("name") or p.parent.name, + "desc": fm.get("description", ""), + "hint": fm.get("argument-hint", ""), + "body_chars": len(text), + "path": rel, + }) + return readable, phantom + + +def distinct(readable: list[dict]) -> dict[str, dict]: + """First record per skill name, in scan order.""" + out: dict[str, dict] = {} + for r in readable: + out.setdefault(r["name"], r) + return out diff --git a/research/attention-budget/test_skillmeta.py b/research/attention-budget/test_skillmeta.py new file mode 100644 index 0000000..1866eea --- /dev/null +++ b/research/attention-budget/test_skillmeta.py @@ -0,0 +1,78 @@ +"""Tests for the frontmatter reader — pinning the block-scalar bug it was written to fix. + +Run: python3 -m pytest research/attention-budget/test_skillmeta.py -q +""" + +from __future__ import annotations + +import pathlib +import sys + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) + +from skillmeta import distinct, load_skills, parse_frontmatter # noqa: E402 + + +def test_plain_scalar(): + assert parse_frontmatter("---\nname: a\ndescription: hello there\n---\nbody\n")["description"] == "hello there" + + +def test_quoted_scalar_is_unquoted(): + assert parse_frontmatter('---\ndescription: "quoted desc"\n---\n')["description"] == "quoted desc" + assert parse_frontmatter("---\ndescription: 'single'\n---\n")["description"] == "single" + + +def test_literal_block_scalar(): + """THE BUG: this used to return '|' -- a one-character description.""" + fm = parse_frontmatter("---\nname: x\ndescription: |\n first line\n second line\nversion: 1\n---\nbody\n") + assert fm["description"] == "first line\nsecond line" + assert fm["name"] == "x" + assert fm["version"] == "1" # the key AFTER the block is still parsed + + +def test_folded_block_scalar_joins_with_spaces(): + fm = parse_frontmatter("---\ndescription: >-\n folded one\n folded two\n---\n") + assert fm["description"] == "folded one folded two" + + +def test_block_indicator_variants(): + for ind in ("|", "|-", "|+", ">", ">-", ">+", "|2", "|2-"): + fm = parse_frontmatter(f"---\ndescription: {ind}\n content here\n---\n") + assert fm["description"] == "content here", ind + + +def test_no_frontmatter_is_empty_not_an_error(): + assert parse_frontmatter("# just markdown\n") == {} + + +def test_missing_description_is_absent_not_a_fake_value(): + assert "description" not in parse_frontmatter("---\nname: only\n---\n") + + +def test_phantom_is_reported_not_skipped(tmp_path): + good = tmp_path / "good" + good.mkdir() + (good / "SKILL.md").write_text("---\nname: good\ndescription: fine\n---\n") + bad = tmp_path / "bad" + bad.mkdir() + (bad / "SKILL.md").symlink_to(tmp_path / "nowhere" / "SKILL.md") + + readable, phantom = load_skills(tmp_path) + assert [r["name"] for r in readable] == ["good"] + assert len(phantom) == 1 and phantom[0]["dir"] == "bad" + + +def test_distinct_keeps_first_occurrence(): + recs = [{"name": "a", "desc": "first"}, {"name": "a", "desc": "second"}] + assert distinct(recs)["a"]["desc"] == "first" + + +def test_real_library_has_no_indicator_only_descriptions(): + """Regression guard against the shipped bug: if the reader ever regresses, block-scalar + skills reappear as 1-2 char descriptions. Skipped when no library is present.""" + root = pathlib.Path.home() / ".claude" + if not root.is_dir(): + return + readable, _ = load_skills(root) + stubs = [r["name"] for r in readable if r["desc"] in ("|", "|-", ">", ">-", "|+", ">+")] + assert not stubs, f"indicator-only descriptions leaked through: {stubs[:10]}" diff --git a/research/attention-budget/trigger_reliability.py b/research/attention-budget/trigger_reliability.py new file mode 100644 index 0000000..04734bc --- /dev/null +++ b/research/attention-budget/trigger_reliability.py @@ -0,0 +1,371 @@ +#!/usr/bin/env python3 +"""Measure trigger reliability as a function of installed-skill count. + +WHY THIS EXISTS +--------------- +The @skills paper (arXiv:2608.12610, Yin et al., 12 Aug 2026) argues that agent +skills compete for "fewer than 100 reliable auto-trigger slots per agent" and is +explicit that this number is NOT measured: + + "Our central quantity, the number of reliable auto-trigger slots, is bounded + by argument and by the literature rather than measured by us" + +and it names the missing experiment as future work: + + "The measurements this argument invites are trigger reliability as a function + of installed-skill count" + +This script runs that experiment against a REAL skill corpus (the operator's own +installed skills), not a synthetic one. + +DESIGN +------ +For each trial: build a menu of N skill (name, description) pairs that always +contains one target, hand the subject model a first-person user request that the +target is meant to serve, and ask which single skill should fire. Score top-1. + +Two independent model families, so a result is not one vendor's quirk: + * generator (writes the probe requests) — Gemini, never sees the distractors + * subject (picks the skill) — Groq/gpt-oss, never sees the target label +That split is maker != checker applied to the data itself. + +DECLARED BIAS (read before quoting any number) +---------------------------------------------- +Probe requests are generated FROM the target's own description, so the target's +trigger wording leaks into the request. That makes every trial EASIER than reality, +where a user's phrasing is not derived from the description at all. So the accuracy +reported here is an OPTIMISTIC UPPER BOUND: whatever degradation appears, the real +degradation is at least that large. This is stated in the output, not buried. + +HONESTY RULES +------------- +* A failed API call is recorded as an error and excluded from the denominator -- + never silently scored as a miss (that would manufacture the paper's conclusion). +* Every response is cached to disk keyed by its exact prompt, so a re-run is free + and the numbers are reproducible rather than re-sampled. +* If fewer than MIN_TRIALS usable trials land for an N, that N reports "insufficient + data" instead of a number. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import pathlib +import random +import re +import sys +import time +import urllib.error +import urllib.request + +from skillmeta import distinct as _distinct +from skillmeta import load_skills as _load_skills + +HERE = pathlib.Path(__file__).resolve().parent +CACHE = HERE / ".cache" +CACHE.mkdir(exist_ok=True) + +SUBJECT_MODEL = "openai/gpt-oss-120b" # Groq +GENERATOR_MODEL = "gemini-3.6-flash" # Google +FALLBACK_GENERATOR = "claude-haiku-4-5-20251001" # Anthropic - only if Gemini quota is spent +GENERATOR_USED: set[str] = set() +N_SWEEP = [10, 40, 80, 176] +SEEDS = [11, 22] +MIN_TRIALS = 8 + + +# ----- skill corpus ------------------------------------------------------- + + +def load_skills() -> dict[str, str]: + """Distinct {name: description} via the shared frontmatter reader. + + A local regex here once captured YAML block-scalar INDICATORS ("|", ">-") as the + description, which handed several unrelated targets the same meaningless probe. + See skillmeta.py.""" + readable, _ = _load_skills(pathlib.Path.home() / ".claude") + return {n: r["desc"] for n, r in _distinct(readable).items() if r["desc"]} + + +# ----- transport ---------------------------------------------------------- + + +class ProviderError(RuntimeError): + """A provider call failed. Recorded, never scored as a model miss.""" + + +class RateLimited(ProviderError): + """429. Carries the server's own Retry-After when it supplies one.""" + + def __init__(self, retry_after: float | None, msg: str): + super().__init__(msg) + self.retry_after = retry_after + + +def with_backoff(fn, tries: int = 7): + """Retry ONLY rate limits, waiting the duration the server asks for. A free tier + that says 'wait 20s' is not an error to report -- it is an instruction.""" + delay = 5.0 + for attempt in range(tries): + try: + return fn() + except RateLimited as e: + if attempt == tries - 1: + raise + time.sleep(e.retry_after if e.retry_after else delay) + delay = min(delay * 1.8, 90) + raise ProviderError("unreachable") + + +# Groq sits behind Cloudflare, which 403s (error 1010) on urllib's default +# User-Agent. Measured, not guessed: the identical payload succeeds under curl. +_UA = "attention-budget-research/1.0 (+loop-engineering-anything)" + + +def _post(url: str, key: str, payload: dict, timeout: int = 90, extra: dict | None = None) -> dict: + body = json.dumps(payload).encode() + headers = { + "Authorization": f"Bearer {key}", + "Content-Type": "application/json", + "User-Agent": _UA, + "Accept": "application/json", + } + headers.update(extra or {}) + req = urllib.request.Request(url, data=body, headers=headers) + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.loads(r.read()) + except urllib.error.HTTPError as e: + detail = e.read()[:200] + if e.code == 429: + wait = e.headers.get("retry-after") + raise RateLimited(float(wait) if wait and wait.replace('.','',1).isdigit() else None, + f"HTTP 429: {detail!r}") from e + raise ProviderError(f"HTTP {e.code}: {detail!r}") from e + except Exception as e: # noqa: BLE001 -- transport is diverse; report precisely + raise ProviderError(str(e)) from e + + +def _cached(tag: str, prompt_key: str, fn): + h = hashlib.sha256(f"{tag}\0{prompt_key}".encode()).hexdigest()[:20] + f = CACHE / f"{tag}-{h}.json" + if f.exists(): + return json.loads(f.read_text())["value"] + value = fn() + f.write_text(json.dumps({"value": value})) + return value + + +def ask_subject(prompt: str) -> str: + """Groq. Needs a generous max_tokens: reasoning tokens are billed to the same + budget and a small cap yields an EMPTY content field (measured, not assumed).""" + key = os.environ.get("GROQ_API_KEY") + if not key: + raise ProviderError("GROQ_API_KEY not set") + + def call(): + d = _post( + "https://api.groq.com/openai/v1/chat/completions", key, + {"model": SUBJECT_MODEL, "temperature": 0, "max_tokens": 500, + "messages": [{"role": "user", "content": prompt}]}, + ) + return (d["choices"][0]["message"].get("content") or "").strip() + + return _cached("subject", prompt, lambda: with_backoff(call)) + + +def ask_generator(prompt: str) -> str: + """Probe writer. + + max_tokens is 1200, not 200. MEASURED 2026-08-13: at 200 the Gemini response came + back as a 15-28 char FRAGMENT ("Hey, can you pull"), because reasoning tokens are + billed against the same budget -- so several unrelated targets received identical, + meaningless probes and the resulting accuracy measured nothing but this bug. The + same trap hit the subject model on a different provider. If an answer looks + truncated, suspect the token budget before the model. Deliberately a DIFFERENT model family from the subject, so the + request wording is not authored by the same model that has to route it. + + Chain: Gemini first (free tier); on quota exhaustion fall back to Claude Haiku. + A fallback is recorded in the output -- never silently substituted.""" + errors = [] + + gkey = os.environ.get("GEMINI_API_KEY") + if gkey: + def gemini(): + d = _post( + "https://generativelanguage.googleapis.com/v1beta/openai/chat/completions", gkey, + {"model": GENERATOR_MODEL, "temperature": 0.7, "max_tokens": 1200, + "messages": [{"role": "user", "content": prompt}]}, + ) + return (d["choices"][0]["message"].get("content") or "").strip() + try: + return _cached("gen", prompt, lambda: with_backoff(gemini, tries=2)) + except ProviderError as e: + errors.append(f"gemini: {e}") + + akey = os.environ.get("ANTHROPIC_API_KEY") + if akey: + def claude(): + d = _post( + "https://api.anthropic.com/v1/messages", akey, + {"model": FALLBACK_GENERATOR, "max_tokens": 1200, "temperature": 1.0, + "messages": [{"role": "user", "content": prompt}]}, + extra={"x-api-key": akey, "anthropic-version": "2023-06-01"}, + ) + parts = [b.get("text", "") for b in d.get("content", []) if b.get("type") == "text"] + GENERATOR_USED.add(FALLBACK_GENERATOR) + return "".join(parts).strip() + try: + return _cached("gen2", prompt, lambda: with_backoff(claude)) + except ProviderError as e: + errors.append(f"claude: {e}") + + raise ProviderError("; ".join(errors) or "no generator key set") + + +# ----- the experiment ----------------------------------------------------- + + +def make_probe(name: str, desc: str) -> str: + """A realistic first-person request the target skill is meant to serve.""" + out = ask_generator( + "Below is the description of a tool available to an AI assistant.\n" + "Write ONE realistic first-person message a busy engineer would type to the\n" + "assistant that this tool is meant to handle. Rules: do NOT name the tool, do\n" + "NOT use the word 'skill', write it the way a person actually types (one or two\n" + f"sentences, no preamble, no quotes).\n\nTOOL DESCRIPTION:\n{desc[:900]}" + ) + # BUG FIXED 2026-08-13: this used to be `.split("\n")[0]`, which kept only the + # FIRST LINE of a wrapped generation. That produced truncated, generic requests + # ("Hey, can you pull") that were IDENTICAL across unrelated targets, so the + # ground truth was corrupt and the measured accuracy was measuring this bug. + # Collapse whitespace instead, and keep the whole request. + return " ".join(out.replace("\n", " ").split()).strip('"').strip()[:400] + + +def menu_prompt(menu: list[tuple[str, str]], request: str) -> str: + lines = "\n".join(f"- {n}: {d[:160]}" for n, d in menu) + return ( + "You route a user's message to at most one tool.\n\n" + f"AVAILABLE TOOLS ({len(menu)}):\n{lines}\n\n" + f"USER MESSAGE:\n{request}\n\n" + "Which single tool should handle this? Answer with the tool name exactly as " + "written above, or the word NONE. Output only that one token." + ) + + +def parse_pick(raw: str, valid: set[str]) -> str | None: + t = raw.strip().strip("`*.,:;\"' ").split() + if not t: + return None + for tok in (t[-1], t[0]): + c = tok.strip("`*.,:;\"' ") + if c in valid or c.upper() == "NONE": + return "NONE" if c.upper() == "NONE" else c + for name in valid: # last resort: the answer mentions exactly one valid name + if re.search(rf"\b{re.escape(name)}\b", raw): + return name + return None + + +def _one_trial(target: str, request: str, names: list[str], skills: dict[str, str], + cap: int, rng: random.Random) -> tuple[str, str | None, str | None]: + """Run a single trial. Returns (outcome, pick, error_detail).""" + pool = [x for x in names if x != target] + menu_names = rng.sample(pool, cap - 1) + [target] + rng.shuffle(menu_names) + menu = [(x, skills[x]) for x in menu_names] + try: + raw = ask_subject(menu_prompt(menu, request)) + except ProviderError as e: + return "error", None, str(e)[:120] + pick = parse_pick(raw, set(menu_names)) + if pick == target: + return "hit", pick, None + if pick == "NONE": + return "none", pick, None + if pick is None: + return "unparsed", pick, None + return "wrong", pick, None + + +def _build_probes(targets: list[str], skills: dict[str, str]) -> dict[str, str]: + probes: dict[str, str] = {} + for t in targets: + try: + probes[t] = make_probe(t, skills[t]) + except ProviderError as e: + print(f" ! probe generation failed for {t}: {str(e)[:110]}") + return probes + + +def _sweep(probes: dict[str, str], names: list[str], skills: dict[str, str]) -> tuple[dict, list]: + results = {n: {"hit": 0, "wrong": 0, "none": 0, "unparsed": 0, "error": 0} for n in N_SWEEP} + trials: list[dict] = [] + for n in N_SWEEP: + cap = min(n, len(names)) + for seed in SEEDS: + rng = random.Random(seed) + for target, request in probes.items(): + outcome, pick, detail = _one_trial(target, request, names, skills, cap, rng) + results[n][outcome] += 1 + row = {"n": cap, "seed": seed, "target": target, "outcome": outcome} + if detail: + row["detail"] = detail + else: + row.update({"pick": pick, "request": request[:160]}) + trials.append(row) + if outcome != "error": + time.sleep(2.0) # free tier: stay under the per-minute cap by construction + d = results[n] + usable = d["hit"] + d["wrong"] + d["none"] + d["unparsed"] + acc = f"{100*d['hit']/usable:.1f}%" if usable >= MIN_TRIALS else "insufficient data" + print(f" N={cap:4d} top-1 {acc:>18} (hit {d['hit']} wrong {d['wrong']} " + f"none {d['none']} unparsed {d['unparsed']} errors {d['error']})") + return results, trials + + +def main() -> int: + skills = load_skills() + names = sorted(skills) + print(f"corpus: {len(names)} distinct readable skills with a description") + if len(names) < max(N_SWEEP): + print(f"note: corpus smaller than max N; capping sweep at {len(names)}") + + rng = random.Random(7) + targets = rng.sample(names, min(12, len(names))) + print(f"generating {len(targets)} probe requests ...") + probes = _build_probes(targets, skills) + print(f" {len(probes)} probes ready") + if not probes: + print("no probes -- refusing to report a number") + return 1 + + results, trials = _sweep(probes, names, skills) + + out = { + "paper": "arXiv:2608.12610 (Yin et al., 12 Aug 2026)", + "measures": "trigger reliability as a function of installed-skill count " + "(the paper's own named future work)", + "declared_bias": "probe requests are generated FROM each target's description, so " + "target trigger wording leaks into the request; these accuracies are " + "an OPTIMISTIC UPPER BOUND on real-world routing", + "subject_model": SUBJECT_MODEL, + "generator_model": GENERATOR_MODEL, + "generator_fallbacks_used": sorted(GENERATOR_USED), + "corpus_size": len(names), + "targets": sorted(probes), + "seeds": SEEDS, + "min_trials": MIN_TRIALS, + "by_n": {str(n): results[n] for n in N_SWEEP}, + "trials": trials, + } + (HERE / "results.json").write_text(json.dumps(out, indent=2)) + print(f"\nwrote {HERE/'results.json'}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/audit_loop_rubric.py b/scripts/audit_loop_rubric.py new file mode 100644 index 0000000..50721ef --- /dev/null +++ b/scripts/audit_loop_rubric.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +"""Score this engine against the Agent Loop Engineering rubric — with observed evidence. + +The rubric (docs/rubrics/agent-loop-engineering.yml) encodes the claims of a webinar +summary that asserts what a mature agent loop must have and verifies none of it against a +running system. This script is the missing half: every claim becomes a probe against +shipped code or a real test run, and the score is whatever the probes return. + +Discipline (playbook: operationalizing-a-paper-rubric-checklist-standard): + + * Evidence is OBSERVED -- a symbol at a file:line, or a pytest node that actually + passes. Never a claim in prose. + * No evidence => NO. + * A probe that cannot run is `unmeasured`: excluded from the rate AND blocking. An + unmeasured item is never a silent pass, because "I could not look" is not "it works". + * `expect: absent` items are honest gap declarations. They PASS while the gap is real + and FAIL the moment the gap silently closes, so the published gap list cannot rot. + * Exits non-zero under --gate, so CI can hold the line. + +Usage: + python3 scripts/audit_loop_rubric.py [--json] [--gate] [--out FILE] +""" + +from __future__ import annotations + +import argparse +import json +import pathlib +import re +import subprocess +import sys + +import yaml + +ROOT = pathlib.Path(__file__).resolve().parent.parent +RUBRIC = ROOT / "docs" / "rubrics" / "agent-loop-engineering.yml" + +PASS, FAIL, UNMEASURED = "pass", "fail", "unmeasured" + + +def _iter_files(paths: list[str]) -> list[pathlib.Path]: + out: list[pathlib.Path] = [] + for rel in paths: + p = ROOT / rel + if p.is_dir(): + out.extend(sorted(q for q in p.rglob("*.py") if "__pycache__" not in q.parts)) + elif p.is_file(): + out.append(p) + return out + + +def probe_grep(spec: dict) -> tuple[bool, str]: + """Observed evidence: the first file:line where the pattern appears.""" + files = _iter_files(spec.get("paths", [])) + if not files: + raise FileNotFoundError(f"no such path(s): {spec.get('paths')}") + rx = re.compile(spec["pattern"]) + for f in files: + for n, line in enumerate(f.read_text(encoding="utf-8", errors="replace").splitlines(), 1): + if rx.search(line): + return True, f"{f.relative_to(ROOT)}:{n}" + return False, f"pattern /{spec['pattern']}/ not found in {len(files)} file(s)" + + +def probe_pytest(spec: dict) -> tuple[bool, str]: + """Observed evidence: the test is RUN, not merely present on disk.""" + node = spec["node"] + target = ROOT / node.split("::")[0] + if not target.exists(): + raise FileNotFoundError(f"no such test file: {node}") + py = ROOT / ".venv" / "bin" / "python" + exe = [str(py)] if py.exists() else [sys.executable] + r = subprocess.run(exe + ["-m", "pytest", node, "-q", "--no-header", "-x"], + cwd=ROOT, capture_output=True, text=True, timeout=600) + tail = (r.stdout.strip().splitlines() or ["(no output)"])[-1][:110] + return r.returncode == 0, f"{node} -> {tail}" + + +PROBES = {"grep": probe_grep, "pytest": probe_pytest} + + +def run_item(item: dict) -> dict: + spec = item["probe"] + expect_absent = item.get("expect") == "absent" + fn = PROBES.get(spec.get("type")) + if fn is None: + return {**item, "status": UNMEASURED, "evidence": f"unknown probe type {spec.get('type')!r}"} + try: + found, evidence = fn(spec) + except Exception as e: # noqa: BLE001 -- a probe that cannot run is unmeasured, never a pass + return {**item, "status": UNMEASURED, "evidence": f"probe could not run: {e}"} + if expect_absent: + ok = not found + evidence = ("gap confirmed absent" if ok + else f"GAP CLOSED without updating the rubric: {evidence}") + else: + ok = found + return {**item, "status": PASS if ok else FAIL, "evidence": evidence} + + +def audit() -> dict: + doc = yaml.safe_load(RUBRIC.read_text(encoding="utf-8")) + results = [run_item(i) for i in doc["items"]] + + verifiable = [r for r in results if not r.get("expect")] + implemented = [r for r in verifiable if r["status"] == PASS] + missing = [r for r in verifiable if r["status"] == FAIL] + gaps = [r for r in results if r.get("expect") == "absent"] + unmeasured = [r for r in results if r["status"] == UNMEASURED] + + scored = [r for r in verifiable if r["status"] != UNMEASURED] + rate = len([r for r in scored if r["status"] == PASS]) / len(scored) if scored else 0.0 + gate_ok = not missing and not unmeasured and all(g["status"] == PASS for g in gaps) + + return { + "source": doc["source"], + "total_items": len(results), + "verifiable": len(verifiable), + "implemented": len(implemented), + "missing": len(missing), + "declared_gaps": len(gaps), + "unmeasured": len(unmeasured), + "conformance_rate": round(rate, 3), + "gate_ok": gate_ok, + "results": results, + } + + +def render(a: dict) -> str: + src = a["source"] + L = ["# Agent Loop Engineering — conformance audit", "", + f"Rubric: `docs/rubrics/agent-loop-engineering.yml` — {a['total_items']} items drawn from " + f"*{src['title']}* ({src['publisher']}, received {src['received']}).", "", + f"That source is a **{src['nature']}**. Every claim below is scored against shipped " + "code or a test that was actually executed. No evidence means no.", "", + f"**Conformance: {a['implemented']}/{a['verifiable']} " + f"({a['conformance_rate']*100:.0f}%)** verifiable claims implemented · " + f"{a['declared_gaps']} declared gaps · {a['unmeasured']} unmeasured · " + f"gate **{'PASS' if a['gate_ok'] else 'FAIL'}**", ""] + + for bucket, title in ((PASS, "Implemented — with observed evidence"), + (FAIL, "NOT implemented"), + (UNMEASURED, "Unmeasured — blocks the gate, never counts as a pass")): + rows = [r for r in a["results"] if r["status"] == bucket and not r.get("expect")] + if not rows: + continue + L += [f"## {title}", ""] + for r in rows: + L.append(f"- **{r['id']}** · {r['section']}") + L.append(f" - claim: {r['claim']}") + L.append(f" - evidence: `{r['evidence']}`") + if r.get("rationale"): + L.append(f" - why it is still a miss: {r['rationale'].strip()}") + L.append("") + + gaps = [r for r in a["results"] if r.get("expect") == "absent"] + if gaps: + L += ["## Declared gaps — the engine does NOT do these, and says so", "", + "Each probe passes while the gap is real and fails the moment it silently closes, " + "so this list cannot quietly go stale.", ""] + for r in gaps: + mark = "confirmed absent" if r["status"] == PASS else "STALE — gap closed, update the rubric" + L.append(f"- **{r['id']}** · {r['section']}") + L.append(f" - claim: {r['claim']}") + L.append(f" - status: **{mark}** — {r.get('gap_note', '')}") + L.append("") + return "\n".join(L) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--json", action="store_true") + ap.add_argument("--gate", action="store_true", help="exit 1 unless every item passes") + ap.add_argument("--out") + args = ap.parse_args() + + a = audit() + text = json.dumps(a, indent=2) if args.json else render(a) + if args.out: + pathlib.Path(args.out).write_text(text + "\n", encoding="utf-8") + print(f"wrote {args.out}") + else: + print(text) + if args.gate and not a["gate_ok"]: + print(f"\nGATE FAIL: {a['missing']} missing, {a['unmeasured']} unmeasured", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/loopeng/memory/store.py b/src/loopeng/memory/store.py index 2edfc2e..27493c7 100644 --- a/src/loopeng/memory/store.py +++ b/src/loopeng/memory/store.py @@ -676,6 +676,53 @@ def first_attempt_grade_series(self, target: str) -> list[tuple[int, str]]: ).fetchall() return [(row["run_id"], row["grade"]) for row in rows] + def success_rate(self, target: str | None = None) -> dict: + """Task success RATE across runs -- the cross-run metric the loop lacked. + + Section 14 of the Agent Loop Engineering rubric asks for token cost, wall time, + iteration count AND task success rate. The first three were per-run and already + recorded; this is the fourth, and it is the only one that cannot be read off a + single run. + + Returns counts plus ``rate`` = converged / (converged + stopped + blocked_safety). + + Two deliberate choices, because a success metric is the easiest thing to flatter: + + * ``running`` rows are counted and reported but EXCLUDED from the denominator -- + an unfinished run is not a failure, and folding it in either way would move the + rate without evidence. + * ``blocked_safety`` counts as a FAILURE. A run halted by the safety gate did not + deliver the artifact, and a rate that quietly forgave safety blocks would reward + exactly the outcome the gate exists to prevent. + + ``rate`` is ``None`` when nothing has finished -- never 0.0, which would read as + "we tried and failed" rather than "we have not measured". + """ + sql = "SELECT status, COUNT(*) AS n FROM runs" + params: tuple = () + if target is not None: + sql += " WHERE target = ?" + params = (target,) + sql += " GROUP BY status" + with self._wlock: + rows = self._conn.execute(sql, params).fetchall() + + counts = {r["status"]: r["n"] for r in rows} + converged = counts.get("converged", 0) + stopped = counts.get("stopped", 0) + blocked = counts.get("blocked_safety", 0) + running = counts.get("running", 0) + finished = converged + stopped + blocked + return { + "target": target, + "converged": converged, + "stopped": stopped, + "blocked_safety": blocked, + "running_excluded": running, + "finished": finished, + "rate": round(converged / finished, 4) if finished else None, + } + def record_injected_count(self, run_id: int, count: int) -> None: """Record how many reused prior learnings were injected into this run's briefs (flywheel U5). Observation-only -- never read into convergence.""" diff --git a/tests/test_success_rate.py b/tests/test_success_rate.py new file mode 100644 index 0000000..1284d31 --- /dev/null +++ b/tests/test_success_rate.py @@ -0,0 +1,79 @@ +"""Cross-run task success rate (Agent Loop Engineering rubric, section 14). + +The rubric asks for tokens, time, iterations AND success rate. The first three are +per-run; this is the only one that needs the whole history. These tests pin the two +choices that decide whether the number is honest: + + * an unfinished run must not move the rate in either direction; + * a safety block must count as a failure, or the metric rewards the outcome the + safety gate exists to prevent. +""" + +from __future__ import annotations + +import pytest + +from loopeng.memory.store import MemoryStore + + +@pytest.fixture +def store(tmp_path): + s = MemoryStore(tmp_path / "sr.db") + yield s + s.close() + + +def _run(store, target: str, status: str | None, grade: str | None = None) -> int: + run_id = store.create_run(target, "codebase", "g", "2026-08-21T10:00:00") + if status is not None: + store.finish_run(run_id, status, grade) + return run_id + + +def test_no_finished_runs_reports_none_not_zero(store): + """0.0 reads as 'we tried and failed'. None reads as 'not measured'.""" + assert store.success_rate()["rate"] is None + _run(store, "./a", None) # still running + out = store.success_rate() + assert out["rate"] is None and out["running_excluded"] == 1 + + +def test_rate_is_converged_over_finished(store): + _run(store, "./a", "converged", "A") + _run(store, "./a", "converged", "A") + _run(store, "./a", "stopped", "C") + out = store.success_rate() + assert out["converged"] == 2 and out["stopped"] == 1 + assert out["finished"] == 3 and out["rate"] == pytest.approx(2 / 3, abs=1e-4) + + +def test_running_runs_are_excluded_from_the_denominator(store): + _run(store, "./a", "converged", "A") + before = store.success_rate()["rate"] + _run(store, "./a", None) + after = store.success_rate() + assert after["rate"] == before == 1.0, "an unfinished run must not move the rate" + assert after["running_excluded"] == 1 + + +def test_safety_block_counts_as_failure(store): + """A rate that forgave safety blocks would reward being stopped by the gate.""" + _run(store, "./a", "converged", "A") + _run(store, "./a", "blocked_safety", None) + out = store.success_rate() + assert out["blocked_safety"] == 1 + assert out["rate"] == pytest.approx(0.5), "blocked_safety must sit in the denominator" + + +def test_target_filter_scopes_the_rate(store): + _run(store, "./a", "converged", "A") + _run(store, "./b", "stopped", "D") + assert store.success_rate("./a")["rate"] == 1.0 + assert store.success_rate("./b")["rate"] == 0.0 + assert store.success_rate()["rate"] == pytest.approx(0.5) + + +def test_unknown_target_is_unmeasured_not_zero(store): + _run(store, "./a", "converged", "A") + out = store.success_rate("./never-run") + assert out["finished"] == 0 and out["rate"] is None