From 4a0cd021cb3831fe541eb98d80a52a66b8bde668 Mon Sep 17 00:00:00 2001 From: Kent Date: Thu, 2 Apr 2026 23:59:33 +0800 Subject: [PATCH 1/9] fix(eval): add fallback for patches that create new files `git apply --reverse --3way` fails on patches that create new model files (data-003~006) because the file doesn't exist in the git index. Add fallback to plain `git apply --reverse` when --3way fails. Found during full 7-scenario eval run where 4/7 with-plugin runs failed with "does not exist in index" error. Co-Authored-By: Claude Opus 4.6 (1M context) --- plugins/recce-dev/skills/recce-eval/scripts/run-case.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/recce-dev/skills/recce-eval/scripts/run-case.sh b/plugins/recce-dev/skills/recce-eval/scripts/run-case.sh index 6b0047d..ef8a8c9 100755 --- a/plugins/recce-dev/skills/recce-eval/scripts/run-case.sh +++ b/plugins/recce-dev/skills/recce-eval/scripts/run-case.sh @@ -194,7 +194,10 @@ if [ "$DRY_RUN" = "false" ] && [ "$SKIP_SETUP" = "false" ]; then # Use --full-refresh so incremental models reprocess ALL rows with # the buggy code — otherwise value_diff sees 0 changed rows because # the stored data was computed before the patch was applied. - git apply --reverse --3way "$PATCH_FILE" + # Try --3way first (handles whitespace mismatches), fall back to + # plain apply for patches that create new files (no base in index). + git apply --reverse --3way "$PATCH_FILE" 2>/dev/null \ + || git apply --reverse "$PATCH_FILE" dbt run --target "$TARGET" --full-refresh --quiet dbt docs generate --target "$TARGET" --quiet 2>/dev/null || true # Run dbt test BEFORE MCP starts (avoids DuckDB lock conflict). From d0b8c16acef7f7a44b52c90adb0b74def180dfb2 Mon Sep 17 00:00:00 2001 From: Kent Date: Fri, 3 Apr 2026 10:12:21 +0800 Subject: [PATCH 2/9] fix(eval): teardown cleanup for files created by reverse patches When a reverse patch creates a new file (data-003~006 scenarios), the previous teardown checked `git ls-files --error-unmatch` which found the file in the index (staged by --3way), then tried `git checkout --` to restore from HEAD where it doesn't exist. File stayed in working tree, causing "already exists" error on the next variant's setup. Fix: check `git show HEAD:"$f"` instead of index. Files that don't exist in HEAD are removed with `rm -f`. Co-Authored-By: Claude Opus 4.6 (1M context) --- plugins/recce-dev/skills/recce-eval/scripts/run-case.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/plugins/recce-dev/skills/recce-eval/scripts/run-case.sh b/plugins/recce-dev/skills/recce-eval/scripts/run-case.sh index ef8a8c9..4806682 100755 --- a/plugins/recce-dev/skills/recce-eval/scripts/run-case.sh +++ b/plugins/recce-dev/skills/recce-eval/scripts/run-case.sh @@ -153,13 +153,12 @@ cleanup() { f="${f#"${f%%[![:space:]]*}"}" # trim leading whitespace f="${f%"${f##*[![:space:]]}"}" # trim trailing whitespace if [ -n "$f" ] && git rev-parse --is-inside-work-tree &>/dev/null 2>&1; then - # For tracked files, unstage then restore from git. - # For untracked files (created by reverse-applying "deleted file" - # patches), remove them. - if git ls-files --error-unmatch "$f" &>/dev/null 2>&1; then - git restore --staged -- "$f" 2>/dev/null || true + git restore --staged -- "$f" 2>/dev/null || true + if git show HEAD:"$f" &>/dev/null 2>&1; then + # File exists in HEAD — restore to HEAD state git checkout -- "$f" 2>/dev/null || true else + # File doesn't exist in HEAD (created by patch) — remove rm -f "$f" 2>/dev/null || true fi fi From 1c31d1ad464b160653e047ff640e2b0e874f59ca Mon Sep 17 00:00:00 2001 From: Kent Date: Fri, 3 Apr 2026 23:59:21 +0800 Subject: [PATCH 3/9] fix(recce): reviewer must follow up on data_impact=potential before classifying MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit impact_analysis returns data_impact="potential" for downstream models (value_diff skipped) with a next_action field pointing to profile_diff. The reviewer was treating "potential" as "probably not impacted" and putting models like customers in not_impacted without investigation. Changes: - data_impact interpretation: "potential" → MUST follow up via next_action - Step 2 priority: next_action items before suggested_deep_dives - Cost limit: ALL next_action items are mandatory, only extra deep dives limited to 3 This fixes D1 customers detection (2/3 runs missed downstream propagation because customers had data_impact=potential and agent skipped investigation). Co-Authored-By: Claude Opus 4.6 (1M context) --- plugins/recce/agents/recce-reviewer.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/plugins/recce/agents/recce-reviewer.md b/plugins/recce/agents/recce-reviewer.md index 8645974..3e654c5 100644 --- a/plugins/recce/agents/recce-reviewer.md +++ b/plugins/recce/agents/recce-reviewer.md @@ -73,13 +73,15 @@ This single call returns: **Interpret `data_impact` for each model:** - `confirmed`: value_diff verified actual data changes — prioritize for root cause investigation - `none`: value_diff verified NO data changes — safe, note briefly in summary -- `null` (or absent): couldn't run value_diff (views, no PK) — unknown, use profile_diff to assess +- `potential`: value_diff was skipped (views, downstream models, no PK) — **MUST follow up** using the model's `next_action` before classifying as impacted or not_impacted. Do NOT put `potential` models in `not_impacted` without investigation. If `impacted_models` is empty: output the "No impact detected" summary (see Section 4) and STOP. ### Step 2 — Follow-up Investigation -For each entry in `suggested_deep_dives`: +**Priority order**: Models with `data_impact: potential` and a `next_action` field take priority over `suggested_deep_dives`. Follow every `next_action` — these are models where impact is unknown and classification depends on your investigation. + +Then, for remaining entries in `suggested_deep_dives`: **2a. Value diff** — If `value_diff` in impact_analysis shows `rows_changed > 0` or the suggestion mentions value changes, call: ``` @@ -95,7 +97,7 @@ This gives distributions (min, max, mean, nulls, distinct counts) that reveal th - If `columns` is null in the suggestion: call `profile_diff` on the whole model (omit `columns` parameter). - On any MCP error: record "tool skipped for {model}: {error reason}" and continue. -- Limit to the first 3 suggested deep dives to control cost. +- Always follow ALL `next_action` items from `potential` models. For additional `suggested_deep_dives` beyond that, limit to 3 to control cost. ### Step 3 — Root Cause Diagnosis From 3b7df4d7082707387445773f0d7817b48242ca7e Mon Sep 17 00:00:00 2001 From: Kent Date: Sat, 4 Apr 2026 16:10:15 +0800 Subject: [PATCH 4/9] =?UTF-8?q?feat(eval):=20add=20run-batch.sh=20?= =?UTF-8?q?=E2=80=94=20consolidate=20Steps=205-7=20into=20single=20backgro?= =?UTF-8?q?und-capable=20script?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit run-batch.sh encapsulates prompt rendering, the interleaved run loop, and deterministic scoring into one command. This replaces ~14 sequential Bash tool calls with a single invocation that can run in background. - Phase 1: renders prompts for all scenarios (v1 inline + v2 template) - Phase 2: interleaved loop (baseline→score→with-plugin→score per run) - Phase 3: writes batch-summary.json with timing and pass/fail counts - SKILL.md: 12 steps → 11 steps (old 5+6+7 → new 5+6), renumbered 8-12 → 7-11 Validated with full 7-scenario eval: 14/14 succeeded, 56 min background, baseline 98.6% / plugin 98.6% det. pass rate, plugin 27% cheaper. Co-Authored-By: Claude Opus 4.6 (1M context) --- plugins/recce-dev/skills/recce-eval/SKILL.md | 158 ++++------- .../skills/recce-eval/scripts/run-batch.sh | 258 ++++++++++++++++++ 2 files changed, 307 insertions(+), 109 deletions(-) create mode 100755 plugins/recce-dev/skills/recce-eval/scripts/run-batch.sh diff --git a/plugins/recce-dev/skills/recce-eval/SKILL.md b/plugins/recce-dev/skills/recce-eval/SKILL.md index fc21c18..2d1fcad 100644 --- a/plugins/recce-dev/skills/recce-eval/SKILL.md +++ b/plugins/recce-dev/skills/recce-eval/SKILL.md @@ -127,7 +127,7 @@ If the user selects nothing (cancels), **STOP**. ## Run Flow -This is the core orchestration — 12 steps that set up scenarios, run headless Claude Code, score results, and produce a report. +This is the core orchestration — 11 steps that set up scenarios, run headless Claude Code, score results, and produce a report. ### Step 1: Read Scenario(s) @@ -175,7 +175,7 @@ yq -o=json '{ }' "/.yaml" ``` -When `prompt_template` is non-null (v2), read the template file and substitute vars in Step 5. When `prompt_inline` is non-null (v1), use it directly as the prompt text. +When `prompt_template` is non-null (v2), `run-batch.sh` uses `render-prompt.py` with template+vars. When `prompt_inline` is non-null (v1), it substitutes runtime variables directly. ### Step 1b: Clone & Bootstrap v2 Project (v2 only) @@ -195,9 +195,9 @@ eval "$(bash ${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/scripts/setup-v2-project.sh echo "PROJECT_DIR=$PROJECT_DIR" ``` -Record `PROJECT_DIR` — pass it as `--project-dir "$PROJECT_DIR"` to all `run-case.sh` invocations in Step 7. +Record `PROJECT_DIR` — pass it as `--project-dir "$PROJECT_DIR"` to `run-batch.sh` in Step 6. -**Cleanup**: At the very end of the Run Flow (after Step 12), remove the temp project: +**Cleanup**: At the very end of the Run Flow (after Step 11), remove the temp project: ```bash if [ -n "$WORK_DIR" ] && [[ "$WORK_DIR" == "${TMPDIR:-/tmp}"* ]]; then @@ -263,23 +263,7 @@ echo "BATCH_DIR=$BATCH_DIR" Record `EVAL_ID` and `BATCH_DIR` for later steps. `BATCH_DIR` is always absolute and anchored to the invoking CWD, so eval output survives v2 temp project cleanup. -### Step 5: Prepare Prompt - -Build the prompt text for each scenario, then write to a temp file. - -**v2 (template+vars):** Read the template file from `prompt_template` (relative to `${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/`), then substitute `{variables}` with values from `prompt_vars` and runtime values (`{target}`, `{adapter_description}`). - -**v1 (inline prompt):** Use the `prompt_inline` string directly, substituting only runtime values (`{target}`, `{adapter_description}`). - -```bash -PROMPT_FILE="/tmp/recce-eval-prompt-${EVAL_ID}-${SCENARIO_ID}.txt" -cat > "$PROMPT_FILE" << 'PROMPT_EOF' - -PROMPT_EOF -echo "PROMPT_FILE=$PROMPT_FILE" -``` - -### Step 6: Generate Eval MCP Config +### Step 5: Generate Eval MCP Config Create a temporary MCP config JSON using **stdio** transport for Recce MCP. This avoids DuckDB lock conflicts — claude spawns the MCP server as a child process after run-case.sh setup completes, so `dbt run` in setup never competes for the database lock. @@ -307,98 +291,53 @@ echo "MCP_CONFIG=/tmp/recce-eval-mcp-config.json" **Why `--strict-mcp-config`**: The `--mcp-config` flag is additive and its merge behavior with plugin `.mcp.json` for same-name keys is undocumented. Using `--strict-mcp-config` guarantees the eval config is the sole MCP source. -### Step 7: Interleaved Run Loop +### Step 6: Run Eval Batch -Set `NO_BARE` based on whether the user passed `--no-bare`: -- If `--no-bare` was passed: `NO_BARE=true` (passes `--no-bare --no-clean-profile` to `run-case.sh`, uses OAuth auth) -- Otherwise: `NO_BARE=""` (default `--bare` mode, requires `ANTHROPIC_API_KEY`) +Run all scenarios with both variants using `run-batch.sh`. This script encapsulates prompt rendering, the interleaved run loop, and deterministic scoring into a single background-capable command. -Run each scenario with both variants in interleaved order. For N runs, the execution order is: baseline run1 → with-plugin run1 → baseline run2 → with-plugin run2 → ... This reduces systematic bias from cache warming or temporal effects. +Build a comma-separated list of absolute scenario file paths from the scenarios parsed in Step 1. Determine the `ADAPTER_DESC` string from the adapter detected in Step 2: -For each run number (1 to N), for each variant (`baseline` first, then `with-plugin`): +| Adapter | `ADAPTER_DESC` | +|---------|---------------| +| duckdb | `DuckDB (local file database, target: $TARGET)` | +| snowflake | `Snowflake (cloud data warehouse, target: $TARGET)` | ```bash -# Create scenario output dir -mkdir -p "$BATCH_DIR/$SCENARIO_ID" - -# ---- Baseline variant ---- -# --bare is default: no memory, no CLAUDE.md, pure prompt-driven evaluation -# When user passes --no-bare: add --no-bare --no-clean-profile (uses OAuth, no API key needed) -bash ${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/scripts/run-case.sh \ - --id "$SCENARIO_ID" \ - --case-type "$CASE_TYPE" \ - --variant baseline \ - --prompt-file "$PROMPT_FILE" \ - --setup-strategy "$SETUP_STRATEGY" \ - --patch-file "${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/$PATCH_FILE" \ - --restore-files "$RESTORE_FILES" \ +bash ${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/scripts/run-batch.sh \ + --scenarios "$SCENARIO_LIST" \ + --batch-dir "$BATCH_DIR" \ + --eval-id "$EVAL_ID" \ + --skill-dir "${CLAUDE_PLUGIN_ROOT}/skills/recce-eval" \ + --recce-plugin "$RECCE_PLUGIN_ROOT" \ --target "$TARGET" \ - --max-budget-usd "$MAX_BUDGET" \ - --output-dir "$BATCH_DIR/$SCENARIO_ID" \ - --run-number "$RUN_NUM" \ - ${NO_BARE:+--no-bare --no-clean-profile} \ - ${PROJECT_DIR:+--project-dir "$PROJECT_DIR"} -``` - -Parse the KEY=VALUE output from `run-case.sh`. Record `OUTPUT_FILE`, `JSON_EXTRACTED`, `TOTAL_COST_USD`, `DURATION_MS`. - -Immediately score the baseline run: - -```bash -bash ${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/scripts/score-deterministic.sh \ - --run-file "$BATCH_DIR/$SCENARIO_ID/baseline_run${RUN_NUM}.json" \ - --case-type "$CASE_TYPE" \ - --ground-truth '$GROUND_TRUTH_JSON' -``` - -Then run the with-plugin variant: - -```bash -# ---- With-plugin variant ---- -# --bare is default; --plugin-dir injects the plugin even in bare mode -# When user passes --no-bare: add --no-bare --no-clean-profile (uses OAuth, no API key needed) -bash ${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/scripts/run-case.sh \ - --id "$SCENARIO_ID" \ - --case-type "$CASE_TYPE" \ - --variant with-plugin \ - --prompt-file "$PROMPT_FILE" \ - --setup-strategy "$SETUP_STRATEGY" \ - --patch-file "${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/$PATCH_FILE" \ - --restore-files "$RESTORE_FILES" \ - --target "$TARGET" \ - --max-budget-usd "$MAX_BUDGET" \ - --output-dir "$BATCH_DIR/$SCENARIO_ID" \ - --plugin-dir "$RECCE_PLUGIN_ROOT" \ + --adapter-desc "$ADAPTER_DESC" \ --mcp-config /tmp/recce-eval-mcp-config.json \ - --run-number "$RUN_NUM" \ - ${NO_BARE:+--no-bare --no-clean-profile} \ + -n $N \ + ${MODEL:+--model "$MODEL"} \ + ${NO_BARE:+--no-bare} \ ${PROJECT_DIR:+--project-dir "$PROJECT_DIR"} ``` -Score the with-plugin run: +Where `$SCENARIO_LIST` is a comma-separated list of absolute paths to scenario YAML files (e.g., `${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/scenarios/v2/data-001-double-tax-deduction.yaml,${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/scenarios/v2/data-002-cogs-food-only.yaml,...`). -```bash -bash ${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/scripts/score-deterministic.sh \ - --run-file "$BATCH_DIR/$SCENARIO_ID/with-plugin_run${RUN_NUM}.json" \ - --case-type "$CASE_TYPE" \ - --ground-truth '$GROUND_TRUTH_JSON' -``` +**What `run-batch.sh` does internally:** +1. **Renders prompts** for each scenario (v2: `render-prompt.py` with template+vars; v1: inline with variable substitution) +2. **Runs interleaved loop**: for each run number (1 to N), for each scenario, baseline → score → with-plugin → score. This interleaving reduces systematic bias from cache warming or temporal effects. +3. **Scores each run** immediately with `score-deterministic.sh` +4. **Writes `batch-summary.json`** with run counts, timing, and scenario list -**Important**: The `--ground-truth` value must be a valid JSON string. Extract the `ground_truth` object from the scenario YAML and pass it as a single-quoted JSON string. Example: - -```bash ---ground-truth '{"issue_found":true,"root_cause_keywords":["null","left join","coalesce"],"impacted_models":["orders","orders_daily_summary"],"not_impacted_models":["customers","customer_segments","customer_order_pattern"],"affected_row_count":1584,"all_tests_pass":true}' -``` +**Output files** (all in `$BATCH_DIR`): +- `/baseline_run.json` — per-run JSONs with deterministic scores merged in +- `/with-plugin_run.json` — per-run JSONs with deterministic scores merged in +- `batch-summary.json` — machine-readable batch metadata (succeeded/failed counts, duration, scenario list) -**Handling setup.strategy**: When calling `run-case.sh`: -- If `setup.strategy` is `git_patch`, pass `--patch-file` pointing to `${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/` and `--restore-files` as a comma-separated list from `teardown.restore_files`. -- If `setup.strategy` is `none`, pass `--setup-strategy none`. Omit `--patch-file` and `--restore-files`. +**Running in background**: This command can be run via the Bash tool's `run_in_background` parameter for long batches. When complete, proceed to Step 7. -**Error handling**: If `run-case.sh` fails (non-zero exit), log the error and continue to the next run. The teardown trap inside `run-case.sh` handles file restoration automatically. Do not add separate teardown calls here. +**Error handling**: If a single `run-case.sh` invocation fails, the batch continues to the next run. Failed runs are counted in `batch-summary.json`. The teardown trap inside `run-case.sh` handles file restoration automatically. -Report progress to the user after each run completes: "Run {N} {variant} complete: cost=${cost}, duration=${duration}s, json_extracted={yes/no}". +**Isolation mode**: `--bare` is the default (both variants get identical isolation). `--no-bare` uses OAuth auth with no API key needed. See Isolation Modes section for details. -### Step 8: Dispatch LLM Judge +### Step 7: Dispatch LLM Judge Use the Agent tool to dispatch `recce-dev:eval-judge` with a prompt that includes all the information the judge needs. Group runs by scenario so the judge can compare variants: @@ -426,7 +365,7 @@ If running multiple scenarios, dispatch the judge once per scenario (not once pe **Error handling**: If the judge agent fails or returns invalid JSON, continue without judge scores. The report will note "LLM judge: unavailable" for affected runs. -### Step 9: Merge Judge Scores +### Step 8: Merge Judge Scores Parse the judge's JSON output. For each run entry in the judge's `runs` array, read the corresponding per-run JSON file and merge `scores.llm_judge` into it: @@ -453,7 +392,7 @@ The judge returns scores per run in the format: Write `comparison_notes` to each run's `scores.llm_judge.comparison_notes` as well. -### Step 10: Write meta.json +### Step 9: Write meta.json Write batch metadata to the batch directory: @@ -476,7 +415,7 @@ EOF Where `$SCENARIOS_JSON_ARRAY` is a JSON array of scenario IDs (e.g., `["data-001-double-tax-deduction", "data-002-cogs-food-only"]`), and `$CLAUDE_MODEL` is from `--model` flag or the current session's model. -### Step 11: Generate Report +### Step 10: Generate Report Read all per-run JSONs in the batch directory (now containing both deterministic and judge scores). Follow the structure defined in `${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/references/report-template.md`. @@ -498,7 +437,7 @@ The report includes: 4. **Detailed Scores** with per-run deterministic checks and judge scores 5. **Cross-Eval Comparison** with historical deltas (if available) -### Step 12: Update History and Print Summary +### Step 11: Update History and Print Summary Append a summary entry to `.claude/recce-eval/history.json`: @@ -603,27 +542,27 @@ bash run-case.sh --id ch3-phantom-filter --variant with-plugin \ ## Common Mistakes -- **Shell variables do not persist**: Each Bash tool invocation starts a fresh shell. Re-derive `EVAL_ID`, `BATCH_DIR`, `TARGET`, `ADAPTER`, `RECCE_PLUGIN_ROOT`, and other state in every Bash block that needs them. Do not assume a previous Bash call's variables are available. +- **Shell variables do not persist**: Each Bash tool invocation starts a fresh shell. Re-derive `EVAL_ID`, `BATCH_DIR`, `TARGET`, `ADAPTER`, `RECCE_PLUGIN_ROOT`, and other state in every Bash block that needs them. Do not assume a previous Bash call's variables are available. Note: `run-batch.sh` eliminates this problem for the run loop (Step 6), but Steps 1-4 and 7-11 still run as separate Bash calls. - **Forgetting `eval`**: Running `bash resolve-recce-root.sh` without `eval "$(...)"` does not set `RECCE_PLUGIN_ROOT` in the current shell. - **Platform-specific `md5`**: macOS uses `md5`, Linux uses `md5sum`. The eval scripts handle both — do not simplify to one. -- **MCP config uses `--strict-mcp-config`**: The eval config must be the sole MCP source. `run-case.sh` passes `--strict-mcp-config --mcp-config` so the eval port is guaranteed. The eval config in Step 6 must include both `recce` (eval port) and `recce-docs` (from `$RECCE_PLUGIN_ROOT`). +- **MCP config uses `--strict-mcp-config`**: The eval config must be the sole MCP source. `run-case.sh` passes `--strict-mcp-config --mcp-config` so the eval port is guaranteed. The eval config in Step 5 must include both `recce` (eval port) and `recce-docs` (from `$RECCE_PLUGIN_ROOT`). - **`--mcp-config` is variadic**: `--mcp-config ` consumes subsequent positional arguments. The `--` separator before the prompt in `run-case.sh` prevents the prompt from being parsed as a config argument. Do not remove it. -- **Interleaved order matters**: Run baseline then with-plugin for the same run number before moving to the next run number. Do not group all baselines then all with-plugins — this introduces systematic bias. +- **Interleaved order matters**: `run-batch.sh` handles this automatically — baseline then with-plugin for each run number. If running manually without `run-batch.sh`, do not group all baselines then all with-plugins — this introduces systematic bias. -- **Teardown is trap-based in run-case.sh**: The script restores files even if `claude -p` fails. Do not add separate teardown calls in the SKILL.md orchestration. +- **Teardown is trap-based in run-case.sh**: The script restores files even if `claude -p` fails. Do not add separate teardown calls in the orchestration. -- **Ground truth as JSON string**: When passing `--ground-truth` to `score-deterministic.sh`, the value must be a valid JSON string. Use single quotes around the entire JSON value in bash to prevent shell expansion. +- **Ground truth as JSON string**: `run-batch.sh` handles this automatically via `yq -o=json | jq -c`. If running `score-deterministic.sh` manually, the `--ground-truth` value must be a valid JSON string. - **Adapter detection uses `yq`**: Do not use grep to parse profiles.yml. The target's adapter type depends on the nested YAML structure which requires proper YAML parsing. - **stdio MCP needs no lifecycle management**: With stdio transport, claude spawns/kills the MCP server automatically. No `start-eval-mcp.sh` / `stop-eval-mcp.sh` calls needed. The `start-eval-mcp.sh` and `stop-eval-mcp.sh` scripts are retained for SSE mode fallback only. -- **Prompt file per scenario**: When running `--all`, create a separate prompt file for each scenario (use `${EVAL_ID}-${SCENARIO_ID}` in the filename) since each scenario has a different prompt. +- **Prompt file per scenario**: `run-batch.sh` handles this automatically (naming: `/tmp/recce-eval-prompt-${EVAL_ID}-${SCENARIO_ID}.txt`). If running manually, create a separate prompt file for each scenario. - **v2 project cleanup**: When `--version v2`, clean up `WORK_DIR` at the end of the Run Flow. Always guard with a `$TMPDIR` prefix check before `rm -rf` to avoid accidental deletion outside temp. @@ -637,9 +576,10 @@ bash run-case.sh --id ch3-phantom-filter --variant with-plugin \ ### Scripts +- **`scripts/run-batch.sh`** — Batch eval runner: renders prompts, runs interleaved loop (baseline→score→with-plugin→score per run per scenario), writes `batch-summary.json`. Background-capable. Encapsulates Steps 5-7 from the original orchestration. - **`scripts/list-scenarios.sh`** — List scenarios for a version. Single `yq eval-all` call. Outputs pipe-delimited rows. -- **`scripts/run-case.sh`** — Atomic runner: setup state, invoke `claude -p`, capture output, teardown, write per-run JSON. Outputs KEY=VALUE lines. -- **`scripts/score-deterministic.sh`** — jq-based scoring against ground truth. Reads and updates per-run JSON in-place. Outputs KEY=VALUE lines. +- **`scripts/run-case.sh`** — Atomic runner: setup state, invoke `claude -p`, capture output, teardown, write per-run JSON. Outputs KEY=VALUE lines. Called by `run-batch.sh`. +- **`scripts/score-deterministic.sh`** — jq-based scoring against ground truth. Reads and updates per-run JSON in-place. Outputs KEY=VALUE lines. Called by `run-batch.sh`. - **`scripts/setup-v2-project.sh`** — Clone a dbt project repo to a temp dir and bootstrap (venv, dbt deps, seed). Used by v2 scenarios only. Outputs `PROJECT_DIR=` and `WORK_DIR=`. - **`scripts/start-eval-mcp.sh`** — Start Recce MCP server on eval-specific port (default 8085). Retained for SSE mode fallback only. - **`scripts/stop-eval-mcp.sh`** — Stop eval MCP server. Retained for SSE mode fallback only. diff --git a/plugins/recce-dev/skills/recce-eval/scripts/run-batch.sh b/plugins/recce-dev/skills/recce-eval/scripts/run-batch.sh new file mode 100755 index 0000000..c1550da --- /dev/null +++ b/plugins/recce-dev/skills/recce-eval/scripts/run-batch.sh @@ -0,0 +1,258 @@ +#!/bin/bash +# run-batch.sh — Batch eval runner: render prompts → interleaved run loop → score +# +# Encapsulates SKILL.md Steps 5-7 into a single background-capable script. +# Caller (SKILL.md) handles Steps 1-4 (parse scenarios, bootstrap project, +# detect adapter, create batch dir) and Steps 8-12 (judge, meta, report). +# +# Usage: +# bash run-batch.sh \ +# --scenarios scenario1.yaml,scenario2.yaml \ +# --batch-dir /path/to/batch \ +# --eval-id 20260404-1530 \ +# --skill-dir /path/to/recce-eval \ +# --recce-plugin /path/to/recce-plugin \ +# --target dev \ +# --adapter-desc "DuckDB (local file database, target: dev)" \ +# [--mcp-config /tmp/mcp.json] \ +# [-n 3] [--model claude-sonnet-4-20250514] [--mode real-world] \ +# [--no-bare] [--project-dir /path/to/project] +# +# Output: +# - Per-run JSONs: $BATCH_DIR//_run.json (via run-case.sh) +# - Deterministic scores merged into per-run JSONs (via score-deterministic.sh) +# - Batch summary: $BATCH_DIR/batch-summary.json +# - Progress lines to stdout +set -euo pipefail + +# ========== Argument Parsing ========== +SCENARIOS="" BATCH_DIR="" EVAL_ID="" SKILL_DIR="" RECCE_PLUGIN="" +TARGET="" ADAPTER_DESC="" MCP_CONFIG="" RUNS=1 MODEL="" MODE="real-world" +NO_BARE="" PROJECT_DIR="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --scenarios) SCENARIOS="$2"; shift 2 ;; + --batch-dir) BATCH_DIR="$2"; shift 2 ;; + --eval-id) EVAL_ID="$2"; shift 2 ;; + --skill-dir) SKILL_DIR="$2"; shift 2 ;; + --recce-plugin) RECCE_PLUGIN="$2"; shift 2 ;; + --target) TARGET="$2"; shift 2 ;; + --adapter-desc) ADAPTER_DESC="$2"; shift 2 ;; + --mcp-config) MCP_CONFIG="$2"; shift 2 ;; + -n|--runs) RUNS="$2"; shift 2 ;; + --model) MODEL="$2"; shift 2 ;; + --mode) MODE="$2"; shift 2 ;; + --no-bare) NO_BARE="true"; shift 1 ;; + --project-dir) PROJECT_DIR="$2"; shift 2 ;; + *) echo "ERROR: Unknown argument: $1" >&2; exit 1 ;; + esac +done + +# ========== Validation ========== +MISSING="" +[ -z "$SCENARIOS" ] && MISSING="$MISSING --scenarios" +[ -z "$BATCH_DIR" ] && MISSING="$MISSING --batch-dir" +[ -z "$EVAL_ID" ] && MISSING="$MISSING --eval-id" +[ -z "$SKILL_DIR" ] && MISSING="$MISSING --skill-dir" +[ -z "$RECCE_PLUGIN" ] && MISSING="$MISSING --recce-plugin" +[ -z "$TARGET" ] && MISSING="$MISSING --target" +[ -z "$ADAPTER_DESC" ] && MISSING="$MISSING --adapter-desc" + +if [ -n "$MISSING" ]; then + echo "ERROR: Missing required arguments:$MISSING" >&2 + exit 1 +fi + +for cmd in yq jq python3; do + if ! command -v "$cmd" &>/dev/null; then + echo "ERROR: Required command not found: $cmd" >&2 + exit 1 + fi +done + +IFS=',' read -ra SCENARIO_FILES <<< "$SCENARIOS" +for f in "${SCENARIO_FILES[@]}"; do + if [ ! -f "$f" ]; then + echo "ERROR: Scenario file not found: $f" >&2 + exit 1 + fi +done + +mkdir -p "$BATCH_DIR" + +# ========== Phase 1: Render Prompts ========== +echo "=== Phase 1: Rendering prompts for ${#SCENARIO_FILES[@]} scenarios ===" + +for SCENARIO_FILE in "${SCENARIO_FILES[@]}"; do + SCENARIO_ID=$(yq -r '.id' "$SCENARIO_FILE") + TEMPLATE=$(yq -r '.prompt.template // ""' "$SCENARIO_FILE") + PROMPT_FILE="/tmp/recce-eval-prompt-${EVAL_ID}-${SCENARIO_ID}.txt" + + if [ -n "$TEMPLATE" ]; then + # v2: template + vars substituted by render-prompt.py + python3 "${SKILL_DIR}/scripts/render-prompt.py" \ + "${SKILL_DIR}/${TEMPLATE}" "$SCENARIO_FILE" \ + --var "adapter_description=${ADAPTER_DESC}" \ + --var "target=${TARGET}" \ + > "$PROMPT_FILE" + else + # v1: inline prompt with runtime variable substitution + PROMPT_TEXT=$(yq -r '.prompt' "$SCENARIO_FILE") + PROMPT_TEXT="${PROMPT_TEXT//\{adapter_description\}/$ADAPTER_DESC}" + PROMPT_TEXT="${PROMPT_TEXT//\{target\}/$TARGET}" + printf '%s' "$PROMPT_TEXT" > "$PROMPT_FILE" + fi + + echo " [ok] $SCENARIO_ID" +done + +# ========== Phase 2: Interleaved Run Loop ========== +# Order: for each run_num → for each scenario → baseline then with-plugin. +# Interleaving reduces systematic bias from cache warming or temporal effects. +TOTAL_RUNS=$(( ${#SCENARIO_FILES[@]} * RUNS * 2 )) +echo "" +echo "=== Phase 2: Running $TOTAL_RUNS cases (${#SCENARIO_FILES[@]} scenarios x $RUNS runs x 2 variants) ===" +echo "" + +RUN_INDEX=0 +SUCCEEDED=0 +FAILED=0 +BATCH_START=$(date +%s) + +for (( run_num=1; run_num<=RUNS; run_num++ )); do + for SCENARIO_FILE in "${SCENARIO_FILES[@]}"; do + # Parse scenario metadata once per scenario per run_num + SCENARIO_ID=$(yq -r '.id' "$SCENARIO_FILE") + CASE_TYPE=$(yq -r '.case_type' "$SCENARIO_FILE") + SETUP_STRATEGY=$(yq -r '.setup.strategy' "$SCENARIO_FILE") + PATCH_REL=$(yq -r '.setup.patch_reverse_file // ""' "$SCENARIO_FILE") + SKIP_CTX=$(yq -r '.setup.skip_context // "false"' "$SCENARIO_FILE") + RESTORE=$(yq -r '.teardown.restore_files // [] | join(",")' "$SCENARIO_FILE") + MAX_BUDGET=$(yq -r '.headless.max_budget_usd' "$SCENARIO_FILE") + GT_JSON=$(yq -o=json '.ground_truth' "$SCENARIO_FILE" | jq -c .) + PROMPT_FILE="/tmp/recce-eval-prompt-${EVAL_ID}-${SCENARIO_ID}.txt" + + mkdir -p "$BATCH_DIR/$SCENARIO_ID" + + for VARIANT in baseline with-plugin; do + RUN_INDEX=$(( RUN_INDEX + 1 )) + CASE_START=$(date +%s) + echo "[${RUN_INDEX}/${TOTAL_RUNS}] ${SCENARIO_ID} ${VARIANT} run${run_num} — starting" + + # Build run-case.sh argument list + RUN_ARGS=( + --id "$SCENARIO_ID" + --case-type "$CASE_TYPE" + --variant "$VARIANT" + --prompt-file "$PROMPT_FILE" + --setup-strategy "$SETUP_STRATEGY" + --target "$TARGET" + --max-budget-usd "$MAX_BUDGET" + --output-dir "$BATCH_DIR/$SCENARIO_ID" + --run-number "$run_num" + ) + + # Isolation mode: --bare (default) or --no-bare + if [ -z "$NO_BARE" ]; then + RUN_ARGS+=(--bare) + else + RUN_ARGS+=(--no-bare --no-clean-profile) + fi + + # Patch file (only for git_patch strategy) + if [ "$SETUP_STRATEGY" = "git_patch" ] && [ -n "$PATCH_REL" ] && [ "$PATCH_REL" != "null" ]; then + RUN_ARGS+=(--patch-file "${SKILL_DIR}/${PATCH_REL}") + fi + [ -n "$RESTORE" ] && RUN_ARGS+=(--restore-files "$RESTORE") + + # With-plugin variant: inject plugin + MCP + if [ "$VARIANT" = "with-plugin" ]; then + RUN_ARGS+=(--plugin-dir "$RECCE_PLUGIN") + [ -n "$MCP_CONFIG" ] && RUN_ARGS+=(--mcp-config "$MCP_CONFIG") + fi + + # Optional flags + [ -n "$MODEL" ] && RUN_ARGS+=(--model "$MODEL") + RUN_ARGS+=(--mode "$MODE") + [ -n "$PROJECT_DIR" ] && RUN_ARGS+=(--project-dir "$PROJECT_DIR") + [ "$SKIP_CTX" = "true" ] && RUN_ARGS+=(--skip-setup-context) + + # Execute run-case.sh + RUN_FILE="$BATCH_DIR/$SCENARIO_ID/${VARIANT}_run${run_num}.json" + RUN_OUTPUT="" + if RUN_OUTPUT=$(bash "${SKILL_DIR}/scripts/run-case.sh" "${RUN_ARGS[@]}" 2>&1); then + # Parse KEY=VALUE output from run-case.sh + COST=$(echo "$RUN_OUTPUT" | grep "^TOTAL_COST_USD=" | cut -d= -f2 || echo "?") + DURATION=$(echo "$RUN_OUTPUT" | grep "^DURATION_MS=" | cut -d= -f2 || echo "0") + JSON_OK=$(echo "$RUN_OUTPUT" | grep "^JSON_EXTRACTED=" | cut -d= -f2 || echo "?") + + # Score immediately after each run + SCORE_OUTPUT="" + if SCORE_OUTPUT=$(bash "${SKILL_DIR}/scripts/score-deterministic.sh" \ + --run-file "$RUN_FILE" \ + --case-type "$CASE_TYPE" \ + --ground-truth "$GT_JSON" 2>&1); then + PASS_RATE=$(echo "$SCORE_OUTPUT" | grep "^PASS_RATE=" | cut -d= -f2 || echo "?") + else + PASS_RATE="score-error" + fi + + DURATION_SEC=$(( ${DURATION:-0} / 1000 )) + echo "[${RUN_INDEX}/${TOTAL_RUNS}] ${SCENARIO_ID} ${VARIANT} run${run_num} — DONE cost=\$${COST} duration=${DURATION_SEC}s json=${JSON_OK} pass_rate=${PASS_RATE}" + SUCCEEDED=$(( SUCCEEDED + 1 )) + else + CASE_END=$(date +%s) + WALL_SEC=$(( CASE_END - CASE_START )) + echo "[${RUN_INDEX}/${TOTAL_RUNS}] ${SCENARIO_ID} ${VARIANT} run${run_num} — FAILED after ${WALL_SEC}s" + echo "$RUN_OUTPUT" | tail -3 | sed 's/^/ > /' + FAILED=$(( FAILED + 1 )) + fi + done + done +done + +# ========== Phase 3: Summary ========== +BATCH_END=$(date +%s) +BATCH_DURATION=$(( BATCH_END - BATCH_START )) +BATCH_MINUTES=$(( BATCH_DURATION / 60 )) +BATCH_SECONDS=$(( BATCH_DURATION % 60 )) + +echo "" +echo "=== BATCH COMPLETE ===" +echo "Eval ID: $EVAL_ID" +echo "Succeeded: $SUCCEEDED / $TOTAL_RUNS" +echo "Failed: $FAILED / $TOTAL_RUNS" +echo "Duration: ${BATCH_MINUTES}m ${BATCH_SECONDS}s" +echo "Output: $BATCH_DIR" + +# Write machine-readable summary for SKILL.md Steps 8-12 +SCENARIO_IDS="[]" +for SCENARIO_FILE in "${SCENARIO_FILES[@]}"; do + SID=$(yq -r '.id' "$SCENARIO_FILE") + SCENARIO_IDS=$(echo "$SCENARIO_IDS" | jq --arg id "$SID" '. + [$id]') +done + +jq -n \ + --arg eval_id "$EVAL_ID" \ + --argjson total "$TOTAL_RUNS" \ + --argjson succeeded "$SUCCEEDED" \ + --argjson failed "$FAILED" \ + --argjson runs "$RUNS" \ + --argjson scenarios "$SCENARIO_IDS" \ + --arg batch_dir "$BATCH_DIR" \ + --arg completed_at "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ + --argjson duration_sec "$BATCH_DURATION" \ + '{ + eval_id: $eval_id, + total_runs: $total, + succeeded: $succeeded, + failed: $failed, + runs_per_scenario: $runs, + scenarios: $scenarios, + batch_dir: $batch_dir, + completed_at: $completed_at, + duration_sec: $duration_sec + }' > "$BATCH_DIR/batch-summary.json" + +echo "Summary: $BATCH_DIR/batch-summary.json" From 935fe23c5d8592973bba6806ec3ec3f22ce95770 Mon Sep 17 00:00:00 2001 From: Kent Date: Sat, 4 Apr 2026 16:38:06 +0800 Subject: [PATCH 5/9] =?UTF-8?q?fix(eval):=20baseline=20must=20not=20get=20?= =?UTF-8?q?free=20prod=20schema=20=E2=80=94=20only=20with-plugin=20builds?= =?UTF-8?q?=20comparison=20target?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously run-case.sh built both prod and dev schemas for ALL variants, giving baseline free access to EXCEPT queries between schemas. This neutralized Recce's core value proposition (comparison infrastructure). Now baseline only gets the dev schema — it must reason from code + single- schema data alone, mirroring a real developer without Recce. With-plugin still builds both schemas so Recce MCP tools (value_diff, profile_diff) can compare before/after data. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../skills/recce-eval/scripts/run-case.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/plugins/recce-dev/skills/recce-eval/scripts/run-case.sh b/plugins/recce-dev/skills/recce-eval/scripts/run-case.sh index 4806682..500853a 100755 --- a/plugins/recce-dev/skills/recce-eval/scripts/run-case.sh +++ b/plugins/recce-dev/skills/recce-eval/scripts/run-case.sh @@ -179,16 +179,17 @@ if [ "$DRY_RUN" = "false" ] && [ "$SKIP_SETUP" = "false" ]; then echo "ERROR: Patch file not found: $PATCH_FILE" >&2 exit 1 fi - # Build base state in a SEPARATE schema so Recce can compare data. - # DuckDB uses one file with multiple schemas. Without a separate base - # schema, value_diff compares dev against itself → 0 differences. - # 1. Build clean state in both dev (current) and prod (base) schemas - # 2. Capture base artifacts from prod - # 3. Apply patch and rebuild dev only - # 4. Recce compares dev (buggy) vs prod (clean) for actual data diffs + # Build base state only for with-plugin variant. + # Baseline gets NO comparison target — it must reason from code + + # single-schema data alone. This mirrors reality: without Recce, + # a developer has no pre-built before/after comparison. + # With-plugin gets both prod (clean) and dev (buggy) schemas so + # Recce MCP tools (value_diff, profile_diff) can compare data. BASE_TARGET="prod" - dbt run --target "$BASE_TARGET" --full-refresh --quiet - dbt docs generate --target-path target-base --target "$BASE_TARGET" --quiet 2>/dev/null || true + if [ "$VARIANT" = "with-plugin" ]; then + dbt run --target "$BASE_TARGET" --full-refresh --quiet + dbt docs generate --target-path target-base --target "$BASE_TARGET" --quiet 2>/dev/null || true + fi # Now apply patch (introduces the bug) and rebuild current state. # Use --full-refresh so incremental models reprocess ALL rows with # the buggy code — otherwise value_diff sees 0 changed rows because From 1c20456cb2406cc49e1d3c2b465350f3f716e432 Mon Sep 17 00:00:00 2001 From: Kent Date: Sun, 5 Apr 2026 09:27:50 +0800 Subject: [PATCH 6/9] =?UTF-8?q?feat(eval):=20add=203=20new=20scenario=20ty?= =?UTF-8?q?pes=20=E2=80=94=20grain=20cascade,=20false=20positive=20trap,?= =?UTF-8?q?=20date=20collapse?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - data-007: supply grain fan-out cascade (modifies order_items.sql GROUP BY, doubles dashboard count columns through orders mart) - data-008: precision no-op false positive trap (wraps cents_to_dollars with round() — zero data change, tests agent's ability to confirm safety) - data-009: date grain month collapse (changes date_trunc day→month in stg_orders, tests partial-scope assessment across DAG) These scenarios target gaps identified in 20260404-1644 eval: existing scenarios are "formula bugs" solvable by code review alone. New scenarios test cascade detection, false positive avoidance, and scope assessment — dimensions where Recce comparison tools add value. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../recce-eval/scenarios/v2/SCENARIOS.md | 101 ++++++++++++++++++ .../v2/data-007-supply-grain-fanout.yaml | 72 +++++++++++++ .../scenarios/v2/data-008-precision-noop.yaml | 69 ++++++++++++ .../v2/data-009-date-grain-month.yaml | 78 ++++++++++++++ .../data-007-supply-grain-fanout.patch | 26 +++++ .../v2/patches/data-008-precision-noop.patch | 16 +++ .../patches/data-009-date-grain-month.patch | 12 +++ 7 files changed, 374 insertions(+) create mode 100644 plugins/recce-dev/skills/recce-eval/scenarios/v2/data-007-supply-grain-fanout.yaml create mode 100644 plugins/recce-dev/skills/recce-eval/scenarios/v2/data-008-precision-noop.yaml create mode 100644 plugins/recce-dev/skills/recce-eval/scenarios/v2/data-009-date-grain-month.yaml create mode 100644 plugins/recce-dev/skills/recce-eval/scenarios/v2/patches/data-007-supply-grain-fanout.patch create mode 100644 plugins/recce-dev/skills/recce-eval/scenarios/v2/patches/data-008-precision-noop.patch create mode 100644 plugins/recce-dev/skills/recce-eval/scenarios/v2/patches/data-009-date-grain-month.patch diff --git a/plugins/recce-dev/skills/recce-eval/scenarios/v2/SCENARIOS.md b/plugins/recce-dev/skills/recce-eval/scenarios/v2/SCENARIOS.md index c926381..e68a39a 100644 --- a/plugins/recce-dev/skills/recce-eval/scenarios/v2/SCENARIOS.md +++ b/plugins/recce-dev/skills/recce-eval/scenarios/v2/SCENARIOS.md @@ -237,6 +237,104 @@ where subtotal > 0 --- +## data-007: Supply Cost Breakdown — Hidden Fan-out Cascade + +**GitHub Issue**: [#4 — Add Supply Cost Analysis and Perishable Inventory Tracking](https://github.com/DataRecce/jaffle-shop-simulator/issues/4) + +**Story**: Purchasing Manager requests perishable vs non-perishable supply cost breakdown per order item. A teammate modifies the `order_supplies_summary` CTE in `order_items.sql` to add `is_perishable_supply` to the GROUP BY. + +**Init state (buggy PR)**: +```sql +-- order_items.sql — order_supplies_summary CTE +select + product_id, + is_perishable_supply, + sum(supply_cost) as supply_cost +from supplies +group by 1, 2 +``` + +**The bug**: Adding `is_perishable_supply` to GROUP BY changes the grain from 1 row/product to 2 rows/product (perishable + non-perishable). The downstream `LEFT JOIN` fans out every order_item into 2 rows. This cascades: +- `order_items`: row count approximately doubles +- `orders.order_cost`: UNCHANGED (sum of split costs = original total) +- `orders.count_order_items`: DOUBLED +- `orders.count_food_items`: DOUBLED (dashboard column!) +- `orders.count_drink_items`: DOUBLED (dashboard column!) +- `orders.order_items_subtotal`: DOUBLED (sum of duplicated product_price) +- `customers`: UNCHANGED (uses order-level columns, not order_items) + +**What we expect the agent to find**: +- Issue found: **yes** — data drift +- Root cause: grain change in order_supplies_summary fans out the join +- Impacted: `order_items`, `orders` +- Not impacted: `stg_orders`, `customers`, `products`, `supplies` +- Dashboard impact: **yes** (count_food_items, count_drink_items doubled) +- Detection requires: **data comparison** + +**Difficulty**: hard — the grain change looks innocent (adding a dimension), but cascades through orders into dashboard columns + +--- + +## data-008: Numeric Precision Refactor — Zero-Change False Positive Trap + +**GitHub Issue**: [#2 — Add Tax Summary Report and Cost Accounting Breakdown](https://github.com/DataRecce/jaffle-shop-simulator/issues/2) + +**Story**: Data Engineer wraps all three `cents_to_dollars()` calls in `stg_orders.sql` with `round(..., 2)` for "defensive precision." + +**Init state (buggy PR)**: +```sql +-- stg_orders.sql +round({{ cents_to_dollars('subtotal') }}, 2) as subtotal, +round({{ cents_to_dollars('tax_paid') }}, 2) as tax_paid, +round({{ cents_to_dollars('order_total') }}, 2) as order_total, +``` + +**The bug**: There is NO bug. The `cents_to_dollars` macro already casts to `numeric(16, 2)`. Applying `round(x, 2)` to a value that is already `numeric(16, 2)` is a complete no-op — zero rows change, zero values change across the entire DAG. + +**What we expect the agent to find**: +- Issue found: **no** — the change is a no-op +- Root cause: round() on already-rounded numeric is redundant +- Impacted: none +- Not impacted: `stg_orders`, `orders`, `customers`, `order_items`, `products` +- Dashboard impact: **no** +- Detection requires: **data comparison** (to confirm zero change, not just code reasoning) + +**Difficulty**: medium — the agent must resist the trap of reporting impact based on DAG reasoning alone (stg_orders is root → everything downstream "could" be affected) + +--- + +## data-009: Date Truncation Change — Month Grain Collapses Daily Timeline + +**GitHub Issue**: [#9 — Optimize Date Granularity for Monthly Reporting](https://github.com/DataRecce/jaffle-shop-simulator/issues/9) + +**Story**: Analytics Engineer changes `date_trunc` in `stg_orders.sql` from `'day'` to `'month'` to "reduce cardinality and improve query performance." + +**Init state (buggy PR)**: +```sql +-- stg_orders.sql +{{ dbt.date_trunc('month','ordered_at') }} as ordered_at +``` + +**The bug**: `ordered_at` loses daily granularity — all orders in the same month collapse to the 1st of the month. This propagates through the entire DAG: +- `orders.ordered_at` — month-level (dashboard column!) +- `orders.customer_order_number` — ROW_NUMBER by month becomes non-deterministic +- `order_items.ordered_at` — month-level +- `customers.first_ordered_at` / `last_ordered_at` — month-level only + +Financial columns (subtotal, tax_paid, order_total) are completely unchanged. Row counts are identical — impact is purely value-level on date columns. + +**What we expect the agent to find**: +- Issue found: **yes** — data drift +- Root cause: date_trunc changed from day to month, collapsing daily granularity +- Impacted: `stg_orders`, `orders`, `order_items`, `customers` +- Not impacted: `products`, `supplies`, `locations` +- Dashboard impact: **yes** (ordered_at is a dashboard column) +- Detection requires: **data comparison** + +**Difficulty**: medium — the agent must correctly scope impact to date columns only and avoid false positives on financial metrics + +--- + ## Summary Matrix | ID | Bug Type | Modified/New | Difficulty | Detection | Dashboard? | Affected Rows | @@ -247,4 +345,7 @@ where subtotal > 0 | data-004 | Count ratio vs cost ratio | New `supply_analysis` | medium | data comparison | no | all rows | | data-005 | current_date on historical data | New `customer_segments` | easy | data comparison | no | all rows | | data-006 | Tax instead of COGS in formula | New `financial_orders` | easy | data comparison | no | all rows | +| data-007 | Grain fan-out cascades to dashboard | Modified `order_items` | hard | data comparison | yes | all rows (doubled) | +| data-008 | No-op precision change (false positive trap) | Modified `stg_orders` | medium | data comparison | no | 0 | +| data-009 | Date grain collapse (day→month) | Modified `stg_orders` | medium | data comparison | yes | 658,657 | | code-001 | Wrong filter column (spec deviation) | Modified `stg_orders` | hard | code review | no | 4,155 | diff --git a/plugins/recce-dev/skills/recce-eval/scenarios/v2/data-007-supply-grain-fanout.yaml b/plugins/recce-dev/skills/recce-eval/scenarios/v2/data-007-supply-grain-fanout.yaml new file mode 100644 index 0000000..5cca3ba --- /dev/null +++ b/plugins/recce-dev/skills/recce-eval/scenarios/v2/data-007-supply-grain-fanout.yaml @@ -0,0 +1,72 @@ +id: data-007-supply-grain-fanout +name: "Supply Cost Breakdown — Hidden Fan-out Cascade" +description: "order_items supply summary adds is_perishable_supply to GROUP BY — grain change fans out join, doubling count columns through orders mart into dashboard" +github_issue: https://github.com/DataRecce/jaffle-shop-simulator/issues/4 +layer: review +difficulty: hard +stakeholder: purchasing +case_type: problem_exists + +story: | + The Purchasing Manager (P2) requested a breakdown of perishable vs non-perishable supply + costs per order item, to better understand spoilage risk in the supply chain. + + A teammate modified the `order_supplies_summary` CTE in `order_items.sql` to include + `is_perishable_supply` in the GROUP BY and SELECT. This splits each product's supply cost + into two rows: one for perishable supplies, one for non-perishable supplies. + + The code change looks reasonable — adding a dimension to an aggregation. But it changes + the grain of `order_supplies_summary` from 1 row per product to 2 rows per product. + The downstream LEFT JOIN in the `joined` CTE now produces 2 rows per order_item (one for + each perishable category). This fan-out cascades: + + - `order_items`: row count approximately doubles + - `orders.order_cost`: UNCHANGED (sum of split costs = original total) + - `orders.count_order_items`: DOUBLED (counts duplicated rows) + - `orders.count_food_items`: DOUBLED (dashboard column!) + - `orders.count_drink_items`: DOUBLED (dashboard column!) + - `orders.order_items_subtotal`: DOUBLED (sum of duplicated product_price) + - `customers`: UNCHANGED (aggregates use order-level columns from stg_orders, not order_items) + + The bug is a classic grain mismatch hidden behind an innocent-looking GROUP BY change. + +environment: + repo: DataRecce/jaffle-shop-simulator + ref: eval-base + adapter: duckdb + +setup: + strategy: git_patch + patch_reverse_file: scenarios/v2/patches/data-007-supply-grain-fanout.patch + skip_context: false + +prompt: + template: prompts/review.md + vars: + stakeholder_name: "Purchasing Manager (P2)" + stakeholder_request: "Add perishable vs non-perishable supply cost breakdown per order item for spoilage risk analysis" + pr_description: "Add is_perishable_supply dimension to order_items supply cost aggregation — splits supply_cost into perishable and non-perishable components" + +headless: + max_budget_usd: 5.00 + output_format: json + +ground_truth: + issue_found: true + issue_type: data_drift + root_cause_keywords: ["grain", "fan-out", "group by", "is_perishable_supply", "duplicate", "count", "order_supplies_summary", "double"] + impacted_models: ["order_items", "orders"] + not_impacted_models: ["stg_orders", "customers", "products", "supplies"] + dashboard_impact: true + detection_requires: data_comparison + +judge_criteria: + - "Agent identifies the grain change in order_supplies_summary (1 row/product → 2 rows/product)" + - "Agent recognizes the fan-out cascade: order_items rows doubled → orders count columns doubled" + - "Agent notes that order_cost (sum of supply_cost) is UNCHANGED despite the fan-out — sum of parts equals the original total" + - "Agent identifies that count_food_items and count_drink_items are DOUBLED — these are Executive Dashboard columns" + - "Agent correctly identifies that customers model is NOT impacted" + - "Agent correctly identifies dashboard_impact as true (count_food_items, count_drink_items)" + +teardown: + restore_files: ["models/marts/order_items.sql"] diff --git a/plugins/recce-dev/skills/recce-eval/scenarios/v2/data-008-precision-noop.yaml b/plugins/recce-dev/skills/recce-eval/scenarios/v2/data-008-precision-noop.yaml new file mode 100644 index 0000000..b2195d8 --- /dev/null +++ b/plugins/recce-dev/skills/recce-eval/scenarios/v2/data-008-precision-noop.yaml @@ -0,0 +1,69 @@ +id: data-008-precision-noop +name: "Numeric Precision Refactor — Zero-Change False Positive Trap" +description: "stg_orders wraps cents_to_dollars with round(x, 2) — macro already outputs numeric(16,2) so data is identical, but code diff touches root staging model" +github_issue: https://github.com/DataRecce/jaffle-shop-simulator/issues/2 +layer: review +difficulty: medium +stakeholder: data-engineering +case_type: no_problem + +story: | + A Data Engineer noticed that the `cents_to_dollars` macro returns `::numeric(16, 2)` but + wanted to make the precision "explicit and defensive" by wrapping all three money columns + in `stg_orders.sql` with `round(..., 2)`. + + The PR description says: "Add explicit rounding to money columns for precision safety — + ensures no floating point drift in downstream aggregations." + + The change modifies `stg_orders.sql`, which is the ROOT staging model feeding into + `orders`, `customers`, and every downstream mart. A code-only reviewer seeing a change + to the root financial staging model would reasonably flag this as high-risk and report + potential impact on all downstream models. + + However, `cents_to_dollars` already casts to `numeric(16, 2)`. Applying `round(x, 2)` to + a value that is already `numeric(16, 2)` is a complete no-op — zero rows change, zero + values change, zero downstream impact. The correct assessment is: no issue found. + + This scenario tests whether the agent can use data comparison to CONFIRM safety rather + than relying on DAG reasoning alone (which would produce false positives). + +environment: + repo: DataRecce/jaffle-shop-simulator + ref: eval-base + adapter: duckdb + +setup: + strategy: git_patch + patch_reverse_file: scenarios/v2/patches/data-008-precision-noop.patch + skip_context: false + +prompt: + template: prompts/review.md + vars: + stakeholder_name: "Data Engineer (P3)" + stakeholder_request: "Add explicit rounding to money columns in stg_orders for precision safety" + pr_description: "Wrap cents_to_dollars output with round(x, 2) in stg_orders — defensive precision for downstream financial aggregations" + +headless: + max_budget_usd: 5.00 + output_format: json + +ground_truth: + issue_found: false + issue_type: no_issue + root_cause_keywords: ["no-op", "round", "numeric", "precision", "already", "identical", "no change", "zero"] + impacted_models: [] + not_impacted_models: ["stg_orders", "orders", "customers", "order_items", "products"] + dashboard_impact: false + detection_requires: data_comparison + +judge_criteria: + - "Agent verifies through DATA comparison that all downstream models have zero value changes" + - "Agent recognizes that round(numeric(16,2), 2) is a no-op — the macro already handles precision" + - "Agent does NOT report false positives on orders, customers, or other downstream models" + - "Agent correctly concludes issue_found: false — no data impact despite code change to root model" + - "Agent correctly identifies dashboard_impact as false" + - "Agent avoids the trap of DAG-based reasoning alone (stg_orders is root → everything must be impacted)" + +teardown: + restore_files: ["models/staging/stg_orders.sql"] diff --git a/plugins/recce-dev/skills/recce-eval/scenarios/v2/data-009-date-grain-month.yaml b/plugins/recce-dev/skills/recce-eval/scenarios/v2/data-009-date-grain-month.yaml new file mode 100644 index 0000000..bfd1796 --- /dev/null +++ b/plugins/recce-dev/skills/recce-eval/scenarios/v2/data-009-date-grain-month.yaml @@ -0,0 +1,78 @@ +id: data-009-date-grain-month +name: "Date Truncation Change — Month Grain Collapses Daily Timeline" +description: "stg_orders changes date_trunc from day to month — ordered_at loses daily granularity across entire DAG, but financial columns are unchanged" +github_issue: https://github.com/DataRecce/jaffle-shop-simulator/issues/9 +layer: review +difficulty: medium +stakeholder: analytics +case_type: problem_exists + +story: | + An Analytics Engineer proposed changing the date truncation in `stg_orders.sql` from + `day` to `month` to "reduce cardinality and improve query performance for monthly + reporting dashboards." + + The PR modifies one line in `stg_orders.sql`: + - Before: `date_trunc('day', ordered_at)` + - After: `date_trunc('month', ordered_at)` + + The change compiles fine and all dbt tests pass. The PR description argues this is a + harmless optimization since "most reports aggregate to monthly anyway." + + However, `stg_orders` is the ROOT staging model for the entire orders pipeline. The + `ordered_at` column propagates through: + - `orders.ordered_at` — now month-level (dashboard column!) + - `orders.customer_order_number` — ROW_NUMBER ordered by month becomes non-deterministic + for orders within the same month + - `order_items.ordered_at` — joined from stg_orders, now month-level + - `customers.first_ordered_at` — now month-level only (loses day precision) + - `customers.last_ordered_at` — now month-level only (loses day precision) + + Critically, financial columns (subtotal, tax_paid, order_total, order_cost) are + COMPLETELY UNCHANGED. The agent must correctly scope the impact to date/time columns + only and avoid false positives on financial metrics. + + Row counts are identical across all models — no rows added or removed. The impact is + purely in value changes to the ordered_at column and its derivatives. + +environment: + repo: DataRecce/jaffle-shop-simulator + ref: eval-base + adapter: duckdb + +setup: + strategy: git_patch + patch_reverse_file: scenarios/v2/patches/data-009-date-grain-month.patch + skip_context: false + +prompt: + template: prompts/review.md + vars: + stakeholder_name: "Analytics Engineer (P3)" + stakeholder_request: "Optimize date granularity in stg_orders from daily to monthly for reporting performance" + pr_description: "Change date_trunc from day to month in stg_orders — reduces ordered_at cardinality for faster monthly aggregations" + +headless: + max_budget_usd: 5.00 + output_format: json + +ground_truth: + issue_found: true + issue_type: data_drift + root_cause_keywords: ["date_trunc", "month", "day", "ordered_at", "granularity", "precision", "cardinality"] + impacted_models: ["stg_orders", "orders", "order_items", "customers"] + not_impacted_models: ["products", "supplies", "locations"] + dashboard_impact: true + detection_requires: data_comparison + +judge_criteria: + - "Agent identifies that ordered_at loses daily granularity — collapses to month-level across the DAG" + - "Agent correctly identifies dashboard_impact as true (ordered_at is a dashboard column)" + - "Agent correctly identifies that financial columns (subtotal, tax_paid, order_total) are UNCHANGED" + - "Agent correctly scopes impacted_models to those that use ordered_at: stg_orders, orders, order_items, customers" + - "Agent does NOT falsely report products, supplies, or locations as impacted" + - "Agent notes that customer_order_number becomes non-deterministic for same-month orders" + - "Agent recognizes row counts are unchanged — the impact is value-level, not row-level" + +teardown: + restore_files: ["models/staging/stg_orders.sql"] diff --git a/plugins/recce-dev/skills/recce-eval/scenarios/v2/patches/data-007-supply-grain-fanout.patch b/plugins/recce-dev/skills/recce-eval/scenarios/v2/patches/data-007-supply-grain-fanout.patch new file mode 100644 index 0000000..900c962 --- /dev/null +++ b/plugins/recce-dev/skills/recce-eval/scenarios/v2/patches/data-007-supply-grain-fanout.patch @@ -0,0 +1,26 @@ +diff --git a/models/marts/order_items.sql b/models/marts/order_items.sql +--- a/models/marts/order_items.sql ++++ b/models/marts/order_items.sql +@@ -29,13 +29,12 @@ + + select + product_id, +- is_perishable_supply, + + sum(supply_cost) as supply_cost + + from supplies + +- group by 1, 2 ++ group by 1 + + ), + +@@ -51,7 +50,6 @@ + products.is_food_item, + products.is_drink_item, + +- order_supplies_summary.is_perishable_supply, + order_supplies_summary.supply_cost + + from order_items diff --git a/plugins/recce-dev/skills/recce-eval/scenarios/v2/patches/data-008-precision-noop.patch b/plugins/recce-dev/skills/recce-eval/scenarios/v2/patches/data-008-precision-noop.patch new file mode 100644 index 0000000..acb6c63 --- /dev/null +++ b/plugins/recce-dev/skills/recce-eval/scenarios/v2/patches/data-008-precision-noop.patch @@ -0,0 +1,16 @@ +diff --git a/models/staging/stg_orders.sql b/models/staging/stg_orders.sql +--- a/models/staging/stg_orders.sql ++++ b/models/staging/stg_orders.sql +@@ -19,9 +19,9 @@ + subtotal as subtotal_cents, + tax_paid as tax_paid_cents, + order_total as order_total_cents, +- round({{ cents_to_dollars('subtotal') }}, 2) as subtotal, +- round({{ cents_to_dollars('tax_paid') }}, 2) as tax_paid, +- round({{ cents_to_dollars('order_total') }}, 2) as order_total, ++ {{ cents_to_dollars('subtotal') }} as subtotal, ++ {{ cents_to_dollars('tax_paid') }} as tax_paid, ++ {{ cents_to_dollars('order_total') }} as order_total, + + ---------- timestamps + {{ dbt.date_trunc('day','ordered_at') }} as ordered_at diff --git a/plugins/recce-dev/skills/recce-eval/scenarios/v2/patches/data-009-date-grain-month.patch b/plugins/recce-dev/skills/recce-eval/scenarios/v2/patches/data-009-date-grain-month.patch new file mode 100644 index 0000000..188f689 --- /dev/null +++ b/plugins/recce-dev/skills/recce-eval/scenarios/v2/patches/data-009-date-grain-month.patch @@ -0,0 +1,12 @@ +diff --git a/models/staging/stg_orders.sql b/models/staging/stg_orders.sql +--- a/models/staging/stg_orders.sql ++++ b/models/staging/stg_orders.sql +@@ -24,7 +24,7 @@ + {{ cents_to_dollars('order_total') }} as order_total, + + ---------- timestamps +- {{ dbt.date_trunc('month','ordered_at') }} as ordered_at ++ {{ dbt.date_trunc('day','ordered_at') }} as ordered_at + + from source + From 4661fc6e2eb09ebdb22285ff4b286e264567e1cd Mon Sep 17 00:00:00 2001 From: Kent Date: Sun, 5 Apr 2026 09:28:01 +0800 Subject: [PATCH 7/9] fix(eval): drop stale prod schema before baseline runs in batch mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In interleaved batch execution (baseline → with-plugin per scenario), the with-plugin variant builds a prod schema that persists in the shared DuckDB file. Subsequent baseline runs inherit this stale prod schema, giving them a free comparison target they shouldn't have. Fix: baseline path now drops the prod schema before setup. Verified with re-run: data-008 baseline without prod produced a false positive (stg_orders in impacted_models) that the plugin correctly avoided — confirming the isolation fix reveals real plugin value. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../recce-dev/skills/recce-eval/scripts/run-case.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/plugins/recce-dev/skills/recce-eval/scripts/run-case.sh b/plugins/recce-dev/skills/recce-eval/scripts/run-case.sh index 500853a..673d662 100755 --- a/plugins/recce-dev/skills/recce-eval/scripts/run-case.sh +++ b/plugins/recce-dev/skills/recce-eval/scripts/run-case.sh @@ -186,6 +186,19 @@ if [ "$DRY_RUN" = "false" ] && [ "$SKIP_SETUP" = "false" ]; then # With-plugin gets both prod (clean) and dev (buggy) schemas so # Recce MCP tools (value_diff, profile_diff) can compare data. BASE_TARGET="prod" + if [ "$VARIANT" = "baseline" ]; then + # Drop stale prod schema from prior with-plugin runs. + # In batch mode, interleaved execution (baseline → with-plugin + # per scenario) leaves prod schema in the shared DuckDB file. + # Without this cleanup, baseline gets a free comparison target. + python3 -c " +import os, duckdb +db_path = os.environ.get('JAFFLE_SHOP_DB_PATH', 'data/jaffel-shop.duckdb') +db = duckdb.connect(db_path) +db.execute('DROP SCHEMA IF EXISTS prod CASCADE') +db.close() +" 2>/dev/null || true + fi if [ "$VARIANT" = "with-plugin" ]; then dbt run --target "$BASE_TARGET" --full-refresh --quiet dbt docs generate --target-path target-base --target "$BASE_TARGET" --quiet 2>/dev/null || true From 4cc7621deaf11e6ccf5f894b9081b4cebf43738a Mon Sep 17 00:00:00 2001 From: Kent Date: Mon, 6 Apr 2026 00:40:44 +0800 Subject: [PATCH 8/9] =?UTF-8?q?fix(eval):=20address=20review=20=E2=80=94?= =?UTF-8?q?=20tighten=20patch=20fallback,=20trim=20scenario=20paths?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - run-case.sh: only fall back to plain git apply when stderr contains "does not exist in index" (new-file patches). Other 3-way failures now surface the real error instead of being silently swallowed. - run-batch.sh: trim whitespace from comma-separated --scenarios entries so "a.yaml, b.yaml" format works correctly. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../skills/recce-eval/scripts/run-batch.sh | 8 ++++++-- .../skills/recce-eval/scripts/run-case.sh | 15 ++++++++++++--- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/plugins/recce-dev/skills/recce-eval/scripts/run-batch.sh b/plugins/recce-dev/skills/recce-eval/scripts/run-batch.sh index c1550da..ae76b06 100755 --- a/plugins/recce-dev/skills/recce-eval/scripts/run-batch.sh +++ b/plugins/recce-dev/skills/recce-eval/scripts/run-batch.sh @@ -71,12 +71,16 @@ for cmd in yq jq python3; do fi done -IFS=',' read -ra SCENARIO_FILES <<< "$SCENARIOS" -for f in "${SCENARIO_FILES[@]}"; do +IFS=',' read -ra SCENARIO_FILES_RAW <<< "$SCENARIOS" +SCENARIO_FILES=() +for f in "${SCENARIO_FILES_RAW[@]}"; do + f="${f#"${f%%[![:space:]]*}"}" # trim leading whitespace + f="${f%"${f##*[![:space:]]}"}" # trim trailing whitespace if [ ! -f "$f" ]; then echo "ERROR: Scenario file not found: $f" >&2 exit 1 fi + SCENARIO_FILES+=("$f") done mkdir -p "$BATCH_DIR" diff --git a/plugins/recce-dev/skills/recce-eval/scripts/run-case.sh b/plugins/recce-dev/skills/recce-eval/scripts/run-case.sh index 673d662..9aeab4d 100755 --- a/plugins/recce-dev/skills/recce-eval/scripts/run-case.sh +++ b/plugins/recce-dev/skills/recce-eval/scripts/run-case.sh @@ -208,9 +208,18 @@ db.close() # the buggy code — otherwise value_diff sees 0 changed rows because # the stored data was computed before the patch was applied. # Try --3way first (handles whitespace mismatches), fall back to - # plain apply for patches that create new files (no base in index). - git apply --reverse --3way "$PATCH_FILE" 2>/dev/null \ - || git apply --reverse "$PATCH_FILE" + # plain apply only for patches that create new files (no base in index). + GIT_APPLY_STDERR=$(mktemp) + if git apply --reverse --3way "$PATCH_FILE" 2>"$GIT_APPLY_STDERR"; then + rm -f "$GIT_APPLY_STDERR" + elif grep -q "does not exist in index" "$GIT_APPLY_STDERR"; then + rm -f "$GIT_APPLY_STDERR" + git apply --reverse "$PATCH_FILE" + else + cat "$GIT_APPLY_STDERR" >&2 + rm -f "$GIT_APPLY_STDERR" + exit 1 + fi dbt run --target "$TARGET" --full-refresh --quiet dbt docs generate --target "$TARGET" --quiet 2>/dev/null || true # Run dbt test BEFORE MCP starts (avoids DuckDB lock conflict). From 551dc171331954dc41a2776bae729cf05248e5de Mon Sep 17 00:00:00 2001 From: Kent Date: Fri, 22 May 2026 15:07:12 +0800 Subject: [PATCH 9/9] refactor(recce-eval): v0.2.0 SKILL.md consolidation + script cleanup - Refactor SKILL.md for v0.2.0: condense dependencies section, simplify setup prose - Add argument-hint field for CLI discoverability - Remove legacy start-eval-mcp.sh and stop-eval-mcp.sh scripts - Consolidate MCP server lifecycle into unified scripts (script refactoring tracked separately) Passed kc-plugin-forge Phase 1 validation. Co-Authored-By: Claude Haiku 4.5 --- plugins/recce-dev/skills/recce-eval/SKILL.md | 589 ++++-------------- .../recce-eval/scripts/start-eval-mcp.sh | 107 ---- .../recce-eval/scripts/stop-eval-mcp.sh | 25 - 3 files changed, 124 insertions(+), 597 deletions(-) delete mode 100755 plugins/recce-dev/skills/recce-eval/scripts/start-eval-mcp.sh delete mode 100755 plugins/recce-dev/skills/recce-eval/scripts/stop-eval-mcp.sh diff --git a/plugins/recce-dev/skills/recce-eval/SKILL.md b/plugins/recce-dev/skills/recce-eval/SKILL.md index 2d1fcad..397e9e7 100644 --- a/plugins/recce-dev/skills/recce-eval/SKILL.md +++ b/plugins/recce-dev/skills/recce-eval/SKILL.md @@ -7,300 +7,114 @@ description: > "list eval cases", "show eval history", "run eval case", or wants to measure the Recce Review Agent's effectiveness compared to pure Claude Code without the plugin. -version: 0.1.0 +argument-hint: "[run|score|report|list|history] [--case ] [--all] [-n N] [--version v1|v2]" +version: 0.2.0 --- # /recce-eval — Evaluate Recce Plugin Effectiveness Measure the Recce Review Agent's impact by running headless Claude Code sessions with and without the Recce plugin, then scoring results against known ground truth. -**Relationship to mcp-e2e-validate:** The `mcp-e2e-validate` skill tests whether the plugin *mechanism* works (hooks fire, MCP responds). This skill tests whether the plugin provides *value* (better accuracy, fewer false positives). Run `mcp-e2e-validate` first to confirm plumbing works, then `recce-eval` to measure how much it helps. +**Relationship to mcp-e2e-validate:** `mcp-e2e-validate` tests whether the plugin *mechanism* works (hooks fire, MCP responds). This skill tests whether the plugin provides *value* (accuracy, false positives). Run `mcp-e2e-validate` first, then `recce-eval`. --- ## Dependencies -Eval scripts require: - -- **yq** — YAML processor ([mikefarah/yq](https://github.com/mikefarah/yq)). Install: `brew install yq` -- **jq** — JSON processor. Install: `brew install jq` -- **git** — required for v2 eval flows that clone/manage projects. Install: `brew install git` (or use your OS package manager) -- **Python 3 with venv + pip** — required for v2 eval flows via `setup-v2-project.sh`. Ensure `python3`, `python3 -m venv`, and `pip` are available in your PATH. +- **yq**, **jq** — `brew install yq jq` +- **git** (v2 only — for clone/checkout) +- **Python 3** with venv + pip (v2 only — for `setup-v2-project.sh`) +- **recce** CLI in PATH (for MCP server) +- **claude** CLI in PATH (v2.x with `--plugin-dir`, `--mcp-config`, `--max-budget-usd` support) ## Setup -Read learned patterns before starting: +Read learned patterns and operational notes before starting: ``` Read → ${CLAUDE_PLUGIN_ROOT}/reference/learned-patterns.md +Read → ${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/references/operational-notes.md ``` ## Prerequisites -Before running eval, confirm: - -1. **dbt project with data loaded** — seeds populated, `dbt run` succeeds on the target -2. **Recce installed** — `recce` CLI in PATH (for MCP server) -3. **`target-base/` artifacts exist** — `dbt docs generate --target-path target-base` on the base branch -4. **No other Recce MCP server on eval port** — default 8085 (configurable via `RECCE_EVAL_MCP_PORT`) -5. **Claude Code CLI installed** — `claude` in PATH -6. **Sufficient API budget** — each run costs ~$1-5 depending on scenario complexity +1. dbt project with data loaded — `dbt run` succeeds on the target +2. `target-base/` artifacts exist — `dbt docs generate --target-path target-base` on the base branch +3. Sufficient API budget — each run costs ~$1-5 depending on scenario complexity --- ## Subcommand Routing -Parse user input to determine which flow to execute: +Parse user input to determine flow: -- **`run --case [,,...] [-n N]`** → Run Flow (one or more scenarios by ID) -- **`run --all [-n N]`** → Run Flow (all scenarios) -- **`run --select [-n N]`** → Select Flow → Run Flow (interactive scenario picker) -- **`score `** → Score Flow -- **`report [eval-id]`** → Report Flow -- **`list`** → List Flow (short-circuit) -- **`history`** → History Flow (short-circuit) +| Subcommand | Action | +|-----------|--------| +| `run --case [,...] [-n N]` | Run Flow (specific scenarios) | +| `run --all [-n N]` | Run Flow (all scenarios) | +| `run --select [-n N]` | Select Flow → Run Flow | +| `score ` | Score Flow | +| `report [eval-id]` | Report Flow | +| `list` | List Flow (short-circuit) | +| `history` | History Flow (short-circuit) | -Shared flags (apply to all flows that accept them): +### Shared flags | Flag | Description | Default | |------|-------------|---------| -| `--version` | Scenario version: `v1` or `v2` | `v2` | +| `--version` | `v1` or `v2` | `v2` | | `--target` | dbt target name | `dev-local` (v1), `dev` (v2) | | `--adapter` | Override adapter detection | Auto-detect from profiles.yml | | `--plugin-dir` | Recce plugin path | Auto-resolve via `resolve-recce-root.sh` | -| `--model` | Claude model for headless runs | Inherits from current session | -| `--no-bare` | Disable bare mode — use OAuth auth, no API key needed | `--bare` is ON by default | - -### Version-Based Path Routing - -Based on `--version`, determine the scenario subdirectory and default target: - -- `--version v1`: scenarios live in `skills/recce-eval/scenarios/v1/`, set `DEFAULT_TARGET=dev-local` -- `--version v2` (default): scenarios live in `skills/recce-eval/scenarios/v2/`, set `DEFAULT_TARGET=dev` - -**IMPORTANT**: Throughout this document, all references to the scenarios directory must use the version-appropriate path. Use `scenarios/v1/` for v1 and `scenarios/v2/` for v2 in every scenario path lookup, glob, and `--patch-file` reference. - -The `--target` flag overrides the default if provided. `DEFAULT_TARGET` is used in Step 2 when `--target` is not provided. +| `--model` | Claude model for headless runs | Inherits from session | +| `--no-bare` | Disable bare mode (use OAuth, no API key) | `--bare` is default | ### List Flow (short-circuit) ```bash bash ${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/scripts/list-scenarios.sh --version ``` - -Display results as a table: - -| ID | Name | Case Type | Difficulty | -|----|------|-----------|---------| - -**STOP here.** Do not proceed to the Run Flow. +Display as table: `| ID | Name | Case Type | Difficulty |`. **STOP** — do not proceed to Run Flow. ### History Flow (short-circuit) -Read `.claude/recce-eval/history.json` in the dbt project root: - ```bash cat .claude/recce-eval/history.json 2>/dev/null || echo "NO_HISTORY" ``` +- Missing → "No eval history found. Run `/recce-eval run` first." +- Present → parse JSON array, display as table with Eval ID / Timestamp / Adapter / Scenario / Baseline Det / Plugin Det / Baseline Judge / Plugin Judge. **STOP**. -- If the file is missing or reads `NO_HISTORY`, tell the user: "No eval history found. Run `/recce-eval run` first." -- If present, parse the JSON array and display as a table: - -| Eval ID | Timestamp | Adapter | Scenario | Baseline Det. | Plugin Det. | Baseline Judge | Plugin Judge | -|---------|-----------|---------|----------|--------------|-------------|----------------|--------------| - -**STOP here.** Do not proceed to the Run Flow. - -### Select Flow (interactive picker) - -When `--select` is used, present the user with an interactive scenario picker before entering the Run Flow. - -1. Load all scenario YAML files from the version-appropriate directory (same as List Flow). -2. Use `AskUserQuestion` with `multiSelect: true` to let the user pick scenarios: - - Each option's `label` is the scenario ID - - Each option's `description` is the scenario name and difficulty -3. Parse the selected IDs and proceed to the Run Flow with those scenarios (same as `--case ,,...`). +### Select Flow -If the user selects nothing (cancels), **STOP**. +When `--select` is used: +1. Load scenarios via `list-scenarios.sh` (same as List Flow). +2. Use `AskUserQuestion` with `multiSelect: true` — `label` = scenario ID, `description` = name + difficulty. +3. Proceed to Run Flow with `CASES` set to the comma-joined selected IDs. If user cancels, **STOP**. --- ## Run Flow -This is the core orchestration — 11 steps that set up scenarios, run headless Claude Code, score results, and produce a report. - -### Step 1: Read Scenario(s) - -Use the version-appropriate scenario directory (see Version-Based Path Routing above). - -If `--case ` (single ID): read `/.yaml`. -If `--case ,,...` (comma-separated): read each `/.yaml`. -If `--all`: read all `data-*.yaml` and `code-*.yaml` files in `/` (skip non-scenario files like `eval-config.yaml`). -If `--select`: scenarios were already selected in the Select Flow above. - -Where `` is `${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/scenarios/v1` (v1) or `${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/scenarios/v2` (v2). - -For each scenario file, extract the required fields in a single `yq` call. - -**v2 scenarios** use `prompt.template` + `prompt.vars` (template-based): - -```bash -yq -o=json '{ - "id": .id, - "case_type": .case_type, - "setup_strategy": .setup.strategy, - "patch_file": .setup.patch_reverse_file, - "prompt_template": .prompt.template, - "prompt_vars": .prompt.vars, - "max_budget_usd": .headless.max_budget_usd, - "ground_truth": .ground_truth, - "judge_criteria": .judge_criteria, - "restore_files": .teardown.restore_files -}' "/.yaml" -``` - -**v1 scenarios** use `prompt:` as an inline string (no template/vars): - -```bash -yq -o=json '{ - "id": .id, - "case_type": .case_type, - "setup_strategy": .setup.strategy, - "patch_file": .setup.patch_reverse_file, - "prompt_inline": .prompt, - "max_budget_usd": .headless.max_budget_usd, - "ground_truth": .ground_truth, - "judge_criteria": .judge_criteria, - "restore_files": .teardown.restore_files -}' "/.yaml" -``` - -When `prompt_template` is non-null (v2), `run-batch.sh` uses `render-prompt.py` with template+vars. When `prompt_inline` is non-null (v1), it substitutes runtime variables directly. - -### Step 1b: Clone & Bootstrap v2 Project (v2 only) - -**Skip this step entirely for `--version v1`.** Only execute when `--version v2`. - -v2 scenarios include `environment.repo` and `environment.ref` fields that specify the dbt project to clone. Parse these from the first scenario (all v2 scenarios share the same repo): - -```bash -yq -o=json '{"repo": .environment.repo, "ref": .environment.ref // "main"}' "/.yaml" -``` - -Clone the repo and bootstrap dbt: - -```bash -eval "$(bash ${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/scripts/setup-v2-project.sh \ - --repo "$REPO" --ref "$REF")" -echo "PROJECT_DIR=$PROJECT_DIR" -``` - -Record `PROJECT_DIR` — pass it as `--project-dir "$PROJECT_DIR"` to `run-batch.sh` in Step 6. - -**Cleanup**: At the very end of the Run Flow (after Step 11), remove the temp project: - -```bash -if [ -n "$WORK_DIR" ] && [[ "$WORK_DIR" == "${TMPDIR:-/tmp}"* ]]; then - rm -rf "$WORK_DIR" -fi -``` - -### Step 2: Detect Adapter - -Determine the dbt adapter type from profiles.yml. Use `--adapter` if provided; otherwise auto-detect: - -```bash -# Default target depends on version: dev-local for v1, dev for v2 -TARGET="${USER_TARGET:-${DEFAULT_TARGET:-dev}}" -# Try the requested target first; fall back to the profile's default target -ADAPTER=$(yq " - .. | select(has(\"outputs\")) | - .outputs[\"$TARGET\"].type // - .outputs[.target // \"dev\"].type // - \"unknown\" -" profiles.yml 2>/dev/null | head -1) -ADAPTER="${ADAPTER:-unknown}" -echo "ADAPTER=$ADAPTER" -``` - -Set template variables based on adapter. The `{target}` value comes from `DEFAULT_TARGET` (set by version routing), not from the adapter: - -| Adapter | `{adapter_description}` | -|---------|------------------------| -| duckdb | `DuckDB (local file database, target: {target})` | -| snowflake | `Snowflake (cloud data warehouse, target: {target})` | - -If a custom `--target` was provided, use that value for `{target}` regardless of version defaults. - -### Step 3: Resolve Plugin Dir - -Resolve the sibling `recce` plugin directory. This is needed for the `with-plugin` variant's `--plugin-dir` flag: - -```bash -eval "$(bash ${CLAUDE_PLUGIN_ROOT}/scripts/resolve-recce-root.sh)" -echo "RECCE_PLUGIN_ROOT=$RECCE_PLUGIN_ROOT" -echo "LAYOUT=$LAYOUT" -``` - -If `ERROR=` appears in output, abort with the error message. Cannot run the `with-plugin` variant without a valid plugin directory. - -If the user provided `--plugin-dir`, use that value instead and skip this resolution. +Six steps. Most setup and finalization is encapsulated in helper scripts; the skill orchestrates only the three LLM-driven pieces (judge dispatch, report narrative, user summary). -### Step 4: Create Batch Directory +### Step 1: Prepare Batch -Create a timestamped directory for this eval batch: +One script call resolves scenarios, clones v2 project (if v2), detects adapter, resolves sibling plugin, creates batch dir, and generates MCP config: ```bash -EVAL_ID=$(date +"%Y%m%d-%H%M") -# Always use the invoking CWD (the plugin repo) as the eval output base. -# v2 PROJECT_DIR is a temp dir that gets cleaned up — output must survive cleanup. -EVAL_BASE="$(pwd)" -BATCH_DIR="${EVAL_BASE}/.claude/recce-eval/runs/$EVAL_ID" -mkdir -p "$BATCH_DIR" -echo "EVAL_ID=$EVAL_ID" -echo "BATCH_DIR=$BATCH_DIR" +eval "$(bash ${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/scripts/prepare-batch.sh \ + --version "${VERSION:-v2}" \ + ${CASES:+--cases "$CASES"} \ + ${ALL:+--all} \ + ${TARGET:+--target "$TARGET"} \ + ${PLUGIN_DIR:+--plugin-dir "$PLUGIN_DIR"})" ``` -Record `EVAL_ID` and `BATCH_DIR` for later steps. `BATCH_DIR` is always absolute and anchored to the invoking CWD, so eval output survives v2 temp project cleanup. +After eval, these vars are set in the caller's shell: `EVAL_ID`, `BATCH_DIR`, `VERSION`, `TARGET`, `ADAPTER`, `ADAPTER_DESC`, `SCENARIO_LIST` (comma-sep absolute paths), `SCENARIO_IDS`, `MCP_CONFIG`, `RECCE_PLUGIN_ROOT`, `LAYOUT`, `PROJECT_DIR` (v2 only), `WORK_DIR` (v2 only). -### Step 5: Generate Eval MCP Config +If Selection Flow provided IDs, set `CASES="$SELECTED_IDS"` before calling. Do not combine `--cases` and `--all`. -Create a temporary MCP config JSON using **stdio** transport for Recce MCP. This avoids DuckDB lock conflicts — claude spawns the MCP server as a child process after run-case.sh setup completes, so `dbt run` in setup never competes for the database lock. - -```bash -cat > /tmp/recce-eval-mcp-config.json << EOF -{ - "mcpServers": { - "recce": { - "type": "stdio", - "command": "recce", - "args": ["mcp-server"] - }, - "recce-docs": { - "type": "stdio", - "command": "node", - "args": ["${RECCE_PLUGIN_ROOT}/servers/recce-docs-mcp/dist/cli.js"] - } - } -} -EOF -echo "MCP_CONFIG=/tmp/recce-eval-mcp-config.json" -``` - -**Why stdio, not SSE**: SSE mode (`start-eval-mcp.sh`) keeps a persistent DuckDB read connection that blocks `dbt run`'s exclusive write lock during setup. stdio transport defers MCP startup to claude's process, which runs after setup. No external MCP server lifecycle management needed. - -**Why `--strict-mcp-config`**: The `--mcp-config` flag is additive and its merge behavior with plugin `.mcp.json` for same-name keys is undocumented. Using `--strict-mcp-config` guarantees the eval config is the sole MCP source. - -### Step 6: Run Eval Batch - -Run all scenarios with both variants using `run-batch.sh`. This script encapsulates prompt rendering, the interleaved run loop, and deterministic scoring into a single background-capable command. - -Build a comma-separated list of absolute scenario file paths from the scenarios parsed in Step 1. Determine the `ADAPTER_DESC` string from the adapter detected in Step 2: - -| Adapter | `ADAPTER_DESC` | -|---------|---------------| -| duckdb | `DuckDB (local file database, target: $TARGET)` | -| snowflake | `Snowflake (cloud data warehouse, target: $TARGET)` | +### Step 2: Run Eval Batch ```bash bash ${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/scripts/run-batch.sh \ @@ -311,164 +125,85 @@ bash ${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/scripts/run-batch.sh \ --recce-plugin "$RECCE_PLUGIN_ROOT" \ --target "$TARGET" \ --adapter-desc "$ADAPTER_DESC" \ - --mcp-config /tmp/recce-eval-mcp-config.json \ - -n $N \ + --mcp-config "$MCP_CONFIG" \ + -n "${N:-1}" \ ${MODEL:+--model "$MODEL"} \ ${NO_BARE:+--no-bare} \ ${PROJECT_DIR:+--project-dir "$PROJECT_DIR"} ``` -Where `$SCENARIO_LIST` is a comma-separated list of absolute paths to scenario YAML files (e.g., `${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/scenarios/v2/data-001-double-tax-deduction.yaml,${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/scenarios/v2/data-002-cogs-food-only.yaml,...`). - -**What `run-batch.sh` does internally:** -1. **Renders prompts** for each scenario (v2: `render-prompt.py` with template+vars; v1: inline with variable substitution) -2. **Runs interleaved loop**: for each run number (1 to N), for each scenario, baseline → score → with-plugin → score. This interleaving reduces systematic bias from cache warming or temporal effects. -3. **Scores each run** immediately with `score-deterministic.sh` -4. **Writes `batch-summary.json`** with run counts, timing, and scenario list - -**Output files** (all in `$BATCH_DIR`): -- `/baseline_run.json` — per-run JSONs with deterministic scores merged in -- `/with-plugin_run.json` — per-run JSONs with deterministic scores merged in -- `batch-summary.json` — machine-readable batch metadata (succeeded/failed counts, duration, scenario list) - -**Running in background**: This command can be run via the Bash tool's `run_in_background` parameter for long batches. When complete, proceed to Step 7. - -**Error handling**: If a single `run-case.sh` invocation fails, the batch continues to the next run. Failed runs are counted in `batch-summary.json`. The teardown trap inside `run-case.sh` handles file restoration automatically. - -**Isolation mode**: `--bare` is the default (both variants get identical isolation). `--no-bare` uses OAuth auth with no API key needed. See Isolation Modes section for details. - -### Step 7: Dispatch LLM Judge - -Use the Agent tool to dispatch `recce-dev:eval-judge` with a prompt that includes all the information the judge needs. Group runs by scenario so the judge can compare variants: +`run-batch.sh` handles: prompt rendering (v1 inline, v2 template+vars via `render-prompt.py`), interleaved run loop (for each run number → for each scenario → baseline → with-plugin), per-run JSON output, deterministic scoring merged into each per-run JSON. Can be executed via Bash tool `run_in_background: true` for long batches. -> For scenario `{scenario_id}` (case_type: {case_type}): -> -> Per-run JSON files: -> - {absolute path to baseline_run1.json} -> - {absolute path to with-plugin_run1.json} -> - {absolute path to baseline_run2.json} (if N > 1) -> - {absolute path to with-plugin_run2.json} (if N > 1) -> -> Ground truth: -> ```json -> {ground_truth from scenario YAML} -> ``` -> -> Judge criteria: -> - {criterion 1} -> - {criterion 2} -> - {criterion 3} -> -> Read each file and score according to the eval-judge rubric. +**Output files** in `$BATCH_DIR`: +- `/baseline_run.json` — per-run with deterministic scores +- `/with-plugin_run.json` — per-run with deterministic scores +- `batch-summary.json` — machine-readable batch metadata -If running multiple scenarios, dispatch the judge once per scenario (not once per run) so it can compare variants within the scenario. +### Step 3: Dispatch LLM Judge -**Error handling**: If the judge agent fails or returns invalid JSON, continue without judge scores. The report will note "LLM judge: unavailable" for affected runs. +For each scenario, dispatch `recce-dev:eval-judge` via the Agent tool and save its JSON output to `$BATCH_DIR//judge.json`. -### Step 8: Merge Judge Scores +The dispatch prompt must include: +- **Absolute paths** to all per-run JSONs for that scenario (both variants, all run numbers) +- **Ground truth** extracted from the scenario YAML (`yq -o=json '.ground_truth'`) +- **Judge criteria** from the scenario YAML (`yq -r '.judge_criteria[]'`) +- **Case type** (`problem_exists` or `no_problem`) -Parse the judge's JSON output. For each run entry in the judge's `runs` array, read the corresponding per-run JSON file and merge `scores.llm_judge` into it: +The agent returns a fenced JSON block with `runs[]` + `comparison_notes`. Extract it and save: ```bash -# For each run scored by the judge, merge the scores -jq --argjson judge '' \ - '.scores.llm_judge = $judge' \ - "$RUN_FILE" > "${RUN_FILE}.tmp" && mv "${RUN_FILE}.tmp" "$RUN_FILE" +mkdir -p "$BATCH_DIR/" +echo '' > "$BATCH_DIR//judge.json" ``` -The judge returns scores per run in the format: -```json -{ - "reasoning_chain": {"score": N, "rationale": "..."}, - "evidence_quality": {"score": N, "rationale": "..."}, - "fix_quality": {"score": N, "rationale": "..."}, - "false_positive_discipline": {"score": N, "rationale": "..."}, - "completeness": {"score": N, "rationale": "..."}, - "overall_score": N.N, - "notable_observations": ["..."], - "comparison_notes": "..." -} -``` +**Parallel dispatch**: Dispatching scenarios sequentially is safe but slow. If multiple scenarios are being judged, prefer parallel Agent dispatches in a single tool-use block (one Agent call per scenario) to reduce wall time. -Write `comparison_notes` to each run's `scores.llm_judge.comparison_notes` as well. +**Error handling**: If the judge agent fails or returns invalid JSON, skip saving that scenario's judge.json — `finalize-eval.sh` tolerates missing judge files (history entry shows null for that scenario's judge averages). -### Step 9: Write meta.json +### Step 4: Finalize Batch -Write batch metadata to the batch directory: +One script call merges judge scores into per-run JSONs, writes `meta.json`, appends `history.json`, and updates the `latest` symlink: ```bash -cat > "$BATCH_DIR/meta.json" << EOF -{ - "eval_id": "$EVAL_ID", - "timestamp": "$(date -u +"%Y-%m-%dT%H:%M:%SZ")", - "target": "$TARGET", - "adapter": "$ADAPTER", - "scenarios_run": $SCENARIOS_JSON_ARRAY, - "runs_per_scenario": $N, - "plugin_dir": "$RECCE_PLUGIN_ROOT", - "recce_version": "$(recce --version 2>/dev/null || echo unknown)", - "claude_model": "$CLAUDE_MODEL", - "max_budget_usd_per_run": $MAX_BUDGET -} -EOF -``` - -Where `$SCENARIOS_JSON_ARRAY` is a JSON array of scenario IDs (e.g., `["data-001-double-tax-deduction", "data-002-cogs-food-only"]`), and `$CLAUDE_MODEL` is from `--model` flag or the current session's model. - -### Step 10: Generate Report - -Read all per-run JSONs in the batch directory (now containing both deterministic and judge scores). Follow the structure defined in `${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/references/report-template.md`. - -Read `.claude/recce-eval/history.json` for cross-eval comparison data. If a previous entry exists with the same adapter, compute deltas for the Cross-Eval Comparison section. - -Write `report.md` to the batch directory: - -```bash -# Write the generated report -cat > "$BATCH_DIR/report.md" << 'REPORT_EOF' - -REPORT_EOF +bash ${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/scripts/finalize-eval.sh \ + --batch-dir "$BATCH_DIR" \ + --eval-id "$EVAL_ID" \ + --adapter "$ADAPTER" \ + --target "$TARGET" \ + --claude-model "${MODEL:-inherited}" \ + --recce-plugin-root "$RECCE_PLUGIN_ROOT" \ + --scenarios "$SCENARIO_IDS" \ + --runs-per-scenario "${N:-1}" \ + --max-budget-usd "${MAX_BUDGET:-5}" \ + --eval-base "$(pwd)" ``` -The report includes: -1. **Environment** section with target, adapter, versions, budget -2. **Summary** table with per-scenario per-variant aggregated metrics -3. **Key Findings** with AI-generated analysis of failure patterns and plugin advantages -4. **Detailed Scores** with per-run deterministic checks and judge scores -5. **Cross-Eval Comparison** with historical deltas (if available) +### Step 5: Generate Report -### Step 11: Update History and Print Summary +Read all per-run JSONs in `$BATCH_DIR//*.json` (now containing both deterministic and judge scores). Read `${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/references/report-template.md` for structure. Generate narrative analysis and write to `$BATCH_DIR/report.md`. -Append a summary entry to `.claude/recce-eval/history.json`: +The report should include: +1. **Environment** — target, adapter, versions, budget, isolation mode +2. **Summary table** — per-scenario per-variant metrics (det pass rate, judge avg, cost, turns) +3. **Key Findings** — failure patterns, plugin advantages, false positives avoided +4. **Detailed Scores** — per-run deterministic checks + judge rationale excerpts +5. **Cross-Eval Comparison** — deltas vs. previous runs (read `history.json` for same adapter) -```bash -# Read existing history or start empty array -HISTORY=$(cat .claude/recce-eval/history.json 2>/dev/null || echo "[]") - -# Build new entry and append -NEW_ENTRY='{ - "eval_id": "'"$EVAL_ID"'", - "timestamp": "'"$(date -u +"%Y-%m-%dT%H:%M:%SZ")"'", - "adapter": "'"$ADAPTER"'", - "claude_model": "'"$CLAUDE_MODEL"'", - "summary": { ... per-scenario summary with det_pass_rate and judge_avg ... } -}' - -echo "$HISTORY" | jq --argjson entry "$NEW_ENTRY" '. + [$entry]' > .claude/recce-eval/history.json -``` +### Step 6: Cleanup + User Summary -Update the `latest` symlink: +**v2 only** — remove temp project dir: ```bash -ln -sfn "$EVAL_ID" .claude/recce-eval/runs/latest +if [ -n "${WORK_DIR:-}" ] && [[ "$WORK_DIR" == "${TMPDIR:-/tmp}"* ]]; then + rm -rf "$WORK_DIR" +fi ``` -Print a summary to the user: - +Print to the user: - Total runs completed - Per-scenario comparison table (baseline vs with-plugin) -- Path to the full report: `$BATCH_DIR/report.md` -- Any warnings (judge failures, MCP issues, etc.) +- Path to `$BATCH_DIR/report.md` +- Any warnings (missing judge files, MCP issues) --- @@ -476,22 +211,11 @@ Print a summary to the user: Re-score existing runs without re-running them. Useful after updating scoring logic or ground truth. -1. Read all `*_run*.json` files in the specified directory. -2. For each file, determine `case_type` from the JSON's `case_type` field. -3. Look up the corresponding scenario YAML from the version-appropriate directory (`scenarios/v1/` or `scenarios/v2/`) using the `scenario_id` in the JSON. -4. Extract `ground_truth` from the scenario YAML. -5. Re-run `score-deterministic.sh` on each file: - -```bash -bash ${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/scripts/score-deterministic.sh \ - --run-file "$RUN_FILE" \ - --case-type "$CASE_TYPE" \ - --ground-truth '$GROUND_TRUTH_JSON' -``` - -6. Re-dispatch `recce-dev:eval-judge` on each scenario's runs (group by scenario). -7. Merge judge scores into per-run JSONs. -8. Regenerate `report.md` in the run directory. +1. For each `*_run*.json` in the dir, extract `scenario_id` and `case_type`; look up the scenario YAML under `scenarios/v1/` or `scenarios/v2/`. +2. Re-run `score-deterministic.sh` on each file with the current ground truth. +3. Re-dispatch `eval-judge` per scenario (same as Run Flow Step 3). Save each to `$BATCH_DIR//judge.json`. +4. Run `finalize-eval.sh` with the same args as Run Flow Step 4 to merge + update meta/history. +5. Regenerate `report.md` (same as Run Flow Step 5). --- @@ -499,119 +223,54 @@ bash ${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/scripts/score-deterministic.sh \ Regenerate the report from existing scored runs without re-scoring. -1. If no eval-id provided, resolve the latest batch: - -```bash -LATEST=$(readlink .claude/recce-eval/runs/latest 2>/dev/null || echo "") -``` - -If empty, tell user "No eval runs found." and **STOP**. - +1. If no eval-id provided: `EVAL_ID=$(readlink .claude/recce-eval/runs/latest 2>/dev/null)`. If empty, tell user "No eval runs found." and **STOP**. 2. Set `BATCH_DIR=".claude/recce-eval/runs/$EVAL_ID"`. -3. Read all per-run JSONs in the batch directory. They must already have `scores.deterministic` and optionally `scores.llm_judge`. -4. Read `${CLAUDE_PLUGIN_ROOT}/skills/recce-eval/references/report-template.md` for the report structure. -5. Generate `report.md` following the template. Write to `$BATCH_DIR/report.md`. -6. Print the report to the user. +3. Read all per-run JSONs (must already have `scores.deterministic` and optionally `scores.llm_judge`). +4. Read `references/report-template.md`, generate `report.md`, write to `$BATCH_DIR/report.md`, print to user. --- -## Isolation Modes - -`run-case.sh` supports three isolation modes: - -| Flag | Memory | CLAUDE.md | Plugin Hooks | Auth | Use Case | -|------|--------|-----------|-------------|------|----------| -| `--no-bare` | ✅ | ✅ | ✅ | OAuth | Internal dev testing | -| **`--bare`** (default) | ❌ | ❌ | ❌ | API key | **Recommended: identical isolation for both variants** | -| `--clean-profile` | ❌ | ❌ | ✅ | API key | Deprecated — causes baseline to produce only 1 turn | - -### `--bare` (default, recommended) - -Both variants run with `--bare` for identical isolation. The with-plugin variant adds `--plugin-dir` which injects the Recce plugin even in bare mode — hooks fire, MCP tools are available, but there is no user memory or CLAUDE.md leaking into results. - -```bash -# Baseline: --bare (implicit default) -bash run-case.sh --id ch3-phantom-filter --variant baseline ... - -# With-plugin: --bare + --plugin-dir (plugin injected in bare mode) -bash run-case.sh --id ch3-phantom-filter --variant with-plugin \ - --plugin-dir "$RECCE_PLUGIN_ROOT" --mcp-config /tmp/eval-mcp.json ... -``` - -**Why `--bare` over `--clean-profile`**: `--clean-profile` (HOME override) causes baseline agents to produce only 1 text turn with no structured JSON output, making scoring impossible. `--bare` provides clean isolation while the agent still engages with tools normally. `--bare --plugin-dir` injects the plugin, so with-plugin runs get the full plugin experience. - -## Common Mistakes - -- **Shell variables do not persist**: Each Bash tool invocation starts a fresh shell. Re-derive `EVAL_ID`, `BATCH_DIR`, `TARGET`, `ADAPTER`, `RECCE_PLUGIN_ROOT`, and other state in every Bash block that needs them. Do not assume a previous Bash call's variables are available. Note: `run-batch.sh` eliminates this problem for the run loop (Step 6), but Steps 1-4 and 7-11 still run as separate Bash calls. - -- **Forgetting `eval`**: Running `bash resolve-recce-root.sh` without `eval "$(...)"` does not set `RECCE_PLUGIN_ROOT` in the current shell. - -- **Platform-specific `md5`**: macOS uses `md5`, Linux uses `md5sum`. The eval scripts handle both — do not simplify to one. - -- **MCP config uses `--strict-mcp-config`**: The eval config must be the sole MCP source. `run-case.sh` passes `--strict-mcp-config --mcp-config` so the eval port is guaranteed. The eval config in Step 5 must include both `recce` (eval port) and `recce-docs` (from `$RECCE_PLUGIN_ROOT`). - -- **`--mcp-config` is variadic**: `--mcp-config ` consumes subsequent positional arguments. The `--` separator before the prompt in `run-case.sh` prevents the prompt from being parsed as a config argument. Do not remove it. - -- **Interleaved order matters**: `run-batch.sh` handles this automatically — baseline then with-plugin for each run number. If running manually without `run-batch.sh`, do not group all baselines then all with-plugins — this introduces systematic bias. - -- **Teardown is trap-based in run-case.sh**: The script restores files even if `claude -p` fails. Do not add separate teardown calls in the orchestration. - -- **Ground truth as JSON string**: `run-batch.sh` handles this automatically via `yq -o=json | jq -c`. If running `score-deterministic.sh` manually, the `--ground-truth` value must be a valid JSON string. - -- **Adapter detection uses `yq`**: Do not use grep to parse profiles.yml. The target's adapter type depends on the nested YAML structure which requires proper YAML parsing. - -- **stdio MCP needs no lifecycle management**: With stdio transport, claude spawns/kills the MCP server automatically. No `start-eval-mcp.sh` / `stop-eval-mcp.sh` calls needed. The `start-eval-mcp.sh` and `stop-eval-mcp.sh` scripts are retained for SSE mode fallback only. +## Isolation & Gotchas -- **Prompt file per scenario**: `run-batch.sh` handles this automatically (naming: `/tmp/recce-eval-prompt-${EVAL_ID}-${SCENARIO_ID}.txt`). If running manually, create a separate prompt file for each scenario. +`run-case.sh` defaults to `--bare`. With-plugin variant additionally sets `--plugin-dir`, which injects the Recce plugin into the bare profile — hooks fire, MCP tools are available, but no user memory/CLAUDE.md leak into results. -- **v2 project cleanup**: When `--version v2`, clean up `WORK_DIR` at the end of the Run Flow. Always guard with a `$TMPDIR` prefix check before `rm -rf` to avoid accidental deletion outside temp. +**Top 3 gotchas** (full list in `references/operational-notes.md`): -- **v2 default target is `dev`, not `dev-local`**: The jaffle-shop-simulator profiles.yml uses `dev` as its default target. If `--target` is not provided with `--version v2`, use `dev`. +- **Shell variables don't persist** across Bash tool invocations. `prepare-batch.sh` / `run-batch.sh` / `finalize-eval.sh` each mitigate this per-phase, but the three top-level calls are still three separate shells — re-derive any vars consumed outside those scripts. +- **Always `eval` the script output**: `prepare-batch.sh` emits `printf %q`-escaped KEY=VALUE lines. Use `eval "$(bash ...)"` to import into the caller's shell. +- **Save judge output to `$BATCH_DIR//judge.json`** before calling `finalize-eval.sh`. Missing files are tolerated but expected ones must be in place, otherwise history entries show null judge averages. -- **v2 clone is shared across scenarios**: When running `--all --version v2`, clone the repo ONCE (from the first scenario's `environment.repo` and `environment.ref`) and reuse `PROJECT_DIR` for all scenarios. Do not clone per-scenario. +Full isolation modes table, 18 common mistakes, and MCP transport rationale: see `references/operational-notes.md`. --- ## Additional Resources -### Scripts +### Scripts (`scripts/`) -- **`scripts/run-batch.sh`** — Batch eval runner: renders prompts, runs interleaved loop (baseline→score→with-plugin→score per run per scenario), writes `batch-summary.json`. Background-capable. Encapsulates Steps 5-7 from the original orchestration. -- **`scripts/list-scenarios.sh`** — List scenarios for a version. Single `yq eval-all` call. Outputs pipe-delimited rows. -- **`scripts/run-case.sh`** — Atomic runner: setup state, invoke `claude -p`, capture output, teardown, write per-run JSON. Outputs KEY=VALUE lines. Called by `run-batch.sh`. -- **`scripts/score-deterministic.sh`** — jq-based scoring against ground truth. Reads and updates per-run JSON in-place. Outputs KEY=VALUE lines. Called by `run-batch.sh`. -- **`scripts/setup-v2-project.sh`** — Clone a dbt project repo to a temp dir and bootstrap (venv, dbt deps, seed). Used by v2 scenarios only. Outputs `PROJECT_DIR=` and `WORK_DIR=`. -- **`scripts/start-eval-mcp.sh`** — Start Recce MCP server on eval-specific port (default 8085). Retained for SSE mode fallback only. -- **`scripts/stop-eval-mcp.sh`** — Stop eval MCP server. Retained for SSE mode fallback only. -- **`scripts/resolve-recce-root.sh`** (plugin-level, at `${CLAUDE_PLUGIN_ROOT}/scripts/`) — Locate sibling `recce` plugin across monorepo and cache layouts. +- **`prepare-batch.sh`** — Pre-run orchestration: scenario resolution, v2 clone, adapter detection, plugin dir, batch dir, MCP config. Outputs `eval`-safe KEY=VALUE lines. +- **`run-batch.sh`** — Batch runner: prompt rendering, interleaved run loop, per-run deterministic scoring. Background-capable. +- **`run-case.sh`** — Atomic single-run: setup state, invoke `claude -p`, capture output, teardown, write per-run JSON. Called by `run-batch.sh`. +- **`finalize-eval.sh`** — Post-run orchestration: judge merge, meta.json, history append, latest symlink. +- **`score-deterministic.sh`** — jq-based scoring against ground truth. Called by `run-batch.sh`. +- **`setup-v2-project.sh`** — Clone + bootstrap a dbt project for v2 scenarios. Called by `prepare-batch.sh`. +- **`list-scenarios.sh`** — Pipe-delimited scenario table. Used by List Flow and Select Flow. +- **`render-prompt.py`** — Template + vars substitution for v2 prompts. Called by `run-batch.sh`. +- **`resolve-recce-root.sh`** (plugin-level, at `${CLAUDE_PLUGIN_ROOT}/scripts/`) — Locate sibling `recce` plugin across monorepo and cache layouts. Called by `prepare-batch.sh`. ### Agents -- **`${CLAUDE_PLUGIN_ROOT}/agents/eval-judge.md`** — LLM-as-judge subagent. Scores reasoning quality, evidence quality, fix quality, false positive discipline, and completeness. Dispatched via `recce-dev:eval-judge`. +- **`${CLAUDE_PLUGIN_ROOT}/agents/eval-judge.md`** — LLM judge subagent (sonnet, Read-only). Dispatched as `recce-dev:eval-judge`. Scores 5 dimensions: reasoning chain, evidence quality, fix quality, false positive discipline, completeness. ### References -- **`references/scoring-rubric.md`** — Deterministic scoring rules per case_type and LLM judge dimension definitions. +- **`references/scoring-rubric.md`** — Deterministic scoring rules per case_type + LLM judge dimension definitions. - **`references/report-template.md`** — Report structure guide with placeholder format and generation rules. +- **`references/operational-notes.md`** — Full isolation modes table, MCP transport rationale, 18 common mistakes. ### Scenarios -- **`scenarios/v1/ch1-null-amounts.yaml`** — Case A (problem_exists): broken pipeline with NULL amounts from missing COALESCE. -- **`scenarios/v1/ch1-healthy-audit.yaml`** — Case B (no_problem): healthy pipeline audit that should find no issues. -- **`scenarios/v1/ch2-silent-filter.yaml`** — Case C (problem_exists): WHERE clause silently drops return_pending orders, all tests pass. -- **`scenarios/v1/ch2-amount-misscale.yaml`** — Case D (problem_exists): amount/1000 instead of /100 makes payments 10x too small, all tests pass. -- **`scenarios/v1/ch3-phantom-filter.yaml`** — Case E (problem_exists): WHERE amount > 0 silently drops 2,326 valid $0 transactions, looks like intentional cleanup. -- **`scenarios/v1/ch3-join-shift.yaml`** — Case F (problem_exists): join key typo (customer_id vs order_id) produces plausible but wrong amounts, all tests pass. -- **`scenarios/v1/ch3-count-distinct.yaml`** — Case G (problem_exists): count(*) → count(distinct customer_id) changes metric semantics without changing column name. - -### Patches - -- **`scenarios/v1/patches/ch1-add-coalesce.patch`** — The COALESCE fix. Reverse-applied during setup to create the broken state for ch1-null-amounts. -- **`scenarios/v1/patches/ch2-remove-status-filter.patch`** — Removes the return_pending filter from stg_orders. -- **`scenarios/v1/patches/ch2-fix-amount-scale.patch`** — Fixes amount/1000 → amount/100 in stg_payments. -- **`scenarios/v1/patches/ch3-phantom-filter.patch`** — Removes the WHERE amount > 0 filter from stg_payments. -- **`scenarios/v1/patches/ch3-join-shift.patch`** — Restores correct join key (order_id) in orders.sql. -- **`scenarios/v1/patches/ch3-count-distinct.patch`** — Restores count(*) in orders_daily_summary. +Run `/recce-eval list --version v1` or `--version v2` to enumerate available scenarios. v1 lives in `scenarios/v1/` (7 chapter cases), v2 lives in `scenarios/v2/` (10+ data/code cases on jaffle-shop-simulator). --- diff --git a/plugins/recce-dev/skills/recce-eval/scripts/start-eval-mcp.sh b/plugins/recce-dev/skills/recce-eval/scripts/start-eval-mcp.sh deleted file mode 100755 index 4c92a50..0000000 --- a/plugins/recce-dev/skills/recce-eval/scripts/start-eval-mcp.sh +++ /dev/null @@ -1,107 +0,0 @@ -#!/bin/bash -# Start Recce MCP Server for eval with isolated port and PID namespace -# Does NOT delegate to start-mcp.sh — manages its own PID file. -set -euo pipefail - -# ========== Port Resolution ========== -EVAL_PORT="${RECCE_EVAL_MCP_PORT:-8085}" - -# ========== Project .env Loading ========== -if [ -f ".env" ]; then - set -a - source .env - set +a -fi - -# ========== Eval-scoped PID / Log Files ========== -EVAL_HASH=$(printf '%s-eval' "$PWD" | md5 2>/dev/null | cut -c1-8 \ - || printf '%s-eval' "$PWD" | md5sum | cut -c1-8) -PID_FILE="/tmp/recce-mcp-${EVAL_HASH}.pid" -LOG_FILE="/tmp/recce-mcp-${EVAL_HASH}.log" - -# ========== Venv Auto-Detection ========== -# Always prefer local venv over global tools — global dbt/recce may be -# dbt Cloud CLI or a different version, incompatible with dbt-core projects. -for VENV_DIR in venv .venv; do - if [ -f "$VENV_DIR/bin/activate" ]; then - # shellcheck disable=SC1091 - source "$VENV_DIR/bin/activate" - break - fi -done - -# ========== Prerequisite Checks ========== -if [ ! -f "dbt_project.yml" ]; then - echo "ERROR=NOT_DBT_PROJECT" - echo "MESSAGE=Current directory is not a dbt project" - exit 1 -fi - -if [ ! -f "target/manifest.json" ]; then - echo "ERROR=MISSING_TARGET_ARTIFACTS" - echo "MESSAGE=Missing target/manifest.json" - echo "FIX=Run: dbt build" - exit 1 -fi - -if ! command -v recce &>/dev/null; then - echo "ERROR=RECCE_NOT_INSTALLED" - echo "MESSAGE=Recce is not installed" - echo "FIX=Run: pip install recce" - exit 1 -fi - -# ========== Check if Already Running ========== -if [ -f "$PID_FILE" ]; then - OLD_PID=$(cat "$PID_FILE") - if ps -p "$OLD_PID" > /dev/null 2>&1; then - echo "STATUS=ALREADY_RUNNING" - echo "PORT=$EVAL_PORT" - echo "PID=$OLD_PID" - echo "URL=http://localhost:$EVAL_PORT/sse" - exit 0 - else - rm "$PID_FILE" - fi -fi - -# ========== Check Port Availability ========== -if lsof -i :"$EVAL_PORT" > /dev/null 2>&1; then - echo "ERROR=PORT_IN_USE" - echo "MESSAGE=Eval port $EVAL_PORT is already in use" - echo "FIX=Set RECCE_EVAL_MCP_PORT or stop the process using port $EVAL_PORT" - exit 1 -fi - -# ========== Start MCP Server ========== -nohup recce mcp-server --sse --port "$EVAL_PORT" > "$LOG_FILE" 2>&1 & -MCP_PID=$! -echo "$MCP_PID" > "$PID_FILE" - -echo "STARTING=true" -echo "PORT=$EVAL_PORT" -echo "PID=$MCP_PID" -echo "LOG_FILE=$LOG_FILE" - -# Wait for startup (max 15 seconds) -for i in {1..15}; do - sleep 1 - if ! ps -p "$MCP_PID" > /dev/null 2>&1; then - echo "ERROR=STARTUP_FAILED" - echo "MESSAGE=Recce MCP Server failed to start" - echo "LOG_FILE=$LOG_FILE" - rm -f "$PID_FILE" - exit 1 - fi - HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 2 "http://localhost:$EVAL_PORT/sse" 2>/dev/null) - if [ "$HTTP_CODE" = "200" ]; then - echo "STATUS=STARTED" - echo "URL=http://localhost:$EVAL_PORT/sse" - exit 0 - fi -done - -echo "ERROR=STARTUP_TIMEOUT" -echo "MESSAGE=Recce MCP Server startup timed out (15 seconds)" -echo "LOG_FILE=$LOG_FILE" -exit 1 diff --git a/plugins/recce-dev/skills/recce-eval/scripts/stop-eval-mcp.sh b/plugins/recce-dev/skills/recce-eval/scripts/stop-eval-mcp.sh deleted file mode 100755 index de2720e..0000000 --- a/plugins/recce-dev/skills/recce-eval/scripts/stop-eval-mcp.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# Stop eval Recce MCP Server using eval-scoped PID file -set -euo pipefail - -EVAL_HASH=$(printf '%s-eval' "$PWD" | md5 2>/dev/null | cut -c1-8 \ - || printf '%s-eval' "$PWD" | md5sum | cut -c1-8) -PID_FILE="/tmp/recce-mcp-${EVAL_HASH}.pid" -LOG_FILE="/tmp/recce-mcp-${EVAL_HASH}.log" - -if [ -f "$PID_FILE" ]; then - PID=$(cat "$PID_FILE") - if ps -p "$PID" > /dev/null 2>&1; then - kill "$PID" - rm -f "$PID_FILE" "$LOG_FILE" - echo "STATUS=STOPPED" - echo "MESSAGE=Eval MCP Server stopped (PID: $PID)" - else - rm -f "$PID_FILE" "$LOG_FILE" - echo "STATUS=NOT_RUNNING" - echo "MESSAGE=Eval MCP Server was not running (stale PID file removed)" - fi -else - echo "STATUS=NOT_RUNNING" - echo "MESSAGE=Eval MCP Server is not running (no PID file)" -fi