From 847249c9e5fc94d3cf9a8685c97f084862eaae38 Mon Sep 17 00:00:00 2001 From: even-wei Date: Tue, 12 May 2026 11:35:50 +0800 Subject: [PATCH 1/4] docs(evals): build six PR fixture set via build_fixtures.sh Replaces the previous artifact-committed approach (+377k LOC) with a build-script + gitignore approach. The eval runner runs `build_fixtures.sh` once before each eval run to (re)generate the per-fixture artifacts/ directories from pinned dbt versions against an empty DuckDB. Keep in tree (human-authored, small): - evals/agent-blind-spots/build_fixtures.sh - evals/agent-blind-spots/fixtures/README.md (caveats rolled up) - evals/agent-blind-spots/fixtures//README.md - evals/agent-blind-spots/fixtures//tier-0-baseline.md - evals/agent-blind-spots/fixtures//commits.txt - evals/agent-blind-spots/fixtures//diff.patch - evals/agent-blind-spots/fixtures/pr44-promotion-flags/diff-from-base-to-intermediate.patch Gitignored (regenerable): - evals/agent-blind-spots/.tmp/ - evals/agent-blind-spots/fixtures/*/artifacts/ Fixtures, classes: - pr1-fix-clv (semantic) - pr42-is-closed-filter (row-grain; older base SHA) - pr2-refactor-cte-to-models (refactor; negative control) - pr3-amount-double-to-decimal (type; PR has merge head on top of substantive 6ffc23f) - pr44-promotion-flags (schema-expansion + intermediate row-filter at 23b96ca) - pr46-net-clv-segments (multi-model semantic; stress-test fixture) Build pins (per upstream packages.yml + DRC-3402 capture): - Python 3.11.11, dbt-core 1.11.9, dbt-duckdb 1.10.1, duckdb 1.5.2 - dbt packages: dbt_profiler@0.8.1, dbt_utils@0.9.6, audit_helper@0.11.0 Manifests/catalogs are scrubbed of user_id, invocation_id, root_path. Resolves DRC-3402 Signed-off-by: even-wei --- .gitignore | 4 + evals/agent-blind-spots/README.md | 16 +- evals/agent-blind-spots/build_fixtures.sh | 244 +++++++++++++++++ evals/agent-blind-spots/fixtures/README.md | 78 ++++++ .../fixtures/pr1-fix-clv/README.md | 42 +++ .../fixtures/pr1-fix-clv/commits.txt | 1 + .../fixtures/pr1-fix-clv/diff.patch | 12 + .../fixtures/pr1-fix-clv/tier-0-baseline.md | 41 +++ .../pr2-refactor-cte-to-models/README.md | 41 +++ .../pr2-refactor-cte-to-models/commits.txt | 1 + .../pr2-refactor-cte-to-models/diff.patch | 121 +++++++++ .../tier-0-baseline.md | 41 +++ .../pr3-amount-double-to-decimal/README.md | 61 +++++ .../pr3-amount-double-to-decimal/commits.txt | 2 + .../pr3-amount-double-to-decimal/diff.patch | 13 + .../tier-0-baseline.md | 41 +++ .../fixtures/pr42-is-closed-filter/README.md | 44 ++++ .../pr42-is-closed-filter/commits.txt | 1 + .../fixtures/pr42-is-closed-filter/diff.patch | 38 +++ .../pr42-is-closed-filter/tier-0-baseline.md | 41 +++ .../fixtures/pr44-promotion-flags/README.md | 73 ++++++ .../fixtures/pr44-promotion-flags/commits.txt | 4 + .../diff-from-base-to-intermediate.patch | 47 ++++ .../fixtures/pr44-promotion-flags/diff.patch | 46 ++++ .../pr44-promotion-flags/tier-0-baseline.md | 41 +++ .../fixtures/pr46-net-clv-segments/README.md | 64 +++++ .../pr46-net-clv-segments/commits.txt | 2 + .../fixtures/pr46-net-clv-segments/diff.patch | 246 ++++++++++++++++++ .../pr46-net-clv-segments/tier-0-baseline.md | 41 +++ 29 files changed, 1445 insertions(+), 2 deletions(-) create mode 100755 evals/agent-blind-spots/build_fixtures.sh create mode 100644 evals/agent-blind-spots/fixtures/README.md create mode 100644 evals/agent-blind-spots/fixtures/pr1-fix-clv/README.md create mode 100644 evals/agent-blind-spots/fixtures/pr1-fix-clv/commits.txt create mode 100644 evals/agent-blind-spots/fixtures/pr1-fix-clv/diff.patch create mode 100644 evals/agent-blind-spots/fixtures/pr1-fix-clv/tier-0-baseline.md create mode 100644 evals/agent-blind-spots/fixtures/pr2-refactor-cte-to-models/README.md create mode 100644 evals/agent-blind-spots/fixtures/pr2-refactor-cte-to-models/commits.txt create mode 100644 evals/agent-blind-spots/fixtures/pr2-refactor-cte-to-models/diff.patch create mode 100644 evals/agent-blind-spots/fixtures/pr2-refactor-cte-to-models/tier-0-baseline.md create mode 100644 evals/agent-blind-spots/fixtures/pr3-amount-double-to-decimal/README.md create mode 100644 evals/agent-blind-spots/fixtures/pr3-amount-double-to-decimal/commits.txt create mode 100644 evals/agent-blind-spots/fixtures/pr3-amount-double-to-decimal/diff.patch create mode 100644 evals/agent-blind-spots/fixtures/pr3-amount-double-to-decimal/tier-0-baseline.md create mode 100644 evals/agent-blind-spots/fixtures/pr42-is-closed-filter/README.md create mode 100644 evals/agent-blind-spots/fixtures/pr42-is-closed-filter/commits.txt create mode 100644 evals/agent-blind-spots/fixtures/pr42-is-closed-filter/diff.patch create mode 100644 evals/agent-blind-spots/fixtures/pr42-is-closed-filter/tier-0-baseline.md create mode 100644 evals/agent-blind-spots/fixtures/pr44-promotion-flags/README.md create mode 100644 evals/agent-blind-spots/fixtures/pr44-promotion-flags/commits.txt create mode 100644 evals/agent-blind-spots/fixtures/pr44-promotion-flags/diff-from-base-to-intermediate.patch create mode 100644 evals/agent-blind-spots/fixtures/pr44-promotion-flags/diff.patch create mode 100644 evals/agent-blind-spots/fixtures/pr44-promotion-flags/tier-0-baseline.md create mode 100644 evals/agent-blind-spots/fixtures/pr46-net-clv-segments/README.md create mode 100644 evals/agent-blind-spots/fixtures/pr46-net-clv-segments/commits.txt create mode 100644 evals/agent-blind-spots/fixtures/pr46-net-clv-segments/diff.patch create mode 100644 evals/agent-blind-spots/fixtures/pr46-net-clv-segments/tier-0-baseline.md diff --git a/.gitignore b/.gitignore index ae37427..c0ea6e5 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,7 @@ tmp/ temp/ *.tmp .worktrees/ + +# DRC-3402 eval fixtures — regenerable artifacts (see evals/agent-blind-spots/fixtures/README.md) +evals/agent-blind-spots/.tmp/ +evals/agent-blind-spots/fixtures/*/artifacts/ diff --git a/evals/agent-blind-spots/README.md b/evals/agent-blind-spots/README.md index 439e6fa..863b375 100644 --- a/evals/agent-blind-spots/README.md +++ b/evals/agent-blind-spots/README.md @@ -16,11 +16,15 @@ N=6 PRs is too small for statistics. The eval is a **named-case narrative**, not evals/agent-blind-spots/ ├── README.md ← this file ├── RUBRIC.md ← scoring rules; read before adding or scoring fixtures -├── fixtures/ ← one directory per PR fixture (built in DRC-3402) +├── build_fixtures.sh ← rebuilds the gitignored artifacts/ per fixture (DRC-3402) +├── fixtures/ ← one directory per PR fixture +│ ├── README.md ← fixture-set caveats + build instructions │ └── / │ ├── README.md ← what the PR does + expected verdicts │ ├── tier-0-baseline.md ← frozen agent-only verdict (template in templates/) -│ └── artifacts/ ← manifest snapshots, compiled SQL pre/post, diff +│ ├── commits.txt ← base + head SHAs read by build_fixtures.sh +│ ├── diff.patch ← small source-models diff base..head (committed) +│ └── artifacts/ ← gitignored; produced by build_fixtures.sh ├── templates/ │ ├── tier-0-baseline.md ← per-fixture frozen baseline template │ └── gap-report.md ← gap-report template (target ≤5 entries) @@ -30,6 +34,14 @@ evals/agent-blind-spots/ └── -scoring.md ← per-fixture scoring per RUBRIC.md ``` +Before any eval run, build the gitignored artifacts: + +```bash +cd evals/agent-blind-spots && ./build_fixtures.sh +``` + +See [`fixtures/README.md`](./fixtures/README.md) for the full per-fixture caveats (PR #16 merge head, PR #20 intermediate trap, PR #46 stress test, empty-DuckDB catalog stats, PR #14 older base) and pinned versions. + ## How to run 1. Pick a fixture in `fixtures/`. diff --git a/evals/agent-blind-spots/build_fixtures.sh b/evals/agent-blind-spots/build_fixtures.sh new file mode 100755 index 0000000..87b96c5 --- /dev/null +++ b/evals/agent-blind-spots/build_fixtures.sh @@ -0,0 +1,244 @@ +#!/usr/bin/env bash +# +# build_fixtures.sh — Rebuild the local-dbt-only artifacts for every fixture +# under fixtures//. Idempotent: existing artifacts/ are removed first. +# +# Required: git, uv. Everything else (Python 3.11.11, dbt-core, dbt-duckdb, +# duckdb) is installed into ./.tmp/.venv/ from the pinned versions below. +# +# Output: one "OK " line per fixture. Non-zero exit on any failure. +# +# See fixtures/README.md for the canonical description of the per-fixture +# layout and the gitignored artifact paths. + +set -euo pipefail + +# ---- Pins (DRC-3402) -------------------------------------------------------- +PYTHON_VERSION="3.11.11" +DBT_CORE_VERSION="1.11.9" +DBT_DUCKDB_VERSION="1.10.1" +DUCKDB_VERSION="1.5.2" + +# ---- Paths ------------------------------------------------------------------ +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FIXTURES_DIR="${SCRIPT_DIR}/fixtures" +TMP_DIR="${SCRIPT_DIR}/.tmp" +JSG_DIR="${TMP_DIR}/jaffle_shop_golden" +VENV_DIR="${TMP_DIR}/.venv" +PROFILES_DIR="${TMP_DIR}/profiles" + +JSG_REPO="DataRecce/jaffle_shop_golden" +JSG_URL="https://github.com/${JSG_REPO}.git" + +mkdir -p "${TMP_DIR}" "${PROFILES_DIR}" + +# ---- Step 1: clone / fetch the source repo --------------------------------- +if [[ ! -d "${JSG_DIR}/.git" ]]; then + echo "Cloning ${JSG_REPO} into ${JSG_DIR}..." >&2 + git clone "${JSG_URL}" "${JSG_DIR}" +fi + +# Always fetch PR heads so all fixture SHAs are reachable +echo "Fetching all branches + PR heads from ${JSG_REPO}..." >&2 +git -C "${JSG_DIR}" fetch --quiet origin +git -C "${JSG_DIR}" fetch --quiet origin '+refs/pull/*/head:refs/remotes/origin/pr/*' + +# ---- Step 2: venv with pinned versions ------------------------------------- +if [[ ! -d "${VENV_DIR}" ]]; then + echo "Creating venv with Python ${PYTHON_VERSION}..." >&2 + uv venv --python "${PYTHON_VERSION}" "${VENV_DIR}" +fi + +# uv pip install is idempotent; cheap to run every time. +echo "Installing pinned dbt-duckdb stack into ${VENV_DIR}..." >&2 +VIRTUAL_ENV="${VENV_DIR}" uv pip install --quiet \ + "dbt-core==${DBT_CORE_VERSION}" \ + "dbt-duckdb==${DBT_DUCKDB_VERSION}" \ + "duckdb==${DUCKDB_VERSION}" + +DBT="${VENV_DIR}/bin/dbt" + +# ---- Step 3: DuckDB profile ------------------------------------------------ +# DuckDB path picks a deterministic file under TMP_DIR so the compiled SQL +# references "jaffle_shop_fixture_build" as the database, matching the original +# build. The actual file is gitignored (lives under .tmp/) and gets reused. +cat > "${PROFILES_DIR}/profiles.yml" < "redacted" +# invocation_id -> "redacted" +# root_path -> "" +scrub_json() { + local path="$1" + VIRTUAL_ENV="${VENV_DIR}" "${VENV_DIR}/bin/python" - "${path}" <<'PY' +import json +import sys + +path = sys.argv[1] +with open(path, "r") as f: + data = json.load(f) + +def walk(node): + if isinstance(node, dict): + for k, v in list(node.items()): + if k == "user_id" and isinstance(v, str): + node[k] = "redacted" + elif k == "invocation_id" and isinstance(v, str): + node[k] = "redacted" + elif k == "root_path" and isinstance(v, str): + node[k] = "" + else: + walk(v) + elif isinstance(node, list): + for item in node: + walk(item) + +walk(data) +with open(path, "w") as f: + json.dump(data, f, indent=2, sort_keys=True) +PY +} + +# Compile a single SHA into a temporary target directory and copy the artifacts +# of interest into /. Layout depends on caller (before/after/intermediate). +# +# Args: +# sha commit to check out in jaffle_shop_golden +# manifest_out destination path for manifest.json (file) +# compiled_out destination dir for compiled/ (directory) +# catalog_out destination path for catalog.json (file) +build_at_sha() { + local sha="$1" + local manifest_out="$2" + local compiled_out="$3" + local catalog_out="$4" + + # Detached checkout; abort any local edits from prior runs. + git -C "${JSG_DIR}" reset --quiet --hard + git -C "${JSG_DIR}" clean --quiet -fdx + git -C "${JSG_DIR}" checkout --quiet --detach "${sha}" + + # Swap upstream Snowflake profile for our DuckDB one (kept only inside JSG_DIR). + cp "${PROFILES_DIR}/profiles.yml" "${JSG_DIR}/profiles.yml" + + # Stage seed CSVs so dbt parse resolves seed nodes (dbt doesn't *need* the data + # for parse/compile/docs-generate, but the project expects seeds/ to exist). + mkdir -p "${JSG_DIR}/seeds" + cp "${JSG_DIR}/jaffle-shop-data/"*.csv "${JSG_DIR}/seeds/" + + # dbt deps fetches packages.yml entries into dbt_packages/. + (cd "${JSG_DIR}" && DBT_PROFILES_DIR="${JSG_DIR}" "${DBT}" deps --quiet) + + # Empty target dir so we never silently mix old artifacts. + rm -rf "${JSG_DIR}/target" + + (cd "${JSG_DIR}" && DBT_PROFILES_DIR="${JSG_DIR}" "${DBT}" parse --quiet) + (cd "${JSG_DIR}" && DBT_PROFILES_DIR="${JSG_DIR}" "${DBT}" compile --quiet) + (cd "${JSG_DIR}" && DBT_PROFILES_DIR="${JSG_DIR}" "${DBT}" docs generate --empty-catalog --quiet) + + # Copy outputs to the fixture artifacts/. + mkdir -p "$(dirname "${manifest_out}")" + cp "${JSG_DIR}/target/manifest.json" "${manifest_out}" + scrub_json "${manifest_out}" + + rm -rf "${compiled_out}" + mkdir -p "${compiled_out}" + if [[ -d "${JSG_DIR}/target/compiled" ]]; then + cp -R "${JSG_DIR}/target/compiled/." "${compiled_out}/" + fi + + cp "${JSG_DIR}/target/catalog.json" "${catalog_out}" + scrub_json "${catalog_out}" +} + +# ---- Step 4: build each fixture -------------------------------------------- +build_fixture() { + local slug="$1" + local fdir="${FIXTURES_DIR}/${slug}" + local readme="${fdir}/README.md" + local commits_file="${fdir}/commits.txt" + local artifacts="${fdir}/artifacts" + + if [[ ! -f "${readme}" ]] || [[ ! -f "${commits_file}" ]]; then + echo "FAIL ${slug} (missing README.md or commits.txt)" >&2 + return 1 + fi + + local base_sha + base_sha="$(extract_base_sha "${readme}")" + if [[ -z "${base_sha}" ]]; then + echo "FAIL ${slug} (no Base SHA in README.md)" >&2 + return 1 + fi + + # Head SHA = first non-blank line of commits.txt (already in repo order: newest first). + local head_sha + head_sha="$(awk 'NF>0 {print $1; exit}' "${commits_file}")" + if [[ -z "${head_sha}" ]]; then + echo "FAIL ${slug} (no head SHA in commits.txt)" >&2 + return 1 + fi + + rm -rf "${artifacts}" + mkdir -p "${artifacts}" + + build_at_sha "${base_sha}" \ + "${artifacts}/manifest-before.json" \ + "${artifacts}/compiled-before" \ + "${artifacts}/catalog-before.json" + + build_at_sha "${head_sha}" \ + "${artifacts}/manifest-after.json" \ + "${artifacts}/compiled-after" \ + "${artifacts}/catalog-after.json" + + # PR #20 intermediate snapshot — keyed on the well-known SHA in commits.txt. + if [[ "${slug}" == "pr44-promotion-flags" ]]; then + local intermediate_sha="23b96ca" + # Resolve to a full SHA from commits.txt to avoid ambiguity. + local full_intermediate + full_intermediate="$(awk -v p="${intermediate_sha}" '$1 ~ "^"p { print $1; exit }' "${commits_file}")" + if [[ -z "${full_intermediate}" ]]; then + echo "FAIL ${slug} (commits.txt does not list intermediate ${intermediate_sha})" >&2 + return 1 + fi + local idir="${artifacts}/intermediate-commit-${intermediate_sha}" + mkdir -p "${idir}/compiled" + build_at_sha "${full_intermediate}" \ + "${idir}/manifest.json" \ + "${idir}/compiled" \ + "${idir}/catalog.json" + fi + + echo "OK ${slug}" +} + +cd "${SCRIPT_DIR}" + +FIXTURES=( + pr1-fix-clv + pr2-refactor-cte-to-models + pr3-amount-double-to-decimal + pr42-is-closed-filter + pr44-promotion-flags + pr46-net-clv-segments +) + +for slug in "${FIXTURES[@]}"; do + build_fixture "${slug}" +done diff --git a/evals/agent-blind-spots/fixtures/README.md b/evals/agent-blind-spots/fixtures/README.md new file mode 100644 index 0000000..45179b9 --- /dev/null +++ b/evals/agent-blind-spots/fixtures/README.md @@ -0,0 +1,78 @@ +# Fixtures — Agent Blind Spots / `/recce-verify` v1 + +Six PR fixtures from [`DataRecce/jaffle_shop_golden`](https://github.com/DataRecce/jaffle_shop_golden) covering distinct verification classes (semantic, row-grain, refactor, type, schema-expansion, multi-model). One directory per fixture, each holding a README, a frozen Tier-0 baseline template, the base/head commit SHAs (`commits.txt`), and a small source-models `diff.patch`. + +## Artifacts are not committed — build them locally + +The large dbt artifacts (`manifest-before.json`, `manifest-after.json`, `catalog-*.json`, `compiled-before/`, `compiled-after/`) are **gitignored** and produced by the build script. Run it once before each eval run: + +```bash +cd evals/agent-blind-spots +./build_fixtures.sh +``` + +The script clones `DataRecce/jaffle_shop_golden` into `.tmp/jaffle_shop_golden/` (also gitignored), swaps the upstream Snowflake `profiles.yml` for a local DuckDB profile, then for each fixture checks out the base + head (+ intermediate for PR #20), runs `dbt deps && dbt parse && dbt compile && dbt docs generate` against an empty DuckDB, and writes the outputs into `fixtures//artifacts/`. Host-specific manifest fields (`user_id`, `invocation_id`, `root_path`) are scrubbed. + +Re-running is idempotent — existing `artifacts/` directories are removed and rebuilt. + +## Index + +| Fixture | Source PR | Class | Notes | +|---------|-----------|-------|-------| +| [`pr1-fix-clv`](./pr1-fix-clv/) | [#13](https://github.com/DataRecce/jaffle_shop_golden/pull/13) | semantic | Adds `where status='completed'` inside `customers.customer_payments` CTE. | +| [`pr42-is-closed-filter`](./pr42-is-closed-filter/) | [#14](https://github.com/DataRecce/jaffle_shop_golden/pull/14) | row-grain | New `is_closed` column + `where is_closed=true` on `orders`. Different (older) base SHA than the rest. | +| [`pr2-refactor-cte-to-models`](./pr2-refactor-cte-to-models/) | [#15](https://github.com/DataRecce/jaffle_shop_golden/pull/15) | refactor | Behavior-preserving; the negative control for the rubric. | +| [`pr3-amount-double-to-decimal`](./pr3-amount-double-to-decimal/) | [#16](https://github.com/DataRecce/jaffle_shop_golden/pull/16) | type | `amount` narrowed to `DECIMAL(10,2)`. PR has a mechanical merge commit on top of the substantive change at `6ffc23f`; fixture captures the merge head. | +| [`pr44-promotion-flags`](./pr44-promotion-flags/) | [#20](https://github.com/DataRecce/jaffle_shop_golden/pull/20) | schema-expansion | Plus an *intermediate-commit* artifact snapshot for the row-filter accident at `23b96ca` (reverted by `1500eb4`). Uses Snowflake-specific `boolor_agg` — DuckDB does not validate at compile but would fail at execute. | +| [`pr46-net-clv-segments`](./pr46-net-clv-segments/) | [#2](https://github.com/DataRecce/jaffle_shop_golden/pull/2) | multi-model semantic | "Stress-test" fixture — redefines `customer_lifetime_value` in place, introduces three row filters on payments, copy-pastes a threshold for `net_value_segment`, and adds a `finance_revenue` model with no downstream consumers. | + +## Per-fixture layout + +``` +fixtures// +├── README.md ← what the PR does, expected verdicts without/with Recce, caveats +├── tier-0-baseline.md ← template instance, fields filled in by the eval runner +├── commits.txt ← base + head SHAs (and intermediate for PR #20) +├── diff.patch ← source-model diff base..head (small, reading-friendly) +└── artifacts/ ← gitignored — produced by build_fixtures.sh + ├── manifest-before.json ← `target/manifest.json` from `dbt parse` on base SHA + ├── manifest-after.json ← same on head SHA + ├── compiled-before/ ← `target/compiled/` from `dbt compile` on base SHA + ├── compiled-after/ ← same on head SHA + ├── catalog-before.json ← `target/catalog.json` from `dbt docs generate` on base SHA (empty data) + └── catalog-after.json ← same on head SHA +``` + +`pr44-promotion-flags/` additionally has a top-level `diff-from-base-to-intermediate.patch` (committed) and an `artifacts/intermediate-commit-23b96ca/` directory (gitignored) with `manifest.json`, `compiled/`, and `catalog.json` for the problematic intermediate commit. + +## Caveats — rolled up + +- **PR #16 (`pr3-amount-double-to-decimal`)** — head SHA `1c56861` is a mechanical merge commit; the substantive type narrowing lives at `6ffc23f`. The fixture captures the merge head; the source-models diff is identical. +- **PR #20 (`pr44-promotion-flags`)** — four-commit PR. The row-filter accident lives at the intermediate commit `23b96ca` and was reverted at `1500eb4`. Snowflake-specific `boolor_agg` appears in source but compiles fine under DuckDB (no compile-time function validation); it would fail at execute on DuckDB. +- **PR #2 (`pr46-net-clv-segments`)** — the "stress test" of the set. Redefines `customer_lifetime_value` in place, introduces three row filters on `payments`, copy-pastes a magic threshold for `net_value_segment`, and adds a `finance_revenue` model with no downstream consumers. Largest diff in the set. +- **Catalog row/column stats are zero everywhere** — `dbt docs generate` runs against an empty DuckDB, so `catalog-*.json` carries schema info (column names, types) but **no row counts and no column stats**. Do not score rubric items off catalog row stats. +- **PR #14 (`pr42-is-closed-filter`)** uses an older base SHA (`62d6dc9`) than the rest (`f09861a`). Don't mix bases when computing inter-fixture deltas. + +## dbt environment + +The source repo `jaffle_shop_golden` ships a **Snowflake-only** `profiles.yml`. Compile and docs-generate against Snowflake require warehouse credentials, which the eval baseline explicitly does not have. The fixture build pipeline therefore swaps in a local DuckDB profile *for parse/compile/docs-generate only*. The model SQL is portable between the two adapters with one exception noted above (`boolor_agg` in PR #20). + +Pinned versions for reproducibility: + +| Component | Version | +|-----------|---------| +| Python | 3.11.11 | +| dbt-core | 1.11.9 | +| dbt-duckdb | 1.10.1 | +| duckdb | 1.5.2 | +| dbt packages | `data-mie/dbt_profiler@0.8.1`, `dbt-labs/dbt_utils@0.9.6`, `dbt-labs/audit_helper@0.11.0` (per upstream `packages.yml`) | + +## Reproducing or extending + +- Add a fixture: create `fixtures//{README.md,tier-0-baseline.md,commits.txt,diff.patch}` and re-run `build_fixtures.sh`. The script reads `commits.txt` to discover the SHAs to build against. +- The script's only required system dependency is `uv` (and `git`). Everything else is installed into `evals/agent-blind-spots/.tmp/.venv/` from pinned versions in the script. +- Eval-baseline assets are **warehouse-free** — the local-dbt-only artifacts (manifest, compiled SQL, git diff) are the canonical inputs to a Tier-0 baseline run. + +## `commits.txt` format + +One SHA per line, with an optional message after the first whitespace. The first non-comment line is treated as the head; the build script also reads the per-fixture `README.md` to discover the base SHA (look for ``- Base SHA: `` ``). For PR #20, the intermediate commit is the line with SHA `23b96ca`. diff --git a/evals/agent-blind-spots/fixtures/pr1-fix-clv/README.md b/evals/agent-blind-spots/fixtures/pr1-fix-clv/README.md new file mode 100644 index 0000000..90f5e71 --- /dev/null +++ b/evals/agent-blind-spots/fixtures/pr1-fix-clv/README.md @@ -0,0 +1,42 @@ +# Fixture `pr1-fix-clv` — Fix CLV to completed orders only + +- Source PR: [DataRecce/jaffle_shop_golden#13](https://github.com/DataRecce/jaffle_shop_golden/pull/13) +- Base SHA: `f09861a39b314907719260f19f7d6ef2fd347ab0` +- Head SHA: `2abf877ec9d067f2169d1b690aefdd54a2b3d205` +- Verification class: **semantic** +- Models touched: `customers` (1 file, 1 line added) + +## What the PR does + +Adds `where orders.status = 'completed'` to the `customer_payments` CTE in `models/customers.sql`. The author frames it as a bugfix: previously, `customer_lifetime_value` summed payment amounts from *all* orders (including `placed`, `shipped`, `return_pending`, `returned`). With the fix it only counts payments tied to completed orders. + +## Why this is a "semantic" case + +The diff is one line, syntactically valid, and the schema (column names, types) is unchanged. An agent reading only manifest + git diff sees a filter being added inside a CTE. The intent — "should non-completed orders' payments count toward lifetime value?" — is a business semantic question, not a SQL correctness question. Whether this is a **fix** or a **regression** depends on the business definition of CLV, and on whether downstream consumers depended on the previous (looser) value. + +## Expected agent verdict — Tier 0 (no Recce) + +Anchor — what the Tier-0 baseline run *should* look like; not a live run record. + +- Likely catch quality: **partial**. +- The agent will spot the new `where` clause from the diff and correctly identify it narrows the set of payments aggregated. It can describe the semantic change in prose. +- But without measuring rows or values it cannot say: + - How many customers see their CLV change. + - How big the per-customer value delta is. + - Whether any customer's CLV drops to NULL because they have no completed orders. +- The agent is therefore likely to hedge ("this could be a bugfix or a behavior change depending on intent") rather than commit to a verdict. + +## Expected agent verdict — with Recce + +- Likely catch quality: **catch**. +- Evidence Recce should surface: + - **Row-count diff on `customers`** — unchanged (every customer still appears; left joins preserve rows). + - **Value diff on `customer_lifetime_value`** — non-trivial mismatch percentage; some customers' CLV drops, none rises. + - **Query diff** on average CLV per first-order week (the preset check in `recce.yml`) — values shift downward. +- Expected conclusion: "The change reduces CLV for customers whose orders include non-completed statuses. This is intentional per the PR title but is a behavior change; downstream consumers of `customer_lifetime_value` should be notified." + +## Caveats + +- The PR's compiled `.sql` for `customers.sql` is the only file that changes; nothing in the manifest schema changes. +- `recce.yml` already defines `value_diff` and `query_diff` preset checks on `customer_lifetime_value`, which makes this fixture an unusually friendly target for Recce. Other fixtures will not have this advantage. +- Reproducible without warehouse: yes. `compiled-before`/`compiled-after` SQL diff captures the substantive change. diff --git a/evals/agent-blind-spots/fixtures/pr1-fix-clv/commits.txt b/evals/agent-blind-spots/fixtures/pr1-fix-clv/commits.txt new file mode 100644 index 0000000..14ab5a0 --- /dev/null +++ b/evals/agent-blind-spots/fixtures/pr1-fix-clv/commits.txt @@ -0,0 +1 @@ +2abf877 PR1 diff --git a/evals/agent-blind-spots/fixtures/pr1-fix-clv/diff.patch b/evals/agent-blind-spots/fixtures/pr1-fix-clv/diff.patch new file mode 100644 index 0000000..07bd7b2 --- /dev/null +++ b/evals/agent-blind-spots/fixtures/pr1-fix-clv/diff.patch @@ -0,0 +1,12 @@ +diff --git a/models/customers.sql b/models/customers.sql +index 9aedd70..61a8302 100644 +--- a/models/customers.sql ++++ b/models/customers.sql +@@ -41,6 +41,7 @@ customer_payments as ( + left join orders on + payments.order_id = orders.order_id + ++ where orders.status = 'completed' + group by orders.customer_id + + ), diff --git a/evals/agent-blind-spots/fixtures/pr1-fix-clv/tier-0-baseline.md b/evals/agent-blind-spots/fixtures/pr1-fix-clv/tier-0-baseline.md new file mode 100644 index 0000000..62f2b23 --- /dev/null +++ b/evals/agent-blind-spots/fixtures/pr1-fix-clv/tier-0-baseline.md @@ -0,0 +1,41 @@ +# Tier-0 Baseline — Fixture `pr1-fix-clv` + +Agent-only verdict for the fixture below. **Frozen at commit**: once this file lands on the branch (via the commander's normal change-control flow), the with-Recce run for this fixture can begin, and this file must not be edited even if later evidence suggests it should be revised. Captured before any Recce-aware run so the eval measures delta, not absolute correctness. + +## Fixture + +- PR: +- Title: `PR1 — Fix CLV to completed orders only` +- Verification class: `semantic` + +## Agent run + +- Agent: `` +- Model: `` +- Date captured: `` +- Inputs available to agent: dbt manifest, compiled SQL pre/post, git diff. **No Recce, no warehouse access.** + +## Prompt given to agent + +Verbatim, including any framing about it being a PR review task. + +``` + +``` + +## Verdict + +- Catch / miss / partial: `` +- Action the agent recommended: `` + +## Reasoning the agent gave + +Verbatim. The baseline reasoning is the thing Recce's evidence will or won't shift, so paraphrasing here destroys the signal. If the agent's output is very long, quote the verdict-bearing passage verbatim and link to the full transcript. + +``` + +``` + +## Notes + +`` diff --git a/evals/agent-blind-spots/fixtures/pr2-refactor-cte-to-models/README.md b/evals/agent-blind-spots/fixtures/pr2-refactor-cte-to-models/README.md new file mode 100644 index 0000000..8b4bc3f --- /dev/null +++ b/evals/agent-blind-spots/fixtures/pr2-refactor-cte-to-models/README.md @@ -0,0 +1,41 @@ +# Fixture `pr2-refactor-cte-to-models` — Refactor CTEs into intermediate models + +- Source PR: [DataRecce/jaffle_shop_golden#15](https://github.com/DataRecce/jaffle_shop_golden/pull/15) +- Base SHA: `f09861a39b314907719260f19f7d6ef2fd347ab0` +- Head SHA: `9c386b453ba7f5317784dc5c6ec03e48af0d4903` +- Verification class: **refactor (behavior-preserving, equality expected)** +- Models touched: `customers` (rewritten), `int_customer_orders` (new), `int_customer_payments` (new) + +## What the PR does + +Refactors `customers.sql` by extracting the two inline CTEs (`customer_orders` and `customer_payments`) into two new standalone intermediate models (`int_customer_orders.sql`, `int_customer_payments.sql`). The body of `customers.sql` is reduced from a ~70-line `with ... select` chain to a single 16-line `select ... left join ref(int_*) ...`. + +No filters added, no columns added or removed, no aggregation logic changed. The DAG gets two new nodes between the staging layer and `customers`. + +## Why this is a "refactor / equality-expected" case + +This fixture is the **negative control** for the eval. A Recce-aware agent should *approve* this PR with high confidence and cite evidence that the refactor preserves behavior. A Recce-aware agent that flags this PR as risky has either misread the lineage change or is generating false alarms. + +The verification question: can the agent (with Recce) confidently conclude "behavior preserved"? + +## Expected agent verdict — Tier 0 (no Recce) + +Anchor — what the Tier-0 baseline run *should* look like; not a live run record. + +- Likely catch quality: **catch** (the easiest verdict in the fixture set, but for the wrong reason — usually pattern-matching on "refactor" in the commit message rather than verifying equivalence). +- The agent will recognize the structural rewrite from the diff and the manifest's new node count and assume behavior preservation based on shape alone. It cannot prove equivalence without comparing values. + +## Expected agent verdict — with Recce + +- Likely catch quality: **catch**. +- Evidence Recce should surface: + - **Lineage diff** — two new nodes appear (`int_customer_orders`, `int_customer_payments`); `customers` now depends on them rather than directly on staging. + - **Schema diff on `customers`** — unchanged columns. + - **Row-count diff on `customers`** — unchanged. + - **Value diff on `customer_lifetime_value`** — 100% match (the preset check in `recce.yml`). +- Expected conclusion: "Behavior-preserving refactor. Two intermediate models added; downstream `customers` is bit-for-bit identical. Approve." + +## Caveats + +- This fixture exercises the rubric's `catch → catch` "same" delta bucket, where the case-study Notes should record *what Recce showed that the Tier-0 verdict couldn't* — namely value-level equivalence. If both runs say "catch" but only the with-Recce run produces a 100% match receipt, that's the differentiating signal even though the binary catch didn't move. +- Reproducible without warehouse: yes. diff --git a/evals/agent-blind-spots/fixtures/pr2-refactor-cte-to-models/commits.txt b/evals/agent-blind-spots/fixtures/pr2-refactor-cte-to-models/commits.txt new file mode 100644 index 0000000..fa2c7ba --- /dev/null +++ b/evals/agent-blind-spots/fixtures/pr2-refactor-cte-to-models/commits.txt @@ -0,0 +1 @@ +9c386b4 PR2 diff --git a/evals/agent-blind-spots/fixtures/pr2-refactor-cte-to-models/diff.patch b/evals/agent-blind-spots/fixtures/pr2-refactor-cte-to-models/diff.patch new file mode 100644 index 0000000..ddfe888 --- /dev/null +++ b/evals/agent-blind-spots/fixtures/pr2-refactor-cte-to-models/diff.patch @@ -0,0 +1,121 @@ +diff --git a/models/customers.sql b/models/customers.sql +index 9aedd70..67dd22a 100644 +--- a/models/customers.sql ++++ b/models/customers.sql +@@ -1,69 +1,16 @@ +-with customers as ( +- +- select * from {{ ref('stg_customers') }} +- +-), +- +-orders as ( +- +- select * from {{ ref('stg_orders') }} +- +-), +- +-payments as ( +- +- select * from {{ ref('stg_payments') }} +- +-), +- +-customer_orders as ( +- +- select +- customer_id, +- +- min(order_date) as first_order, +- max(order_date) as most_recent_order, +- count(order_id) as number_of_orders +- from orders +- +- group by customer_id +- +-), +- +-customer_payments as ( +- +- select +- orders.customer_id, +- sum(amount)::bigint as total_amount +- +- from payments +- +- left join orders on +- payments.order_id = orders.order_id +- +- group by orders.customer_id +- +-), +- +-final as ( +- +- select +- customers.customer_id, +- customers.first_name, +- customers.last_name, +- customer_orders.first_order, +- customer_orders.most_recent_order, +- customer_orders.number_of_orders, +- customer_payments.total_amount as customer_lifetime_value +- +- from customers +- +- left join customer_orders +- on customers.customer_id = customer_orders.customer_id +- +- left join customer_payments +- on customers.customer_id = customer_payments.customer_id +- +-) +- +-select * from final ++select ++ customers.customer_id, ++ customers.first_name, ++ customers.last_name, ++ customer_orders.first_order, ++ customer_orders.most_recent_order, ++ customer_orders.number_of_orders, ++ customer_payments.total_amount as customer_lifetime_value ++ ++from {{ ref('stg_customers') }} customers ++ ++left join {{ ref('int_customer_orders') }} customer_orders ++ on customers.customer_id = customer_orders.customer_id ++ ++left join {{ ref('int_customer_payments') }} customer_payments ++ on customers.customer_id = customer_payments.customer_id +diff --git a/models/int_customer_orders.sql b/models/int_customer_orders.sql +new file mode 100644 +index 0000000..eec8bc8 +--- /dev/null ++++ b/models/int_customer_orders.sql +@@ -0,0 +1,9 @@ ++select ++ customer_id, ++ ++ min(order_date) as first_order, ++ max(order_date) as most_recent_order, ++ count(order_id) as number_of_orders ++from {{ ref('stg_orders') }} ++ ++group by customer_id +diff --git a/models/int_customer_payments.sql b/models/int_customer_payments.sql +new file mode 100644 +index 0000000..b36ac82 +--- /dev/null ++++ b/models/int_customer_payments.sql +@@ -0,0 +1,10 @@ ++select ++ orders.customer_id, ++ sum(amount)::bigint as total_amount ++ ++from {{ ref('stg_payments') }} payments ++ ++left join {{ ref('stg_orders') }} orders on ++ payments.order_id = orders.order_id ++ ++group by orders.customer_id diff --git a/evals/agent-blind-spots/fixtures/pr2-refactor-cte-to-models/tier-0-baseline.md b/evals/agent-blind-spots/fixtures/pr2-refactor-cte-to-models/tier-0-baseline.md new file mode 100644 index 0000000..377935b --- /dev/null +++ b/evals/agent-blind-spots/fixtures/pr2-refactor-cte-to-models/tier-0-baseline.md @@ -0,0 +1,41 @@ +# Tier-0 Baseline — Fixture `pr2-refactor-cte-to-models` + +Agent-only verdict for the fixture below. **Frozen at commit**: once this file lands on the branch (via the commander's normal change-control flow), the with-Recce run for this fixture can begin, and this file must not be edited even if later evidence suggests it should be revised. Captured before any Recce-aware run so the eval measures delta, not absolute correctness. + +## Fixture + +- PR: +- Title: `PR2 — Refactor CTEs into intermediate models` +- Verification class: `refactor` + +## Agent run + +- Agent: `` +- Model: `` +- Date captured: `` +- Inputs available to agent: dbt manifest, compiled SQL pre/post, git diff. **No Recce, no warehouse access.** + +## Prompt given to agent + +Verbatim, including any framing about it being a PR review task. + +``` + +``` + +## Verdict + +- Catch / miss / partial: `` +- Action the agent recommended: `` + +## Reasoning the agent gave + +Verbatim. The baseline reasoning is the thing Recce's evidence will or won't shift, so paraphrasing here destroys the signal. If the agent's output is very long, quote the verdict-bearing passage verbatim and link to the full transcript. + +``` + +``` + +## Notes + +`` diff --git a/evals/agent-blind-spots/fixtures/pr3-amount-double-to-decimal/README.md b/evals/agent-blind-spots/fixtures/pr3-amount-double-to-decimal/README.md new file mode 100644 index 0000000..d1e7097 --- /dev/null +++ b/evals/agent-blind-spots/fixtures/pr3-amount-double-to-decimal/README.md @@ -0,0 +1,61 @@ +# Fixture `pr3-amount-double-to-decimal` — Change payment amount from double to decimal + +- Source PR: [DataRecce/jaffle_shop_golden#16](https://github.com/DataRecce/jaffle_shop_golden/pull/16) +- Base SHA: `f09861a39b314907719260f19f7d6ef2fd347ab0` +- Head SHA: `1c56861bf11eb1449eb6d357596d8ff015678c5b` (includes a `main` merge commit on top of `6ffc23f` — see drift note below) +- Verification class: **type / rounding drift** +- Models touched: `stg_payments` (1 file, 1 line) + +## What the PR does + +In `models/staging/stg_payments.sql`, replaces: + +```sql +amount / 100 as amount +``` + +with: + +```sql +(amount / 100)::DECIMAL(10,2) amount +``` + +The previous expression yields a floating-point type (Snowflake `NUMBER(38,4)` on integer division, but for the original raw cents column behaves like double). The new expression coerces to `DECIMAL(10,2)`. + +## Why this is a "type / rounding" case + +Three distinct sub-issues, none catastrophic individually: + +1. **Precision narrowing.** `DECIMAL(10,2)` only fits values up to `99,999,999.99`. Any payment whose dollar value exceeds that overflows and the result is implementation-dependent (Snowflake errors; DuckDB may also error). +2. **Rounding behavior.** Casting `amount / 100` (e.g., `1234 / 100` in integer arithmetic) to `DECIMAL(10,2)` produces `12.34` instead of `12` or `12.3400000`. Whether this is the desired result depends on the upstream type of `amount`. +3. **Missing `as`.** The new statement is `(amount / 100)::DECIMAL(10,2) amount` — no `as`. Snowflake accepts an alias without `as`; this is a stylistic quirk, not a bug. + +The downstream effect propagates through `customer_payments`/`gross_amount`/`customer_lifetime_value`. CLV is cast to `::bigint` further downstream, which truncates fractional cents — so the visible delta in CLV may be small or zero, masking the upstream type change. + +## Expected agent verdict — Tier 0 (no Recce) + +Anchor — what the Tier-0 baseline run *should* look like; not a live run record. + +- Likely catch quality: **partial** or **miss**. +- A diligent agent will spot the type cast and may comment on rounding / overflow possibilities. The agent will *not* know: + - The actual maximum payment amount in the data (and therefore overflow risk). + - Whether any downstream computation changes value because of the cast. + - Whether the `::bigint` cast downstream hides the precision change entirely. +- Most agents will pattern-match on "type change" → "may affect downstream" and hedge. + +## Expected agent verdict — with Recce + +- Likely catch quality: **partial** at best on a *single dev environment* (Tier 1). +- Evidence Recce should surface: + - **Schema diff** — column type for `amount` on `stg_payments` changed to `DECIMAL(10,2)`. + - **Query diff** for `select max(amount) from stg_payments` — same value, but now bounded by precision. +- What single-env Recce **cannot** surface: + - Whether the change introduces overflow on production-scale data (no base env to compare against). + - Whether `customer_lifetime_value` shifts on any customer (requires Tier-2 base comparison; the ::bigint cast may absorb the difference anyway). +- Expected conclusion: "Type narrowed to DECIMAL(10,2). Potential overflow if any payment exceeds 99,999,999.99 dollars. Downstream impact unknown without a base comparison." + +## Caveats + +- **PR has evolved beyond the spec.** Head SHA includes a merge commit (`1c56861`) that brings `main` into the PR branch. The substantive change is at `6ffc23f`; the merge is mechanical. We use the merge head as the fixture head because it is what reviewers see when the PR is opened. +- This fixture is the clearest example in the set where **single-env Tier-1 Recce is honestly degraded** vs Tier-2 (data-diff). The case study should call this out — it's the most useful kind of signal for the gap report. +- Reproducible without warehouse: yes. The type change is visible in the source diff and the `catalog-*.json` files; without real data the empty catalogs will not show realistic precision behavior, so the eval runner should not rely on `catalog-*.json` numbers for this fixture. diff --git a/evals/agent-blind-spots/fixtures/pr3-amount-double-to-decimal/commits.txt b/evals/agent-blind-spots/fixtures/pr3-amount-double-to-decimal/commits.txt new file mode 100644 index 0000000..dcb7810 --- /dev/null +++ b/evals/agent-blind-spots/fixtures/pr3-amount-double-to-decimal/commits.txt @@ -0,0 +1,2 @@ +1c56861 Merge branch 'main' into feature/add-rounding-effect-analysis +6ffc23f PR3 diff --git a/evals/agent-blind-spots/fixtures/pr3-amount-double-to-decimal/diff.patch b/evals/agent-blind-spots/fixtures/pr3-amount-double-to-decimal/diff.patch new file mode 100644 index 0000000..0dfaa27 --- /dev/null +++ b/evals/agent-blind-spots/fixtures/pr3-amount-double-to-decimal/diff.patch @@ -0,0 +1,13 @@ +diff --git a/models/staging/stg_payments.sql b/models/staging/stg_payments.sql +index 28b8e8b..8900825 100644 +--- a/models/staging/stg_payments.sql ++++ b/models/staging/stg_payments.sql +@@ -12,7 +12,7 @@ renamed as ( + payment_method, + + -- `amount` is currently stored in cents, so we convert it to dollars +- amount / 100 as amount ++ (amount / 100)::DECIMAL(10,2) amount + + from source + diff --git a/evals/agent-blind-spots/fixtures/pr3-amount-double-to-decimal/tier-0-baseline.md b/evals/agent-blind-spots/fixtures/pr3-amount-double-to-decimal/tier-0-baseline.md new file mode 100644 index 0000000..93dbd7b --- /dev/null +++ b/evals/agent-blind-spots/fixtures/pr3-amount-double-to-decimal/tier-0-baseline.md @@ -0,0 +1,41 @@ +# Tier-0 Baseline — Fixture `pr3-amount-double-to-decimal` + +Agent-only verdict for the fixture below. **Frozen at commit**: once this file lands on the branch (via the commander's normal change-control flow), the with-Recce run for this fixture can begin, and this file must not be edited even if later evidence suggests it should be revised. Captured before any Recce-aware run so the eval measures delta, not absolute correctness. + +## Fixture + +- PR: +- Title: `PR3 — Change payment amount from double to decimal` +- Verification class: `type` + +## Agent run + +- Agent: `` +- Model: `` +- Date captured: `` +- Inputs available to agent: dbt manifest, compiled SQL pre/post, git diff. **No Recce, no warehouse access.** + +## Prompt given to agent + +Verbatim, including any framing about it being a PR review task. + +``` + +``` + +## Verdict + +- Catch / miss / partial: `` +- Action the agent recommended: `` + +## Reasoning the agent gave + +Verbatim. The baseline reasoning is the thing Recce's evidence will or won't shift, so paraphrasing here destroys the signal. If the agent's output is very long, quote the verdict-bearing passage verbatim and link to the full transcript. + +``` + +``` + +## Notes + +`` diff --git a/evals/agent-blind-spots/fixtures/pr42-is-closed-filter/README.md b/evals/agent-blind-spots/fixtures/pr42-is-closed-filter/README.md new file mode 100644 index 0000000..fa82079 --- /dev/null +++ b/evals/agent-blind-spots/fixtures/pr42-is-closed-filter/README.md @@ -0,0 +1,44 @@ +# Fixture `pr42-is-closed-filter` — Add `is_closed` and filter orders + +- Source PR: [DataRecce/jaffle_shop_golden#14](https://github.com/DataRecce/jaffle_shop_golden/pull/14) +- Base SHA: `62d6dc9367cb6a35fc56942ad437900f9c1fd8cb` +- Head SHA: `d2be60a0f338ef6bf5e1dcf143c5bc0a17a55060` +- Verification class: **row-grain** +- Models touched: `orders`, `stg_orders` (2 files) + +## What the PR does + +Two changes packaged together: + +1. `stg_orders.sql` adds a new derived column `is_closed`, computed as `status = 'completed'`. +2. `orders.sql` (a) surfaces `is_closed` in the model's output, then (b) adds `where is_closed = true` at the bottom of the final CTE. + +The net effect on `orders`: every row whose status is anything other than `completed` is dropped. The row grain of `orders` changes from "one row per order" to "one row per **completed** order." A new column also appears in the schema. + +## Why this is a "row-grain" case + +Both changes are syntactically clean. The compiled-after SQL still reads naturally. The verification question — "does the agent notice that the model now drops ~half the rows?" — depends on whether the agent reasons about set semantics from the new `where` clause, not on a SQL parse error. + +Adding the `is_closed` column to the schema is a separate concern (downstream consumers that `select *` from `orders` now have an extra column). The row drop is the more dangerous of the two. + +## Expected agent verdict — Tier 0 (no Recce) + +Anchor — what the Tier-0 baseline run *should* look like; not a live run record. + +- Likely catch quality: **partial** (best case) or **miss** (typical). +- An attentive agent reading the diff will see the new `where is_closed = true` and flag it. A less careful agent will frame the change as "adds an `is_closed` indicator," focusing on the schema addition and treating the filter as obvious / intentional. +- Without row counts the agent has no way to quantify the impact ("filters down to maybe 30% of orders? 90%?") and cannot point to a downstream consumer that breaks. + +## Expected agent verdict — with Recce + +- Likely catch quality: **catch**. +- Evidence Recce should surface: + - **Row-count diff on `orders`** — large negative delta (every non-completed order is dropped). + - **Schema diff** — new column `is_closed` on both `stg_orders` and `orders`. + - **Lineage** — any model downstream of `orders` that depends on non-completed orders now silently sees fewer rows. +- Expected conclusion: "This PR drops X% of rows from `orders` because of the new `where is_closed = true` filter. The `is_closed` column itself is fine; the filter is the breaking change. Either remove the filter or update the model's contract." + +## Caveats + +- Base SHA for this PR is `62d6dc936...`, an older `main` head before `f09861a` (`feat: add avg_order_amount to orders_daily_summary`). That earlier merge added two columns to `orders_daily_summary` that are absent in this fixture's base manifest. The other five fixtures use `f09861a` as base. +- Reproducible without warehouse: yes. The compiled SQL and the source diff make the filter visible. diff --git a/evals/agent-blind-spots/fixtures/pr42-is-closed-filter/commits.txt b/evals/agent-blind-spots/fixtures/pr42-is-closed-filter/commits.txt new file mode 100644 index 0000000..6e69c46 --- /dev/null +++ b/evals/agent-blind-spots/fixtures/pr42-is-closed-filter/commits.txt @@ -0,0 +1 @@ +d2be60a PR42 diff --git a/evals/agent-blind-spots/fixtures/pr42-is-closed-filter/diff.patch b/evals/agent-blind-spots/fixtures/pr42-is-closed-filter/diff.patch new file mode 100644 index 0000000..df32d56 --- /dev/null +++ b/evals/agent-blind-spots/fixtures/pr42-is-closed-filter/diff.patch @@ -0,0 +1,38 @@ +diff --git a/models/orders.sql b/models/orders.sql +index cbb2934..5b0beee 100644 +--- a/models/orders.sql ++++ b/models/orders.sql +@@ -36,6 +36,7 @@ final as ( + orders.customer_id, + orders.order_date, + orders.status, ++ orders.is_closed, + + {% for payment_method in payment_methods -%} + +@@ -50,7 +51,7 @@ final as ( + + left join order_payments + on orders.order_id = order_payments.order_id +- ++ where is_closed = true + ) + + select * from final +diff --git a/models/staging/stg_orders.sql b/models/staging/stg_orders.sql +index ec77ac2..7d4eb7d 100644 +--- a/models/staging/stg_orders.sql ++++ b/models/staging/stg_orders.sql +@@ -10,10 +10,9 @@ renamed as ( + id as order_id, + user_id as customer_id, + order_date, +- status +- ++ status, ++ status = 'completed' as is_closed + from source +- + ) + + select * from renamed diff --git a/evals/agent-blind-spots/fixtures/pr42-is-closed-filter/tier-0-baseline.md b/evals/agent-blind-spots/fixtures/pr42-is-closed-filter/tier-0-baseline.md new file mode 100644 index 0000000..a388ef2 --- /dev/null +++ b/evals/agent-blind-spots/fixtures/pr42-is-closed-filter/tier-0-baseline.md @@ -0,0 +1,41 @@ +# Tier-0 Baseline — Fixture `pr42-is-closed-filter` + +Agent-only verdict for the fixture below. **Frozen at commit**: once this file lands on the branch (via the commander's normal change-control flow), the with-Recce run for this fixture can begin, and this file must not be edited even if later evidence suggests it should be revised. Captured before any Recce-aware run so the eval measures delta, not absolute correctness. + +## Fixture + +- PR: +- Title: `PR42 — Add is_closed and filter orders` +- Verification class: `row-grain` + +## Agent run + +- Agent: `` +- Model: `` +- Date captured: `` +- Inputs available to agent: dbt manifest, compiled SQL pre/post, git diff. **No Recce, no warehouse access.** + +## Prompt given to agent + +Verbatim, including any framing about it being a PR review task. + +``` + +``` + +## Verdict + +- Catch / miss / partial: `` +- Action the agent recommended: `` + +## Reasoning the agent gave + +Verbatim. The baseline reasoning is the thing Recce's evidence will or won't shift, so paraphrasing here destroys the signal. If the agent's output is very long, quote the verdict-bearing passage verbatim and link to the full transcript. + +``` + +``` + +## Notes + +`` diff --git a/evals/agent-blind-spots/fixtures/pr44-promotion-flags/README.md b/evals/agent-blind-spots/fixtures/pr44-promotion-flags/README.md new file mode 100644 index 0000000..f32c92b --- /dev/null +++ b/evals/agent-blind-spots/fixtures/pr44-promotion-flags/README.md @@ -0,0 +1,73 @@ +# Fixture `pr44-promotion-flags` — Add promotion payment flag + customer has-promotion flag + +- Source PR: [DataRecce/jaffle_shop_golden#20](https://github.com/DataRecce/jaffle_shop_golden/pull/20) +- Base SHA: `f09861a39b314907719260f19f7d6ef2fd347ab0` +- Head SHA: `bd407ac2a40ce52cc24ee9c40393e9412706c4e3` +- Verification class: **schema-expansion + (intermediate row-filter accident)** +- Models touched: `stg_payments`, `customers` (2 files at head) + +## What the PR does (head state) + +Two schema additions at head: + +1. `stg_payments` gains an `is_promotion` column derived as `payment_method = 'coupon'`. +2. `customers` gains a `has_promoted_orders` column, computed via `boolor_agg(is_promotion)` in the `customer_payments` CTE, surfaced in the final select. + +At head, no row filter is present in `customers.sql`. Schema and lineage both expand; no rows are dropped. + +## The intermediate-commit accident (the interesting bit) + +PR #20 has four commits. The interesting case for this eval is **not** the head state — it is the state after commit `23b96ca02b` ("Add promotion information"), which: + +- Added the two schema columns as described above. +- **Also added** `where has_promoted_orders = true` at the bottom of the `customers.sql` final CTE. + +That `where` clause means: only customers with at least one coupon-paid order remain in the `customers` model. Every customer who paid by card / bank-transfer / gift-card with no coupon ever is dropped. + +Commit `1500eb444c` ("Remove where condition") reverted the filter — but only because the author re-read their own diff. An agent reviewing commit-by-commit (or any reviewer who looks only at the first commit's preview before more were pushed) would face the same row-filter trap as `pr42-is-closed-filter`, packaged inside what looks like a benign schema-expansion PR. + +Artifacts for both states are produced by `build_fixtures.sh` (not committed): + +- `artifacts/manifest-after.json` + `artifacts/compiled-after/` — head (`bd407ac`). Schema expansion only, no row drop. +- `artifacts/intermediate-commit-23b96ca/` — the problematic intermediate. Same schema additions **plus** the row filter. +- `diff-from-base-to-intermediate.patch` (committed, top-level) — the diff `base..23b96ca` so the eval can present the intermediate commit as if it were the PR head. + +## Why this is a "schema-expansion + accidental row filter" case + +The dangerous pattern is the *combination*. A new column is a low-stakes change; reviewers tend to approve it on diff alone. Wrapping a row filter inside the same commit hides the filter behind the schema noise. Recce's row-count diff is the cheap, decisive signal that catches it. + +## Expected agent verdict — Tier 0 (no Recce) + +Anchor — what the Tier-0 baseline run *should* look like; not a live run record. **Run separately against each of the two artifact snapshots; the headline finding is the contrast between them.** + +### Against head (`bd407ac`) + +- Likely catch quality: **catch** (correctly approves schema expansion). +- Reasoning: agent sees two new columns, no `where` clause introduced, approves with a note about new columns. + +### Against intermediate (`23b96ca`) + +- Likely catch quality: **miss** or **partial**. +- The agent sees a schema-expansion diff that *also* contains `where has_promoted_orders = true`. Diligent agents catch the filter; many will read the diff as "adds promotion flags" and miss the trailing two lines. +- Without row counts they cannot quantify impact. + +## Expected agent verdict — with Recce + +### Against head (`bd407ac`) + +- Likely catch quality: **catch**. +- Evidence Recce should surface: schema diff showing two new columns, row count unchanged. +- Conclusion: "Approve — schema expansion, no row impact." + +### Against intermediate (`23b96ca`) + +- Likely catch quality: **catch**. +- Evidence Recce should surface: schema diff (new columns) **and** row-count diff on `customers` showing a large drop. +- Conclusion: "Block — accidental row filter introduced in the same commit as the schema expansion. Either remove `where has_promoted_orders = true` or scope the new column to a separate downstream model." + +## Caveats + +- The PR has **four commits**, not one. Per the spec, the row-filter case lives in the middle. The eval runner has the choice to (a) treat `head` as the canonical PR (clean schema expansion, no row drop) or (b) replay commit `23b96ca` as the PR head to exercise the row-filter case. The author of the spec wanted **both**. +- Tier-2 (base comparison) gives the cleanest evidence, but the row-filter trap is also visible from single-env Recce: a row count of `customers` against the dev environment's prior state would surface the drop. Whether the agent can construct that comparison without a base env is itself a case-study question. +- `boolor_agg` is a Snowflake function. The fixture build pipeline overrides the dbt profile to DuckDB for offline compile; DuckDB accepts `bool_or` as the canonical aggregate. The compiled SQL captured in `compiled-after/` will show `boolor_agg` as written in source — note this in the agent prompt if the eval runs a DuckDB-backed Recce. +- Reproducible without warehouse: yes for compile artifacts. Schema diff and lineage diff are visible; row-count diff requires a live env, which v1 Tier-1 single-env Recce can do against any dev DB. diff --git a/evals/agent-blind-spots/fixtures/pr44-promotion-flags/commits.txt b/evals/agent-blind-spots/fixtures/pr44-promotion-flags/commits.txt new file mode 100644 index 0000000..02ba8cf --- /dev/null +++ b/evals/agent-blind-spots/fixtures/pr44-promotion-flags/commits.txt @@ -0,0 +1,4 @@ +bd407ac Fix Snowflake compatibility: use = operator and boolor_agg +274e750 Fix equality operator in stg_payments.sql for Snowflake compatibility +1500eb4 Remove where condition +23b96ca Add promotion information diff --git a/evals/agent-blind-spots/fixtures/pr44-promotion-flags/diff-from-base-to-intermediate.patch b/evals/agent-blind-spots/fixtures/pr44-promotion-flags/diff-from-base-to-intermediate.patch new file mode 100644 index 0000000..52f93e4 --- /dev/null +++ b/evals/agent-blind-spots/fixtures/pr44-promotion-flags/diff-from-base-to-intermediate.patch @@ -0,0 +1,47 @@ +diff --git a/models/customers.sql b/models/customers.sql +index 9aedd70..1d4cb29 100644 +--- a/models/customers.sql ++++ b/models/customers.sql +@@ -34,7 +34,8 @@ customer_payments as ( + + select + orders.customer_id, +- sum(amount)::bigint as total_amount ++ sum(amount)::bigint as total_amount, ++ bool_or(is_promotion) as has_promoted_orders + + from payments + +@@ -54,7 +55,8 @@ final as ( + customer_orders.first_order, + customer_orders.most_recent_order, + customer_orders.number_of_orders, +- customer_payments.total_amount as customer_lifetime_value ++ customer_payments.total_amount as customer_lifetime_value, ++ customer_payments.has_promoted_orders + + from customers + +@@ -63,7 +65,7 @@ final as ( + + left join customer_payments + on customers.customer_id = customer_payments.customer_id +- ++ where has_promoted_orders = true + ) + + select * from final +diff --git a/models/staging/stg_payments.sql b/models/staging/stg_payments.sql +index 28b8e8b..8a8d4be 100644 +--- a/models/staging/stg_payments.sql ++++ b/models/staging/stg_payments.sql +@@ -12,7 +12,8 @@ renamed as ( + payment_method, + + -- `amount` is currently stored in cents, so we convert it to dollars +- amount / 100 as amount ++ amount / 100 as amount, ++ payment_method == 'coupon' as is_promotion + + from source + diff --git a/evals/agent-blind-spots/fixtures/pr44-promotion-flags/diff.patch b/evals/agent-blind-spots/fixtures/pr44-promotion-flags/diff.patch new file mode 100644 index 0000000..c5f372c --- /dev/null +++ b/evals/agent-blind-spots/fixtures/pr44-promotion-flags/diff.patch @@ -0,0 +1,46 @@ +diff --git a/models/customers.sql b/models/customers.sql +index 9aedd70..9b440c2 100644 +--- a/models/customers.sql ++++ b/models/customers.sql +@@ -34,7 +34,8 @@ customer_payments as ( + + select + orders.customer_id, +- sum(amount)::bigint as total_amount ++ sum(amount)::bigint as total_amount, ++ boolor_agg(is_promotion) as has_promoted_orders + + from payments + +@@ -54,7 +55,8 @@ final as ( + customer_orders.first_order, + customer_orders.most_recent_order, + customer_orders.number_of_orders, +- customer_payments.total_amount as customer_lifetime_value ++ customer_payments.total_amount as customer_lifetime_value, ++ customer_payments.has_promoted_orders + + from customers + +@@ -63,7 +65,6 @@ final as ( + + left join customer_payments + on customers.customer_id = customer_payments.customer_id +- + ) + + select * from final +diff --git a/models/staging/stg_payments.sql b/models/staging/stg_payments.sql +index 28b8e8b..9977ba8 100644 +--- a/models/staging/stg_payments.sql ++++ b/models/staging/stg_payments.sql +@@ -12,7 +12,8 @@ renamed as ( + payment_method, + + -- `amount` is currently stored in cents, so we convert it to dollars +- amount / 100 as amount ++ amount / 100 as amount, ++ payment_method = 'coupon' as is_promotion + + from source + diff --git a/evals/agent-blind-spots/fixtures/pr44-promotion-flags/tier-0-baseline.md b/evals/agent-blind-spots/fixtures/pr44-promotion-flags/tier-0-baseline.md new file mode 100644 index 0000000..553c7cf --- /dev/null +++ b/evals/agent-blind-spots/fixtures/pr44-promotion-flags/tier-0-baseline.md @@ -0,0 +1,41 @@ +# Tier-0 Baseline — Fixture `pr44-promotion-flags` + +Agent-only verdict for the fixture below. **Frozen at commit**: once this file lands on the branch (via the commander's normal change-control flow), the with-Recce run for this fixture can begin, and this file must not be edited even if later evidence suggests it should be revised. Captured before any Recce-aware run so the eval measures delta, not absolute correctness. + +## Fixture + +- PR: +- Title: `PR44 — Add promotion payment flag + customer has-promotion flag` +- Verification class: `schema-expansion` + +## Agent run + +- Agent: `` +- Model: `` +- Date captured: `` +- Inputs available to agent: dbt manifest, compiled SQL pre/post, git diff. **No Recce, no warehouse access.** + +## Prompt given to agent + +Verbatim, including any framing about it being a PR review task. + +``` + +``` + +## Verdict + +- Catch / miss / partial: `` +- Action the agent recommended: `` + +## Reasoning the agent gave + +Verbatim. The baseline reasoning is the thing Recce's evidence will or won't shift, so paraphrasing here destroys the signal. If the agent's output is very long, quote the verdict-bearing passage verbatim and link to the full transcript. + +``` + +``` + +## Notes + +`` diff --git a/evals/agent-blind-spots/fixtures/pr46-net-clv-segments/README.md b/evals/agent-blind-spots/fixtures/pr46-net-clv-segments/README.md new file mode 100644 index 0000000..cbd28f9 --- /dev/null +++ b/evals/agent-blind-spots/fixtures/pr46-net-clv-segments/README.md @@ -0,0 +1,64 @@ +# Fixture `pr46-net-clv-segments` — Net revenue, net CLV, customer segments + +- Source PR: [DataRecce/jaffle_shop_golden#2](https://github.com/DataRecce/jaffle_shop_golden/pull/2) +- Base SHA: `f09861a39b314907719260f19f7d6ef2fd347ab0` +- Head SHA: `297eb54e868f7f6070cc1d2bb6a46aade7cc97b1` +- Verification class: **multi-model semantic** +- Models touched: `stg_payments`, `customers`, `customer_segments`, `finance_revenue` (new), schema YAML (2 files) + +## What the PR does + +The most behavior-rich PR in the fixture set. Multiple co-changed models, each with its own semantic concern: + +1. **`stg_payments`** gains a `coupon_amount` column: `(payment_method = 'coupon')::int * (amount / 100)`. Pure schema add — no row impact. + +2. **`customers`** is rewritten more than the diff suggests: + - The intermediate `customer_payments` CTE renames `total_amount` → `gross_amount` and adds `net_amount = sum(amount - coupon_amount)`. + - The join is qualified with `and orders.status = 'completed'` — silently introducing the same "completed-only" semantic as PR #13 but inside the join condition (not a `where`). + - Adds `where payments.amount is not null and payments.amount > 0` — two row filters on the payments side of the join, before aggregation. + - The final select **renames** the user-visible `customer_lifetime_value` to be sourced from `gross_amount` (was `total_amount`), and adds `net_customer_lifetime_value` from `net_amount`. The column name `customer_lifetime_value` is preserved at the model boundary; its definition changed underneath. + +3. **`customer_segments`** adds `net_customer_lifetime_value` and a new `net_value_segment` column with the same threshold logic as the existing `value_segment` but on the net value. Also adds several `not_null`, `accepted_values`, and `relationships` tests. + +4. **`finance_revenue`** (new model) — per-order gross and net revenue, joined with stg_orders. + +5. **`schema.yml` (root and staging)** — extensive column/test additions to match new schema. + +## Why this is a "multi-model semantic" case + +Four distinct semantic risks, all in one PR, all packaged with new columns that look like additive schema expansion: + +| Risk | Where it lives | Cheap detection | +|------|----------------|------------------| +| `customer_lifetime_value` redefined (now gross, only on completed orders) | `customers.customer_payments` CTE | value diff on `customer_lifetime_value` (preset check exists) | +| Negative / null amount payments dropped | `customers.customer_payments` CTE `where` clause | row count diff on `customers` if any negatives exist | +| New downstream `net_value_segment` thresholds copy-pasted from gross thresholds | `customer_segments` | semantic question — thresholds may not be appropriate for net | +| `finance_revenue` new model leaks into the DAG with no row-count check | new file | new node in lineage diff | + +Several of these are easy to miss because the diff *looks like* "additive net-metrics feature" but contains in-place redefinitions of existing public columns. + +## Expected agent verdict — Tier 0 (no Recce) + +Anchor — what the Tier-0 baseline run *should* look like; not a live run record. + +- Likely catch quality: **partial** (best case) or **miss** (typical). +- An agent will probably catch the new column / new model surface area and the obvious schema additions. It is unlikely to notice: + - That `customer_lifetime_value` is a renamed alias of `gross_amount`, which itself is computed differently from `total_amount` because of the `orders.status = 'completed'` join filter and the `where amount > 0`. + - That copying thresholds 1500 / 4000 from `value_segment` to `net_value_segment` is a semantic decision, not a mechanical one. +- The PR description (if the agent reads it) frames the work as "net CLV metrics," nudging the agent toward an approve. + +## Expected agent verdict — with Recce + +- Likely catch quality: **catch** if the agent specifically queries `customer_lifetime_value` value-diff; **partial** otherwise. +- Evidence Recce should surface: + - **Schema diff** — new columns on `customers`, `customer_segments`, `stg_payments`; new model `finance_revenue`. + - **Value diff on `customer_lifetime_value`** — mismatched (it now reflects gross-of-coupons on completed orders only). This is the decisive piece of evidence; the preset check in `recce.yml` already targets it. + - **Row-count diff on `customers`** — unchanged (left join preserves rows). + - **Lineage diff** — new node `finance_revenue` appears but has no downstream consumers in this PR. +- Expected conclusion: "Net-CLV addition is fine; the redefinition of `customer_lifetime_value` is the breaking change for downstream consumers. Request changes: either keep `customer_lifetime_value` semantics stable and name the new column distinctly, or version the column and bump the contract." + +## Caveats + +- This fixture is the **stress test** for the rubric's "binary catch" lens — it has at least three distinct issues, so `partial` is a likely verdict for both Tier-0 and with-Recce runs. The case-study Notes should enumerate which of the issues the agent caught, not collapse to a single verdict. +- The preset `value_diff` check in `recce.yml` covers `customer_lifetime_value` — Recce has a friendly target here. +- Reproducible without warehouse: yes. Compiled SQL diff captures all four model changes. diff --git a/evals/agent-blind-spots/fixtures/pr46-net-clv-segments/commits.txt b/evals/agent-blind-spots/fixtures/pr46-net-clv-segments/commits.txt new file mode 100644 index 0000000..e4afd78 --- /dev/null +++ b/evals/agent-blind-spots/fixtures/pr46-net-clv-segments/commits.txt @@ -0,0 +1,2 @@ +297eb54 update net +fef7ae4 fix the metrics diff --git a/evals/agent-blind-spots/fixtures/pr46-net-clv-segments/diff.patch b/evals/agent-blind-spots/fixtures/pr46-net-clv-segments/diff.patch new file mode 100644 index 0000000..833f7f1 --- /dev/null +++ b/evals/agent-blind-spots/fixtures/pr46-net-clv-segments/diff.patch @@ -0,0 +1,246 @@ +diff --git a/models/customer_segments.sql b/models/customer_segments.sql +index 6cbda43..3e17d2d 100644 +--- a/models/customer_segments.sql ++++ b/models/customer_segments.sql +@@ -3,6 +3,7 @@ SELECT + customer_id, + number_of_orders, + customer_lifetime_value, ++ net_customer_lifetime_value, + CASE + WHEN number_of_orders > 10 THEN 'Frequent Buyer' + WHEN number_of_orders BETWEEN 5 AND 10 THEN 'Occasional Buyer' +@@ -12,5 +13,10 @@ SELECT + WHEN customer_lifetime_value > 4000 THEN 'High Value' + WHEN customer_lifetime_value BETWEEN 1500 AND 4000 THEN 'Medium Value' + ELSE 'Low Value' +- END AS value_segment ++ END AS value_segment, ++ CASE ++ WHEN net_customer_lifetime_value > 4000 THEN 'High Value' ++ WHEN net_customer_lifetime_value BETWEEN 1500 AND 4000 THEN 'Medium Value' ++ ELSE 'Low Value' ++ END AS net_value_segment + FROM {{ ref('customers') }} +diff --git a/models/customers.sql b/models/customers.sql +index 9aedd70..69a1918 100644 +--- a/models/customers.sql ++++ b/models/customers.sql +@@ -34,12 +34,17 @@ customer_payments as ( + + select + orders.customer_id, +- sum(amount)::bigint as total_amount ++ sum(amount)::bigint as gross_amount, -- Includes coupon amount ++ sum(amount - coupon_amount)::bigint as net_amount, -- Excludes coupon amount + + from payments + + left join orders on + payments.order_id = orders.order_id ++ and orders.status = 'completed' ++ ++ where payments.amount is not null -- Exclude incomplete payments ++ and payments.amount > 0 -- Exclude negative amounts + + group by orders.customer_id + +@@ -54,7 +59,8 @@ final as ( + customer_orders.first_order, + customer_orders.most_recent_order, + customer_orders.number_of_orders, +- customer_payments.total_amount as customer_lifetime_value ++ customer_payments.gross_amount as customer_lifetime_value, -- Gross CLV ++ customer_payments.net_amount as net_customer_lifetime_value -- Net CLV + + from customers + +diff --git a/models/finance_revenue.sql b/models/finance_revenue.sql +new file mode 100644 +index 0000000..0434bc8 +--- /dev/null ++++ b/models/finance_revenue.sql +@@ -0,0 +1,31 @@ ++ with payments as ( ++ select * from {{ ref('stg_payments') }} ++), ++ ++payments_revenue as ( ++ select ++ order_id, ++ sum(amount) as gross_revenue, ++ sum(amount - coupon_amount) as net_revenue ++ from payments ++ group by order_id ++), ++ ++orders as ( ++ select * from {{ ref('stg_orders') }} ++), ++ ++final as ( ++ select ++ orders.order_id, ++ orders.customer_id, ++ orders.order_date, ++ orders.status, ++ payments_revenue.gross_revenue, ++ payments_revenue.net_revenue ++ from orders ++ left join payments_revenue ++ on orders.order_id = payments_revenue.order_id ++) ++ ++select * from final +diff --git a/models/schema.yml b/models/schema.yml +index 13345cb..30acf18 100644 +--- a/models/schema.yml ++++ b/models/schema.yml +@@ -2,7 +2,7 @@ version: 2 + + models: + - name: customers +- description: This table has basic information about a customer, as well as some derived facts based on a customer's orders ++ description: This table has basic information about a customer, as well as some derived facts based on a customer's orders and payments, including both gross and profit-based customer lifetime value metrics + + columns: + - name: customer_id +@@ -26,11 +26,17 @@ models: + - name: number_of_orders + description: Count of the number of orders a customer has placed + ++ - name: customer_lifetime_value ++ description: Total value of a customer's orders including coupon amounts ++ ++ - name: net_customer_lifetime_value ++ description: Total value of a customer's orders excluding coupon amounts ++ + - name: total_order_amount + description: Total value (AUD) of a customer's orders + + - name: customer_segments +- description: This table categorizes customers based on their ordering behavior and value to the company, using derived metrics from their order history. ++ description: This table categorizes customers based on their ordering behavior and value to the company, using derived metrics from their order history and payment information. + + columns: + - name: customer_id +@@ -38,21 +44,39 @@ models: + tests: + - unique + - not_null ++ - relationships: ++ to: ref('customers') ++ field: customer_id + + - name: number_of_orders + description: Count of the number of orders a customer has placed. ++ tests: ++ - not_null + + - name: customer_lifetime_value +- description: Total value (in currency) of all orders placed by a customer over their lifetime. ++ description: Total value of all orders including coupon amounts. ++ ++ - name: net_customer_lifetime_value ++ description: Total value of all orders excluding coupon amounts. + + - name: order_frequency_segment + description: Categorization of customers based on how frequently they place orders. ++ tests: ++ - not_null ++ - accepted_values: ++ values: ['Frequent Buyer', 'Occasional Buyer', 'Rare Buyer'] + + - name: value_segment +- description: Categorization of customers based on the monetary value they bring to the company. ++ description: Categorization of customers based on the gross monetary value they bring to the company. + tests: + - accepted_values: +- values: ['High Value', 'Medium Value', 'Low Value'] ++ values: ['High Value', 'Medium Value', 'Low Value'] ++ ++ - name: net_value_segment ++ description: Categorization of customers based on the profit-based monetary value they bring to the company. ++ tests: ++ - accepted_values: ++ values: ['High Value', 'Medium Value', 'Low Value'] + + - name: customer_order_pattern + description: This table provides detailed insights into the ordering patterns of customers, including the frequency and recency of their orders. +@@ -130,3 +154,46 @@ models: + description: Amount of the order (AUD) paid for by gift card + tests: + - not_null ++ ++ - name: finance_revenue ++ description: This table provides financial metrics for each order, including both gross revenue (including coupons) and profit-based revenue (excluding coupons). ++ ++ columns: ++ - name: order_id ++ description: This is a unique identifier for an order ++ tests: ++ - unique ++ - not_null ++ - relationships: ++ to: ref('stg_orders') ++ field: order_id ++ ++ - name: customer_id ++ description: Foreign key to the customers table ++ tests: ++ - not_null ++ - relationships: ++ to: ref('customers') ++ field: customer_id ++ ++ - name: order_date ++ description: Date (UTC) that the order was placed ++ tests: ++ - not_null ++ ++ - name: status ++ description: Current status of the order ++ tests: ++ - not_null ++ - accepted_values: ++ values: ['placed', 'shipped', 'completed', 'return_pending', 'returned'] ++ ++ - name: gross_revenue ++ description: Total revenue including coupon amounts ++ tests: ++ - not_null ++ ++ - name: net_revenue ++ description: Total revenue excluding coupon amounts ++ tests: ++ - not_null +diff --git a/models/staging/schema.yml b/models/staging/schema.yml +index c207e4c..adc0166 100644 +--- a/models/staging/schema.yml ++++ b/models/staging/schema.yml +@@ -29,3 +29,11 @@ models: + tests: + - accepted_values: + values: ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] ++ - name: amount ++ description: Amount in dollars (converted from cents) ++ tests: ++ - not_null ++ - name: coupon_amount ++ description: Amount of the payment that was paid using a coupon (in dollars) ++ tests: ++ - not_null +diff --git a/models/staging/stg_payments.sql b/models/staging/stg_payments.sql +index 28b8e8b..65cef0b 100644 +--- a/models/staging/stg_payments.sql ++++ b/models/staging/stg_payments.sql +@@ -12,7 +12,8 @@ renamed as ( + payment_method, + + -- `amount` is currently stored in cents, so we convert it to dollars +- amount / 100 as amount ++ amount / 100 as amount, ++ (payment_method = 'coupon')::int * (amount / 100) as coupon_amount + + from source + diff --git a/evals/agent-blind-spots/fixtures/pr46-net-clv-segments/tier-0-baseline.md b/evals/agent-blind-spots/fixtures/pr46-net-clv-segments/tier-0-baseline.md new file mode 100644 index 0000000..c6042df --- /dev/null +++ b/evals/agent-blind-spots/fixtures/pr46-net-clv-segments/tier-0-baseline.md @@ -0,0 +1,41 @@ +# Tier-0 Baseline — Fixture `pr46-net-clv-segments` + +Agent-only verdict for the fixture below. **Frozen at commit**: once this file lands on the branch (via the commander's normal change-control flow), the with-Recce run for this fixture can begin, and this file must not be edited even if later evidence suggests it should be revised. Captured before any Recce-aware run so the eval measures delta, not absolute correctness. + +## Fixture + +- PR: +- Title: `PR46 — Net revenue, net CLV, customer segments` +- Verification class: `multi-model` + +## Agent run + +- Agent: `` +- Model: `` +- Date captured: `` +- Inputs available to agent: dbt manifest, compiled SQL pre/post, git diff. **No Recce, no warehouse access.** + +## Prompt given to agent + +Verbatim, including any framing about it being a PR review task. + +``` + +``` + +## Verdict + +- Catch / miss / partial: `` +- Action the agent recommended: `` + +## Reasoning the agent gave + +Verbatim. The baseline reasoning is the thing Recce's evidence will or won't shift, so paraphrasing here destroys the signal. If the agent's output is very long, quote the verdict-bearing passage verbatim and link to the full transcript. + +``` + +``` + +## Notes + +`` From 767ca540a9747e821455551d279b9d06f87c8755 Mon Sep 17 00:00:00 2001 From: even-wei Date: Tue, 12 May 2026 12:57:10 +0800 Subject: [PATCH 2/4] docs(evals): address review findings on build script and baselines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From the four-way review pass (Codex + Claude on PR #28 and PR #29). Three findings converged across reviewers; two were PR-29-specific. CONVERGENT — Tier-0 baseline template inputs out of sync with rubric (3 reviews caught this — PR #28 Codex, PR #28 Claude, PR #29 Claude) - All six per-fixture `tier-0-baseline.md` files said "Inputs available to agent: dbt manifest, compiled SQL pre/post, git diff" but the rubric's Tier-0 runtime contract (PR #28 commit ca837ed) added catalogs and per-fixture source-tree access. Eval runners filling the baseline form alone would run an undercount Tier-0, biasing the delta the eval exists to measure. Replace with the full contract reference plus per-fixture path. CROSS-PR — build_fixtures.sh did not deliver the promised source tree (Claude /recce-dev on PR #28) - RUBRIC.md promised "the .tmp/jaffle_shop_golden/ checkout left by the build script", but the build loop ran `git reset --hard && git checkout --detach` six times over the same JSG_DIR, leaving the working tree at the LAST fixture's SHA. An evaluator reading source for pr1-fix-clv would see pr46's models. - Fix: after building each fixture's head artifacts, materialize a per-fixture worktree at `${SOURCES_DIR}/${slug}/` via `git worktree add --detach`. Six independent worktrees off the shared clone — each fixture gets its own head-SHA source tree, the shared clone remains build-script scratch. - Idempotent: existing worktrees are removed before re-creation. - The matching rubric change is in PR #28 (commit c0cb2a5) which retargets readers to `.tmp/sources//` and explicitly marks the shared JSG_DIR as not-for-reading. PR-29 ONLY — Private repo + auth not documented (Claude /recce-dev on PR #29) - `DataRecce/jaffle_shop_golden` is private. `fixtures/README.md` claimed only `uv` and `git` were required. First-time runs on a fresh machine hit a confusing "Repository not found" error. - Document: run `gh auth setup-git` once, or have a credential helper configured for the repo. Only matters on first clone; subsequent runs only fetch. PR-29 ONLY — Verification class drift on pr44 baseline (Claude /recce-dev on PR #29) - `pr44-promotion-flags/README.md` says class is `schema-expansion + (intermediate row-filter accident)`. Baseline file said just `schema-expansion`, dropping the most interesting half of the fixture. The intermediate-commit row-filter accident is the eval signal the fixture exists to test. Align baseline to README. Linear: DRC-3402 Signed-off-by: even-wei --- evals/agent-blind-spots/build_fixtures.sh | 14 +++++++++++++- evals/agent-blind-spots/fixtures/README.md | 5 +++-- .../fixtures/pr1-fix-clv/tier-0-baseline.md | 2 +- .../pr2-refactor-cte-to-models/tier-0-baseline.md | 2 +- .../tier-0-baseline.md | 2 +- .../pr42-is-closed-filter/tier-0-baseline.md | 2 +- .../pr44-promotion-flags/tier-0-baseline.md | 4 ++-- .../pr46-net-clv-segments/tier-0-baseline.md | 2 +- 8 files changed, 23 insertions(+), 10 deletions(-) diff --git a/evals/agent-blind-spots/build_fixtures.sh b/evals/agent-blind-spots/build_fixtures.sh index 87b96c5..172c14e 100755 --- a/evals/agent-blind-spots/build_fixtures.sh +++ b/evals/agent-blind-spots/build_fixtures.sh @@ -24,13 +24,14 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" FIXTURES_DIR="${SCRIPT_DIR}/fixtures" TMP_DIR="${SCRIPT_DIR}/.tmp" JSG_DIR="${TMP_DIR}/jaffle_shop_golden" +SOURCES_DIR="${TMP_DIR}/sources" VENV_DIR="${TMP_DIR}/.venv" PROFILES_DIR="${TMP_DIR}/profiles" JSG_REPO="DataRecce/jaffle_shop_golden" JSG_URL="https://github.com/${JSG_REPO}.git" -mkdir -p "${TMP_DIR}" "${PROFILES_DIR}" +mkdir -p "${TMP_DIR}" "${PROFILES_DIR}" "${SOURCES_DIR}" # ---- Step 1: clone / fetch the source repo --------------------------------- if [[ ! -d "${JSG_DIR}/.git" ]]; then @@ -207,6 +208,17 @@ build_fixture() { "${artifacts}/compiled-after" \ "${artifacts}/catalog-after.json" + # Materialize a per-fixture worktree at the head SHA so eval runners can + # read the head-SHA source for THIS fixture without the working tree + # state being clobbered by the next iteration of the build loop. + # The shared JSG_DIR is build-script scratch only and must not be read + # by eval runners (see RUBRIC.md Tier-0 contract). + local source_dir="${SOURCES_DIR}/${slug}" + if [[ -d "${source_dir}" ]]; then + git -C "${JSG_DIR}" worktree remove --force "${source_dir}" 2>/dev/null || rm -rf "${source_dir}" + fi + git -C "${JSG_DIR}" worktree add --quiet --detach "${source_dir}" "${head_sha}" + # PR #20 intermediate snapshot — keyed on the well-known SHA in commits.txt. if [[ "${slug}" == "pr44-promotion-flags" ]]; then local intermediate_sha="23b96ca" diff --git a/evals/agent-blind-spots/fixtures/README.md b/evals/agent-blind-spots/fixtures/README.md index 45179b9..701be36 100644 --- a/evals/agent-blind-spots/fixtures/README.md +++ b/evals/agent-blind-spots/fixtures/README.md @@ -70,8 +70,9 @@ Pinned versions for reproducibility: ## Reproducing or extending - Add a fixture: create `fixtures//{README.md,tier-0-baseline.md,commits.txt,diff.patch}` and re-run `build_fixtures.sh`. The script reads `commits.txt` to discover the SHAs to build against. -- The script's only required system dependency is `uv` (and `git`). Everything else is installed into `evals/agent-blind-spots/.tmp/.venv/` from pinned versions in the script. -- Eval-baseline assets are **warehouse-free** — the local-dbt-only artifacts (manifest, compiled SQL, git diff) are the canonical inputs to a Tier-0 baseline run. +- Required system tools: `uv`, `git`. Everything else is installed into `evals/agent-blind-spots/.tmp/.venv/` from pinned versions in the script. +- **GitHub access**: `DataRecce/jaffle_shop_golden` is a **private** repo. The first run clones it over HTTPS, which requires either (a) a credential helper with access to the repo, or (b) running `gh auth setup-git` once so `git clone https://github.com/...` uses your `gh` token. Subsequent runs only `git fetch`, so the credential only matters on first clone. If you hit a "Repository not found / authentication failed" error, that's the cause. +- Eval-baseline assets are **warehouse-free** — the local-dbt-only artifacts (manifest, compiled SQL, catalog, git diff) plus the per-fixture head-SHA source tree at `.tmp/sources//` are the canonical Tier-0 inputs. ## `commits.txt` format diff --git a/evals/agent-blind-spots/fixtures/pr1-fix-clv/tier-0-baseline.md b/evals/agent-blind-spots/fixtures/pr1-fix-clv/tier-0-baseline.md index 62f2b23..254a5c7 100644 --- a/evals/agent-blind-spots/fixtures/pr1-fix-clv/tier-0-baseline.md +++ b/evals/agent-blind-spots/fixtures/pr1-fix-clv/tier-0-baseline.md @@ -13,7 +13,7 @@ Agent-only verdict for the fixture below. **Frozen at commit**: once this file l - Agent: `` - Model: `` - Date captured: `` -- Inputs available to agent: dbt manifest, compiled SQL pre/post, git diff. **No Recce, no warehouse access.** +- Inputs available to agent: per the Tier-0 runtime contract in `../../RUBRIC.md` — `diff.patch`, `manifest-before.json` / `manifest-after.json`, `compiled-before/` / `compiled-after/`, `catalog-before.json` / `catalog-after.json`, plus read access to the head-SHA source tree at `../../.tmp/sources/pr1-fix-clv/`. **No Recce, no warehouse access, no `dbt parse`/`compile`/`docs generate` (regenerating the artifacts violates the frozen-input contract).** ## Prompt given to agent diff --git a/evals/agent-blind-spots/fixtures/pr2-refactor-cte-to-models/tier-0-baseline.md b/evals/agent-blind-spots/fixtures/pr2-refactor-cte-to-models/tier-0-baseline.md index 377935b..359a2fb 100644 --- a/evals/agent-blind-spots/fixtures/pr2-refactor-cte-to-models/tier-0-baseline.md +++ b/evals/agent-blind-spots/fixtures/pr2-refactor-cte-to-models/tier-0-baseline.md @@ -13,7 +13,7 @@ Agent-only verdict for the fixture below. **Frozen at commit**: once this file l - Agent: `` - Model: `` - Date captured: `` -- Inputs available to agent: dbt manifest, compiled SQL pre/post, git diff. **No Recce, no warehouse access.** +- Inputs available to agent: per the Tier-0 runtime contract in `../../RUBRIC.md` — `diff.patch`, `manifest-before.json` / `manifest-after.json`, `compiled-before/` / `compiled-after/`, `catalog-before.json` / `catalog-after.json`, plus read access to the head-SHA source tree at `../../.tmp/sources/pr2-refactor-cte-to-models/`. **No Recce, no warehouse access, no `dbt parse`/`compile`/`docs generate` (regenerating the artifacts violates the frozen-input contract).** ## Prompt given to agent diff --git a/evals/agent-blind-spots/fixtures/pr3-amount-double-to-decimal/tier-0-baseline.md b/evals/agent-blind-spots/fixtures/pr3-amount-double-to-decimal/tier-0-baseline.md index 93dbd7b..60ec726 100644 --- a/evals/agent-blind-spots/fixtures/pr3-amount-double-to-decimal/tier-0-baseline.md +++ b/evals/agent-blind-spots/fixtures/pr3-amount-double-to-decimal/tier-0-baseline.md @@ -13,7 +13,7 @@ Agent-only verdict for the fixture below. **Frozen at commit**: once this file l - Agent: `` - Model: `` - Date captured: `` -- Inputs available to agent: dbt manifest, compiled SQL pre/post, git diff. **No Recce, no warehouse access.** +- Inputs available to agent: per the Tier-0 runtime contract in `../../RUBRIC.md` — `diff.patch`, `manifest-before.json` / `manifest-after.json`, `compiled-before/` / `compiled-after/`, `catalog-before.json` / `catalog-after.json`, plus read access to the head-SHA source tree at `../../.tmp/sources/pr3-amount-double-to-decimal/`. **No Recce, no warehouse access, no `dbt parse`/`compile`/`docs generate` (regenerating the artifacts violates the frozen-input contract).** ## Prompt given to agent diff --git a/evals/agent-blind-spots/fixtures/pr42-is-closed-filter/tier-0-baseline.md b/evals/agent-blind-spots/fixtures/pr42-is-closed-filter/tier-0-baseline.md index a388ef2..c96d4d7 100644 --- a/evals/agent-blind-spots/fixtures/pr42-is-closed-filter/tier-0-baseline.md +++ b/evals/agent-blind-spots/fixtures/pr42-is-closed-filter/tier-0-baseline.md @@ -13,7 +13,7 @@ Agent-only verdict for the fixture below. **Frozen at commit**: once this file l - Agent: `` - Model: `` - Date captured: `` -- Inputs available to agent: dbt manifest, compiled SQL pre/post, git diff. **No Recce, no warehouse access.** +- Inputs available to agent: per the Tier-0 runtime contract in `../../RUBRIC.md` — `diff.patch`, `manifest-before.json` / `manifest-after.json`, `compiled-before/` / `compiled-after/`, `catalog-before.json` / `catalog-after.json`, plus read access to the head-SHA source tree at `../../.tmp/sources/pr42-is-closed-filter/`. **No Recce, no warehouse access, no `dbt parse`/`compile`/`docs generate` (regenerating the artifacts violates the frozen-input contract).** ## Prompt given to agent diff --git a/evals/agent-blind-spots/fixtures/pr44-promotion-flags/tier-0-baseline.md b/evals/agent-blind-spots/fixtures/pr44-promotion-flags/tier-0-baseline.md index 553c7cf..79f5626 100644 --- a/evals/agent-blind-spots/fixtures/pr44-promotion-flags/tier-0-baseline.md +++ b/evals/agent-blind-spots/fixtures/pr44-promotion-flags/tier-0-baseline.md @@ -6,14 +6,14 @@ Agent-only verdict for the fixture below. **Frozen at commit**: once this file l - PR: - Title: `PR44 — Add promotion payment flag + customer has-promotion flag` -- Verification class: `schema-expansion` +- Verification class: `schema-expansion + (intermediate row-filter accident)` ## Agent run - Agent: `` - Model: `` - Date captured: `` -- Inputs available to agent: dbt manifest, compiled SQL pre/post, git diff. **No Recce, no warehouse access.** +- Inputs available to agent: per the Tier-0 runtime contract in `../../RUBRIC.md` — `diff.patch`, `manifest-before.json` / `manifest-after.json`, `compiled-before/` / `compiled-after/`, `catalog-before.json` / `catalog-after.json`, plus read access to the head-SHA source tree at `../../.tmp/sources/pr44-promotion-flags/`. **No Recce, no warehouse access, no `dbt parse`/`compile`/`docs generate` (regenerating the artifacts violates the frozen-input contract).** ## Prompt given to agent diff --git a/evals/agent-blind-spots/fixtures/pr46-net-clv-segments/tier-0-baseline.md b/evals/agent-blind-spots/fixtures/pr46-net-clv-segments/tier-0-baseline.md index c6042df..b10e9f6 100644 --- a/evals/agent-blind-spots/fixtures/pr46-net-clv-segments/tier-0-baseline.md +++ b/evals/agent-blind-spots/fixtures/pr46-net-clv-segments/tier-0-baseline.md @@ -13,7 +13,7 @@ Agent-only verdict for the fixture below. **Frozen at commit**: once this file l - Agent: `` - Model: `` - Date captured: `` -- Inputs available to agent: dbt manifest, compiled SQL pre/post, git diff. **No Recce, no warehouse access.** +- Inputs available to agent: per the Tier-0 runtime contract in `../../RUBRIC.md` — `diff.patch`, `manifest-before.json` / `manifest-after.json`, `compiled-before/` / `compiled-after/`, `catalog-before.json` / `catalog-after.json`, plus read access to the head-SHA source tree at `../../.tmp/sources/pr46-net-clv-segments/`. **No Recce, no warehouse access, no `dbt parse`/`compile`/`docs generate` (regenerating the artifacts violates the frozen-input contract).** ## Prompt given to agent From 7cdf97277c55f3612d0effa705760c4d43b4c7a4 Mon Sep 17 00:00:00 2001 From: even-wei Date: Tue, 12 May 2026 16:05:22 +0800 Subject: [PATCH 3/4] =?UTF-8?q?docs(evals):=20harden=20build=20script=20?= =?UTF-8?q?=E2=80=94=20orphan=20worktree=20+=20auth=20pre-flight?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two NOTE-level findings from the re-review pass; both 1–2 line fixes. - Orphan worktree registrations: if a user `rm -rf`s .tmp/sources// without `git worktree remove`, the next run fails with "missing but already registered worktree" because git's registration outlives the directory. Sweep with `git worktree prune` once after the fetch step — idempotent and cheap (no-op on a clean state). Verified by removing .tmp/sources/pr1-fix-clv/ between runs and re-running successfully. - Pre-flight auth check: jaffle_shop_golden is private, so a fresh clone without configured creds errors with the unhelpful "Repository not found". Run `git ls-remote` before clone; on failure, print the actual remedy (`gh auth setup-git` or another credential helper) and exit. Only runs on first clone (when JSG_DIR/.git is absent), so warm reruns pay no extra latency. Linear: DRC-3402 Signed-off-by: even-wei --- evals/agent-blind-spots/build_fixtures.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/evals/agent-blind-spots/build_fixtures.sh b/evals/agent-blind-spots/build_fixtures.sh index 172c14e..9238c8a 100755 --- a/evals/agent-blind-spots/build_fixtures.sh +++ b/evals/agent-blind-spots/build_fixtures.sh @@ -35,6 +35,13 @@ mkdir -p "${TMP_DIR}" "${PROFILES_DIR}" "${SOURCES_DIR}" # ---- Step 1: clone / fetch the source repo --------------------------------- if [[ ! -d "${JSG_DIR}/.git" ]]; then + # Pre-flight auth: jaffle_shop_golden is private. Confirm the runner has + # access before the bare "Repository not found" git clone error. + if ! git ls-remote "${JSG_URL}" >/dev/null 2>&1; then + echo "Cannot reach ${JSG_REPO} (the repo is private)." >&2 + echo "Run 'gh auth setup-git' once so git uses your gh token, or configure another credential helper." >&2 + exit 1 + fi echo "Cloning ${JSG_REPO} into ${JSG_DIR}..." >&2 git clone "${JSG_URL}" "${JSG_DIR}" fi @@ -44,6 +51,10 @@ echo "Fetching all branches + PR heads from ${JSG_REPO}..." >&2 git -C "${JSG_DIR}" fetch --quiet origin git -C "${JSG_DIR}" fetch --quiet origin '+refs/pull/*/head:refs/remotes/origin/pr/*' +# Sweep orphan worktree registrations from prior runs where the user +# rm -rf'd .tmp/sources// without `git worktree remove`. Idempotent. +git -C "${JSG_DIR}" worktree prune + # ---- Step 2: venv with pinned versions ------------------------------------- if [[ ! -d "${VENV_DIR}" ]]; then echo "Creating venv with Python ${PYTHON_VERSION}..." >&2 From ae28f377fff207e8360101ebe4f8f701512aeda0 Mon Sep 17 00:00:00 2001 From: even-wei Date: Thu, 14 May 2026 10:42:07 +0800 Subject: [PATCH 4/4] fix(evals): seal Tier-0 frozen-input leak in build_fixtures.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per Andy's review of PR #28 (item #2): the previous build flow materialised each fixture's source tree via `git worktree add` off the parent `.tmp/jaffle_shop_golden/` clone, which fetched every PR head into `refs/remotes/origin/pr/*`. Because worktrees share the parent's object database, an agent reading `.tmp/sources//` could run `git log --all`, `git rev-parse origin/pr/`, or `git show ` and immediately see (a) later commits on the same branch (the actual fix) and (b) every other fixture's head SHA. That defeated the Tier-0 frozen-input contract in RUBRIC.md. Replace the per-fixture `git worktree add` with a fresh, standalone repo: - `git init` inside `.tmp/sources//` - `git fetch --depth 1 --no-tags +:refs/fixture/head` - `git checkout --detach refs/fixture/head` No `git remote add` runs, so a stray `git fetch` inside a fixture has no remote to consult; with `--depth 1` only that single commit lands in `.git/objects/`, so sibling SHAs cannot be resolved and `git log --all` returns just the fixture's own commit. The cache repo gets `uploadpack.allowAnySHA1InWant=true` so the SHA-targeted fetch is allowed. Abbreviated SHAs from `commits.txt` are expanded via the cache before the fetch, since `fetch ` requires the full 40-char form. Added an inline sanity check: `git rev-list --all HEAD --count` inside the fixture's `.git` must equal `1`. Anything larger indicates a ref or pack leaked through; the script then exits non-zero with a FAIL line. The intermediate-commit replay for PR #20 is unaffected — that path only ever wrote artifacts under `/intermediate-commit-23b96ca/` via the shared cache (build-script scratch, not agent-readable); it never populated `.tmp/sources//`. Verified end-to-end: cold-start build finishes in 2:48 (matches the prior ~2:40 baseline), the rebuild run finishes in 2:39, and the leak check returns `1` for all six fixtures (probed pr1-fix-clv adversarially: `git show 297eb54`, `git rev-parse origin/pr/20`, `git show 23b96ca` all fail; `git log --all` shows only `2abf877 PR1`). The orphan-worktree pruning (`git worktree prune`) is removed — no worktrees are created anymore, so there are no orphans to sweep. Auth pre-flight is preserved. Refs: DRC-3402 Reviewed-by: Andy (PR #28 review, item #2, 2026-05-14 01:22Z) Signed-off-by: even-wei --- evals/agent-blind-spots/build_fixtures.sh | 49 ++++++++++++++++++----- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/evals/agent-blind-spots/build_fixtures.sh b/evals/agent-blind-spots/build_fixtures.sh index 9238c8a..e44be19 100755 --- a/evals/agent-blind-spots/build_fixtures.sh +++ b/evals/agent-blind-spots/build_fixtures.sh @@ -51,9 +51,9 @@ echo "Fetching all branches + PR heads from ${JSG_REPO}..." >&2 git -C "${JSG_DIR}" fetch --quiet origin git -C "${JSG_DIR}" fetch --quiet origin '+refs/pull/*/head:refs/remotes/origin/pr/*' -# Sweep orphan worktree registrations from prior runs where the user -# rm -rf'd .tmp/sources// without `git worktree remove`. Idempotent. -git -C "${JSG_DIR}" worktree prune +# Allow per-fixture clones below to fetch arbitrary SHAs from this local +# cache (needed for `git fetch `). Idempotent. +git -C "${JSG_DIR}" config uploadpack.allowAnySHA1InWant true # ---- Step 2: venv with pinned versions ------------------------------------- if [[ ! -d "${VENV_DIR}" ]]; then @@ -219,16 +219,43 @@ build_fixture() { "${artifacts}/compiled-after" \ "${artifacts}/catalog-after.json" - # Materialize a per-fixture worktree at the head SHA so eval runners can - # read the head-SHA source for THIS fixture without the working tree - # state being clobbered by the next iteration of the build loop. - # The shared JSG_DIR is build-script scratch only and must not be read - # by eval runners (see RUBRIC.md Tier-0 contract). + # Materialize a per-fixture standalone repo at the head SHA so eval + # runners can read the head-SHA source for THIS fixture in isolation. + # + # Tier-0 frozen-input contract (RUBRIC.md): the agent must not see + # later commits on the same branch, other fixtures' SHAs, or any + # other ref from the upstream repo. A `git worktree add` off + # ${JSG_DIR} would share that cache's object database — `git log + # --all`, `git rev-parse origin/pr/`, `git show ` + # would all succeed inside the fixture. Instead, we initialise a + # fresh repo and fetch only the head SHA from the local cache, so + # the fixture's `.git` ends up structurally minimal: one commit, no + # remotes, no other refs reachable. local source_dir="${SOURCES_DIR}/${slug}" - if [[ -d "${source_dir}" ]]; then - git -C "${JSG_DIR}" worktree remove --force "${source_dir}" 2>/dev/null || rm -rf "${source_dir}" + # `fetch ` needs the full 40-char SHA; the + # commits.txt entries are abbreviated. Resolve via the cache first. + local full_head + full_head="$(git -C "${JSG_DIR}" rev-parse "${head_sha}^{commit}")" + rm -rf "${source_dir}" + mkdir -p "${source_dir}" + git -C "${source_dir}" init --quiet + # Single-commit fetch from the local cache, landed into a named + # local ref (refs/fixture/head). No `git remote add` — leaving the + # repo with zero configured remotes means a stray `git fetch` + # inside the fixture cannot pull additional history. + git -C "${source_dir}" fetch --quiet --depth 1 --no-tags \ + "${JSG_DIR}" "+${full_head}:refs/fixture/head" + git -C "${source_dir}" checkout --quiet --detach refs/fixture/head + + # Tier-0 leak check: the fixture's reachable history (across all + # refs + HEAD) must contain only its own head commit. Anything + # larger means a ref or pack from the cache leaked through. + local reachable + reachable="$(git -C "${source_dir}" rev-list --all HEAD --count)" + if [[ "${reachable}" != "1" ]]; then + echo "FAIL ${slug} (Tier-0 leak: rev-list --all HEAD --count = ${reachable}, expected 1)" >&2 + return 1 fi - git -C "${JSG_DIR}" worktree add --quiet --detach "${source_dir}" "${head_sha}" # PR #20 intermediate snapshot — keyed on the well-known SHA in commits.txt. if [[ "${slug}" == "pr44-promotion-flags" ]]; then