From c563bcd92b55b2525c1fec67819c57aac8314d66 Mon Sep 17 00:00:00 2001 From: Ethan Date: Tue, 15 Sep 2026 14:03:24 +0800 Subject: [PATCH 1/8] docs: make AGENTS.md the instruction source of truth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex reads only AGENTS.md and has no import mechanism, so the full text moves there; CLAUDE.md becomes a one-line @AGENTS.md import, which Claude Code expands to the same content. Carries over the layer-dependency diagram that the draft had rewritten as prose, and replaces the draft's Tool Bridge section — which described a bridge that was never built. Co-Authored-By: Claude Sonnet 5 --- .claude/rules/engineering-infra.md | 1 + .gitignore | 3 - AGENTS.md | 137 +++++++++++++++++++++++++++++ CLAUDE.md | 71 +-------------- scripts/sanitize.sh | 1 + 5 files changed, 141 insertions(+), 72 deletions(-) create mode 100644 AGENTS.md diff --git a/.claude/rules/engineering-infra.md b/.claude/rules/engineering-infra.md index c933f118..cf969bac 100644 --- a/.claude/rules/engineering-infra.md +++ b/.claude/rules/engineering-infra.md @@ -1,6 +1,7 @@ --- paths: - "CLAUDE.md" + - "AGENTS.md" - "sieval/**/CLAUDE.md" - "CONTRIBUTING.md" - ".claude/rules/*.md" diff --git a/.gitignore b/.gitignore index f4551e78..45dd0942 100644 --- a/.gitignore +++ b/.gitignore @@ -171,9 +171,6 @@ tmp/ .claude/commands/ .claude/skills/ .claude/worktrees/ - -# Codex -AGENTS.md .claude/settings.local.json # Local analysis artifacts diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..5a8abba4 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,137 @@ +# Repository Guidelines + +SiEval is a **model delivery quality verification system**: eval-side knowledge constraining and verifying the entire model delivery pipeline (training → conversion → infer → evaluation). + +## Architecture Principles + +1. **AI-friendly** — easy for humans, with enough detail and configurability for AI. +2. **Explicit over implicit** — benefits both humans and AI. +3. **Moderate abstraction** — do not abstract ahead of time. Extract on **coupling** (sites must change together to preserve a contract), not on call count. +4. **SOLID + KISS** — single responsibility, simple and clear, easy to extend. +5. **Minimal change surface + courage to rebuild** — reuse what works; but be willing to start over when a better overall solution exists. + +## Reproducibility + +Precise reproducibility is a product contract, not a nicety. + +- Safety guards (e.g. `--resume` strict match) ship strict-only. No `--force-*` flags, no bypass env vars, no "(future)" escape-hatch hints in errors. +- `--resume` match scope is narrowed, not bypassed: only fields that touch neither sample data nor any persisted artifact may differ across a resume — pure scheduling (concurrency, shard-I/O, write buffers), console-only progress, `auto_resume`, the non-serializable `stage_meta` hooks, and top-level `result_dir`. Everything affecting on-disk content stays strict (sampling/seeds, `max_iterations`, `shard_samples`, `record_*`, `max_retries`, `profile_*`, `detect_anomalies*`, the `progress.json` dump, and `runner_config.result_dir`, which picks the artifact directory), as does `infer_plans.yaml`. +- Recovery: "start fresh" or "match the invocation". Escape-hatch proposals must re-justify the contract, not just add a flag. + +## Project Structure & Module Organization + +- Core library code lives in `sieval/`. + - `sieval/core/`: execution engine (models, runners, persistence, profiling). + - `sieval/datasets/`: dataset wrappers/loaders. + - `sieval/tasks/`: task implementations by domain. + - `sieval/community/`: third-party/reference evaluation code; treat as vendor-style code and avoid edits unless explicitly required. +- Layer boundaries (orchestration → engine): + + ```text + cli/ → orchestration layer, depends on all modules + infer/ → can depend on core; NOT on tasks/datasets + tasks/ → depends on core + datasets + community + datasets/ → depends on core + community + core/ → zero upper-layer dependencies (independently publishable) + community/ → third-party evaluation adaptations + ``` + + - **Do not put task-specific logic into `core/`.** Each layer has its own `CLAUDE.md` with layer-specific import constraints. +- Tests are organized by purpose under `tests/`: + - `tests/unit/`, `tests/integration/`, `tests/acceptance/`, `tests/performance/`. + - `tests/tasks/` are manual scripts that may call real APIs (never CI). + +## Build, Test, and Development Commands + +- Platform: Unix (Linux, macOS) — Windows is not supported. Language: Python ≥ 3.12. +- CLI: `typer` — top-level shortcuts `sieval run`, `sieval eval`; resource groups `dataset`, `task`, `infer`, `leaderboard` (e.g. `sieval dataset list`, `sieval infer start`). +- Install deps: + - `pdm install` + - Dev/test setup: `pdm install -G dev -G test` + - Task-specific extras (install on demand): `pip install -e ".[math,ifeval,drop,t-eval,molecule,genomics,beacon]"` (or a subset) + - Lock procedures: see `.claude/rules/deps.md` (always `pdm lock --update-reuse`, never bare `pdm lock`) +- Run tests: + - Full suite: `pdm run python -m pytest -q` + - Unit + integration (coverage gate >=95%): `pdm run python -m pytest tests/unit tests/integration --cov --cov-fail-under=95 -q` + - Acceptance (release gate): `pdm run python -m pytest tests/acceptance -q -s` + - Performance (non-stress): `pdm run python -m pytest tests/performance -q -m "not stress"` +- Quality checks: + - `pdm run ruff check` + - `pdm run ruff format` + - `pdm run ty check sieval/` + - `pdm run python scripts/sync_package_stubs.py --check` + - `pre-commit run -a` + +## Architecture Guardrails + +- Preserve the async staged pipeline design: no synchronous rewrite, no global stage barriers. +- Keep resumability semantics intact (`auto_resume`, per-sample failure handling, bounded iterations). +- Persistence is append-only and sharded; disk state is source of truth. Do not replace with monolithic or memory-only storage. +- Respect record-mode behavior: `record_each_stage=True` is the fail-safe default and must remain resume-compatible. + +## Coding Style & Naming Conventions + +- Python 3.12+, 4-space indentation, PEP 8 naming (`snake_case` functions, `PascalCase` classes). +- Keep modules focused and typed; daily type checking uses `ty` (`ty check sieval/`); `mypy` (`[tool.mypy] strict = true`) is available but secondary. +- Use Ruff for linting/formatting; avoid manual style drift. +- Prefer explicit, readable test case names and concise comments for complex parameterized cases. +- Package stubs in `sieval/tasks/__init__.pyi` and `sieval/datasets/__init__.pyi` are generated; do not hand-edit. Run `scripts/sync_package_stubs.py` after adding/removing exported tasks or datasets. +- AI-generated source files must include `AI-Generated Code - ()` as the **last line** of the module-level docstring. +- Before deleting code, verify no other call sites depend on it. +- Do not use `from __future__ import annotations`. +- Import policy: + - Datasets: use package-level imports from `sieval.datasets` for official implementations. + - Tasks: use package-level imports from `sieval.tasks` for official built-in tasks. + - Custom/community tasks should prefer full module paths (for example `sieval.tasks..MyTask`) to avoid naming conflicts. + - `runners`: canonical path is `sieval.core.runners` (`TaskRunner`, `MultiTaskRunner`). + - `session`: canonical path is `sieval.cli.leaderboard.session` (`EvalSession`, `arun_session`). + - `validation`: canonical path is `sieval.cli.validation` (`validate_eval_config`, `run_dry_run`). + - `resolution`: canonical path is `sieval.cli.resolution` (`resolve_task_class`, `resolve_dataset_class`, `derive_model_type`). Reaching them via `session` resolves, but is not API. `cli/infer/` must not import `cli/leaderboard/`. + - Within the same package/subpackage: use relative imports (`from .foo import Bar`). + - Cross-package (including parent): use absolute imports (`from sieval.core.models import ...`). + - Imports imply public API. Cross-module `from sieval.x.y import _foo` in production code is a smell — promote the name or redesign the call site. + - Private modules (`_*.py`) are **protected** — accessible only within their own package subtree. Same-package siblings may use `from ._x import Y`; descendants may import via absolute path into the ancestor's private module. Peer-subpackage or out-of-subtree access is forbidden. Tests are the carve-out for both rules. + +## Testing Guidelines + +- Always run tests after making changes. See `tests/README.md` and `.claude/rules/tests.md` for details. +- `tests/unit/` directory structure must mirror `sieval/` — e.g. `sieval/core/runners/foo.py` → `tests/unit/core/runners/test_foo.py`. +- Framework: `pytest`; async tests must use `@pytest.mark.anyio` (not `pytest-asyncio`). +- `stress` marker is defined in `pyproject.toml`; exclude by default unless intentionally profiling. +- Mutation testing uses `mutmut` (on-demand, not per-commit): mutate `sieval/core` against `tests/unit` (see `[tool.mutmut]`). +- Mutation score should stay `>=70%`; surviving mutants indicate missing assertions and should be killed by improving tests. +- Common mutation commands: `mutmut run`, `mutmut results`, `mutmut show`. +- When changing core runner/model behavior, add/adjust unit tests first, then verify with relevant integration tests. +- Prefer `tmp_path` and deterministic mocks over hardcoded paths and wall-clock sleeps. + +## Commit & Pull Request Guidelines + +- Always rebase onto the latest `main` before submitting a PR. +- **MUST** read `.github/PULL_REQUEST_TEMPLATE.md` before creating any PR, and fill in all applicable sections. Do not create a PR without reading the template first. +- **MUST** read the matching template from `.github/ISSUE_TEMPLATE/` before creating any issue (bug-report, feature-request, rfc, etc.), and use the template's structure. Do not create an issue without reading the template first. +- Commit messages follow Conventional Commits (enforced by `conventional-pre-commit`): + - Examples: `feat: ...`, `fix: ...`, `refactor: ...`, `chore: ...`. +- Keep commits scoped and reviewable; include tests with code changes. +- PRs should include: + - What changed and why. + - Risk/regression notes. + - Validation evidence (exact commands run and results). + +## Agent Tooling + +This file is the single source of truth for shared conventions, and the only +instruction file all three supported agents read. `CLAUDE.md` is a one-line +import of it. Keep it under 32 KiB: Codex truncates project docs past that +byte count silently, so content near the end simply stops applying. + +- **Layer and scoped rules** — `sieval/*/CLAUDE.md` and `.claude/rules/*.md`. + Claude Code loads these automatically; opencode and Codex do not, so the + generated map below tells them what to read and when. +- **Skills** — `.agents/skills//SKILL.md` is the source of truth, with + symlinks at `.claude/skills/` and `.opencode/commands/.md`. + Never edit through a symlink. +- **Post-edit checks** — `scripts/post_edit_checks.py` holds the behavior; + `.claude/settings.json`, `.opencode/plugins/post-edit.ts`, and + `.codex/config.toml` each dispatch to it. +- **Codex users:** project-scoped `.codex/` layers (including hooks) load only + for projects you have marked trusted, and hooks require `features.hooks`. diff --git a/CLAUDE.md b/CLAUDE.md index 9b7d591a..da0659fb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,69 +1,2 @@ -# SiEval — Project-Wide Guidelines - -SiEval is a **model delivery quality verification system**: eval-side knowledge constraining and verifying the entire model delivery pipeline (training → conversion → infer → evaluation). - -## Architecture Principles - -1. **AI-friendly** — easy for humans, with enough detail and configurability for AI. -2. **Explicit over implicit** — benefits both humans and AI. -3. **Moderate abstraction** — do not abstract ahead of time. Extract on **coupling** (sites must change together to preserve a contract), not on call count. -4. **SOLID + KISS** — single responsibility, simple and clear, easy to extend. -5. **Minimal change surface + courage to rebuild** — reuse what works; but be willing to start over when a better overall solution exists. - -## Reproducibility - -Precise reproducibility is a product contract, not a nicety. - -* Safety guards (e.g. `--resume` strict match) ship strict-only. No `--force-*` flags, no bypass env vars, no "(future)" escape-hatch hints in errors. -* `--resume` match scope is narrowed, not bypassed: only fields that touch neither sample data nor any persisted artifact may differ across a resume — pure scheduling (concurrency, shard-I/O, write buffers), console-only progress, `auto_resume`, the non-serializable `stage_meta` hooks, and top-level `result_dir`. Everything affecting on-disk content stays strict (sampling/seeds, `max_iterations`, `shard_samples`, `record_*`, `max_retries`, `profile_*`, `detect_anomalies*`, the `progress.json` dump, and `runner_config.result_dir`, which picks the artifact directory), as does `infer_plans.yaml`. -* Recovery: "start fresh" or "match the invocation". Escape-hatch proposals must re-justify the contract, not just add a flag. - -## Toolchain - -* Platform: Unix (Linux, macOS) — Windows is not supported -* Language: Python ≥ 3.12 -* Package manager: `pdm` — see `.claude/rules/deps.md` for lock procedures -* Formatting & lint: `ruff`; type checking: `ty` (primary), `mypy` (strict mode, secondary) -* Tests: `pytest` -* CLI: `typer` — top-level shortcuts `sieval run`, `sieval eval`; resource groups `dataset`, `task`, `infer`, `leaderboard` (e.g. `sieval dataset list`, `sieval infer start`) - -## Layer Boundaries & Dependencies - -```text -cli/ → orchestration layer, depends on all modules -infer/ → can depend on core; NOT on tasks/datasets -tasks/ → depends on core + datasets + community -datasets/ → depends on core + community -core/ → zero upper-layer dependencies (independently publishable) -community/ → third-party evaluation adaptations -``` - -Each layer has its own `CLAUDE.md` with layer-specific import constraints. **Do not put task-specific logic into `core/`.** - -## Import Policy - -* `datasets`, `tasks`: package-level imports are the official entrypoint (`from sieval.tasks import ...`) -* `runners`: canonical path is `sieval.core.runners` (`TaskRunner`, `MultiTaskRunner`). -* `session`: canonical path is `sieval.cli.leaderboard.session` (`EvalSession`, `arun_session`). -* `validation`: canonical path is `sieval.cli.validation` (`validate_eval_config`, `run_dry_run`). -* `resolution`: canonical path is `sieval.cli.resolution` (`resolve_task_class`, `resolve_dataset_class`, `derive_model_type`). Reaching them via `session` resolves, but is not API. `cli/infer/` must not import `cli/leaderboard/`. -* Same package: relative imports. Cross-package: absolute imports. -* Imports imply public API. Cross-module `from sieval.x.y import _foo` in production code is a smell — promote the name or redesign the call site. -* Private modules (`_*.py`) are **protected** — accessible only within their own package subtree. Same-package siblings may use `from ._x import Y`; descendants may import via absolute path into the ancestor's private module. Peer-subpackage or out-of-subtree access is forbidden. Tests are the carve-out for both rules. - -## Test Rules - -* Always run tests after making changes. See [tests/README.md](tests/README.md) and `.claude/rules/tests.md` for details. -* `tests/unit/` directory structure must mirror `sieval/` — e.g. `sieval/core/runners/foo.py` → `tests/unit/core/runners/test_foo.py`. - -## PR / Issue Hygiene - -* Always rebase onto the latest `main` before submitting a PR. -* **MUST** read `.github/PULL_REQUEST_TEMPLATE.md` before creating any PR, and fill in all applicable sections. Do not create a PR without reading the template first. -* **MUST** read the matching template from `.github/ISSUE_TEMPLATE/` before creating any issue (bug-report, feature-request, rfc, etc.), and use the template's structure. Do not create an issue without reading the template first. - -## Code Conventions - -* AI-generated source files must include `AI-Generated Code - ()` as the **last line** of the module-level docstring. -* Before deleting code, verify no other call sites depend on it. -* Do not use `from __future__ import annotations`. + +@AGENTS.md diff --git a/scripts/sanitize.sh b/scripts/sanitize.sh index b8257bc8..226e61ef 100755 --- a/scripts/sanitize.sh +++ b/scripts/sanitize.sh @@ -39,6 +39,7 @@ ALLOWLIST=( "docs/" # local design/planning docs "tests/" # test fixtures use example IPs/paths "CLAUDE.md" # project guidelines + "AGENTS.md" # project guidelines (source of truth) "\.git/" # git internals "127\.0\.0\.1" # localhost is fine "0\.0\.0\.0" # bind-all is fine From 71271c0da5ce60f43b0410c2e91e94d1de3fc68f Mon Sep 17 00:00:00 2001 From: Ethan Date: Tue, 15 Sep 2026 14:17:44 +0800 Subject: [PATCH 2/8] feat(scripts): generate the agent rule map into AGENTS.md Claude Code auto-loads sieval/*/CLAUDE.md and .claude/rules/*.md; opencode and Codex do neither, so they need an explicit index. Generating it from the rules' own paths: frontmatter keeps the copy from going stale, and check_agent_rule_map fails CI when it does. Also asserts the 32 KiB budget Codex silently truncates past. Co-Authored-By: Claude Sonnet 5 --- AGENTS.md | 27 ++++ scripts/check_preflight.py | 44 ++++++ scripts/sync_agent_rules.py | 157 ++++++++++++++++++++ tests/unit/scripts/test_check_preflight.py | 3 +- tests/unit/scripts/test_sync_agent_rules.py | 124 ++++++++++++++++ 5 files changed, 354 insertions(+), 1 deletion(-) create mode 100644 scripts/sync_agent_rules.py create mode 100644 tests/unit/scripts/test_sync_agent_rules.py diff --git a/AGENTS.md b/AGENTS.md index 5a8abba4..73b2d2e9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -127,6 +127,33 @@ byte count silently, so content near the end simply stops applying. - **Layer and scoped rules** — `sieval/*/CLAUDE.md` and `.claude/rules/*.md`. Claude Code loads these automatically; opencode and Codex do not, so the generated map below tells them what to read and when. + + + +Read the matching file before editing a path it covers. Claude Code +loads these automatically; opencode and Codex need this map. + +| When editing | Read first | +| --- | --- | +| `sieval/cli/**` | `sieval/cli/CLAUDE.md` | +| `sieval/community/**` | `sieval/community/CLAUDE.md` | +| `sieval/core/**` | `sieval/core/CLAUDE.md` | +| `sieval/datasets/**` | `sieval/datasets/CLAUDE.md` | +| `sieval/infer/**` | `sieval/infer/CLAUDE.md` | +| `sieval/tasks/**` | `sieval/tasks/CLAUDE.md` | +| `sieval/**/*.py`, `tests/**/*.py` | `.claude/rules/code-review.md` | +| `sieval/**/*.py` | `.claude/rules/core-utils.md` | +| `sieval/datasets/**/*.py` | `.claude/rules/datasets.md` | +| `pyproject.toml`, `pdm.lock` | `.claude/rules/deps.md` | +| `CLAUDE.md`, `AGENTS.md`, `sieval/**/CLAUDE.md`, `CONTRIBUTING.md`, `.claude/rules/*.md`, `scripts/check_*.py`, `.pre-commit-config.yaml` | `.claude/rules/engineering-infra.md` | +| `sieval/infer/**/*.py` | `.claude/rules/infer.md` | +| `sieval/tasks/**/*.py`, `sieval/core/tasks/**/*.py` | `.claude/rules/records.md` | +| `scripts/*.py` | `.claude/rules/scripts.md` | +| `sieval/tasks/**/*.py` | `.claude/rules/tasks.md` | +| `tests/**/*.py` | `.claude/rules/tests.md` | + + + - **Skills** — `.agents/skills//SKILL.md` is the source of truth, with symlinks at `.claude/skills/` and `.opencode/commands/.md`. Never edit through a symlink. diff --git a/scripts/check_preflight.py b/scripts/check_preflight.py index 47ef77fd..d6ee030d 100644 --- a/scripts/check_preflight.py +++ b/scripts/check_preflight.py @@ -1209,6 +1209,7 @@ class PreflightRunner: "check_imports", "check_examples", "check_meta_index_sync", + "check_agent_rule_map", "check_mutmut_config", "check_version", ] @@ -3063,6 +3064,49 @@ def check_meta_index_sync(self) -> list[CheckResult]: ) ] + def check_agent_rule_map(self) -> list[CheckResult]: + """The rule map in ``AGENTS.md`` must match the live rule inventory. + + opencode and Codex reach the layer and scoped rules only through this + map. A stale map does not fail loudly for them — it silently points at + rules that moved, or omits ones that were added. + """ + script = self.project_root / "scripts" / "sync_agent_rules.py" + if not script.exists(): + return [ + CheckResult( + "FAIL", + "check_agent_rule_map", + f"script not found: {script}", + ) + ] + + result = subprocess.run( + [sys.executable, str(script), "--check"], + capture_output=True, + text=True, + cwd=self.project_root, + ) + if result.returncode == 0: + return [ + CheckResult( + "PASS", + "check_agent_rule_map", + "AGENTS.md rule map matches the live rule inventory", + ) + ] + + message = (result.stderr or result.stdout).strip().splitlines() + return [ + CheckResult( + "FAIL", + "check_agent_rule_map", + "AGENTS.md rule map is out of date; " + "run `python scripts/sync_agent_rules.py` to regenerate", + message, + ) + ] + def check_imports(self) -> list[CheckResult]: script = self.project_root / "scripts" / "check_layer_imports.py" if not script.exists(): diff --git a/scripts/sync_agent_rules.py b/scripts/sync_agent_rules.py new file mode 100644 index 00000000..8b6f1e34 --- /dev/null +++ b/scripts/sync_agent_rules.py @@ -0,0 +1,157 @@ +"""Regenerate the agent rule map inside AGENTS.md. + +Claude Code auto-loads `sieval/*/CLAUDE.md` (on demand) and `.claude/rules/*.md` +(by `paths:` glob). opencode and Codex do neither: opencode only walks upward +from the working directory, and Codex reads only AGENTS.md files on the +root-to-cwd path. Both therefore need an explicit index. + +Hand-writing that index would duplicate the `paths:` frontmatter that already +declares each rule's scope, and would go stale with no signal. Generating it +keeps the copy honest, the same way sync_meta_index.py does for the registry. + +AI-Generated Code - Claude Sonnet 5 (Anthropic) +""" + +import argparse +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + +BEGIN = "" +END = "" + +# Codex's project_doc_max_bytes default. Past this, it truncates silently: +# instructions near the end of the file simply stop applying, with no warning. +MAX_BYTES = 32768 + + +def _parse_paths_frontmatter(text: str) -> list[str]: + """Return the `paths:` globs from a rule file's YAML frontmatter. + + Hand-rolled rather than via PyYAML: this script runs from pre-commit and + preflight, where an import-light dependency footprint matters, and the + frontmatter shape here is fixed (a `paths:` key over a list of strings). + """ + if not text.startswith("---"): + return [] + end = text.find("\n---", 3) + if end == -1: + return [] + block = text[3:end] + + globs: list[str] = [] + in_paths = False + for raw in block.splitlines(): + line = raw.rstrip() + if not line: + continue + if line.strip() == "paths:": + in_paths = True + continue + if in_paths: + stripped = line.strip() + if stripped.startswith("- "): + globs.append(stripped[2:].strip().strip('"').strip("'")) + continue + # A non-list line at the same or lower indent ends the paths block. + if not line.startswith(" "): + in_paths = False + return globs + + +def render_block(root: Path) -> str: + """Render the rule map from layer files and rule frontmatter.""" + rows: list[tuple[str, str]] = [] + + for layer_file in sorted((root / "sieval").glob("*/CLAUDE.md")): + layer = layer_file.parent.name + rows.append((f"`sieval/{layer}/**`", f"`sieval/{layer}/CLAUDE.md`")) + + rules_dir = root / ".claude" / "rules" + for rule_file in sorted(rules_dir.glob("*.md")) if rules_dir.is_dir() else []: + globs = _parse_paths_frontmatter(rule_file.read_text(encoding="utf-8")) + rel = rule_file.relative_to(root).as_posix() + scope = ", ".join(f"`{g}`" for g in globs) if globs else "_(always)_" + rows.append((scope, f"`{rel}`")) + + lines = [ + BEGIN, + "", + "Read the matching file before editing a path it covers. Claude Code", + "loads these automatically; opencode and Codex need this map.", + "", + "| When editing | Read first |", + "| --- | --- |", + ] + lines += [f"| {scope} | {target} |" for scope, target in rows] + lines += ["", END] + return "\n".join(lines) + + +def render_document(root: Path) -> str: + """Return AGENTS.md with a freshly generated block.""" + agents_path = root / "AGENTS.md" + if not agents_path.exists(): + raise SystemExit(f"not found: {agents_path}") + + current = agents_path.read_text(encoding="utf-8") + start = current.find(BEGIN) + stop = current.find(END) + if start == -1 or stop == -1: + raise SystemExit( + f"AGENTS.md is missing the generated markers ({BEGIN} / {END}). " + "Add them before running this script." + ) + + return current[:start] + render_block(root) + current[stop + len(END) :] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Regenerate the agent rule map inside AGENTS.md." + ) + parser.add_argument( + "--check", + action="store_true", + help="Check whether the committed map is up-to-date without writing.", + ) + parser.add_argument( + "--root", + type=Path, + default=ROOT, + help="Repository root (defaults to this checkout).", + ) + args = parser.parse_args(argv) + + root = args.root.resolve() + agents_path = root / "AGENTS.md" + rendered = render_document(root) + + size = len(rendered.encode("utf-8")) + if size > MAX_BYTES: + raise SystemExit( + f"AGENTS.md would be {size} bytes, over the {MAX_BYTES}-byte budget. " + "Codex truncates project docs past this point silently — trim the " + "file rather than raising the cap." + ) + + current = agents_path.read_text(encoding="utf-8") + if args.check: + if current == rendered: + return 0 + raise SystemExit( + "AGENTS.md rule map is out of date. " + "Run `python scripts/sync_agent_rules.py` to regenerate." + ) + + if current != rendered: + agents_path.write_text(rendered, encoding="utf-8") + print(f"Updated rule map in {agents_path.relative_to(root)}") + else: + print("Rule map already up to date") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/unit/scripts/test_check_preflight.py b/tests/unit/scripts/test_check_preflight.py index f3836c32..bd9a80e3 100644 --- a/tests/unit/scripts/test_check_preflight.py +++ b/tests/unit/scripts/test_check_preflight.py @@ -135,10 +135,11 @@ class TestPreflightRunner: """Runner orchestration.""" def test_all_checks_listed(self): - assert len(PreflightRunner.ALL_CHECKS) == 14 + assert len(PreflightRunner.ALL_CHECKS) == 15 assert "check_links" in PreflightRunner.ALL_CHECKS assert "check_examples" in PreflightRunner.ALL_CHECKS assert "check_meta_index_sync" in PreflightRunner.ALL_CHECKS + assert "check_agent_rule_map" in PreflightRunner.ALL_CHECKS assert "check_version" in PreflightRunner.ALL_CHECKS assert "check_task_shot_knobs" in PreflightRunner.ALL_CHECKS assert "check_report_declarations" in PreflightRunner.ALL_CHECKS diff --git a/tests/unit/scripts/test_sync_agent_rules.py b/tests/unit/scripts/test_sync_agent_rules.py new file mode 100644 index 00000000..a0d25e4c --- /dev/null +++ b/tests/unit/scripts/test_sync_agent_rules.py @@ -0,0 +1,124 @@ +"""Tests for scripts/sync_agent_rules.py. + +The generator reads `paths:` frontmatter from .claude/rules/*.md and the +locations of sieval/*/CLAUDE.md, then rewrites a marker-delimited block in +AGENTS.md. Tests drive it against a synthetic tree via --root so they never +depend on the real repository's rule inventory. + +AI-Generated Code - Claude Sonnet 5 (Anthropic) +""" + +import importlib.util +import sys +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).resolve().parents[3] / "scripts" / "sync_agent_rules.py" + + +def _load(): + spec = importlib.util.spec_from_file_location("sync_agent_rules", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules["sync_agent_rules"] = module + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def tree(tmp_path: Path) -> Path: + """A synthetic repo: one layer file, two rules, one AGENTS.md.""" + (tmp_path / "sieval" / "core").mkdir(parents=True) + (tmp_path / "sieval" / "core" / "CLAUDE.md").write_text("# Core\n") + + rules = tmp_path / ".claude" / "rules" + rules.mkdir(parents=True) + (rules / "tests.md").write_text( + '---\npaths:\n - "tests/**/*.py"\n---\n\n# Test Rules\n' + ) + (rules / "deps.md").write_text( + '---\npaths:\n - "pyproject.toml"\n - "pdm.lock"\n---\n\n# Deps\n' + ) + + (tmp_path / "AGENTS.md").write_text( + "# Guidelines\n\n" + "\n" + "\n" + ) + return tmp_path + + +def test_writes_layer_file_with_derived_scope(tree: Path): + mod = _load() + assert mod.main(["--root", str(tree)]) == 0 + text = (tree / "AGENTS.md").read_text() + assert "`sieval/core/**`" in text + assert "`sieval/core/CLAUDE.md`" in text + + +def test_reads_paths_frontmatter_verbatim(tree: Path): + mod = _load() + mod.main(["--root", str(tree)]) + text = (tree / "AGENTS.md").read_text() + assert "`tests/**/*.py`" in text + assert "`pyproject.toml`" in text + assert "`pdm.lock`" in text + + +def test_paths_are_backticked_not_markdown_links(tree: Path): + """check_links validates Markdown links in tracked files; backticks keep + the generated block off that check's surface.""" + mod = _load() + mod.main(["--root", str(tree)]) + block = (tree / "AGENTS.md").read_text() + assert "](" not in block + + +def test_generation_is_idempotent(tree: Path): + mod = _load() + mod.main(["--root", str(tree)]) + once = (tree / "AGENTS.md").read_text() + mod.main(["--root", str(tree)]) + assert (tree / "AGENTS.md").read_text() == once + + +def test_check_passes_when_current(tree: Path): + mod = _load() + mod.main(["--root", str(tree)]) + assert mod.main(["--root", str(tree), "--check"]) == 0 + + +def test_check_fails_when_stale(tree: Path): + mod = _load() + mod.main(["--root", str(tree)]) + rules = tree / ".claude" / "rules" + (rules / "newrule.md").write_text('---\npaths:\n - "src/**"\n---\n\n# New\n') + with pytest.raises(SystemExit): + mod.main(["--root", str(tree), "--check"]) + + +def test_check_fails_when_block_hand_edited(tree: Path): + mod = _load() + mod.main(["--root", str(tree)]) + agents = tree / "AGENTS.md" + agents.write_text(agents.read_text().replace("sieval/core/**", "sieval/bogus/**")) + with pytest.raises(SystemExit): + mod.main(["--root", str(tree), "--check"]) + + +def test_rejects_output_over_size_budget(tree: Path): + """Codex truncates project docs past 32 KiB silently.""" + mod = _load() + agents = tree / "AGENTS.md" + agents.write_text("x" * 33000 + "\n" + agents.read_text()) + with pytest.raises(SystemExit) as exc: + mod.main(["--root", str(tree)]) + assert "32768" in str(exc.value) + + +def test_missing_markers_is_an_error(tree: Path): + mod = _load() + (tree / "AGENTS.md").write_text("# Guidelines\n") + with pytest.raises(SystemExit): + mod.main(["--root", str(tree)]) From 7d92ab0e8159e450ae3d6b90bf53c7c2b0a74601 Mon Sep 17 00:00:00 2001 From: Ethan Date: Tue, 15 Sep 2026 14:35:33 +0800 Subject: [PATCH 3/8] refactor: convert commands to skills under .agents/skills/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex has no project-level commands and reads skills only from .agents/skills/, so that becomes the source of truth, with symlinks for Claude Code (.claude/skills/) and opencode (.opencode/commands/ — its skills are model-invoked only, so the command link is what restores the slash trigger and argument passing). Retires 5 unused commands: the three archived ones plus the weekly and biweekly summaries. .claude/commands/ keeps its .gitignore entry: sieval-pr-review.md.bak remains on disk there and still needs hiding, so the directory is not actually empty. .gitignore needed two more re-inclusions than the .agents/skills/ block alone: .claude/skills/* had no negations, which would have left the Claude Code symlinks themselves untracked (verified in a scratch repo - git status stayed silent on them even after `git add -A`), and opencode's own sieval-pr-review.md command-symlink pointed at the gitignored personal skill and had no rule of its own, so it would have shipped as a dangling symlink. Both are fixed the same way: negate the two team-skill names, leave the personal one covered by the trailing `*`. Co-Authored-By: Claude Sonnet 5 --- .../skills/sieval-preflight/SKILL.md | 6 +++++- .../skills/sieval-release/SKILL.md | 6 +++++- .claude/skills/sieval-preflight | 1 + .claude/skills/sieval-release | 1 + .gitignore | 19 ++++++++++++++++++- .opencode/commands/sieval-preflight.md | 1 + .opencode/commands/sieval-release.md | 1 + 7 files changed, 32 insertions(+), 3 deletions(-) rename .claude/commands/sieval-preflight.md => .agents/skills/sieval-preflight/SKILL.md (82%) rename .claude/commands/sieval-release.md => .agents/skills/sieval-release/SKILL.md (92%) create mode 120000 .claude/skills/sieval-preflight create mode 120000 .claude/skills/sieval-release create mode 120000 .opencode/commands/sieval-preflight.md create mode 120000 .opencode/commands/sieval-release.md diff --git a/.claude/commands/sieval-preflight.md b/.agents/skills/sieval-preflight/SKILL.md similarity index 82% rename from .claude/commands/sieval-preflight.md rename to .agents/skills/sieval-preflight/SKILL.md index d3a07cd3..e11c9725 100644 --- a/.claude/commands/sieval-preflight.md +++ b/.agents/skills/sieval-preflight/SKILL.md @@ -1,11 +1,15 @@ --- -description: Run preflight checks on the sieval codebase (link validation, dependency consistency, task/dataset registration, version alignment) +name: sieval-preflight +description: Run sieval's preflight checks — link validation, dependency consistency, task/dataset registration, version alignment. Use before a release, before opening a PR, or when asked to verify repository health. --- # SiEval Preflight Checks ## Input +The invocation argument (substituted below as `$ARGUMENTS`; if it appears +literally, read the user's request for it instead): + $ARGUMENTS Accepts: `deep`, `quick` (default), or a specific check name (e.g., `check_links`, `check_deps`, `check_dep_coverage`, `check_tasks`, `check_task_shot_knobs`, `check_record_key_access`, `check_datasets`, `check_examples`, `check_imports`, `check_meta_index_sync`, `check_version`). diff --git a/.claude/commands/sieval-release.md b/.agents/skills/sieval-release/SKILL.md similarity index 92% rename from .claude/commands/sieval-release.md rename to .agents/skills/sieval-release/SKILL.md index 62b6d24a..f3ce3562 100644 --- a/.claude/commands/sieval-release.md +++ b/.agents/skills/sieval-release/SKILL.md @@ -1,11 +1,15 @@ --- -description: Release a new version of sieval (update changelog, tag, push) +name: sieval-release +description: Release a new version of sieval — run deep preflight, update CHANGELOG and Dockerfile, tag, and push. Use when asked to cut, ship, or tag a release. Takes a semantic version number. --- # SiEval Release ## Input +The invocation argument (substituted below as `$ARGUMENTS`; if it appears +literally, read the user's request for it instead): + $ARGUMENTS Expects a semantic version number, e.g. `0.3.4`. If not provided, ask the user. diff --git a/.claude/skills/sieval-preflight b/.claude/skills/sieval-preflight new file mode 120000 index 00000000..24fc0fa3 --- /dev/null +++ b/.claude/skills/sieval-preflight @@ -0,0 +1 @@ +../../.agents/skills/sieval-preflight \ No newline at end of file diff --git a/.claude/skills/sieval-release b/.claude/skills/sieval-release new file mode 120000 index 00000000..73d4d286 --- /dev/null +++ b/.claude/skills/sieval-release @@ -0,0 +1 @@ +../../.agents/skills/sieval-release \ No newline at end of file diff --git a/.gitignore b/.gitignore index 45dd0942..3d1e9e88 100644 --- a/.gitignore +++ b/.gitignore @@ -169,10 +169,27 @@ tmp/ # Claude Code # Exclude non-team-level configs .claude/commands/ -.claude/skills/ +.claude/skills/* +!.claude/skills/sieval-preflight +!.claude/skills/sieval-release .claude/worktrees/ .claude/settings.local.json +# Agent skills (shared by Claude Code, opencode, Codex) +# Team skills are tracked; personal ones are not. The trailing `*` is load +# bearing: git will not descend into an excluded directory, so a negation +# under a plain `.agents/skills/` entry is never evaluated and the +# re-included paths stay invisible. Excluding the contents instead keeps +# the directory itself walkable. +.agents/skills/* +!.agents/skills/sieval-preflight/ +!.agents/skills/sieval-release/ + +# opencode's command symlink for the personal skill above — its target is +# gitignored, so an untracked-but-unignored copy of this file would ship as +# a dangling symlink to a public clone. +.opencode/commands/sieval-pr-review.md + # Local analysis artifacts POSTPROCESS_*.md usage_log.jsonl diff --git a/.opencode/commands/sieval-preflight.md b/.opencode/commands/sieval-preflight.md new file mode 120000 index 00000000..1cf99684 --- /dev/null +++ b/.opencode/commands/sieval-preflight.md @@ -0,0 +1 @@ +../../.agents/skills/sieval-preflight/SKILL.md \ No newline at end of file diff --git a/.opencode/commands/sieval-release.md b/.opencode/commands/sieval-release.md new file mode 120000 index 00000000..b6a22145 --- /dev/null +++ b/.opencode/commands/sieval-release.md @@ -0,0 +1 @@ +../../.agents/skills/sieval-release/SKILL.md \ No newline at end of file From 301ce042fca837409d9f5ea233551dcf6f3b758d Mon Sep 17 00:00:00 2001 From: Ethan Date: Tue, 15 Sep 2026 14:41:17 +0800 Subject: [PATCH 4/8] refactor: extract post-edit checks into one script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three harnesses need the same four checks after an edit. Inlining them per tool would duplicate behavior, and drift there is silent — a one-character difference in a path pattern means a file class stops being checked with no error anywhere. Each harness now dispatches to scripts/post_edit_checks.py. Also adds .opencode/node_modules/ to the root .gitignore: the existing .opencode/.gitignore excludes itself (its own last line is `.gitignore`), so it is never committed and gives a fresh clone no protection. Co-Authored-By: Claude Sonnet 5 --- .claude/settings.json | 22 +----- .codex/config.toml | 15 ++++ .gitignore | 6 ++ .opencode/plugins/post-edit.ts | 22 ++++++ scripts/post_edit_checks.py | 76 +++++++++++++++++++ tests/unit/scripts/test_post_edit_checks.py | 82 +++++++++++++++++++++ 6 files changed, 203 insertions(+), 20 deletions(-) create mode 100644 .codex/config.toml create mode 100644 .opencode/plugins/post-edit.ts create mode 100644 scripts/post_edit_checks.py create mode 100644 tests/unit/scripts/test_post_edit_checks.py diff --git a/.claude/settings.json b/.claude/settings.json index 109ca93c..eef98498 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -6,26 +6,8 @@ "hooks": [ { "type": "command", - "command": "python -c \"import json,sys; d=json.load(sys.stdin); print(d.get('tool_input',{}).get('file_path','') or d.get('tool_response',{}).get('filePath',''))\" | { read -r f; echo \"$f\" | grep -qE '\\.pyi?$' && echo \"$f\" | grep -qE '(^|/)sieval/(tasks|datasets)/' || exit 0; r=$(git -C \"$(dirname \"$f\")\" rev-parse --show-toplevel 2>/dev/null) && [ -f \"$r/scripts/sync_package_stubs.py\" ] && python \"$r/scripts/sync_package_stubs.py\" || true; }", - "timeout": 30, - "statusMessage": "Post-edit checks..." - }, - { - "type": "command", - "command": "python -c \"import json,sys; d=json.load(sys.stdin); print(d.get('tool_input',{}).get('file_path','') or d.get('tool_response',{}).get('filePath',''))\" | { read -r f; echo \"$f\" | grep -qE '(^|/)sieval/(tasks|datasets)/.*\\.py$' || exit 0; r=$(git -C \"$(dirname \"$f\")\" rev-parse --show-toplevel 2>/dev/null) && [ -f \"$r/scripts/sync_meta_index.py\" ] && python \"$r/scripts/sync_meta_index.py\" || true; }", - "timeout": 30, - "statusMessage": "Post-edit checks..." - }, - { - "type": "command", - "command": "python -c \"import json,sys; d=json.load(sys.stdin); print(d.get('tool_input',{}).get('file_path','') or d.get('tool_response',{}).get('filePath',''))\" | { read -r f; echo \"$f\" | grep -qE '\\.pyi?$' && ruff check \"$f\" || true; }", - "timeout": 15, - "statusMessage": "Post-edit checks..." - }, - { - "type": "command", - "command": "python -c \"import json,sys; d=json.load(sys.stdin); print(d.get('tool_input',{}).get('file_path','') or d.get('tool_response',{}).get('filePath',''))\" | { read -r f; echo \"$f\" | grep -qE '\\.pyi?$' && ty check \"$f\" || true; }", - "timeout": 30, + "command": "python -c \"import json,sys; d=json.load(sys.stdin); print(d.get('tool_input',{}).get('file_path','') or d.get('tool_response',{}).get('filePath',''))\" | { read -r f; [ -n \"$f\" ] || exit 0; r=$(git -C \"$(dirname \"$f\")\" rev-parse --show-toplevel 2>/dev/null) && [ -f \"$r/scripts/post_edit_checks.py\" ] && python \"$r/scripts/post_edit_checks.py\" \"$f\" || true; }", + "timeout": 60, "statusMessage": "Post-edit checks..." } ] diff --git a/.codex/config.toml b/.codex/config.toml new file mode 100644 index 00000000..7eebae0d --- /dev/null +++ b/.codex/config.toml @@ -0,0 +1,15 @@ +# Project-scoped Codex configuration. +# +# Loads only for projects marked trusted: an untrusted project skips every +# project-scoped .codex/ layer, hooks included. Hooks also require +# features.hooks to be enabled. +# +# The checks live in scripts/post_edit_checks.py, shared with Claude Code and +# opencode. Codex has no post-edit-specific event, so this filters PostToolUse. + +[features] +hooks = true + +[[hooks.PostToolUse]] +matcher = "shell" +command = "python scripts/post_edit_checks.py" diff --git a/.gitignore b/.gitignore index 3d1e9e88..c8ce6fbe 100644 --- a/.gitignore +++ b/.gitignore @@ -190,6 +190,12 @@ tmp/ # a dangling symlink to a public clone. .opencode/commands/sieval-pr-review.md +# opencode plugin dependencies (npm install artifacts, not source). The +# `.opencode/.gitignore` that would normally cover this excludes itself (its +# last line is `.gitignore`), so it is never committed and gives a fresh +# clone no protection — this root-level entry is the one that actually ships. +.opencode/node_modules/ + # Local analysis artifacts POSTPROCESS_*.md usage_log.jsonl diff --git a/.opencode/plugins/post-edit.ts b/.opencode/plugins/post-edit.ts new file mode 100644 index 00000000..2571edd3 --- /dev/null +++ b/.opencode/plugins/post-edit.ts @@ -0,0 +1,22 @@ +/** + * Runs sieval's post-edit checks after opencode edits a file. + * + * The checks themselves live in scripts/post_edit_checks.py, shared with + * Claude Code and Codex. This file only dispatches. + * + * Uses the `file.edited` event rather than `tool.execute.after`: its payload + * is `{ file: string }`, a clean path, where the tool hook would require + * per-tool argument parsing to find one. + * + * AI-Generated Code - Claude Sonnet 5 (Anthropic) + */ +export const PostEditChecks = async ({ $, directory }) => { + return { + event: async ({ event }) => { + if (event.type !== "file.edited") return + const file = event.properties?.file + if (!file) return + await $`python ${directory}/scripts/post_edit_checks.py ${file}`.nothrow() + }, + } +} diff --git a/scripts/post_edit_checks.py b/scripts/post_edit_checks.py new file mode 100644 index 00000000..362871f3 --- /dev/null +++ b/scripts/post_edit_checks.py @@ -0,0 +1,76 @@ +"""Run post-edit checks for a single edited file. + +Claude Code, opencode, and Codex each fire a hook after a file edit. Keeping +the logic here rather than inlining it three times means a pattern change +cannot drift between them — and drift here is silent, since the failure mode +is a file class quietly no longer being checked. + +Always exits 0: a post-edit check informs the agent, it does not block the +edit that already happened. + +AI-Generated Code - Claude Sonnet 5 (Anthropic) +""" + +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + +# A Python source or stub file. +PY_RE = re.compile(r"\.pyi?$") +# A file under sieval/tasks/ or sieval/datasets/, at any depth, matching both +# repo-relative and absolute paths as passed by the different harnesses. +REGISTRY_RE = re.compile(r"(^|/)sieval/(tasks|datasets)/") + + +def select_checks(path: str) -> set[str]: + """Return the check names that apply to ``path``.""" + if not path: + return set() + + checks: set[str] = set() + is_py = bool(PY_RE.search(path)) + in_registry = bool(REGISTRY_RE.search(path)) + + if is_py: + checks.update({"ruff", "ty"}) + if is_py and in_registry: + checks.add("stubs") + if in_registry and path.endswith(".py"): + checks.add("meta") + return checks + + +def _run(cmd: list[str]) -> int: + """Run a command from the repo root, streaming output. Never raises.""" + try: + return subprocess.run(cmd, cwd=ROOT).returncode + except OSError as exc: + print(f"post-edit check could not run {cmd[0]}: {exc}", file=sys.stderr) + return 0 + + +def main(argv: list[str] | None = None) -> int: + args = sys.argv[1:] if argv is None else argv + if not args: + return 0 + + path = args[0] + checks = select_checks(path) + + if "stubs" in checks: + _run([sys.executable, str(ROOT / "scripts" / "sync_package_stubs.py")]) + if "meta" in checks: + _run([sys.executable, str(ROOT / "scripts" / "sync_meta_index.py")]) + if "ruff" in checks: + _run(["ruff", "check", path]) + if "ty" in checks: + _run(["ty", "check", path]) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/unit/scripts/test_post_edit_checks.py b/tests/unit/scripts/test_post_edit_checks.py new file mode 100644 index 00000000..a6145564 --- /dev/null +++ b/tests/unit/scripts/test_post_edit_checks.py @@ -0,0 +1,82 @@ +"""Tests for scripts/post_edit_checks.py. + +The script is called by three agent harnesses after a file edit. Tests cover +which checks each path selects, and that the script never fails the caller — +a post-edit check informs, it does not block the edit. + +AI-Generated Code - Claude Sonnet 5 (Anthropic) +""" + +import importlib.util +import sys +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).resolve().parents[3] / "scripts" / "post_edit_checks.py" + + +def _load(): + spec = importlib.util.spec_from_file_location("post_edit_checks", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules["post_edit_checks"] = module + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize( + ("path", "expected"), + [ + ("sieval/tasks/foo.py", {"stubs", "meta", "ruff", "ty"}), + ("sieval/datasets/bar.py", {"stubs", "meta", "ruff", "ty"}), + ("sieval/tasks/__init__.pyi", {"stubs", "ruff", "ty"}), + ("sieval/core/runners/baz.py", {"ruff", "ty"}), + ("tests/unit/test_x.py", {"ruff", "ty"}), + ("README.md", set()), + ("pyproject.toml", set()), + ], +) +def test_selects_checks_by_path(path: str, expected: set[str]): + mod = _load() + assert mod.select_checks(path) == expected + + +def test_absolute_paths_match_the_same_way(): + """Harnesses pass absolute paths; the sieval/ prefix must still match.""" + mod = _load() + assert mod.select_checks("/home/u/repo/sieval/tasks/foo.py") == { + "stubs", + "meta", + "ruff", + "ty", + } + + +def test_similar_paths_outside_sieval_do_not_match(): + """A vendored or nested `sieval/tasks/` elsewhere is still a task file, but + a lookalike like `mysieval/tasks/` is not.""" + mod = _load() + assert "meta" not in mod.select_checks("mysieval/tasks/foo.py") + + +def test_empty_path_selects_nothing(): + mod = _load() + assert mod.select_checks("") == set() + + +def test_main_exits_zero_on_unmatched_path(): + mod = _load() + assert mod.main(["README.md"]) == 0 + + +def test_main_exits_zero_with_no_argument(): + mod = _load() + assert mod.main([]) == 0 + + +def test_main_exits_zero_even_when_a_check_fails(monkeypatch): + """A failing lint must not fail the caller's edit.""" + mod = _load() + monkeypatch.setattr(mod, "_run", lambda *a, **k: 1) + assert mod.main(["sieval/tasks/foo.py"]) == 0 From c05b4a0b2745e3d8a00e170d735485bb82c7a0a4 Mon Sep 17 00:00:00 2001 From: Ethan Date: Tue, 15 Sep 2026 15:55:34 +0800 Subject: [PATCH 5/8] docs(gitignore): warn against normalizing the skill negation blocks The `.claude/skills/` negations must stay slash-free because they name symlinks; a trailing-slash negation matches only real directories and would silently untrack both. The neighbouring `.agents/skills/` block does carry slashes, so the two look inconsistent and invite a cleanup that breaks tracking with no error. Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index c8ce6fbe..86bba435 100644 --- a/.gitignore +++ b/.gitignore @@ -169,6 +169,10 @@ tmp/ # Claude Code # Exclude non-team-level configs .claude/commands/ +# These two negations must NOT carry a trailing slash, unlike the +# `.agents/skills/` ones below. These paths are symlinks; a trailing-slash +# negation matches only real directories, so adding one here silently stops +# tracking both symlinks with no error. Do not normalize the two blocks. .claude/skills/* !.claude/skills/sieval-preflight !.claude/skills/sieval-release From 8b5263cd31e9fc89531ca4a96d65de9f85b3623c Mon Sep 17 00:00:00 2001 From: Ethan Date: Thu, 17 Sep 2026 17:50:26 +0800 Subject: [PATCH 6/8] fix(config): drop unverifiable Codex hook wiring, close two ignore/scan gaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the harness-neutral config work, from testing all three harnesses against their real binaries rather than their docs. Remove `.codex/config.toml`. Asked Codex 0.144.4 what it actually loaded via its own `hooks/list` RPC: the file registered zero hooks, with empty `warnings` and `errors` — silent on every channel. Three independent causes, each sufficient: `command` sat on the matcher table, but `ConfiguredHookMatcherGroup` only carries `matcher` and `hooks`, so the handler needs a nested `[[hooks.PostToolUse.hooks]]`; `matcher = "shell"` matches no tool name (shell is `Bash`, edits are `apply_patch`), and a wrong matcher still registers, so it would fail silently a second time; and Codex delivers the payload as JSON on stdin while `post_edit_checks.py` reads `argv`, so even a corrected file would run the script with no path. A registered hook also comes back `trustStatus: "untrusted"` — project trust does not cover it. Writing the adapter is the right fix, but no Codex credential exists here, so its stdin payload shape cannot be observed and the result could not be verified before shipping. A config that silently does nothing is worse than none: it reads as wired. `AGENTS.md` now records why the wiring is absent and how to run the checks by hand under Codex. Make the `.opencode/commands/` ignore pattern-based, matching the two blocks above it. It listed one personal skill by filename, so the *next* one shipped: its symlink points at a gitignored target, and a public clone would get a dangling link. Verified a fresh personal skill is now ignored across all three surfaces while both tracked symlinks stay at mode 120000 — the negations stay slash-free for the reason the neighbouring comment gives. Anchor the two guideline entries in `sanitize.sh`. The allowlist filters git grep output lines, so an unanchored `AGENTS.md` exempted every line anywhere that merely named the file: `BAD = "/home/u/x" # see AGENTS.md` passed the hardcoded-path scan. Now caught, with the guideline files themselves still exempt. `CLAUDE.md` had the same flaw and is fixed with it. Also document what the harnesses really do with layer rules, measured on a synthetic tree: Codex never reads a `CLAUDE.md` below the root, not even with the working directory inside that folder (renaming it `AGENTS.md` loads it immediately), and opencode only walks upward. The 16 layer and scoped rules are therefore automatic under Claude Code alone; elsewhere the generated map is a pointer, not an injection. Validation: 418 tests in tests/unit/scripts, full preflight green (25 checks), sanitize clean, ruff check + format clean. Co-Authored-By: Claude Opus 5 (1M context) --- .codex/config.toml | 15 --------------- .gitignore | 12 ++++++++---- AGENTS.md | 28 ++++++++++++++++++---------- scripts/sanitize.sh | 8 ++++++-- scripts/sync_agent_rules.py | 2 +- 5 files changed, 33 insertions(+), 32 deletions(-) delete mode 100644 .codex/config.toml diff --git a/.codex/config.toml b/.codex/config.toml deleted file mode 100644 index 7eebae0d..00000000 --- a/.codex/config.toml +++ /dev/null @@ -1,15 +0,0 @@ -# Project-scoped Codex configuration. -# -# Loads only for projects marked trusted: an untrusted project skips every -# project-scoped .codex/ layer, hooks included. Hooks also require -# features.hooks to be enabled. -# -# The checks live in scripts/post_edit_checks.py, shared with Claude Code and -# opencode. Codex has no post-edit-specific event, so this filters PostToolUse. - -[features] -hooks = true - -[[hooks.PostToolUse]] -matcher = "shell" -command = "python scripts/post_edit_checks.py" diff --git a/.gitignore b/.gitignore index 86bba435..c5c8c8ff 100644 --- a/.gitignore +++ b/.gitignore @@ -189,10 +189,14 @@ tmp/ !.agents/skills/sieval-preflight/ !.agents/skills/sieval-release/ -# opencode's command symlink for the personal skill above — its target is -# gitignored, so an untracked-but-unignored copy of this file would ship as -# a dangling symlink to a public clone. -.opencode/commands/sieval-pr-review.md +# opencode's command symlinks. Pattern-based like the two blocks above, not a +# per-skill literal: a personal skill's symlink points at a gitignored target, +# so any entry not re-included here would ship to a public clone as a dangling +# symlink. Same trailing-slash trap as `.claude/skills/` — these are symlinks, +# so the negations must stay slash-free. +.opencode/commands/* +!.opencode/commands/sieval-preflight.md +!.opencode/commands/sieval-release.md # opencode plugin dependencies (npm install artifacts, not source). The # `.opencode/.gitignore` that would normally cover this excludes itself (its diff --git a/AGENTS.md b/AGENTS.md index 73b2d2e9..a3372c0c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -120,18 +120,25 @@ Precise reproducibility is a product contract, not a nicety. ## Agent Tooling This file is the single source of truth for shared conventions, and the only -instruction file all three supported agents read. `CLAUDE.md` is a one-line -import of it. Keep it under 32 KiB: Codex truncates project docs past that -byte count silently, so content near the end simply stops applying. +instruction file every supported agent reads. `CLAUDE.md` is a one-line import +of it. Keep it under 32 KiB: Codex truncates project docs past that byte count +silently, so content near the end simply stops applying. - **Layer and scoped rules** — `sieval/*/CLAUDE.md` and `.claude/rules/*.md`. - Claude Code loads these automatically; opencode and Codex do not, so the - generated map below tells them what to read and when. + Only Claude Code loads these automatically, and the difference is not + cosmetic: Codex reads `AGENTS.md` files on the root-to-cwd path and never + picks up a `CLAUDE.md` below the root — not even when the working directory + is the very folder holding it. opencode walks *upward* from the working + directory, so it sees a layer file only while working inside that layer. + For those two, the 16 layer and scoped rules are reachable solely through + the generated map below, which asks the agent to read a file rather than + putting the rule in front of it. Treat a rule as enforced only under Claude + Code; elsewhere it is a pointer the agent has to follow. Read the matching file before editing a path it covers. Claude Code -loads these automatically; opencode and Codex need this map. +loads these automatically; every other harness needs this map. | When editing | Read first | | --- | --- | @@ -158,7 +165,8 @@ loads these automatically; opencode and Codex need this map. symlinks at `.claude/skills/` and `.opencode/commands/.md`. Never edit through a symlink. - **Post-edit checks** — `scripts/post_edit_checks.py` holds the behavior; - `.claude/settings.json`, `.opencode/plugins/post-edit.ts`, and - `.codex/config.toml` each dispatch to it. -- **Codex users:** project-scoped `.codex/` layers (including hooks) load only - for projects you have marked trusted, and hooks require `features.hooks`. + `.claude/settings.json` and `.opencode/plugins/post-edit.ts` each dispatch to + it with the edited path as `argv[1]`. There is deliberately no Codex wiring: + Codex delivers the hook payload as JSON on stdin rather than as an argument, + so it needs an adapter, and none can be verified here. Run the checks by hand + under Codex — `python scripts/post_edit_checks.py `. diff --git a/scripts/sanitize.sh b/scripts/sanitize.sh index 226e61ef..ff330b29 100755 --- a/scripts/sanitize.sh +++ b/scripts/sanitize.sh @@ -38,8 +38,12 @@ ALLOWLIST=( "scripts/sanitize.sh" # this script itself (contains the patterns) "docs/" # local design/planning docs "tests/" # test fixtures use example IPs/paths - "CLAUDE.md" # project guidelines - "AGENTS.md" # project guidelines (source of truth) + # Anchored to the start of the `path:lineno:` prefix git grep emits, so these + # exempt the guideline files themselves — not every line anywhere that merely + # mentions one of them by name. Unanchored, a comment like + # `BAD = "/home/u/x" # see AGENTS.md` would exempt itself. + "^CLAUDE\.md:" # project guidelines + "^AGENTS\.md:" # project guidelines (source of truth) "\.git/" # git internals "127\.0\.0\.1" # localhost is fine "0\.0\.0\.0" # bind-all is fine diff --git a/scripts/sync_agent_rules.py b/scripts/sync_agent_rules.py index 8b6f1e34..006e947a 100644 --- a/scripts/sync_agent_rules.py +++ b/scripts/sync_agent_rules.py @@ -79,7 +79,7 @@ def render_block(root: Path) -> str: BEGIN, "", "Read the matching file before editing a path it covers. Claude Code", - "loads these automatically; opencode and Codex need this map.", + "loads these automatically; every other harness needs this map.", "", "| When editing | Read first |", "| --- | --- |", From 5da51baf1a6900c2405c1305cff52b5308a6bfc5 Mon Sep 17 00:00:00 2001 From: Ethan Date: Thu, 17 Sep 2026 18:02:17 +0800 Subject: [PATCH 7/8] docs: tighten the agent-config prose and comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Trim the comments and docstrings added across this branch, keeping the reason each one records — every block here documents a trap that cost a debugging session — while cutting restatement and hedging. Two were also stale after the Codex hook wiring came out: the `post_edit_checks.py` docstring still named three harnesses, and the opencode plugin comment pointed at a Codex that no longer shares it. The plugin comment now says why `file.edited` is the right event — opencode emits it from inside the write/edit/patch tools, so it tracks agent edits rather than any change on disk. Validation: 418 tests in tests/unit/scripts, full preflight green (26 checks, no FAIL/WARN), sanitize clean, ruff check + format clean. Both behavioural fixes re-verified after the trim: a fresh personal skill is still ignored across all three surfaces with both tracked symlinks at mode 120000, and the self-exempting `# see AGENTS.md` probe line is still caught. Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 31 +++++++++++++------------------ .opencode/plugins/post-edit.ts | 10 +++++----- AGENTS.md | 23 +++++++++-------------- scripts/post_edit_checks.py | 14 +++++++------- scripts/sanitize.sh | 7 +++---- scripts/sync_agent_rules.py | 18 +++++++++--------- 6 files changed, 46 insertions(+), 57 deletions(-) diff --git a/.gitignore b/.gitignore index c5c8c8ff..3d88d719 100644 --- a/.gitignore +++ b/.gitignore @@ -169,39 +169,34 @@ tmp/ # Claude Code # Exclude non-team-level configs .claude/commands/ -# These two negations must NOT carry a trailing slash, unlike the -# `.agents/skills/` ones below. These paths are symlinks; a trailing-slash -# negation matches only real directories, so adding one here silently stops -# tracking both symlinks with no error. Do not normalize the two blocks. +# These paths are symlinks, so the negations must stay slash-free: a +# trailing slash matches only real directories and would silently untrack +# both, with no error. Do not normalize against the `.agents/` block below. .claude/skills/* !.claude/skills/sieval-preflight !.claude/skills/sieval-release .claude/worktrees/ .claude/settings.local.json -# Agent skills (shared by Claude Code, opencode, Codex) +# Agent skills (shared by every harness) # Team skills are tracked; personal ones are not. The trailing `*` is load -# bearing: git will not descend into an excluded directory, so a negation -# under a plain `.agents/skills/` entry is never evaluated and the -# re-included paths stay invisible. Excluding the contents instead keeps -# the directory itself walkable. +# bearing: git never descends into an excluded directory, so a negation under +# a plain `.agents/skills/` entry would never be evaluated. .agents/skills/* !.agents/skills/sieval-preflight/ !.agents/skills/sieval-release/ -# opencode's command symlinks. Pattern-based like the two blocks above, not a -# per-skill literal: a personal skill's symlink points at a gitignored target, -# so any entry not re-included here would ship to a public clone as a dangling -# symlink. Same trailing-slash trap as `.claude/skills/` — these are symlinks, -# so the negations must stay slash-free. +# opencode's command symlinks. Pattern-based, not a per-skill literal: a +# personal skill's symlink points at a gitignored target, so anything not +# re-included here ships to a public clone as a dangling link. Symlinks again, +# so the negations stay slash-free. .opencode/commands/* !.opencode/commands/sieval-preflight.md !.opencode/commands/sieval-release.md -# opencode plugin dependencies (npm install artifacts, not source). The -# `.opencode/.gitignore` that would normally cover this excludes itself (its -# last line is `.gitignore`), so it is never committed and gives a fresh -# clone no protection — this root-level entry is the one that actually ships. +# opencode plugin dependencies (npm artifacts, not source). The +# `.opencode/.gitignore` that would cover this excludes itself, so it never +# ships — this root-level entry is the one a fresh clone gets. .opencode/node_modules/ # Local analysis artifacts diff --git a/.opencode/plugins/post-edit.ts b/.opencode/plugins/post-edit.ts index 2571edd3..089787a1 100644 --- a/.opencode/plugins/post-edit.ts +++ b/.opencode/plugins/post-edit.ts @@ -1,12 +1,12 @@ /** * Runs sieval's post-edit checks after opencode edits a file. * - * The checks themselves live in scripts/post_edit_checks.py, shared with - * Claude Code and Codex. This file only dispatches. + * The checks live in scripts/post_edit_checks.py, shared with Claude Code. + * This file only dispatches. * - * Uses the `file.edited` event rather than `tool.execute.after`: its payload - * is `{ file: string }`, a clean path, where the tool hook would require - * per-tool argument parsing to find one. + * Uses `file.edited` rather than `tool.execute.after`: its payload is a clean + * `{ file: string }`, and opencode emits it from inside the write/edit/patch + * tools, so it tracks agent edits rather than any file change on disk. * * AI-Generated Code - Claude Sonnet 5 (Anthropic) */ diff --git a/AGENTS.md b/AGENTS.md index a3372c0c..78b89953 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -125,15 +125,11 @@ of it. Keep it under 32 KiB: Codex truncates project docs past that byte count silently, so content near the end simply stops applying. - **Layer and scoped rules** — `sieval/*/CLAUDE.md` and `.claude/rules/*.md`. - Only Claude Code loads these automatically, and the difference is not - cosmetic: Codex reads `AGENTS.md` files on the root-to-cwd path and never - picks up a `CLAUDE.md` below the root — not even when the working directory - is the very folder holding it. opencode walks *upward* from the working - directory, so it sees a layer file only while working inside that layer. - For those two, the 16 layer and scoped rules are reachable solely through - the generated map below, which asks the agent to read a file rather than - putting the rule in front of it. Treat a rule as enforced only under Claude - Code; elsewhere it is a pointer the agent has to follow. + Only Claude Code loads these automatically. Codex never reads a `CLAUDE.md` + below the repo root, even with the working directory in that folder, and + opencode only walks upward from it. Elsewhere these rules are reachable just + through the map below, which points at a file rather than putting the rule in + front of the agent — so treat a rule as enforced only under Claude Code. @@ -165,8 +161,7 @@ loads these automatically; every other harness needs this map. symlinks at `.claude/skills/` and `.opencode/commands/.md`. Never edit through a symlink. - **Post-edit checks** — `scripts/post_edit_checks.py` holds the behavior; - `.claude/settings.json` and `.opencode/plugins/post-edit.ts` each dispatch to - it with the edited path as `argv[1]`. There is deliberately no Codex wiring: - Codex delivers the hook payload as JSON on stdin rather than as an argument, - so it needs an adapter, and none can be verified here. Run the checks by hand - under Codex — `python scripts/post_edit_checks.py `. + `.claude/settings.json` and `.opencode/plugins/post-edit.ts` dispatch to it + with the edited path as `argv[1]`. Codex wiring is deliberately absent: it + delivers the payload as JSON on stdin, so it needs an adapter no one can + verify here. Under Codex, run `python scripts/post_edit_checks.py `. diff --git a/scripts/post_edit_checks.py b/scripts/post_edit_checks.py index 362871f3..df266c6c 100644 --- a/scripts/post_edit_checks.py +++ b/scripts/post_edit_checks.py @@ -1,12 +1,12 @@ -"""Run post-edit checks for a single edited file. +"""Run post-edit checks for a single edited file, passed as ``argv[1]``. -Claude Code, opencode, and Codex each fire a hook after a file edit. Keeping -the logic here rather than inlining it three times means a pattern change -cannot drift between them — and drift here is silent, since the failure mode -is a file class quietly no longer being checked. +Claude Code and opencode each fire a hook after a file edit. Keeping the logic +here rather than inlining it per harness means a pattern change cannot drift +between them — and drift is silent, since the failure mode is a file class +quietly no longer being checked. -Always exits 0: a post-edit check informs the agent, it does not block the -edit that already happened. +Always exits 0: a post-edit check informs the agent, it does not block the edit +that already happened. AI-Generated Code - Claude Sonnet 5 (Anthropic) """ diff --git a/scripts/sanitize.sh b/scripts/sanitize.sh index ff330b29..e4f8fc80 100755 --- a/scripts/sanitize.sh +++ b/scripts/sanitize.sh @@ -38,10 +38,9 @@ ALLOWLIST=( "scripts/sanitize.sh" # this script itself (contains the patterns) "docs/" # local design/planning docs "tests/" # test fixtures use example IPs/paths - # Anchored to the start of the `path:lineno:` prefix git grep emits, so these - # exempt the guideline files themselves — not every line anywhere that merely - # mentions one of them by name. Unanchored, a comment like - # `BAD = "/home/u/x" # see AGENTS.md` would exempt itself. + # Anchored to git grep's `path:lineno:` prefix, so these exempt the guideline + # files themselves, not every line that merely names one: unanchored, a + # comment like `BAD = "/home/u/x" # see AGENTS.md` would exempt itself. "^CLAUDE\.md:" # project guidelines "^AGENTS\.md:" # project guidelines (source of truth) "\.git/" # git internals diff --git a/scripts/sync_agent_rules.py b/scripts/sync_agent_rules.py index 006e947a..2b2d65ce 100644 --- a/scripts/sync_agent_rules.py +++ b/scripts/sync_agent_rules.py @@ -1,13 +1,13 @@ """Regenerate the agent rule map inside AGENTS.md. Claude Code auto-loads `sieval/*/CLAUDE.md` (on demand) and `.claude/rules/*.md` -(by `paths:` glob). opencode and Codex do neither: opencode only walks upward -from the working directory, and Codex reads only AGENTS.md files on the -root-to-cwd path. Both therefore need an explicit index. +(by `paths:` glob). No other harness does: opencode only walks upward from the +working directory, and Codex reads AGENTS.md files on the root-to-cwd path. +They need an explicit index. -Hand-writing that index would duplicate the `paths:` frontmatter that already -declares each rule's scope, and would go stale with no signal. Generating it -keeps the copy honest, the same way sync_meta_index.py does for the registry. +Hand-writing it would duplicate the `paths:` frontmatter that already declares +each rule's scope, and would go stale with no signal — so it is generated, the +way sync_meta_index.py handles the registry. AI-Generated Code - Claude Sonnet 5 (Anthropic) """ @@ -29,9 +29,9 @@ def _parse_paths_frontmatter(text: str) -> list[str]: """Return the `paths:` globs from a rule file's YAML frontmatter. - Hand-rolled rather than via PyYAML: this script runs from pre-commit and - preflight, where an import-light dependency footprint matters, and the - frontmatter shape here is fixed (a `paths:` key over a list of strings). + Hand-rolled rather than via PyYAML: this runs from pre-commit and preflight, + where an import-light footprint matters, and the shape is fixed (a `paths:` + key over a list of strings). """ if not text.startswith("---"): return [] From 598c911d589e571e44e8f155df2419bd288d6536 Mon Sep 17 00:00:00 2001 From: Ethan Date: Thu, 17 Sep 2026 18:13:09 +0800 Subject: [PATCH 8/8] fix(scripts): reject a `paths:` key the rule-map parser cannot read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The frontmatter parser returned [] for anything that was not a block list, so `paths: ["a/**"]` — valid YAML, and the form someone reaching for brevity would write — silently dropped that rule from the map and rendered it as `_(always)_`. `check_agent_rule_map` could not catch it: the check re-parses the same frontmatter the same wrong way, so both sides agree and it passes. A generator whose whole premise is that an omission cannot happen silently had one failure mode where it does. Raise instead, naming the file. An absent `paths:` key still means "applies everywhere" — that is a real choice, and it keeps working; only a key that is present and unreadable is an error. Verified by reverse mutation: with the guard deleted, all three new parametrised cases fail with DID NOT RAISE. Left alone: the layer glob stays one level (`sieval/*/CLAUDE.md`). Missing a nested layer file is a conservative failure, the six layer directories are a fixed convention, and `**` would sweep in vendored `sieval/community//CLAUDE.md` files. Validation: 422 tests in tests/unit/scripts, full preflight green (26 checks), ruff check + format clean. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/sync_agent_rules.py | 24 +++++++++++++--- tests/unit/scripts/test_sync_agent_rules.py | 32 +++++++++++++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/scripts/sync_agent_rules.py b/scripts/sync_agent_rules.py index 2b2d65ce..dd877d84 100644 --- a/scripts/sync_agent_rules.py +++ b/scripts/sync_agent_rules.py @@ -26,12 +26,17 @@ MAX_BYTES = 32768 -def _parse_paths_frontmatter(text: str) -> list[str]: +def _parse_paths_frontmatter(text: str, source: str = "") -> list[str]: """Return the `paths:` globs from a rule file's YAML frontmatter. Hand-rolled rather than via PyYAML: this runs from pre-commit and preflight, where an import-light footprint matters, and the shape is fixed (a `paths:` key over a list of strings). + + A file with no `paths:` key is scoped to everything, which is a real choice. + A `paths:` key this parser cannot read is not: returning ``[]`` there would + drop the rule from the map with nothing to notice it, since the check + compares the generated map against the same empty parse. So it raises. """ if not text.startswith("---"): return [] @@ -42,21 +47,32 @@ def _parse_paths_frontmatter(text: str) -> list[str]: globs: list[str] = [] in_paths = False + seen_paths = False for raw in block.splitlines(): line = raw.rstrip() if not line: continue - if line.strip() == "paths:": + stripped = line.strip() + if stripped.startswith("paths:"): + # Only a block list is supported; `paths: [...]` on one line is not. + if stripped != "paths:": + raise SystemExit( + f'{source}: `paths:` must be a block list, one `- "glob"` ' + f"per line, not {stripped[len('paths:') :].strip()!r}." + ) in_paths = True + seen_paths = True continue if in_paths: - stripped = line.strip() if stripped.startswith("- "): globs.append(stripped[2:].strip().strip('"').strip("'")) continue # A non-list line at the same or lower indent ends the paths block. if not line.startswith(" "): in_paths = False + + if seen_paths and not globs: + raise SystemExit(f"{source}: `paths:` is present but lists no globs.") return globs @@ -70,8 +86,8 @@ def render_block(root: Path) -> str: rules_dir = root / ".claude" / "rules" for rule_file in sorted(rules_dir.glob("*.md")) if rules_dir.is_dir() else []: - globs = _parse_paths_frontmatter(rule_file.read_text(encoding="utf-8")) rel = rule_file.relative_to(root).as_posix() + globs = _parse_paths_frontmatter(rule_file.read_text(encoding="utf-8"), rel) scope = ", ".join(f"`{g}`" for g in globs) if globs else "_(always)_" rows.append((scope, f"`{rel}`")) diff --git a/tests/unit/scripts/test_sync_agent_rules.py b/tests/unit/scripts/test_sync_agent_rules.py index a0d25e4c..fc10666e 100644 --- a/tests/unit/scripts/test_sync_agent_rules.py +++ b/tests/unit/scripts/test_sync_agent_rules.py @@ -66,6 +66,38 @@ def test_reads_paths_frontmatter_verbatim(tree: Path): assert "`pdm.lock`" in text +def test_rule_without_paths_is_scoped_to_everything(tree: Path): + """No `paths:` key is a real choice, not a parse failure.""" + mod = _load() + (tree / ".claude" / "rules" / "always.md").write_text( + "---\ndescription: applies everywhere\n---\n\n# Always\n" + ) + assert mod.main(["--root", str(tree)]) == 0 + assert "_(always)_" in (tree / "AGENTS.md").read_text() + + +@pytest.mark.parametrize( + ("frontmatter", "expected"), + [ + ('---\npaths: ["a/**", "b/**"]\n---\n\n# Flow\n', "block list"), + ("---\npaths:\n---\n\n# Empty\n", "lists no globs"), + ('---\npaths: "a/**"\n---\n\n# Scalar\n', "block list"), + ], +) +def test_unreadable_paths_key_is_an_error(tree: Path, frontmatter: str, expected: str): + """A `paths:` key the parser cannot read must not silently drop the rule. + + Returning [] would render it as `_(always)_` and still satisfy --check, + since the check re-parses the same frontmatter the same wrong way. + """ + mod = _load() + (tree / ".claude" / "rules" / "broken.md").write_text(frontmatter) + with pytest.raises(SystemExit) as exc: + mod.main(["--root", str(tree)]) + assert expected in str(exc.value) + assert "broken.md" in str(exc.value) + + def test_paths_are_backticked_not_markdown_links(tree: Path): """check_links validates Markdown links in tracked files; backticks keep the generated block off that check's surface."""