diff --git a/.claude/commands/sieval-preflight.md b/.agents/skills/sieval-preflight/SKILL.md similarity index 82% rename from .claude/commands/sieval-preflight.md rename to .agents/skills/sieval-preflight/SKILL.md index d3a07cd3..e11c9725 100644 --- a/.claude/commands/sieval-preflight.md +++ b/.agents/skills/sieval-preflight/SKILL.md @@ -1,11 +1,15 @@ --- -description: Run preflight checks on the sieval codebase (link validation, dependency consistency, task/dataset registration, version alignment) +name: sieval-preflight +description: Run sieval's preflight checks — link validation, dependency consistency, task/dataset registration, version alignment. Use before a release, before opening a PR, or when asked to verify repository health. --- # SiEval Preflight Checks ## Input +The invocation argument (substituted below as `$ARGUMENTS`; if it appears +literally, read the user's request for it instead): + $ARGUMENTS Accepts: `deep`, `quick` (default), or a specific check name (e.g., `check_links`, `check_deps`, `check_dep_coverage`, `check_tasks`, `check_task_shot_knobs`, `check_record_key_access`, `check_datasets`, `check_examples`, `check_imports`, `check_meta_index_sync`, `check_version`). diff --git a/.claude/commands/sieval-release.md b/.agents/skills/sieval-release/SKILL.md similarity index 92% rename from .claude/commands/sieval-release.md rename to .agents/skills/sieval-release/SKILL.md index 62b6d24a..f3ce3562 100644 --- a/.claude/commands/sieval-release.md +++ b/.agents/skills/sieval-release/SKILL.md @@ -1,11 +1,15 @@ --- -description: Release a new version of sieval (update changelog, tag, push) +name: sieval-release +description: Release a new version of sieval — run deep preflight, update CHANGELOG and Dockerfile, tag, and push. Use when asked to cut, ship, or tag a release. Takes a semantic version number. --- # SiEval Release ## Input +The invocation argument (substituted below as `$ARGUMENTS`; if it appears +literally, read the user's request for it instead): + $ARGUMENTS Expects a semantic version number, e.g. `0.3.4`. If not provided, ask the user. diff --git a/.claude/rules/engineering-infra.md b/.claude/rules/engineering-infra.md index c933f118..cf969bac 100644 --- a/.claude/rules/engineering-infra.md +++ b/.claude/rules/engineering-infra.md @@ -1,6 +1,7 @@ --- paths: - "CLAUDE.md" + - "AGENTS.md" - "sieval/**/CLAUDE.md" - "CONTRIBUTING.md" - ".claude/rules/*.md" diff --git a/.claude/settings.json b/.claude/settings.json index 109ca93c..eef98498 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -6,26 +6,8 @@ "hooks": [ { "type": "command", - "command": "python -c \"import json,sys; d=json.load(sys.stdin); print(d.get('tool_input',{}).get('file_path','') or d.get('tool_response',{}).get('filePath',''))\" | { read -r f; echo \"$f\" | grep -qE '\\.pyi?$' && echo \"$f\" | grep -qE '(^|/)sieval/(tasks|datasets)/' || exit 0; r=$(git -C \"$(dirname \"$f\")\" rev-parse --show-toplevel 2>/dev/null) && [ -f \"$r/scripts/sync_package_stubs.py\" ] && python \"$r/scripts/sync_package_stubs.py\" || true; }", - "timeout": 30, - "statusMessage": "Post-edit checks..." - }, - { - "type": "command", - "command": "python -c \"import json,sys; d=json.load(sys.stdin); print(d.get('tool_input',{}).get('file_path','') or d.get('tool_response',{}).get('filePath',''))\" | { read -r f; echo \"$f\" | grep -qE '(^|/)sieval/(tasks|datasets)/.*\\.py$' || exit 0; r=$(git -C \"$(dirname \"$f\")\" rev-parse --show-toplevel 2>/dev/null) && [ -f \"$r/scripts/sync_meta_index.py\" ] && python \"$r/scripts/sync_meta_index.py\" || true; }", - "timeout": 30, - "statusMessage": "Post-edit checks..." - }, - { - "type": "command", - "command": "python -c \"import json,sys; d=json.load(sys.stdin); print(d.get('tool_input',{}).get('file_path','') or d.get('tool_response',{}).get('filePath',''))\" | { read -r f; echo \"$f\" | grep -qE '\\.pyi?$' && ruff check \"$f\" || true; }", - "timeout": 15, - "statusMessage": "Post-edit checks..." - }, - { - "type": "command", - "command": "python -c \"import json,sys; d=json.load(sys.stdin); print(d.get('tool_input',{}).get('file_path','') or d.get('tool_response',{}).get('filePath',''))\" | { read -r f; echo \"$f\" | grep -qE '\\.pyi?$' && ty check \"$f\" || true; }", - "timeout": 30, + "command": "python -c \"import json,sys; d=json.load(sys.stdin); print(d.get('tool_input',{}).get('file_path','') or d.get('tool_response',{}).get('filePath',''))\" | { read -r f; [ -n \"$f\" ] || exit 0; r=$(git -C \"$(dirname \"$f\")\" rev-parse --show-toplevel 2>/dev/null) && [ -f \"$r/scripts/post_edit_checks.py\" ] && python \"$r/scripts/post_edit_checks.py\" \"$f\" || true; }", + "timeout": 60, "statusMessage": "Post-edit checks..." } ] diff --git a/.claude/skills/sieval-preflight b/.claude/skills/sieval-preflight new file mode 120000 index 00000000..24fc0fa3 --- /dev/null +++ b/.claude/skills/sieval-preflight @@ -0,0 +1 @@ +../../.agents/skills/sieval-preflight \ No newline at end of file diff --git a/.claude/skills/sieval-release b/.claude/skills/sieval-release new file mode 120000 index 00000000..73d4d286 --- /dev/null +++ b/.claude/skills/sieval-release @@ -0,0 +1 @@ +../../.agents/skills/sieval-release \ No newline at end of file diff --git a/.gitignore b/.gitignore index f4551e78..3d88d719 100644 --- a/.gitignore +++ b/.gitignore @@ -169,13 +169,36 @@ tmp/ # Claude Code # Exclude non-team-level configs .claude/commands/ -.claude/skills/ +# These paths are symlinks, so the negations must stay slash-free: a +# trailing slash matches only real directories and would silently untrack +# both, with no error. Do not normalize against the `.agents/` block below. +.claude/skills/* +!.claude/skills/sieval-preflight +!.claude/skills/sieval-release .claude/worktrees/ - -# Codex -AGENTS.md .claude/settings.local.json +# Agent skills (shared by every harness) +# Team skills are tracked; personal ones are not. The trailing `*` is load +# bearing: git never descends into an excluded directory, so a negation under +# a plain `.agents/skills/` entry would never be evaluated. +.agents/skills/* +!.agents/skills/sieval-preflight/ +!.agents/skills/sieval-release/ + +# opencode's command symlinks. Pattern-based, not a per-skill literal: a +# personal skill's symlink points at a gitignored target, so anything not +# re-included here ships to a public clone as a dangling link. Symlinks again, +# so the negations stay slash-free. +.opencode/commands/* +!.opencode/commands/sieval-preflight.md +!.opencode/commands/sieval-release.md + +# opencode plugin dependencies (npm artifacts, not source). The +# `.opencode/.gitignore` that would cover this excludes itself, so it never +# ships — this root-level entry is the one a fresh clone gets. +.opencode/node_modules/ + # Local analysis artifacts POSTPROCESS_*.md usage_log.jsonl diff --git a/.opencode/commands/sieval-preflight.md b/.opencode/commands/sieval-preflight.md new file mode 120000 index 00000000..1cf99684 --- /dev/null +++ b/.opencode/commands/sieval-preflight.md @@ -0,0 +1 @@ +../../.agents/skills/sieval-preflight/SKILL.md \ No newline at end of file diff --git a/.opencode/commands/sieval-release.md b/.opencode/commands/sieval-release.md new file mode 120000 index 00000000..b6a22145 --- /dev/null +++ b/.opencode/commands/sieval-release.md @@ -0,0 +1 @@ +../../.agents/skills/sieval-release/SKILL.md \ No newline at end of file diff --git a/.opencode/plugins/post-edit.ts b/.opencode/plugins/post-edit.ts new file mode 100644 index 00000000..089787a1 --- /dev/null +++ b/.opencode/plugins/post-edit.ts @@ -0,0 +1,22 @@ +/** + * Runs sieval's post-edit checks after opencode edits a file. + * + * The checks live in scripts/post_edit_checks.py, shared with Claude Code. + * This file only dispatches. + * + * Uses `file.edited` rather than `tool.execute.after`: its payload is a clean + * `{ file: string }`, and opencode emits it from inside the write/edit/patch + * tools, so it tracks agent edits rather than any file change on disk. + * + * AI-Generated Code - Claude Sonnet 5 (Anthropic) + */ +export const PostEditChecks = async ({ $, directory }) => { + return { + event: async ({ event }) => { + if (event.type !== "file.edited") return + const file = event.properties?.file + if (!file) return + await $`python ${directory}/scripts/post_edit_checks.py ${file}`.nothrow() + }, + } +} diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..78b89953 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,167 @@ +# Repository Guidelines + +SiEval is a **model delivery quality verification system**: eval-side knowledge constraining and verifying the entire model delivery pipeline (training → conversion → infer → evaluation). + +## Architecture Principles + +1. **AI-friendly** — easy for humans, with enough detail and configurability for AI. +2. **Explicit over implicit** — benefits both humans and AI. +3. **Moderate abstraction** — do not abstract ahead of time. Extract on **coupling** (sites must change together to preserve a contract), not on call count. +4. **SOLID + KISS** — single responsibility, simple and clear, easy to extend. +5. **Minimal change surface + courage to rebuild** — reuse what works; but be willing to start over when a better overall solution exists. + +## Reproducibility + +Precise reproducibility is a product contract, not a nicety. + +- Safety guards (e.g. `--resume` strict match) ship strict-only. No `--force-*` flags, no bypass env vars, no "(future)" escape-hatch hints in errors. +- `--resume` match scope is narrowed, not bypassed: only fields that touch neither sample data nor any persisted artifact may differ across a resume — pure scheduling (concurrency, shard-I/O, write buffers), console-only progress, `auto_resume`, the non-serializable `stage_meta` hooks, and top-level `result_dir`. Everything affecting on-disk content stays strict (sampling/seeds, `max_iterations`, `shard_samples`, `record_*`, `max_retries`, `profile_*`, `detect_anomalies*`, the `progress.json` dump, and `runner_config.result_dir`, which picks the artifact directory), as does `infer_plans.yaml`. +- Recovery: "start fresh" or "match the invocation". Escape-hatch proposals must re-justify the contract, not just add a flag. + +## Project Structure & Module Organization + +- Core library code lives in `sieval/`. + - `sieval/core/`: execution engine (models, runners, persistence, profiling). + - `sieval/datasets/`: dataset wrappers/loaders. + - `sieval/tasks/`: task implementations by domain. + - `sieval/community/`: third-party/reference evaluation code; treat as vendor-style code and avoid edits unless explicitly required. +- Layer boundaries (orchestration → engine): + + ```text + cli/ → orchestration layer, depends on all modules + infer/ → can depend on core; NOT on tasks/datasets + tasks/ → depends on core + datasets + community + datasets/ → depends on core + community + core/ → zero upper-layer dependencies (independently publishable) + community/ → third-party evaluation adaptations + ``` + + - **Do not put task-specific logic into `core/`.** Each layer has its own `CLAUDE.md` with layer-specific import constraints. +- Tests are organized by purpose under `tests/`: + - `tests/unit/`, `tests/integration/`, `tests/acceptance/`, `tests/performance/`. + - `tests/tasks/` are manual scripts that may call real APIs (never CI). + +## Build, Test, and Development Commands + +- Platform: Unix (Linux, macOS) — Windows is not supported. Language: Python ≥ 3.12. +- CLI: `typer` — top-level shortcuts `sieval run`, `sieval eval`; resource groups `dataset`, `task`, `infer`, `leaderboard` (e.g. `sieval dataset list`, `sieval infer start`). +- Install deps: + - `pdm install` + - Dev/test setup: `pdm install -G dev -G test` + - Task-specific extras (install on demand): `pip install -e ".[math,ifeval,drop,t-eval,molecule,genomics,beacon]"` (or a subset) + - Lock procedures: see `.claude/rules/deps.md` (always `pdm lock --update-reuse`, never bare `pdm lock`) +- Run tests: + - Full suite: `pdm run python -m pytest -q` + - Unit + integration (coverage gate >=95%): `pdm run python -m pytest tests/unit tests/integration --cov --cov-fail-under=95 -q` + - Acceptance (release gate): `pdm run python -m pytest tests/acceptance -q -s` + - Performance (non-stress): `pdm run python -m pytest tests/performance -q -m "not stress"` +- Quality checks: + - `pdm run ruff check` + - `pdm run ruff format` + - `pdm run ty check sieval/` + - `pdm run python scripts/sync_package_stubs.py --check` + - `pre-commit run -a` + +## Architecture Guardrails + +- Preserve the async staged pipeline design: no synchronous rewrite, no global stage barriers. +- Keep resumability semantics intact (`auto_resume`, per-sample failure handling, bounded iterations). +- Persistence is append-only and sharded; disk state is source of truth. Do not replace with monolithic or memory-only storage. +- Respect record-mode behavior: `record_each_stage=True` is the fail-safe default and must remain resume-compatible. + +## Coding Style & Naming Conventions + +- Python 3.12+, 4-space indentation, PEP 8 naming (`snake_case` functions, `PascalCase` classes). +- Keep modules focused and typed; daily type checking uses `ty` (`ty check sieval/`); `mypy` (`[tool.mypy] strict = true`) is available but secondary. +- Use Ruff for linting/formatting; avoid manual style drift. +- Prefer explicit, readable test case names and concise comments for complex parameterized cases. +- Package stubs in `sieval/tasks/__init__.pyi` and `sieval/datasets/__init__.pyi` are generated; do not hand-edit. Run `scripts/sync_package_stubs.py` after adding/removing exported tasks or datasets. +- AI-generated source files must include `AI-Generated Code - ()` as the **last line** of the module-level docstring. +- Before deleting code, verify no other call sites depend on it. +- Do not use `from __future__ import annotations`. +- Import policy: + - Datasets: use package-level imports from `sieval.datasets` for official implementations. + - Tasks: use package-level imports from `sieval.tasks` for official built-in tasks. + - Custom/community tasks should prefer full module paths (for example `sieval.tasks..MyTask`) to avoid naming conflicts. + - `runners`: canonical path is `sieval.core.runners` (`TaskRunner`, `MultiTaskRunner`). + - `session`: canonical path is `sieval.cli.leaderboard.session` (`EvalSession`, `arun_session`). + - `validation`: canonical path is `sieval.cli.validation` (`validate_eval_config`, `run_dry_run`). + - `resolution`: canonical path is `sieval.cli.resolution` (`resolve_task_class`, `resolve_dataset_class`, `derive_model_type`). Reaching them via `session` resolves, but is not API. `cli/infer/` must not import `cli/leaderboard/`. + - Within the same package/subpackage: use relative imports (`from .foo import Bar`). + - Cross-package (including parent): use absolute imports (`from sieval.core.models import ...`). + - Imports imply public API. Cross-module `from sieval.x.y import _foo` in production code is a smell — promote the name or redesign the call site. + - Private modules (`_*.py`) are **protected** — accessible only within their own package subtree. Same-package siblings may use `from ._x import Y`; descendants may import via absolute path into the ancestor's private module. Peer-subpackage or out-of-subtree access is forbidden. Tests are the carve-out for both rules. + +## Testing Guidelines + +- Always run tests after making changes. See `tests/README.md` and `.claude/rules/tests.md` for details. +- `tests/unit/` directory structure must mirror `sieval/` — e.g. `sieval/core/runners/foo.py` → `tests/unit/core/runners/test_foo.py`. +- Framework: `pytest`; async tests must use `@pytest.mark.anyio` (not `pytest-asyncio`). +- `stress` marker is defined in `pyproject.toml`; exclude by default unless intentionally profiling. +- Mutation testing uses `mutmut` (on-demand, not per-commit): mutate `sieval/core` against `tests/unit` (see `[tool.mutmut]`). +- Mutation score should stay `>=70%`; surviving mutants indicate missing assertions and should be killed by improving tests. +- Common mutation commands: `mutmut run`, `mutmut results`, `mutmut show`. +- When changing core runner/model behavior, add/adjust unit tests first, then verify with relevant integration tests. +- Prefer `tmp_path` and deterministic mocks over hardcoded paths and wall-clock sleeps. + +## Commit & Pull Request Guidelines + +- Always rebase onto the latest `main` before submitting a PR. +- **MUST** read `.github/PULL_REQUEST_TEMPLATE.md` before creating any PR, and fill in all applicable sections. Do not create a PR without reading the template first. +- **MUST** read the matching template from `.github/ISSUE_TEMPLATE/` before creating any issue (bug-report, feature-request, rfc, etc.), and use the template's structure. Do not create an issue without reading the template first. +- Commit messages follow Conventional Commits (enforced by `conventional-pre-commit`): + - Examples: `feat: ...`, `fix: ...`, `refactor: ...`, `chore: ...`. +- Keep commits scoped and reviewable; include tests with code changes. +- PRs should include: + - What changed and why. + - Risk/regression notes. + - Validation evidence (exact commands run and results). + +## Agent Tooling + +This file is the single source of truth for shared conventions, and the only +instruction file every supported agent reads. `CLAUDE.md` is a one-line import +of it. Keep it under 32 KiB: Codex truncates project docs past that byte count +silently, so content near the end simply stops applying. + +- **Layer and scoped rules** — `sieval/*/CLAUDE.md` and `.claude/rules/*.md`. + Only Claude Code loads these automatically. Codex never reads a `CLAUDE.md` + below the repo root, even with the working directory in that folder, and + opencode only walks upward from it. Elsewhere these rules are reachable just + through the map below, which points at a file rather than putting the rule in + front of the agent — so treat a rule as enforced only under Claude Code. + + + +Read the matching file before editing a path it covers. Claude Code +loads these automatically; every other harness needs this map. + +| When editing | Read first | +| --- | --- | +| `sieval/cli/**` | `sieval/cli/CLAUDE.md` | +| `sieval/community/**` | `sieval/community/CLAUDE.md` | +| `sieval/core/**` | `sieval/core/CLAUDE.md` | +| `sieval/datasets/**` | `sieval/datasets/CLAUDE.md` | +| `sieval/infer/**` | `sieval/infer/CLAUDE.md` | +| `sieval/tasks/**` | `sieval/tasks/CLAUDE.md` | +| `sieval/**/*.py`, `tests/**/*.py` | `.claude/rules/code-review.md` | +| `sieval/**/*.py` | `.claude/rules/core-utils.md` | +| `sieval/datasets/**/*.py` | `.claude/rules/datasets.md` | +| `pyproject.toml`, `pdm.lock` | `.claude/rules/deps.md` | +| `CLAUDE.md`, `AGENTS.md`, `sieval/**/CLAUDE.md`, `CONTRIBUTING.md`, `.claude/rules/*.md`, `scripts/check_*.py`, `.pre-commit-config.yaml` | `.claude/rules/engineering-infra.md` | +| `sieval/infer/**/*.py` | `.claude/rules/infer.md` | +| `sieval/tasks/**/*.py`, `sieval/core/tasks/**/*.py` | `.claude/rules/records.md` | +| `scripts/*.py` | `.claude/rules/scripts.md` | +| `sieval/tasks/**/*.py` | `.claude/rules/tasks.md` | +| `tests/**/*.py` | `.claude/rules/tests.md` | + + + +- **Skills** — `.agents/skills//SKILL.md` is the source of truth, with + symlinks at `.claude/skills/` and `.opencode/commands/.md`. + Never edit through a symlink. +- **Post-edit checks** — `scripts/post_edit_checks.py` holds the behavior; + `.claude/settings.json` and `.opencode/plugins/post-edit.ts` dispatch to it + with the edited path as `argv[1]`. Codex wiring is deliberately absent: it + delivers the payload as JSON on stdin, so it needs an adapter no one can + verify here. Under Codex, run `python scripts/post_edit_checks.py `. diff --git a/CLAUDE.md b/CLAUDE.md index 9b7d591a..da0659fb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,69 +1,2 @@ -# SiEval — Project-Wide Guidelines - -SiEval is a **model delivery quality verification system**: eval-side knowledge constraining and verifying the entire model delivery pipeline (training → conversion → infer → evaluation). - -## Architecture Principles - -1. **AI-friendly** — easy for humans, with enough detail and configurability for AI. -2. **Explicit over implicit** — benefits both humans and AI. -3. **Moderate abstraction** — do not abstract ahead of time. Extract on **coupling** (sites must change together to preserve a contract), not on call count. -4. **SOLID + KISS** — single responsibility, simple and clear, easy to extend. -5. **Minimal change surface + courage to rebuild** — reuse what works; but be willing to start over when a better overall solution exists. - -## Reproducibility - -Precise reproducibility is a product contract, not a nicety. - -* Safety guards (e.g. `--resume` strict match) ship strict-only. No `--force-*` flags, no bypass env vars, no "(future)" escape-hatch hints in errors. -* `--resume` match scope is narrowed, not bypassed: only fields that touch neither sample data nor any persisted artifact may differ across a resume — pure scheduling (concurrency, shard-I/O, write buffers), console-only progress, `auto_resume`, the non-serializable `stage_meta` hooks, and top-level `result_dir`. Everything affecting on-disk content stays strict (sampling/seeds, `max_iterations`, `shard_samples`, `record_*`, `max_retries`, `profile_*`, `detect_anomalies*`, the `progress.json` dump, and `runner_config.result_dir`, which picks the artifact directory), as does `infer_plans.yaml`. -* Recovery: "start fresh" or "match the invocation". Escape-hatch proposals must re-justify the contract, not just add a flag. - -## Toolchain - -* Platform: Unix (Linux, macOS) — Windows is not supported -* Language: Python ≥ 3.12 -* Package manager: `pdm` — see `.claude/rules/deps.md` for lock procedures -* Formatting & lint: `ruff`; type checking: `ty` (primary), `mypy` (strict mode, secondary) -* Tests: `pytest` -* CLI: `typer` — top-level shortcuts `sieval run`, `sieval eval`; resource groups `dataset`, `task`, `infer`, `leaderboard` (e.g. `sieval dataset list`, `sieval infer start`) - -## Layer Boundaries & Dependencies - -```text -cli/ → orchestration layer, depends on all modules -infer/ → can depend on core; NOT on tasks/datasets -tasks/ → depends on core + datasets + community -datasets/ → depends on core + community -core/ → zero upper-layer dependencies (independently publishable) -community/ → third-party evaluation adaptations -``` - -Each layer has its own `CLAUDE.md` with layer-specific import constraints. **Do not put task-specific logic into `core/`.** - -## Import Policy - -* `datasets`, `tasks`: package-level imports are the official entrypoint (`from sieval.tasks import ...`) -* `runners`: canonical path is `sieval.core.runners` (`TaskRunner`, `MultiTaskRunner`). -* `session`: canonical path is `sieval.cli.leaderboard.session` (`EvalSession`, `arun_session`). -* `validation`: canonical path is `sieval.cli.validation` (`validate_eval_config`, `run_dry_run`). -* `resolution`: canonical path is `sieval.cli.resolution` (`resolve_task_class`, `resolve_dataset_class`, `derive_model_type`). Reaching them via `session` resolves, but is not API. `cli/infer/` must not import `cli/leaderboard/`. -* Same package: relative imports. Cross-package: absolute imports. -* Imports imply public API. Cross-module `from sieval.x.y import _foo` in production code is a smell — promote the name or redesign the call site. -* Private modules (`_*.py`) are **protected** — accessible only within their own package subtree. Same-package siblings may use `from ._x import Y`; descendants may import via absolute path into the ancestor's private module. Peer-subpackage or out-of-subtree access is forbidden. Tests are the carve-out for both rules. - -## Test Rules - -* Always run tests after making changes. See [tests/README.md](tests/README.md) and `.claude/rules/tests.md` for details. -* `tests/unit/` directory structure must mirror `sieval/` — e.g. `sieval/core/runners/foo.py` → `tests/unit/core/runners/test_foo.py`. - -## PR / Issue Hygiene - -* Always rebase onto the latest `main` before submitting a PR. -* **MUST** read `.github/PULL_REQUEST_TEMPLATE.md` before creating any PR, and fill in all applicable sections. Do not create a PR without reading the template first. -* **MUST** read the matching template from `.github/ISSUE_TEMPLATE/` before creating any issue (bug-report, feature-request, rfc, etc.), and use the template's structure. Do not create an issue without reading the template first. - -## Code Conventions - -* AI-generated source files must include `AI-Generated Code - ()` as the **last line** of the module-level docstring. -* Before deleting code, verify no other call sites depend on it. -* Do not use `from __future__ import annotations`. + +@AGENTS.md diff --git a/scripts/check_preflight.py b/scripts/check_preflight.py index 47ef77fd..d6ee030d 100644 --- a/scripts/check_preflight.py +++ b/scripts/check_preflight.py @@ -1209,6 +1209,7 @@ class PreflightRunner: "check_imports", "check_examples", "check_meta_index_sync", + "check_agent_rule_map", "check_mutmut_config", "check_version", ] @@ -3063,6 +3064,49 @@ def check_meta_index_sync(self) -> list[CheckResult]: ) ] + def check_agent_rule_map(self) -> list[CheckResult]: + """The rule map in ``AGENTS.md`` must match the live rule inventory. + + opencode and Codex reach the layer and scoped rules only through this + map. A stale map does not fail loudly for them — it silently points at + rules that moved, or omits ones that were added. + """ + script = self.project_root / "scripts" / "sync_agent_rules.py" + if not script.exists(): + return [ + CheckResult( + "FAIL", + "check_agent_rule_map", + f"script not found: {script}", + ) + ] + + result = subprocess.run( + [sys.executable, str(script), "--check"], + capture_output=True, + text=True, + cwd=self.project_root, + ) + if result.returncode == 0: + return [ + CheckResult( + "PASS", + "check_agent_rule_map", + "AGENTS.md rule map matches the live rule inventory", + ) + ] + + message = (result.stderr or result.stdout).strip().splitlines() + return [ + CheckResult( + "FAIL", + "check_agent_rule_map", + "AGENTS.md rule map is out of date; " + "run `python scripts/sync_agent_rules.py` to regenerate", + message, + ) + ] + def check_imports(self) -> list[CheckResult]: script = self.project_root / "scripts" / "check_layer_imports.py" if not script.exists(): diff --git a/scripts/post_edit_checks.py b/scripts/post_edit_checks.py new file mode 100644 index 00000000..df266c6c --- /dev/null +++ b/scripts/post_edit_checks.py @@ -0,0 +1,76 @@ +"""Run post-edit checks for a single edited file, passed as ``argv[1]``. + +Claude Code and opencode each fire a hook after a file edit. Keeping the logic +here rather than inlining it per harness means a pattern change cannot drift +between them — and drift is silent, since the failure mode is a file class +quietly no longer being checked. + +Always exits 0: a post-edit check informs the agent, it does not block the edit +that already happened. + +AI-Generated Code - Claude Sonnet 5 (Anthropic) +""" + +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + +# A Python source or stub file. +PY_RE = re.compile(r"\.pyi?$") +# A file under sieval/tasks/ or sieval/datasets/, at any depth, matching both +# repo-relative and absolute paths as passed by the different harnesses. +REGISTRY_RE = re.compile(r"(^|/)sieval/(tasks|datasets)/") + + +def select_checks(path: str) -> set[str]: + """Return the check names that apply to ``path``.""" + if not path: + return set() + + checks: set[str] = set() + is_py = bool(PY_RE.search(path)) + in_registry = bool(REGISTRY_RE.search(path)) + + if is_py: + checks.update({"ruff", "ty"}) + if is_py and in_registry: + checks.add("stubs") + if in_registry and path.endswith(".py"): + checks.add("meta") + return checks + + +def _run(cmd: list[str]) -> int: + """Run a command from the repo root, streaming output. Never raises.""" + try: + return subprocess.run(cmd, cwd=ROOT).returncode + except OSError as exc: + print(f"post-edit check could not run {cmd[0]}: {exc}", file=sys.stderr) + return 0 + + +def main(argv: list[str] | None = None) -> int: + args = sys.argv[1:] if argv is None else argv + if not args: + return 0 + + path = args[0] + checks = select_checks(path) + + if "stubs" in checks: + _run([sys.executable, str(ROOT / "scripts" / "sync_package_stubs.py")]) + if "meta" in checks: + _run([sys.executable, str(ROOT / "scripts" / "sync_meta_index.py")]) + if "ruff" in checks: + _run(["ruff", "check", path]) + if "ty" in checks: + _run(["ty", "check", path]) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/sanitize.sh b/scripts/sanitize.sh index b8257bc8..e4f8fc80 100755 --- a/scripts/sanitize.sh +++ b/scripts/sanitize.sh @@ -38,7 +38,11 @@ ALLOWLIST=( "scripts/sanitize.sh" # this script itself (contains the patterns) "docs/" # local design/planning docs "tests/" # test fixtures use example IPs/paths - "CLAUDE.md" # project guidelines + # Anchored to git grep's `path:lineno:` prefix, so these exempt the guideline + # files themselves, not every line that merely names one: unanchored, a + # comment like `BAD = "/home/u/x" # see AGENTS.md` would exempt itself. + "^CLAUDE\.md:" # project guidelines + "^AGENTS\.md:" # project guidelines (source of truth) "\.git/" # git internals "127\.0\.0\.1" # localhost is fine "0\.0\.0\.0" # bind-all is fine diff --git a/scripts/sync_agent_rules.py b/scripts/sync_agent_rules.py new file mode 100644 index 00000000..dd877d84 --- /dev/null +++ b/scripts/sync_agent_rules.py @@ -0,0 +1,173 @@ +"""Regenerate the agent rule map inside AGENTS.md. + +Claude Code auto-loads `sieval/*/CLAUDE.md` (on demand) and `.claude/rules/*.md` +(by `paths:` glob). No other harness does: opencode only walks upward from the +working directory, and Codex reads AGENTS.md files on the root-to-cwd path. +They need an explicit index. + +Hand-writing it would duplicate the `paths:` frontmatter that already declares +each rule's scope, and would go stale with no signal — so it is generated, the +way sync_meta_index.py handles the registry. + +AI-Generated Code - Claude Sonnet 5 (Anthropic) +""" + +import argparse +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + +BEGIN = "" +END = "" + +# Codex's project_doc_max_bytes default. Past this, it truncates silently: +# instructions near the end of the file simply stop applying, with no warning. +MAX_BYTES = 32768 + + +def _parse_paths_frontmatter(text: str, source: str = "") -> list[str]: + """Return the `paths:` globs from a rule file's YAML frontmatter. + + Hand-rolled rather than via PyYAML: this runs from pre-commit and preflight, + where an import-light footprint matters, and the shape is fixed (a `paths:` + key over a list of strings). + + A file with no `paths:` key is scoped to everything, which is a real choice. + A `paths:` key this parser cannot read is not: returning ``[]`` there would + drop the rule from the map with nothing to notice it, since the check + compares the generated map against the same empty parse. So it raises. + """ + if not text.startswith("---"): + return [] + end = text.find("\n---", 3) + if end == -1: + return [] + block = text[3:end] + + globs: list[str] = [] + in_paths = False + seen_paths = False + for raw in block.splitlines(): + line = raw.rstrip() + if not line: + continue + stripped = line.strip() + if stripped.startswith("paths:"): + # Only a block list is supported; `paths: [...]` on one line is not. + if stripped != "paths:": + raise SystemExit( + f'{source}: `paths:` must be a block list, one `- "glob"` ' + f"per line, not {stripped[len('paths:') :].strip()!r}." + ) + in_paths = True + seen_paths = True + continue + if in_paths: + if stripped.startswith("- "): + globs.append(stripped[2:].strip().strip('"').strip("'")) + continue + # A non-list line at the same or lower indent ends the paths block. + if not line.startswith(" "): + in_paths = False + + if seen_paths and not globs: + raise SystemExit(f"{source}: `paths:` is present but lists no globs.") + return globs + + +def render_block(root: Path) -> str: + """Render the rule map from layer files and rule frontmatter.""" + rows: list[tuple[str, str]] = [] + + for layer_file in sorted((root / "sieval").glob("*/CLAUDE.md")): + layer = layer_file.parent.name + rows.append((f"`sieval/{layer}/**`", f"`sieval/{layer}/CLAUDE.md`")) + + rules_dir = root / ".claude" / "rules" + for rule_file in sorted(rules_dir.glob("*.md")) if rules_dir.is_dir() else []: + rel = rule_file.relative_to(root).as_posix() + globs = _parse_paths_frontmatter(rule_file.read_text(encoding="utf-8"), rel) + scope = ", ".join(f"`{g}`" for g in globs) if globs else "_(always)_" + rows.append((scope, f"`{rel}`")) + + lines = [ + BEGIN, + "", + "Read the matching file before editing a path it covers. Claude Code", + "loads these automatically; every other harness needs this map.", + "", + "| When editing | Read first |", + "| --- | --- |", + ] + lines += [f"| {scope} | {target} |" for scope, target in rows] + lines += ["", END] + return "\n".join(lines) + + +def render_document(root: Path) -> str: + """Return AGENTS.md with a freshly generated block.""" + agents_path = root / "AGENTS.md" + if not agents_path.exists(): + raise SystemExit(f"not found: {agents_path}") + + current = agents_path.read_text(encoding="utf-8") + start = current.find(BEGIN) + stop = current.find(END) + if start == -1 or stop == -1: + raise SystemExit( + f"AGENTS.md is missing the generated markers ({BEGIN} / {END}). " + "Add them before running this script." + ) + + return current[:start] + render_block(root) + current[stop + len(END) :] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Regenerate the agent rule map inside AGENTS.md." + ) + parser.add_argument( + "--check", + action="store_true", + help="Check whether the committed map is up-to-date without writing.", + ) + parser.add_argument( + "--root", + type=Path, + default=ROOT, + help="Repository root (defaults to this checkout).", + ) + args = parser.parse_args(argv) + + root = args.root.resolve() + agents_path = root / "AGENTS.md" + rendered = render_document(root) + + size = len(rendered.encode("utf-8")) + if size > MAX_BYTES: + raise SystemExit( + f"AGENTS.md would be {size} bytes, over the {MAX_BYTES}-byte budget. " + "Codex truncates project docs past this point silently — trim the " + "file rather than raising the cap." + ) + + current = agents_path.read_text(encoding="utf-8") + if args.check: + if current == rendered: + return 0 + raise SystemExit( + "AGENTS.md rule map is out of date. " + "Run `python scripts/sync_agent_rules.py` to regenerate." + ) + + if current != rendered: + agents_path.write_text(rendered, encoding="utf-8") + print(f"Updated rule map in {agents_path.relative_to(root)}") + else: + print("Rule map already up to date") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/unit/scripts/test_check_preflight.py b/tests/unit/scripts/test_check_preflight.py index f3836c32..bd9a80e3 100644 --- a/tests/unit/scripts/test_check_preflight.py +++ b/tests/unit/scripts/test_check_preflight.py @@ -135,10 +135,11 @@ class TestPreflightRunner: """Runner orchestration.""" def test_all_checks_listed(self): - assert len(PreflightRunner.ALL_CHECKS) == 14 + assert len(PreflightRunner.ALL_CHECKS) == 15 assert "check_links" in PreflightRunner.ALL_CHECKS assert "check_examples" in PreflightRunner.ALL_CHECKS assert "check_meta_index_sync" in PreflightRunner.ALL_CHECKS + assert "check_agent_rule_map" in PreflightRunner.ALL_CHECKS assert "check_version" in PreflightRunner.ALL_CHECKS assert "check_task_shot_knobs" in PreflightRunner.ALL_CHECKS assert "check_report_declarations" in PreflightRunner.ALL_CHECKS diff --git a/tests/unit/scripts/test_post_edit_checks.py b/tests/unit/scripts/test_post_edit_checks.py new file mode 100644 index 00000000..a6145564 --- /dev/null +++ b/tests/unit/scripts/test_post_edit_checks.py @@ -0,0 +1,82 @@ +"""Tests for scripts/post_edit_checks.py. + +The script is called by three agent harnesses after a file edit. Tests cover +which checks each path selects, and that the script never fails the caller — +a post-edit check informs, it does not block the edit. + +AI-Generated Code - Claude Sonnet 5 (Anthropic) +""" + +import importlib.util +import sys +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).resolve().parents[3] / "scripts" / "post_edit_checks.py" + + +def _load(): + spec = importlib.util.spec_from_file_location("post_edit_checks", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules["post_edit_checks"] = module + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize( + ("path", "expected"), + [ + ("sieval/tasks/foo.py", {"stubs", "meta", "ruff", "ty"}), + ("sieval/datasets/bar.py", {"stubs", "meta", "ruff", "ty"}), + ("sieval/tasks/__init__.pyi", {"stubs", "ruff", "ty"}), + ("sieval/core/runners/baz.py", {"ruff", "ty"}), + ("tests/unit/test_x.py", {"ruff", "ty"}), + ("README.md", set()), + ("pyproject.toml", set()), + ], +) +def test_selects_checks_by_path(path: str, expected: set[str]): + mod = _load() + assert mod.select_checks(path) == expected + + +def test_absolute_paths_match_the_same_way(): + """Harnesses pass absolute paths; the sieval/ prefix must still match.""" + mod = _load() + assert mod.select_checks("/home/u/repo/sieval/tasks/foo.py") == { + "stubs", + "meta", + "ruff", + "ty", + } + + +def test_similar_paths_outside_sieval_do_not_match(): + """A vendored or nested `sieval/tasks/` elsewhere is still a task file, but + a lookalike like `mysieval/tasks/` is not.""" + mod = _load() + assert "meta" not in mod.select_checks("mysieval/tasks/foo.py") + + +def test_empty_path_selects_nothing(): + mod = _load() + assert mod.select_checks("") == set() + + +def test_main_exits_zero_on_unmatched_path(): + mod = _load() + assert mod.main(["README.md"]) == 0 + + +def test_main_exits_zero_with_no_argument(): + mod = _load() + assert mod.main([]) == 0 + + +def test_main_exits_zero_even_when_a_check_fails(monkeypatch): + """A failing lint must not fail the caller's edit.""" + mod = _load() + monkeypatch.setattr(mod, "_run", lambda *a, **k: 1) + assert mod.main(["sieval/tasks/foo.py"]) == 0 diff --git a/tests/unit/scripts/test_sync_agent_rules.py b/tests/unit/scripts/test_sync_agent_rules.py new file mode 100644 index 00000000..fc10666e --- /dev/null +++ b/tests/unit/scripts/test_sync_agent_rules.py @@ -0,0 +1,156 @@ +"""Tests for scripts/sync_agent_rules.py. + +The generator reads `paths:` frontmatter from .claude/rules/*.md and the +locations of sieval/*/CLAUDE.md, then rewrites a marker-delimited block in +AGENTS.md. Tests drive it against a synthetic tree via --root so they never +depend on the real repository's rule inventory. + +AI-Generated Code - Claude Sonnet 5 (Anthropic) +""" + +import importlib.util +import sys +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).resolve().parents[3] / "scripts" / "sync_agent_rules.py" + + +def _load(): + spec = importlib.util.spec_from_file_location("sync_agent_rules", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules["sync_agent_rules"] = module + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def tree(tmp_path: Path) -> Path: + """A synthetic repo: one layer file, two rules, one AGENTS.md.""" + (tmp_path / "sieval" / "core").mkdir(parents=True) + (tmp_path / "sieval" / "core" / "CLAUDE.md").write_text("# Core\n") + + rules = tmp_path / ".claude" / "rules" + rules.mkdir(parents=True) + (rules / "tests.md").write_text( + '---\npaths:\n - "tests/**/*.py"\n---\n\n# Test Rules\n' + ) + (rules / "deps.md").write_text( + '---\npaths:\n - "pyproject.toml"\n - "pdm.lock"\n---\n\n# Deps\n' + ) + + (tmp_path / "AGENTS.md").write_text( + "# Guidelines\n\n" + "\n" + "\n" + ) + return tmp_path + + +def test_writes_layer_file_with_derived_scope(tree: Path): + mod = _load() + assert mod.main(["--root", str(tree)]) == 0 + text = (tree / "AGENTS.md").read_text() + assert "`sieval/core/**`" in text + assert "`sieval/core/CLAUDE.md`" in text + + +def test_reads_paths_frontmatter_verbatim(tree: Path): + mod = _load() + mod.main(["--root", str(tree)]) + text = (tree / "AGENTS.md").read_text() + assert "`tests/**/*.py`" in text + assert "`pyproject.toml`" in text + assert "`pdm.lock`" in text + + +def test_rule_without_paths_is_scoped_to_everything(tree: Path): + """No `paths:` key is a real choice, not a parse failure.""" + mod = _load() + (tree / ".claude" / "rules" / "always.md").write_text( + "---\ndescription: applies everywhere\n---\n\n# Always\n" + ) + assert mod.main(["--root", str(tree)]) == 0 + assert "_(always)_" in (tree / "AGENTS.md").read_text() + + +@pytest.mark.parametrize( + ("frontmatter", "expected"), + [ + ('---\npaths: ["a/**", "b/**"]\n---\n\n# Flow\n', "block list"), + ("---\npaths:\n---\n\n# Empty\n", "lists no globs"), + ('---\npaths: "a/**"\n---\n\n# Scalar\n', "block list"), + ], +) +def test_unreadable_paths_key_is_an_error(tree: Path, frontmatter: str, expected: str): + """A `paths:` key the parser cannot read must not silently drop the rule. + + Returning [] would render it as `_(always)_` and still satisfy --check, + since the check re-parses the same frontmatter the same wrong way. + """ + mod = _load() + (tree / ".claude" / "rules" / "broken.md").write_text(frontmatter) + with pytest.raises(SystemExit) as exc: + mod.main(["--root", str(tree)]) + assert expected in str(exc.value) + assert "broken.md" in str(exc.value) + + +def test_paths_are_backticked_not_markdown_links(tree: Path): + """check_links validates Markdown links in tracked files; backticks keep + the generated block off that check's surface.""" + mod = _load() + mod.main(["--root", str(tree)]) + block = (tree / "AGENTS.md").read_text() + assert "](" not in block + + +def test_generation_is_idempotent(tree: Path): + mod = _load() + mod.main(["--root", str(tree)]) + once = (tree / "AGENTS.md").read_text() + mod.main(["--root", str(tree)]) + assert (tree / "AGENTS.md").read_text() == once + + +def test_check_passes_when_current(tree: Path): + mod = _load() + mod.main(["--root", str(tree)]) + assert mod.main(["--root", str(tree), "--check"]) == 0 + + +def test_check_fails_when_stale(tree: Path): + mod = _load() + mod.main(["--root", str(tree)]) + rules = tree / ".claude" / "rules" + (rules / "newrule.md").write_text('---\npaths:\n - "src/**"\n---\n\n# New\n') + with pytest.raises(SystemExit): + mod.main(["--root", str(tree), "--check"]) + + +def test_check_fails_when_block_hand_edited(tree: Path): + mod = _load() + mod.main(["--root", str(tree)]) + agents = tree / "AGENTS.md" + agents.write_text(agents.read_text().replace("sieval/core/**", "sieval/bogus/**")) + with pytest.raises(SystemExit): + mod.main(["--root", str(tree), "--check"]) + + +def test_rejects_output_over_size_budget(tree: Path): + """Codex truncates project docs past 32 KiB silently.""" + mod = _load() + agents = tree / "AGENTS.md" + agents.write_text("x" * 33000 + "\n" + agents.read_text()) + with pytest.raises(SystemExit) as exc: + mod.main(["--root", str(tree)]) + assert "32768" in str(exc.value) + + +def test_missing_markers_is_an_error(tree: Path): + mod = _load() + (tree / "AGENTS.md").write_text("# Guidelines\n") + with pytest.raises(SystemExit): + mod.main(["--root", str(tree)])