refactor(config): make the agent config harness-neutral

.claude/commands/sieval-preflight.md → .agents/skills/sieval-preflight/SKILL.md

@@ -1,11 +1,15 @@
 ---
-description: Run preflight checks on the sieval codebase (link validation, dependency consistency, task/dataset registration, version alignment)
+name: sieval-preflight
+description: Run sieval's preflight checks — link validation, dependency consistency, task/dataset registration, version alignment. Use before a release, before opening a PR, or when asked to verify repository health.
 ---
 
 # SiEval Preflight Checks
 
 ## Input
 
+The invocation argument (substituted below as `$ARGUMENTS`; if it appears
+literally, read the user's request for it instead):
+
 $ARGUMENTS
 
 Accepts: `deep`, `quick` (default), or a specific check name (e.g., `check_links`, `check_deps`, `check_dep_coverage`, `check_tasks`, `check_task_shot_knobs`, `check_record_key_access`, `check_datasets`, `check_examples`, `check_imports`, `check_meta_index_sync`, `check_version`).

.claude/commands/sieval-release.md → .agents/skills/sieval-release/SKILL.md

@@ -1,11 +1,15 @@
 ---
-description: Release a new version of sieval (update changelog, tag, push)
+name: sieval-release
+description: Release a new version of sieval — run deep preflight, update CHANGELOG and Dockerfile, tag, and push. Use when asked to cut, ship, or tag a release. Takes a semantic version number.
 ---
 
 # SiEval Release
 
 ## Input
 
+The invocation argument (substituted below as `$ARGUMENTS`; if it appears
+literally, read the user's request for it instead):
+
 $ARGUMENTS
 
 Expects a semantic version number, e.g. `0.3.4`. If not provided, ask the user.

.claude/rules/engineering-infra.md

@@ -1,6 +1,7 @@
 ---
 paths:
   - "CLAUDE.md"
+  - "AGENTS.md"
   - "sieval/**/CLAUDE.md"
   - "CONTRIBUTING.md"
   - ".claude/rules/*.md"

.claude/settings.json

@@ -6,26 +6,8 @@
         "hooks": [
           {
             "type": "command",
-            "command": "python -c \"import json,sys; d=json.load(sys.stdin); print(d.get('tool_input',{}).get('file_path','') or d.get('tool_response',{}).get('filePath',''))\" | { read -r f; echo \"$f\" | grep -qE '\\.pyi?$' && echo \"$f\" | grep -qE '(^|/)sieval/(tasks|datasets)/' || exit 0; r=$(git -C \"$(dirname \"$f\")\" rev-parse --show-toplevel 2>/dev/null) && [ -f \"$r/scripts/sync_package_stubs.py\" ] && python \"$r/scripts/sync_package_stubs.py\" || true; }",
-            "timeout": 30,
-            "statusMessage": "Post-edit checks..."
-          },
-          {
-            "type": "command",
-            "command": "python -c \"import json,sys; d=json.load(sys.stdin); print(d.get('tool_input',{}).get('file_path','') or d.get('tool_response',{}).get('filePath',''))\" | { read -r f; echo \"$f\" | grep -qE '(^|/)sieval/(tasks|datasets)/.*\\.py$' || exit 0; r=$(git -C \"$(dirname \"$f\")\" rev-parse --show-toplevel 2>/dev/null) && [ -f \"$r/scripts/sync_meta_index.py\" ] && python \"$r/scripts/sync_meta_index.py\" || true; }",
-            "timeout": 30,
-            "statusMessage": "Post-edit checks..."
-          },
-          {
-            "type": "command",
-            "command": "python -c \"import json,sys; d=json.load(sys.stdin); print(d.get('tool_input',{}).get('file_path','') or d.get('tool_response',{}).get('filePath',''))\" | { read -r f; echo \"$f\" | grep -qE '\\.pyi?$' && ruff check \"$f\" || true; }",
-            "timeout": 15,
-            "statusMessage": "Post-edit checks..."
-          },
-          {
-            "type": "command",
-            "command": "python -c \"import json,sys; d=json.load(sys.stdin); print(d.get('tool_input',{}).get('file_path','') or d.get('tool_response',{}).get('filePath',''))\" | { read -r f; echo \"$f\" | grep -qE '\\.pyi?$' && ty check \"$f\" || true; }",
-            "timeout": 30,
+            "command": "python -c \"import json,sys; d=json.load(sys.stdin); print(d.get('tool_input',{}).get('file_path','') or d.get('tool_response',{}).get('filePath',''))\" | { read -r f; [ -n \"$f\" ] || exit 0; r=$(git -C \"$(dirname \"$f\")\" rev-parse --show-toplevel 2>/dev/null) && [ -f \"$r/scripts/post_edit_checks.py\" ] && python \"$r/scripts/post_edit_checks.py\" \"$f\" || true; }",
+            "timeout": 60,
             "statusMessage": "Post-edit checks..."
           }
         ]

.claude/skills/sieval-preflight

@@ -0,0 +1 @@
+../../.agents/skills/sieval-preflight

.claude/skills/sieval-release

@@ -0,0 +1 @@
+../../.agents/skills/sieval-release

.gitignore

@@ -169,13 +169,36 @@ tmp/
 # Claude Code
 #   Exclude non-team-level configs
 .claude/commands/
-.claude/skills/
+#   These paths are symlinks, so the negations must stay slash-free: a
+#   trailing slash matches only real directories and would silently untrack
+#   both, with no error. Do not normalize against the `.agents/` block below.
+.claude/skills/*
+!.claude/skills/sieval-preflight
+!.claude/skills/sieval-release
 .claude/worktrees/
-
-# Codex
-AGENTS.md
 .claude/settings.local.json
 
+# Agent skills (shared by every harness)
+#   Team skills are tracked; personal ones are not. The trailing `*` is load
+#   bearing: git never descends into an excluded directory, so a negation under
+#   a plain `.agents/skills/` entry would never be evaluated.
+.agents/skills/*
+!.agents/skills/sieval-preflight/
+!.agents/skills/sieval-release/
+
+# opencode's command symlinks. Pattern-based, not a per-skill literal: a
+# personal skill's symlink points at a gitignored target, so anything not
+# re-included here ships to a public clone as a dangling link. Symlinks again,
+# so the negations stay slash-free.
+.opencode/commands/*
+!.opencode/commands/sieval-preflight.md
+!.opencode/commands/sieval-release.md
+
+# opencode plugin dependencies (npm artifacts, not source). The
+# `.opencode/.gitignore` that would cover this excludes itself, so it never
+# ships — this root-level entry is the one a fresh clone gets.
+.opencode/node_modules/
+
 # Local analysis artifacts
 POSTPROCESS_*.md
 usage_log.jsonl

.opencode/commands/sieval-preflight.md

@@ -0,0 +1 @@
+../../.agents/skills/sieval-preflight/SKILL.md

.opencode/commands/sieval-release.md

@@ -0,0 +1 @@
+../../.agents/skills/sieval-release/SKILL.md

.opencode/plugins/post-edit.ts

@@ -0,0 +1,22 @@
+/**
+ * Runs sieval's post-edit checks after opencode edits a file.
+ *
+ * The checks live in scripts/post_edit_checks.py, shared with Claude Code.
+ * This file only dispatches.
+ *
+ * Uses `file.edited` rather than `tool.execute.after`: its payload is a clean
+ * `{ file: string }`, and opencode emits it from inside the write/edit/patch
+ * tools, so it tracks agent edits rather than any file change on disk.
+ *
+ * AI-Generated Code - Claude Sonnet 5 (Anthropic)
+ */
+export const PostEditChecks = async ({ $, directory }) => {
+  return {
+    event: async ({ event }) => {
+      if (event.type !== "file.edited") return
+      const file = event.properties?.file
+      if (!file) return
+      await $`python ${directory}/scripts/post_edit_checks.py ${file}`.nothrow()
+    },
+  }
+}

AGENTS.md

@@ -0,0 +1,167 @@
+# Repository Guidelines
+
+SiEval is a **model delivery quality verification system**: eval-side knowledge constraining and verifying the entire model delivery pipeline (training → conversion → infer → evaluation).
+
+## Architecture Principles
+
+1. **AI-friendly** — easy for humans, with enough detail and configurability for AI.
+2. **Explicit over implicit** — benefits both humans and AI.
+3. **Moderate abstraction** — do not abstract ahead of time. Extract on **coupling** (sites must change together to preserve a contract), not on call count.
+4. **SOLID + KISS** — single responsibility, simple and clear, easy to extend.
+5. **Minimal change surface + courage to rebuild** — reuse what works; but be willing to start over when a better overall solution exists.
+
+## Reproducibility
+
+Precise reproducibility is a product contract, not a nicety.
+
+- Safety guards (e.g. `--resume` strict match) ship strict-only. No `--force-*` flags, no bypass env vars, no "(future)" escape-hatch hints in errors.
+- `--resume` match scope is narrowed, not bypassed: only fields that touch neither sample data nor any persisted artifact may differ across a resume — pure scheduling (concurrency, shard-I/O, write buffers), console-only progress, `auto_resume`, the non-serializable `stage_meta` hooks, and top-level `result_dir`. Everything affecting on-disk content stays strict (sampling/seeds, `max_iterations`, `shard_samples`, `record_*`, `max_retries`, `profile_*`, `detect_anomalies*`, the `progress.json` dump, and `runner_config.result_dir`, which picks the artifact directory), as does `infer_plans.yaml`.
+- Recovery: "start fresh" or "match the invocation". Escape-hatch proposals must re-justify the contract, not just add a flag.
+
+## Project Structure & Module Organization
+
+- Core library code lives in `sieval/`.
+    - `sieval/core/`: execution engine (models, runners, persistence, profiling).
+    - `sieval/datasets/`: dataset wrappers/loaders.
+    - `sieval/tasks/`: task implementations by domain.
+    - `sieval/community/`: third-party/reference evaluation code; treat as vendor-style code and avoid edits unless explicitly required.
+- Layer boundaries (orchestration → engine):
+
+    ```text
+    cli/          → orchestration layer, depends on all modules
+    infer/        → can depend on core; NOT on tasks/datasets
+    tasks/        → depends on core + datasets + community
+    datasets/     → depends on core + community
+    core/         → zero upper-layer dependencies (independently publishable)
+    community/    → third-party evaluation adaptations
+    ```
+
+    - **Do not put task-specific logic into `core/`.** Each layer has its own `CLAUDE.md` with layer-specific import constraints.
+- Tests are organized by purpose under `tests/`:
+    - `tests/unit/`, `tests/integration/`, `tests/acceptance/`, `tests/performance/`.
+    - `tests/tasks/` are manual scripts that may call real APIs (never CI).
+
+## Build, Test, and Development Commands
+
+- Platform: Unix (Linux, macOS) — Windows is not supported. Language: Python ≥ 3.12.
+- CLI: `typer` — top-level shortcuts `sieval run`, `sieval eval`; resource groups `dataset`, `task`, `infer`, `leaderboard` (e.g. `sieval dataset list`, `sieval infer start`).
+- Install deps:
+    - `pdm install`
+    - Dev/test setup: `pdm install -G dev -G test`
+    - Task-specific extras (install on demand): `pip install -e ".[math,ifeval,drop,t-eval,molecule,genomics,beacon]"` (or a subset)
+    - Lock procedures: see `.claude/rules/deps.md` (always `pdm lock --update-reuse`, never bare `pdm lock`)
+- Run tests:
+    - Full suite: `pdm run python -m pytest -q`
+    - Unit + integration (coverage gate >=95%): `pdm run python -m pytest tests/unit tests/integration --cov --cov-fail-under=95 -q`
+    - Acceptance (release gate): `pdm run python -m pytest tests/acceptance -q -s`
+    - Performance (non-stress): `pdm run python -m pytest tests/performance -q -m "not stress"`
+- Quality checks:
+    - `pdm run ruff check`
+    - `pdm run ruff format`
+    - `pdm run ty check sieval/`
+    - `pdm run python scripts/sync_package_stubs.py --check`
+    - `pre-commit run -a`
+
+## Architecture Guardrails
+
+- Preserve the async staged pipeline design: no synchronous rewrite, no global stage barriers.
+- Keep resumability semantics intact (`auto_resume`, per-sample failure handling, bounded iterations).
+- Persistence is append-only and sharded; disk state is source of truth. Do not replace with monolithic or memory-only storage.
+- Respect record-mode behavior: `record_each_stage=True` is the fail-safe default and must remain resume-compatible.
+
+## Coding Style & Naming Conventions
+
+- Python 3.12+, 4-space indentation, PEP 8 naming (`snake_case` functions, `PascalCase` classes).
+- Keep modules focused and typed; daily type checking uses `ty` (`ty check sieval/`); `mypy` (`[tool.mypy] strict = true`) is available but secondary.
+- Use Ruff for linting/formatting; avoid manual style drift.
+- Prefer explicit, readable test case names and concise comments for complex parameterized cases.
+- Package stubs in `sieval/tasks/__init__.pyi` and `sieval/datasets/__init__.pyi` are generated; do not hand-edit. Run `scripts/sync_package_stubs.py` after adding/removing exported tasks or datasets.
+- AI-generated source files must include `AI-Generated Code - <model> (<provider>)` as the **last line** of the module-level docstring.
+- Before deleting code, verify no other call sites depend on it.
+- Do not use `from __future__ import annotations`.
+- Import policy:
+    - Datasets: use package-level imports from `sieval.datasets` for official implementations.
+    - Tasks: use package-level imports from `sieval.tasks` for official built-in tasks.
+    - Custom/community tasks should prefer full module paths (for example `sieval.tasks.<module>.MyTask`) to avoid naming conflicts.
+    - `runners`: canonical path is `sieval.core.runners` (`TaskRunner`, `MultiTaskRunner`).
+    - `session`: canonical path is `sieval.cli.leaderboard.session` (`EvalSession`, `arun_session`).
+    - `validation`: canonical path is `sieval.cli.validation` (`validate_eval_config`, `run_dry_run`).
+    - `resolution`: canonical path is `sieval.cli.resolution` (`resolve_task_class`, `resolve_dataset_class`, `derive_model_type`). Reaching them via `session` resolves, but is not API. `cli/infer/` must not import `cli/leaderboard/`.
+    - Within the same package/subpackage: use relative imports (`from .foo import Bar`).
+    - Cross-package (including parent): use absolute imports (`from sieval.core.models import ...`).
+    - Imports imply public API. Cross-module `from sieval.x.y import _foo` in production code is a smell — promote the name or redesign the call site.
+    - Private modules (`_*.py`) are **protected** — accessible only within their own package subtree. Same-package siblings may use `from ._x import Y`; descendants may import via absolute path into the ancestor's private module. Peer-subpackage or out-of-subtree access is forbidden. Tests are the carve-out for both rules.
+
+## Testing Guidelines
+
+- Always run tests after making changes. See `tests/README.md` and `.claude/rules/tests.md` for details.
+- `tests/unit/` directory structure must mirror `sieval/` — e.g. `sieval/core/runners/foo.py` → `tests/unit/core/runners/test_foo.py`.
+- Framework: `pytest`; async tests must use `@pytest.mark.anyio` (not `pytest-asyncio`).
+- `stress` marker is defined in `pyproject.toml`; exclude by default unless intentionally profiling.
+- Mutation testing uses `mutmut` (on-demand, not per-commit): mutate `sieval/core` against `tests/unit` (see `[tool.mutmut]`).
+- Mutation score should stay `>=70%`; surviving mutants indicate missing assertions and should be killed by improving tests.
+- Common mutation commands: `mutmut run`, `mutmut results`, `mutmut show`.
+- When changing core runner/model behavior, add/adjust unit tests first, then verify with relevant integration tests.
+- Prefer `tmp_path` and deterministic mocks over hardcoded paths and wall-clock sleeps.
+
+## Commit & Pull Request Guidelines
+
+- Always rebase onto the latest `main` before submitting a PR.
+- **MUST** read `.github/PULL_REQUEST_TEMPLATE.md` before creating any PR, and fill in all applicable sections. Do not create a PR without reading the template first.
+- **MUST** read the matching template from `.github/ISSUE_TEMPLATE/` before creating any issue (bug-report, feature-request, rfc, etc.), and use the template's structure. Do not create an issue without reading the template first.
+- Commit messages follow Conventional Commits (enforced by `conventional-pre-commit`):
+    - Examples: `feat: ...`, `fix: ...`, `refactor: ...`, `chore: ...`.
+- Keep commits scoped and reviewable; include tests with code changes.
+- PRs should include:
+    - What changed and why.
+    - Risk/regression notes.
+    - Validation evidence (exact commands run and results).
+
+## Agent Tooling
+
+This file is the single source of truth for shared conventions, and the only
+instruction file every supported agent reads. `CLAUDE.md` is a one-line import
+of it. Keep it under 32 KiB: Codex truncates project docs past that byte count
+silently, so content near the end simply stops applying.
+
+- **Layer and scoped rules** — `sieval/*/CLAUDE.md` and `.claude/rules/*.md`.
+  Only Claude Code loads these automatically. Codex never reads a `CLAUDE.md`
+  below the repo root, even with the working directory in that folder, and
+  opencode only walks upward from it. Elsewhere these rules are reachable just
+  through the map below, which points at a file rather than putting the rule in
+  front of the agent — so treat a rule as enforced only under Claude Code.
+
+<!-- BEGIN generated: rule-map -->
+
+Read the matching file before editing a path it covers. Claude Code
+loads these automatically; every other harness needs this map.
+
+| When editing | Read first |
+| --- | --- |
+| `sieval/cli/**` | `sieval/cli/CLAUDE.md` |
+| `sieval/community/**` | `sieval/community/CLAUDE.md` |
+| `sieval/core/**` | `sieval/core/CLAUDE.md` |
+| `sieval/datasets/**` | `sieval/datasets/CLAUDE.md` |
+| `sieval/infer/**` | `sieval/infer/CLAUDE.md` |
+| `sieval/tasks/**` | `sieval/tasks/CLAUDE.md` |
+| `sieval/**/*.py`, `tests/**/*.py` | `.claude/rules/code-review.md` |
+| `sieval/**/*.py` | `.claude/rules/core-utils.md` |
+| `sieval/datasets/**/*.py` | `.claude/rules/datasets.md` |
+| `pyproject.toml`, `pdm.lock` | `.claude/rules/deps.md` |
+| `CLAUDE.md`, `AGENTS.md`, `sieval/**/CLAUDE.md`, `CONTRIBUTING.md`, `.claude/rules/*.md`, `scripts/check_*.py`, `.pre-commit-config.yaml` | `.claude/rules/engineering-infra.md` |
+| `sieval/infer/**/*.py` | `.claude/rules/infer.md` |
+| `sieval/tasks/**/*.py`, `sieval/core/tasks/**/*.py` | `.claude/rules/records.md` |
+| `scripts/*.py` | `.claude/rules/scripts.md` |
+| `sieval/tasks/**/*.py` | `.claude/rules/tasks.md` |
+| `tests/**/*.py` | `.claude/rules/tests.md` |
+
+<!-- END generated: rule-map -->
+
+- **Skills** — `.agents/skills/<name>/SKILL.md` is the source of truth, with
+  symlinks at `.claude/skills/<name>` and `.opencode/commands/<name>.md`.
+  Never edit through a symlink.
+- **Post-edit checks** — `scripts/post_edit_checks.py` holds the behavior;
+  `.claude/settings.json` and `.opencode/plugins/post-edit.ts` dispatch to it
+  with the edited path as `argv[1]`. Codex wiring is deliberately absent: it
+  delivers the payload as JSON on stdin, so it needs an adapter no one can
+  verify here. Under Codex, run `python scripts/post_edit_checks.py <path>`.