diff --git a/.agents/skills/watchlist-md/LICENSE.txt b/.agents/skills/watchlist-md/LICENSE.txt new file mode 100644 index 0000000..3fffaab --- /dev/null +++ b/.agents/skills/watchlist-md/LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Injae + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/.agents/skills/watchlist-md/SKILL.md b/.agents/skills/watchlist-md/SKILL.md index e5537d6..586154e 100644 --- a/.agents/skills/watchlist-md/SKILL.md +++ b/.agents/skills/watchlist-md/SKILL.md @@ -1,7 +1,6 @@ --- name: watchlist-md -description: >- - Use when recording, reviewing, or updating WATCHLIST.md/WL-YYYYMMDD-NNN deferred checks for CI/deploy/job/sync/order/PR/ticket/email 후속 체크; not generic calendars/wakeups/polling or lifecycle words unless WATCHLIST-scoped. +description: Record, review, or update WATCHLIST.md/WL-YYYYMMDD-NNN deferred checks and 후속 체크; not generic reminders, wakeups, polling, or unscoped lifecycle requests. --- # WATCHLIST.md @@ -44,7 +43,7 @@ unused `WL-YYYYMMDD-NNN`, edit `## Open` only. ### WL-YYYYMMDD-NNN — Short title - status: open - priority: P1 -- owner: user|assistant_on_review|both|external +- owner: assistant_on_review - due_at: YYYY-MM-DDTHH:MM:SS+09:00 - created_at: YYYY-MM-DDTHH:MM:SS+09:00 - source: short stable pointer, safe link, file, PR, issue, or conversation note diff --git a/.agents/skills/watchlist-md/agents/openai.yaml b/.agents/skills/watchlist-md/agents/openai.yaml index 1532cac..758e5fd 100644 --- a/.agents/skills/watchlist-md/agents/openai.yaml +++ b/.agents/skills/watchlist-md/agents/openai.yaml @@ -1,11 +1,4 @@ interface: display_name: "WATCHLIST.md" short_description: "Track deferred checks and 후속 체크 in WATCHLIST.md" - default_prompt: >- - Use $watchlist-md when a user asks to add, review, complete, drop, delete, - archive, or check WATCHLIST.md entries, or when the request mentions a - WL-YYYYMMDD-NNN item ID. Generic lifecycle words such as 완료, 삭제, 취소, 드롭, 차단, 연기, - 보관, or 아카이브 should not trigger this skill unless they clearly refer to - WATCHLIST.md or a WL-YYYYMMDD-NNN item. This skill records Markdown notes only; it does - not schedule reminders, wakeups, or background checks. Do not store secrets, - signed URLs, raw logs, raw emails, or private dashboard excerpts. + default_prompt: "Use $watchlist-md to record a deferred CI check in WATCHLIST.md and report its ID, due_at, and scheduler status." diff --git a/.agents/skills/watchlist-md/assets/WATCHLIST.template.md b/.agents/skills/watchlist-md/assets/WATCHLIST.template.md index f81c1fd..f8f9f6a 100644 --- a/.agents/skills/watchlist-md/assets/WATCHLIST.template.md +++ b/.agents/skills/watchlist-md/assets/WATCHLIST.template.md @@ -21,7 +21,7 @@ Do not copy the literal ID or timestamps; generate fresh values when adding an i ### WL-20260514-001 — GitHub Actions 결과 확인 - status: open - priority: P1 -- owner: user|assistant_on_review|both|external +- owner: assistant_on_review - due_at: 2026-05-14T17:00:00+09:00 - created_at: 2026-05-14T16:30:00+09:00 - source: GitHub Actions run for PR #123 diff --git a/.agents/skills/watchlist-md/references/format.md b/.agents/skills/watchlist-md/references/format.md index ccdd7fd..241bb37 100644 --- a/.agents/skills/watchlist-md/references/format.md +++ b/.agents/skills/watchlist-md/references/format.md @@ -22,6 +22,10 @@ Allowed `archive_policy` values: When `archive_policy: suggest` is used, add `archive_after_days: N` with a positive integer. +Use a non-empty IANA time-zone name such as `Asia/Seoul` for `timezone`. The +maintainer validator currently checks this field for presence only; it does not +resolve the name against a host timezone database. + ## Sections Required sections: @@ -84,17 +88,23 @@ Keep item fields in this order: `priority`: -- `P0` -- `P1` -- `P2` -- `P3` +- `P0`: critical or urgent +- `P1`: high or time-sensitive +- `P2`: normal +- `P3`: low or optional + +Priority expresses review urgency, not permission to run automatically. Preserve +an explicit user or repository convention; otherwise choose conservatively. `owner`: -- `user` -- `assistant_on_review` -- `both` -- `external` +- `user`: the user acts at the next explicit review +- `assistant_on_review`: the assistant acts when the item is explicitly reviewed +- `both`: user and assistant each have a review-time action +- `external`: another person or system owns the next action + +Owner describes who acts during an explicit WATCHLIST review. It does not grant +background execution, wakeups, or access to an external system. ## Required populated values diff --git a/.agents/skills/watchlist-md/references/lifecycle.md b/.agents/skills/watchlist-md/references/lifecycle.md index d287829..5b613b3 100644 --- a/.agents/skills/watchlist-md/references/lifecycle.md +++ b/.agents/skills/watchlist-md/references/lifecycle.md @@ -29,13 +29,18 @@ an update, a check is performed, or the result is known. | `blocked` | `snoozed` | blocker remains but the next review time is known | `due_at`, `last_checked_at`, `result` | | `blocked` | `done` | `done_when` is satisfied or the user reports completion | `last_checked_at`, `result` | | any active status | `dropped` | user says to drop, cancel, or ignore | `result` | -| `done` or `dropped` | active status | user explicitly asks to reopen | `result` describing the reopen reason | +| `done` or `dropped` | active status | user explicitly asks to reopen | `result` describing the reopen reason, plus the target status requirements in `format.md` | -When marking an item done, the default lifecycle update is: set `status: done`, -fill `last_checked_at`, fill `result`, and move the completed item under `## Done` -if that section exists. If the user explicitly says to change only the status, -keep the item in place, or preserve section placement, leave it in its original -section. +By default, section placement follows status: + +- Keep `open`, `snoozed`, and `blocked` items under `## Open`. +- Move `done` and `dropped` items under `## Done` when that section exists. +- Move a reopened `done` or `dropped` item back under `## Open`. +- Move terminal items under `## Archive` only after an explicit archive request. + +These moves are part of the default lifecycle update. If the user explicitly +asks to change only the status, keep an item in place, or preserve section +placement, leave it in its original section. ## Archive Policy diff --git a/CHANGELOG.md b/CHANGELOG.md index 31d231a..018dc62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ ### Added +- Added regression coverage for validator false negatives, trigger-boundary + pairs, exact package contents, and template/example drift. +- Added the MIT notice to the standalone runtime bundle. - Added English and Korean semantic coverage for default local/private watchlist creation when no WATCHLIST file exists. - Added a broad staging self-check to keep private `.watchlist/WATCHLIST.md` data @@ -17,10 +20,25 @@ ### Changed +- Aligned runtime metadata, lifecycle/enum guidance, vendor installation paths, + and compatibility claims with their documented contracts and smoke status. +- Clarified that semantic-case checks lint declared fixtures and contracts; they + do not execute an LLM, agent, or runtime. - Made the installable skill bundle py-free and slimmer by moving the WATCHLIST validator and self-check prompts to source-repository maintainer tooling, trimming `SKILL.md`, tightening package guards, and preserving CI validation. +### Fixed + +- Rejected invalid skeleton values, misplaced top-level fields, empty open-item + semantics, and overflowing timezone-offset minutes in the maintainer validator. +- Scoped positive trigger cases to explicit WATCHLIST context and preserved the + original generic prompts as negative routing cases. +- Replaced the clean-clone-invalid contributor validation command and enforced + the documented exact runtime package allowlist. +- Allowed explicit WATCHLIST negation as a declared no-trigger intent and made + the dependency-free YAML subset validator track parent container types. + ## [0.4.2] - 2026-06-10 ### Changed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4d1a312..c180992 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,15 +2,19 @@ Keep changes narrow and update the files that define the same behavior. -When changing WATCHLIST lifecycle behavior, update: +When changing WATCHLIST behavior or runtime packaging, update the applicable sources: - `.agents/skills/watchlist-md/SKILL.md` +- `.agents/skills/watchlist-md/LICENSE.txt` when repository licensing changes - `.agents/skills/watchlist-md/agents/openai.yaml` if trigger or boundary text changes - `README.md` - `README.ko.md` - `.agents/skills/watchlist-md/assets/WATCHLIST.template.md` if the file format changes +- `.agents/skills/watchlist-md/references/format.md` or `lifecycle.md` when its contract changes +- `examples/WATCHLIST.example.md`, which must mirror the canonical runtime template - `evals/prompts.csv` - `evals/self_checks.yaml` +- `evals/cases/*.json` and `evals/trigger_cases.json` - `evals/test_check_watchlist.py` - `CHANGELOG.md` @@ -20,7 +24,7 @@ prerequisites for the py-free skill bundle. ```bash PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s evals -p 'test_*.py' -python3 evals/check_watchlist.py .watchlist/WATCHLIST.md --strict-format --strict-safety --require-archive-section +python3 evals/check_watchlist.py examples/WATCHLIST.example.md --strict-format --strict-safety --require-archive-section python3 evals/check_watchlist.py .agents/skills/watchlist-md/assets/WATCHLIST.template.md --strict-format --strict-safety --require-archive-section python3 evals/check_release_metadata.py python3 evals/check_policy_markers.py diff --git a/README.ko.md b/README.ko.md index 804a445..ac88259 100644 --- a/README.ko.md +++ b/README.ko.md @@ -6,7 +6,7 @@ [English README](README.md) -`WATCHLIST.md`는 deferred check를 기록하기 위한 경량 **AI Agent Skill**이자 AgentSkills 호환 Markdown workflow입니다. Codex, Claude Code, OpenClaw, Gemini CLI, Kilo, Hermes가 CI 후속 확인, 배포 검증, PR 확인, 티켓, 작업, 데이터 동기화, 이메일을 scheduler, daemon, database, MCP server 없이 추적하도록 돕습니다. +`WATCHLIST.md`는 deferred check를 기록하기 위한 경량 **AI Agent Skill**이자 AgentSkills 호환 Markdown workflow입니다. Codex, Claude Code, OpenClaw, Gemini CLI, Kilo, Hermes에서 CI 후속 확인, 배포 검증, PR 확인, 티켓, 작업, 데이터 동기화, 이메일을 scheduler, daemon, database, MCP server 없이 추적하도록 설계됐습니다. 문서상 format/path 호환성과 실제 runtime 검증은 구분하며, runtime smoke matrix의 pending 상태를 기준으로 봅니다. 이 스킬은 자율 스케줄러, 자율 알림, daemon, database, cron job, UI, background worker가 아닙니다. 나중에 확인할 일을 기록할 뿐이며, 스스로 깨어나거나 polling, 알림, 확인 실행을 하지 않습니다. @@ -38,10 +38,11 @@ python3 evals/check_watchlist.py examples/WATCHLIST.example.md .agents/skills/watchlist-md ``` -runtime bundle에는 스킬 지시문, 템플릿, OpenAI 메타데이터, 짧은 reference가 들어갑니다: +runtime bundle에는 스킬 지시문, 라이선스 고지, 템플릿, OpenAI 메타데이터, 짧은 reference가 들어갑니다: ```text .agents/skills/watchlist-md/SKILL.md +.agents/skills/watchlist-md/LICENSE.txt .agents/skills/watchlist-md/assets/WATCHLIST.template.md .agents/skills/watchlist-md/agents/openai.yaml .agents/skills/watchlist-md/references/format.md @@ -71,11 +72,11 @@ Markdown 편집으로 add, review, complete, blocked, snoozed, dropped, explicit `.agents/skills/watchlist-md/`에 CLI, MCP server, browser automation, bundled validator, smoke transcript, screenshot, 긴 eval corpus를 넣지 마세요. -Gemini CLI, Kilo, OpenClaw, Hermes 같은 AgentSkills 호환 런타임은 가능하면 같은 스킬 디렉토리를 사용하세요. OpenClaw와 Hermes는 runtime smoke 전까지 AgentSkills 호환/manual 지원으로 보고, 리포지토리 루트가 아니라 `SKILL.md`가 루트에 있는 스킬 디렉토리를 설치하세요. +각 벤더가 문서화한 discovery path 또는 install flow를 사용하세요. format/path 호환성은 runtime smoke pass가 아닙니다. 설치 세부사항과 pending 검증 matrix는 `docs/install.md`와 `docs/runtime-smoke.md`에 있습니다. ## Docs -- [Installation](docs/install.md): Codex, Claude Code, OpenAI Skills zip packaging, AgentSkills-compatible runtime notes. +- [Installation](docs/install.md): vendor discovery path, Codex·Claude Code setup, standalone zip packaging. - [Storage and privacy](docs/storage-and-privacy.md): generated `.watchlist/WATCHLIST.md`, shared root watchlists, archive policy, concurrent edits, retention. - [Validation](docs/validation.md): validator commands, strict-safety behavior, semantic cases, item format expectations. - [Runtime smoke](docs/runtime-smoke.md): transcript나 raw log 없는 compact vendor/runtime smoke matrix. diff --git a/README.md b/README.md index 2913d4b..358d9af 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [Korean README](README.ko.md) -`WATCHLIST.md` is a lightweight **AI Agent Skill** and AgentSkills-compatible Markdown workflow for recording deferred checks. It helps Codex, Claude Code, OpenClaw, Gemini CLI, Kilo, and Hermes track CI follow-ups, deployment verification, PR checks, tickets, jobs, data syncs, and emails without creating a scheduler, daemon, database, or MCP server. +`WATCHLIST.md` is a lightweight **AI Agent Skill** and AgentSkills-compatible Markdown workflow for recording deferred checks. It is designed for Codex, Claude Code, OpenClaw, Gemini CLI, Kilo, and Hermes to track CI follow-ups, deployment verification, PR checks, tickets, jobs, data syncs, and emails without creating a scheduler, daemon, database, or MCP server. Documented format and path compatibility is separate from real runtime verification; see the pending rows in the runtime smoke matrix. It is not an autonomous scheduler, reminder service, daemon, database, cron job, UI, or background worker. It records what should be checked later; it does not wake up, poll, notify, or run checks by itself. @@ -38,10 +38,11 @@ Install or copy the skill directory whose root contains `SKILL.md`, not the repo .agents/skills/watchlist-md ``` -The runtime bundle contains the skill instructions, template, OpenAI metadata, and compact references: +The runtime bundle contains the skill instructions, license notice, template, OpenAI metadata, and compact references: ```text .agents/skills/watchlist-md/SKILL.md +.agents/skills/watchlist-md/LICENSE.txt .agents/skills/watchlist-md/assets/WATCHLIST.template.md .agents/skills/watchlist-md/agents/openai.yaml .agents/skills/watchlist-md/references/format.md @@ -71,11 +72,11 @@ The installable runtime skill stays Python-free. Agents edit Markdown directly f Do not add a CLI, MCP server, browser automation, bundled validator, smoke transcript, screenshot, or long eval corpus to `.agents/skills/watchlist-md/`. -AgentSkills-compatible runtimes such as Gemini CLI, Kilo, OpenClaw, and Hermes should use the same skill directory when possible. For OpenClaw and Hermes, treat support as AgentSkills-compatible/manual until runtime-smoked; install the skill directory whose root contains `SKILL.md`, not the repository root. +Use each vendor's documented discovery path or install flow. Format/path compatibility does not count as a runtime smoke pass; install details and the pending verification matrix are in `docs/install.md` and `docs/runtime-smoke.md`. ## Docs -- [Installation](docs/install.md): Codex, Claude Code, OpenAI Skills zip packaging, and AgentSkills-compatible runtime notes. +- [Installation](docs/install.md): vendor discovery paths, Codex and Claude Code setup, and standalone zip packaging. - [Storage and privacy](docs/storage-and-privacy.md): generated `.watchlist/WATCHLIST.md`, shared root watchlists, archive policy, concurrent edits, and retention. - [Validation](docs/validation.md): validator commands, strict-safety behavior, semantic cases, and item format expectations. - [Runtime smoke](docs/runtime-smoke.md): compact vendor/runtime smoke matrix without transcripts or raw logs. diff --git a/docs/install.md b/docs/install.md index a676141..058932d 100644 --- a/docs/install.md +++ b/docs/install.md @@ -12,7 +12,18 @@ Install or copy the skill directory whose root contains `SKILL.md`, not the repo Install `watchlist-md` in the primary agent runtime you actually use. Avoid copying the same skill into every runtime by default; duplicate installs can drift. Repositories should usually contain watchlist data, not runtime-specific skill copies. -AgentSkills-compatible runtimes such as Gemini CLI, Kilo, OpenClaw, and Hermes should use the same skill directory when possible. Avoid vendor-specific copies unless a runtime requires a different install location. Track real runtime smoke results in `docs/runtime-smoke.md`. +Use a vendor-specific copy only when that runtime requires a different location. Documented format/path support is not evidence of runtime behavior; track real results separately in `docs/runtime-smoke.md`. + +## Vendor Paths And Guides + +| Runtime | Documented project or user path / flow | Official guide | +| --- | --- | --- | +| Codex | `.agents/skills/` or `$HOME/.agents/skills/` | [Build skills](https://learn.chatgpt.com/docs/build-skills) | +| Claude Code | `.claude/skills/` or `~/.claude/skills/` | [Extend Claude with skills](https://code.claude.com/docs/en/skills) | +| Gemini CLI | Project: `.agents/skills/` or `.gemini/skills/`; user: `~/.agents/skills/` or `~/.gemini/skills/`; `gemini skills link` | [Managing Agent Skills](https://geminicli.com/docs/cli/using-agent-skills/) | +| Kilo | Project: `.agents/skills/` or `.kilo/skills/`; user: `~/.kilo/skills/` | [Skills](https://kilo.ai/docs/customize/skills) | +| OpenClaw | `/.agents/skills/` or `~/.agents/skills/` | [Skills](https://docs.openclaw.ai/skills) | +| Hermes | `~/.hermes/skills/` or `hermes skills install` | [Working with Skills](https://hermes-agent.nousresearch.com/docs/guides/work-with-skills) | ## Installation For Codex @@ -22,7 +33,8 @@ Pass the skill directory URL, not only the repository root: $skill-installer install https://github.com/dd3ok/WATCHLIST.md/tree/main/.agents/skills/watchlist-md ``` -Restart Codex after installation so the new skill is detected. +Codex detects newly installed skills automatically. If the skill does not appear, +restart Codex. The bundle includes `assets/WATCHLIST.template.md`, so an agent can create a new WATCHLIST.md even when only `.agents/skills/watchlist-md` is installed. @@ -61,22 +73,54 @@ cp -R .agents/skills/watchlist-md ~/.claude/skills/watchlist-md The `agents/openai.yaml` file is Codex UI metadata. It is safe if copied with the directory. -## Installation For ChatGPT / OpenAI Skills +Claude Code watches existing skill directories for changes. Restart only if the +top-level skills directory did not exist when the session started. + +## Installation For Gemini CLI, Kilo, And OpenClaw + +In a checkout of this repository, all three runtimes can discover the existing +`.agents/skills/watchlist-md` project path. Gemini CLI can reload with +`/skills reload`; Kilo can reload with `/reload`. OpenClaw normally refreshes +watched skill changes for the next turn or discovers them in a new session. + +For a personal install, copy or link the complete `watchlist-md` directory to one +of the user paths in the vendor table. Keep `SKILL.md`, `assets/`, `references/`, +and `LICENSE.txt` together. + +## Installation For Hermes + +Hermes uses its own user skill directory: + +```bash +mkdir -p ~/.hermes/skills +cp -R .agents/skills/watchlist-md ~/.hermes/skills/watchlist-md +``` + +Hermes also supports `hermes skills install` for hub and direct HTTP(S) +`SKILL.md` sources. Start a new Hermes session after copying, then verify with +`hermes skills list`; do not treat a successful copy as a runtime behavior smoke +pass. + +## Standalone Zip Packaging -OpenAI skill surfaces do not automatically sync with Codex or Claude Code installs. When uploading a skill bundle as a zip, package one top-level skill directory: +This repository defines a standalone zip shape for consumers that accept skill +directories as archives. This is a repository packaging contract, not a claim +that every vendor supports zip upload. Package one top-level skill directory: ```bash cd .agents/skills zip -r watchlist-md-skill.zip watchlist-md ``` -Upload the resulting zip through the OpenAI skill management UI or workflow you use. The archive should contain `watchlist-md/SKILL.md` at its top-level folder. The uploaded skill bundle is Python-free. +The archive must contain `watchlist-md/SKILL.md` and +`watchlist-md/LICENSE.txt` at its top-level folder. The standalone bundle is +Python-free. Repository-level `tools/validate_watchlist.py` and `evals/` are source-repository maintainer checks only. Do not package runtime `scripts/` validators; the runtime bundle intentionally has no validator script. -Test after upload: +If a chosen surface accepts the standalone bundle, test explicit invocation +using that surface's documented syntax. For Codex: ```text -/watchlist-md -Add this to WATCHLIST.md. Check GitHub Actions results today at 17:00. +$watchlist-md Add this to WATCHLIST.md. Check GitHub Actions results today at 17:00. ``` diff --git a/docs/maintainers/release.md b/docs/maintainers/release.md index ac9a8c1..dc1f672 100644 --- a/docs/maintainers/release.md +++ b/docs/maintainers/release.md @@ -8,6 +8,7 @@ The installable skill bundle is intentionally Python-free. It should contain onl ```text watchlist-md/SKILL.md +watchlist-md/LICENSE.txt watchlist-md/agents/openai.yaml watchlist-md/assets/WATCHLIST.template.md watchlist-md/references/format.md @@ -15,6 +16,9 @@ watchlist-md/references/lifecycle.md watchlist-md/references/safety.md ``` +`evals/check_skill_package.py` treats this as an exact file allowlist. Update the +checker and this list together before adding any runtime file. + Repository-only files must stay outside `.agents/skills/watchlist-md/`: `tools/`, `evals/`, `.github/`, `docs/`, examples, smoke notes, release notes, transcripts, screenshots, and raw logs. ## Checks @@ -41,7 +45,7 @@ git diff --name-only -- .agents/skills/watchlist-md If the PR targets a branch other than `main`, replace `origin/main` with the actual base ref. -## OpenAI Skills Zip +## Standalone Skill Zip When uploading a skill bundle as a zip, package one top-level skill directory: @@ -50,4 +54,7 @@ cd .agents/skills zip -r watchlist-md-skill.zip watchlist-md ``` -The archive should contain `watchlist-md/SKILL.md` at its top-level folder. Repository-level `tools/validate_watchlist.py` and `evals/` are source-repository maintainer checks only. Do not package runtime `scripts/`, `docs/`, `evals/`, screenshots, transcripts, or raw runtime logs. +The archive should contain `watchlist-md/SKILL.md` and `watchlist-md/LICENSE.txt` +at its top-level folder. Repository-level `tools/validate_watchlist.py` and +`evals/` are source-repository maintainer checks only. Do not package runtime +`scripts/`, `docs/`, `evals/`, screenshots, transcripts, or raw runtime logs. diff --git a/docs/maintainers/self-checks.md b/docs/maintainers/self-checks.md index 17b20c2..350fde0 100644 --- a/docs/maintainers/self-checks.md +++ b/docs/maintainers/self-checks.md @@ -4,9 +4,9 @@ Use these prompts when validating changes to this skill. 1. `WATCHLIST.md에 추가해줘. 오늘 17:00에 GitHub Actions 결과 확인. 실패하면 로그 요약하고 수정 여부 물어봐.` - Expected: creates or updates WATCHLIST.md, appends one `open` item under `## Open`, converts time to ISO-8601 with timezone when current date is available, confirms the ID, scheduler status, and does not promise automatic execution. -2. `배포가 방금 시작됐어. 30분 뒤에 에러 로그 확인해야 해.` - - Expected: treats the explicit future-check request as permission to record a deferred check with a concrete `due_at` if current time is available; otherwise uses `due_at: unscheduled` and mentions the ambiguity. -3. `코드 수정하고 CI가 돌기 시작하면, 아직 결과가 안 나왔을 때 필요한 후속 체크를 남겨.` +2. `WATCHLIST.md에 남겨줘. 배포가 방금 시작됐어. 30분 뒤에 에러 로그 확인해야 해.` + - Expected: treats the explicit WATCHLIST recording request as permission to record a deferred check with a concrete `due_at` if current time is available; otherwise uses `due_at: unscheduled` and mentions the ambiguity. +3. `이 저장소의 WATCHLIST workflow에서 코드 수정 후 CI 결과가 아직 안 나왔으면 필요한 후속 체크를 남겨.` - Expected: records a check only when CI is actually pending because the user opted into this workflow, includes source/context and a concrete `done_when`, and avoids unrelated file changes. 4. `오늘 확인할 WATCHLIST.md 보여줘.` - Expected: groups `open`, `snoozed`, and `blocked` items into overdue, due today, upcoming, and unscheduled without mutating list-only reviews. @@ -14,13 +14,13 @@ Use these prompts when validating changes to this skill. - Expected: sets `status: done`, fills `result` and `last_checked_at`, moves the item under `## Done` by default when that section exists, and does not delete the item. 6. `WATCHLIST.md에 추가했는데 git status에 .watchlist/WATCHLIST.md가 untracked로 떠. 이거 커밋해야 해?` - Expected: explains that this is normal after the skill creates the file, recommends not committing personal/private watchlists, and suggests `$HOME/.watchlist/WATCHLIST.md`, `.git/info/exclude`, or `.gitignore` depending on whether the rule is personal or team-wide. -7. `내 토큰 ABC123을 저장해뒀다가 나중에 써줘.` +7. `내 토큰 ABC123을 WATCHLIST.md에 저장해뒀다가 나중에 써줘.` - Expected: refuses to store the token; if a deferred check is still useful, records only a stable non-secret pointer. 8. `WL-20260507-001 더 이상 필요 없어. 드롭 처리해.` - Expected: sets `status: dropped`, fills a short `result`, preserves the record, and does not delete the item. 9. `WL-20260507-001 기록 자체를 삭제해줘.` - Expected: deletes the item only because the user explicitly asked to remove the record itself, without rewriting unrelated items. -10. `30일 지난 done/dropped 항목을 Archive로 옮겨줘.` +10. `WATCHLIST.md에서 30일 지난 done/dropped 항목을 Archive로 옮겨줘.` - Expected: treats this as an explicit archive request, creates `## Archive` if needed, moves only requested `done` or `dropped` items, and leaves active items in place. 11. `오늘 확인할 WATCHLIST.md 보여줘.` - Expected: if `archive_policy: suggest` is present, suggests old `done` or `dropped` archive candidates during explicit review without mutating the file. diff --git a/docs/runtime-smoke.md b/docs/runtime-smoke.md index 9dccb46..321e82d 100644 --- a/docs/runtime-smoke.md +++ b/docs/runtime-smoke.md @@ -8,10 +8,10 @@ This file tracks manual smoke checks in real agent runtimes. Record only real ru | --- | --- | --- | --- | --- | --- | --- | | Codex | `$skill-installer install https://github.com/dd3ok/WATCHLIST.md/tree/main/.agents/skills/watchlist-md` | `Add this to WATCHLIST.md. Check GitHub Actions results today at 17:00.` | Creates or updates the selected WATCHLIST target and reports `scheduler: none`. | pending | - | - | | Claude Code | Copy `watchlist-md` to `.claude/skills/watchlist-md` or `~/.claude/skills/watchlist-md` | `Review WATCHLIST.md.` | Lists items without mutation unless a check is explicitly performed. | pending | - | - | -| Gemini CLI | Install or copy the AgentSkills-compatible `watchlist-md` directory | `Add this to watchlist. Check the data sync result tomorrow.` | Records a deferred check without promising a wakeup. | pending | - | - | -| Kilo | Install or copy the AgentSkills-compatible `watchlist-md` directory | `Show WATCHLIST.md items due today.` | Reviews due items without mutating list-only output. | pending | - | - | -| OpenClaw | Install or copy the AgentSkills-compatible `watchlist-md` directory | `Add a local/private watchlist item for test logs today at 18:00.` | Uses `.watchlist/WATCHLIST.md` when no shared-team intent exists. | pending | - | - | -| Hermes | Install or copy the AgentSkills-compatible `watchlist-md` directory | `Mark WL-20260507-001 done after confirming CI passed.` | Updates lifecycle fields without deleting the item. | pending | - | - | +| Gemini CLI | Use `.agents/skills/watchlist-md` or `gemini skills link ` | `Add this to watchlist. Check the data sync result tomorrow.` | Records a deferred check without promising a wakeup. | pending | - | - | +| Kilo | Use `.agents/skills/watchlist-md` or `.kilo/skills/watchlist-md` | `Show WATCHLIST.md items due today.` | Reviews due items without mutating list-only output. | pending | - | - | +| OpenClaw | Use `/.agents/skills/watchlist-md` or `~/.agents/skills/watchlist-md` | `Add a local/private watchlist item for test logs today at 18:00.` | Uses `.watchlist/WATCHLIST.md` when no shared-team intent exists. | pending | - | - | +| Hermes | Copy to `~/.hermes/skills/watchlist-md` or use `hermes skills install` | `Mark WL-20260507-001 done after confirming CI passed.` | Updates lifecycle fields without deleting the item. | pending | - | - | ## Pass Criteria diff --git a/docs/storage-and-privacy.md b/docs/storage-and-privacy.md index e852146..917dc98 100644 --- a/docs/storage-and-privacy.md +++ b/docs/storage-and-privacy.md @@ -41,7 +41,9 @@ git rm --cached .watchlist/WATCHLIST.md Do not add a full CLI or MCP server for the MVP flow. The installable skill bundle is intentionally Python-free; agents edit Markdown directly using the documented contract, and source-repository maintainers run `tools/validate_watchlist.py` or `evals/check_watchlist.py` for deterministic checks. -AgentSkills-compatible runtimes such as Gemini CLI, Kilo, OpenClaw, and Hermes should use the same skill directory when possible. Runtime smoke status is tracked separately in `docs/runtime-smoke.md`. +Gemini CLI, Kilo, and OpenClaw document `.agents/skills` discovery. Hermes uses +`~/.hermes/skills` or its own installer. These path claims are not runtime smoke +results; status is tracked separately in `docs/runtime-smoke.md`. ## Archive Policy diff --git a/docs/validation.md b/docs/validation.md index b40ed1d..2a5951a 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -16,7 +16,24 @@ python3 evals/check_semantic_cases.py python3 evals/check_skill_package.py ``` -`evals/prompts.csv`, `evals/rubric.md`, `evals/self_checks.yaml`, `evals/cases/*.json`, and `evals/trigger_cases.json` are small prompt and trigger regression sets. The Semantic case checker validates the expected trigger and operation contract; it does not run an LLM, agent, browser, network call, or runtime integration. +`evals/cases/*.json` is the canonical deterministic operation-contract corpus. +`evals/prompts.csv` and `evals/self_checks.yaml` mirror its IDs and prompts for +manual runs. Their expected fields are manual expectations, not executed agent +assertions; `expected.should_trigger_skill`, when present in YAML, is cross-checked +for routing parity. +`evals/trigger_cases.json` is a small prompt-routing contract corpus. +For semantic operation cases, `should_trigger_skill` is the declared intent; +mentioning WATCHLIST is not automatically positive when the prompt explicitly +rejects using it. Prompt-only routing exceptions use named trigger reasons. + +The evaluation contract linter (`evals/check_semantic_cases.py`) checks corpus +shape, prompt/trigger alignment, declared contract rules, and fixture validity. +It does not run an LLM, agent, browser, network call, or runtime integration. +`fixed_now` is validated as data but is not injected into an agent. A passing +lint therefore must not be reported as an agent-behavior or runtime smoke pass. +To stay dependency-free, `self_checks.yaml` is checked against the limited YAML +structure used by this repository. The checker tracks mapping/sequence parents +and rejects children under scalar values; it does not accept arbitrary YAML. ## Item Format @@ -38,7 +55,9 @@ python3 evals/check_skill_package.py - next_step_on_fail: Summarize the logs and confirm whether the user wants a fix ``` -The validator requires every field key in the stable order shown above, but not every field needs a populated value for an open item. +The validator requires every field key. Default mode reports field-order drift as +a warning; `--strict-format` promotes it to an error. For an open item, only +`last_checked_at` and `result` are normally left blank. Required values for open items are `status`, `priority`, `owner`, `due_at`, `created_at`, `source`, `trigger`, `action`, and `done_when`. Recommended when known: `next_step_on_fail`. Normally blank until checked: `last_checked_at` and `result`. diff --git a/evals/cases/add-kr-02.json b/evals/cases/add-kr-02.json index aa63427..0ccf712 100644 --- a/evals/cases/add-kr-02.json +++ b/evals/cases/add-kr-02.json @@ -1,6 +1,6 @@ { "id": "add-kr-02", - "prompt": "배포가 방금 시작됐어. 30분 뒤에 에러 로그 확인해야 해.", + "prompt": "WATCHLIST.md에 남겨줘. 배포가 방금 시작됐어. 30분 뒤에 에러 로그 확인해야 해.", "locale": "ko", "fixed_now": "2026-05-14T16:30:00+09:00", "fixture": "empty.watchlist.md", diff --git a/evals/cases/archive-kr-01.json b/evals/cases/archive-kr-01.json index 05635ea..03fceec 100644 --- a/evals/cases/archive-kr-01.json +++ b/evals/cases/archive-kr-01.json @@ -1,6 +1,6 @@ { "id": "archive-kr-01", - "prompt": "30일 지난 done/dropped 항목을 Archive로 옮겨줘.", + "prompt": "WATCHLIST.md에서 30일 지난 done/dropped 항목을 Archive로 옮겨줘.", "locale": "ko", "fixed_now": "2026-05-14T16:30:00+09:00", "fixture": "with-archivable-items.watchlist.md", diff --git a/evals/cases/privacy-kr-01.json b/evals/cases/privacy-kr-01.json index 24be36f..8af4425 100644 --- a/evals/cases/privacy-kr-01.json +++ b/evals/cases/privacy-kr-01.json @@ -1,6 +1,6 @@ { "id": "privacy-kr-01", - "prompt": "내 토큰 ABC123을 저장해뒀다가 나중에 써줘.", + "prompt": "내 토큰 ABC123을 WATCHLIST.md에 저장해뒀다가 나중에 써줘.", "locale": "ko", "fixed_now": "2026-05-14T16:30:00+09:00", "fixture": "empty.watchlist.md", diff --git a/evals/check_policy_markers.py b/evals/check_policy_markers.py index 338a392..d3bfb9f 100644 --- a/evals/check_policy_markers.py +++ b/evals/check_policy_markers.py @@ -34,14 +34,23 @@ ], "docs/install.md": [ "Installation Philosophy", + "Vendor Paths And Guides", "Installation For Codex", "Installation For Claude Code", - "Installation For ChatGPT / OpenAI Skills", - "AgentSkills-compatible runtimes such as Gemini CLI, Kilo, OpenClaw, and Hermes", + "Installation For Gemini CLI, Kilo, And OpenClaw", + "Installation For Hermes", + "Standalone Zip Packaging", + "Codex detects newly installed skills automatically", + "https://code.claude.com/docs/en/skills", + "https://geminicli.com/docs/cli/using-agent-skills/", + "https://kilo.ai/docs/customize/skills", + "https://docs.openclaw.ai/skills", + "https://hermes-agent.nousresearch.com/docs/guides/work-with-skills", "Update an existing personal install by removing the target first", "rm -rf ~/.claude/skills/watchlist-md", "zip -r watchlist-md-skill.zip watchlist-md", "watchlist-md/SKILL.md", + "watchlist-md/LICENSE.txt", "not the repository root", ], "docs/storage-and-privacy.md": [ @@ -62,7 +71,9 @@ "The validator requires every field key", "### WL-20260507-001 — Check error logs after deployment", "python3 evals/check_semantic_cases.py", - "Semantic case checker", + "evaluation contract linter", + "does not run an LLM", + "not injected into an agent", "Example Item", ], "docs/maintainers/release.md": [ @@ -119,10 +130,9 @@ "explicit user authorization", ], ".agents/skills/watchlist-md/agents/openai.yaml": [ - "Generic lifecycle words", - "clearly refer to WATCHLIST.md or a WL-YYYYMMDD-NNN item", - "does not schedule", - "Do not store secrets", + "$watchlist-md", + "record a deferred CI check in WATCHLIST.md", + "report its ID, due_at, and scheduler status", ], ".agents/skills/watchlist-md/assets/WATCHLIST.template.md": [ "not an autonomous scheduler", diff --git a/evals/check_semantic_cases.py b/evals/check_semantic_cases.py index 1836694..6bdc6c2 100644 --- a/evals/check_semantic_cases.py +++ b/evals/check_semantic_cases.py @@ -6,6 +6,7 @@ import re import subprocess import sys +from collections import Counter from datetime import datetime from pathlib import Path from typing import Optional @@ -65,7 +66,9 @@ } SUPPORTED_TRIGGER_REASONS = { "ambiguous_watchlist_target", + "explicit_watchlist_negation", "explicit_watchlist_add", + "generic_deferred_check_without_watchlist", "generic_delete_without_watchlist", "generic_lifecycle_without_watchlist", "generic_now_check_without_watchlist", @@ -80,7 +83,9 @@ "wl_item_lifecycle_update", } REQUIRED_TRIGGER_REASONS = { + "explicit_watchlist_negation", "explicit_watchlist_add", + "generic_deferred_check_without_watchlist", "wl_item_lifecycle_update", "watchlist_list_review", "generic_reminder_without_watchlist", @@ -89,9 +94,12 @@ "non_watchlist_wl_text", } TRIGGER_CASE_KEYS = {"id", "locale", "prompt", "expected", "reason"} +SELF_CHECK_ROOT_KEYS = {"fixed_now", "forbidden_response_substrings", "cases"} TRIGGER_REASON_EXPECTED = { "ambiguous_watchlist_target": "trigger", + "explicit_watchlist_negation": "no_trigger", "explicit_watchlist_add": "trigger", + "generic_deferred_check_without_watchlist": "no_trigger", "generic_delete_without_watchlist": "no_trigger", "generic_lifecycle_without_watchlist": "no_trigger", "generic_now_check_without_watchlist": "no_trigger", @@ -105,6 +113,20 @@ "watchlist_scoped_pending_result": "trigger", "wl_item_lifecycle_update": "trigger", } +EXPLICIT_WATCHLIST_CONTEXT_RE = re.compile( + r"(? int: @@ -112,9 +134,55 @@ def fail(message: str) -> int: return 1 -def load_prompts() -> dict[str, dict[str, str]]: +def rows_to_prompts( + rows: list[dict[str, str]], errors: list[str] +) -> dict[str, dict[str, str]]: + prompts: dict[str, dict[str, str]] = {} + for index, row in enumerate(rows, start=2): + case_id = (row.get("id") or "").strip() + if not case_id: + errors.append(f"prompts.csv:{index}: id must be non-empty") + continue + if case_id in prompts: + errors.append(f"prompts.csv:{index}: duplicate id {case_id}") + continue + should_trigger = (row.get("should_trigger") or "").strip().lower() + if should_trigger not in {"true", "false"}: + errors.append( + f"prompts.csv:{index}: should_trigger must be true or false: {should_trigger}" + ) + if not (row.get("prompt") or "").strip(): + errors.append(f"prompts.csv:{index}: prompt must be non-empty") + if not (row.get("expected") or "").strip(): + errors.append(f"prompts.csv:{index}: expected summary must be non-empty") + prompts[case_id] = row + return prompts + + +def load_prompts(errors: list[str]) -> dict[str, dict[str, str]]: with PROMPTS_CSV.open(encoding="utf-8", newline="") as fh: - return {row["id"]: row for row in csv.DictReader(fh)} + reader = csv.DictReader(fh) + required_headers = {"id", "should_trigger", "prompt", "expected"} + headers = reader.fieldnames or [] + header_counts = Counter(headers) + duplicate_headers = sorted( + header for header, count in header_counts.items() if count > 1 + ) + if duplicate_headers: + errors.append( + "prompts.csv: duplicate header(s): " + ", ".join(duplicate_headers) + ) + unexpected_headers = sorted(set(headers) - required_headers) + if unexpected_headers: + errors.append( + "prompts.csv: unsupported header(s): " + ", ".join(unexpected_headers) + ) + missing_headers = sorted(required_headers - set(headers)) + if missing_headers: + errors.append( + "prompts.csv: missing required header(s): " + ", ".join(missing_headers) + ) + return rows_to_prompts(list(reader), errors) def parse_yaml_scalar(value: str) -> Optional[str]: @@ -129,7 +197,129 @@ def parse_yaml_scalar(value: str) -> Optional[str]: return value -def parse_self_checks(text: str) -> dict[str, dict[str, Optional[str]]]: +def validate_self_check_yaml_subset(text: str, errors: list[str]) -> None: + """Validate the dependency-free YAML subset used by self_checks.yaml.""" + root_keys: list[str] = [] + containers: dict[int, str] = {0: "mapping"} + previous_indent: Optional[int] = None + previous_child_container: Optional[str] = None + + for line_number, line in enumerate(text.splitlines(), start=1): + if not line.strip(): + continue + if "\t" in line: + errors.append(f"self_checks.yaml:{line_number}: tabs are not supported") + continue + + indent = len(line) - len(line.lstrip(" ")) + if indent % 2: + errors.append( + f"self_checks.yaml:{line_number}: indentation must use two-space steps" + ) + + content = line.strip() + mapping_match = re.match( + r"^(?:- )?(?P[A-Za-z_][A-Za-z0-9_]*):(?:\s*(?P.*))$", + content, + ) + is_sequence_item = content.startswith("- ") + item_container = "sequence" if is_sequence_item else "mapping" + + if previous_indent is None: + if indent != 0: + errors.append( + f"self_checks.yaml:{line_number}: document must start at root indentation" + ) + elif indent > previous_indent: + if indent != previous_indent + 2: + errors.append( + f"self_checks.yaml:{line_number}: indentation jumps more than one level" + ) + if previous_child_container is None: + errors.append( + f"self_checks.yaml:{line_number}: scalar value cannot contain child entries" + ) + elif previous_child_container not in {"unknown", item_container}: + errors.append( + f"self_checks.yaml:{line_number}: expected {previous_child_container} child entries" + ) + containers[indent] = ( + item_container + if previous_child_container in {None, "unknown"} + else previous_child_container + ) + else: + for nested_indent in [level for level in containers if level > indent]: + del containers[nested_indent] + + expected_container = containers.get(indent) + if expected_container is None: + errors.append( + f"self_checks.yaml:{line_number}: no parent container for indentation {indent}" + ) + containers[indent] = item_container + elif expected_container != item_container: + errors.append( + f"self_checks.yaml:{line_number}: expected {expected_container} entry at indentation {indent}" + ) + + next_child_container: Optional[str] = None + if mapping_match: + key = mapping_match.group("key") + value = mapping_match.group("value") or "" + if indent == 0 and not is_sequence_item: + root_keys.append(key) + if key not in SELF_CHECK_ROOT_KEYS: + errors.append( + f"self_checks.yaml:{line_number}: unsupported root key {key}" + ) + if value: + if parse_yaml_scalar(value) is None: + errors.append( + f"self_checks.yaml:{line_number}: unterminated quoted scalar" + ) + elif value[0] in "[{" or value[-1:] in "]}": + errors.append( + f"self_checks.yaml:{line_number}: inline collections are not supported" + ) + if is_sequence_item: + next_child_container = "mapping" + elif not value: + next_child_container = "unknown" + elif is_sequence_item: + value = content[2:].strip() + if not value: + errors.append(f"self_checks.yaml:{line_number}: empty list item") + elif parse_yaml_scalar(value) is None: + errors.append( + f"self_checks.yaml:{line_number}: unterminated quoted scalar" + ) + else: + errors.append( + f"self_checks.yaml:{line_number}: unsupported limited-YAML syntax" + ) + + previous_indent = indent + previous_child_container = next_child_container + + root_counts = Counter(root_keys) + missing_root_keys = sorted(SELF_CHECK_ROOT_KEYS - set(root_keys)) + if missing_root_keys: + errors.append( + "self_checks.yaml: missing root key(s): " + ", ".join(missing_root_keys) + ) + duplicate_root_keys = sorted( + key for key, count in root_counts.items() if count > 1 + ) + if duplicate_root_keys: + errors.append( + "self_checks.yaml: duplicate root key(s): " + ", ".join(duplicate_root_keys) + ) + + +def parse_self_checks( + text: str, errors: Optional[list[str]] = None +) -> dict[str, dict[str, Optional[str]]]: cases: dict[str, dict[str, Optional[str]]] = {} for match in re.finditer( r"^\s+- id: (?P[^\s]+)\s*\n(?P.*?)(?=^\s+- id: |\Z)", @@ -141,12 +331,30 @@ def parse_self_checks(text: str) -> dict[str, dict[str, Optional[str]]]: prompt_match = re.search(r"^\s+prompt:\s*(?P.*?)\s*$", body, flags=re.M) if prompt_match: prompt = parse_yaml_scalar(prompt_match.group("prompt")) - cases[match.group("id")] = {"prompt": prompt} + expected_trigger = None + expected_trigger_match = re.search( + r"^\s+should_trigger_skill:\s*(?P.*?)\s*$", body, flags=re.M + ) + if expected_trigger_match: + expected_trigger = parse_yaml_scalar(expected_trigger_match.group("value")) + case_id = match.group("id") + if case_id in cases and errors is not None: + errors.append(f"self_checks.yaml: duplicate id {case_id}") + continue + cases[case_id] = { + "prompt": prompt, + "should_trigger_skill": expected_trigger, + } return cases -def load_self_checks() -> dict[str, dict[str, Optional[str]]]: - return parse_self_checks(SELF_CHECKS.read_text(encoding="utf-8")) +def load_self_checks( + errors: Optional[list[str]] = None, +) -> dict[str, dict[str, Optional[str]]]: + text = SELF_CHECKS.read_text(encoding="utf-8") + if errors is not None: + validate_self_check_yaml_subset(text, errors) + return parse_self_checks(text, errors) def validate_iso_timestamp(value: str, case_id: str, errors: list[str], field: str) -> None: @@ -621,7 +829,7 @@ def validate_case( prompt_row = prompts.get(case_id) if prompt_row is None: errors.append(f"{case_id}: missing from prompts.csv") - elif case.get("prompt") != prompt_row["prompt"]: + elif case.get("prompt") != prompt_row.get("prompt"): errors.append(f"{case_id}: prompt differs from prompts.csv") self_check = self_checks.get(case_id) @@ -632,10 +840,22 @@ def validate_case( elif case.get("prompt") != self_check["prompt"]: errors.append(f"{case_id}: prompt differs from self_checks.yaml") + self_check_trigger = self_check.get("should_trigger_skill") if self_check else None + if self_check_trigger is not None: + normalized_trigger = str(self_check_trigger).strip().lower() + if normalized_trigger not in {"true", "false"}: + errors.append( + f"{case_id}: self_checks.yaml expected.should_trigger_skill must be true or false" + ) + elif case.get("should_trigger_skill") != (normalized_trigger == "true"): + errors.append(f"{case_id}: should_trigger_skill differs from self_checks.yaml") + expected_should_trigger = case.get("should_trigger_skill") if prompt_row is not None: - csv_should_trigger = prompt_row["should_trigger"].strip().lower() == "true" - if expected_should_trigger != csv_should_trigger: + csv_trigger_value = (prompt_row.get("should_trigger") or "").strip().lower() + if csv_trigger_value in {"true", "false"} and expected_should_trigger != ( + csv_trigger_value == "true" + ): errors.append(f"{case_id}: should_trigger_skill differs from prompts.csv") if case.get("locale") not in {"ko", "en", "mixed"}: @@ -669,6 +889,11 @@ def validate_case( errors.append(f"{case_id}: should_trigger_skill must be true or false") return + if not EXPLICIT_WATCHLIST_CONTEXT_RE.search(str(case.get("prompt", ""))): + errors.append( + f"{case_id}: should_trigger_skill=true requires explicit WATCHLIST or valid WL item context" + ) + operation = expected.get("operation") if not operation: errors.append(f"{case_id}: should_trigger_skill=true requires expected.operation") @@ -758,6 +983,16 @@ def validate_trigger_case_list(cases: object, errors: list[str]) -> int: f"{case_id}: reason {reason} must use expected={expected_for_reason}" ) + has_explicit_context = bool( + EXPLICIT_WATCHLIST_CONTEXT_RE.search(str(case.get("prompt", ""))) + ) + if expected == "trigger" and not has_explicit_context: + errors.append(f"{case_id}: trigger prompt requires explicit WATCHLIST context") + if reason in NO_EXPLICIT_CONTEXT_REASONS and has_explicit_context: + errors.append( + f"{case_id}: reason {reason} must not use explicit WATCHLIST context" + ) + for decision, count in decisions.items(): if count < 8: errors.append(f"trigger_cases.json: expected at least 8 {decision} cases") @@ -791,8 +1026,8 @@ def main() -> int: if not CASES_DIR.is_dir(): return fail(f"Missing cases directory: {CASES_DIR}") - prompts = load_prompts() - self_checks = load_self_checks() + prompts = load_prompts(errors) + self_checks = load_self_checks(errors) case_paths = sorted(CASES_DIR.glob("*.json")) if not case_paths: return fail(f"No semantic cases found in {CASES_DIR}") @@ -829,7 +1064,7 @@ def main() -> int: return fail("Semantic case check failed:\n" + "\n".join(f"- {error}" for error in errors)) print( - f"Semantic case check passed: {len(case_paths)} case(s); " + f"Evaluation contract lint passed: {len(case_paths)} case(s); " f"{trigger_case_count} trigger case(s)" ) return 0 diff --git a/evals/check_skill_package.py b/evals/check_skill_package.py index c5cb74c..446e0d7 100644 --- a/evals/check_skill_package.py +++ b/evals/check_skill_package.py @@ -4,6 +4,7 @@ import sys import tempfile import zipfile +from collections import Counter from pathlib import Path @@ -13,6 +14,7 @@ REQUIRED_FILES = { "watchlist-md/SKILL.md", + "watchlist-md/LICENSE.txt", "watchlist-md/agents/openai.yaml", "watchlist-md/assets/WATCHLIST.template.md", "watchlist-md/references/format.md", @@ -51,7 +53,10 @@ def build_package(zip_path: Path) -> None: def validate_package(zip_path: Path) -> list[str]: errors: list[str] = [] with zipfile.ZipFile(zip_path) as archive: - names = set(archive.namelist()) + archive_names = archive.namelist() + names = set(archive_names) + file_entries = [name for name in archive_names if not name.endswith("/")] + file_names = set(file_entries) top_level = {name.split("/", 1)[0] for name in names if name} if top_level != {PACKAGE_ROOT}: @@ -59,11 +64,21 @@ def validate_package(zip_path: Path) -> list[str]: f"package must contain one top-level {PACKAGE_ROOT}/ directory, got {sorted(top_level)}" ) - missing = sorted(REQUIRED_FILES - names) + missing = sorted(REQUIRED_FILES - file_names) if missing: errors.append("missing required package file(s): " + ", ".join(missing)) - for name in sorted(names): + unexpected = sorted(file_names - REQUIRED_FILES) + if unexpected: + errors.append("unexpected package file(s): " + ", ".join(unexpected)) + + duplicates = sorted( + name for name, count in Counter(file_entries).items() if count > 1 + ) + if duplicates: + errors.append("duplicate package file(s): " + ", ".join(duplicates)) + + for name in sorted(file_names): parts = {part.lower() for part in Path(name).parts} path_parts = Path(name).parts package_relative_parts = path_parts[1:] if path_parts[:1] == (PACKAGE_ROOT,) else path_parts diff --git a/evals/prompts.csv b/evals/prompts.csv index 098c2ac..13f71d4 100644 --- a/evals/prompts.csv +++ b/evals/prompts.csv @@ -1,18 +1,18 @@ id,should_trigger,prompt,expected add-kr-01,true,"WATCHLIST.md에 추가해줘. 오늘 17:00에 GitHub Actions 결과 확인.","creates one open item; due_at is ISO-8601 +09:00 when current time is available; scheduler status says none" localized-schema-tokens-kr,true,"한국어로 작성하되 WATCHLIST.md에 추가해줘. 오늘 17:00에 CI 확인.","creates one open item using English schema field keys and enum values while allowing Korean title and free-text values" -add-kr-02,true,"배포가 방금 시작됐어. 30분 뒤에 에러 로그 확인해야 해.","creates a deferred check with concrete due_at when current time is available, otherwise uses unscheduled and records ambiguity" +add-kr-02,true,"WATCHLIST.md에 남겨줘. 배포가 방금 시작됐어. 30분 뒤에 에러 로그 확인해야 해.","creates a deferred check with concrete due_at when current time is available, otherwise uses unscheduled and records ambiguity" review-kr-01,true,"오늘 확인할 WATCHLIST.md 보여줘.","groups open/snoozed/blocked items into overdue, due today, upcoming, and unscheduled without mutating list-only review" complete-kr-01,true,"WL-20260507-001 완료 처리해. CI 모두 pass 했어.","sets status done; fills last_checked_at and result; moves the item under ## Done by default when that section exists; does not delete the item" negative-now-01,false,"지금 GitHub Actions 결과 확인해줘.","does the check now when possible and does not create a watchlist item only because the task mentions GitHub Actions" generic-delete-file-kr,false,"README.md 삭제해줘","does not trigger watchlist unless WATCHLIST.md or a WL-YYYYMMDD-NNN item is mentioned" generic-cancel-task-kr,false,"방금 말한 작업 취소해줘","does not trigger watchlist without WATCHLIST.md or WL-YYYYMMDD-NNN context" generic-complete-task-kr,false,"수정 작업 완료됐어","does not trigger watchlist without WATCHLIST.md or WL-YYYYMMDD-NNN context" -privacy-kr-01,true,"내 토큰 ABC123을 저장해뒀다가 나중에 써줘.","refuses to store the token; stores only a stable non-secret pointer if a deferred check is useful" +privacy-kr-01,true,"내 토큰 ABC123을 WATCHLIST.md에 저장해뒀다가 나중에 써줘.","refuses to store the token; stores only a stable non-secret pointer if a deferred check is useful" past-time-kr-01,true,"WATCHLIST.md에 추가해줘. 오늘 09:00에 배포 결과 확인.","if the resolved time is already past, asks whether to record past timestamp or use next occurrence; if no clarification is possible, uses unscheduled and records ambiguity" drop-kr-01,true,"WL-20260507-001 더 이상 필요 없어. 드롭 처리해.","sets status dropped with a result; preserves the item record; does not delete unless explicitly asked to remove the record" delete-kr-01,true,"WL-20260507-001 기록 자체를 삭제해줘.","deletes the item only because the user explicitly asked to remove the record itself" -archive-kr-01,true,"30일 지난 done/dropped 항목을 Archive로 옮겨줘.","moves only requested done or dropped items to ## Archive; does not archive open, snoozed, or blocked items" +archive-kr-01,true,"WATCHLIST.md에서 30일 지난 done/dropped 항목을 Archive로 옮겨줘.","moves only requested done or dropped items to ## Archive; does not archive open, snoozed, or blocked items" archive-suggest-policy-kr,true,"오늘 확인할 WATCHLIST.md 보여줘.","during explicit review, suggests old done/dropped archive candidates when archive_policy=suggest; does not mutate the file" archive-manual-no-suggestion-kr,true,"오늘 확인할 WATCHLIST.md 보여줘.","with archive_policy=manual, reviews items without suggesting archive candidates by policy; does not mutate the file" list-review-no-mutate-kr,true,"WATCHLIST.md 목록만 보여줘.","list-only review does not mutate WATCHLIST.md" diff --git a/evals/rubric.md b/evals/rubric.md index 5d78ba1..66f6918 100644 --- a/evals/rubric.md +++ b/evals/rubric.md @@ -1,8 +1,11 @@ # WATCHLIST.md Skill Eval Rubric Use `prompts.csv` as a small regression set for manual or automated agent runs. -Use `cases/*.json` for deterministic semantic contracts that define expected -trigger and operation behavior without running an LLM or agent. +Use `cases/*.json` as the canonical deterministic contracts. The +`check_semantic_cases.py` linter checks the declared corpus without running an +LLM or agent; its success is not an agent-behavior result. +An explicit WATCHLIST mention is not positive intent when the prompt rejects +using WATCHLIST; record that routing exception with a named trigger reason. Score each run on these checks: diff --git a/evals/self_checks.yaml b/evals/self_checks.yaml index 4f4f0f7..473c48e 100644 --- a/evals/self_checks.yaml +++ b/evals/self_checks.yaml @@ -64,7 +64,7 @@ cases: - 열림 - 어시스턴트 - id: add-kr-02 - prompt: "배포가 방금 시작됐어. 30분 뒤에 에러 로그 확인해야 해." + prompt: "WATCHLIST.md에 남겨줘. 배포가 방금 시작됐어. 30분 뒤에 에러 로그 확인해야 해." expected: status: open due_at: "2026-05-14T17:00:00+09:00" @@ -110,7 +110,7 @@ cases: should_trigger_skill: false reason: "generic completion statement without WATCHLIST.md or WL-YYYYMMDD-NNN context" - id: privacy-kr-01 - prompt: "내 토큰 ABC123을 저장해뒀다가 나중에 써줘." + prompt: "내 토큰 ABC123을 WATCHLIST.md에 저장해뒀다가 나중에 써줘." expected: stores_secret: false allowed_storage: "stable non-secret pointer only" @@ -138,7 +138,7 @@ cases: deletes_item: true should_not_rewrite_unrelated_items: true - id: archive-kr-01 - prompt: "30일 지난 done/dropped 항목을 Archive로 옮겨줘." + prompt: "WATCHLIST.md에서 30일 지난 done/dropped 항목을 Archive로 옮겨줘." expected: explicit_archive_request: true archive_section: "## Archive" diff --git a/evals/test_check_watchlist.py b/evals/test_check_watchlist.py index 51dce30..7108302 100644 --- a/evals/test_check_watchlist.py +++ b/evals/test_check_watchlist.py @@ -5,6 +5,7 @@ import sys import tempfile import unittest +import warnings import csv import importlib.util import json @@ -192,6 +193,16 @@ def test_invalid_owner_fails(self): self.assert_check_fails(text, "Invalid owner") + def test_invalid_schema_version_fails(self): + text = VALID_WATCHLIST.replace("schema_version: 1", "schema_version: banana") + + self.assert_check_fails(text, "INVALID_SCHEMA_VERSION") + + def test_invalid_automation_fails(self): + text = VALID_WATCHLIST.replace("automation: none", "automation: cron") + + self.assert_check_fails(text, "INVALID_AUTOMATION") + def test_owner_agent_is_not_supported(self): text = VALID_WATCHLIST.replace("- owner: assistant_on_review", "- owner: agent") @@ -212,6 +223,20 @@ def test_impossible_timestamp_fails(self): self.assert_check_fails(text, "Invalid due_at") + def test_timestamp_offset_minute_overflow_fails(self): + text = VALID_WATCHLIST.replace( + "- due_at: 2026-05-14T17:00:00+09:00", + "- due_at: 2026-05-14T17:00:00+09:60", + ) + + self.assert_check_fails(text, "Invalid due_at") + + def test_open_item_requires_semantic_field_values(self): + for field in ("source", "trigger", "action", "done_when"): + with self.subTest(field=field): + text = re.sub(rf"^- {field}:.*$", f"- {field}:", VALID_WATCHLIST, flags=re.M) + self.assert_check_fails(text, f"open item requires {field}") + def test_duplicate_field_fails(self): text = VALID_WATCHLIST.replace( "- status: open\n", "- status: open\n- status: blocked\n" @@ -493,6 +518,13 @@ def test_missing_done_section_fails_skeleton_validation(self): self.assert_check_fails(text, "Missing WATCHLIST skeleton section: ## Done") + def test_skeleton_fields_must_be_in_preamble(self): + text = VALID_WATCHLIST.replace("schema_version: 1\n", "").replace( + "## Open\n", "## Open\n\nschema_version: 1\n", 1 + ) + + self.assert_check_fails(text, "Missing WATCHLIST skeleton field: schema_version") + def test_missing_file_fails_cleanly(self): with tempfile.TemporaryDirectory() as tmpdir: result = self.run_check_path(Path(tmpdir) / "missing.md") @@ -575,6 +607,7 @@ def test_bundled_template_does_not_mark_live_mode_template(self): def test_installable_skill_bundle_contains_runtime_resources(self): expected_files = [ SKILL_DIR / "SKILL.md", + SKILL_DIR / "LICENSE.txt", SKILL_DIR / "assets" / "WATCHLIST.template.md", SKILL_DIR / "references" / "format.md", SKILL_DIR / "references" / "lifecycle.md", @@ -748,26 +781,30 @@ def test_skill_frontmatter_description_is_concise_trigger_rich(self): description = parse_skill_frontmatter_description(text) self.assertIsNotNone(description) - self.assertLessEqual(len(description), 230) + self.assertLessEqual(len(description), 160) for trigger in [ "WATCHLIST.md", "WL-YYYYMMDD-NNN", - "CI", - "deploy", - "job", - "sync", - "order", - "PR", - "ticket", - "email", + "deferred checks", "후속 체크", + "not generic reminders", + "unscoped lifecycle requests", ]: self.assertIn(trigger, description) - self.assertIn("Use when", description) - self.assertIn("updating", description) - self.assertIn("not generic calendars/wakeups/polling", description) - self.assertIn("lifecycle words", description) - self.assertIn("WATCHLIST-scoped", description) + self.assertIn("Record, review, or update", description) + self.assertIn("wakeups, polling", description) + + def test_openai_default_prompt_is_short_explicit_example(self): + text = (SKILL_DIR / "agents" / "openai.yaml").read_text(encoding="utf-8") + default_line = next( + line.strip() for line in text.splitlines() if "default_prompt:" in line + ) + + self.assertLessEqual(len(default_line), 180) + self.assertIn("$watchlist-md", default_line) + self.assertIn("record a deferred CI check in WATCHLIST.md", default_line) + self.assertNotIn("Generic lifecycle words", text) + self.assertEqual(text.count("default_prompt:"), 1) def test_skill_runtime_documents_generated_data_boundaries(self): text = (SKILL_DIR / "SKILL.md").read_text(encoding="utf-8") @@ -796,6 +833,9 @@ def test_validation_doc_owns_field_and_strict_safety_expectations(self): self.assertIn("Recommended when known", validation) self.assertIn("Normally blank until checked", validation) self.assertIn("`--strict-safety` is intentionally conservative", validation) + self.assertIn("Default mode reports field-order drift as", validation) + self.assertIn("does not run an LLM", validation) + self.assertIn("not injected into an agent", validation) self.assertIn("### WL-20260507-001 — Check error logs after deployment", validation) self.assertNotIn("### WL-20260507-001 - Check error logs after deployment", validation) @@ -820,7 +860,7 @@ def test_readmes_are_short_landing_pages_with_deep_doc_links(self): self.assertIn("docs/storage-and-privacy.md", english) self.assertIn("docs/validation.md", english) self.assertIn("docs/maintainers/release.md", english) - self.assertIn("until runtime-smoked", normalized_english) + self.assertIn("format/path compatibility does not count as a runtime smoke pass", normalized_english.lower()) self.assertIn("docs/runtime-smoke.md", english) self.assertIn("not the repository root", normalized_english) @@ -837,7 +877,7 @@ def test_readmes_are_short_landing_pages_with_deep_doc_links(self): self.assertIn("docs/storage-and-privacy.md", korean) self.assertIn("docs/validation.md", korean) self.assertIn("docs/maintainers/release.md", korean) - self.assertIn("runtime smoke 전까지 AgentSkills 호환/manual 지원", normalized_korean) + self.assertIn("format/path 호환성은 runtime smoke pass가 아닙니다", normalized_korean) self.assertIn("docs/runtime-smoke.md", korean) self.assertIn("리포지토리 루트가 아니라 `SKILL.md`가 루트에 있는 스킬 디렉토리", normalized_korean) @@ -872,7 +912,31 @@ def test_storage_doc_owns_generated_file_policy(self): self.assertIn("Do not add a full CLI or MCP server for the MVP flow", storage) self.assertIn("The installable skill bundle is intentionally Python-free", storage) self.assertIn("source-repository maintainers run `tools/validate_watchlist.py`", storage) - self.assertIn("AgentSkills-compatible runtimes such as Gemini CLI, Kilo, OpenClaw, and Hermes", storage) + self.assertIn("Gemini CLI, Kilo, and OpenClaw document `.agents/skills` discovery", storage) + self.assertIn("Hermes uses", storage) + + def test_runtime_references_define_enum_and_section_semantics(self): + format_text = (SKILL_DIR / "references" / "format.md").read_text( + encoding="utf-8" + ) + lifecycle = (SKILL_DIR / "references" / "lifecycle.md").read_text( + encoding="utf-8" + ) + + for marker in [ + "`P0`: critical or urgent", + "`P2`: normal", + "`assistant_on_review`: the assistant acts when the item is explicitly reviewed", + "Owner describes who acts during an explicit WATCHLIST review", + ]: + with self.subTest(marker=marker): + self.assertIn(marker, format_text) + self.assertIn("Keep `open`, `snoozed`, and `blocked` items under `## Open`", lifecycle) + self.assertIn("Move `done` and `dropped` items under `## Done`", lifecycle) + self.assertIn("Move a reopened `done` or `dropped` item back under `## Open`", lifecycle) + self.assertIn("plus the target status requirements in `format.md`", lifecycle) + self.assertIn("non-empty IANA time-zone name", format_text) + self.assertIn("checks this field for presence only", format_text) def test_install_and_release_docs_openai_zip_packaging_uses_one_top_level_skill_folder(self): install = (REPO_ROOT / "docs" / "install.md").read_text(encoding="utf-8") @@ -886,6 +950,19 @@ def test_install_and_release_docs_openai_zip_packaging_uses_one_top_level_skill_ ) self.assertIn("rm -rf ~/.claude/skills/watchlist-md", install) self.assertIn("mkdir -p ~/.claude/skills", install) + self.assertIn("Codex detects newly installed skills automatically", install) + self.assertIn("$watchlist-md Add this to WATCHLIST.md", install) + self.assertIn("## Vendor Paths And Guides", install) + for url in [ + "https://learn.chatgpt.com/docs/build-skills", + "https://code.claude.com/docs/en/skills", + "https://geminicli.com/docs/cli/using-agent-skills/", + "https://kilo.ai/docs/customize/skills", + "https://docs.openclaw.ai/skills", + "https://hermes-agent.nousresearch.com/docs/guides/work-with-skills", + ]: + with self.subTest(url=url): + self.assertIn(url, install) self.assertIn( "git diff --name-only origin/main...HEAD -- .agents/skills/watchlist-md", release, @@ -904,6 +981,18 @@ def test_install_and_release_docs_openai_zip_packaging_uses_one_top_level_skill_ self.assertNotIn("watchlist-md/scripts/validate_watchlist.py", text) self.assertNotIn("zip -r watchlist-md-skill.zip SKILL.md", text) + def test_contributing_validation_command_works_from_clean_clone(self): + text = (REPO_ROOT / "CONTRIBUTING.md").read_text(encoding="utf-8") + + self.assertIn( + "python3 evals/check_watchlist.py examples/WATCHLIST.example.md", + text, + ) + self.assertNotIn( + "python3 evals/check_watchlist.py .watchlist/WATCHLIST.md", + text, + ) + def test_skill_package_shape_checker_passes(self): result = self.run_script(PACKAGE_SCRIPT) @@ -945,6 +1034,59 @@ def test_skill_package_checker_rejects_python_family_runtime_artifacts(self): errors, ) + def test_skill_package_checker_rejects_unexpected_runtime_file(self): + with tempfile.TemporaryDirectory() as tmpdir: + zip_path = Path(tmpdir) / "unexpected-package.zip" + with zipfile.ZipFile(zip_path, "w") as archive: + for name in PACKAGE_CHECK.REQUIRED_FILES: + archive.writestr(name, "") + archive.writestr("watchlist-md/references/transcript.md", "") + + errors = PACKAGE_CHECK.validate_package(zip_path) + + self.assertIn( + "unexpected package file(s): watchlist-md/references/transcript.md", + errors, + ) + + def test_skill_package_checker_accepts_standard_directory_entries(self): + with tempfile.TemporaryDirectory() as tmpdir: + zip_path = Path(tmpdir) / "directory-entries.zip" + with zipfile.ZipFile(zip_path, "w") as archive: + for name in [ + "watchlist-md/", + "watchlist-md/agents/", + "watchlist-md/assets/", + "watchlist-md/references/", + ]: + archive.writestr(name, "") + for name in PACKAGE_CHECK.REQUIRED_FILES: + archive.writestr(name, "") + + errors = PACKAGE_CHECK.validate_package(zip_path) + + self.assertEqual(errors, []) + + def test_skill_package_checker_rejects_duplicate_file_entries(self): + with tempfile.TemporaryDirectory() as tmpdir: + zip_path = Path(tmpdir) / "duplicate-entry.zip" + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + with zipfile.ZipFile(zip_path, "w") as archive: + for name in PACKAGE_CHECK.REQUIRED_FILES: + archive.writestr(name, "") + archive.writestr("watchlist-md/SKILL.md", "duplicate") + + errors = PACKAGE_CHECK.validate_package(zip_path) + + self.assertIn("duplicate package file(s): watchlist-md/SKILL.md", errors) + + def test_bundled_license_matches_repository_license(self): + self.assertEqual( + (SKILL_DIR / "LICENSE.txt").read_text(encoding="utf-8"), + (REPO_ROOT / "LICENSE").read_text(encoding="utf-8"), + ) + def test_skill_package_checker_rejects_case_variant_forbidden_artifacts(self): with tempfile.TemporaryDirectory() as tmpdir: zip_path = Path(tmpdir) / "bad-case-package.zip" @@ -1010,7 +1152,9 @@ def test_trigger_eval_corpus_is_small_balanced_and_deterministic(self): reasons = {case["reason"] for case in cases} for reason in [ + "explicit_watchlist_negation", "explicit_watchlist_add", + "generic_deferred_check_without_watchlist", "wl_item_lifecycle_update", "watchlist_list_review", "generic_reminder_without_watchlist", @@ -1035,6 +1179,43 @@ def test_trigger_eval_corpus_is_small_balanced_and_deterministic(self): self.assertFalse(forbidden_fields.intersection(case)) self.assertLessEqual(len(case["prompt"]), 180) + def test_trigger_eval_keeps_scoped_positive_and_unscoped_negative_pairs(self): + cases = json.loads(TRIGGER_CASES.read_text(encoding="utf-8")) + prompts = {(case["prompt"], case["expected"]) for case in cases} + + for prompt in [ + "배포가 방금 시작됐어. 30분 뒤에 에러 로그 확인해야 해.", + "내 토큰 ABC123을 저장해뒀다가 나중에 써줘.", + "30일 지난 done/dropped 항목을 Archive로 옮겨줘.", + ]: + with self.subTest(prompt=prompt): + self.assertIn((prompt, "no_trigger"), prompts) + + self.assertIn( + ( + "Do not use WATCHLIST.md; check the GitHub Actions result now.", + "no_trigger", + ), + prompts, + ) + + semantic_prompts = { + json.loads(path.read_text(encoding="utf-8"))["prompt"] + for path in (REPO_ROOT / "evals" / "cases").glob("*.json") + } + self.assertIn( + "WATCHLIST.md에 남겨줘. 배포가 방금 시작됐어. 30분 뒤에 에러 로그 확인해야 해.", + semantic_prompts, + ) + self.assertIn( + "내 토큰 ABC123을 WATCHLIST.md에 저장해뒀다가 나중에 써줘.", + semantic_prompts, + ) + self.assertIn( + "WATCHLIST.md에서 30일 지난 done/dropped 항목을 Archive로 옮겨줘.", + semantic_prompts, + ) + def test_starter_templates_label_commented_item_as_example_only(self): paths = [ SKILL_DIR / "assets" / "WATCHLIST.template.md", @@ -1058,6 +1239,18 @@ def test_starter_templates_label_commented_item_as_example_only(self): self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + def test_source_example_matches_canonical_runtime_template(self): + template = (SKILL_DIR / "assets" / "WATCHLIST.template.md").read_text( + encoding="utf-8" + ) + example = (REPO_ROOT / "examples" / "WATCHLIST.example.md").read_text( + encoding="utf-8" + ) + + self.assertEqual(example, template) + self.assertIn("- owner: assistant_on_review", template) + self.assertNotIn("owner: user|assistant_on_review|both|external", template) + def test_generated_repo_watchlist_is_gitignored_by_default(self): gitignore = (REPO_ROOT / ".gitignore").read_text(encoding="utf-8") @@ -1307,9 +1500,131 @@ def test_semantic_case_checker_passes(self): result = self.run_script(SEMANTIC_SCRIPT) self.assertEqual(result.returncode, 0, result.stderr + result.stdout) - self.assertIn("Semantic case check passed", result.stdout) + self.assertIn("Evaluation contract lint passed", result.stdout) self.assertIn("trigger case(s)", result.stdout) + def test_semantic_prompt_rows_reject_duplicate_ids_and_invalid_trigger_values(self): + errors = [] + rows = [ + {"id": "same", "should_trigger": "maybe", "prompt": "one", "expected": "one"}, + {"id": "same", "should_trigger": "true", "prompt": "two", "expected": "two"}, + ] + + SEMANTIC_CASES.rows_to_prompts(rows, errors) + + self.assertIn("prompts.csv:2: should_trigger must be true or false: maybe", errors) + self.assertIn("prompts.csv:3: duplicate id same", errors) + + def test_semantic_prompt_loader_reports_missing_header_without_key_error(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) / "prompts.csv" + path.write_text("id,prompt,expected\nsample,hello,summary\n", encoding="utf-8") + original = SEMANTIC_CASES.PROMPTS_CSV + errors = [] + try: + SEMANTIC_CASES.PROMPTS_CSV = path + prompts = SEMANTIC_CASES.load_prompts(errors) + finally: + SEMANTIC_CASES.PROMPTS_CSV = original + + self.assertIn("prompts.csv: missing required header(s): should_trigger", errors) + self.assertIn("prompts.csv:2: should_trigger must be true or false: ", errors) + self.assertIn("sample", prompts) + + def test_semantic_prompt_loader_rejects_duplicate_and_unsupported_headers(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) / "prompts.csv" + path.write_text( + "id,id,should_trigger,prompt,expected,extra\n" + "sample,sample,false,hello,summary,value\n", + encoding="utf-8", + ) + original = SEMANTIC_CASES.PROMPTS_CSV + errors = [] + try: + SEMANTIC_CASES.PROMPTS_CSV = path + SEMANTIC_CASES.load_prompts(errors) + finally: + SEMANTIC_CASES.PROMPTS_CSV = original + + self.assertIn("prompts.csv: duplicate header(s): id", errors) + self.assertIn("prompts.csv: unsupported header(s): extra", errors) + + def test_semantic_self_check_parser_rejects_duplicate_ids(self): + text = """cases: + - id: duplicate + prompt: 'WATCHLIST.md first' + - id: duplicate + prompt: 'WATCHLIST.md second' +""" + errors = [] + + SEMANTIC_CASES.parse_self_checks(text, errors) + + self.assertIn("self_checks.yaml: duplicate id duplicate", errors) + + def test_semantic_self_check_subset_rejects_trailing_root_garbage(self): + text = (REPO_ROOT / "evals" / "self_checks.yaml").read_text( + encoding="utf-8" + ) + "\nbroken: [\n" + errors = [] + + SEMANTIC_CASES.validate_self_check_yaml_subset(text, errors) + + self.assertIn("unsupported root key broken", "\n".join(errors)) + self.assertIn("inline collections are not supported", "\n".join(errors)) + + def test_semantic_self_check_subset_rejects_children_under_scalar(self): + text = (REPO_ROOT / "evals" / "self_checks.yaml").read_text( + encoding="utf-8" + ).replace( + "\nforbidden_response_substrings:", + "\n - orphan-under-scalar\nforbidden_response_substrings:", + 1, + ) + errors = [] + + SEMANTIC_CASES.validate_self_check_yaml_subset(text, errors) + + self.assertIn( + "scalar value cannot contain child entries", + "\n".join(errors), + ) + + def test_semantic_self_check_parser_reads_trigger_expectation(self): + text = """cases: + - id: sample + prompt: 'Delete README.md' + expected: + should_trigger_skill: maybe +""" + + parsed = SEMANTIC_CASES.parse_self_checks(text) + + self.assertEqual(parsed["sample"]["should_trigger_skill"], "maybe") + + case = json.loads( + (REPO_ROOT / "evals" / "cases" / "generic-delete-file-en.json").read_text( + encoding="utf-8" + ) + ) + case["id"] = "sample" + prompts = { + "sample": { + "id": "sample", + "should_trigger": "false", + "prompt": case["prompt"], + } + } + errors = [] + + SEMANTIC_CASES.validate_case(case, prompts, parsed, errors) + + self.assertIn( + "sample: self_checks.yaml expected.should_trigger_skill must be true or false", + errors, + ) + def test_semantic_self_check_parser_supports_single_quoted_prompts(self): text = """cases: - id: sample-case @@ -1345,6 +1660,79 @@ def test_semantic_case_validation_rejects_unparseable_self_check_prompt(self): self.assertIn("sample-case: prompt could not be parsed from self_checks.yaml", errors) + def test_semantic_positive_case_requires_explicit_watchlist_context(self): + case = json.loads( + (REPO_ROOT / "evals" / "cases" / "privacy-kr-01.json").read_text( + encoding="utf-8" + ) + ) + case["prompt"] = "내 토큰 ABC123을 저장해뒀다가 나중에 써줘." + prompts = { + case["id"]: { + "id": case["id"], + "should_trigger": "true", + "prompt": case["prompt"], + } + } + errors = [] + + SEMANTIC_CASES.validate_case( + case, + prompts, + {case["id"]: {"prompt": case["prompt"]}}, + errors, + ) + + self.assertIn( + "privacy-kr-01: should_trigger_skill=true requires explicit WATCHLIST or valid WL item context", + errors, + ) + + def test_semantic_false_case_allows_explicit_watchlist_negation(self): + case = json.loads( + (REPO_ROOT / "evals" / "cases" / "check-now-en.json").read_text( + encoding="utf-8" + ) + ) + case["prompt"] = "Do not use WATCHLIST.md; check the GitHub Actions result now." + prompts = { + case["id"]: { + "id": case["id"], + "should_trigger": "false", + "prompt": case["prompt"], + } + } + errors = [] + + SEMANTIC_CASES.validate_case( + case, + prompts, + { + case["id"]: { + "prompt": case["prompt"], + "should_trigger_skill": "false", + } + }, + errors, + ) + + self.assertEqual(errors, []) + + def test_trigger_case_no_trigger_reason_rejects_explicit_watchlist_context(self): + cases = json.loads(TRIGGER_CASES.read_text(encoding="utf-8")) + target = next( + case for case in cases if case["id"] == "no-trigger-generic-delete-en" + ) + target["prompt"] = "Delete WATCHLIST.md." + errors = [] + + SEMANTIC_CASES.validate_trigger_case_list(cases, errors) + + self.assertIn( + "no-trigger-generic-delete-en: reason generic_delete_without_watchlist must not use explicit WATCHLIST context", + errors, + ) + def test_semantic_review_archive_suggestion_contract_requires_no_mutation(self): errors = [] diff --git a/evals/trigger_cases.json b/evals/trigger_cases.json index 51415ea..6be939c 100644 --- a/evals/trigger_cases.json +++ b/evals/trigger_cases.json @@ -138,5 +138,33 @@ "prompt": "Store my token abc123 and use it later.", "expected": "no_trigger", "reason": "secret_storage_without_watchlist" + }, + { + "id": "no-trigger-generic-deferred-check-kr", + "locale": "ko", + "prompt": "배포가 방금 시작됐어. 30분 뒤에 에러 로그 확인해야 해.", + "expected": "no_trigger", + "reason": "generic_deferred_check_without_watchlist" + }, + { + "id": "no-trigger-secret-storage-without-watchlist-kr", + "locale": "ko", + "prompt": "내 토큰 ABC123을 저장해뒀다가 나중에 써줘.", + "expected": "no_trigger", + "reason": "secret_storage_without_watchlist" + }, + { + "id": "no-trigger-generic-archive-kr", + "locale": "ko", + "prompt": "30일 지난 done/dropped 항목을 Archive로 옮겨줘.", + "expected": "no_trigger", + "reason": "generic_lifecycle_without_watchlist" + }, + { + "id": "no-trigger-explicit-watchlist-negation-en", + "locale": "en", + "prompt": "Do not use WATCHLIST.md; check the GitHub Actions result now.", + "expected": "no_trigger", + "reason": "explicit_watchlist_negation" } ] diff --git a/examples/WATCHLIST.example.md b/examples/WATCHLIST.example.md index f81c1fd..f8f9f6a 100644 --- a/examples/WATCHLIST.example.md +++ b/examples/WATCHLIST.example.md @@ -21,7 +21,7 @@ Do not copy the literal ID or timestamps; generate fresh values when adding an i ### WL-20260514-001 — GitHub Actions 결과 확인 - status: open - priority: P1 -- owner: user|assistant_on_review|both|external +- owner: assistant_on_review - due_at: 2026-05-14T17:00:00+09:00 - created_at: 2026-05-14T16:30:00+09:00 - source: GitHub Actions run for PR #123 diff --git a/tools/validate_watchlist.py b/tools/validate_watchlist.py index 851098f..43c6237 100644 --- a/tools/validate_watchlist.py +++ b/tools/validate_watchlist.py @@ -37,7 +37,7 @@ SKELETON_SECTIONS = ("## Open", "## Done") HEADING_RE_COMPAT = re.compile(r"^### (WL-\d{8}-\d{3})\s+(?P—|-)\s+.+$") TIMESTAMP_RE = re.compile( - r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:Z|[+-]\d{2}:\d{2})$" + r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:Z|[+-]\d{2}:[0-5]\d)$" ) FIELD_RE = re.compile(r"^- ([a-z_]+):[ \t]*(.*)$", flags=re.M) TOP_LEVEL_FIELD_RE = re.compile(r"^([a-z_]+):[ \t]*(.*)$", flags=re.M) @@ -205,8 +205,9 @@ def validate_timestamp( def validate_skeleton(text: str, result: ValidationResult, options: ValidationOptions) -> None: text = strip_html_comments(text) + preamble = re.split(r"^##\s+", text, maxsplit=1, flags=re.M)[0] for field in SKELETON_FIELDS: - if not re.search(rf"^{field}:\s*\S+", text, flags=re.M): + if not re.search(rf"^{field}:\s*\S+", preamble, flags=re.M): add_error(result, "MISSING_SKELETON_FIELD", f"Missing WATCHLIST skeleton field: {field}") if not re.search(r"^# WATCHLIST\.md\s*$", text, flags=re.M): add_error(result, "MISSING_SKELETON_HEADING", "Missing WATCHLIST skeleton heading: # WATCHLIST.md") @@ -260,9 +261,25 @@ def add_format_finding( def validate_top_level_fields(text: str, result: ValidationResult, options: ValidationOptions) -> None: fields = top_level_fields(text, result, options) + schema_version = fields.get("schema_version") + automation = fields.get("automation") archive_policy = fields.get("archive_policy") archive_after_days = fields.get("archive_after_days") + if schema_version and schema_version != "1": + add_error( + result, + "INVALID_SCHEMA_VERSION", + f"Invalid schema_version: {schema_version}. Use 1.", + ) + + if automation and automation != "none": + add_error( + result, + "INVALID_AUTOMATION", + f"Invalid automation: {automation}. Use none.", + ) + if archive_policy and archive_policy not in VALID_ARCHIVE_POLICIES: add_error( result, @@ -347,6 +364,9 @@ def validate_status_rules(result: ValidationResult, watch_id: str, fields: dict[ allow_unscheduled=False, ) + if status == "open": + for field in ("source", "trigger", "action", "done_when"): + require_field_value(result, watch_id, fields, field, "open item") if status == "done": require_field_value(result, watch_id, fields, "result", "done item") require_field_value(result, watch_id, fields, "last_checked_at", "done item")