diff --git a/kits/agent-kiro-kit/README.md b/kits/agent-kiro-kit/README.md index b31311a..27aa2ad 100644 --- a/kits/agent-kiro-kit/README.md +++ b/kits/agent-kiro-kit/README.md @@ -63,7 +63,7 @@ strategy the step failure `BEDDEL-EXEC-002` wraps it as its cause. Operator install/verify: `PYTHONPATH=/python python3 -m beddel_agent_kiro.agent_config install|verify`; supports `--home`, `--pointers`, `--replace`. Pointer `/.json`; policy `beddel.kiro-hook-policy/1`, first-match-wins, no rules deny all; Python 3.11+; errors `KIRO_AGENT_INVALID`/`KIRO_CLI_MISSING`. -Pointer or log path security failures deny without writing through the pointer. Logs use +Pointer or log path security failures deny without writing through the pointer. Each agent-stage policy must declare a positive `max_tool_calls`; the call past the cap is denied with `cap.max_tool_calls` and counted as `denied`. Logs use `beddel.kiro-hook/1` and 16 fields; results include `type=kiro-hook`, counts, and `log_sha256`. Exit 2 means deny/error; raw values are never logged. `policy_sha256` is the SHA-256 of raw policy bytes. diff --git a/kits/agent-kiro-kit/python/beddel_agent_kiro/hook.py b/kits/agent-kiro-kit/python/beddel_agent_kiro/hook.py index aafa269..8835069 100644 --- a/kits/agent-kiro-kit/python/beddel_agent_kiro/hook.py +++ b/kits/agent-kiro-kit/python/beddel_agent_kiro/hook.py @@ -18,7 +18,7 @@ SCHEMA = "beddel.kiro-hook/1" _POLICY_SCHEMA = "beddel.kiro-hook-policy/1" _POINTER_KEYS = frozenset({"policy", "log_path", "stage"}) -_POLICY_KEYS = frozenset({"schema", "rules"}) +_POLICY_KEYS = frozenset({"schema", "rules", "max_tool_calls"}) _RULE_KEYS = frozenset({"id", "effect", "tool_class", "argv_prefix"}) _ID_RE = re.compile(r"[a-z0-9._-]{1,64}") _CMD_RE = re.compile(r"[a-z0-9._-]{1,32}") @@ -139,6 +139,9 @@ def _validate_policy(policy: Any) -> None: policy.get("rules"), list ): raise ValueError("invalid policy schema") + max_tool_calls = policy.get("max_tool_calls") + if type(max_tool_calls) is not int or max_tool_calls < 1: + raise ValueError("invalid max tool calls") seen: set[str] = set() for rule in policy["rules"]: if not isinstance(rule, dict) or not set(rule) <= _RULE_KEYS: @@ -282,7 +285,9 @@ def _record( return {key: values[key] for key in _LOG_KEYS} -def _append_log(path: Path, record: dict[str, Any]) -> None: +def _append_log( + path: Path, record: dict[str, Any], max_tool_calls: int | None +) -> tuple[str, str]: import fcntl if path.is_symlink() or not path.parent.is_dir(): @@ -306,15 +311,27 @@ def _append_log(path: Path, record: dict[str, Any]) -> None: locked = True read_fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) maximum = 0 + pre_calls = 0 with os.fdopen(read_fd, "r", encoding="utf-8", errors="replace") as stream: read_fd = None for line in stream: try: value = json.loads(line) - if isinstance(value, dict) and isinstance(value.get("seq"), int): - maximum = max(maximum, value["seq"]) + if isinstance(value, dict): + if isinstance(value.get("seq"), int): + maximum = max(maximum, value["seq"]) + if value.get("event") == "pre": + pre_calls += 1 except (json.JSONDecodeError, TypeError): pass + if ( + record.get("event") == "pre" + and record.get("decision") != "error" + and max_tool_calls is not None + and pre_calls >= max_tool_calls + ): + record["decision"] = "deny" + record["rule_id"] = "cap.max_tool_calls" record["seq"] = maximum + 1 raw = ( json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n" @@ -322,6 +339,7 @@ def _append_log(path: Path, record: dict[str, Any]) -> None: while raw: raw = raw[os.write(fd, raw) :] os.fsync(fd) + return str(record["decision"]), str(record["rule_id"]) finally: if locked: fcntl.flock(fd, fcntl.LOCK_UN) @@ -350,6 +368,7 @@ def _run(payload: dict[str, Any], pointers: Path) -> int: event = _event_name(payload.get("hook_event_name")) log_path: Path | None = None policy_sha: str | None = None + policy: dict[str, Any] | None = None loaded = False try: if not _secure_file(pointer): @@ -371,7 +390,7 @@ def _run(payload: dict[str, Any], pointers: Path) -> int: if event == "pre" else ("allow", f"hook.{event}") ) - _append_log( + decision, rule = _append_log( log_path, _record( event, @@ -383,6 +402,7 @@ def _run(payload: dict[str, Any], pointers: Path) -> int: command, tool_class, ), + policy["max_tool_calls"], ) _emit(decision, rule) return 0 if decision == "allow" else 2 @@ -403,6 +423,7 @@ def _run(payload: dict[str, Any], pointers: Path) -> int: None, None, ), + None, ) except BaseException: # noqa: BLE001 log_path = None diff --git a/kits/agent-kiro-kit/tests/test_delivery_stage.py b/kits/agent-kiro-kit/tests/test_delivery_stage.py index 2993f4e..b675fe5 100644 --- a/kits/agent-kiro-kit/tests/test_delivery_stage.py +++ b/kits/agent-kiro-kit/tests/test_delivery_stage.py @@ -201,6 +201,7 @@ async def test_trust_all_hook_policy_counts( json.dumps( { "schema": "beddel.kiro-hook-policy/1", + "max_tool_calls": 100, "rules": [ { "id": "shell.git-push", diff --git a/kits/agent-kiro-kit/tests/test_hook.py b/kits/agent-kiro-kit/tests/test_hook.py index 91255df..48a193f 100644 --- a/kits/agent-kiro-kit/tests/test_hook.py +++ b/kits/agent-kiro-kit/tests/test_hook.py @@ -13,9 +13,13 @@ import pytest from beddel_agent_kiro import hook +_OMIT_CAP = object() + def _setup( - tmp_path: Path, rules: list[dict[str, Any]] | None = None + tmp_path: Path, + rules: list[dict[str, Any]] | None = None, + cap: Any = 100, ) -> tuple[Path, Path, Path]: pointers = tmp_path / "pointers" pointers.mkdir(parents=True, mode=0o700) @@ -28,6 +32,8 @@ def _setup( {"id": "read.any", "effect": "allow", "tool_class": "read"}, ], } + if cap is not _OMIT_CAP: + policy["max_tool_calls"] = cap policy_path.write_text(json.dumps(policy), encoding="utf-8") os.chmod(policy_path, 0o600) pointer = hook.pointer_path(pointers, str(tmp_path)) @@ -220,3 +226,58 @@ def test_fifo_paths_do_not_hang(tmp_path: Path, target: str) -> None: timeout=5, ) assert completed.returncode == 2 + + +def test_cap_max_tool_calls_denies_past_the_cap(tmp_path: Path) -> None: + rules = [ + { + "id": "shell.git-push", + "effect": "deny", + "tool_class": "shell", + "argv_prefix": ["git", "push"], + }, + {"id": "shell.any", "effect": "allow", "tool_class": "shell"}, + ] + pointers, _policy, log = _setup(tmp_path, rules, cap=2) + assert _call(pointers, _event(tmp_path, command="true"))[0] == 0 + assert _call(pointers, _event(tmp_path, command="true"))[0] == 0 + code, _out, err = _call(pointers, _event(tmp_path, command="git push origin x")) + assert code == 2 and "cap.max_tool_calls" in err + assert _call(pointers, _event(tmp_path, command="true", name="postToolUse"))[0] == 0 + assert _call(pointers, _event(tmp_path, name="stop"))[0] == 0 + records = [json.loads(line) for line in log.read_text().splitlines()] + assert records[2]["decision"] == "deny" + assert records[2]["rule_id"] == "cap.max_tool_calls" + + +@pytest.mark.parametrize("cap", [_OMIT_CAP, 0, -1, True, 1.5, "2", None]) +def test_cap_missing_or_invalid_policy_denies(tmp_path: Path, cap: Any) -> None: + pointers, _policy, log = _setup(tmp_path, cap=cap) + code, _out, err = _call(pointers, _event(tmp_path, command="true")) + assert code == 2 and "policy.invalid" in err + code, _out, err = _call(pointers, _event(tmp_path, command="true")) + assert code == 2 and "policy.invalid" in err + records = [json.loads(line) for line in log.read_text().splitlines()] + assert [record["rule_id"] for record in records] == [ + "policy.invalid", + "policy.invalid", + ] + + +def test_internal_error_past_the_cap_stays_an_error( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + pointers, _policy, log = _setup(tmp_path, cap=1) + assert _call(pointers, _event(tmp_path, command="true"))[0] == 0 + + def fail(*_args: Any) -> tuple[str, str]: + raise RuntimeError("test failure") + + monkeypatch.setattr(hook, "_evaluate", fail) + code, _out, err = _call(pointers, _event(tmp_path, command="true")) + assert code == 2 and "hook.internal" in err + records = [json.loads(line) for line in log.read_text().splitlines()] + assert records[-1]["decision"] == "error" + assert records[-1]["rule_id"] == "hook.internal" + summary = hook.summarize(pointers, tmp_path) + assert summary["errors"] == 1 and summary["denied"] == 0 diff --git a/kits/profile-dev-kit/README.md b/kits/profile-dev-kit/README.md index ffac19c..9182191 100644 --- a/kits/profile-dev-kit/README.md +++ b/kits/profile-dev-kit/README.md @@ -460,7 +460,10 @@ beddel run -i ... workflows/plan-story.yaml The plan, implement and verify-and-fix stages use tracked policies at `policies/plan.json`, `policies/implement.json` and -`policies/verify-and-fix.json`. Each policy denies shell `git push` and `gh pr +`policies/verify-and-fix.json`. Each policy declares a positive +`max_tool_calls` cap. The hook denies the call past the cap with +`cap.max_tool_calls`, counted in `hook_denied`, and overrides any rule +decision. Each policy denies shell `git push` and `gh pr merge` before allowing the other declared shell, read and write classes. The policy copy and `hook.jsonl` live in one fresh 0700 run directory under `~/.local/state/beddel/kiro-hook/runs/`; the policy digest is recorded in the @@ -574,3 +577,16 @@ once. Each run ends with a `deliver` record. One Change per run. beddel run -i ... workflows/deliver-story.yaml ``` +## Delivery: summary (Story 8.7) + +`read-delivery-summary` is a read-only entry workflow. Given `repo_path` and +`story_key`, it reads the existing Stage outcomes, Reports and Gate outcomes +and prints one row per stage plus totals. It reports wall seconds, agent calls, +denied calls, Review rounds and Gate exit codes. Missing records or fields are +reported as `missing`; Kiro credits and tokens are always `not observable`. +The workflow creates no records and does not change the records tree. + +```text +beddel run -i repo_path= -i story_key= workflows/read-delivery-summary.yaml +``` + diff --git a/kits/profile-dev-kit/kit.yaml b/kits/profile-dev-kit/kit.yaml index c0c4039..f5d0012 100644 --- a/kits/profile-dev-kit/kit.yaml +++ b/kits/profile-dev-kit/kit.yaml @@ -161,6 +161,10 @@ tools: target: "beddel_profile_dev.tools.deliver:profile_deliver_approval" description: "Check and settle the Delivery Approval stage" category: profile + - name: profile_read_delivery_summary + target: "beddel_profile_dev.tools.summary:profile_read_delivery_summary" + description: "Read a delivery cost summary from existing profile records" + category: profile workflows: - name: verify-change @@ -202,6 +206,9 @@ workflows: - name: deliver-story path: workflows/deliver-story.yaml description: Chains the Delivery stages or runs one selected stage, waits for Approval and records the complete Delivery outcome. + - name: read-delivery-summary + path: workflows/read-delivery-summary.yaml + description: Reads a delivery cost summary from existing profile records. targets: python: @@ -273,6 +280,8 @@ targets: target: "beddel_profile_dev.tools.deliver:profile_deliver_gate" - name: profile_deliver_approval target: "beddel_profile_dev.tools.deliver:profile_deliver_approval" + - name: profile_read_delivery_summary + target: "beddel_profile_dev.tools.summary:profile_read_delivery_summary" typescript: status: unavailable unavailable_reason: > diff --git a/kits/profile-dev-kit/policies/implement.json b/kits/profile-dev-kit/policies/implement.json index c6d27de..d50262e 100644 --- a/kits/profile-dev-kit/policies/implement.json +++ b/kits/profile-dev-kit/policies/implement.json @@ -1 +1 @@ -{"schema":"beddel.kiro-hook-policy/1","rules":[{"id":"shell.git-push","effect":"deny","tool_class":"shell","argv_prefix":["git","push"]},{"id":"shell.gh-pr-merge","effect":"deny","tool_class":"shell","argv_prefix":["gh","pr","merge"]},{"id":"shell.any","effect":"allow","tool_class":"shell"},{"id":"read.any","effect":"allow","tool_class":"read"},{"id":"write.any","effect":"allow","tool_class":"write"}]} +{"schema":"beddel.kiro-hook-policy/1","rules":[{"id":"shell.git-push","effect":"deny","tool_class":"shell","argv_prefix":["git","push"]},{"id":"shell.gh-pr-merge","effect":"deny","tool_class":"shell","argv_prefix":["gh","pr","merge"]},{"id":"shell.any","effect":"allow","tool_class":"shell"},{"id":"read.any","effect":"allow","tool_class":"read"},{"id":"write.any","effect":"allow","tool_class":"write"}],"max_tool_calls":400} diff --git a/kits/profile-dev-kit/policies/plan.json b/kits/profile-dev-kit/policies/plan.json index c6d27de..b409ca3 100644 --- a/kits/profile-dev-kit/policies/plan.json +++ b/kits/profile-dev-kit/policies/plan.json @@ -1 +1 @@ -{"schema":"beddel.kiro-hook-policy/1","rules":[{"id":"shell.git-push","effect":"deny","tool_class":"shell","argv_prefix":["git","push"]},{"id":"shell.gh-pr-merge","effect":"deny","tool_class":"shell","argv_prefix":["gh","pr","merge"]},{"id":"shell.any","effect":"allow","tool_class":"shell"},{"id":"read.any","effect":"allow","tool_class":"read"},{"id":"write.any","effect":"allow","tool_class":"write"}]} +{"schema":"beddel.kiro-hook-policy/1","rules":[{"id":"shell.git-push","effect":"deny","tool_class":"shell","argv_prefix":["git","push"]},{"id":"shell.gh-pr-merge","effect":"deny","tool_class":"shell","argv_prefix":["gh","pr","merge"]},{"id":"shell.any","effect":"allow","tool_class":"shell"},{"id":"read.any","effect":"allow","tool_class":"read"},{"id":"write.any","effect":"allow","tool_class":"write"}],"max_tool_calls":200} diff --git a/kits/profile-dev-kit/policies/verify-and-fix.json b/kits/profile-dev-kit/policies/verify-and-fix.json index c6d27de..d50262e 100644 --- a/kits/profile-dev-kit/policies/verify-and-fix.json +++ b/kits/profile-dev-kit/policies/verify-and-fix.json @@ -1 +1 @@ -{"schema":"beddel.kiro-hook-policy/1","rules":[{"id":"shell.git-push","effect":"deny","tool_class":"shell","argv_prefix":["git","push"]},{"id":"shell.gh-pr-merge","effect":"deny","tool_class":"shell","argv_prefix":["gh","pr","merge"]},{"id":"shell.any","effect":"allow","tool_class":"shell"},{"id":"read.any","effect":"allow","tool_class":"read"},{"id":"write.any","effect":"allow","tool_class":"write"}]} +{"schema":"beddel.kiro-hook-policy/1","rules":[{"id":"shell.git-push","effect":"deny","tool_class":"shell","argv_prefix":["git","push"]},{"id":"shell.gh-pr-merge","effect":"deny","tool_class":"shell","argv_prefix":["gh","pr","merge"]},{"id":"shell.any","effect":"allow","tool_class":"shell"},{"id":"read.any","effect":"allow","tool_class":"read"},{"id":"write.any","effect":"allow","tool_class":"write"}],"max_tool_calls":400} diff --git a/kits/profile-dev-kit/python/beddel_profile_dev/tools/summary.py b/kits/profile-dev-kit/python/beddel_profile_dev/tools/summary.py new file mode 100644 index 0000000..289ff0f --- /dev/null +++ b/kits/profile-dev-kit/python/beddel_profile_dev/tools/summary.py @@ -0,0 +1,349 @@ +"""Read-only delivery cost summary assembled from persisted profile records.""" + +from __future__ import annotations + +import json +import re +from collections.abc import Mapping, Sequence +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from beddel.tools import beddel_tool + +from beddel_profile_dev._errors import PROFILE_REPOSITORY_INVALID, ProfileError +from beddel_profile_dev._io import confined, read_confined, real_repo_root, records_root +from beddel_profile_dev.config import load_config +from beddel_profile_dev.deliver import STAGES +from beddel_profile_dev.inventory import is_valid_id +from beddel_profile_dev.tools.deliver import _records +from beddel_profile_dev.tools.implement import _valid_story +from beddel_profile_dev.tools.ratify import TIMESTAMP_FORMAT +from beddel_profile_dev.tools.report import _validate_report_content + +__all__ = ["profile_read_delivery_summary"] + +_SUMMARY_STAGES = (*STAGES, "deliver") +_AGENT_STAGES = {"plan", "implement", "verify-and-fix"} +_GATE_SCHEMA = "profile-dev/attempt-gate-outcome/1" +_FULL_SHA = re.compile(r"(?:[0-9a-f]{40}|[0-9a-f]{64})") +_DIGEST = re.compile(r"[0-9a-f]{64}") +_REPORT_ORDINAL = re.compile(r"([1-9][0-9]*)\.json") +_MISSING = "missing" +_NOT_OBSERVABLE = "not observable" + + +def _invalid() -> ProfileError: + return ProfileError(PROFILE_REPOSITORY_INVALID, "repository or revision invalid") + + +def _timestamp(value: object) -> datetime: + if not isinstance(value, str): + raise _invalid() + try: + return datetime.strptime(value, TIMESTAMP_FORMAT).replace(tzinfo=UTC) + except ValueError: + raise _invalid() from None + + +def _report_records(root: Path, change_id: str) -> list[tuple[int, dict[str, Any]]]: + relative = f"changes/{change_id}/report" + directory = confined(root, relative) + if not directory.exists(): + return [] + if not directory.is_dir(): + raise _invalid() + found: list[tuple[int, dict[str, Any]]] = [] + for entry in directory.iterdir(): + match = _REPORT_ORDINAL.fullmatch(entry.name) + if match is None: + continue + ordinal = int(match.group(1)) + raw = read_confined(root, f"{relative}/{entry.name}") + try: + content = json.loads(raw) if raw is not None else None + except (TypeError, ValueError, RecursionError): + content = None + found.append((ordinal, _validate_report_content(content, ordinal, change_id))) + return sorted(found) + + +def _gate_codes( + root: Path, change_id: str, report: Mapping[str, Any] +) -> dict[str, Any] | None: + revision, attempt = report.get("revision"), report.get("attempt") + if not isinstance(revision, str) or _FULL_SHA.fullmatch(revision) is None: + raise _invalid() + if attempt is None: + return None + if type(attempt) is not int or attempt < 1: + raise _invalid() + relative = f"changes/{change_id}/revisions/{revision}/attempts/{attempt}/gate.json" + gate_path = confined(root, relative) + if not gate_path.exists(): + if gate_path.is_symlink(): + raise _invalid() + return None + if not gate_path.is_file(): + raise _invalid() + raw = read_confined(root, relative) + if raw is None: + raise _invalid() + try: + data = json.loads(raw) + except (TypeError, ValueError, RecursionError): + raise _invalid() from None + expected = { + "schema", + "attempt", + "output_attempt", + "change_id", + "revision", + "inventory_version", + "inventory_sha256", + "status", + "commands", + "gate_ref", + } + if not isinstance(data, dict) or set(data) != expected: + raise _invalid() + output_attempt = data.get("output_attempt") + if ( + data.get("schema") != _GATE_SCHEMA + or type(data.get("attempt")) is not int + or data["attempt"] != attempt + or data.get("change_id") != change_id + or not isinstance(revision, str) + or data.get("revision") != revision + or _FULL_SHA.fullmatch(revision) is None + or type(output_attempt) is not int + or output_attempt < 1 + or data.get("status") != "completed" + or not is_valid_id(data.get("inventory_version")) + or not isinstance(data.get("inventory_sha256"), str) + or _DIGEST.fullmatch(data["inventory_sha256"]) is None + or data.get("gate_ref") + != f"changes/{change_id}/revisions/{revision}/gate-output/{output_attempt}" + ): + raise _invalid() + commands = data.get("commands") + if not isinstance(commands, list) or not commands: + raise _invalid() + values: dict[str, Any] = {} + for command in commands: + if ( + not isinstance(command, dict) + or set(command) != {"name", "argv", "exit_code", "timed_out"} + or not isinstance(command.get("name"), str) + or command["name"] in values + or not isinstance(command.get("argv"), list) + or not all(isinstance(arg, str) for arg in command["argv"]) + or type(command.get("timed_out")) is not bool + ): + raise _invalid() + exit_code = command["exit_code"] + if command["timed_out"]: + values[command["name"]] = "timeout" + elif type(exit_code) is int: + values[command["name"]] = exit_code + else: + raise _invalid() + return values + + +def _wall_seconds( + record: Mapping[str, Any], runs: list[tuple[int, int, list[Mapping[str, Any]]]] +) -> int | str: + ordinal = record.get("record") + if type(ordinal) is not int: + return _MISSING + for first, last, run_records in runs: + if first <= ordinal <= last: + position = next( + ( + index + for index, item in enumerate(run_records) + if item.get("record") == ordinal + ), + None, + ) + if position is None or position == 0: + return _MISSING + previous = _timestamp(run_records[position - 1].get("recorded_at")) + current = _timestamp(record.get("recorded_at")) + seconds = (current - previous).total_seconds() + return int(seconds) if seconds >= 0 and seconds.is_integer() else _MISSING + return _MISSING + + +def _agent_counts( + stage: str, records: list[Mapping[str, Any]] +) -> tuple[int | str, int | str]: + if stage not in _AGENT_STAGES: + return 0, 0 + totals = [0, 0] + for record in records: + refs = record.get("refs") + reason = record.get("reason") + if not isinstance(refs, Mapping): + return _MISSING, _MISSING + for index, key in enumerate(("hook_stop", "hook_denied")): + if key not in refs: + return _MISSING, _MISSING + value = refs[key] + if value is None: + if reason == "hooks.missing": + return _MISSING, _MISSING + continue + if type(value) is not int or value < 0: + return _MISSING, _MISSING + totals[index] += value + return totals[0], totals[1] + + +def _review_data( + root: Path, records: list[Mapping[str, Any]] +) -> tuple[int | str, list[dict[str, Any]] | str]: + change_ids: set[str] = set() + for record in records: + refs = record.get("refs") + value = refs.get("change_id") if isinstance(refs, Mapping) else None + if not isinstance(value, str) or not is_valid_id(value): + return _MISSING, _MISSING + change_ids.add(value) + if not change_ids: + return 0, [] + report_values: list[tuple[str, int, dict[str, Any]]] = [] + for change_id in sorted(change_ids): + found = _report_records(root, change_id) + if not found: + return _MISSING, _MISSING + report_values.extend((change_id, ordinal, report) for ordinal, report in found) + reports: list[dict[str, Any]] = [] + for change_id, ordinal, report in report_values: + codes = _gate_codes(root, change_id, report) + if codes is None: + return len(report_values), _MISSING + reports.append({"change_id": change_id, "report": ordinal, "commands": codes}) + reports.sort(key=lambda item: (item["change_id"], item["report"])) + return len(reports), reports + + +def _row( + stage: str, + records: list[Mapping[str, Any]], + root: Path, + runs: list[tuple[int, int, list[Mapping[str, Any]]]], +) -> dict[str, Any]: + if not records: + values: dict[str, Any] = { + "wall_seconds": _MISSING, + "agent_calls": _MISSING, + "denied_calls": _MISSING, + "review_rounds": _MISSING, + "gate_exit_codes": _MISSING, + } + else: + walls = [_wall_seconds(record, runs) for record in records] + wall = ( + sum(value for value in walls if isinstance(value, int)) + if all(isinstance(value, int) for value in walls) + else _MISSING + ) + calls, denied = _agent_counts(stage, records) + review_rounds, gates = ( + _review_data(root, records) if stage == "verify-and-fix" else (0, []) + ) + values = { + "wall_seconds": wall, + "agent_calls": calls, + "denied_calls": denied, + "review_rounds": review_rounds, + "gate_exit_codes": gates, + } + return { + "stage": stage, + "records": [record["record"] for record in records], + **values, + "kiro_credits": _NOT_OBSERVABLE, + "tokens": _NOT_OBSERVABLE, + } + + +def _total(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + fields = ("wall_seconds", "agent_calls", "denied_calls", "review_rounds") + result: dict[str, Any] = {} + for field in fields: + values = [row[field] for row in rows] + result[field] = ( + sum(values) if all(type(value) is int for value in values) else _MISSING + ) + gates = [row["gate_exit_codes"] for row in rows] + result["gate_exit_codes"] = ( + [item for value in gates if isinstance(value, list) for item in value] + if all(isinstance(value, list) for value in gates) + else _MISSING + ) + result["kiro_credits"] = _NOT_OBSERVABLE + result["tokens"] = _NOT_OBSERVABLE + return result + + +@beddel_tool( + name="profile_read_delivery_summary", + description="Read a delivery cost summary from existing profile records", + category="profile", +) +def profile_read_delivery_summary(repo_path: str, story_key: str) -> dict[str, Any]: + """Read delivery records without creating or modifying any record.""" + if not _valid_story(story_key): + raise ProfileError(PROFILE_REPOSITORY_INVALID, "story key invalid") + repo = real_repo_root(repo_path) + root = records_root(repo, load_config().records_dir) + records = [ + record + for record in _records(root) + if record.get("story_key") == story_key + and record.get("stage") in _SUMMARY_STAGES + ] + records.sort(key=lambda record: record["record"]) + runs: list[tuple[int, int, list[Mapping[str, Any]]]] = [] + for deliver in (record for record in records if record.get("stage") == "deliver"): + refs = deliver.get("refs") + first = refs.get("first_record") if isinstance(refs, Mapping) else None + last = deliver.get("record") + if type(first) is int and type(last) is int and first <= last: + runs.append( + ( + first, + last, + [record for record in records if first <= record["record"] <= last], + ) + ) + rows = [ + _row( + stage, + [record for record in records if record.get("stage") == stage], + root, + runs, + ) + for stage in _SUMMARY_STAGES + ] + missing = [ + f"{row['stage']}.{field}" + for row in rows + for field in ( + "wall_seconds", + "agent_calls", + "denied_calls", + "review_rounds", + "gate_exit_codes", + ) + if row[field] == _MISSING + ] + return { + "story_key": story_key, + "stages": rows, + "total": _total(rows), + "missing": missing, + } diff --git a/kits/profile-dev-kit/tests/test_delivery_summary.py b/kits/profile-dev-kit/tests/test_delivery_summary.py new file mode 100644 index 0000000..893bee4 --- /dev/null +++ b/kits/profile-dev-kit/tests/test_delivery_summary.py @@ -0,0 +1,399 @@ +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +from pathlib import Path + +import pytest +from beddel_profile_dev._errors import PROFILE_REPOSITORY_INVALID, ProfileError +from beddel_profile_dev._io import canonical_json +from beddel_profile_dev.tools.stage_outcome import profile_record_stage_outcome +from beddel_profile_dev.tools.summary import profile_read_delivery_summary + +from ._helpers import commit_all, init_repo, records_root_for +from .test_run_cli import _beddel_argv + +KIT_ROOT = Path(__file__).resolve().parent.parent +WORKFLOW = KIT_ROOT / "workflows" / "read-delivery-summary.yaml" +STORY = "8-7-summary-fixture" + + +def _repo(tmp_path: Path) -> Path: + repo = init_repo(tmp_path / "repo") + (repo / "story.md").write_text("story\n") + commit_all(repo) + return repo + + +def _record( + repo: Path, + stage: str, + refs: dict[str, object], + *, + reason: str = "stage.complete", + recorded_at: str | None = None, +) -> None: + result = profile_record_stage_outcome( + str(repo), STORY, stage, "succeeded", reason, refs + ) + if recorded_at is not None: + path = records_root_for(repo) / result["path"] + data = json.loads(path.read_text()) + data["recorded_at"] = recorded_at + path.write_bytes(canonical_json(data)) + + +def _complete(repo: Path) -> Path: + hooks = {"hook_stop": 2, "hook_denied": 1} + _record( + repo, + "plan", + {**hooks}, + recorded_at="2026-01-01T00:00:10Z", + ) + _record( + repo, + "implement", + {"hook_stop": 3, "hook_denied": 0}, + recorded_at="2026-01-01T00:00:20Z", + ) + _record( + repo, + "verify-and-fix", + {"hook_stop": 1, "hook_denied": 0, "change_id": "c1"}, + recorded_at="2026-01-01T00:00:30Z", + ) + _record(repo, "approval", {}, recorded_at="2026-01-01T00:00:40Z") + _record(repo, "integrate", {}, recorded_at="2026-01-01T00:00:50Z") + _record( + repo, + "deliver", + {"first_record": 1}, + recorded_at="2026-01-01T00:01:20Z", + ) + root = records_root_for(repo) + revision = "a" * 40 + report = { + "schema": "profile-dev/verification-report/1", + "record": 1, + "change_id": "c1", + "revision": revision, + "attempt": 1, + } + report_path = root / "changes" / "c1" / "report" / "1.json" + report_path.parent.mkdir(parents=True, exist_ok=True) + report_path.write_bytes(canonical_json(report)) + gate = { + "schema": "profile-dev/attempt-gate-outcome/1", + "attempt": 1, + "output_attempt": 1, + "change_id": "c1", + "revision": revision, + "inventory_version": "v1", + "inventory_sha256": "b" * 64, + "status": "completed", + "commands": [ + { + "name": "01-ruff", + "argv": ["ruff", "check", "."], + "exit_code": 0, + "timed_out": False, + }, + { + "name": "02-pytest", + "argv": ["pytest", "-q"], + "exit_code": None, + "timed_out": True, + }, + ], + "gate_ref": "changes/c1/revisions/" + revision + "/gate-output/1", + } + gate_path = ( + root + / "changes" + / "c1" + / "revisions" + / revision + / "attempts" + / "1" + / "gate.json" + ) + gate_path.parent.mkdir(parents=True, exist_ok=True) + gate_path.write_bytes(canonical_json(gate)) + return root + + +def _tree_digest(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(root.rglob("*")): + relative = path.relative_to(root).as_posix().encode() + digest.update(relative) + if path.is_dir(): + digest.update(b"/") + elif path.is_file(): + digest.update(path.read_bytes()) + return digest.hexdigest() + + +def test_complete_records_summary_has_values_and_totals(tmp_path: Path) -> None: + repo = _repo(tmp_path) + _complete(repo) + summary = profile_read_delivery_summary(str(repo), STORY) + rows = {row["stage"]: row for row in summary["stages"]} + assert { + stage: { + "records": row["records"], + "wall_seconds": row["wall_seconds"], + "agent_calls": row["agent_calls"], + "denied_calls": row["denied_calls"], + "review_rounds": row["review_rounds"], + "gate_exit_codes": row["gate_exit_codes"], + } + for stage, row in rows.items() + } == { + "plan": { + "records": [1], + "wall_seconds": "missing", + "agent_calls": 2, + "denied_calls": 1, + "review_rounds": 0, + "gate_exit_codes": [], + }, + "implement": { + "records": [2], + "wall_seconds": 10, + "agent_calls": 3, + "denied_calls": 0, + "review_rounds": 0, + "gate_exit_codes": [], + }, + "verify-and-fix": { + "records": [3], + "wall_seconds": 10, + "agent_calls": 1, + "denied_calls": 0, + "review_rounds": 1, + "gate_exit_codes": [ + { + "change_id": "c1", + "report": 1, + "commands": {"01-ruff": 0, "02-pytest": "timeout"}, + } + ], + }, + "approval": { + "records": [4], + "wall_seconds": 10, + "agent_calls": 0, + "denied_calls": 0, + "review_rounds": 0, + "gate_exit_codes": [], + }, + "integrate": { + "records": [5], + "wall_seconds": 10, + "agent_calls": 0, + "denied_calls": 0, + "review_rounds": 0, + "gate_exit_codes": [], + }, + "deliver": { + "records": [6], + "wall_seconds": 30, + "agent_calls": 0, + "denied_calls": 0, + "review_rounds": 0, + "gate_exit_codes": [], + }, + } + assert all( + row["kiro_credits"] == "not observable" and row["tokens"] == "not observable" + for row in summary["stages"] + ) + assert summary["total"] == { + "wall_seconds": "missing", + "agent_calls": 6, + "denied_calls": 1, + "review_rounds": 1, + "gate_exit_codes": [ + { + "change_id": "c1", + "report": 1, + "commands": {"01-ruff": 0, "02-pytest": "timeout"}, + } + ], + "kiro_credits": "not observable", + "tokens": "not observable", + } + assert summary["missing"] == ["plan.wall_seconds"] + + +def test_missing_records_and_fields_are_reported(tmp_path: Path) -> None: + repo = _repo(tmp_path) + _record(repo, "plan", {"hook_stop": 1, "hook_denied": 0}) + _record(repo, "verify-and-fix", {}) + _record(repo, "deliver", {"first_record": 1}) + summary = profile_read_delivery_summary(str(repo), STORY) + implement = next(row for row in summary["stages"] if row["stage"] == "implement") + verify = next(row for row in summary["stages"] if row["stage"] == "verify-and-fix") + assert all( + implement[field] == "missing" + for field in ( + "wall_seconds", + "agent_calls", + "denied_calls", + "review_rounds", + "gate_exit_codes", + ) + ) + assert verify["agent_calls"] == "missing" + assert verify["review_rounds"] == "missing" + assert verify["gate_exit_codes"] == "missing" + assert "implement.wall_seconds" in summary["missing"] + assert summary["total"]["agent_calls"] == "missing" + assert summary["stages"][0]["kiro_credits"] == "not observable" + + +def test_cli_summary_does_not_change_records(tmp_path: Path, monkeypatch) -> None: + home = tmp_path / "home" + monkeypatch.setenv("HOME", str(home)) + repo = _repo(tmp_path) + root = _complete(repo) + before = _tree_digest(root) + config_dir = home / ".config" / "beddel" + config_dir.mkdir(parents=True, exist_ok=True) + (config_dir / "config.json").write_text( + json.dumps({"kits_paths": [str(KIT_ROOT.parent)]}) + ) + env = { + key: value + for key, value in os.environ.items() + if not key.startswith(("GIT_", "BEDDEL_", "PYTHON")) and key != "KIRO_API_KEY" + } + env.update({"HOME": str(home), "XDG_CONFIG_HOME": str(home / ".config")}) + result = subprocess.run( + [ + *_beddel_argv(), + "run", + "--json-output", + "-i", + f"repo_path={repo}", + "-i", + f"story_key={STORY}", + str(WORKFLOW), + ], + capture_output=True, + text=True, + cwd=tmp_path, + env=env, + timeout=60, + check=False, + ) + assert result.returncode == 0, result.stderr + output = json.loads(result.stdout) + summary = output["summary"]["result"] + rows = {row["stage"]: row for row in summary["stages"]} + assert rows["deliver"]["wall_seconds"] == 30 + assert rows["verify-and-fix"]["review_rounds"] == 1 + assert summary["total"]["gate_exit_codes"][0]["change_id"] == "c1" + assert summary["total"]["gate_exit_codes"][0]["commands"]["01-ruff"] == 0 + assert _tree_digest(root) == before + + +def test_hook_null_reason_controls_missing_vs_zero(tmp_path: Path) -> None: + repo = _repo(tmp_path) + _record( + repo, "plan", {"hook_stop": None, "hook_denied": None}, reason="hooks.missing" + ) + _record(repo, "implement", {"hook_stop": None, "hook_denied": None}) + summary = profile_read_delivery_summary(str(repo), STORY) + rows = {row["stage"]: row for row in summary["stages"]} + assert rows["plan"]["agent_calls"] == "missing" + assert rows["plan"]["denied_calls"] == "missing" + assert rows["implement"]["agent_calls"] == 0 + assert rows["implement"]["denied_calls"] == 0 + + +def test_absent_gate_with_report_is_missing_but_report_is_counted( + tmp_path: Path, +) -> None: + repo = _repo(tmp_path) + root = _complete(repo) + gate_path = ( + root + / "changes" + / "c1" + / "revisions" + / ("a" * 40) + / "attempts" + / "1" + / "gate.json" + ) + gate_path.unlink() + summary = profile_read_delivery_summary(str(repo), STORY) + verify = next(row for row in summary["stages"] if row["stage"] == "verify-and-fix") + assert verify["review_rounds"] == 1 + assert verify["gate_exit_codes"] == "missing" + assert summary["total"]["review_rounds"] == 1 + assert summary["total"]["gate_exit_codes"] == "missing" + + +def test_record_outside_closed_run_has_missing_wall_time(tmp_path: Path) -> None: + repo = _repo(tmp_path) + _record( + repo, + "plan", + {"hook_stop": 1, "hook_denied": 0}, + recorded_at="2026-01-01T00:00:10Z", + ) + _record(repo, "deliver", {"first_record": 1}, recorded_at="2026-01-01T00:00:20Z") + _record( + repo, + "plan", + {"hook_stop": 1, "hook_denied": 0}, + recorded_at="2026-01-01T00:00:30Z", + ) + summary = profile_read_delivery_summary(str(repo), STORY) + plan = next(row for row in summary["stages"] if row["stage"] == "plan") + assert plan["records"] == [1, 3] + assert plan["wall_seconds"] == "missing" + + +def test_malformed_gate_outcome_fails_closed_without_writing( + tmp_path: Path, +) -> None: + repo = _repo(tmp_path) + root = _complete(repo) + gate_path = ( + root + / "changes" + / "c1" + / "revisions" + / ("a" * 40) + / "attempts" + / "1" + / "gate.json" + ) + gate_path.write_text("{}") + before = _tree_digest(root) + with pytest.raises(ProfileError) as error: + profile_read_delivery_summary(str(repo), STORY) + assert error.value.code == PROFILE_REPOSITORY_INVALID + assert _tree_digest(root) == before + + +def test_malformed_stage_outcome_fails_closed_without_writing( + tmp_path: Path, +) -> None: + repo = _repo(tmp_path) + root = _complete(repo) + stage_files = sorted((root / "stage-outcomes").rglob("*.json")) + assert stage_files + stage_files[0].write_text("{") + before = _tree_digest(root) + with pytest.raises(ProfileError) as error: + profile_read_delivery_summary(str(repo), STORY) + assert error.value.code == PROFILE_REPOSITORY_INVALID + assert _tree_digest(root) == before diff --git a/kits/profile-dev-kit/tests/test_hook_proof.py b/kits/profile-dev-kit/tests/test_hook_proof.py index e731914..c0a2ad4 100644 --- a/kits/profile-dev-kit/tests/test_hook_proof.py +++ b/kits/profile-dev-kit/tests/test_hook_proof.py @@ -1,6 +1,7 @@ from __future__ import annotations import hashlib +import importlib.util import json import os import stat @@ -90,11 +91,12 @@ def test_run_pointer_lifecycle_and_foreign_pointer( ) proof = profile_hook_proof("implement", first, [armed], _settled()) assert (proof["outcome"], proof["refs"]["hook_denied"]) == ("succeeded", 2) - assert ( - policy_path("plan").read_bytes() - == policy_path("implement").read_bytes() - == policy_path("verify-and-fix").read_bytes() - ) + policies = [ + json.loads(policy_path(stage).read_text()) + for stage in ("plan", "implement", "verify-and-fix") + ] + assert policies[0]["rules"] == policies[1]["rules"] == policies[2]["rules"] + assert [policy["max_tool_calls"] for policy in policies] == [200, 400, 400] assert all( x in Path(first["log_path"]).read_text() for x in ("shell.git-push", "shell.gh-pr-merge", "shell.any") @@ -184,3 +186,41 @@ def test_digest_sessions_cleanup_and_record_boundary( refused = profile_hook_arm(str(repo), "implement", opened) monkeypatch.setattr(os, "fsync", original) assert not refused["armed"] and not pointer_path(repo).exists() + + +def test_tracked_policies_declare_cap_and_cap_denials_count( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + hook_path = ( + Path(__file__).resolve().parents[2] + / "agent-kiro-kit" + / "python" + / "beddel_agent_kiro" + / "hook.py" + ) + spec = importlib.util.spec_from_file_location("hook_cap_test", hook_path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + for stage in ("plan", "implement", "verify-and-fix"): + data = json.loads(policy_path(stage).read_text()) + assert type(data["max_tool_calls"]) is int and data["max_tool_calls"] > 0 + module._validate_policy(data) + + cap_policy = tmp_path / "cap-2.json" + policy = json.loads(policy_path("implement").read_text()) + policy["max_tool_calls"] = 2 + cap_policy.write_text(json.dumps(policy)) + cap_policy.chmod(0o600) + import beddel_profile_dev.hook_proof as proof_module + + monkeypatch.setattr(proof_module, "policy_path", lambda _stage: cap_policy) + repo = _repo(tmp_path) + monkeypatch.chdir(repo) + opened, armed = _open_arm(repo) + emit_hook_events(repo, ["pre", "pre", "pre", "stop"]) + profile_hook_disarm(str(repo), armed) + proof = profile_hook_proof("implement", opened, [armed], _settled()) + assert proof["outcome"] == "succeeded" + assert proof["refs"]["hook_denied"] == 1 + assert set(proof["refs"]) == set({**_settled()["refs"], **null_keys()}) diff --git a/kits/profile-dev-kit/tests/test_manifest.py b/kits/profile-dev-kit/tests/test_manifest.py index f34ec20..2b32f01 100644 --- a/kits/profile-dev-kit/tests/test_manifest.py +++ b/kits/profile-dev-kit/tests/test_manifest.py @@ -46,6 +46,7 @@ "profile_integrate_settle", "profile_deliver_gate", "profile_deliver_approval", + "profile_read_delivery_summary", ] @@ -124,6 +125,7 @@ def test_manifest_declares_all_workflows_and_python_targets() -> None: "plan-story", "integrate-story", "deliver-story", + "read-delivery-summary", ] for tool in manifest["tools"]: assert tool["category"] == "profile" diff --git a/kits/profile-dev-kit/tests/test_workflow.py b/kits/profile-dev-kit/tests/test_workflow.py index f2cff98..47a6acb 100644 --- a/kits/profile-dev-kit/tests/test_workflow.py +++ b/kits/profile-dev-kit/tests/test_workflow.py @@ -113,7 +113,7 @@ def test_allowlist_equals_the_tools_the_steps_name() -> None: "profile_capture_evidence", "profile_record_report", ] - assert len(list(WORKFLOWS_DIR.glob("*.yaml"))) == 13 + assert len(list(WORKFLOWS_DIR.glob("*.yaml"))) == 14 def test_review_step_outlives_the_largest_accepted_review_budget() -> None: diff --git a/kits/profile-dev-kit/workflows/read-delivery-summary.yaml b/kits/profile-dev-kit/workflows/read-delivery-summary.yaml new file mode 100644 index 0000000..b86d412 --- /dev/null +++ b/kits/profile-dev-kit/workflows/read-delivery-summary.yaml @@ -0,0 +1,19 @@ +id: read-delivery-summary +name: Read Delivery Summary +description: Reads a delivery cost summary from existing Stage outcomes, Reports and Gate outcomes without running a stage or writing records. +version: "1.0" +requires_kits: + - profile-dev-kit +allowed_tools: + - profile_read_delivery_summary +inputs: + - repo_path + - story_key +steps: + - id: summary + primitive: tool + config: + tool: profile_read_delivery_summary + arguments: + repo_path: "$input.repo_path" + story_key: "$input.story_key"