diff --git a/kits/profile-dev-kit/README.md b/kits/profile-dev-kit/README.md index d48414b..688aca9 100644 --- a/kits/profile-dev-kit/README.md +++ b/kits/profile-dev-kit/README.md @@ -468,3 +468,28 @@ A baseline-red Gate is advisory; only a green command that becomes red blocks. ```text beddel run -i ... workflows/implement-story.yaml ``` + +## Delivery: verify-and-fix stage + +`verify-and-fix-story` takes `repo_path`, `story_key`, `spec_path`, `branch`, +`base_ref`, `change_id` and `inventory_version`. It invokes `verify-change.yaml` unchanged through the core nested +`call-agent` primitive, reads its Report through `profile_read_report`, and +records one Stage outcome. The stage never writes a Change, binding, Impact, +VerificationAttempt, Gate, Review, Evidence, Report or Verdict. + +A `consistent` Report succeeds. A `has pendency` Report starts a fix round only +when it contains a `finding_blocking` or `finding_unclassified` Pendency. The +Coding agent receives only those recorded Findings and the union of the spec +unit paths. Kit checks run again before `profile_fix_settle` creates one commit +with `Story-Key` and `Fix-Round` trailers. At most two rounds invoke `verify-change` at the new commit so it opens a fresh +VerificationAttempt. Earlier Evidence is never copied. Other Pendencies stop with `pendency.no_fixes`; the bound stops with +`fix.bound_reached`. Stage refs contain identifiers, digests, counts and paths +only. + +Owner-ratified Review classes: +Blind Hunter, Edge Case Hunter, Verification Gap and Acceptance Auditor. Agents +never ratify or write inventory versions. + +```text +beddel run -i ... workflows/verify-and-fix-story.yaml +``` diff --git a/kits/profile-dev-kit/kit.yaml b/kits/profile-dev-kit/kit.yaml index 263ddf3..bb6a16c 100644 --- a/kits/profile-dev-kit/kit.yaml +++ b/kits/profile-dev-kit/kit.yaml @@ -117,6 +117,14 @@ tools: target: "beddel_profile_dev.tools.implement:profile_implement_settle" description: "Review and commit one implementation unit" category: profile + - name: profile_fix_reconcile + target: "beddel_profile_dev.tools.fix:profile_fix_reconcile" + description: "Reconcile one bounded verification fix round" + category: profile + - name: profile_fix_settle + target: "beddel_profile_dev.tools.fix:profile_fix_settle" + description: "Validate and commit one verification fix round" + category: profile workflows: - name: verify-change @@ -146,6 +154,9 @@ workflows: - name: implement-story path: workflows/implement-story.yaml description: Implements one unit of one spec through a Coding agent and commits only its declared paths; never pushes, verifies or integrates. + - name: verify-and-fix-story + path: workflows/verify-and-fix-story.yaml + description: Runs verify-change unchanged and applies at most two bounded fix rounds with fresh verification attempts; never approves, ratifies, pushes or integrates. - name: plan-story path: workflows/plan-story.yaml description: Plans one selected story into one implementation spec through a Coding agent; never implements, verifies or commits. @@ -198,6 +209,10 @@ targets: target: "beddel_profile_dev.tools.implement:profile_implement_prepare" - name: profile_implement_settle target: "beddel_profile_dev.tools.implement:profile_implement_settle" + - name: profile_fix_reconcile + target: "beddel_profile_dev.tools.fix:profile_fix_reconcile" + - name: profile_fix_settle + target: "beddel_profile_dev.tools.fix:profile_fix_settle" typescript: status: unavailable unavailable_reason: > diff --git a/kits/profile-dev-kit/python/beddel_profile_dev/fix.py b/kits/profile-dev-kit/python/beddel_profile_dev/fix.py new file mode 100644 index 0000000..4ff8a1b --- /dev/null +++ b/kits/profile-dev-kit/python/beddel_profile_dev/fix.py @@ -0,0 +1,83 @@ +"""Pure fix contracts.""" + +# fmt: off +from __future__ import annotations + +import re +from collections.abc import Iterable, Mapping +from typing import Any + +from beddel_profile_dev.implement import STORY_TRAILER, Unit + +FIX_TRAILER = "Fix-Round" +MAX_FIX_ROUNDS = 2 +MAX_FINDINGS_BYTES = 32768 +FIXABLE_REASONS = frozenset({"finding_blocking", "finding_unclassified"}) +_ROUND = re.compile(r"[1-2]") + + +def fix_message(story_key: str, round_: int) -> tuple[str, str]: + """Return the subject and trailers for a fix commit.""" + return (f"fix: apply verification fixes (round {round_})", f"{STORY_TRAILER}: {story_key}\n{FIX_TRAILER}: {round_}") + + +def trailer_round(message: str, story_key: str) -> int | None: + """Return the valid fix-round trailer in a commit message.""" + if not isinstance(message, str): + return None + lines = message.splitlines() + story = [line for line in lines if line == f"{STORY_TRAILER}: {story_key}"] + rounds = [line for line in lines if line.startswith(f"{FIX_TRAILER}: ")] + if len(story) != 1 or len(rounds) != 1: + return None + value = rounds[0][len(f"{FIX_TRAILER}: ") :] + return int(value) if _ROUND.fullmatch(value) else None + + +def report_at(history: object, head: str) -> dict[str, Any] | None: + """Return the last report recorded at ``head``.""" + selected: dict[str, Any] | None = None + if not isinstance(history, Iterable) or isinstance(history, (str, bytes)): + return None + for entry in history: + content = entry.get("content") if isinstance(entry, Mapping) else None + if isinstance(content, Mapping) and content.get("revision") == head: + selected = dict(content) + return selected + + +def fix_findings(report: object) -> list[dict[str, Any]]: + """Return Findings named by fixable Pendencies in index order.""" + if not isinstance(report, Mapping) or not isinstance(report.get("items"), Mapping): + return [] + items = report["items"] + divergences, pendencies = items.get("divergences"), items.get("pendencies") + if not isinstance(divergences, Mapping) or not isinstance(pendencies, Mapping): + return [] + values, pending = divergences.get("value"), pendencies.get("value") + if divergences.get("status") != "present" or pendencies.get("status") != "present" or not isinstance(values, list) or not isinstance(pending, list): + return [] + indexes = {item["finding_index"] for item in pending if isinstance(item, Mapping) and item.get("reason") in FIXABLE_REASONS and type(item.get("finding_index")) is int and item["finding_index"] >= 0} + keys = ("source", "association", "class", "affected_point", "criterion", "kind", "between", "detail") + return [{key: values[index].get(key) for key in keys} for index in sorted(indexes) if index < len(values) and isinstance(values[index], Mapping)] + + +def decision(report: object, rounds: int, max_rounds: int) -> tuple[str | None, str | None]: + """Return the stage decision for one Report.""" + if not isinstance(report, Mapping): + return "failed", "report.unreadable" + verdict = report.get("verdict") + if verdict == "consistent": + return "succeeded", "verdict.consistent" + if verdict != "has pendency": + return "failed", "report.unreadable" + if not fix_findings(report): + return "stopped", "pendency.no_fixes" + if rounds >= max_rounds: + return "stopped", "fix.bound_reached" + return None, None + + +def scope_paths(units: Iterable[Unit]) -> tuple[str, ...]: + return tuple(sorted({path for unit in units for path in unit.paths})) +# fmt: on diff --git a/kits/profile-dev-kit/python/beddel_profile_dev/tools/fix.py b/kits/profile-dev-kit/python/beddel_profile_dev/tools/fix.py new file mode 100644 index 0000000..8dc65ab --- /dev/null +++ b/kits/profile-dev-kit/python/beddel_profile_dev/tools/fix.py @@ -0,0 +1,194 @@ +# fmt: off +# ruff: noqa: I001 +import hashlib +import json +import os +from collections.abc import Mapping +from pathlib import Path +from typing import Any +from beddel.tools import beddel_tool +from beddel_profile_dev._errors import PROFILE_IMPACT_UNBOUND, ProfileError +from beddel_profile_dev._io import changed_paths, diff_text, git_run, real_repo_root, resolve_revision +from beddel_profile_dev.config import load_config +from beddel_profile_dev.fix import MAX_FINDINGS_BYTES, MAX_FIX_ROUNDS, decision, fix_findings, fix_message, report_at, scope_paths, trailer_round +from beddel_profile_dev.implement import branch_reason, covered, spec_units +from beddel_profile_dev.inventory import is_valid_id +from beddel_profile_dev.plan import front_matter, relative_path, spec_complete +from beddel_profile_dev.tools._subprocess import resolve_executable +from beddel_profile_dev.tools.implement import _TOKEN, _candidate_tree, _completed, _git_text, _read_spec, _repo_error, _run_gates, _symbolic_branch, _valid_base, _valid_story +from beddel_profile_dev.tools.report import profile_read_report + +def _refs(**values: Any) -> dict[str, Any]: + names = ("spec_path", "spec_sha256", "branch", "base", "head", "commit", "change_id", "report", "report_sha256", "round", "rounds", "findings", "diff_bytes", "out_of_scope", "regressed") + return {name: values.get(name) for name in names} + +def _result(story: str | None, outcome: str | None, reason: str | None, refs: dict[str, Any], **values: Any) -> dict[str, Any]: + return {"stage": "verify-and-fix", "story_key": story, "verify": values.get("verify", False), "fix": values.get("fix", False), "outcome": outcome, "reason": reason, "refs": refs, "spec_sha256": values.get("spec_sha256"), "base": values.get("base"), "head": values.get("head"), "round": values.get("round"), "rounds": values.get("rounds", 0), "findings": values.get("findings", ""), "scope_paths": values.get("scope", ""), "baseline": values.get("baseline")} + +def _bad(story: str | None, reason: str, **values: Any) -> dict[str, Any]: + return _result(story, "failed", reason, _refs(**values), **values) + +def _refused(story: str | None, reason: str, **values: Any) -> dict[str, Any]: + return _result(story, "refused", reason, _refs(**values), **values) + +def _inputs(repo_path: str, story: object, spec: object, branch: object, base: object, change: object, bound: object): + s = story if isinstance(story, str) and _valid_story(story) else None + p = spec if isinstance(spec, str) and len(spec) <= 512 and relative_path(spec) else None + b = branch if isinstance(branch, str) and len(branch) <= 128 else None + c = change if isinstance(change, str) and is_valid_id(change) else None + if s is None: return _refused(None, "story_key.invalid", spec_path=p, branch=b, change_id=c) + if p is None: return _refused(s, "path.invalid", branch=b, change_id=c) + if not isinstance(branch, str) or branch_reason(branch, base) == "branch.invalid": return _refused(s, "branch.invalid", spec_path=p, change_id=c) + branch_error = branch_reason(branch, base) + if branch_error is not None: return _refused(s, branch_error, spec_path=p, change_id=c) + if not _valid_base(base): return _refused(s, "base.invalid", spec_path=p, change_id=c) + if c is None: return _refused(s, "change_id.invalid", spec_path=p, branch=branch) + if type(bound) is not int or not 0 <= bound <= MAX_FIX_ROUNDS: return _refused(s, "bound.invalid", spec_path=p, change_id=c) + return real_repo_root(repo_path), s, p, branch, str(base), c, bound + +def _report(repo: Path, change: str, head: str): + try: data = profile_read_report(str(repo), change) + except ProfileError as error: + if error.code == PROFILE_IMPACT_UNBOUND: return None, None + raise + history = data.get("history") if isinstance(data, Mapping) else None + if not isinstance(history, list): return None, None + item = next((item for item in reversed(history) if isinstance(item, Mapping) and isinstance(item.get("content"), Mapping) and item["content"].get("revision") == head), None) + return (report_at(history, head) if item is not None else None), item + +def _state_refs(spec: str, digest: str, branch: str, base: str, head: str, change: str, rounds: int, **values: Any): + item = values.pop("report", None) + if isinstance(item, Mapping): values.update(report=item.get("ref"), report_sha256=item.get("sha256")) + return _refs(spec_path=spec, spec_sha256=digest, branch=branch, base=base, head=head, change_id=change, rounds=rounds, **values) + +def _repo_state(repo: Path, story: str, spec: str, branch: str, base_ref: str, change: str): + refused = lambda reason: _refused(story, reason, spec_path=spec, branch=branch, change_id=change) + base0 = resolve_revision(repo, base_ref) + result = git_run(repo, "show-ref", "--verify", "--quiet", f"refs/heads/{branch}") + if result is None or result.returncode not in (0, 1): raise _repo_error() + if result.returncode == 1: return refused("branch.absent") + if _symbolic_branch(repo) != branch: return refused("branch.not_current") + if Path(os.path.realpath(os.getcwd())) != repo: return refused("cwd.mismatch") + data, error = _read_spec(repo, spec) + if error is not None or data is None: return refused(error or "spec.unreadable") + if not spec_complete(data, story): return refused("spec.incomplete") + units = spec_units(data, story) + if units is None: return refused("units.invalid") + config = load_config() + if any(resolve_executable(argv[0]) is None for argv in config.gate_commands): return refused("gate.unavailable") + fail = lambda reason, **values: _bad(story, reason, **values) + if changed_paths(repo) - {spec}: return fail("worktree.dirty", head=resolve_revision(repo, "HEAD")) + head = resolve_revision(repo, "HEAD") + base = _git_text(git_run(repo, "merge-base", base0, head)) + if base is None: return refused("base.invalid") + if any(unit.id not in _completed(repo, base, story) for unit in units): return refused("units.incomplete") + log = git_run(repo, "log", "-z", "--format=%H%n%B", f"{base}..HEAD") + if log is None or log.returncode != 0: raise _repo_error() + rounds = sum(trailer_round(raw.decode("utf-8", "replace"), story) is not None for raw in log.stdout.split(b"\x00") if raw) + return data, hashlib.sha256(data).hexdigest(), units, config, base, head, rounds + +@beddel_tool(name="profile_fix_reconcile", description="fix", category="profile") +def profile_fix_reconcile(repo_path: str, story_key: str, spec_path: str, branch: str, base_ref: str, change_id: str, max_fix_rounds: int, prior: Mapping[str, Any] | None) -> dict[str, Any]: + checked = _inputs(repo_path, story_key, spec_path, branch, base_ref, change_id, max_fix_rounds) + if isinstance(checked, dict): return checked + repo, story, spec, branch, base_ref, change, bound = checked + fail = lambda reason, **values: _bad(story, reason, **values) + if prior is None: return fail("reconcile.failed") + if prior != {} and (not isinstance(prior, Mapping) or prior.get("stage") != "verify-and-fix" or prior.get("story_key") != story): return fail("state.mismatch") + try: + if prior != {} and (prior.get("outcome") is not None or prior.get("fix") is True): + if prior.get("outcome") in ("failed", "refused") and prior.get("fix") is not True and prior.get("head") is None: + return dict(prior) + head = resolve_revision(repo, "HEAD") + return fail("head.moved", head=head) if head != prior.get("head") else dict(prior) + if prior != {} and prior.get("verify") not in (False, True): return fail("state.mismatch") + state = _repo_state(repo, story, spec, branch, base_ref, change) + if isinstance(state, dict): return state + _data, digest, units, config, base, head, rounds = state + if prior.get("verify") is True and head != prior.get("head"): return fail("head.moved", base=base, head=head, rounds=rounds) + try: report, item = _report(repo, change, head) + except ProfileError: return fail("report.unreadable", base=base, head=head, rounds=rounds) + scope = ", ".join(scope_paths(units)) + if report is None: + refs = _state_refs(spec, digest, branch, base, head, change, rounds) + return fail("verify.failed", **refs) if prior.get("verify") is True else _result(story, None, None, refs, verify=True, spec_sha256=digest, base=base, head=head, rounds=rounds, scope=scope) + outcome, reason = decision(report, rounds, bound) + findings = fix_findings(report) + text = json.dumps(findings, sort_keys=True, separators=(",", ":")) + prior_refs = prior.get("refs") if prior.get("verify") is True else None + commit = prior_refs.get("commit") if isinstance(prior_refs, Mapping) else None + refs = _state_refs(spec, digest, branch, base, head, change, rounds, report=item, findings=len(findings), commit=commit) + if outcome is not None: return _result(story, outcome, reason, refs, spec_sha256=digest, base=base, head=head, rounds=rounds, findings=text, scope=scope) + if len(text.encode()) > MAX_FINDINGS_BYTES: return fail("findings.too_large", **refs) + if _TOKEN.search(text): return fail("secret.detected", **refs) + baseline = _run_gates(repo, config) + if changed_paths(repo) - {spec}: return fail("gate.changed_tree", **refs) + return _result(story, None, None, {**refs, "round": rounds + 1}, verify=False, fix=True, spec_sha256=digest, base=base, head=head, round=rounds + 1, rounds=rounds, findings=text, scope=scope, baseline=baseline) + except ProfileError: return fail("reconcile.failed") + except Exception: # noqa: BLE001 + return fail("reconcile.failed") + +@beddel_tool(name="profile_fix_settle", description="fix", category="profile") +def profile_fix_settle(repo_path: str, story_key: str, spec_path: str, branch: str, base_ref: str, change_id: str, prepare: Mapping[str, Any] | None, agent: Mapping[str, Any] | None = None) -> dict[str, Any]: + checked = _inputs(repo_path, story_key, spec_path, branch, base_ref, change_id, MAX_FIX_ROUNDS) + if isinstance(checked, dict): return checked + repo, story, spec, branch, base_ref, change, _ = checked + fail = lambda reason, **values: _bad(story, reason, **values) + if prepare is None: return fail("reconcile.failed") + if not isinstance(prepare, Mapping) or prepare.get("stage") != "verify-and-fix" or prepare.get("story_key") != story: return fail("state.mismatch") + if prepare.get("fix") is not True: return dict(prepare) + if not isinstance(agent, Mapping): return fail("agent.failed") + try: + head = resolve_revision(repo, "HEAD") + if _symbolic_branch(repo) != branch: return fail("branch.moved", head=head) + if head != prepare.get("head"): return fail("head.moved", head=head) + data, error = _read_spec(repo, spec) + if error is not None or data is None: return fail("spec.changed", head=head) + digest = hashlib.sha256(data).hexdigest() + if prepare.get("spec_sha256") != digest: return fail("spec.changed", head=head) + units = spec_units(data, story) + if units is None: return fail("spec.changed", head=head) + paths, scope = scope_paths(units), ", ".join(scope_paths(units)) + round_, rounds = prepare.get("round"), prepare.get("rounds") + if prepare.get("scope_paths") != scope or type(round_) is not int or not 1 <= round_ <= MAX_FIX_ROUNDS or type(rounds) is not int or rounds != round_ - 1: return fail("state.mismatch", head=head) + config = load_config() + names = [f"{i:02d}-{argv[0]}" for i, argv in enumerate(config.gate_commands, 1)] + baseline = prepare.get("baseline") + if not isinstance(baseline, list) or any(not isinstance(item, Mapping) for item in baseline) or [item.get("name") for item in baseline] != names: return fail("state.mismatch", head=head, rounds=round_) + if any(resolve_executable(argv[0]) is None for argv in config.gate_commands): return fail("gate.unavailable", head=head, rounds=round_) + current = changed_paths(repo) - {spec} + if not current: return fail("fix.empty", head=head, rounds=round_) + outside = sum(not covered(paths, path) for path in current) + if outside: return fail("scope.exceeded", head=head, rounds=round_, out_of_scope=outside) + candidate = _candidate_tree(repo, tuple(sorted(current))) + metadata = front_matter(data) + if metadata is None: return fail("spec.changed", head=head, rounds=round_) + base = prepare.get("base") + if not isinstance(base, str): return fail("state.mismatch", head=head, rounds=round_) + diff, truncated = diff_text(repo, base, candidate, int(metadata["size_cap_bytes"])) + diff_bytes = len(diff.encode()) + if truncated: + full, still = diff_text(repo, base, candidate, 262144) + return fail("size.exceeded", base=base, head=head, rounds=round_, diff_bytes=len(full.encode()) if not still else int(metadata["size_cap_bytes"]) + 1) + if _TOKEN.search("\n".join(line for line in diff.splitlines() if line.startswith("+"))): return fail("secret.detected", base=base, head=head, rounds=round_, diff_bytes=diff_bytes) + gates = _run_gates(repo, config) + after = changed_paths(repo) - {spec} + if after != current or _candidate_tree(repo, tuple(sorted(after))) != candidate: return fail("gate.changed_tree", base=base, head=head, rounds=round_, diff_bytes=diff_bytes) + regressed = [entry["name"] for entry, old in zip(gates, baseline) if old.get("exit_code") == 0 and (entry["timed_out"] or entry["exit_code"] != 0)] + if regressed: return fail("gate.regressed", base=base, head=head, rounds=round_, diff_bytes=diff_bytes, regressed=", ".join(regressed)) + subject, trailers = fix_message(story, round_) + present = [path for path in sorted(current) if os.path.lexists(repo / path)] + added = git_run(repo, "add", "-A", "--", *present) if present else None + if present and (added is None or added.returncode != 0): return fail("commit.failed", base=base, head=head, rounds=round_, diff_bytes=diff_bytes) + committed = git_run(repo, "commit", "-q", "-m", subject, "-m", trailers, "--only", "--", *sorted(current), timeout=120) + if committed is None or committed.returncode != 0: return fail("commit.failed", base=base, head=head, rounds=round_, diff_bytes=diff_bytes) + new_head = resolve_revision(repo, "HEAD") + parent, tree = _git_text(git_run(repo, "rev-parse", "HEAD^")), _git_text(git_run(repo, "rev-parse", "HEAD^{tree}")) + message = _git_text(git_run(repo, "show", "-s", "--format=%B", "HEAD")) + if parent != head or tree != candidate or message is None or trailer_round(message, story) != round_: return fail("commit.changed", base=base, head=new_head, commit=new_head, rounds=round_, diff_bytes=diff_bytes) + refs = _refs(spec_path=spec, branch=branch, base=base, head=new_head, commit=new_head, change_id=change, round=round_, rounds=round_, diff_bytes=diff_bytes) + return _result(story, None, None, refs, verify=True, spec_sha256=digest, base=base, head=new_head, round=round_, rounds=round_, scope=scope) + except ProfileError: return fail("settle.failed") + except Exception: # noqa: BLE001 + return fail("settle.failed") +# fmt: on diff --git a/kits/profile-dev-kit/tests/test_fix.py b/kits/profile-dev-kit/tests/test_fix.py new file mode 100644 index 0000000..e23b1aa --- /dev/null +++ b/kits/profile-dev-kit/tests/test_fix.py @@ -0,0 +1,406 @@ +"""Pure and boundary tests for the verify-and-fix stage.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pytest +from beddel_profile_dev.fix import ( + FIXABLE_REASONS, + MAX_FIX_ROUNDS, + decision, + fix_findings, + fix_message, + report_at, + scope_paths, + trailer_round, +) +from beddel_profile_dev.implement import Unit +from beddel_profile_dev.inventory import validate_inventory +from beddel_profile_dev.tools.fix import profile_fix_reconcile, profile_fix_settle + +from ._helpers import commit_all, git, good, init_repo, stub_gate_tools +from .test_implement import BRANCH, SPEC_PATH, STORY, spec_bytes + + +def test_fix_message_and_trailer_round_contract() -> None: + subject, trailers = fix_message(STORY, 2) + assert subject == "fix: apply verification fixes (round 2)" + assert trailer_round(f"{subject}\n\n{trailers}\n", STORY) == 2 + assert trailer_round(f"{trailers}\n{trailers}\n", STORY) is None + assert trailer_round(f"Story-Key: {STORY}\nFix-Round: 3", STORY) is None + assert trailer_round("Story-Key: other\nFix-Round: 1", STORY) is None + + +def _report(revision: str, verdict: str = "has pendency") -> dict[str, object]: + finding = { + "source": "gate", + "association": "requirement", + "class": "gate-unit", + "affected_point": "api", + "criterion": "tests-pass", + "kind": "gate", + "between": ["implementation", "validation"], + "detail": {"description": "failed"}, + "classification": "blocking", + } + return { + "revision": revision, + "verdict": verdict, + "items": { + "divergences": {"status": "present", "value": [finding, {"x": 2}]}, + "pendencies": { + "status": "present", + "value": [ + {"reason": "finding_blocking", "finding_index": 0}, + {"reason": "evidence_unrecorded"}, + {"reason": "finding_unclassified", "finding_index": 0}, + {"reason": "finding_blocking", "finding_index": 1}, + ], + }, + }, + } + + +def test_report_selection_findings_scope_and_decision() -> None: + first = _report("a" * 40) + history = [ + {"ref": "one", "sha256": "a", "content": first}, + {"ref": "two", "sha256": "b", "content": _report("b" * 40, "consistent")}, + {"ref": "three", "sha256": "c", "content": first}, + ] + assert report_at(history, "a" * 40) == first + findings = fix_findings(first) + assert set(findings[0]) == { + "source", + "association", + "class", + "affected_point", + "criterion", + "kind", + "between", + "detail", + } + assert findings[0]["source"] == "gate" + assert findings[1]["detail"] is None + assert all("classification" not in finding for finding in findings) + assert decision(first, 0, MAX_FIX_ROUNDS) == (None, None) + assert decision(first, 2, MAX_FIX_ROUNDS) == ("stopped", "fix.bound_reached") + assert decision(_report("a" * 40, "consistent"), 0, 2) == ( + "succeeded", + "verdict.consistent", + ) + no_fix = _report("a" * 40) + no_fix["items"]["pendencies"]["value"] = [{"reason": "evidence_unrecorded"}] # type: ignore[index] + assert decision(no_fix, 0, 2) == ("stopped", "pendency.no_fixes") + assert decision({"verdict": "unknown"}, 0, 2) == ("failed", "report.unreadable") + assert FIXABLE_REASONS == {"finding_blocking", "finding_unclassified"} + assert scope_paths( + [Unit("b", ("tests", "src/b.py"), "feat: b"), Unit("a", ("src",), "feat: a")] + ) == ("src", "src/b.py", "tests") + + +def _complete_repo(root: Path) -> tuple[Path, str]: + repo = init_repo(root / "repo") + git(repo, "config", "user.name", "Fixture") + git(repo, "config", "user.email", "fixture@example.invalid") + (repo / "src").mkdir() + (repo / "src" / "base.py").write_text("base = True\n") + (repo / SPEC_PATH).parent.mkdir() + (repo / SPEC_PATH).write_bytes(spec_bytes()) + base = commit_all(repo, "base") + git(repo, "switch", "-q", "-c", BRANCH) + (repo / "src" / "one.py").write_text("one = 1\n") + git(repo, "add", "--", "src/one.py") + git( + repo, + "commit", + "-q", + "-m", + "feat: one", + "-m", + f"Story-Key: {STORY}\nImplementation-Unit: u1", + ) + (repo / "src" / "two.py").write_text("two = 2\n") + git(repo, "add", "--", "src/two.py") + git( + repo, + "commit", + "-q", + "-m", + "feat: two", + "-m", + f"Story-Key: {STORY}\nImplementation-Unit: u2", + ) + return repo, base + + +def test_reconcile_refusals_and_settle_state_checks( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + repo, base = _complete_repo(tmp_path) + monkeypatch.chdir(repo) + monkeypatch.setenv("PATH", stub_gate_tools(tmp_path)) + monkeypatch.setenv("HOME", str(tmp_path / "home")) + assert ( + profile_fix_reconcile(str(repo), STORY, SPEC_PATH, BRANCH, base, "c1", 3, {})[ + "reason" + ] + == "bound.invalid" + ) + assert ( + profile_fix_reconcile(str(repo), STORY, SPEC_PATH, BRANCH, base, "c1", 2, None)[ + "reason" + ] + == "reconcile.failed" + ) + prepared = profile_fix_reconcile( + str(repo), STORY, SPEC_PATH, BRANCH, base, "c1", 2, {} + ) + assert prepared["verify"] is True, (prepared["reason"], prepared["refs"]) + assert ( + profile_fix_settle(str(repo), STORY, SPEC_PATH, BRANCH, base, "c1", prepared) + == prepared + ) + assert json.loads(json.dumps(prepared))["stage"] == "verify-and-fix" + + +def test_ac2_review_lens_fixture_is_a_valid_inventory() -> None: + data = good() + ids = ["blind-hunter", "edge-case-hunter", "verification-gap", "acceptance-auditor"] + data["classes"] = [{"id": item, "kind": "review"} for item in ids] + data["obligations"] = [ + {"class": item, "affected_point": "api", "criterion": "arch-ok"} for item in ids + ] + inventory, failures = validate_inventory(data, "v1") + assert inventory is not None + assert not failures + + +# fmt: off +@pytest.mark.parametrize("bound", [-1, True, "2", 3]) +def test_reconcile_rejects_every_invalid_fix_bound( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, bound: object +) -> None: + repo, base = _complete_repo(tmp_path) + monkeypatch.chdir(repo) + monkeypatch.setenv("PATH", stub_gate_tools(tmp_path)) + result = profile_fix_reconcile( + str(repo), STORY, SPEC_PATH, BRANCH, base, "c1", bound, {} + ) + assert result["reason"] == "bound.invalid" + + +@pytest.mark.parametrize( + ("field", "value", "expected"), + [ + ("story_key", "BAD!", "story_key.invalid"), + ("spec_path", "../bad", "path.invalid"), + ("branch", "BAD", "branch.invalid"), + ("branch", "main", "branch.protected"), + ("base_ref", "", "base.invalid"), + ("change_id", "bad!", "change_id.invalid"), + ], +) +def test_reconcile_rejects_input_refusal_reasons( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + field: str, + value: object, + expected: str, +) -> None: + repo, base = _complete_repo(tmp_path) + monkeypatch.chdir(repo) + monkeypatch.setenv("PATH", stub_gate_tools(tmp_path)) + args: dict[str, object] = { + "repo_path": str(repo), "story_key": STORY, "spec_path": SPEC_PATH, + "branch": BRANCH, "base_ref": base, "change_id": "c1", "max_fix_rounds": 2, "prior": {}, + } + args[field] = value + result = profile_fix_reconcile(**args) # type: ignore[arg-type] + assert result["reason"] == expected + + +@pytest.mark.parametrize("kind", [ + "branch.absent", "branch.not_current", "cwd.mismatch", "spec.absent", + "spec.unreadable", "spec.incomplete", "units.invalid", "units.incomplete", + "gate.unavailable", +]) +def test_reconcile_refusals_forward_without_head( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, kind: str +) -> None: + repo, base = _complete_repo(tmp_path) + monkeypatch.chdir(repo) + monkeypatch.setenv("PATH", stub_gate_tools(tmp_path)) + if kind == "branch.absent": branch = "feat/missing" + elif kind == "branch.not_current": + branch = "feat/other" + git(repo, "branch", branch) + else: branch = BRANCH + if kind == "cwd.mismatch": monkeypatch.chdir(tmp_path) + elif kind == "spec.absent": (repo / SPEC_PATH).unlink() + elif kind == "spec.unreadable": + (repo / SPEC_PATH).unlink() + (repo / SPEC_PATH).mkdir() + elif kind == "spec.incomplete": (repo / SPEC_PATH).write_text("---\nstory_key: wrong\n---\n") + elif kind == "units.invalid": monkeypatch.setattr("beddel_profile_dev.tools.fix.spec_units", lambda *_: None) + elif kind == "units.incomplete": monkeypatch.setattr("beddel_profile_dev.tools.fix._completed", lambda *_: set()) + elif kind == "gate.unavailable": monkeypatch.setattr("beddel_profile_dev.tools.fix.resolve_executable", lambda *_: None) + first = profile_fix_reconcile(str(repo), STORY, SPEC_PATH, branch, base, "c1", 2, {}) + assert first["reason"] == kind + assert first["head"] is None + second = profile_fix_reconcile(str(repo), STORY, SPEC_PATH, branch, base, "c1", 2, first) + assert second == first + assert first["refs"]["spec_path"] == SPEC_PATH + assert first["refs"]["branch"] == branch + assert first["refs"]["change_id"] == "c1" + + +def test_reconcile_gate_changed_tree_is_a_recordable_failure( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + repo, base = _complete_repo(tmp_path) + monkeypatch.chdir(repo) + monkeypatch.setenv("PATH", stub_gate_tools(tmp_path)) + monkeypatch.setattr( + "beddel_profile_dev.tools.fix.profile_read_report", + lambda *_: {"history": [{"ref": "report-1", "sha256": "hash", "content": _report(git(repo, "rev-parse", "HEAD"))}]}, + ) + def dirty_gate(*_: object) -> list[dict[str, object]]: + (repo / "src" / "gate-created.py").write_text("x = 1\n") + return [] + monkeypatch.setattr("beddel_profile_dev.tools.fix._run_gates", dirty_gate) + result = profile_fix_reconcile(str(repo), STORY, SPEC_PATH, BRANCH, base, "c1", 2, {}) + assert result["reason"] == "gate.changed_tree" + assert result["refs"]["findings"] == 2 + assert not isinstance(result["refs"]["findings"], str) + + +def test_reconcile_unexpected_exception_is_recordable( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + repo, base = _complete_repo(tmp_path) + monkeypatch.chdir(repo) + monkeypatch.setenv("PATH", stub_gate_tools(tmp_path)) + monkeypatch.setattr("beddel_profile_dev.tools.fix._report", lambda *_: (_ for _ in ()).throw(RuntimeError("boom"))) + result = profile_fix_reconcile(str(repo), STORY, SPEC_PATH, BRANCH, base, "c1", 2, {}) + assert result["reason"] == "reconcile.failed" + + +def test_settle_unexpected_exception_is_recordable( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + repo, base = _complete_repo(tmp_path) + monkeypatch.chdir(repo) + monkeypatch.setenv("PATH", stub_gate_tools(tmp_path)) + prepare = { + "stage": "verify-and-fix", "story_key": STORY, "fix": True, + "head": git(repo, "rev-parse", "HEAD"), + "spec_sha256": hashlib.sha256(spec_bytes()).hexdigest(), + "scope_paths": "src/one.py, src/two.py", "round": 1, "rounds": 0, + "base": base, + "baseline": [{"name": name} for name in ("01-ruff", "02-ruff", "03-mypy", "04-pytest")], + } + monkeypatch.setattr("beddel_profile_dev.tools.fix.changed_paths", lambda *_: (_ for _ in ()).throw(RuntimeError("boom"))) + result = profile_fix_settle(str(repo), STORY, SPEC_PATH, BRANCH, base, "c1", prepare, {}) + assert result["reason"] == "settle.failed" + + +def _due_prepare( + repo: Path, base: str, monkeypatch: pytest.MonkeyPatch +) -> dict[str, object]: + head = git(repo, "rev-parse", "HEAD") + monkeypatch.setattr( + "beddel_profile_dev.tools.fix.profile_read_report", + lambda *_: {"history": [{"ref": "r", "sha256": "s", "content": _report(head)}]}, + ) + return profile_fix_reconcile(str(repo), STORY, SPEC_PATH, BRANCH, base, "c1", 2, {}) + + +@pytest.mark.parametrize("case", [ + "agent.failed", "branch.moved", "head.moved", "spec.changed", "fix.empty", + "scope.exceeded", "size.exceeded", "secret.detected", "gate.regressed", + "base-red", "gate.changed_tree", "commit.failed", +]) +def test_settle_boundary_table( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, case: str +) -> None: + repo, base = _complete_repo(tmp_path) + monkeypatch.chdir(repo) + monkeypatch.setenv("PATH", stub_gate_tools(tmp_path)) + if case == "base-red": monkeypatch.setenv("STUB_EXIT_PYTEST", "1") + prepare = _due_prepare(repo, base, monkeypatch) + assert prepare["fix"] is True + if case == "branch.moved": + git(repo, "switch", "-q", "-c", "feat/other") + elif case == "head.moved": + commit_all(repo, "moved") + elif case == "spec.changed": + (repo / SPEC_PATH).write_bytes(spec_bytes() + b"changed\n") + elif case == "scope.exceeded": + (repo / "README.md").write_text("outside\n") + elif case == "size.exceeded": + monkeypatch.setattr("beddel_profile_dev.tools.fix.diff_text", lambda *_: ("diff", True)) + (repo / "src" / "one.py").write_text("one = 2\n") + elif case == "secret.detected": + (repo / "src" / "one.py").write_text("token = 'github_pat_" + "x" * 11 + "'\n") + elif case in {"gate.regressed", "base-red"}: + if case == "gate.regressed": monkeypatch.setenv("STUB_EXIT_PYTEST", "1") + (repo / "src" / "one.py").write_text("one = 2\n") + elif case == "gate.changed_tree": + (repo / "src" / "one.py").write_text("one = 2\n") + def dirty_gate(*_: object) -> list[dict[str, object]]: + (repo / "src" / "extra.py").write_text("extra = 1\n") + return prepare["baseline"] # type: ignore[return-value] + monkeypatch.setattr("beddel_profile_dev.tools.fix._run_gates", dirty_gate) + elif case == "commit.failed": + (repo / "src" / "one.py").write_text("one = 2\n") + hook = repo / ".git" / "hooks" / "pre-commit" + hook.write_text("#!/bin/sh\nexit 1\n") + hook.chmod(0o755) + result = profile_fix_settle( + str(repo), STORY, SPEC_PATH, BRANCH, base, "c1", prepare, + None if case == "agent.failed" else {}, + ) + expected = None if case == "base-red" else ("agent.failed" if case == "agent.failed" else case) + assert result["reason"] == expected + if case == "base-red": assert result["verify"] is True + + +def test_reconcile_dirty_state_and_state_chain_checks( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + repo, base = _complete_repo(tmp_path) + monkeypatch.chdir(repo) + monkeypatch.setenv("PATH", stub_gate_tools(tmp_path)) + (repo / "src" / "untracked.py").write_text("x = 1\n") + dirty = profile_fix_reconcile(str(repo), STORY, SPEC_PATH, BRANCH, base, "c1", 2, {}) + assert dirty["reason"] == "worktree.dirty" + (repo / "src" / "untracked.py").unlink() + clean = profile_fix_reconcile(str(repo), STORY, SPEC_PATH, BRANCH, base, "c1", 2, {}) + assert profile_fix_reconcile(str(repo), STORY, SPEC_PATH, BRANCH, base, "c1", 2, {"stage": "x"})["reason"] == "state.mismatch" + moved = dict(clean) + moved["head"] = "f" * 40 + assert profile_fix_reconcile(str(repo), STORY, SPEC_PATH, BRANCH, base, "c1", 2, moved)["reason"] == "head.moved" + + +# fmt: on + + +# fmt: off +@pytest.mark.parametrize("kind", ["findings.too_large", "secret.detected"]) +def test_findings_handoff_limits(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, kind: str) -> None: + repo, base = _complete_repo(tmp_path) + monkeypatch.chdir(repo) + monkeypatch.setenv("PATH", stub_gate_tools(tmp_path)) + head = git(repo, "rev-parse", "HEAD") + report = _report(head) + value = "x" * 33000 if kind == "findings.too_large" else "github_pat_" + "x" * 11 + report["items"]["divergences"]["value"][0]["detail"] = value # type: ignore[index] + monkeypatch.setattr("beddel_profile_dev.tools.fix.profile_read_report", lambda *_: {"history": [{"ref": "r", "sha256": "s", "content": report}]}) + result = profile_fix_reconcile(str(repo), STORY, SPEC_PATH, BRANCH, base, "c1", 2, {}) + assert result["reason"] == kind + +# fmt: on diff --git a/kits/profile-dev-kit/tests/test_manifest.py b/kits/profile-dev-kit/tests/test_manifest.py index d144b8b..338bb6e 100644 --- a/kits/profile-dev-kit/tests/test_manifest.py +++ b/kits/profile-dev-kit/tests/test_manifest.py @@ -35,6 +35,8 @@ "profile_record_stage_outcome", "profile_implement_prepare", "profile_implement_settle", + "profile_fix_reconcile", + "profile_fix_settle", ] @@ -109,6 +111,7 @@ def test_manifest_declares_all_workflows_and_python_targets() -> None: "record-approval-decision", "read-approval", "implement-story", + "verify-and-fix-story", "plan-story", ] declared = {t["name"] for t in manifest["tools"]} diff --git a/kits/profile-dev-kit/tests/test_verify_fix_workflow.py b/kits/profile-dev-kit/tests/test_verify_fix_workflow.py new file mode 100644 index 0000000..dde46ac --- /dev/null +++ b/kits/profile-dev-kit/tests/test_verify_fix_workflow.py @@ -0,0 +1,395 @@ +"""Shape and safety coverage for the verify-and-fix entry workflow.""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from typing import Any + +import pytest +import yaml +from beddel.domain.kit import parse_kit_manifest +from beddel.domain.models import AgentResult, DefaultDependencies, StrategyType +from beddel.domain.parser import WorkflowParser +from beddel.domain.registry import PrimitiveRegistry +from beddel.primitives import register_builtins +from beddel.tools.kits import load_kit +from beddel_profile_dev import _adapters +from beddel_profile_dev.findings import ( + ASSESSMENT_BEGIN, + ASSESSMENT_END, + REVIEW_ASSESSMENT_SCHEMA, +) + +from ._helpers import ( + FakeReviewAdapter, + commit_all, + git, + good, + init_repo, + make_source, + stub_gate_tools, + write_version, +) + +KIT_ROOT = Path(__file__).resolve().parents[1] +WORKFLOW_PATH = KIT_ROOT / "workflows" / "verify-and-fix-story.yaml" + + +def test_workflow_shape_and_bound() -> None: + document = yaml.safe_load(WORKFLOW_PATH.read_text()) + assert document["id"] == "verify-and-fix-story" + assert document["requires_kits"] == ["profile-dev-kit", "agent-kiro-kit"] + assert document["inputs"] == [ + "repo_path", + "story_key", + "spec_path", + "branch", + "base_ref", + "change_id", + "inventory_version", + ] + assert document["allowed_tools"] == [ + "profile_fix_reconcile", + "profile_fix_settle", + "profile_record_stage_outcome", + ] + assert "mcp_servers" not in document + assert not any( + word in document["allowed_tools"] + for word in ("approve", "ratify", "integrate", "push", "publish") + ) + steps = WorkflowParser.parse(WORKFLOW_PATH.read_text()).steps + assert [step.id for step in steps] == [ + "reconcile", + "verify", + "round_1", + "fix_1", + "settle_1", + "settle_1_stop", + "verify_1", + "round_2", + "fix_2", + "settle_2", + "settle_2_stop", + "verify_2", + "close", + "record", + ] + assert all( + step.config.get("arguments", {}).get("max_fix_rounds") == 2 + for step in steps + if step.id in {"reconcile", "round_1", "round_2", "close"} + ) + agents = [step for step in steps if step.primitive == "agent-exec"] + assert len(agents) == 2 + assert all( + step.config["adapter"] == "kiro-cli" + and step.config["sandbox"] == "workspace-write" + and step.timeout <= 600 + for step in agents + ) + nested = [step for step in steps if step.primitive == "call-agent"] + assert len(nested) == 3 + assert all(step.config["workflow"] == "verify-change.yaml" for step in nested) + assert all( + list(step.config["inputs"]) + == [ + "repo_path", + "change_id", + "base", + "revision", + "inventory_version", + ] + for step in nested + ) + assert all(step.execution_strategy.type is StrategyType.SKIP for step in nested) + assert steps[4].output_key == steps[5].output_key == "fixed_1" + assert steps[9].output_key == steps[10].output_key == "fixed_2" + assert { + step.id: step.if_condition for step in steps if step.if_condition is not None + } == { + "verify": "$stepResult.reconcile.result.verify == true", + "fix_1": "$stepResult.round_1.result.fix == true", + "settle_1": "$stepResult.round_1.result.fix == true", + "settle_1_stop": "$stepResult.round_1.result.fix != true", + "verify_1": "$stepResult.fixed_1.result.verify == true", + "fix_2": "$stepResult.round_2.result.fix == true", + "settle_2": "$stepResult.round_2.result.fix == true", + "settle_2_stop": "$stepResult.round_2.result.fix != true", + "verify_2": "$stepResult.fixed_2.result.verify == true", + } + assert all( + step.execution_strategy.type is StrategyType.FAIL + for step in steps + if step.primitive == "tool" and step.id != "reconcile" + ) + + +STORY = "7-5-verify-fix-fixture" +SPEC_PATH = "specs/story.md" +BRANCH = "feat/story-7-5" + + +def _spec() -> bytes: + return ( + ( + f"---\nstory_key: {STORY}\nrequirements: [FR-14]\n" + "repositories: [. ]\ndependencies: []\nsize_cap_bytes: 64000\n" + 'units:\n - id: u1\n paths: [src/api.py]\n subject: "feat: update api"\n---\n\n# Fixture\n' + ) + .replace("[. ]", "[.]") + .encode() + ) + + +def _repo(tmp_path: Path) -> tuple[Path, str]: + repo = init_repo(tmp_path / "repo") + git(repo, "config", "user.name", "Fixture") + git(repo, "config", "user.email", "fixture@example.invalid") + make_source(repo) + write_version(repo, "v1", good()) + from beddel_profile_dev.tools.ratify import profile_ratify_inventory + + profile_ratify_inventory(str(repo), "v1", "owner") + (repo / SPEC_PATH).parent.mkdir() + (repo / SPEC_PATH).write_bytes(_spec()) + base = commit_all(repo, "base") + git(repo, "switch", "-q", "-c", BRANCH) + (repo / "src/api.py").write_text("x = 2\n") + git(repo, "add", "--", "src/api.py") + git( + repo, + "commit", + "-q", + "-m", + "feat: update api", + "-m", + f"Story-Key: {STORY}\nImplementation-Unit: u1", + ) + return repo, base + + +class FixAgent: + def __init__(self, repo: Path, monkeypatch: pytest.MonkeyPatch) -> None: + self.repo = repo + self.monkeypatch = monkeypatch + self.calls = 0 + + async def execute( + self, + prompt: str, + *, + model: str | None = None, + sandbox: str = "read-only", + tools: list[str] | None = None, + output_schema: dict[str, Any] | None = None, + ) -> AgentResult: + self.calls += 1 + (self.repo / "src/api.py").write_text("x = 3\n") + self.monkeypatch.setenv("STUB_EXIT_PYTEST", "0") + return AgentResult(0, "", [], [], {}, "fake") + + +def _run_stage( + repo: Path, fake: FixAgent, base: str, **overrides: object +) -> dict[str, Any]: + workflow = WorkflowParser.parse(WORKFLOW_PATH.read_text()) + registry = PrimitiveRegistry() + register_builtins(registry) + tools = load_kit(parse_kit_manifest(KIT_ROOT / "kit.yaml")) + dependencies = DefaultDependencies( + agent_registry={"kiro-cli": fake}, + tool_registry=tools, + registry=registry, + workflow_loader=lambda name: WorkflowParser.parse( + (KIT_ROOT / "workflows" / name).read_text() + ), + ) + from beddel.domain.executor import WorkflowExecutor + + payload = { + "repo_path": str(repo), + "story_key": STORY, + "spec_path": SPEC_PATH, + "branch": BRANCH, + "base_ref": base, + "change_id": "c1", + "inventory_version": "v1", + } + payload.update(overrides) + return asyncio.run( + WorkflowExecutor(registry, deps=dependencies).execute(workflow, payload) + ) + + +def _review_output() -> str: + payload = { + "schema": REVIEW_ASSESSMENT_SCHEMA, + "assessments": [ + { + "class": "review-arch", + "affected_point": "api", + "criterion": "arch-ok", + "divergences": [], + } + ], + "unassociated": [], + } + return f"{ASSESSMENT_BEGIN}\n{json.dumps(payload)}\n{ASSESSMENT_END}\n" + + +def test_in_process_consistent_run_has_one_report_and_no_fix_agent( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + repo, base = _repo(tmp_path) + monkeypatch.chdir(repo) + monkeypatch.setenv("PATH", stub_gate_tools(tmp_path)) + monkeypatch.setenv("HOME", str(tmp_path / "home")) + monkeypatch.setenv("STUB_EXIT_PYTEST", "0") + monkeypatch.setattr( + _adapters, + "get_review_adapter", + lambda agent_name, *, timeout_seconds: FakeReviewAdapter( + "completed", output=_review_output() + ), + ) + fake = FixAgent(repo, monkeypatch) + result = _run_stage(repo, fake, base) + close = result["step_results"]["close"]["result"] + assert (close["outcome"], close["reason"], fake.calls) == ( + "succeeded", + "verdict.consistent", + 0, + ) + assert len(list((tmp_path / "home").rglob("changes/c1/report/*.json"))) == 1 + + +# fmt: off +def test_in_process_fix_round_commits_and_reverifies( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + repo, base = _repo(tmp_path) + monkeypatch.chdir(repo) + monkeypatch.setenv("PATH", stub_gate_tools(tmp_path)) + monkeypatch.setenv("HOME", str(tmp_path / "home")) + monkeypatch.setenv("STUB_EXIT_PYTEST", "1") + monkeypatch.setattr( + _adapters, + "get_review_adapter", + lambda agent_name, *, timeout_seconds: FakeReviewAdapter( + "completed", output=_review_output() + ), + ) + fake = FixAgent(repo, monkeypatch) + result = _run_stage(repo, fake, base) + close = result["step_results"]["close"]["result"] + reports = sorted((tmp_path / "home").rglob("changes/c1/report/*.json")) + assert (close["outcome"], close["reason"], fake.calls) == ( + "succeeded", + "verdict.consistent", + 1, + ) + assert len(reports) == 2 + assert "Fix-Round: 1" in git(repo, "show", "-s", "--format=%B", "HEAD") + assert json.loads(reports[1].read_text())["reverification"]["status"] == "recorded" + first, second = (json.loads(path.read_text()) for path in reports) + assert first["revision"] != second["revision"] + links = [link for link in second["reverification"]["links"] if link["class"] == "gate-unit"] + assert links and all(link["result"] == "resolved" for link in links) + assert all(not item["ref"].startswith(f"revisions/{first['revision']}/") for item in second["reads"] if "ref" in item) +class ErrorAgent(FixAgent): + async def execute(self, prompt: str, **kwargs: Any) -> AgentResult: + from beddel.domain.errors import AgentError + from beddel.error_codes import AGENT_EXECUTION_FAILED + + self.calls += 1 + raise AgentError(code=AGENT_EXECUTION_FAILED, message="x") + + +class PersistentAgent(FixAgent): + async def execute(self, prompt: str, **kwargs: Any) -> AgentResult: + self.calls += 1 + (self.repo / "src/api.py").write_text(f"x = {self.calls + 2}\n") + return AgentResult(0, "", [], [], {}, "fake") + + +class OutOfScopeAgent(FixAgent): + async def execute(self, prompt: str, **kwargs: Any) -> AgentResult: + self.calls += 1 + (self.repo / "README.md").write_text("outside\n") + return AgentResult(0, "", [], [], {}, "fake") + + +def _install_review(monkeypatch: pytest.MonkeyPatch, behavior: str = "completed") -> None: + monkeypatch.setattr( + _adapters, + "get_review_adapter", + lambda agent_name, *, timeout_seconds: FakeReviewAdapter( + behavior, output=_review_output() + ), + ) + + +def test_workflow_persistent_finding_hits_bound_and_rerun_reuses_report( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + repo, base = _repo(tmp_path) + monkeypatch.chdir(repo) + monkeypatch.setenv("PATH", stub_gate_tools(tmp_path)) + home = tmp_path / "home" + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("STUB_EXIT_PYTEST", "1") + _install_review(monkeypatch) + fake = PersistentAgent(repo, monkeypatch) + result = _run_stage(repo, fake, base) + close = result["step_results"]["close"]["result"] + assert (close["outcome"], close["reason"], fake.calls) == ("stopped", "fix.bound_reached", 2) + assert len(list(home.rglob("changes/c1/report/*.json"))) == 3 + assert not list(home.rglob("changes/c1/approval.json")) + rerun = _run_stage(repo, fake, base) + assert len(list(home.rglob("changes/c1/report/*.json"))) == 3 + assert [path.name for path in sorted((tmp_path / "home").rglob("stage-outcomes/*.json"))] == ["1.json", "2.json"] + assert rerun["step_results"]["record"]["result"]["record"] == 2 + + +def test_workflow_maps_agent_error_refusal_timeout_inventory_and_scope( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + cases = ["timeout", "agent", "refusal", "inventory", "scope"] + for case in cases: + root = tmp_path / case + repo, base = _repo(root) + monkeypatch.chdir(repo) + monkeypatch.setenv("PATH", stub_gate_tools(root)) + monkeypatch.setenv("HOME", str(root / "home")) + if case == "timeout": + _install_review(monkeypatch, "timeout") + fake: FixAgent = FixAgent(repo, monkeypatch) + expected = "pendency.no_fixes" + elif case == "agent": + monkeypatch.setenv("STUB_EXIT_PYTEST", "1") + _install_review(monkeypatch) + fake = ErrorAgent(repo, monkeypatch) + expected = "agent.failed" + elif case == "scope": + monkeypatch.setenv("STUB_EXIT_PYTEST", "1") + _install_review(monkeypatch) + fake = OutOfScopeAgent(repo, monkeypatch) + expected = "scope.exceeded" + else: + _install_review(monkeypatch) + fake = FixAgent(repo, monkeypatch) + expected = "branch.protected" if case == "refusal" else "verify.failed" + overrides: dict[str, object] = {} + if case == "refusal": overrides["branch"] = "main" + if case == "inventory": overrides["inventory_version"] = "missing" + result = _run_stage(repo, fake, base, **overrides) + close = result["step_results"]["close"]["result"] + assert close["reason"] == expected + if case in ("agent", "timeout"): assert fake.calls == (case == "agent") + if case == "inventory": assert "verify" in str(result.get("absorbed_errors")) + + +# fmt: on diff --git a/kits/profile-dev-kit/tests/test_workflow.py b/kits/profile-dev-kit/tests/test_workflow.py index 1a4d994..0c4d565 100644 --- a/kits/profile-dev-kit/tests/test_workflow.py +++ b/kits/profile-dev-kit/tests/test_workflow.py @@ -113,7 +113,7 @@ def test_allowlist_equals_the_tools_the_steps_name() -> None: "profile_capture_evidence", "profile_record_report", ] - assert len(list(WORKFLOWS_DIR.glob("*.yaml"))) == 10 + assert len(list(WORKFLOWS_DIR.glob("*.yaml"))) == 11 def test_review_step_outlives_the_largest_accepted_review_budget() -> None: diff --git a/kits/profile-dev-kit/workflows/verify-and-fix-story.yaml b/kits/profile-dev-kit/workflows/verify-and-fix-story.yaml new file mode 100644 index 0000000..e215ad6 --- /dev/null +++ b/kits/profile-dev-kit/workflows/verify-and-fix-story.yaml @@ -0,0 +1,230 @@ +id: verify-and-fix-story +name: Verify And Fix Story +description: Entry verify-and-fix stage that runs verify-change unchanged and, when Findings need fixes, runs at most 2 fix rounds through a Coding agent with a fresh verify-change attempt each. It never approves, ratifies, pushes or integrates. +version: "1.0" +requires_kits: + - profile-dev-kit + - agent-kiro-kit +allowed_tools: + - profile_fix_reconcile + - profile_fix_settle + - profile_record_stage_outcome +inputs: + - repo_path + - story_key + - spec_path + - branch + - base_ref + - change_id + - inventory_version +steps: + - id: reconcile + primitive: tool + execution_strategy: + type: skip + config: + tool: profile_fix_reconcile + arguments: + repo_path: "$input.repo_path" + story_key: "$input.story_key" + spec_path: "$input.spec_path" + branch: "$input.branch" + base_ref: "$input.base_ref" + change_id: "$input.change_id" + max_fix_rounds: 2 + prior: {} + - id: verify + primitive: call-agent + if: "$stepResult.reconcile.result.verify == true" + execution_strategy: + type: skip + config: + workflow: verify-change.yaml + inputs: + repo_path: "$input.repo_path" + change_id: "$input.change_id" + base: "$stepResult.reconcile.result.base" + revision: "$stepResult.reconcile.result.head" + inventory_version: "$input.inventory_version" + - id: round_1 + primitive: tool + execution_strategy: + type: fail + config: + tool: profile_fix_reconcile + arguments: + repo_path: "$input.repo_path" + story_key: "$input.story_key" + spec_path: "$input.spec_path" + branch: "$input.branch" + base_ref: "$input.base_ref" + change_id: "$input.change_id" + max_fix_rounds: 2 + prior: "$stepResult.reconcile.result" + - id: fix_1 + primitive: agent-exec + if: "$stepResult.round_1.result.fix == true" + execution_strategy: + type: skip + timeout: 600 + config: + adapter: kiro-cli + sandbox: workspace-write + prompt: >- + You are the Coding agent for one bounded verification fix round. Read + the spec at $input.spec_path; apply only the recorded Findings in + $stepResult.round_1.result.findings within the paths + $stepResult.round_1.result.scope_paths; change only those paths and + leave the spec unchanged. Do not stage, commit, push, create or switch + branches, approve, verify, ratify or integrate. + - id: settle_1 + primitive: tool + if: "$stepResult.round_1.result.fix == true" + execution_strategy: + type: fail + output_key: fixed_1 + config: + tool: profile_fix_settle + arguments: + repo_path: "$input.repo_path" + story_key: "$input.story_key" + spec_path: "$input.spec_path" + branch: "$input.branch" + base_ref: "$input.base_ref" + change_id: "$input.change_id" + prepare: "$stepResult.round_1.result" + agent: "$stepResult.fix_1" + - id: settle_1_stop + primitive: tool + if: "$stepResult.round_1.result.fix != true" + execution_strategy: + type: fail + output_key: fixed_1 + config: + tool: profile_fix_settle + arguments: + repo_path: "$input.repo_path" + story_key: "$input.story_key" + spec_path: "$input.spec_path" + branch: "$input.branch" + base_ref: "$input.base_ref" + change_id: "$input.change_id" + prepare: "$stepResult.round_1.result" + - id: verify_1 + primitive: call-agent + if: "$stepResult.fixed_1.result.verify == true" + execution_strategy: + type: skip + config: + workflow: verify-change.yaml + inputs: + repo_path: "$input.repo_path" + change_id: "$input.change_id" + base: "$stepResult.fixed_1.result.base" + revision: "$stepResult.fixed_1.result.head" + inventory_version: "$input.inventory_version" + - id: round_2 + primitive: tool + execution_strategy: + type: fail + config: + tool: profile_fix_reconcile + arguments: + repo_path: "$input.repo_path" + story_key: "$input.story_key" + spec_path: "$input.spec_path" + branch: "$input.branch" + base_ref: "$input.base_ref" + change_id: "$input.change_id" + max_fix_rounds: 2 + prior: "$stepResult.fixed_1.result" + - id: fix_2 + primitive: agent-exec + if: "$stepResult.round_2.result.fix == true" + execution_strategy: + type: skip + timeout: 600 + config: + adapter: kiro-cli + sandbox: workspace-write + prompt: >- + You are the Coding agent for one bounded verification fix round. Read + the spec at $input.spec_path; apply only the recorded Findings in + $stepResult.round_2.result.findings within the paths + $stepResult.round_2.result.scope_paths; change only those paths and + leave the spec unchanged. Do not stage, commit, push, create or switch + branches, approve, verify, ratify or integrate. + - id: settle_2 + primitive: tool + if: "$stepResult.round_2.result.fix == true" + execution_strategy: + type: fail + output_key: fixed_2 + config: + tool: profile_fix_settle + arguments: + repo_path: "$input.repo_path" + story_key: "$input.story_key" + spec_path: "$input.spec_path" + branch: "$input.branch" + base_ref: "$input.base_ref" + change_id: "$input.change_id" + prepare: "$stepResult.round_2.result" + agent: "$stepResult.fix_2" + - id: settle_2_stop + primitive: tool + if: "$stepResult.round_2.result.fix != true" + execution_strategy: + type: fail + output_key: fixed_2 + config: + tool: profile_fix_settle + arguments: + repo_path: "$input.repo_path" + story_key: "$input.story_key" + spec_path: "$input.spec_path" + branch: "$input.branch" + base_ref: "$input.base_ref" + change_id: "$input.change_id" + prepare: "$stepResult.round_2.result" + - id: verify_2 + primitive: call-agent + if: "$stepResult.fixed_2.result.verify == true" + execution_strategy: + type: skip + config: + workflow: verify-change.yaml + inputs: + repo_path: "$input.repo_path" + change_id: "$input.change_id" + base: "$stepResult.fixed_2.result.base" + revision: "$stepResult.fixed_2.result.head" + inventory_version: "$input.inventory_version" + - id: close + primitive: tool + execution_strategy: + type: fail + config: + tool: profile_fix_reconcile + arguments: + repo_path: "$input.repo_path" + story_key: "$input.story_key" + spec_path: "$input.spec_path" + branch: "$input.branch" + base_ref: "$input.base_ref" + change_id: "$input.change_id" + max_fix_rounds: 2 + prior: "$stepResult.fixed_2.result" + - id: record + primitive: tool + execution_strategy: + type: fail + config: + tool: profile_record_stage_outcome + arguments: + repo_path: "$input.repo_path" + story_key: "$stepResult.close.result.story_key" + stage: verify-and-fix + outcome: "$stepResult.close.result.outcome" + reason: "$stepResult.close.result.reason" + refs: "$stepResult.close.result.refs"