diff --git a/plugins/aidd-dev/skills/05-review/actions/03-review-functional.md b/plugins/aidd-dev/skills/05-review/actions/03-review-functional.md index 4d459dc52..74cf6327e 100644 --- a/plugins/aidd-dev/skills/05-review/actions/03-review-functional.md +++ b/plugins/aidd-dev/skills/05-review/actions/03-review-functional.md @@ -4,7 +4,7 @@ Trace the diff against the plan's acceptance criteria. ## Input -The plan, its phase files, and the diff prepare resolved. +The plan, its phase files, any run recorded beside them, and the diff prepare resolved. ## Output @@ -18,16 +18,17 @@ The round's criteria and score. - A phase whose `## Architecture projection` files the diff leaves untouched: out of the diff, no box. A phase projecting no file is traced, never out of the diff. - Met by the files as the diff leaves them, an added line or a context line: checked. - Unmet, partial, its subject absent, or a name echoing it with no behaviour behind: unchecked, never a finding. - - Its subject there, its property one no diff could ever show: checked, not-applicable. -3. **Score.** Count met + unmet + out of the diff = the plan's whole criteria total. - - A not-applicable criterion counts as met. + - Its subject there, its outcome one only a run could show, what a screen displays, a timing, an outside system's answer: unverified, never checked from the code. + - A run recorded beside the plan on the reviewed head shows it passing: checked, citing the record; failing: unchecked; recorded on another head: unverified. +3. **Score.** Count met + unmet + unverified + out of the diff = the plan's whole criteria total. ## Test | Case | Pass | | --- | --- | -| A plan of 6 criteria, the diff touching one phase of 2 with 1 met | `1/6 met, 1 unmet, 4 out of the diff`, the untouched phases out of the diff in plan order | +| A plan of 6 criteria, the diff touching one phase of 2 with 1 met | `1/6 met, 1 unmet, 0 unverified at runtime, 4 out of the diff`, the untouched phases out of the diff in plan order | | A criterion unmet, its subject absent, or a name echoing it with no behaviour behind | unchecked, naming the gap, never a finding | -| A criterion whose property no diff could show | checked, not-applicable, counted as met | +| A criterion only a run could show, and no run recorded | unverified, never checked | +| The same, a recorded run passing it | checked, citing the record | | No plan and no criteria given | not scored, and no criteria | | A second round over the same plan | every box follows the current diff, none carried from the earlier round | diff --git a/plugins/aidd-dev/skills/05-review/assets/review-template.md b/plugins/aidd-dev/skills/05-review/assets/review-template.md index b3a51292d..9d097eca8 100644 --- a/plugins/aidd-dev/skills/05-review/assets/review-template.md +++ b/plugins/aidd-dev/skills/05-review/assets/review-template.md @@ -10,13 +10,13 @@ - Diff: `{{base}}...{{head}}`{{ plus the working tree, when it held changes}} - Axes: {{the axes that ran, from code, functional, relevancy}} - Verdict: {{verdict}} -- Score: {{n_met}}/{{n_plan}} met, {{n_unmet}} unmet, {{n_out}} out of the diff, or not scored +- Score: {{n_met}}/{{n_plan}} met, {{n_unmet}} unmet, {{n_unverified}} unverified at runtime, {{n_out}} out of the diff, or not scored ### Criteria -- [x] {{criterion met}} — {{file:line}} +- [x] {{criterion met}} — {{file:line, or the run record}} - [ ] {{criterion unmet}} — {{gap}} -- [x] {{criterion no diff could show}} — not-applicable, {{why}} +- Unverified at runtime: {{criterion}} — {{what a run would show}} - Out of the diff: {{phase-name}} ({{n}} criteria) ### Findings diff --git a/plugins/aidd-dev/skills/05-review/references/report-contract.md b/plugins/aidd-dev/skills/05-review/references/report-contract.md index 7b7b96682..dfd33a699 100644 --- a/plugins/aidd-dev/skills/05-review/references/report-contract.md +++ b/plugins/aidd-dev/skills/05-review/references/report-contract.md @@ -23,5 +23,5 @@ Where a review lands, what it reviews, and how a round joins it. | Rule | Why | | --- | --- | -| Append a round, derived from the plan and the current diff alone | a round states what this diff shows, and nothing it inherited | +| Append a round, derived from the plan, the current diff and a run recorded on its head alone | a round states what this candidate shows, and nothing it inherited | | Count the `## Round` headings, never read their bodies | an earlier round's boxes describe a diff that no longer exists, and copying one is the defect | diff --git a/plugins/aidd-dev/skills/05-review/references/review-rubric.md b/plugins/aidd-dev/skills/05-review/references/review-rubric.md index ef44734e9..523b9c357 100644 --- a/plugins/aidd-dev/skills/05-review/references/review-rubric.md +++ b/plugins/aidd-dev/skills/05-review/references/review-rubric.md @@ -21,4 +21,5 @@ One overall verdict, the strictest across the axes run. | Case | Counts as | | --- | --- | | A criterion out of the diff | neither unchecked nor blocking | +| A criterion unverified at runtime | in scope, neither unchecked nor blocking | | A round without functional | a verdict from its findings alone | diff --git a/scripts/__tests__/a-review-appends-a-round-and-scores-the-whole-plan.test.js b/scripts/__tests__/a-review-appends-a-round-and-scores-the-whole-plan.test.js index d32b5c939..22252f852 100644 --- a/scripts/__tests__/a-review-appends-a-round-and-scores-the-whole-plan.test.js +++ b/scripts/__tests__/a-review-appends-a-round-and-scores-the-whole-plan.test.js @@ -49,7 +49,7 @@ function inSkill(file) { describe("a review appends a round and scores the whole plan", () => { it("a round is appended, and an earlier one is never read back", () => { const reference = read("references/report-contract.md"); - assert.match(reference, /Append a round, derived from the plan and the current diff alone/u); + assert.match(reference, /Append a round, derived from the plan, the current diff and a run recorded on its head alone/u); assert.match( reference, /Count the `## Round` headings, never read their bodies/u, @@ -205,10 +205,21 @@ describe("a review appends a round and scores the whole plan", () => { ); }); - it("a criterion no diff could show is not-applicable, not a gap", () => { + it("a criterion only a run could show is unverified, never met on the code alone", () => { + const trace = step("actions/03-review-functional.md", "Trace"); + assert.match(trace, /its outcome one only a run could show[^\n]*: unverified, never checked from the code/u); + assert.match(trace, /only a run could show, what a screen displays, a timing, an outside system's answer:/u, "a display is a run's to show, even beside a timing"); + assert.match(trace, /A run recorded beside the plan on the reviewed head shows it passing: checked, citing the record; failing: unchecked; recorded on another head: unverified/u); + const strays = everyMarkdown(SKILL, []) + .filter((file) => /not-applicable/u.test(fs.readFileSync(file, "utf8"))) + .map(inSkill); + assert.deepEqual(strays, [], "one state for a criterion the code cannot prove"); + }); + + it("an unverified criterion neither blocks nor asks for changes", () => { assert.match( - step("actions/03-review-functional.md", "Trace"), - /Its subject there, its property one no diff could ever show: checked, not-applicable/u, + read("references/review-rubric.md"), + /^\| A criterion unverified at runtime \| in scope, neither unchecked nor blocking \|$/mu, ); }); @@ -265,14 +276,13 @@ describe("a review appends a round and scores the whole plan", () => { it("the template renders every form a round can hold", () => { const template = read("assets/review-template.md"); assert.match(template, /or not scored/u); - assert.match(template, /not-applicable, \{\{why\}\}/u); + assert.match(template, /^- Unverified at runtime: \{\{criterion\}\} — \{\{what a run would show\}\}$/mu); }); it("the score counts over the whole plan, and the template holds its form", () => { const score = step("actions/03-review-functional.md", "Score"); assert.match(score, /whole criteria total/u, "the count is over the plan, not over what is in scope"); - assert.match(score, /met \+ unmet \+ out of the diff = the plan's whole criteria total/u, "and the three numbers are checkable against it"); - assert.match(score, /counts as met/u, "and a not-applicable criterion is counted"); + assert.match(score, /met \+ unmet \+ unverified \+ out of the diff = the plan's whole criteria total/u, "and the four numbers are checkable against it"); assert.doesNotMatch(score, /\{\{/u, "a rule names no template placeholder"); assert.match( step("actions/03-review-functional.md", "Read"), @@ -281,7 +291,7 @@ describe("a review appends a round and scores the whole plan", () => { ); assert.match( read("assets/review-template.md"), - /\{\{n_met\}\}\/\{\{n_plan\}\} met, \{\{n_unmet\}\} unmet, \{\{n_out\}\} out of the diff/u, + /\{\{n_met\}\}\/\{\{n_plan\}\} met, \{\{n_unmet\}\} unmet, \{\{n_unverified\}\} unverified at runtime, \{\{n_out\}\} out of the diff/u, "the form is rendered by the template, not spelled out in a rule", ); });