diff --git a/skills/rudder/SKILL.md b/skills/rudder/SKILL.md index a973e70..2ec49b3 100644 --- a/skills/rudder/SKILL.md +++ b/skills/rudder/SKILL.md @@ -127,6 +127,62 @@ For every new or changed expectation: Otherwise, run the applicable related and full test commands. 5. Measure coverage only after the suite is green. +## Prepare the prompt report + +End every Rudder test-generation run by preparing a temporary Markdown report. +Create or refresh it only after all rewrites have joined and the final test results are known. +Do not create it while a user answer or rewrite is pending. +If generation ends early because intent is missing, declined, or not captured, or because the user stops the flow, still create the stopped-run report. + +1. Rerun `scripts/context.mjs` so the report includes the latest captured follow-up answers: + + ```text + node /scripts/context.mjs \ + --cwd \ + --phase refresh \ + --run-id \ + --base + ``` + +2. Inspect the final affected test paths. + Include each generated, rewritten, or restored test case only when it still has an immediately preceding Rudder source-intent tag. + Match that exact `//` tag to a captured prompt record; use the tag, not memory or test-file proximity, to choose the prompt group. +3. Consolidate the included tests into one group per prompt. + List every final test case once under its matched prompt, and omit prompts that inspired no final test. + Use each test's human-readable title or description, never its code body. + Copy `promptText` exactly from the matched database record; do not truncate, summarize, or paraphrase it. + Make a concise, faithful representation of `previousAgentOutput`. + Do not reproduce long raw previous output or add context from model memory, the implementation, or the test diff. + When `previousAgentOutput` is null, use `N/A` as the context representation. +4. Write the report to a uniquely named Markdown file in the operating system's temporary directory, outside the repository worktree. + Never stage it or include it among the repository files changed. + Use this shape: + + ```markdown + # The tests your prompts inspired: + + ## Prompt 1 + + | Exact prompt text | Prior agent context | + | --- | --- | + | | | + + - [ (:)]() + - [ (:)]() + + ## Prompt 2 + + | Exact prompt text | Prior agent context | + | --- | --- | + | | | + + - [ (:)]() + ``` + + Escape table-cell, test-title, and path text when needed to keep the Markdown valid. + Format every bullet's visible text as ` (:)`. + Link that entire text to the test case's starting line when the host supports local file links; otherwise render the same text without a link. + If there are no final tagged tests, write `No prompt-backed tests were generated.` below the report heading. ## Run the workflow 1. Determine the repository root and target branch from the request. @@ -206,7 +262,8 @@ For every new or changed expectation: Continue asking independent questions until the batch must join. After joining, run the combined suites and coverage before selecting another uncovered behavior. Continue until the target passes or the user tells you to stop the flow. -12. Before the final report, record the verified Rudder outcome: +12. Prepare the final report with the prompt report as its lead item, but do not send it. +13. After the final report is prepared and before sending it, record the verified Rudder outcome: ```text node /scripts/telemetry.mjs complete \ @@ -219,9 +276,12 @@ For every new or changed expectation: --questions-asked ``` - Use `completed` when the workflow reaches its normal report, `stopped` when it ends by user choice or missing intent, and `blocked` only for an external blocker. + Use `completed` when generation finishes normally and the report is ready to present. + Use `stopped` when generation ends by user choice or missing intent and the stopped-run report is ready to present. + Use `blocked` only for an external blocker, including failure to create the temporary report. Set test and coverage values only from command output already observed during this run. Telemetry is best-effort; do not change the workflow result if this helper is unavailable. +14. Send the prepared final report, showing the prompt report contents and its temporary file path or local file link. Report the requirements derived from intent and all files changed. Report commands run, coverage, unanswered ambiguities, and the backup location. diff --git a/skills/rudder/scripts/context.mjs b/skills/rudder/scripts/context.mjs index c7e7918..1e4904c 100644 --- a/skills/rudder/scripts/context.mjs +++ b/skills/rudder/scripts/context.mjs @@ -85,6 +85,7 @@ function storedPrompts(repository, branch) { session_id AS sessionId, prompt_id AS promptId, prompt_text AS promptText, + previous_agent_output AS previousAgentOutput, submitted_at AS submittedAt, reconciled_at AS reconciledAt FROM prompt_branches diff --git a/test/skill-runtime.test.ts b/test/skill-runtime.test.ts index a669ce4..84aa462 100644 --- a/test/skill-runtime.test.ts +++ b/test/skill-runtime.test.ts @@ -182,11 +182,28 @@ test('data controls do not permit disabling prompt capture', () => { // codex/019fb36f-4dfe-7c91-8674-5caaf68fcced/019fb39d-12e9-7673-b7d7-04d6c3f27243 test('the skill helper returns intent, run identity, and initial test lines', () => { const originalCaptureDisabled = process.env.RUDDER_DISABLE_PROMPT_CAPTURE; + const transcriptPath = join(root, 'skill-context.jsonl'); mkdirSync(stateRoot, { recursive: true }); writeFileSync( join(stateRoot, 'prompt-capture-disabled'), 'legacy preference\n' ); + writeFileSync( + transcriptPath, + JSON.stringify({ + type: 'response_item', + payload: { + type: 'message', + role: 'assistant', + content: [ + { + type: 'output_text', + text: 'The request currently throws when the cache times out.', + }, + ], + }, + }) + ); process.env.RUDDER_DISABLE_PROMPT_CAPTURE = '1'; try { assert.notEqual( @@ -195,6 +212,7 @@ test('the skill helper returns intent, run identity, and initial test lines', () session_id: 'skill-context', turn_id: 'skill-turn', prompt: 'Return cached data when the request times out.', + transcript_path: transcriptPath, cwd: repo, }), null @@ -241,6 +259,7 @@ test('the skill helper returns intent, run identity, and initial test lines', () sessionId: string; promptId: string; promptText: string; + previousAgentOutput: string | null; }>; }; @@ -267,6 +286,10 @@ test('the skill helper returns intent, run identity, and initial test lines', () context.prompts[0]?.promptText, 'Return cached data when the request times out.' ); + assert.equal( + context.prompts[0]?.previousAgentOutput, + 'The request currently throws when the cache times out.' + ); }); // codex/019faf66-7413-7a31-a0ea-b5fe1c9b66d6/019faf8b-5c9c-7962-b60a-19e540b127d5