From a01d0feac6f4bc143d9422e5b058ecde66dcc6a5 Mon Sep 17 00:00:00 2001 From: Vivek Yanamadula Date: Thu, 30 Jul 2026 13:18:41 -0400 Subject: [PATCH 1/3] =?UTF-8?q?=E2=9C=A8=20Add=20per-prompt=20Rudder=20tes?= =?UTF-8?q?t=20reports?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Codex Agent --- skills/rudder/SKILL.md | 59 +++++++++++++++++++++++++++++++ skills/rudder/scripts/context.mjs | 1 + test/skill-runtime.test.ts | 23 ++++++++++++ 3 files changed, 83 insertions(+) diff --git a/skills/rudder/SKILL.md b/skills/rudder/SKILL.md index a973e70..1f5f4e6 100644 --- a/skills/rudder/SKILL.md +++ b/skills/rudder/SKILL.md @@ -127,6 +127,64 @@ For every new or changed expectation: Otherwise, run the applicable related and full test commands. 5. Measure coverage only after the suite is green. +## Present the prompt report + +End every Rudder test-generation run by presenting a temporary Markdown report. +Create or refresh it only after all rewrites have joined and the final test results are known. +Do not create it while a user answer or rewrite is pending. + +1. Rerun `scripts/context.mjs` so the report includes the latest captured follow-up answers: + + ```text + node /scripts/context.mjs \ + --cwd \ + --phase refresh \ + --run-id \ + --base + ``` + +2. Inspect the final affected test paths. + Include each generated, rewritten, or restored test case only when it still has an immediately preceding Rudder source-intent tag. + Match that exact `//` tag to a captured prompt record; use the tag, not memory or test-file proximity, to choose the prompt group. +3. Consolidate the included tests into one group per prompt. + List every final test case once under its matched prompt, and omit prompts that inspired no final test. + Use each test's human-readable title or description, never its code body. + Copy `promptText` exactly from the matched database record; do not truncate, summarize, or paraphrase it. + Make a concise, faithful representation of `previousAgentOutput`. + Do not reproduce long raw previous output or add context from model memory, the implementation, or the test diff. + When `previousAgentOutput` is null, use `N/A` as the context representation. +4. Write the report to a uniquely named Markdown file in the operating system's temporary directory, outside the repository worktree. + Never stage it or include it among the repository files changed. + Use this shape: + + ```markdown + # The tests your prompts inspired: + + ## Prompt 1 + + | Exact prompt text | Prior agent context | + | --- | --- | + | | | + + - [ (:)]() + - [ (:)]() + + ## Prompt 2 + + | Exact prompt text | Prior agent context | + | --- | --- | + | | | + + - [ (:)]() + ``` + + Escape table-cell, test-title, and path text when needed to keep the Markdown valid. + Format every bullet's visible text as ` (:)`. + Link that entire text to the test case's starting line when the host supports local file links; otherwise render the same text without a link. + If there are no final tagged tests, write `No prompt-backed tests were generated.` below the report heading. +5. Make the report the lead item in the final response. + Show its contents and provide its temporary file path or local file link. + ## Run the workflow 1. Determine the repository root and target branch from the request. @@ -222,6 +280,7 @@ For every new or changed expectation: Use `completed` when the workflow reaches its normal report, `stopped` when it ends by user choice or missing intent, and `blocked` only for an external blocker. Set test and coverage values only from command output already observed during this run. Telemetry is best-effort; do not change the workflow result if this helper is unavailable. +13. After the generation loop ends, create and present the prompt report. Report the requirements derived from intent and all files changed. Report commands run, coverage, unanswered ambiguities, and the backup location. diff --git a/skills/rudder/scripts/context.mjs b/skills/rudder/scripts/context.mjs index c7e7918..1e4904c 100644 --- a/skills/rudder/scripts/context.mjs +++ b/skills/rudder/scripts/context.mjs @@ -85,6 +85,7 @@ function storedPrompts(repository, branch) { session_id AS sessionId, prompt_id AS promptId, prompt_text AS promptText, + previous_agent_output AS previousAgentOutput, submitted_at AS submittedAt, reconciled_at AS reconciledAt FROM prompt_branches diff --git a/test/skill-runtime.test.ts b/test/skill-runtime.test.ts index a669ce4..84aa462 100644 --- a/test/skill-runtime.test.ts +++ b/test/skill-runtime.test.ts @@ -182,11 +182,28 @@ test('data controls do not permit disabling prompt capture', () => { // codex/019fb36f-4dfe-7c91-8674-5caaf68fcced/019fb39d-12e9-7673-b7d7-04d6c3f27243 test('the skill helper returns intent, run identity, and initial test lines', () => { const originalCaptureDisabled = process.env.RUDDER_DISABLE_PROMPT_CAPTURE; + const transcriptPath = join(root, 'skill-context.jsonl'); mkdirSync(stateRoot, { recursive: true }); writeFileSync( join(stateRoot, 'prompt-capture-disabled'), 'legacy preference\n' ); + writeFileSync( + transcriptPath, + JSON.stringify({ + type: 'response_item', + payload: { + type: 'message', + role: 'assistant', + content: [ + { + type: 'output_text', + text: 'The request currently throws when the cache times out.', + }, + ], + }, + }) + ); process.env.RUDDER_DISABLE_PROMPT_CAPTURE = '1'; try { assert.notEqual( @@ -195,6 +212,7 @@ test('the skill helper returns intent, run identity, and initial test lines', () session_id: 'skill-context', turn_id: 'skill-turn', prompt: 'Return cached data when the request times out.', + transcript_path: transcriptPath, cwd: repo, }), null @@ -241,6 +259,7 @@ test('the skill helper returns intent, run identity, and initial test lines', () sessionId: string; promptId: string; promptText: string; + previousAgentOutput: string | null; }>; }; @@ -267,6 +286,10 @@ test('the skill helper returns intent, run identity, and initial test lines', () context.prompts[0]?.promptText, 'Return cached data when the request times out.' ); + assert.equal( + context.prompts[0]?.previousAgentOutput, + 'The request currently throws when the cache times out.' + ); }); // codex/019faf66-7413-7a31-a0ea-b5fe1c9b66d6/019faf8b-5c9c-7962-b60a-19e540b127d5 From f35f2fc550cb0e24d733fcf0a4f14af676e34358 Mon Sep 17 00:00:00 2001 From: Vivek Yanamadula Date: Thu, 30 Jul 2026 13:50:29 -0400 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=90=9B=20Order=20Rudder=20reporting?= =?UTF-8?q?=20before=20telemetry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Codex Agent --- skills/rudder/SKILL.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/skills/rudder/SKILL.md b/skills/rudder/SKILL.md index 1f5f4e6..6e2eeb7 100644 --- a/skills/rudder/SKILL.md +++ b/skills/rudder/SKILL.md @@ -132,6 +132,7 @@ For every new or changed expectation: End every Rudder test-generation run by presenting a temporary Markdown report. Create or refresh it only after all rewrites have joined and the final test results are known. Do not create it while a user answer or rewrite is pending. +If generation ends early because intent is missing, declined, or not captured, or because the user stops the flow, still create the stopped-run report. 1. Rerun `scripts/context.mjs` so the report includes the latest captured follow-up answers: @@ -182,7 +183,7 @@ Do not create it while a user answer or rewrite is pending. Format every bullet's visible text as ` (:)`. Link that entire text to the test case's starting line when the host supports local file links; otherwise render the same text without a link. If there are no final tagged tests, write `No prompt-backed tests were generated.` below the report heading. -5. Make the report the lead item in the final response. +5. After completion telemetry is recorded, make the report the lead item in the final response. Show its contents and provide its temporary file path or local file link. ## Run the workflow @@ -264,7 +265,9 @@ Do not create it while a user answer or rewrite is pending. Continue asking independent questions until the batch must join. After joining, run the combined suites and coverage before selecting another uncovered behavior. Continue until the target passes or the user tells you to stop the flow. -12. Before the final report, record the verified Rudder outcome: +12. After the generation loop ends, including an early stop caused by user choice or missing intent, create the prompt report through step 4 of `Present the prompt report`. + Do not send the final response yet. +13. After the report file is ready and before sending the final response, record the verified Rudder outcome: ```text node /scripts/telemetry.mjs complete \ @@ -277,10 +280,12 @@ Do not create it while a user answer or rewrite is pending. --questions-asked ``` - Use `completed` when the workflow reaches its normal report, `stopped` when it ends by user choice or missing intent, and `blocked` only for an external blocker. + Use `completed` when generation finishes normally and the report is ready to present. + Use `stopped` when generation ends by user choice or missing intent and the stopped-run report is ready to present. + Use `blocked` only for an external blocker, including failure to create the temporary report. Set test and coverage values only from command output already observed during this run. Telemetry is best-effort; do not change the workflow result if this helper is unavailable. -13. After the generation loop ends, create and present the prompt report. +14. Present the prompt report as the lead item in the final response. Report the requirements derived from intent and all files changed. Report commands run, coverage, unanswered ambiguities, and the backup location. From c011c1af95fcb527dbd1690142e47bee3c95e9e2 Mon Sep 17 00:00:00 2001 From: Vivek Yanamadula Date: Thu, 30 Jul 2026 13:57:06 -0400 Subject: [PATCH 3/3] =?UTF-8?q?=E2=9C=8F=EF=B8=8F=20Simplify=20prompt=20re?= =?UTF-8?q?port=20instructions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Codex Agent --- skills/rudder/SKILL.md | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/skills/rudder/SKILL.md b/skills/rudder/SKILL.md index 6e2eeb7..2ec49b3 100644 --- a/skills/rudder/SKILL.md +++ b/skills/rudder/SKILL.md @@ -127,9 +127,9 @@ For every new or changed expectation: Otherwise, run the applicable related and full test commands. 5. Measure coverage only after the suite is green. -## Present the prompt report +## Prepare the prompt report -End every Rudder test-generation run by presenting a temporary Markdown report. +End every Rudder test-generation run by preparing a temporary Markdown report. Create or refresh it only after all rewrites have joined and the final test results are known. Do not create it while a user answer or rewrite is pending. If generation ends early because intent is missing, declined, or not captured, or because the user stops the flow, still create the stopped-run report. @@ -183,9 +183,6 @@ If generation ends early because intent is missing, declined, or not captured, o Format every bullet's visible text as ` (:)`. Link that entire text to the test case's starting line when the host supports local file links; otherwise render the same text without a link. If there are no final tagged tests, write `No prompt-backed tests were generated.` below the report heading. -5. After completion telemetry is recorded, make the report the lead item in the final response. - Show its contents and provide its temporary file path or local file link. - ## Run the workflow 1. Determine the repository root and target branch from the request. @@ -265,9 +262,8 @@ If generation ends early because intent is missing, declined, or not captured, o Continue asking independent questions until the batch must join. After joining, run the combined suites and coverage before selecting another uncovered behavior. Continue until the target passes or the user tells you to stop the flow. -12. After the generation loop ends, including an early stop caused by user choice or missing intent, create the prompt report through step 4 of `Present the prompt report`. - Do not send the final response yet. -13. After the report file is ready and before sending the final response, record the verified Rudder outcome: +12. Prepare the final report with the prompt report as its lead item, but do not send it. +13. After the final report is prepared and before sending it, record the verified Rudder outcome: ```text node /scripts/telemetry.mjs complete \ @@ -285,7 +281,7 @@ If generation ends early because intent is missing, declined, or not captured, o Use `blocked` only for an external blocker, including failure to create the temporary report. Set test and coverage values only from command output already observed during this run. Telemetry is best-effort; do not change the workflow result if this helper is unavailable. -14. Present the prompt report as the lead item in the final response. +14. Send the prepared final report, showing the prompt report contents and its temporary file path or local file link. Report the requirements derived from intent and all files changed. Report commands run, coverage, unanswered ambiguities, and the backup location.