diff --git a/resources/extensions/glm-loop/index.ts b/resources/extensions/glm-loop/index.ts index 42e97a6..5fe023e 100644 --- a/resources/extensions/glm-loop/index.ts +++ b/resources/extensions/glm-loop/index.ts @@ -15,6 +15,18 @@ type LoopProfileName = "code"; type LoopFailureMode = "handoff" | "fail"; type LoopExecutionStatus = "succeeded" | "handoff" | "failed"; type LoopPhase = "run" | "verify" | "repair"; +type VerificationArtifactReference = { + kind: "verification"; + id: string; + path: string; + createdAt: string; + scenario?: string; + command?: string; + exitCode?: number; + summary: string; + stdoutSummary?: string; + stderrSummary?: string; +}; type VerificationResult = | { kind: "pass"; @@ -22,6 +34,7 @@ type VerificationResult = exitCode: 0; summary: string; artifactPath?: string; + artifactRef?: VerificationArtifactReference; stdout?: string; stderr?: string; } @@ -31,6 +44,7 @@ type VerificationResult = exitCode: number; summary: string; artifactPath?: string; + artifactRef?: VerificationArtifactReference; stdout?: string; stderr?: string; } @@ -39,6 +53,7 @@ type VerificationResult = command?: string; summary: string; artifactPath?: string; + artifactRef?: VerificationArtifactReference; stdout?: string; stderr?: string; }; @@ -71,6 +86,7 @@ type LoopVerificationRecord = { command?: string; exitCode?: number; summary: string; + artifact?: VerificationArtifactReference; artifactPath?: string; stdoutSummary?: string; stderrSummary?: string; @@ -228,6 +244,7 @@ function isLoopResultRecord(value: unknown): value is LoopResultRecord { if (!value || typeof value !== "object") return false; const maybe = value as Partial; const verification = maybe.verification as Partial | undefined; + const artifact = verification?.artifact as Partial | undefined; return ( (maybe.status === "succeeded" || maybe.status === "handoff" || maybe.status === "failed") && typeof maybe.task === "string" && @@ -242,6 +259,16 @@ function isLoopResultRecord(value: unknown): value is LoopResultRecord { (verification.command === undefined || typeof verification.command === "string") && (verification.exitCode === undefined || typeof verification.exitCode === "number") && typeof verification.summary === "string" && + (!artifact || + (artifact.kind === "verification" && + typeof artifact.id === "string" && + typeof artifact.path === "string" && + typeof artifact.createdAt === "string" && + (artifact.command === undefined || typeof artifact.command === "string") && + (artifact.exitCode === undefined || typeof artifact.exitCode === "number") && + typeof artifact.summary === "string" && + (artifact.stdoutSummary === undefined || typeof artifact.stdoutSummary === "string") && + (artifact.stderrSummary === undefined || typeof artifact.stderrSummary === "string"))) && (verification.artifactPath === undefined || typeof verification.artifactPath === "string") && (verification.stdoutSummary === undefined || typeof verification.stdoutSummary === "string") && (verification.stderrSummary === undefined || typeof verification.stderrSummary === "string") && @@ -354,18 +381,18 @@ function summarizeOutputText( } function createLoopVerificationRecord(result: VerificationResult): LoopVerificationRecord { + const stdoutSummary = result.artifactRef?.stdoutSummary ?? summarizeOutputText(result.stdout); + const stderrSummary = result.artifactRef?.stderrSummary ?? summarizeOutputText(result.stderr); + return { kind: result.kind, ...(result.command ? { command: result.command } : {}), ...(result.exitCode === undefined ? {} : { exitCode: result.exitCode }), summary: result.summary, + ...(result.artifactRef ? { artifact: result.artifactRef } : {}), ...(result.artifactPath ? { artifactPath: result.artifactPath } : {}), - ...(summarizeOutputText(result.stdout) - ? { stdoutSummary: summarizeOutputText(result.stdout) } - : {}), - ...(summarizeOutputText(result.stderr) - ? { stderrSummary: summarizeOutputText(result.stderr) } - : {}), + ...(stdoutSummary ? { stdoutSummary } : {}), + ...(stderrSummary ? { stderrSummary } : {}), }; } @@ -455,6 +482,9 @@ function buildLastResultLines(result: LoopResultRecord): string[] { result.verification.command ? `Last verifier: ${result.verification.command}` : undefined, `Last verification: ${result.verification.kind}`, `Last summary: ${result.verification.summary}`, + result.verification.artifact + ? `Last artifact: verification | ${result.verification.artifact.path}` + : undefined, result.verification.stdoutSummary ? `Last stdout summary: ${result.verification.stdoutSummary}` : undefined, @@ -504,6 +534,9 @@ function buildShowLines(results: LoopResultRecord[], index: number): string[] { ? [] : [`Exit code: ${result.verification.exitCode}`]), `Summary: ${result.verification.summary}`, + ...(result.verification.artifact + ? [`Artifact reference: verification | ${result.verification.artifact.path}`] + : []), ...(result.verification.stdoutSummary ? [`Stdout summary: ${result.verification.stdoutSummary}`] : []), @@ -700,6 +733,18 @@ function buildRepairPrompt(result: VerificationResult, nextRound: number): strin `Verification failed. Begin repair round ${nextRound}.`, result.command ? `Verifier: ${result.command}` : "Verifier: unavailable", `Summary: ${result.summary}`, + ...(result.artifactRef + ? [ + `Artifact reference: verification | ${result.artifactRef.path}`, + ...(result.artifactRef.stdoutSummary + ? [`Artifact stdout summary: ${result.artifactRef.stdoutSummary}`] + : []), + ...(result.artifactRef.stderrSummary + ? [`Artifact stderr summary: ${result.artifactRef.stderrSummary}`] + : []), + "Use the artifact summary first. Inspect the artifact file only if you need full verifier output.", + ] + : []), "", "Instructions:", "- Fix only the failure reported by the verifier.", @@ -734,6 +779,18 @@ function buildFailureSummary(args: { `Rounds attempted: ${args.rounds.length}`, args.lastResult.command ? `Last verifier: ${args.lastResult.command}` : undefined, `Last result: ${args.lastResult.summary}`, + args.lastResult.artifactRef + ? `Artifact reference: verification | ${args.lastResult.artifactRef.path}` + : undefined, + args.lastResult.artifactRef?.stdoutSummary + ? `Artifact stdout summary: ${args.lastResult.artifactRef.stdoutSummary}` + : undefined, + args.lastResult.artifactRef?.stderrSummary + ? `Artifact stderr summary: ${args.lastResult.artifactRef.stderrSummary}` + : undefined, + args.lastResult.artifactRef + ? "Use the artifact summary first. Inspect the artifact file only if deeper verifier output is required." + : undefined, "Recommended next step: inspect the latest verifier output, apply a focused fix, and rerun verification.", ] .filter(Boolean) @@ -866,7 +923,11 @@ async function runVerification( resolution: { kind: "command", command, source: "loop" }, verification, }); - return { ...verification, artifactPath: artifact.artifactPath }; + return { + ...verification, + artifactPath: artifact.artifactPath, + artifactRef: artifact.artifactRef, + }; } const verification: VerificationResult = { @@ -882,7 +943,11 @@ async function runVerification( resolution: { kind: "command", command, source: "loop" }, verification, }); - return { ...verification, artifactPath: artifact.artifactPath }; + return { + ...verification, + artifactPath: artifact.artifactPath, + artifactRef: artifact.artifactRef, + }; } async function resolveVerifier( diff --git a/src/commands/verify.ts b/src/commands/verify.ts index 8321121..287aae2 100644 --- a/src/commands/verify.ts +++ b/src/commands/verify.ts @@ -1,8 +1,12 @@ import { readConfigFile, type GlmConfigFile } from "../app/config-store.js"; +import { + formatVerificationArtifactReference, + writeVerificationArtifact, + type VerificationArtifact, +} from "../harness/artifacts.js"; import { detectCodeVerifier } from "../loop/verify-detect.js"; import { runVerificationCommand } from "../loop/verify-runner.js"; import type { VerificationCommandResolution, VerificationResult } from "../loop/types.js"; -import { writeVerificationArtifact, type VerificationArtifact } from "../harness/artifacts.js"; import { resolveVerifyScenario, type VerifyScenarioName } from "../harness/scenarios.js"; export type VerifyCommandArgs = { @@ -18,6 +22,7 @@ type VerifyResolution = { verification: VerificationResult; artifact: VerificationArtifact; artifactPath: string; + artifactRef: NonNullable; }; type VerifyDependencies = { @@ -33,27 +38,6 @@ type VerifyDependencies = { writeArtifact: typeof writeVerificationArtifact; }; -function summarizeOutputText( - value: string | undefined, - maxLines = 2, - maxChars = 160, -): string | undefined { - if (!value) return undefined; - const lines = value - .split(/\r?\n/) - .map((line) => line.trim()) - .filter(Boolean) - .slice(0, maxLines); - if (lines.length === 0) return undefined; - - const summary = lines.join(" | "); - if (summary.length <= maxChars) { - return summary; - } - - return `${summary.slice(0, maxChars - 1)}…`; -} - async function resolveVerifier(args: { cwd: string; explicit?: string; @@ -115,7 +99,11 @@ export async function verifyProject( return { resolution, - verification, + verification: { + ...verification, + artifactPath: artifactResult.artifactPath, + artifactRef: artifactResult.artifactRef, + }, ...artifactResult, }; } @@ -132,7 +120,15 @@ export async function verifyProject( verification, }); - return { resolution, verification, ...artifactResult }; + return { + resolution, + verification: { + ...verification, + artifactPath: artifactResult.artifactPath, + artifactRef: artifactResult.artifactRef, + }, + ...artifactResult, + }; } export async function runVerifyCommand( @@ -141,25 +137,22 @@ export async function runVerifyCommand( ): Promise { const log = deps?.log ?? console.log; const { resolution, verification, artifact, artifactPath } = await verifyProject(input, deps); + const artifactRef = verification.artifactRef; if (input.json) { - log(JSON.stringify({ resolution, verification, artifact, artifactPath }, null, 2)); + log(JSON.stringify({ resolution, verification, artifact, artifactPath, artifactRef }, null, 2)); return verification.kind === "pass" ? 0 : 1; } - const stdoutSummary = summarizeOutputText(verification.stdout); - const stderrSummary = summarizeOutputText(verification.stderr); - const lines = [ resolution.kind === "command" ? `Verifier: ${resolution.command} (${resolution.source})` : `Verifier: ${resolution.summary}`, `Result: ${verification.kind}`, `Summary: ${verification.summary}`, - ...(verification.exitCode === undefined ? [] : [`Exit code: ${verification.exitCode}`]), - `Artifact: ${artifactPath}`, - ...(stdoutSummary ? [`Stdout summary: ${stdoutSummary}`] : []), - ...(stderrSummary ? [`Stderr summary: ${stderrSummary}`] : []), + ...(artifactRef + ? formatVerificationArtifactReference(artifactRef) + : [`Artifact reference: verification | ${artifactPath}`]), ]; log(lines.join("\n")); diff --git a/src/harness/artifacts.ts b/src/harness/artifacts.ts index 17e8f80..8c90d75 100644 --- a/src/harness/artifacts.ts +++ b/src/harness/artifacts.ts @@ -1,6 +1,10 @@ import { mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import type { VerificationCommandResolution, VerificationResult } from "../loop/types.js"; +import type { + VerificationArtifactReference, + VerificationCommandResolution, + VerificationResult, +} from "../loop/types.js"; import { resolveGlmSessionPaths } from "../session/session-paths.js"; export type VerificationArtifact = { @@ -12,6 +16,13 @@ export type VerificationArtifact = { scenario?: string; resolution: VerificationCommandResolution; verification: VerificationResult; + digest: { + command?: string; + exitCode?: number; + summary: string; + stdoutSummary?: string; + stderrSummary?: string; + }; }; function createArtifactId(now = new Date()): string { @@ -20,13 +31,84 @@ function createArtifactId(now = new Date()): string { return `verify-${timestamp}-${suffix}`; } +function summarizeOutputText( + value: string | undefined, + maxLines = 2, + maxChars = 160, +): string | undefined { + if (!value) return undefined; + const lines = value + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean) + .slice(0, maxLines); + if (lines.length === 0) return undefined; + + const summary = lines.join(" | "); + if (summary.length <= maxChars) { + return summary; + } + + return `${summary.slice(0, maxChars - 1)}…`; +} + +export function buildVerificationArtifactReference(args: { + artifact: VerificationArtifact; + artifactPath: string; +}): VerificationArtifactReference { + return { + kind: "verification", + id: args.artifact.id, + path: args.artifactPath, + createdAt: args.artifact.createdAt, + ...(args.artifact.scenario ? { scenario: args.artifact.scenario } : {}), + ...(args.artifact.digest.command ? { command: args.artifact.digest.command } : {}), + ...(args.artifact.digest.exitCode === undefined + ? {} + : { exitCode: args.artifact.digest.exitCode }), + summary: args.artifact.digest.summary, + ...(args.artifact.digest.stdoutSummary + ? { stdoutSummary: args.artifact.digest.stdoutSummary } + : {}), + ...(args.artifact.digest.stderrSummary + ? { stderrSummary: args.artifact.digest.stderrSummary } + : {}), + }; +} + +export function formatVerificationArtifactReference(ref: VerificationArtifactReference): string[] { + return [ + `Artifact reference: verification | ${ref.path}`, + ref.command ? `Artifact verifier: ${ref.command}` : undefined, + ref.exitCode === undefined ? undefined : `Artifact exit code: ${ref.exitCode}`, + `Artifact summary: ${ref.summary}`, + ref.stdoutSummary ? `Artifact stdout summary: ${ref.stdoutSummary}` : undefined, + ref.stderrSummary ? `Artifact stderr summary: ${ref.stderrSummary}` : undefined, + ].filter((line): line is string => Boolean(line)); +} + export async function writeVerificationArtifact(args: { cwd: string; scenario?: string; resolution: VerificationCommandResolution; verification: VerificationResult; -}): Promise<{ artifact: VerificationArtifact; artifactPath: string }> { +}): Promise<{ + artifact: VerificationArtifact; + artifactPath: string; + artifactRef: VerificationArtifactReference; +}> { const id = createArtifactId(); + const digest = { + ...(args.verification.command ? { command: args.verification.command } : {}), + ...(args.verification.exitCode === undefined ? {} : { exitCode: args.verification.exitCode }), + summary: args.verification.summary, + ...(summarizeOutputText(args.verification.stdout) + ? { stdoutSummary: summarizeOutputText(args.verification.stdout) } + : {}), + ...(summarizeOutputText(args.verification.stderr) + ? { stderrSummary: summarizeOutputText(args.verification.stderr) } + : {}), + }; const artifact: VerificationArtifact = { kind: "verification", version: 1, @@ -36,12 +118,17 @@ export async function writeVerificationArtifact(args: { ...(args.scenario ? { scenario: args.scenario } : {}), resolution: args.resolution, verification: args.verification, + digest, }; const artifactDir = join(resolveGlmSessionPaths(args.cwd).sessionDir, "artifacts"); await mkdir(artifactDir, { recursive: true }); const artifactPath = join(artifactDir, `${id}.json`); await writeFile(artifactPath, `${JSON.stringify(artifact, null, 2)}\n`, "utf8"); + const artifactRef = buildVerificationArtifactReference({ + artifact, + artifactPath, + }); - return { artifact, artifactPath }; + return { artifact, artifactPath, artifactRef }; } diff --git a/src/loop/controller.ts b/src/loop/controller.ts index 99bd1ca..3b9dfed 100644 --- a/src/loop/controller.ts +++ b/src/loop/controller.ts @@ -61,6 +61,8 @@ export async function runLoopController( const budgetStop: VerificationResult = { kind: "unavailable", ...(verification.command ? { command: verification.command } : {}), + ...(verification.artifactPath ? { artifactPath: verification.artifactPath } : {}), + ...(verification.artifactRef ? { artifactRef: verification.artifactRef } : {}), summary: `Verification budget exceeded (maxVerifyRuns=${input.maxVerifyRuns}). Last result: ${verification.summary}`, }; diff --git a/src/loop/profiles/code.ts b/src/loop/profiles/code.ts index 57d5e69..f61cace 100644 --- a/src/loop/profiles/code.ts +++ b/src/loop/profiles/code.ts @@ -32,6 +32,18 @@ export function createCodeLoopProfile(promptMode: PromptMode = "intensive"): Loo `Rounds attempted: ${rounds.length}`, lastResult.command ? `Last verifier: ${lastResult.command}` : undefined, `Last result: ${lastResult.summary}`, + lastResult.artifactRef + ? `Artifact reference: verification | ${lastResult.artifactRef.path}` + : undefined, + lastResult.artifactRef?.stdoutSummary + ? `Artifact stdout summary: ${lastResult.artifactRef.stdoutSummary}` + : undefined, + lastResult.artifactRef?.stderrSummary + ? `Artifact stderr summary: ${lastResult.artifactRef.stderrSummary}` + : undefined, + lastResult.artifactRef + ? "Use the artifact summary first. Inspect the artifact file only if deeper verifier output is required." + : undefined, "Recommended next step: inspect the latest verifier output, apply a focused fix, and rerun verification.", ] .filter(Boolean) diff --git a/src/loop/types.ts b/src/loop/types.ts index 49b2059..e18e9df 100644 --- a/src/loop/types.ts +++ b/src/loop/types.ts @@ -1,3 +1,16 @@ +export type VerificationArtifactReference = { + kind: "verification"; + id: string; + path: string; + createdAt: string; + scenario?: string; + command?: string; + exitCode?: number; + summary: string; + stdoutSummary?: string; + stderrSummary?: string; +}; + export type VerificationResult = | { kind: "pass"; @@ -5,6 +18,7 @@ export type VerificationResult = exitCode: 0; summary: string; artifactPath?: string; + artifactRef?: VerificationArtifactReference; stdout?: string; stderr?: string; } @@ -14,6 +28,7 @@ export type VerificationResult = exitCode: number; summary: string; artifactPath?: string; + artifactRef?: VerificationArtifactReference; stdout?: string; stderr?: string; } @@ -23,6 +38,7 @@ export type VerificationResult = exitCode?: number; summary: string; artifactPath?: string; + artifactRef?: VerificationArtifactReference; stdout?: string; stderr?: string; }; diff --git a/src/prompt/verification-overlay.ts b/src/prompt/verification-overlay.ts index d32ce85..e72d969 100644 --- a/src/prompt/verification-overlay.ts +++ b/src/prompt/verification-overlay.ts @@ -5,6 +5,18 @@ export function buildVerificationOverlay(result: VerificationResult, nextRound: `Verification overlay: repair round ${nextRound}.`, result.command ? `Verifier: ${result.command}` : "Verifier: unavailable", `Failure summary: ${result.summary}`, + ...(result.artifactRef + ? [ + `Artifact reference: verification | ${result.artifactRef.path}`, + ...(result.artifactRef.stdoutSummary + ? [`Artifact stdout summary: ${result.artifactRef.stdoutSummary}`] + : []), + ...(result.artifactRef.stderrSummary + ? [`Artifact stderr summary: ${result.artifactRef.stderrSummary}`] + : []), + "Use the artifact summary first. Inspect the artifact file only if you need full verifier output.", + ] + : []), "", "Repair instructions:", "- Fix only the verifier-reported failure.", diff --git a/tests/commands/verify.test.ts b/tests/commands/verify.test.ts index dd43d6b..13bf4a6 100644 --- a/tests/commands/verify.test.ts +++ b/tests/commands/verify.test.ts @@ -98,6 +98,11 @@ describe("verifyProject", () => { const persisted = JSON.parse(readFileSync(result.artifactPath, "utf8")); expect(persisted).toMatchObject({ kind: "verification", + digest: { + command: "pnpm test", + summary: "Verification passed.", + stdoutSummary: "ok", + }, resolution: { command: "pnpm test", }, @@ -105,6 +110,13 @@ describe("verifyProject", () => { stdout: "ok\n", }, }); + expect(result.verification.artifactRef).toMatchObject({ + kind: "verification", + path: result.artifactPath, + command: "pnpm test", + summary: "Verification passed.", + stdoutSummary: "ok", + }); }); }); @@ -138,5 +150,12 @@ describe("runVerifyCommand", () => { const payload = JSON.parse(log.mock.calls[0][0]); expect(payload.artifactPath).toContain("verify-"); expect(payload.artifact.verification.stderr).toBe("failed\n"); + expect(payload.artifactRef).toMatchObject({ + kind: "verification", + path: payload.artifactPath, + command: "pnpm test", + summary: "Verification failed with exit code 1.", + stderrSummary: "failed", + }); }); }); diff --git a/tests/loop/controller.test.ts b/tests/loop/controller.test.ts index 88d9237..fe8037c 100644 --- a/tests/loop/controller.test.ts +++ b/tests/loop/controller.test.ts @@ -52,12 +52,23 @@ describe("runLoopController", () => { command: "pnpm test", exitCode: 1, summary: "still failing", + artifactRef: { + kind: "verification", + id: "verify-1", + path: "/tmp/repo/artifacts/verify-1.json", + createdAt: "2026-05-03T00:00:00.000Z", + command: "pnpm test", + exitCode: 1, + summary: "still failing", + stderrSummary: "1 failing test", + }, }), }); expect(result.status).toBe("handoff"); expect(result.summary).toContain("pnpm test"); expect(result.summary).toContain("still failing"); + expect(result.summary).toContain("/tmp/repo/artifacts/verify-1.json"); }); test("stops before repair when maxVerifyRuns would be exceeded", async () => { diff --git a/tests/runtime/prompt.test.ts b/tests/runtime/prompt.test.ts index b772af3..9c54ec0 100644 --- a/tests/runtime/prompt.test.ts +++ b/tests/runtime/prompt.test.ts @@ -65,6 +65,16 @@ describe("runtime prompt stack", () => { command: "pnpm test", exitCode: 1, summary: "1 test failed", + artifactRef: { + kind: "verification", + id: "verify-1", + path: "/tmp/repo/artifacts/verify-1.json", + createdAt: "2026-05-03T00:00:00.000Z", + command: "pnpm test", + exitCode: 1, + summary: "1 test failed", + stderrSummary: "1 failing test", + }, }, 2, ); @@ -72,6 +82,8 @@ describe("runtime prompt stack", () => { expect(repairPrompt).toContain("Verification overlay: repair round 2."); expect(repairPrompt).toContain("pnpm test"); expect(repairPrompt).toContain("1 test failed"); + expect(repairPrompt).toContain("/tmp/repo/artifacts/verify-1.json"); + expect(repairPrompt).toContain("Use the artifact summary first."); expect(repairPrompt).not.toContain("Start with a short explicit plan before editing."); }); });