Skip to content

feat: add artifact-first verifier references #102

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
angeiv wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: add artifact-first verifier references #102

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 73 additions & 8 deletions resources/extensions/glm-loop/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,26 @@ type LoopProfileName = "code";
type LoopFailureMode = "handoff" | "fail";
type LoopExecutionStatus = "succeeded" | "handoff" | "failed";
type LoopPhase = "run" | "verify" | "repair";
type VerificationArtifactReference = {
kind: "verification";
id: string;
path: string;
createdAt: string;
scenario?: string;
command?: string;
exitCode?: number;
summary: string;
stdoutSummary?: string;
stderrSummary?: string;
};
type VerificationResult =
| {
kind: "pass";
command: string;
exitCode: 0;
summary: string;
artifactPath?: string;
artifactRef?: VerificationArtifactReference;
stdout?: string;
stderr?: string;
}
Expand All @@ -31,6 +44,7 @@ type VerificationResult =
exitCode: number;
summary: string;
artifactPath?: string;
artifactRef?: VerificationArtifactReference;
stdout?: string;
stderr?: string;
}
Expand All @@ -39,6 +53,7 @@ type VerificationResult =
command?: string;
summary: string;
artifactPath?: string;
artifactRef?: VerificationArtifactReference;
stdout?: string;
stderr?: string;
};
Expand Down Expand Up @@ -71,6 +86,7 @@ type LoopVerificationRecord = {
command?: string;
exitCode?: number;
summary: string;
artifact?: VerificationArtifactReference;
artifactPath?: string;
stdoutSummary?: string;
stderrSummary?: string;
Expand Down Expand Up @@ -228,6 +244,7 @@ function isLoopResultRecord(value: unknown): value is LoopResultRecord {
if (!value || typeof value !== "object") return false;
const maybe = value as Partial<LoopResultRecord>;
const verification = maybe.verification as Partial<LoopVerificationRecord> | undefined;
const artifact = verification?.artifact as Partial<VerificationArtifactReference> | undefined;
return (
(maybe.status === "succeeded" || maybe.status === "handoff" || maybe.status === "failed") &&
typeof maybe.task === "string" &&
Expand All @@ -242,6 +259,16 @@ function isLoopResultRecord(value: unknown): value is LoopResultRecord {
(verification.command === undefined || typeof verification.command === "string") &&
(verification.exitCode === undefined || typeof verification.exitCode === "number") &&
typeof verification.summary === "string" &&
(!artifact ||
(artifact.kind === "verification" &&
typeof artifact.id === "string" &&
typeof artifact.path === "string" &&
typeof artifact.createdAt === "string" &&
(artifact.command === undefined || typeof artifact.command === "string") &&
(artifact.exitCode === undefined || typeof artifact.exitCode === "number") &&
typeof artifact.summary === "string" &&
(artifact.stdoutSummary === undefined || typeof artifact.stdoutSummary === "string") &&
(artifact.stderrSummary === undefined || typeof artifact.stderrSummary === "string"))) &&
(verification.artifactPath === undefined || typeof verification.artifactPath === "string") &&
(verification.stdoutSummary === undefined || typeof verification.stdoutSummary === "string") &&
(verification.stderrSummary === undefined || typeof verification.stderrSummary === "string") &&
Expand Down Expand Up @@ -354,18 +381,18 @@ function summarizeOutputText(
}

function createLoopVerificationRecord(result: VerificationResult): LoopVerificationRecord {
const stdoutSummary = result.artifactRef?.stdoutSummary ?? summarizeOutputText(result.stdout);
const stderrSummary = result.artifactRef?.stderrSummary ?? summarizeOutputText(result.stderr);

return {
kind: result.kind,
...(result.command ? { command: result.command } : {}),
...(result.exitCode === undefined ? {} : { exitCode: result.exitCode }),
summary: result.summary,
...(result.artifactRef ? { artifact: result.artifactRef } : {}),
...(result.artifactPath ? { artifactPath: result.artifactPath } : {}),
...(summarizeOutputText(result.stdout)
? { stdoutSummary: summarizeOutputText(result.stdout) }
: {}),
...(summarizeOutputText(result.stderr)
? { stderrSummary: summarizeOutputText(result.stderr) }
: {}),
...(stdoutSummary ? { stdoutSummary } : {}),
...(stderrSummary ? { stderrSummary } : {}),
};
}

Expand Down Expand Up @@ -455,6 +482,9 @@ function buildLastResultLines(result: LoopResultRecord): string[] {
result.verification.command ? `Last verifier: ${result.verification.command}` : undefined,
`Last verification: ${result.verification.kind}`,
`Last summary: ${result.verification.summary}`,
result.verification.artifact
? `Last artifact: verification | ${result.verification.artifact.path}`
: undefined,
result.verification.stdoutSummary
? `Last stdout summary: ${result.verification.stdoutSummary}`
: undefined,
Expand Down Expand Up @@ -504,6 +534,9 @@ function buildShowLines(results: LoopResultRecord[], index: number): string[] {
? []
: [`Exit code: ${result.verification.exitCode}`]),
`Summary: ${result.verification.summary}`,
...(result.verification.artifact
? [`Artifact reference: verification | ${result.verification.artifact.path}`]
: []),
...(result.verification.stdoutSummary
? [`Stdout summary: ${result.verification.stdoutSummary}`]
: []),
Expand Down Expand Up @@ -700,6 +733,18 @@ function buildRepairPrompt(result: VerificationResult, nextRound: number): strin
`Verification failed. Begin repair round ${nextRound}.`,
result.command ? `Verifier: ${result.command}` : "Verifier: unavailable",
`Summary: ${result.summary}`,
...(result.artifactRef
? [
`Artifact reference: verification | ${result.artifactRef.path}`,
...(result.artifactRef.stdoutSummary
? [`Artifact stdout summary: ${result.artifactRef.stdoutSummary}`]
: []),
...(result.artifactRef.stderrSummary
? [`Artifact stderr summary: ${result.artifactRef.stderrSummary}`]
: []),
"Use the artifact summary first. Inspect the artifact file only if you need full verifier output.",
]
: []),
"",
"Instructions:",
"- Fix only the failure reported by the verifier.",
Expand Down Expand Up @@ -734,6 +779,18 @@ function buildFailureSummary(args: {
`Rounds attempted: ${args.rounds.length}`,
args.lastResult.command ? `Last verifier: ${args.lastResult.command}` : undefined,
`Last result: ${args.lastResult.summary}`,
args.lastResult.artifactRef
? `Artifact reference: verification | ${args.lastResult.artifactRef.path}`
: undefined,
args.lastResult.artifactRef?.stdoutSummary
? `Artifact stdout summary: ${args.lastResult.artifactRef.stdoutSummary}`
: undefined,
args.lastResult.artifactRef?.stderrSummary
? `Artifact stderr summary: ${args.lastResult.artifactRef.stderrSummary}`
: undefined,
args.lastResult.artifactRef
? "Use the artifact summary first. Inspect the artifact file only if deeper verifier output is required."
: undefined,
"Recommended next step: inspect the latest verifier output, apply a focused fix, and rerun verification.",
]
.filter(Boolean)
Expand Down Expand Up @@ -866,7 +923,11 @@ async function runVerification(
resolution: { kind: "command", command, source: "loop" },
verification,
});
return { ...verification, artifactPath: artifact.artifactPath };
return {
...verification,
artifactPath: artifact.artifactPath,
artifactRef: artifact.artifactRef,
};
}

const verification: VerificationResult = {
Expand All @@ -882,7 +943,11 @@ async function runVerification(
resolution: { kind: "command", command, source: "loop" },
verification,
});
return { ...verification, artifactPath: artifact.artifactPath };
return {
...verification,
artifactPath: artifact.artifactPath,
artifactRef: artifact.artifactRef,
};
}

async function resolveVerifier(
Expand Down
57 changes: 25 additions & 32 deletions src/commands/verify.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,12 @@
import { readConfigFile, type GlmConfigFile } from "../app/config-store.js";
import {
formatVerificationArtifactReference,
writeVerificationArtifact,
type VerificationArtifact,
} from "../harness/artifacts.js";
import { detectCodeVerifier } from "../loop/verify-detect.js";
import { runVerificationCommand } from "../loop/verify-runner.js";
import type { VerificationCommandResolution, VerificationResult } from "../loop/types.js";
import { writeVerificationArtifact, type VerificationArtifact } from "../harness/artifacts.js";
import { resolveVerifyScenario, type VerifyScenarioName } from "../harness/scenarios.js";

export type VerifyCommandArgs = {
Expand All @@ -18,6 +22,7 @@ type VerifyResolution = {
verification: VerificationResult;
artifact: VerificationArtifact;
artifactPath: string;
artifactRef: NonNullable<VerificationResult["artifactRef"]>;
};

type VerifyDependencies = {
Expand All @@ -33,27 +38,6 @@ type VerifyDependencies = {
writeArtifact: typeof writeVerificationArtifact;
};

function summarizeOutputText(
value: string | undefined,
maxLines = 2,
maxChars = 160,
): string | undefined {
if (!value) return undefined;
const lines = value
.split(/\r?\n/)
.map((line) => line.trim())
.filter(Boolean)
.slice(0, maxLines);
if (lines.length === 0) return undefined;

const summary = lines.join(" | ");
if (summary.length <= maxChars) {
return summary;
}

return `${summary.slice(0, maxChars - 1)}…`;
}

async function resolveVerifier(args: {
cwd: string;
explicit?: string;
Expand Down Expand Up @@ -115,7 +99,11 @@ export async function verifyProject(

return {
resolution,
verification,
verification: {
...verification,
artifactPath: artifactResult.artifactPath,
artifactRef: artifactResult.artifactRef,
},
...artifactResult,
};
}
Expand All @@ -132,7 +120,15 @@ export async function verifyProject(
verification,
});

return { resolution, verification, ...artifactResult };
return {
resolution,
verification: {
...verification,
artifactPath: artifactResult.artifactPath,
artifactRef: artifactResult.artifactRef,
},
...artifactResult,
};
}

export async function runVerifyCommand(
Expand All @@ -141,25 +137,22 @@ export async function runVerifyCommand(
): Promise<number> {
const log = deps?.log ?? console.log;
const { resolution, verification, artifact, artifactPath } = await verifyProject(input, deps);
const artifactRef = verification.artifactRef;

if (input.json) {
log(JSON.stringify({ resolution, verification, artifact, artifactPath }, null, 2));
log(JSON.stringify({ resolution, verification, artifact, artifactPath, artifactRef }, null, 2));
return verification.kind === "pass" ? 0 : 1;
}

const stdoutSummary = summarizeOutputText(verification.stdout);
const stderrSummary = summarizeOutputText(verification.stderr);

const lines = [
resolution.kind === "command"
? `Verifier: ${resolution.command} (${resolution.source})`
: `Verifier: ${resolution.summary}`,
`Result: ${verification.kind}`,
`Summary: ${verification.summary}`,
...(verification.exitCode === undefined ? [] : [`Exit code: ${verification.exitCode}`]),
`Artifact: ${artifactPath}`,
...(stdoutSummary ? [`Stdout summary: ${stdoutSummary}`] : []),
...(stderrSummary ? [`Stderr summary: ${stderrSummary}`] : []),
...(artifactRef
? formatVerificationArtifactReference(artifactRef)
: [`Artifact reference: verification | ${artifactPath}`]),
];

log(lines.join("\n"));
Expand Down
Loading
Loading