Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 27 additions & 1 deletion resources/extensions/shared/runtime-state.js
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,31 @@ export function patchRuntimeLoopStatus(patch) {
};
}

function formatCapabilityFlag(enabled) {
return enabled ? "on" : "off";
}

function formatCapabilityMatrixLine(status) {
const matrix = status.resolvedModel?.capabilityMatrix;
if (!matrix) {
return "Capability matrix: unavailable";
}

return [
`Capability matrix: input=${(matrix.modalities || []).join(",") || "none"}`,
`thinking=${formatCapabilityFlag(Boolean(matrix.thinking))}`,
`preservedThinking=${formatCapabilityFlag(Boolean(matrix.preservedThinking))}`,
`streaming=${formatCapabilityFlag(Boolean(matrix.streaming))}`,
`toolCall=${formatCapabilityFlag(Boolean(matrix.toolCall))}`,
`toolStream=${formatCapabilityFlag(Boolean(matrix.toolStream))}`,
`struct=${formatCapabilityFlag(Boolean(matrix.structuredOutput))}`,
`cache=${formatCapabilityFlag(Boolean(matrix.cache))}`,
`mcp=${formatCapabilityFlag(Boolean(matrix.mcp))}`,
`zhipuNativePatch=${formatCapabilityFlag(Boolean(matrix.zhipuNativePatch))}`,
`dashscopeCompatPatch=${formatCapabilityFlag(Boolean(matrix.dashscopeCompatPatch))}`,
].join(" | ");
}

export function buildRuntimeStatusLines(status) {
const verifier = status.loop.verifyCommand
? status.loop.verifyCommand
Expand Down Expand Up @@ -174,10 +199,11 @@ export function buildRuntimeStatusLines(status) {
`Cwd: ${status.cwd}`,
`Provider: ${status.provider}`,
`Model: ${status.model}`,
`Resolved: canonical=${status.resolvedModel?.canonicalModelId ?? "none"} | platform=${status.resolvedModel?.platform ?? "unknown"} | upstream=${status.resolvedModel?.upstreamVendor ?? "unknown"} | patch=${status.resolvedModel?.payloadPatchPolicy ?? "safe-openai-compatible"} | confidence=${status.resolvedModel?.confidence ?? "low"}`,
`Resolved: family=${status.resolvedModel?.family ?? "generic"} | transport=${status.resolvedModel?.transport ?? "openai-completions"} | gateway=${status.resolvedModel?.gateway ?? status.resolvedModel?.platform ?? "unknown"} | canonical=${status.resolvedModel?.canonicalModelId ?? "none"} | upstream=${status.resolvedModel?.upstreamVendor ?? "unknown"} | patch=${status.resolvedModel?.payloadPatchPolicy ?? "safe-openai-compatible"} | confidence=${status.resolvedModel?.confidence ?? "low"}`,
status.resolvedModel?.contextWindow
? `Model caps: contextWindow=${status.resolvedModel.contextWindow} | maxOutputTokens=${status.resolvedModel.maxOutputTokens}`
: "Model caps: unknown",
formatCapabilityMatrixLine(status),
generationLine,
glmLine,
`Approval policy: ${status.approvalPolicy}`,
Expand Down
116 changes: 87 additions & 29 deletions src/diagnostics/runtime-status.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ import {
type RuntimeConfig,
} from "../app/env.js";
import { appendRuntimeEvent, getRuntimeEvents } from "./event-log.js";
import { resolveRuntimeModelProfile } from "../models/runtime-model-profile.js";
import { resolveModelDiscoveryStatus } from "../models/model-discovery.js";
import { resolveGlmProfileV2 } from "../models/resolve-glm-profile-v2.js";
import { formatCompactionSource, resolveRuntimeCompactionStatus } from "./compaction-settings.js";
import {
getMcpMetadataCachePath,
Expand All @@ -16,6 +16,7 @@ import {
import { readLatestVerificationArtifact } from "../harness/artifact-index.js";
import { resolveProviderBaseUrl } from "../providers/settings.js";
import {
type ApiKind,
getProviderDefaultApi,
isProviderName,
normalizeApiKind,
Expand All @@ -28,6 +29,7 @@ import type {
RuntimeLoopStatus,
RuntimeNotificationStatus,
RuntimePaths,
RuntimeResolvedModelStatus,
RuntimeStatus,
RuntimeVerificationStatus,
} from "./types.js";
Expand Down Expand Up @@ -207,6 +209,81 @@ function parseOptionalNumber(value: string | undefined): number | undefined {
return Number.isFinite(parsed) ? parsed : undefined;
}

function formatCapabilityFlag(enabled: boolean): "on" | "off" {
return enabled ? "on" : "off";
}

function buildResolvedModelStatus(args: {
provider: string;
api: ApiKind;
model: string;
baseUrl?: string;
overrides?: GlmConfigFile["modelOverrides"];
}): RuntimeResolvedModelStatus {
const profile = resolveRuntimeModelProfile({
provider: args.provider,
api: args.api,
modelId: args.model,
baseUrl: args.baseUrl,
overrides: args.overrides,
});

const capabilityMatrix = {
modalities: profile.effectiveModalities,
thinking: profile.effectiveCaps.supportsThinking,
preservedThinking: profile.effectiveCaps.supportsPreservedThinking,
streaming: profile.effectiveCaps.supportsStreaming,
toolCall: profile.effectiveCaps.supportsToolCall,
toolStream: profile.effectiveCaps.supportsToolStream,
structuredOutput: profile.effectiveCaps.supportsStructuredOutput,
cache: profile.effectiveCaps.supportsCache,
mcp: profile.effectiveCaps.supportsMcp,
zhipuNativePatch: profile.patchPipeline.zhipuNative,
dashscopeCompatPatch: profile.patchPipeline.dashscopeCompat,
};

return {
family: profile.family,
transport: profile.transport,
gateway: profile.gateway,
canonicalModelId: profile.canonicalModelId,
platform: profile.evidence.platform,
upstreamVendor: profile.evidence.upstreamVendor,
payloadPatchPolicy: profile.payloadPatchPolicy,
confidence: profile.evidence.confidence,
modalities: profile.effectiveModalities,
patchPipeline: profile.patchPipeline,
capabilityMatrix,
contextWindow: profile.effectiveCaps.contextWindow,
maxOutputTokens: profile.effectiveCaps.maxOutputTokens,
supportsThinking: profile.effectiveCaps.supportsThinking,
supportsPreservedThinking: profile.effectiveCaps.supportsPreservedThinking,
supportsStreaming: profile.effectiveCaps.supportsStreaming,
supportsToolCall: profile.effectiveCaps.supportsToolCall,
supportsToolStream: profile.effectiveCaps.supportsToolStream,
supportsCache: profile.effectiveCaps.supportsCache,
supportsStructuredOutput: profile.effectiveCaps.supportsStructuredOutput,
supportsMcp: profile.effectiveCaps.supportsMcp,
};
}

function formatCapabilityMatrixLine(status: RuntimeStatus): string {
const matrix = status.resolvedModel.capabilityMatrix;
return [
`Capability matrix: input=${matrix.modalities.join(",") || "none"}`,
`thinking=${formatCapabilityFlag(matrix.thinking)}`,
`preservedThinking=${formatCapabilityFlag(matrix.preservedThinking)}`,
`streaming=${formatCapabilityFlag(matrix.streaming)}`,
`toolCall=${formatCapabilityFlag(matrix.toolCall)}`,
`toolStream=${formatCapabilityFlag(matrix.toolStream)}`,
`struct=${formatCapabilityFlag(matrix.structuredOutput)}`,
`cache=${formatCapabilityFlag(matrix.cache)}`,
`mcp=${formatCapabilityFlag(matrix.mcp)}`,
`zhipuNativePatch=${formatCapabilityFlag(matrix.zhipuNativePatch)}`,
`dashscopeCompatPatch=${formatCapabilityFlag(matrix.dashscopeCompatPatch)}`,
].join(" | ");
}

export async function buildRuntimeStatus(args: {
cwd: string;
runtime: RuntimeConfig;
Expand Down Expand Up @@ -280,34 +357,14 @@ export async function buildRuntimeStatus(args: {
api: effectiveApi,
model: args.runtime.model,
baseUrl,
resolvedModel: buildResolvedModelStatus({
provider: args.runtime.provider,
api: effectiveApi,
model: args.runtime.model,
baseUrl,
overrides: args.config?.modelOverrides,
}),
...(modelDiscovery ? { modelDiscovery } : {}),
resolvedModel: (() => {
const profile = resolveGlmProfileV2({
provider: args.runtime.provider,
api: effectiveApi,
modelId: args.runtime.model,
baseUrl,
overrides: args.config?.modelOverrides,
});

return {
canonicalModelId: profile.canonicalModelId,
platform: profile.evidence.platform,
upstreamVendor: profile.evidence.upstreamVendor,
payloadPatchPolicy: profile.payloadPatchPolicy,
confidence: profile.evidence.confidence,
contextWindow: profile.effectiveCaps.contextWindow,
maxOutputTokens: profile.effectiveCaps.maxOutputTokens,
supportsThinking: profile.effectiveCaps.supportsThinking,
supportsPreservedThinking: profile.effectiveCaps.supportsPreservedThinking,
supportsStreaming: profile.effectiveCaps.supportsStreaming,
supportsToolCall: profile.effectiveCaps.supportsToolCall,
supportsToolStream: profile.effectiveCaps.supportsToolStream,
supportsCache: profile.effectiveCaps.supportsCache,
supportsStructuredOutput: profile.effectiveCaps.supportsStructuredOutput,
supportsMcp: profile.effectiveCaps.supportsMcp,
};
})(),
generation,
glmCapabilities,
toolSignature,
Expand Down Expand Up @@ -463,8 +520,9 @@ export function formatRuntimeStatusLines(status: RuntimeStatus): string[] {
`API: ${status.api}`,
`Model: ${status.model}`,
`Base URL: ${status.baseUrl ?? "default"}`,
`Resolved: canonical=${status.resolvedModel.canonicalModelId ?? "none"} | platform=${status.resolvedModel.platform} | upstream=${status.resolvedModel.upstreamVendor} | patch=${status.resolvedModel.payloadPatchPolicy} | confidence=${status.resolvedModel.confidence}`,
`Resolved: family=${status.resolvedModel.family} | transport=${status.resolvedModel.transport} | gateway=${status.resolvedModel.gateway} | canonical=${status.resolvedModel.canonicalModelId ?? "none"} | upstream=${status.resolvedModel.upstreamVendor} | patch=${status.resolvedModel.payloadPatchPolicy} | confidence=${status.resolvedModel.confidence}`,
`Model caps: contextWindow=${status.resolvedModel.contextWindow} | maxOutputTokens=${status.resolvedModel.maxOutputTokens} | thinking=${status.resolvedModel.supportsThinking ? "on" : "off"} | preservedThinking=${status.resolvedModel.supportsPreservedThinking ? "on" : "off"} | toolCall=${status.resolvedModel.supportsToolCall ? "on" : "off"} | toolStream=${status.resolvedModel.supportsToolStream ? "on" : "off"} | struct=${status.resolvedModel.supportsStructuredOutput ? "on" : "off"} | cache=${status.resolvedModel.supportsCache ? "on" : "off"} | mcp=${status.resolvedModel.supportsMcp ? "on" : "off"}`,
formatCapabilityMatrixLine(status),
modelDiscoveryLine,
generationLine,
glmLine,
Expand Down
26 changes: 26 additions & 0 deletions src/diagnostics/types.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
import type { ApprovalPolicy, LoopFailureMode, LoopProfileName } from "../app/config-store.js";
import type {
GlmInputModality,
RuntimeModelFamily,
RuntimeTransport,
} from "../models/model-profile-types.js";
import type { RuntimeToolSignature } from "./tool-signature.js";

export type RuntimeDiagnosticsConfig = {
Expand Down Expand Up @@ -82,11 +87,32 @@ export type RuntimePaths = {
};

export type RuntimeResolvedModelStatus = {
family: RuntimeModelFamily;
transport: RuntimeTransport;
gateway: string;
canonicalModelId?: string;
platform: string;
upstreamVendor: string;
payloadPatchPolicy: "glm-native" | "safe-openai-compatible";
confidence: "high" | "medium" | "low";
modalities: GlmInputModality[];
patchPipeline: {
zhipuNative: boolean;
dashscopeCompat: boolean;
};
capabilityMatrix: {
modalities: GlmInputModality[];
thinking: boolean;
preservedThinking: boolean;
streaming: boolean;
toolCall: boolean;
toolStream: boolean;
structuredOutput: boolean;
cache: boolean;
mcp: boolean;
zhipuNativePatch: boolean;
dashscopeCompatPatch: boolean;
};
contextWindow: number;
maxOutputTokens: number;
supportsThinking: boolean;
Expand Down
16 changes: 16 additions & 0 deletions tests/commands/inspect.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,9 +52,17 @@ describe("inspectRuntime", () => {
expect(status.provider).toBe("openai-compatible");
expect(status.model).toBe("glm-openai");
expect(status.resolvedModel).toMatchObject({
family: "generic",
transport: "openai-completions",
gateway: "gateway-other",
platform: "gateway-other",
payloadPatchPolicy: "safe-openai-compatible",
confidence: "low",
capabilityMatrix: {
modalities: ["text", "image"],
thinking: false,
zhipuNativePatch: false,
},
});
expect(status.approvalPolicy).toBe("never");
expect(status.loop).toMatchObject({
Expand Down Expand Up @@ -108,8 +116,16 @@ describe("runInspectCommand", () => {
provider: "glm",
model: "glm-5.1",
resolvedModel: expect.objectContaining({
family: "glm",
transport: "openai-completions",
gateway: "native-bigmodel",
canonicalModelId: "glm-5.1",
payloadPatchPolicy: "glm-native",
capabilityMatrix: expect.objectContaining({
modalities: ["text"],
thinking: true,
zhipuNativePatch: true,
}),
}),
loop: expect.objectContaining({
profile: "code",
Expand Down
80 changes: 79 additions & 1 deletion tests/diagnostics/runtime-status.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -115,11 +115,32 @@ describe("buildRuntimeStatus", () => {
expect(status.provider).toBe("glm");
expect(status.model).toBe("glm-5.1");
expect(status.resolvedModel).toMatchObject({
family: "glm",
transport: "openai-completions",
gateway: "native-bigmodel",
canonicalModelId: "glm-5.1",
platform: "native-bigmodel",
upstreamVendor: "unknown",
payloadPatchPolicy: "glm-native",
confidence: "high",
modalities: ["text"],
patchPipeline: {
zhipuNative: true,
dashscopeCompat: false,
},
capabilityMatrix: {
modalities: ["text"],
thinking: true,
preservedThinking: true,
streaming: true,
toolCall: true,
toolStream: true,
structuredOutput: true,
cache: true,
mcp: true,
zhipuNativePatch: true,
dashscopeCompatPatch: false,
},
contextWindow: 204_800,
maxOutputTokens: 131_072,
});
Expand Down Expand Up @@ -171,17 +192,72 @@ describe("buildRuntimeStatus", () => {
reserveTokens: 16_384,
keepRecentTokens: 20_000,
});
expect(formatRuntimeStatusLines(status)).toEqual(
const lines = formatRuntimeStatusLines(status);
expect(lines).toEqual(
expect.arrayContaining([
expect.stringContaining("Model discovery: unsupported"),
expect.stringContaining(
"Capability matrix: input=text | thinking=on | preservedThinking=on",
),
expect.stringContaining(
`Verification: smoke | fail | pnpm test | tests failed | ${artifactPath}`,
),
]),
);
expect(lines.filter((line) => line.startsWith("Capability matrix:"))).toHaveLength(1);
expect(status.paths.sessionDir).toBe("/tmp/.glm/sessions/demo");
});

test("summarizes multimodal and dashscope capability matrix for qwen routes", async () => {
const status = await buildRuntimeStatus({
cwd: "/tmp/repo",
runtime: {
provider: "bailian",
model: "qwen/qwen3.5-122b-a10b",
approvalPolicy: "ask",
},
loop: {
enabled: false,
profile: "code",
maxRounds: 3,
failureMode: "handoff",
autoVerify: true,
},
diagnostics: {
debugRuntime: false,
eventLogLimit: 10,
},
notifications: {
enabled: false,
onTurnEnd: true,
onLoopResult: true,
},
paths: resolveGlmSessionPaths("/tmp/repo"),
env: {},
});

expect(status.resolvedModel).toMatchObject({
family: "qwen",
transport: "openai-completions",
gateway: "gateway-dashscope",
modalities: ["text", "image", "video"],
patchPipeline: {
zhipuNative: false,
dashscopeCompat: true,
},
capabilityMatrix: {
modalities: ["text", "image", "video"],
dashscopeCompatPatch: true,
zhipuNativePatch: false,
},
});
expect(formatRuntimeStatusLines(status)).toEqual(
expect.arrayContaining([
expect.stringContaining("Capability matrix: input=text,image,video"),
]),
);
});

test("patchRuntimeLoopStatus updates the in-process runtime status store", async () => {
const status = await buildRuntimeStatus({
cwd: "/tmp/repo",
Expand Down Expand Up @@ -276,6 +352,8 @@ describe("buildRuntimeStatus", () => {

expect(status.resolvedModel).toMatchObject({
canonicalModelId: "glm-5",
family: "glm",
transport: "anthropic-messages",
platform: "gateway-modelscope-openai",
confidence: "medium",
});
Expand Down
Loading