diff --git a/apps/judge/src/judge-strategies.test.ts b/apps/judge/src/judge-strategies.test.ts index ee5f7528..8dce4df6 100644 --- a/apps/judge/src/judge-strategies.test.ts +++ b/apps/judge/src/judge-strategies.test.ts @@ -470,6 +470,9 @@ class TestableBundledStrategy extends BundledStrategy { publicBuildSystemPrompt(hasToolOutputs: boolean, hasAgentResponse?: boolean) { return (this as any).buildSystemPrompt(undefined, hasToolOutputs, hasAgentResponse); } + publicParseJsonResponse(response: string, criteria: { id: string; prompt: string }[]) { + return (this as any).parseJsonResponse(response, criteria); + } publicBuildUserPrompt( criteria: { id: string; prompt: string }[] = [ { id: "c1", prompt: "does the code work" }, @@ -480,6 +483,31 @@ class TestableBundledStrategy extends BundledStrategy { } } +describe("BundledStrategy JSON parsing (issue #167)", () => { + const criteria = [{ id: "c1", prompt: "criterion one" }]; + + it("parses fenced JSON with nested objects", () => { + const response = '```json\n{"results":[{"criterion":"c1","passed":true,"feedback":"details"}],"meta":{"tokens":1}}\n```'; + const result = new TestableBundledStrategy("test-model").publicParseJsonResponse(response, criteria); + expect(result.allPassed).toBe(true); + expect(result.results[0].evaluated).toBe(true); + }); + + it("ignores braces inside JSON strings", () => { + const response = '{"results":[{"criterion":"c1","passed":true,"feedback":"mongoose } config { is present"}]} trailing text'; + const result = new TestableBundledStrategy("test-model").publicParseJsonResponse(response, criteria); + expect(result.allPassed).toBe(true); + expect(result.results[0].feedback).toContain("mongoose } config {"); + }); + + it("treats truncated JSON as unevaluated", () => { + const response = '```json\n{"results":[{"criterion":"c1","passed":false,"feedback":"truncated'; + const result = new TestableBundledStrategy("test-model").publicParseJsonResponse(response, criteria); + expect(result.allPassed).toBe(false); + expect(result.results[0].evaluated).toBe(false); + }); +}); + describe("judge tool-outputs guidance (issue #1125)", () => { // The headless judge cannot run commands; it must decide from the codebase // plus the coding agent's captured tool outputs. The guidance must be generic diff --git a/apps/judge/src/judge-strategies.ts b/apps/judge/src/judge-strategies.ts index a231fa2f..24986c55 100644 --- a/apps/judge/src/judge-strategies.ts +++ b/apps/judge/src/judge-strategies.ts @@ -1002,23 +1002,39 @@ IMPORTANT: Return ONLY the JSON, no additional text before or after.`; ${criteriaList}${historySection}`; } + /** Extract the first complete JSON object, respecting strings and escapes. */ + private extractJsonObject(response: string): string | undefined { + const start = response.indexOf("{"); + if (start < 0) return undefined; + + let depth = 0; + let inString = false; + let escaped = false; + for (let i = start; i < response.length; i++) { + const ch = response[i]; + if (inString) { + if (escaped) escaped = false; + else if (ch === "\\") escaped = true; + else if (ch === '"') inString = false; + continue; + } + if (ch === '"') inString = true; + else if (ch === "{") depth++; + else if (ch === "}" && --depth === 0) return response.slice(start, i + 1); + } + return undefined; + } + private parseJsonResponse( response: string, criteria: CriteriaConfig[], onProgress?: (result: CriterionResult) => void, ): DetailedEvaluationResult { - // Try to extract JSON from markdown code blocks - let jsonStr = response.trim(); - const jsonMatch = jsonStr.match(/```(?:json)?\s*(\{[\s\S]*?\})\s*```/); - if (jsonMatch) { - jsonStr = jsonMatch[1]; - } else { - // Try to find raw JSON - const rawJsonMatch = jsonStr.match(/(\{[\s\S]*\})/); - if (rawJsonMatch) { - jsonStr = rawJsonMatch[1]; - } - } + // Extract a complete JSON object without relying on markdown fences. A + // brace-balanced scan handles nested objects and braces inside feedback strings, + // while an incomplete/truncated response deliberately falls through to the + // existing parse-error path. + const jsonStr = this.extractJsonObject(response) ?? response.trim(); try { const data = JSON.parse(jsonStr);