diff --git a/packages/rules/src/reviewSchema.js b/packages/rules/src/reviewSchema.js index 0888c922..aad9f4e5 100644 --- a/packages/rules/src/reviewSchema.js +++ b/packages/rules/src/reviewSchema.js @@ -240,13 +240,63 @@ export function reportToLegacy(report) { // JSON Extraction Cascade // ════════════════════════════════════════════════════════════════════════════ +/** + * Deterministic fenced-block scanner (CodeQL js/polynomial-redos #1/#2). + * + * Replaces the two regex strategies — the fenced matcher in single-match + * and matchAll form. That regex's greedy \s* (which matches newlines) + * interacting with the following \n? made the whitespace bridge ambiguous: + * on pathological inputs (a fence followed by very long whitespace runs, no + * closing fence) the engine re-splits the whitespace while the lazy body + * expands, giving quadratic — and with matchAll, multiplied — work on the + * shared review-worker event loop. This scanner pairs opening and closing + * fences with monotonic indexOf cursors: every character is visited a + * bounded number of times, so worst-case work is linear in the input + * regardless of whitespace shape. + * + * Semantics preserved from the regex pair (pinned by + * tests/review-schema-fence-scanner.test.js alongside the pre-existing + * fence tests in reviewSchema.test.js): + * - fences pair sequentially: an opening ``` runs to the NEXT ``` (its + * close); scanning resumes after the close — identical to matchAll's + * continuation after each completed match + * - a `json` label immediately after the opening fence is stripped from the + * body (the regex's (?:json)? consumption); JSON content never begins + * with the letters "json", so the strip cannot destroy a parseable body + * - callers trim bodies before JSON.parse exactly as before, so exact + * boundary whitespace is irrelevant (and was under the regex too) + * - an unclosed trailing fence yields no block; odd fence counts pair + * floor(n/2) blocks, identical to matchAll's completed matches + * + * @param {string} text - full LLM output + * @returns {string[]} fence bodies, in document order (untrimmed) + */ +export function scanFencedBlocks(text) { + const blocks = []; + let cursor = 0; + for (;;) { + const open = text.indexOf("```", cursor); + if (open === -1) break; + const close = text.indexOf("```", open + 3); + if (close === -1) break; // trailing unclosed fence — no complete block remains + let body = text.slice(open + 3, close); + // Strip the immediate optional "json" label plus its following + // whitespace run (down to zero), mirroring the regex's label branch. + body = body.replace(/^json\s*/, ""); + blocks.push(body); + cursor = close + 3; + } + return blocks; +} + /** * Extract a JSON review report from LLM output text. * Tries multiple extraction strategies in order. * * Strategies: * 1. Direct JSON.parse - * 2. Strip markdown fences (```json ... ```) + * 2. Strip markdown fences (```json ... ```) — deterministic scanner + * (scanFencedBlocks), not a regex * 3. Extract first {...} block at top level * 4. JSONL stream parsing (one JSON object per line) * 5. Nested structured_output/result unwrapping @@ -269,11 +319,13 @@ export function extractReviewJSON(text) { // continue } - // Strategy 2: Strip markdown fences - const fenceMatch = trimmed.match(/```(?:json)?\s*\n?([\s\S]*?)\n?\s*```/); - if (fenceMatch) { + // Strategy 2: Strip markdown fences — deterministic scanner (linear + // worst-case; the old regex pair was polynomial on adversarial whitespace, + // CodeQL #1/#2). + const fencedBlocks = scanFencedBlocks(trimmed); + if (fencedBlocks.length >= 1) { try { - const parsed = JSON.parse(fenceMatch[1].trim()); + const parsed = JSON.parse(fencedBlocks[0].trim()); return { json: unwrapNested(parsed), strategy: "fenced" }; } catch (_e) { // continue @@ -281,11 +333,10 @@ export function extractReviewJSON(text) { } // Strategy 2b: Multiple fences — try last one (often the actual result) - const allFences = [...trimmed.matchAll(/```(?:json)?\s*\n?([\s\S]*?)\n?\s*```/g)]; - if (allFences.length > 1) { - for (let i = allFences.length - 1; i >= 0; i--) { + if (fencedBlocks.length > 1) { + for (let i = fencedBlocks.length - 1; i >= 0; i -= 1) { try { - const parsed = JSON.parse(allFences[i][1].trim()); + const parsed = JSON.parse(fencedBlocks[i].trim()); return { json: unwrapNested(parsed), strategy: "fenced_multi" }; } catch (_e) { // continue diff --git a/packages/rules/tests/review-schema-fence-scanner.test.js b/packages/rules/tests/review-schema-fence-scanner.test.js new file mode 100644 index 00000000..4b63ba96 --- /dev/null +++ b/packages/rules/tests/review-schema-fence-scanner.test.js @@ -0,0 +1,162 @@ +// tests/review-schema-fence-scanner.test.js +// #425 unit (#1+#2) regressions: the deterministic fenced-block scanner +// replaces the two polynomial regexes in extractReviewJSON. Two jobs: +// +// 1. EQUIVALENCE — the old regex pair and the scanner must agree on the +// extraction outcome (strategy + parsed json) across a corpus covering +// labels, whitespace shapes, multi-fence, unclosed fences, and nesting. +// The old regexes are reproduced here verbatim so the equivalence is +// pinned in-tree, not asserted from memory. +// 2. BOUNDEDNESS — the adversarial shapes that made the regex polynomial +// (long whitespace runs after an unclosed fence, repeated opening fences +// without closures, long newlines) must complete in linear time, with a +// generous wall-clock bound that the old regex would blow through. + +import { + scanFencedBlocks, + extractReviewJSON, +} from "../src/reviewSchema.js"; + +// The replaced regex pair, verbatim from reviewSchema.js pre-remediation. +const OLD_FENCE = /```(?:json)?\s*\n?([\s\S]*?)\n?\s*```/; +const OLD_FENCE_G = /```(?:json)?\s*\n?([\s\S]*?)\n?\s*```/g; + +/** Outcome of the OLD extraction strategies 2/2b for equivalence checks. */ +function oldFencedOutcome(text) { + const trimmed = text.trim(); + const fenceMatch = trimmed.match(OLD_FENCE); + if (fenceMatch) { + try { + return { strategy: "fenced", json: JSON.parse(fenceMatch[1].trim()) }; + } catch (_e) { /* fall through */ } + } + const allFences = [...trimmed.matchAll(OLD_FENCE_G)]; + if (allFences.length > 1) { + for (let i = allFences.length - 1; i >= 0; i -= 1) { + try { + return { strategy: "fenced_multi", json: JSON.parse(allFences[i][1].trim()) }; + } catch (_e) { /* keep walking backwards */ } + } + } + return null; +} + +/** Full-pipeline equivalence: old strategies 2/2b vs new, on one input. */ +function assertFenceEquivalence(input) { + const oldOutcome = oldFencedOutcome(input); + const extracted = extractReviewJSON(input); + if (oldOutcome === null) { + // Old code fell through to strategies 3-5; the new code must reach the + // SAME subsequent strategies. For corpus members designed to be decided + // at the fence stage, falling through means both sides continue — the + // extraction result must then still agree, so compare full results. + return extracted; + } + expect(extracted.strategy).toBe(oldOutcome.strategy); + expect(extracted.json).toEqual(oldOutcome.json); + return extracted; +} + +// Wall-clock bound generous enough for CI variance yet far below the old +// regex's seconds-scale blowup at these sizes (the maintainer's microbench: +// ~2.06s for ONE unclosed fence + 2,000 spaces). +const TIME_BUDGET_MS = 500; + +function timed(fn) { + const t0 = Date.now(); + const out = fn(); + return { out, ms: Date.now() - t0 }; +} + +describe("EQUIVALENCE: scanner vs replaced regex pair", () => { + const corpus = [ + '```json\n{"findings": [], "overall_correctness": "pass"}\n```', + '```\n{"a": 1}\n```', + 'Here is the review:\n```json\n{"a": 1}\n```\nThanks', + '```\nnot json\n```\n```json\n{"b": 2}\n```', // first invalid, last valid → fenced_multi + '```json\n{"c": 3}\n```\n```json\nbroken\n```', // first valid → fenced + '```json{"d": 4}```', // label with no newline + '``` json\n{"e": 5}\n```', // label NOT immediate (spaces first) — both must fail fence parse identically + 'leading ```\n{"f": 6}\n``` trailing', + '```\n```', // empty body pair + 'unclosed ``` json body without end', // no complete fence + '```x``````y```', // adjacent pairs + '```\n{"g": 7}\n\n```', // blank line before close + 'text before ``` mid-line {"h": 8} ``` text after', + ]; + + test.each(corpus.map((c) => [c]))("corpus %j", (input) => { + assertFenceEquivalence(input); + }); + + it("strategy identifiers are unchanged (fenced / fenced_multi)", () => { + expect(extractReviewJSON('```json\n{"a": 1}\n```').strategy).toBe("fenced"); + expect(extractReviewJSON('```\nnope\n```\n```\n{"b": 2}\n```').strategy).toBe("fenced_multi"); + }); + + it("scanFencedBlocks pairing matches matchAll continuation semantics", () => { + // Odd fence count: matchAll produced floor(n/2) completed matches. + expect(scanFencedBlocks("```a```b```")).toEqual(["a"]); + expect(scanFencedBlocks("```a```b```c```")).toEqual(["a", "c"]); + expect(scanFencedBlocks("no fences")).toEqual([]); + expect(scanFencedBlocks("``````")).toEqual([""]); + }); + + it("label stripping matches the regex's (?:json)? consumption", () => { + expect(scanFencedBlocks("```json\n{}\n```")).toEqual(["{}\n"]); // label + following ws run stripped (greedy, like the regex) + expect(scanFencedBlocks("```json{}```")).toEqual(["{}"]); // label, zero whitespace + expect(scanFencedBlocks("``` json{}\n```")).toEqual([" json{}\n"]); // spaces first: NOT a label + }); +}); + +describe("BOUNDEDNESS: adversarial shapes complete in linear time", () => { + it("unclosed fence + very long SPACE run (the maintainer's reproduced case, scaled)", () => { + const input = "```" + " ".repeat(200_000); + const { out, ms } = timed(() => extractReviewJSON(input)); + expect(out.json).toBeNull(); + expect(ms).toBeLessThan(TIME_BUDGET_MS); + }); + + it("unclosed fence + very long NEWLINE run (CodeQL's stated attack shape)", () => { + const input = "```json" + "\n".repeat(200_000); + const { out, ms } = timed(() => extractReviewJSON(input)); + expect(out.json).toBeNull(); + expect(ms).toBeLessThan(TIME_BUDGET_MS); + }); + + it("repeated opening fences without closures", () => { + const input = "```json".repeat(30_000); + const { out, ms } = timed(() => extractReviewJSON(input)); + expect(ms).toBeLessThan(TIME_BUDGET_MS); + expect(scanFencedBlocks(input)).toHaveLength(15_000); + }); + + it("valid JSON buried after a pathological unclosed prefix still extracts", () => { + const input = "```json" + "\n".repeat(100_000) + "trailing text"; + // No closing fence anywhere: nothing extractable at the fence stage, and + // the cascade must still terminate quickly (strategies 3-5 run on the + // same string). + const { out, ms } = timed(() => extractReviewJSON(input)); + expect(ms).toBeLessThan(TIME_BUDGET_MS); + expect(out.json).toBeNull(); + }); + + it("valid multi-fence response at scale (many fences, last one valid)", () => { + const parts = []; + for (let i = 0; i < 5_000; i += 1) parts.push("```\nnot-json-" + i + "\n```"); + parts.push("```json\n{\"findings\": [], \"overall_correctness\": \"pass\"}\n```"); + const input = parts.join("\n"); + const { out, ms } = timed(() => extractReviewJSON(input)); + expect(out.strategy).toBe("fenced_multi"); + expect(out.json).toEqual({ findings: [], overall_correctness: "pass" }); + expect(ms).toBeLessThan(TIME_BUDGET_MS); + }); + + it("long whitespace run INSIDE a valid fence still extracts (equivalence at scale)", () => { + const input = "```json\n" + " ".repeat(150_000) + "\n{\"deep\": true}\n```"; + const { out, ms } = timed(() => extractReviewJSON(input)); + expect(out.strategy).toBe("fenced"); + expect(out.json).toEqual({ deep: true }); + expect(ms).toBeLessThan(TIME_BUDGET_MS); + }); +});