Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
146 changes: 132 additions & 14 deletions packages/rules/src/reviewSchema.js
Original file line number Diff line number Diff line change
Expand Up @@ -240,13 +240,129 @@ export function reportToLegacy(report) {
// JSON Extraction Cascade
// ════════════════════════════════════════════════════════════════════════════

/**
* Deterministic fenced-block scanner (CodeQL js/polynomial-redos #1/#2).
*
* Replaces the two regex strategies — the fenced matcher in single-match
* and matchAll form. That regex's greedy \s* (which matches newlines)
* interacting with the following \n? made the whitespace bridge ambiguous:
* on pathological inputs (a fence followed by very long whitespace runs, no
* closing fence) the engine re-splits the whitespace while the lazy body
* expands, giving quadratic — and with matchAll, multiplied — work on the
* shared review-worker event loop. This scanner pairs opening and closing
* fences with monotonic indexOf cursors: every character is visited a
* bounded number of times, so worst-case work is linear in the input
* regardless of whitespace shape.
*
* Semantics preserved from the regex pair (pinned by
* tests/review-schema-fence-scanner.test.js alongside the pre-existing
* fence tests in reviewSchema.test.js):
* - fences pair sequentially: an opening ``` runs to the NEXT ``` (its
* close); scanning resumes after the close — identical to matchAll's
* continuation after each completed match
* - a `json` label immediately after the opening fence is stripped from the
* body (the regex's (?:json)? consumption); JSON content never begins
* with the letters "json", so the strip cannot destroy a parseable body
* - callers trim bodies before JSON.parse exactly as before, so exact
* boundary whitespace is irrelevant (and was under the regex too)
* - an unclosed trailing fence yields no block; odd fence counts pair
* floor(n/2) blocks, identical to matchAll's completed matches
*
* @param {string} text - full LLM output
* @returns {string[]} fence bodies, in document order (untrimmed)
*/
export function scanFencedBlocks(text) {
const blocks = [];
let cursor = 0;
for (;;) {
const open = text.indexOf("```", cursor);
if (open === -1) break;
const close = text.indexOf("```", open + 3);
if (close === -1) break; // trailing unclosed fence — no complete block remains
let body = text.slice(open + 3, close);
// Strip the immediate optional "json" label plus its following
// whitespace run (down to zero), mirroring the regex's label branch.
body = body.replace(/^json\s*/, "");
blocks.push(body);
cursor = close + 3;
}
return blocks;
}

const WS = /\s/;

/**
* Bounded scan for the embedded-findings pattern (CodeQL
* js/polynomial-redos #3).
*
* Replaces `text.match(/\{"findings"\s*:\s*\[[\s\S]*?\]\s*,\s*"overall_correctness"/)`.
* That regex retried its literal prefix at EVERY index, and per candidate
* rescanned the remaining suffix with the lazy body — quadratic on inputs
* that repeat `{"findings":[` without ever completing the pattern. This
* scanner never rescans: two monotonic indexOf passes (one over the tail
* literal, one over the head literal) with O(1) shape checks per candidate,
* so total work is linear in the input.
*
* Semantics preserved from the regex (pinned by
* tests/review-schema-findings-pattern.test.js):
* - a match exists at a `{"findings"` occurrence iff skipping one
* whitespace run reaches `:`, then one more whitespace run reaches `[`,
* and some `]` follows that `[` with only whitespace, a comma, and
* whitespace before the literal `"overall_correctness"`
* - the FIRST such occurrence (in position order) is returned, identical to
* the regex engine's first-match start. (The old code's extra
* `trimmed.indexOf(matchText)` was provably redundant: an earlier
* occurrence of the full matched text would itself have been an earlier
* pattern match, contradicting first-match selection.)
*
* @param {string} text - full trimmed LLM output
* @returns {number} start index of the first pattern match, or -1
*/
function findFindingsPatternStart(text) {
// Pass 1 — every valid close: a `]` immediately followed (modulo the two
// whitespace runs) by `,"overall_correctness"`. Scan the tail LITERAL once
// and walk left; each character is visited a constant number of times.
const TAIL = '"overall_correctness"';
const validCloses = [];
for (let t = text.indexOf(TAIL); t !== -1; t = text.indexOf(TAIL, t + 1)) {
let i = t - 1;
while (i >= 0 && WS.test(text[i])) i -= 1; // whitespace before the quote
if (i >= 0 && text[i] === ",") {
i -= 1;
while (i >= 0 && WS.test(text[i])) i -= 1; // whitespace before the comma
if (i >= 0 && text[i] === "]") validCloses.push(i);
}
}
if (validCloses.length === 0) return -1;
validCloses.sort((a, b) => a - b); // collected right-to-left

// Pass 2 — candidate heads in order; O(1) shape check plus an
// existence test against the sorted close list.
const HEAD = '{"findings"';
const lastClose = validCloses[validCloses.length - 1];
for (let s = text.indexOf(HEAD); s !== -1; s = text.indexOf(HEAD, s + 1)) {
let p = s + HEAD.length;
while (p < text.length && WS.test(text[p])) p += 1; // \s* before ':'
if (text[p] !== ":") continue;
p += 1;
while (p < text.length && WS.test(text[p])) p += 1; // \s* before '['
if (text[p] !== "[") continue;
p += 1;
// A valid close at or after the body start — the lazy body may be
// empty (`[]` puts the close exactly at p).
if (lastClose >= p) return s;
}
return -1;
}

/**
* Extract a JSON review report from LLM output text.
* Tries multiple extraction strategies in order.
*
* Strategies:
* 1. Direct JSON.parse
* 2. Strip markdown fences (```json ... ```)
* 2. Strip markdown fences (```json ... ```) — deterministic scanner
* (scanFencedBlocks), not a regex
* 3. Extract first {...} block at top level
* 4. JSONL stream parsing (one JSON object per line)
* 5. Nested structured_output/result unwrapping
Expand All @@ -269,23 +385,24 @@ export function extractReviewJSON(text) {
// continue
}

// Strategy 2: Strip markdown fences
const fenceMatch = trimmed.match(/```(?:json)?\s*\n?([\s\S]*?)\n?\s*```/);
if (fenceMatch) {
// Strategy 2: Strip markdown fences — deterministic scanner (linear
// worst-case; the old regex pair was polynomial on adversarial whitespace,
// CodeQL #1/#2).
const fencedBlocks = scanFencedBlocks(trimmed);
if (fencedBlocks.length >= 1) {
try {
const parsed = JSON.parse(fenceMatch[1].trim());
const parsed = JSON.parse(fencedBlocks[0].trim());
return { json: unwrapNested(parsed), strategy: "fenced" };
} catch (_e) {
// continue
}
}

// Strategy 2b: Multiple fences — try last one (often the actual result)
const allFences = [...trimmed.matchAll(/```(?:json)?\s*\n?([\s\S]*?)\n?\s*```/g)];
if (allFences.length > 1) {
for (let i = allFences.length - 1; i >= 0; i--) {
if (fencedBlocks.length > 1) {
for (let i = fencedBlocks.length - 1; i >= 0; i -= 1) {
try {
const parsed = JSON.parse(allFences[i][1].trim());
const parsed = JSON.parse(fencedBlocks[i].trim());
return { json: unwrapNested(parsed), strategy: "fenced_multi" };
} catch (_e) {
// continue
Expand Down Expand Up @@ -321,11 +438,12 @@ export function extractReviewJSON(text) {

// Strategy 5: Try to find JSON embedded in text (common with Claude)
// Look for patterns like: {"findings": [...], "overall_correctness": ...}
const jsonPattern = /\{"findings"\s*:\s*\[[\s\S]*?\]\s*,\s*"overall_correctness"/;
const jsonMatch = trimmed.match(jsonPattern);
if (jsonMatch) {
const startIdx = trimmed.indexOf(jsonMatch[0]);
const subStr = trimmed.slice(startIdx);
// Bounded scan (findFindingsPatternStart) — the replaced regex retried its
// literal prefix at every index and rescanned the remaining suffix per
// candidate, quadratic on repeated incomplete prefixes (CodeQL #3).
const matchStart = findFindingsPatternStart(trimmed);
if (matchStart !== -1) {
const subStr = trimmed.slice(matchStart);
const endBrace = findMatchingBrace(subStr);
if (endBrace > 0) {
try {
Expand Down
162 changes: 162 additions & 0 deletions packages/rules/tests/review-schema-fence-scanner.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
// tests/review-schema-fence-scanner.test.js
// #425 unit (#1+#2) regressions: the deterministic fenced-block scanner
// replaces the two polynomial regexes in extractReviewJSON. Two jobs:
//
// 1. EQUIVALENCE — the old regex pair and the scanner must agree on the
// extraction outcome (strategy + parsed json) across a corpus covering
// labels, whitespace shapes, multi-fence, unclosed fences, and nesting.
// The old regexes are reproduced here verbatim so the equivalence is
// pinned in-tree, not asserted from memory.
// 2. BOUNDEDNESS — the adversarial shapes that made the regex polynomial
// (long whitespace runs after an unclosed fence, repeated opening fences
// without closures, long newlines) must complete in linear time, with a
// generous wall-clock bound that the old regex would blow through.

import {
scanFencedBlocks,
extractReviewJSON,
} from "../src/reviewSchema.js";

// The replaced regex pair, verbatim from reviewSchema.js pre-remediation.
const OLD_FENCE = /```(?:json)?\s*\n?([\s\S]*?)\n?\s*```/;
const OLD_FENCE_G = /```(?:json)?\s*\n?([\s\S]*?)\n?\s*```/g;

/** Outcome of the OLD extraction strategies 2/2b for equivalence checks. */
function oldFencedOutcome(text) {
const trimmed = text.trim();
const fenceMatch = trimmed.match(OLD_FENCE);
if (fenceMatch) {
try {
return { strategy: "fenced", json: JSON.parse(fenceMatch[1].trim()) };
} catch (_e) { /* fall through */ }
}
const allFences = [...trimmed.matchAll(OLD_FENCE_G)];
if (allFences.length > 1) {
for (let i = allFences.length - 1; i >= 0; i -= 1) {
try {
return { strategy: "fenced_multi", json: JSON.parse(allFences[i][1].trim()) };
} catch (_e) { /* keep walking backwards */ }
}
}
return null;
}

/** Full-pipeline equivalence: old strategies 2/2b vs new, on one input. */
function assertFenceEquivalence(input) {
const oldOutcome = oldFencedOutcome(input);
const extracted = extractReviewJSON(input);
if (oldOutcome === null) {
// Old code fell through to strategies 3-5; the new code must reach the
// SAME subsequent strategies. For corpus members designed to be decided

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[LOW] Fence equivalence helper silently skips comparison when the old fence stage yields no parseable outcome

In assertFenceEquivalence, the oldOutcome === null branch returns extracted without any assertion, despite the comment claiming 'the extraction result must then still agree, so compare full results'. Corpus members designed to fail fence parsing on both sides (e.g. ' json\n{"e": 5}\n' with the space-before-label, 'unclosed ``` json body without end') therefore pin nothing about the fence stage: if scanFencedBlocks ever diverged by parsing a body the old regex rejected, these cases would still pass silently. The direct label-stripping assertions on scanFencedBlocks partially mitigate this, but the null branch should assert the negative invariant for fence-decided corpus entries: expect(extracted.strategy).not.toBe('fenced') and .not.toBe('fenced_multi'). Suggested fix: add those two assertions in the null branch (optionally gated per corpus entry with a flag for inputs genuinely decided by later strategies).

// at the fence stage, falling through means both sides continue — the
// extraction result must then still agree, so compare full results.
return extracted;
}
expect(extracted.strategy).toBe(oldOutcome.strategy);
expect(extracted.json).toEqual(oldOutcome.json);
return extracted;
}

// Wall-clock bound generous enough for CI variance yet far below the old
// regex's seconds-scale blowup at these sizes (the maintainer's microbench:
// ~2.06s for ONE unclosed fence + 2,000 spaces).
const TIME_BUDGET_MS = 500;

function timed(fn) {
const t0 = Date.now();
const out = fn();
return { out, ms: Date.now() - t0 };
}

describe("EQUIVALENCE: scanner vs replaced regex pair", () => {
const corpus = [
'```json\n{"findings": [], "overall_correctness": "pass"}\n```',
'```\n{"a": 1}\n```',
'Here is the review:\n```json\n{"a": 1}\n```\nThanks',
'```\nnot json\n```\n```json\n{"b": 2}\n```', // first invalid, last valid → fenced_multi
'```json\n{"c": 3}\n```\n```json\nbroken\n```', // first valid → fenced
'```json{"d": 4}```', // label with no newline
'``` json\n{"e": 5}\n```', // label NOT immediate (spaces first) — both must fail fence parse identically
'leading ```\n{"f": 6}\n``` trailing',
'```\n```', // empty body pair
'unclosed ``` json body without end', // no complete fence
'```x``````y```', // adjacent pairs
'```\n{"g": 7}\n\n```', // blank line before close
'text before ``` mid-line {"h": 8} ``` text after',
];

test.each(corpus.map((c) => [c]))("corpus %j", (input) => {
assertFenceEquivalence(input);
});

it("strategy identifiers are unchanged (fenced / fenced_multi)", () => {
expect(extractReviewJSON('```json\n{"a": 1}\n```').strategy).toBe("fenced");
expect(extractReviewJSON('```\nnope\n```\n```\n{"b": 2}\n```').strategy).toBe("fenced_multi");
});

it("scanFencedBlocks pairing matches matchAll continuation semantics", () => {
// Odd fence count: matchAll produced floor(n/2) completed matches.
expect(scanFencedBlocks("```a```b```")).toEqual(["a"]);
expect(scanFencedBlocks("```a```b```c```")).toEqual(["a", "c"]);
expect(scanFencedBlocks("no fences")).toEqual([]);
expect(scanFencedBlocks("``````")).toEqual([""]);
});

it("label stripping matches the regex's (?:json)? consumption", () => {
expect(scanFencedBlocks("```json\n{}\n```")).toEqual(["{}\n"]); // label + following ws run stripped (greedy, like the regex)
expect(scanFencedBlocks("```json{}```")).toEqual(["{}"]); // label, zero whitespace
expect(scanFencedBlocks("``` json{}\n```")).toEqual([" json{}\n"]); // spaces first: NOT a label
});
});

describe("BOUNDEDNESS: adversarial shapes complete in linear time", () => {
it("unclosed fence + very long SPACE run (the maintainer's reproduced case, scaled)", () => {
const input = "```" + " ".repeat(200_000);
const { out, ms } = timed(() => extractReviewJSON(input));
expect(out.json).toBeNull();
expect(ms).toBeLessThan(TIME_BUDGET_MS);
});

it("unclosed fence + very long NEWLINE run (CodeQL's stated attack shape)", () => {
const input = "```json" + "\n".repeat(200_000);
const { out, ms } = timed(() => extractReviewJSON(input));
expect(out.json).toBeNull();
expect(ms).toBeLessThan(TIME_BUDGET_MS);
});

it("repeated opening fences without closures", () => {
const input = "```json".repeat(30_000);
const { out, ms } = timed(() => extractReviewJSON(input));
expect(ms).toBeLessThan(TIME_BUDGET_MS);
expect(scanFencedBlocks(input)).toHaveLength(15_000);
});

it("valid JSON buried after a pathological unclosed prefix still extracts", () => {
const input = "```json" + "\n".repeat(100_000) + "trailing text";
// No closing fence anywhere: nothing extractable at the fence stage, and
// the cascade must still terminate quickly (strategies 3-5 run on the
// same string).
const { out, ms } = timed(() => extractReviewJSON(input));
expect(ms).toBeLessThan(TIME_BUDGET_MS);
expect(out.json).toBeNull();
});

it("valid multi-fence response at scale (many fences, last one valid)", () => {
const parts = [];
for (let i = 0; i < 5_000; i += 1) parts.push("```\nnot-json-" + i + "\n```");
parts.push("```json\n{\"findings\": [], \"overall_correctness\": \"pass\"}\n```");
const input = parts.join("\n");
const { out, ms } = timed(() => extractReviewJSON(input));
expect(out.strategy).toBe("fenced_multi");
expect(out.json).toEqual({ findings: [], overall_correctness: "pass" });
expect(ms).toBeLessThan(TIME_BUDGET_MS);
});

it("long whitespace run INSIDE a valid fence still extracts (equivalence at scale)", () => {
const input = "```json\n" + " ".repeat(150_000) + "\n{\"deep\": true}\n```";
const { out, ms } = timed(() => extractReviewJSON(input));
expect(out.strategy).toBe("fenced");
expect(out.json).toEqual({ deep: true });
expect(ms).toBeLessThan(TIME_BUDGET_MS);
});
});
Loading
Loading