Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 72 additions & 5 deletions packages/rules/src/reviewSchema.js
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,72 @@ export function reportToLegacy(report) {
// JSON Extraction Cascade
// ════════════════════════════════════════════════════════════════════════════

const WS = /\s/;

/**
* Bounded scan for the embedded-findings pattern (CodeQL
* js/polynomial-redos #3).
*
* Replaces `text.match(/\{"findings"\s*:\s*\[[\s\S]*?\]\s*,\s*"overall_correctness"/)`.
* That regex retried its literal prefix at EVERY index, and per candidate
* rescanned the remaining suffix with the lazy body — quadratic on inputs
* that repeat `{"findings":[` without ever completing the pattern. This
* scanner never rescans: two monotonic indexOf passes (one over the tail
* literal, one over the head literal) with O(1) shape checks per candidate,
* so total work is linear in the input.
*
* Semantics preserved from the regex (pinned by
* tests/review-schema-findings-pattern.test.js):
* - a match exists at a `{"findings"` occurrence iff skipping one
* whitespace run reaches `:`, then one more whitespace run reaches `[`,
* and some `]` follows that `[` with only whitespace, a comma, and
* whitespace before the literal `"overall_correctness"`
* - the FIRST such occurrence (in position order) is returned, identical to
* the regex engine's first-match start. (The old code's extra
* `trimmed.indexOf(matchText)` was provably redundant: an earlier
* occurrence of the full matched text would itself have been an earlier
* pattern match, contradicting first-match selection.)
*
* @param {string} text - full trimmed LLM output
* @returns {number} start index of the first pattern match, or -1
*/
function findFindingsPatternStart(text) {
// Pass 1 — every valid close: a `]` immediately followed (modulo the two
// whitespace runs) by `,"overall_correctness"`. Scan the tail LITERAL once
// and walk left; each character is visited a constant number of times.
const TAIL = '"overall_correctness"';
const validCloses = [];
for (let t = text.indexOf(TAIL); t !== -1; t = text.indexOf(TAIL, t + 1)) {
let i = t - 1;
while (i >= 0 && WS.test(text[i])) i -= 1; // whitespace before the quote
if (i >= 0 && text[i] === ",") {
i -= 1;
while (i >= 0 && WS.test(text[i])) i -= 1; // whitespace before the comma
if (i >= 0 && text[i] === "]") validCloses.push(i);
}
}
if (validCloses.length === 0) return -1;
validCloses.sort((a, b) => a - b); // collected right-to-left

// Pass 2 — candidate heads in order; O(1) shape check plus an
// existence test against the sorted close list.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[LOW] Misleading invariant comment: validCloses are collected left-to-right, not right-to-left

In findFindingsPatternStart(), the comment // collected right-to-left on the sort line is factually wrong: the Pass 1 loop enumerates TAIL occurrences via indexOf in strictly ascending order, and each pushed close index i satisfies i < t with all closes strictly increasing (a later close cannot precede an earlier one, since the chars between a close and its tail must be whitespace/comma, which excludes an earlier close's ]). So validCloses is already sorted ascending and the sort is a defensive no-op — harmless, but the comment misstates the invariant in a security-sensitive bounded-scan algorithm and could mislead a future maintainer into reasoning incorrectly about ordering (e.g., 'fixing' the code to match the comment, or removing the sort while assuming the wrong order). Suggested fix: change the comment to something like // already ascending (indexOf enumerates tails left-to-right); sort is defensive or delete the sort with an accurate note. No functional impact.

const HEAD = '{"findings"';
const lastClose = validCloses[validCloses.length - 1];
for (let s = text.indexOf(HEAD); s !== -1; s = text.indexOf(HEAD, s + 1)) {
let p = s + HEAD.length;
while (p < text.length && WS.test(text[p])) p += 1; // \s* before ':'
if (text[p] !== ":") continue;
p += 1;
while (p < text.length && WS.test(text[p])) p += 1; // \s* before '['
if (text[p] !== "[") continue;
p += 1;
// A valid close at or after the body start — the lazy body may be
// empty (`[]` puts the close exactly at p).
if (lastClose >= p) return s;
}
return -1;
}

/**
* Extract a JSON review report from LLM output text.
* Tries multiple extraction strategies in order.
Expand Down Expand Up @@ -321,11 +387,12 @@ export function extractReviewJSON(text) {

// Strategy 5: Try to find JSON embedded in text (common with Claude)
// Look for patterns like: {"findings": [...], "overall_correctness": ...}
const jsonPattern = /\{"findings"\s*:\s*\[[\s\S]*?\]\s*,\s*"overall_correctness"/;
const jsonMatch = trimmed.match(jsonPattern);
if (jsonMatch) {
const startIdx = trimmed.indexOf(jsonMatch[0]);
const subStr = trimmed.slice(startIdx);
// Bounded scan (findFindingsPatternStart) — the replaced regex retried its
// literal prefix at every index and rescanned the remaining suffix per
// candidate, quadratic on repeated incomplete prefixes (CodeQL #3).
const matchStart = findFindingsPatternStart(trimmed);
if (matchStart !== -1) {
const subStr = trimmed.slice(matchStart);
const endBrace = findMatchingBrace(subStr);
if (endBrace > 0) {
try {
Expand Down
147 changes: 147 additions & 0 deletions packages/rules/tests/review-schema-findings-pattern.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
// tests/review-schema-findings-pattern.test.js
// #425 unit (#3) regressions: the bounded findings-pattern scan replaces
// the regex /\{"findings"\s*:\s*\[[\s\S]*?\]\s*,\s*"overall_correctness"/
// whose per-index prefix retry and per-candidate suffix rescan made it
// quadratic on repeated incomplete prefixes. Two jobs, mirroring the fence
// scanner suite:
//
// 1. EQUIVALENCE — the old regex (reproduced verbatim) driving the old
// strategy-5 flow (match → slice from match start → brace balance →
// JSON.parse) and the new scan driving the same flow must agree on
// whether strategy 5 fires, across a corpus covering whitespace shapes,
// first-candidate-completes-via-later-text, unparseable tails, and
// no-match fall-through. Strategy 5 is the LAST resort: corpus members
// that assert pattern_match are built to fail strategies 1-4 first.
// 2. BOUNDEDNESS — the repeated-incomplete-prefix attack input completes in
// linear time under a generous wall-clock bound.

import { extractReviewJSON } from "../src/reviewSchema.js";

// The replaced regex, verbatim.
const OLD_PATTERN = /\{"findings"\s*:\s*\[[\s\S]*?\]\s*,\s*"overall_correctness"/;

// findMatchingBrace, replicated from reviewSchema.js (unexported there).
function findMatchingBrace(str) {
let depth = 0;
let inString = false;
let escape = false;
for (let i = 0; i < str.length; i += 1) {
const ch = str[i];
if (escape) { escape = false; continue; }
if (ch === "\\") { if (inString) escape = true; continue; }
if (ch === '"') { inString = !inString; continue; }
if (inString) continue;
if (ch === "{") depth += 1;
else if (ch === "}") { depth -= 1; if (depth === 0) return i; }
}
return -1;
}

/** TRUE old strategy-5 outcome: fires (and parses) or not. */
function oldStrategyFiveFires(text) {
const trimmed = text.trim();
const jsonMatch = trimmed.match(OLD_PATTERN);
if (!jsonMatch) return false;
const startIdx = trimmed.indexOf(jsonMatch[0]);
const subStr = trimmed.slice(startIdx);
const endBrace = findMatchingBrace(subStr);
if (endBrace <= 0) return false;
try {
JSON.parse(subStr.slice(0, endBrace + 1));
return true;
} catch (_e) {
return false;
}
}

// Wall-clock bound generous for CI variance, far below the old regex's
// blowup at these sizes.
const TIME_BUDGET_MS = 500;

function timed(fn) {
const t0 = Date.now();
const out = fn();
return { out, ms: Date.now() - t0 };
}

// Strategy 5 fires only when strategies 1-4 all failed: embed the pattern
// on ONE line with surrounding text (brace_extract spans to the LAST brace
// and fails; the line is not valid JSONL).
const FIRING = 'intro {"findings": [], "overall_correctness": "pass"} x { trailing }'; // trailing broken brace defeats brace_extract/jsonl so strategy 5 decides
const FIRING_WS = 'x { "findings" : [ {"id": 1} ] , "overall_correctness" : "pass" } y { tail }';

describe("EQUIVALENCE: bounded scan vs replaced regex (full old flow)", () => {
const corpus = [
FIRING,
FIRING_WS,
'first {"findings": [no completion intro second {"findings": [], "overall_correctness": "pass"} tail', // first candidate completes via later text (lazy body)
'{"findings" : [ ] , "overall_correctness"}', // shape completes, JSON invalid → both must not fire
'{"findings": not-an-array, "overall_correctness": "x"}', // ':' then no '[' → no match → neither fires
'prefix text with no pattern at all', // neither fires
'noise {"findings": [], "overall_correctness": "pass"} trailing {"findings": [] and no tail x }', // later partial candidate; trailing x } defeats brace_extract
];

test.each(corpus.map((c) => [c]))("corpus %j", (input) => {
const oldFires = oldStrategyFiveFires(input);
const extracted = extractReviewJSON(input);
if (oldFires) {
expect(extracted.strategy).toBe("pattern_match");
} else {
expect(extracted.strategy).not.toBe("pattern_match");
}
});

it("fires with strategy identifier preserved: pattern_match", () => {
expect(extractReviewJSON(FIRING).strategy).toBe("pattern_match");
expect(extractReviewJSON(FIRING).json).toEqual({ findings: [], overall_correctness: "pass" });
});

it("whitespace-shape tolerance matches the regex (tabs/spacing between tokens)", () => {
// Single line with tab spreads: defeats jsonl (one unparseable line) and
// brace_extract (trailing broken brace), so strategy 5 decides. A
// multi-line spread is decided by the untouched jsonl strategy instead.
const spread = 'a {"findings"\t:\t[\t{"id": 1}\t]\t,\t"overall_correctness"\t:\t"pass"\t} b { tail }';
expect(extractReviewJSON(spread).strategy).toBe("pattern_match");
expect(oldStrategyFiveFires(spread)).toBe(true);
});

it("first candidate completing wins even when a later candidate exists", () => {
const input = FIRING + " mid " + FIRING;
expect(extractReviewJSON(input).strategy).toBe("pattern_match");
expect(oldStrategyFiveFires(input)).toBe(true);
});
});

describe("BOUNDEDNESS: repeated incomplete prefixes (the attack shape)", () => {
it("30,000 repeated {\"findings\":[ prefixes with no completion — linear time, falls through", () => {
const input = '{"findings":['.repeat(30_000);
const { out, ms } = timed(() => extractReviewJSON(input));
expect(out.json).toBeNull();
expect(ms).toBeLessThan(TIME_BUDGET_MS);
});

it("30,000 prefixes WITH a completion: scan matches (old regex matches too) but brace balance cannot close 30k opens — both flows fall through, bounded", () => {
const input = "lead " + '{"findings":['.repeat(30_000) + "], \"overall_correctness\": \"pass\"}";
const { out, ms } = timed(() => extractReviewJSON(input));
// OLD flow on this input: regex matches at candidate 1, the same slice
// cannot brace-balance (30k opens, 1 close), strategy 5 falls through —
// identical terminal result here.
expect(OLD_PATTERN.test(input)).toBe(true); // the shape DOES complete
expect(out.json).toBeNull();
expect(ms).toBeLessThan(TIME_BUDGET_MS);
});

it("single candidate completing at scale resolves pattern_match", () => {
const input = "lead " + '{"findings":[ {"id":1} ], "overall_correctness": "pass"} tail '.repeat(1_000) + "{ broken";
const { out, ms } = timed(() => extractReviewJSON(input));
expect(out.strategy).toBe("pattern_match");
expect(ms).toBeLessThan(TIME_BUDGET_MS);
});

it("repeated prefixes with a tail literal but no valid close shape — linear", () => {
const input = '{"findings":['.repeat(20_000) + '"overall_correctness"'.repeat(500);
const { out, ms } = timed(() => extractReviewJSON(input));
expect(out.json).toBeNull();
expect(ms).toBeLessThan(TIME_BUDGET_MS);
});
});
Loading