From ed08efc674a75cd2c878c5c4b12ca39418a4251f Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Mon, 10 Aug 2026 07:39:05 +0200 Subject: [PATCH 1/4] Fix screenshot gate to key off gui/ file diffs Require UI screenshots only when changed paths include gui/, not when title or body merely mention GUI. Wire changedFilePaths through enforce-pr-target and align unit and integration tests. --- .github/scripts/enforce-pr-target.test.cjs | 1 + .github/scripts/pr-quality-messages.cjs | 2 +- .github/scripts/pr-quality.cjs | 45 ++++++++---- .github/scripts/pr-quality.test.cjs | 71 +++++++++++++++++-- .github/workflows/enforce-pr-target.yml | 15 ++-- .../content/docs/contributing/pr-quality.md | 8 +-- tests/ci-workflows.test.ts | 64 +++++++++++++++-- 7 files changed, 174 insertions(+), 32 deletions(-) diff --git a/.github/scripts/enforce-pr-target.test.cjs b/.github/scripts/enforce-pr-target.test.cjs index bf5497d7ff..4b3d602f8c 100644 --- a/.github/scripts/enforce-pr-target.test.cjs +++ b/.github/scripts/enforce-pr-target.test.cjs @@ -197,6 +197,7 @@ describe("enforce-pr-target workflow", () => { ); assert.ok(qualityCall, "must call collectPrQualityFailures"); assert.match(qualityCall[1], /stackedBase/); + assert.match(qualityCall[1], /changedFilePaths/); }); it("strips stale WRONG BRANCH prefix on failure when base is corrected", () => { diff --git a/.github/scripts/pr-quality-messages.cjs b/.github/scripts/pr-quality-messages.cjs index aec05331b8..40b3c747b0 100644 --- a/.github/scripts/pr-quality-messages.cjs +++ b/.github/scripts/pr-quality-messages.cjs @@ -229,7 +229,7 @@ function buildFailureSections(failures, { pr, allowedBases, defaultBase }) { sections.push( "⚠️ **UI screenshot required**", "", - `This pull request mentions ${inlineCode("gui")} in its title or description, so it is treated as a GUI change.`, + `This pull request changes files under ${inlineCode("gui/")}, so it is treated as a GUI change.`, "", `@${pr.user.login} Please add a screenshot of the UI change to the description — drag and drop the image into the description editor, or paste a markdown image such as ${inlineCode("![Screenshot](https://example.com/after.png)")}. The check re-runs automatically once the description is edited.` ); diff --git a/.github/scripts/pr-quality.cjs b/.github/scripts/pr-quality.cjs index eaceab06ad..6106317d02 100644 --- a/.github/scripts/pr-quality.cjs +++ b/.github/scripts/pr-quality.cjs @@ -168,16 +168,34 @@ function assessPrDescription(body) { return { ok: false, reason: "thin" }; } +/** + * True when any changed path is the gui directory or inside it (slash-guarded). + * Mirrors `guiPathsChanged` in `scripts/doctor-gui-if-changed.ts`. + */ +function guiPathsChanged(files) { + return files.some( + (file) => file === "gui" || file.startsWith("gui/") + ); +} + /** * True when the PR title or description names the GUI surface as a whole word. * The description is template-stripped first so the template's own screenshot - * instruction cannot arm the gate on its own. + * instruction cannot arm the gate on its own. Negated phrases such as "no gui + * changes" are not treated as cues (see `segmentHasAffirmativeGuiCue`). */ +function segmentHasAffirmativeGuiCue(text) { + if (typeof text !== "string" || !text.trim()) return false; + const segments = text.split(/(?<=[.!?\n])/); + return segments.some((segment) => { + if (!GUI_CUE_RE.test(segment)) return false; + const withoutNegated = segment.replace(GUI_OVERRIDE_RE, ""); + return GUI_CUE_RE.test(withoutNegated); + }); +} + function hasGuiCue(title, body) { - return ( - (typeof title === "string" && GUI_CUE_RE.test(title)) || - (typeof body === "string" && GUI_CUE_RE.test(body)) - ); + return segmentHasAffirmativeGuiCue(title) || segmentHasAffirmativeGuiCue(body); } /** @@ -447,7 +465,9 @@ function collectPrQualityFailures({ /** True when baseRef is another open PR's head (stacked child). */ stackedBase = false, /** Issue comments; a maintainer comment waives the GUI-screenshot gate. */ - guiOverrideComments = [] + guiOverrideComments = [], + /** Changed file paths from `pulls.listFiles` (repo-relative). */ + changedFilePaths = [] }) { const failures = []; const wrongBase = !allowedBases.includes(baseRef) && !stackedBase; @@ -475,14 +495,12 @@ function collectPrQualityFailures({ failures.push({ code: "bad_description", reason: desc.reason }); } - // GUI-cued PRs must prove the UI change visually. The template's own - // screenshot instruction is boilerplate, so it cannot trigger this gate. A - // maintainer comment saying the change does not touch the GUI waives it. + // PRs that change gui/ must prove the UI change visually. Text cues in the + // title or description are not enough — "no gui changes" in the body must + // not arm the gate when the diff is backend-only. A maintainer comment saying + // the change does not touch the GUI still waives a gui/ diff false positive. if ( - hasGuiCue( - title, - typeof body === "string" ? stripPrTemplateBoilerplate(body) : "", - ) && + guiPathsChanged(changedFilePaths) && !hasScreenshotEvidence(body) && !hasGuiOverride({ comments: guiOverrideComments }) ) { @@ -500,6 +518,7 @@ module.exports = { isWrongAncestry, authorHasPushPermission, assessPrDescription, + guiPathsChanged, hasGuiCue, hasGuiOverride, hasScreenshotEvidence, diff --git a/.github/scripts/pr-quality.test.cjs b/.github/scripts/pr-quality.test.cjs index e013fab88a..329633ea9c 100644 --- a/.github/scripts/pr-quality.test.cjs +++ b/.github/scripts/pr-quality.test.cjs @@ -9,6 +9,7 @@ const { authorHasPushPermission, assessPrDescription, hasGuiCue, + guiPathsChanged, hasGuiOverride, hasScreenshotEvidence, buildReviewReadinessSection, @@ -137,6 +138,16 @@ describe("hasGuiCue", () => { ); }); + it("does not match negated gui phrases", () => { + assert.equal(hasGuiCue("", "no gui changes in this PR"), false); + assert.equal(hasGuiCue("", "Without gui changes"), false); + assert.equal(hasGuiCue("No GUI changes", ""), false); + assert.equal( + hasGuiCue("", "This does not change the API. Please add a gui screenshot."), + true, + ); + }); + it("does not match gui inside other words", () => { assert.equal(hasGuiCue("Add contributor guidance", ""), false); assert.equal(hasGuiCue("", "Fix the guild invitation bug"), false); @@ -149,6 +160,17 @@ describe("hasGuiCue", () => { }); }); +describe("guiPathsChanged", () => { + it("matches gui/ paths with a slash guard", () => { + assert.equal(guiPathsChanged(["gui/src/App.tsx"]), true); + assert.equal(guiPathsChanged(["gui"]), true); + assert.equal(guiPathsChanged(["scripts/foo.ts", "gui/package.json"]), true); + assert.equal(guiPathsChanged(["scripts/foo.ts"]), false); + assert.equal(guiPathsChanged(["guitools/x.ts"]), false); + assert.equal(guiPathsChanged([]), false); + }); +}); + describe("hasGuiOverride", () => { const owner = { author_association: "OWNER", body: "Not touching gui here." }; const collaborator = { author_association: "COLLABORATOR", body: "no gui changes needed" }; @@ -792,20 +814,55 @@ describe("collectPrQualityFailures", () => { assert.ok(failures.some((f) => f.code === "wrong_base")); }); - it("flags a gui title without a screenshot", () => { + it("flags gui/ file changes without a screenshot", () => { const failures = collectPrQualityFailures({ baseRef: "dev", allowedBases: allowed, - title: "GUI: fix provider list spacing", + title: "Fix dashboard spacing", body: richBody, behindMain: 0, behindBase: 0, authorPermission: "read", + changedFilePaths: ["gui/src/App.tsx"], }); assert.ok(failures.some((f) => f.code === "missing_ui_screenshot")); }); - it("flags a gui mention in the body without a screenshot", () => { + it("does not flag a gui title when no gui/ files changed", () => { + const failures = collectPrQualityFailures({ + baseRef: "dev", + allowedBases: allowed, + title: "GUI: fix provider list spacing", + body: richBody, + behindMain: 0, + behindBase: 0, + authorPermission: "read", + changedFilePaths: ["scripts/foo.ts"], + }); + assert.ok(!failures.some((f) => f.code === "missing_ui_screenshot")); + }); + + it("does not flag no gui changes text without gui/ file changes", () => { + const failures = collectPrQualityFailures({ + baseRef: "dev", + allowedBases: allowed, + title: "Fix proxy routing", + body: [ + "## Summary", + "No gui changes in this PR; proxy routing only.", + "", + "## Test plan", + "- Ran bun test tests/ci-workflows.test.ts", + ].join("\n"), + behindMain: 0, + behindBase: 0, + authorPermission: "read", + changedFilePaths: ["scripts/foo.ts"], + }); + assert.ok(!failures.some((f) => f.code === "missing_ui_screenshot")); + }); + + it("flags a gui mention in the body without a screenshot when gui/ changed", () => { const failures = collectPrQualityFailures({ baseRef: "dev", allowedBases: allowed, @@ -820,6 +877,7 @@ describe("collectPrQualityFailures", () => { behindMain: 0, behindBase: 0, authorPermission: "read", + changedFilePaths: ["gui/src/styles.css"], }); assert.ok(failures.some((f) => f.code === "missing_ui_screenshot")); }); @@ -839,6 +897,7 @@ describe("collectPrQualityFailures", () => { behindMain: 0, behindBase: 0, authorPermission: "read", + changedFilePaths: ["gui/src/App.tsx"], guiOverrideComments: [ { author_association: "OWNER", body: "no gui changes here" }, ], @@ -861,6 +920,7 @@ describe("collectPrQualityFailures", () => { behindMain: 0, behindBase: 0, authorPermission: "read", + changedFilePaths: ["gui/src/App.tsx"], guiOverrideComments: [ { author_association: "CONTRIBUTOR", body: "no gui changes here" }, ], @@ -885,6 +945,7 @@ describe("collectPrQualityFailures", () => { behindMain: 0, behindBase: 0, authorPermission: "read", + changedFilePaths: ["gui/src/App.tsx"], }); assert.ok(!failures.some((f) => f.code === "missing_ui_screenshot")); }); @@ -908,11 +969,12 @@ describe("collectPrQualityFailures", () => { behindMain: 0, behindBase: 0, authorPermission: "read", + changedFilePaths: ["gui/src/App.tsx"], }); assert.ok(!failures.some((f) => f.code === "missing_ui_screenshot")); }); - it("still flags a gui title when image syntax is only inside a code fence", () => { + it("still flags gui/ changes when image syntax is only inside a code fence", () => { const failures = collectPrQualityFailures({ baseRef: "dev", allowedBases: allowed, @@ -931,6 +993,7 @@ describe("collectPrQualityFailures", () => { behindMain: 0, behindBase: 0, authorPermission: "read", + changedFilePaths: ["gui/src/App.tsx"], }); assert.ok(failures.some((f) => f.code === "missing_ui_screenshot")); }); diff --git a/.github/workflows/enforce-pr-target.yml b/.github/workflows/enforce-pr-target.yml index a6e93d8993..02a05e21c8 100644 --- a/.github/workflows/enforce-pr-target.yml +++ b/.github/workflows/enforce-pr-target.yml @@ -534,6 +534,14 @@ jobs: } } + const changedFiles = await github.paginate( + github.rest.pulls.listFiles, + { owner, repo, pull_number, per_page: 100 } + ); + const changedFilePaths = changedFiles + .map(file => file.filename) + .filter(Boolean); + let failures = collectPrQualityFailures({ baseRef: pr.base.ref, allowedBases: ALLOWED_BASES, @@ -548,17 +556,14 @@ jobs: stackedBase, // A maintainer issue comment ("not touching gui") waives the // GUI-screenshot gate; the comments are already fetched above. - guiOverrideComments: comments + guiOverrideComments: comments, + changedFilePaths }); // Hygiene is a separate workflow that owns the blocked label and // the Hygiene comment section, but Ready / review-ready must not // clear while those checks fail. Re-assess here from the same // trusted scripts so the gate cannot race ahead of hygiene. - const changedFiles = await github.paginate( - github.rest.pulls.listFiles, - { owner, repo, pull_number, per_page: 100 } - ); const labelNames = (pr.labels ?? []).map(label => label.name); failures = [ ...failures, diff --git a/docs-site/src/content/docs/contributing/pr-quality.md b/docs-site/src/content/docs/contributing/pr-quality.md index e29619e5a2..aeb47a23f5 100644 --- a/docs-site/src/content/docs/contributing/pr-quality.md +++ b/docs-site/src/content/docs/contributing/pr-quality.md @@ -37,10 +37,10 @@ tells you exactly what to change: - **PR quality (`enforce-target`).** Pull requests must target `dev` and carry a real description: a **Summary** of what changed and why, plus a **Test - plan** (or equivalent substance). When the title or description mentions - `gui`, the description must include a screenshot of the UI change; the check - keeps the PR a draft and comments until the screenshot is present. A - maintainer can waive a false-positive GUI cue by adding the + plan** (or equivalent substance). When the diff changes files under `gui/`, + the description must include a screenshot of the UI change; the check keeps + the PR a draft and comments until the screenshot is present. A maintainer can + waive a false-positive GUI cue by adding the `gui-screenshot-waived` label; adding or removing that label immediately re-evaluates the gate. Legacy maintainer comments such as "no gui changes" are still recognised on the next PR event for compatibility, but comments diff --git a/tests/ci-workflows.test.ts b/tests/ci-workflows.test.ts index e83017ed27..c0f41e56f1 100644 --- a/tests/ci-workflows.test.ts +++ b/tests/ci-workflows.test.ts @@ -1090,8 +1090,8 @@ describe("GitHub Actions hardening", () => { expect(script).toContain("collectPrQualityFailures"); expect(script).toContain("collectDeterministicHygieneFailures"); expect(script).toContain("github.rest.pulls.listFiles"); - // The GUI screenshot gate reads the title as well as the body. - expect(script).toContain("title: pr.title"); + // The GUI screenshot gate reads changed file paths under gui/. + expect(script).toContain("changedFilePaths"); expect(script).toContain("github.rest.repos.getCollaboratorPermissionLevel"); expect(script).toContain("github.rest.repos.compareCommitsWithBasehead"); // The allow-list is the gate's whole policy, so it is pinned by value and @@ -1301,6 +1301,11 @@ describe("GitHub Actions hardening", () => { "- [@Wibias](https://github.com/Wibias) was added as a maintainer.", ].join("\n"); + const GUI_CHANGED_FILES = [ + { filename: "gui/src/App.tsx" }, + { filename: "tests/smoke.test.ts" }, + ]; + /** A PR body whose readiness checklist has exactly `checked` boxes ticked. */ function readinessChecklistBody(checked: number, base = CONTRIBUTOR_BODY): string { const boxes = CHECKLIST_ITEMS.map((item, index) => @@ -2458,6 +2463,7 @@ describe("GitHub Actions hardening", () => { title: "GUI: fix provider list spacing", body: readinessChecklistBody(4), }, + files: GUI_CHANGED_FILES, }); expect(methodsOf(result)).toEqual(readsAllowedBase([ @@ -2617,6 +2623,7 @@ describe("GitHub Actions hardening", () => { }, authorPermission: "read", maintainersFile: MAINTAINERS_FIXTURE, + files: GUI_CHANGED_FILES, }); expect( result.warnings.some( @@ -2842,9 +2849,10 @@ describe("GitHub Actions hardening", () => { expect(injected.body).toContain(CHECKLIST_START); }); - test("gui in the title without a screenshot fails and drafts", async () => { + test("gui/ changes without a screenshot fail and draft", async () => { const result = await run({ - pr: { base: { ref: "dev" }, title: "GUI: fix provider list spacing" }, + pr: { base: { ref: "dev" }, title: "Fix dashboard spacing" }, + files: GUI_CHANGED_FILES, }); expect(result.warnings.some((w) => w.startsWith("setFailed:"))).toBe(true); @@ -2855,7 +2863,42 @@ describe("GitHub Actions hardening", () => { expect(callsTo(result, "graphql")).toHaveLength(1); }); - test("gui in the body without a screenshot fails", async () => { + test("gui in the title alone does not demand a screenshot", async () => { + const result = await run({ + pr: { base: { ref: "dev" }, title: "GUI: fix provider list spacing" }, + authorPermission: "write", + files: [{ filename: "scripts/foo.ts" }], + }); + + expect(methodsOf(result)).toEqual(readsAllowedBase()); + expect(result.warnings.some((w) => w.startsWith("setFailed:") && w.includes("screenshot"))).toBe(false); + }); + + test("no gui changes text without gui/ file changes does not demand a screenshot", async () => { + const result = await run({ + pr: { + base: { ref: "dev" }, + title: "Fix proxy routing", + body: [ + "## Summary", + "", + "This change adjusts proxy routing only; there are no gui changes in this PR.", + "The handler in scripts/foo.ts keeps the same public surface while fixing retry semantics.", + "", + "## Test plan", + "", + "- Ran \`bun test tests/ci-workflows.test.ts\`", + ].join("\n"), + }, + authorPermission: "write", + files: [{ filename: "scripts/foo.ts" }], + }); + + expect(methodsOf(result)).toEqual(readsAllowedBase()); + expect(result.warnings.some((w) => w.startsWith("setFailed:") && w.includes("screenshot"))).toBe(false); + }); + + test("gui in the body without a screenshot fails when gui/ changed", async () => { const result = await run({ pr: { base: { ref: "dev" }, @@ -2868,6 +2911,7 @@ describe("GitHub Actions hardening", () => { "- Ran bun test tests/ci-workflows.test.ts", ].join("\n"), }, + files: GUI_CHANGED_FILES, }); expect(result.warnings.some((w) => w.startsWith("setFailed:"))).toBe(true); @@ -2890,6 +2934,7 @@ describe("GitHub Actions hardening", () => { comments: [ { id: 1, user: { login: "lidge-jun" }, author_association: "OWNER", body: "Not touching gui here." }, ], + files: GUI_CHANGED_FILES, }); // The screenshot failure is gone: no setFailed for it, and the comment @@ -2915,6 +2960,7 @@ describe("GitHub Actions hardening", () => { comments: [ { id: 1, user: { login: "wibias" }, author_association: "COLLABORATOR", body: "no gui changes needed" }, ], + files: GUI_CHANGED_FILES, }); expect(result.warnings.some((w) => w.startsWith("setFailed:") && w.includes("screenshot"))).toBe(false); @@ -2947,6 +2993,7 @@ describe("GitHub Actions hardening", () => { actor: { login: "lidge-jun" }, label: { name: "gui-screenshot-waived" }, }], + files: GUI_CHANGED_FILES, }); expect(result.warnings.some((w) => w.startsWith("setFailed:") && w.includes("screenshot"))).toBe(false); @@ -2975,6 +3022,7 @@ describe("GitHub Actions hardening", () => { actor: { login: "lidge-jun" }, label: { name: "gui-screenshot-waived" }, }], + files: GUI_CHANGED_FILES, }); expect(result.warnings.some((w) => w.startsWith("setFailed:") && w.includes("screenshot"))).toBe(false); @@ -3008,6 +3056,7 @@ describe("GitHub Actions hardening", () => { actor: { login: "unauthorized-contributor" }, label: { name: "gui-screenshot-waived" }, }], + files: GUI_CHANGED_FILES, }); // The screenshot failure must remain because the label was applied by @@ -3199,6 +3248,7 @@ describe("GitHub Actions hardening", () => { comments: [ { id: 1, user: { login: "contributor" }, author_association: "CONTRIBUTOR", body: "Not touching gui here." }, ], + files: GUI_CHANGED_FILES, }); expect(result.warnings.some((w) => w.startsWith("setFailed:") && w.includes("screenshot"))).toBe(true); @@ -3222,6 +3272,7 @@ describe("GitHub Actions hardening", () => { comments: [ { id: 1, user: { login: "lidge-jun" }, author_association: "OWNER", body: "This is gui related, please add a screenshot." }, ], + files: GUI_CHANGED_FILES, }); expect(result.warnings.some((w) => w.startsWith("setFailed:") && w.includes("screenshot"))).toBe(true); @@ -3244,6 +3295,7 @@ describe("GitHub Actions hardening", () => { ].join("\n"), }, authorPermission: "write", + files: GUI_CHANGED_FILES, }); expect(methodsOf(result)).toEqual(readsAllowedBase()); @@ -3268,6 +3320,7 @@ describe("GitHub Actions hardening", () => { ].join("\n"), }, authorPermission: "write", + files: GUI_CHANGED_FILES, }); expect(methodsOf(result)).toEqual(readsAllowedBase()); @@ -3291,6 +3344,7 @@ describe("GitHub Actions hardening", () => { "- Ran bun test tests/ci-workflows.test.ts", ].join("\n"), }, + files: GUI_CHANGED_FILES, }); expect(result.warnings.some((w) => w.startsWith("setFailed:"))).toBe(true); From 68368baef7b50eab98897a651f6d04aa875bb413 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Mon, 10 Aug 2026 08:12:05 +0200 Subject: [PATCH 2/4] fix(ci): fail closed when PR file list is truncated for screenshot gate --- .github/scripts/enforce-pr-target.test.cjs | 1 + .github/scripts/pr-quality-messages.cjs | 2 +- .github/scripts/pr-quality.cjs | 9 +++-- .github/scripts/pr-quality.test.cjs | 33 +++++++++++++++++++ .github/workflows/enforce-pr-target.yml | 6 +++- .../content/docs/contributing/pr-quality.md | 3 +- tests/ci-workflows.test.ts | 2 ++ 7 files changed, 51 insertions(+), 5 deletions(-) diff --git a/.github/scripts/enforce-pr-target.test.cjs b/.github/scripts/enforce-pr-target.test.cjs index 4b3d602f8c..8c4ef5b0e3 100644 --- a/.github/scripts/enforce-pr-target.test.cjs +++ b/.github/scripts/enforce-pr-target.test.cjs @@ -198,6 +198,7 @@ describe("enforce-pr-target workflow", () => { assert.ok(qualityCall, "must call collectPrQualityFailures"); assert.match(qualityCall[1], /stackedBase/); assert.match(qualityCall[1], /changedFilePaths/); + assert.match(qualityCall[1], /filesTruncated/); }); it("strips stale WRONG BRANCH prefix on failure when base is corrected", () => { diff --git a/.github/scripts/pr-quality-messages.cjs b/.github/scripts/pr-quality-messages.cjs index 40b3c747b0..6d9b2fff02 100644 --- a/.github/scripts/pr-quality-messages.cjs +++ b/.github/scripts/pr-quality-messages.cjs @@ -229,7 +229,7 @@ function buildFailureSections(failures, { pr, allowedBases, defaultBase }) { sections.push( "⚠️ **UI screenshot required**", "", - `This pull request changes files under ${inlineCode("gui/")}, so it is treated as a GUI change.`, + `This pull request changes files under ${inlineCode("gui/")}, or GitHub returned an incomplete changed-file list for a large diff, so it is treated as a GUI change.`, "", `@${pr.user.login} Please add a screenshot of the UI change to the description — drag and drop the image into the description editor, or paste a markdown image such as ${inlineCode("![Screenshot](https://example.com/after.png)")}. The check re-runs automatically once the description is edited.` ); diff --git a/.github/scripts/pr-quality.cjs b/.github/scripts/pr-quality.cjs index 6106317d02..3012ba8cac 100644 --- a/.github/scripts/pr-quality.cjs +++ b/.github/scripts/pr-quality.cjs @@ -467,7 +467,12 @@ function collectPrQualityFailures({ /** Issue comments; a maintainer comment waives the GUI-screenshot gate. */ guiOverrideComments = [], /** Changed file paths from `pulls.listFiles` (repo-relative). */ - changedFilePaths = [] + changedFilePaths = [], + /** + * True when `pulls.listFiles` returned fewer paths than `pulls.get` + * `changed_files` (GitHub caps the file list at 3,000 entries). + */ + filesTruncated = false }) { const failures = []; const wrongBase = !allowedBases.includes(baseRef) && !stackedBase; @@ -500,7 +505,7 @@ function collectPrQualityFailures({ // not arm the gate when the diff is backend-only. A maintainer comment saying // the change does not touch the GUI still waives a gui/ diff false positive. if ( - guiPathsChanged(changedFilePaths) && + (guiPathsChanged(changedFilePaths) || filesTruncated) && !hasScreenshotEvidence(body) && !hasGuiOverride({ comments: guiOverrideComments }) ) { diff --git a/.github/scripts/pr-quality.test.cjs b/.github/scripts/pr-quality.test.cjs index 329633ea9c..da46ca48bb 100644 --- a/.github/scripts/pr-quality.test.cjs +++ b/.github/scripts/pr-quality.test.cjs @@ -842,6 +842,39 @@ describe("collectPrQualityFailures", () => { assert.ok(!failures.some((f) => f.code === "missing_ui_screenshot")); }); + it("flags truncated file lists even when gui/ is not in the partial list", () => { + const truncatedPaths = Array.from({ length: 3000 }, (_, index) => `scripts/file-${index}.ts`); + const failures = collectPrQualityFailures({ + baseRef: "dev", + allowedBases: allowed, + title: "Large refactor", + body: richBody, + behindMain: 0, + behindBase: 0, + authorPermission: "read", + changedFilePaths: truncatedPaths, + filesTruncated: true, + }); + assert.ok(failures.some((f) => f.code === "missing_ui_screenshot")); + }); + + it("flags truncated file lists when gui/ appears in the partial list", () => { + const truncatedPaths = Array.from({ length: 2999 }, (_, index) => `scripts/file-${index}.ts`); + truncatedPaths.push("gui/src/App.tsx"); + const failures = collectPrQualityFailures({ + baseRef: "dev", + allowedBases: allowed, + title: "Large refactor with gui tweak", + body: richBody, + behindMain: 0, + behindBase: 0, + authorPermission: "read", + changedFilePaths: truncatedPaths, + filesTruncated: true, + }); + assert.ok(failures.some((f) => f.code === "missing_ui_screenshot")); + }); + it("does not flag no gui changes text without gui/ file changes", () => { const failures = collectPrQualityFailures({ baseRef: "dev", diff --git a/.github/workflows/enforce-pr-target.yml b/.github/workflows/enforce-pr-target.yml index 02a05e21c8..9ee7769abb 100644 --- a/.github/workflows/enforce-pr-target.yml +++ b/.github/workflows/enforce-pr-target.yml @@ -541,6 +541,9 @@ jobs: const changedFilePaths = changedFiles .map(file => file.filename) .filter(Boolean); + const filesTruncated = + typeof pr.changed_files === "number" && + pr.changed_files > changedFiles.length; let failures = collectPrQualityFailures({ baseRef: pr.base.ref, @@ -557,7 +560,8 @@ jobs: // A maintainer issue comment ("not touching gui") waives the // GUI-screenshot gate; the comments are already fetched above. guiOverrideComments: comments, - changedFilePaths + changedFilePaths, + filesTruncated }); // Hygiene is a separate workflow that owns the blocked label and diff --git a/docs-site/src/content/docs/contributing/pr-quality.md b/docs-site/src/content/docs/contributing/pr-quality.md index aeb47a23f5..3d7e6e64bf 100644 --- a/docs-site/src/content/docs/contributing/pr-quality.md +++ b/docs-site/src/content/docs/contributing/pr-quality.md @@ -40,7 +40,8 @@ tells you exactly what to change: plan** (or equivalent substance). When the diff changes files under `gui/`, the description must include a screenshot of the UI change; the check keeps the PR a draft and comments until the screenshot is present. A maintainer can - waive a false-positive GUI cue by adding the + waive the screenshot requirement for a `gui/` change, or for a false-positive + GUI-path classification, by adding the `gui-screenshot-waived` label; adding or removing that label immediately re-evaluates the gate. Legacy maintainer comments such as "no gui changes" are still recognised on the next PR event for compatibility, but comments diff --git a/tests/ci-workflows.test.ts b/tests/ci-workflows.test.ts index c0f41e56f1..9327952efc 100644 --- a/tests/ci-workflows.test.ts +++ b/tests/ci-workflows.test.ts @@ -1092,6 +1092,8 @@ describe("GitHub Actions hardening", () => { expect(script).toContain("github.rest.pulls.listFiles"); // The GUI screenshot gate reads changed file paths under gui/. expect(script).toContain("changedFilePaths"); + expect(script).toContain("filesTruncated"); + expect(script).toContain("pr.changed_files"); expect(script).toContain("github.rest.repos.getCollaboratorPermissionLevel"); expect(script).toContain("github.rest.repos.compareCommitsWithBasehead"); // The allow-list is the gate's whole policy, so it is pinned by value and From 4a8653d61b868d4a11165559e64b85fbd25ddfb3 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Mon, 10 Aug 2026 08:28:17 +0200 Subject: [PATCH 3/4] fix(ci): bind screenshot file-list truncation to stable PR head --- .github/scripts/enforce-pr-target.test.cjs | 1 + .github/scripts/pr-quality.cjs | 13 +++++ .github/scripts/pr-quality.test.cjs | 12 ++++ .github/workflows/enforce-pr-target.yml | 55 +++++++++++++++---- .../content/docs/contributing/pr-quality.md | 12 ++-- tests/ci-workflows.test.ts | 11 +++- tests/helpers/enforce-pr-target-harness.ts | 7 +++ 7 files changed, 95 insertions(+), 16 deletions(-) diff --git a/.github/scripts/enforce-pr-target.test.cjs b/.github/scripts/enforce-pr-target.test.cjs index 8c4ef5b0e3..a2e835f9eb 100644 --- a/.github/scripts/enforce-pr-target.test.cjs +++ b/.github/scripts/enforce-pr-target.test.cjs @@ -199,6 +199,7 @@ describe("enforce-pr-target workflow", () => { assert.match(qualityCall[1], /stackedBase/); assert.match(qualityCall[1], /changedFilePaths/); assert.match(qualityCall[1], /filesTruncated/); + assert.match(workflow, /isChangedFileListTruncated/); }); it("strips stale WRONG BRANCH prefix on failure when base is corrected", () => { diff --git a/.github/scripts/pr-quality.cjs b/.github/scripts/pr-quality.cjs index 3012ba8cac..cc841e3809 100644 --- a/.github/scripts/pr-quality.cjs +++ b/.github/scripts/pr-quality.cjs @@ -178,6 +178,18 @@ function guiPathsChanged(files) { ); } +/** + * True when the changed-file list from `pulls.listFiles` cannot be trusted to + * be complete for screenshot gating. Missing or non-integer counts, a head + * mismatch between the count snapshot and the paginated list, or a count above + * the returned list length all fail closed. + */ +function isChangedFileListTruncated(changedFilesCount, listedLength, headMatches = true) { + if (!headMatches) return true; + if (!Number.isInteger(changedFilesCount) || changedFilesCount < 0) return true; + return changedFilesCount > listedLength; +} + /** * True when the PR title or description names the GUI surface as a whole word. * The description is template-stripped first so the template's own screenshot @@ -524,6 +536,7 @@ module.exports = { authorHasPushPermission, assessPrDescription, guiPathsChanged, + isChangedFileListTruncated, hasGuiCue, hasGuiOverride, hasScreenshotEvidence, diff --git a/.github/scripts/pr-quality.test.cjs b/.github/scripts/pr-quality.test.cjs index da46ca48bb..7d61925758 100644 --- a/.github/scripts/pr-quality.test.cjs +++ b/.github/scripts/pr-quality.test.cjs @@ -10,6 +10,7 @@ const { assessPrDescription, hasGuiCue, guiPathsChanged, + isChangedFileListTruncated, hasGuiOverride, hasScreenshotEvidence, buildReviewReadinessSection, @@ -171,6 +172,17 @@ describe("guiPathsChanged", () => { }); }); +describe("isChangedFileListTruncated", () => { + it("treats head drift, invalid counts, and oversized lists as truncated", () => { + assert.equal(isChangedFileListTruncated(10, 10, false), true); + assert.equal(isChangedFileListTruncated(undefined, 10, true), true); + assert.equal(isChangedFileListTruncated(10.5, 10, true), true); + assert.equal(isChangedFileListTruncated(11, 10, true), true); + assert.equal(isChangedFileListTruncated(10, 10, true), false); + assert.equal(isChangedFileListTruncated(5, 10, true), false); + }); +}); + describe("hasGuiOverride", () => { const owner = { author_association: "OWNER", body: "Not touching gui here." }; const collaborator = { author_association: "COLLABORATOR", body: "no gui changes needed" }; diff --git a/.github/workflows/enforce-pr-target.yml b/.github/workflows/enforce-pr-target.yml index 9ee7769abb..d7eb5d3453 100644 --- a/.github/workflows/enforce-pr-target.yml +++ b/.github/workflows/enforce-pr-target.yml @@ -139,6 +139,7 @@ jobs: collectPrQualityFailures, authorHasPushPermission, hasGuiOverride, + isChangedFileListTruncated, extractReviewReadiness, appendReviewReadinessSection, stripReviewReadinessSection, @@ -534,16 +535,50 @@ jobs: } } - const changedFiles = await github.paginate( - github.rest.pulls.listFiles, - { owner, repo, pull_number, per_page: 100 } - ); - const changedFilePaths = changedFiles - .map(file => file.filename) - .filter(Boolean); - const filesTruncated = - typeof pr.changed_files === "number" && - pr.changed_files > changedFiles.length; + const changedFiles = []; + const changedFilePaths = []; + let filesTruncated = true; + for (let attempt = 0; attempt < 2; attempt += 1) { + const { data: fileSnapshot } = await github.rest.pulls.get({ + owner, + repo, + pull_number + }); + const headShaForFiles = fileSnapshot.head?.sha ?? ""; + const listedFiles = await github.paginate( + github.rest.pulls.listFiles, + { owner, repo, pull_number, per_page: 100 } + ); + const { data: fileVerify } = await github.rest.pulls.get({ + owner, + repo, + pull_number + }); + const headMatches = fileVerify.head?.sha === headShaForFiles; + if (!headMatches && attempt === 0) { + core.info( + "PR head moved while listing changed files; retrying once." + ); + continue; + } + if (!headMatches) { + core.warning( + "PR head moved during changed-file snapshot; treating file list as truncated." + ); + } + changedFiles.length = 0; + changedFiles.push(...listedFiles); + changedFilePaths.length = 0; + changedFilePaths.push( + ...listedFiles.map(file => file.filename).filter(Boolean) + ); + filesTruncated = isChangedFileListTruncated( + fileSnapshot.changed_files, + listedFiles.length, + headMatches + ); + break; + } let failures = collectPrQualityFailures({ baseRef: pr.base.ref, diff --git a/docs-site/src/content/docs/contributing/pr-quality.md b/docs-site/src/content/docs/contributing/pr-quality.md index 3d7e6e64bf..b8b5c39ce5 100644 --- a/docs-site/src/content/docs/contributing/pr-quality.md +++ b/docs-site/src/content/docs/contributing/pr-quality.md @@ -37,11 +37,13 @@ tells you exactly what to change: - **PR quality (`enforce-target`).** Pull requests must target `dev` and carry a real description: a **Summary** of what changed and why, plus a **Test - plan** (or equivalent substance). When the diff changes files under `gui/`, - the description must include a screenshot of the UI change; the check keeps - the PR a draft and comments until the screenshot is present. A maintainer can - waive the screenshot requirement for a `gui/` change, or for a false-positive - GUI-path classification, by adding the + plan** (or equivalent substance). When the diff changes files under `gui/`, or + when GitHub returns an incomplete changed-file list for a large diff, the + description must include a screenshot of the UI change; the check keeps + the PR a draft and comments until the screenshot is present. Incomplete file + lists are treated conservatively as a GUI change. A maintainer can waive the + screenshot requirement for a `gui/` change, for a false-positive GUI-path + classification, or for an incomplete-file-list false positive, by adding the `gui-screenshot-waived` label; adding or removing that label immediately re-evaluates the gate. Legacy maintainer comments such as "no gui changes" are still recognised on the next PR event for compatibility, but comments diff --git a/tests/ci-workflows.test.ts b/tests/ci-workflows.test.ts index 9327952efc..dc9c1c3643 100644 --- a/tests/ci-workflows.test.ts +++ b/tests/ci-workflows.test.ts @@ -788,7 +788,9 @@ describe("GitHub Actions hardening", () => { "repos.getCollaboratorPermissionLevel", "repos.compareCommitsWithBasehead", "repos.compareCommitsWithBasehead", + "pulls.get", "pulls.listFiles", + "pulls.get", ...tail, ]; } @@ -801,7 +803,9 @@ describe("GitHub Actions hardening", () => { "issues.listComments", "repos.getCollaboratorPermissionLevel", "pulls.list", + "pulls.get", "pulls.listFiles", + "pulls.get", ...tail, ]; } @@ -818,8 +822,10 @@ describe("GitHub Actions hardening", () => { "repos.compareCommitsWithBasehead", // The harness walks every paginate call across the same page count, so // listFiles appears once per comment page even when the file list is empty. + "pulls.get", "pulls.listFiles", "pulls.listFiles", + "pulls.get", ...tail, ]; } @@ -1093,7 +1099,8 @@ describe("GitHub Actions hardening", () => { // The GUI screenshot gate reads changed file paths under gui/. expect(script).toContain("changedFilePaths"); expect(script).toContain("filesTruncated"); - expect(script).toContain("pr.changed_files"); + expect(script).toContain("isChangedFileListTruncated"); + expect(script).toContain("PR head moved while listing changed files"); expect(script).toContain("github.rest.repos.getCollaboratorPermissionLevel"); expect(script).toContain("github.rest.repos.compareCommitsWithBasehead"); // The allow-list is the gate's whole policy, so it is pinned by value and @@ -3092,6 +3099,8 @@ describe("GitHub Actions hardening", () => { expect(callsTo(result, "pulls.get")).toEqual([ { owner: "lidge-jun", repo: "opencodex", pull_number: 4242 }, + { owner: "lidge-jun", repo: "opencodex", pull_number: 4242 }, + { owner: "lidge-jun", repo: "opencodex", pull_number: 4242 }, ]); expect(callsTo(result, "repos.listPullRequestsAssociatedWithCommit")).toEqual([]); expect(methodsOf(result)).toContain("issues.listComments"); diff --git a/tests/helpers/enforce-pr-target-harness.ts b/tests/helpers/enforce-pr-target-harness.ts index 79a788f1fd..a27b2010f9 100644 --- a/tests/helpers/enforce-pr-target-harness.ts +++ b/tests/helpers/enforce-pr-target-harness.ts @@ -653,6 +653,13 @@ export async function runEnforcePrTarget( const filePages: unknown[][] = options.filePages ?? (options.files && options.files.length > 0 ? [options.files] : [[]]); + const listedFileCount = filePages.flat().length; + const prChangedFiles = (options.pr as { changed_files?: number }).changed_files; + if (Number.isInteger(prChangedFiles)) { + (pr as { changed_files: number }).changed_files = prChangedFiles!; + } else { + (pr as { changed_files: number }).changed_files = listedFileCount; + } const checkRunPages = (options.checkRunPages ?? [options.checkRuns ?? DEFAULT_GREEN_CHECKS]) .map(page => page.map(check => ({ ...check, From 7dd266270b5077d98fafe30cfccafe2812c3ec3c Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Mon, 10 Aug 2026 08:36:02 +0200 Subject: [PATCH 4/4] test(ci): preserve malformed changed_files in PR gate harness --- tests/ci-workflows.test.ts | 14 ++++++++++++++ tests/helpers/enforce-pr-target-harness.ts | 8 +++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/ci-workflows.test.ts b/tests/ci-workflows.test.ts index dc9c1c3643..5652814903 100644 --- a/tests/ci-workflows.test.ts +++ b/tests/ci-workflows.test.ts @@ -2907,6 +2907,20 @@ describe("GitHub Actions hardening", () => { expect(result.warnings.some((w) => w.startsWith("setFailed:") && w.includes("screenshot"))).toBe(false); }); + test("malformed changed_files metadata fails closed on the screenshot gate", async () => { + const result = await run({ + pr: { + base: { ref: "dev" }, + changed_files: 2.5, + }, + files: [{ filename: "scripts/foo.ts" }], + authorPermission: "write", + }); + + expect(result.warnings.some((w) => w.startsWith("setFailed:") && w.includes("screenshot"))).toBe(true); + expect(lastEnforcerCommentBody(result)).toContain("UI screenshot required"); + }); + test("gui in the body without a screenshot fails when gui/ changed", async () => { const result = await run({ pr: { diff --git a/tests/helpers/enforce-pr-target-harness.ts b/tests/helpers/enforce-pr-target-harness.ts index a27b2010f9..ceac6073b6 100644 --- a/tests/helpers/enforce-pr-target-harness.ts +++ b/tests/helpers/enforce-pr-target-harness.ts @@ -53,6 +53,8 @@ export type PullRequestState = { draft?: boolean; base?: { ref: string }; user?: { login: string }; + /** `pulls.get` changed_files; omit to default to listed file count in harness. */ + changed_files?: number; }; export type Comment = { @@ -654,9 +656,9 @@ export async function runEnforcePrTarget( options.filePages ?? (options.files && options.files.length > 0 ? [options.files] : [[]]); const listedFileCount = filePages.flat().length; - const prChangedFiles = (options.pr as { changed_files?: number }).changed_files; - if (Number.isInteger(prChangedFiles)) { - (pr as { changed_files: number }).changed_files = prChangedFiles!; + const prInput = options.pr as Record; + if (Object.prototype.hasOwnProperty.call(prInput, "changed_files")) { + (pr as Record).changed_files = prInput.changed_files; } else { (pr as { changed_files: number }).changed_files = listedFileCount; }