diff --git a/.github/workflows/mypy_primer_pr.yaml b/.github/workflows/mypy_primer_pr.yaml index 930b9524e1f8..68fde777de31 100644 --- a/.github/workflows/mypy_primer_pr.yaml +++ b/.github/workflows/mypy_primer_pr.yaml @@ -42,9 +42,17 @@ jobs: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest + concurrency: + group: pr-follow-up-mypy_primer-${{ github.event.workflow_run.head_sha }} + cancel-in-progress: false + permissions: + contents: read + pull-requests: read + checks: write outputs: pr-number: ${{ steps.resolve.outputs.pr-number }} merge-sha: ${{ steps.resolve.outputs.merge-sha }} + check-id: ${{ steps.resolve.outputs.check-id }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -58,16 +66,13 @@ jobs: const resolve = require('./build/resolveValidatedPullRequest.js') const validated = await resolve({ github, context, core }) if (!validated) return - const { pullRequest, files } = validated - const paths = [ - 'packages/pyright/', - 'packages/pyright-internal/src/', - 'packages/pyright-internal/typeshed-fallback/', - ] - if (!files.some((file) => paths.some((path) => file.filename.startsWith(path)))) { - core.notice('The pull request does not change primer inputs') - return - } + const { pullRequest } = validated + const report = require('./build/reportPullRequestCheck.js') + const checkId = await report({ + github, context, name: 'mypy_primer', headSha: pullRequest.head.sha, + status: 'in_progress', summary: 'Running mypy_primer after successful Validation.', + }) + core.setOutput('check-id', checkId) if (!pullRequest.merge_commit_sha) { core.setFailed('The pull request does not have a merge commit to test') return @@ -181,3 +186,33 @@ jobs: with: name: mypy_primer_diffs_pr_number path: pr_number.txt + + report: + name: Report mypy_primer check + needs: [resolve, mypy_primer] + if: ${{ always() && needs.resolve.outputs.check-id != '' }} + runs-on: ubuntu-latest + permissions: + contents: read + checks: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + CHECK_ID: ${{ needs.resolve.outputs.check-id }} + RESOLVE_RESULT: ${{ needs.resolve.result }} + PRIMER_RESULT: ${{ needs.mypy_primer.result }} + with: + script: | + const report = require('./build/reportPullRequestCheck.js') + const results = [process.env.RESOLVE_RESULT, process.env.PRIMER_RESULT] + const conclusion = results.includes('failure') ? 'failure' + : results.includes('cancelled') ? 'cancelled' + : process.env.PRIMER_RESULT + await report({ + github, context, name: 'mypy_primer', checkId: process.env.CHECK_ID, + conclusion, summary: `mypy_primer: ${conclusion}. See the workflow run for shard results.`, + }) diff --git a/.github/workflows/pr_follow_up_checks.yml b/.github/workflows/pr_follow_up_checks.yml new file mode 100644 index 000000000000..88e6da66366d --- /dev/null +++ b/.github/workflows/pr_follow_up_checks.yml @@ -0,0 +1,58 @@ +name: Queue PR follow-up checks + +on: + pull_request_target: + branches: [main] + types: [opened, synchronize, reopened] + workflow_run: + workflows: [Validation] + types: [completed] + +permissions: + contents: read + pull-requests: read + checks: write + +jobs: + report: + if: >- + ${{ github.repository == 'microsoft/pyright' && + (github.event_name == 'pull_request_target' || + (github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion != 'success')) }} + runs-on: ubuntu-latest + strategy: + matrix: + check: [mypy_primer, Type checker benchmark] + concurrency: + group: pr-follow-up-${{ matrix.check }}-${{ github.event.pull_request.head.sha || github.event.workflow_run.head_sha }} + cancel-in-progress: false + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + CHECK_NAME: ${{ matrix.check }} + with: + script: | + const report = require('./build/reportPullRequestCheck.js') + let headSha + const failedValidation = context.eventName === 'workflow_run' + if (failedValidation) { + const resolve = require('./build/resolveValidatedPullRequest.js') + const validated = await resolve({ github, context, core, requireSuccess: false }) + if (!validated) return + headSha = validated.pullRequest.head.sha + } else { + headSha = context.payload.pull_request.head.sha + } + await report({ + github, context, name: process.env.CHECK_NAME, headSha, + status: failedValidation ? 'completed' : 'queued', + conclusion: failedValidation ? 'skipped' : undefined, + summary: failedValidation + ? 'Not run because Validation did not pass.' + : 'Waiting for Validation to pass.', + }) diff --git a/.github/workflows/typecheck_benchmark_trigger.yml b/.github/workflows/typecheck_benchmark_trigger.yml index ca5b850af767..0ba15d51cfdb 100644 --- a/.github/workflows/typecheck_benchmark_trigger.yml +++ b/.github/workflows/typecheck_benchmark_trigger.yml @@ -27,12 +27,17 @@ jobs: actions: read contents: read pull-requests: read + checks: write runs-on: ubuntu-latest + concurrency: + group: pr-follow-up-Type checker benchmark-${{ github.event.workflow_run.head_sha || github.event.issue.number }} + cancel-in-progress: false outputs: pr-number: ${{ steps.resolve.outputs.pr-number }} head-sha: ${{ steps.resolve.outputs.head-sha }} base-sha: ${{ steps.resolve.outputs.base-sha }} merge-sha: ${{ steps.resolve.outputs.merge-sha }} + check-id: ${{ steps.resolve.outputs.check-id }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -52,6 +57,12 @@ jobs: const { pullRequest, files } = validated issueNumber = pullRequest.number if (!files.some((file) => file.filename.startsWith('packages/pyright-internal/src/analyzer/'))) { + const report = require('./build/reportPullRequestCheck.js') + await report({ + github, context, name: 'Type checker benchmark', headSha: pullRequest.head.sha, + status: 'completed', conclusion: 'skipped', + summary: 'Not run because the pull request does not change analyzer sources.', + }) core.notice('The pull request does not change analyzer sources') return } @@ -105,6 +116,12 @@ jobs: return } } + const report = require('./build/reportPullRequestCheck.js') + const checkId = await report({ + github, context, name: 'Type checker benchmark', headSha: pullRequest.data.head.sha, + status: 'in_progress', summary: 'Running analyzer benchmarks after successful Validation.', + }) + core.setOutput('check-id', checkId) let candidateRef if (pullRequest.data.merged) { candidateRef = pullRequest.data.merge_commit_sha @@ -165,3 +182,33 @@ jobs: head_sha: ${{ needs.trigger.outputs.head-sha }} base_sha: ${{ needs.trigger.outputs.base-sha }} merge_sha: ${{ needs.trigger.outputs.merge-sha }} + + report: + name: Report benchmark check + needs: [trigger, benchmark] + if: ${{ always() && needs.trigger.outputs.check-id != '' }} + runs-on: ubuntu-latest + permissions: + contents: read + checks: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + CHECK_ID: ${{ needs.trigger.outputs.check-id }} + TRIGGER_RESULT: ${{ needs.trigger.result }} + BENCHMARK_RESULT: ${{ needs.benchmark.result }} + with: + script: | + const report = require('./build/reportPullRequestCheck.js') + const results = [process.env.TRIGGER_RESULT, process.env.BENCHMARK_RESULT] + const conclusion = results.includes('failure') ? 'failure' + : results.includes('cancelled') ? 'cancelled' + : process.env.BENCHMARK_RESULT + await report({ + github, context, name: 'Type checker benchmark', checkId: process.env.CHECK_ID, + conclusion, summary: `Type checker benchmark: ${conclusion}. See the workflow run for results.`, + }) diff --git a/build/benchmark/README.md b/build/benchmark/README.md index 294e0e5dfe73..6df8f7b37369 100644 --- a/build/benchmark/README.md +++ b/build/benchmark/README.md @@ -119,11 +119,17 @@ result contract, and the comparator rejects mismatched environments. Pull requests that change `packages/pyright-internal/src/analyzer/` automatically run the hosted benchmark only after `Validation` succeeds, including all test platforms, type checking, style, and -the build. `mypy_primer` also waits for successful Validation and retains its own path filters. For +the build. `mypy_primer` also waits for successful Validation and runs on every pull request. For external contributors, both wait for a maintainer to approve workflows to run before Validation can start. Subsequent commits require successful Validation for the new head before either check runs. The trusted triggers run from the base repository and reject outdated PR heads. +Both follow-up checks appear on the PR head as queued while Validation runs, then report their +execution results with links to the workflow runs. If Validation does not pass, both are skipped; +the benchmark is also skipped for PRs without analyzer changes. Check reporting does not change +branch protection. Only trusted reporting jobs receive permission to write checks; jobs executing +the candidate code keep read-only repository access. + To run the benchmark again, or to run it for another pull request, a maintainer can comment exactly `/benchmark` on an open or merged pull request. The command must be the entire comment. The trusted command workflow checks that the commenter has `write`, `maintain`, or `admin` repository permission diff --git a/build/benchmark/test_compare_benchmarks.py b/build/benchmark/test_compare_benchmarks.py index b964d2e802be..69af26c54666 100644 --- a/build/benchmark/test_compare_benchmarks.py +++ b/build/benchmark/test_compare_benchmarks.py @@ -1052,7 +1052,12 @@ def test_pr_benchmark_requires_authorized_comment(self) -> None: ) self.assertEqual( trigger_workflow_data["jobs"]["trigger"]["permissions"], - {"actions": "read", "contents": "read", "pull-requests": "read"}, + { + "actions": "read", + "contents": "read", + "pull-requests": "read", + "checks": "write", + }, ) self.assertNotIn("actions: write", trigger_workflow) self.assertNotIn("createWorkflowDispatch", trigger_workflow) @@ -1138,7 +1143,7 @@ def test_primer_waits_for_validation_and_uses_read_only_candidate(self) -> None: self.assertNotIn("GITHUB_SHA", script) self.assertIn('git branch new_commit "$PRIMER_SHA"', script) - def test_validated_pr_resolution_and_separate_path_filters(self) -> None: + def test_validated_pr_resolution_and_benchmark_only_path_filter(self) -> None: script = r""" const assert = require('assert').strict; const fs = require('fs'); @@ -1148,7 +1153,7 @@ def test_validated_pr_resolution_and_separate_path_filters(self) -> None: const workflows = ['typecheck_benchmark_trigger.yml', 'mypy_primer_pr.yaml'].map( (name) => YAML.parse(fs.readFileSync(`.github/workflows/${name}`, 'utf8')) ); -async function check(runChanges = {}, prChanges = {}, filenames = [], script) { +async function check(runChanges = {}, prChanges = {}, filenames = [], script, requireSuccess = true) { const run = { name: 'Validation', path: '.github/workflows/validation.yml', event: 'pull_request', conclusion: 'success', @@ -1179,12 +1184,17 @@ def test_validated_pr_resolution_and_separate_path_filters(self) -> None: parents: [{ sha: 'c'.repeat(40) }, { sha: pr.head.sha }], } }), }, + checks: { + listForRef: 'checks', + create: async (parameters) => ({ data: { ...parameters, id: 42 } }), + }, }, paginate: async (method, parameters) => { if (method === 'list') { assert.equal(parameters.head, 'external:feature'); return [pr]; } + if (method === 'checks') return []; assert.equal(method, 'files'); return filenames.map((filename) => ({ filename })); }, @@ -1195,12 +1205,13 @@ def test_validated_pr_resolution_and_separate_path_filters(self) -> None: ); return outputs['pr-number']; } - return resolve({ github, context, core }); + return resolve({ github, context, core, requireSuccess }); } (async () => { assert.equal((await check()).pullRequest.number, 7); for (const conclusion of ['failure', 'cancelled', null]) { assert.equal(await check({ conclusion }), undefined); + assert.equal((await check({ conclusion }, {}, [], undefined, false)).pullRequest.number, 7); } assert.equal(await check({ event: 'push' }), undefined); assert.equal(await check({ path: '.github/workflows/other.yml' }), undefined); @@ -1217,13 +1228,173 @@ def test_validated_pr_resolution_and_separate_path_filters(self) -> None: ['packages/pyright-internal/src/tests/sample.py', [undefined, 7]], ['packages/pyright/index.js', [undefined, 7]], ['packages/pyright-internal/typeshed-fallback/stdlib/os.pyi', [undefined, 7]], - ['README.md', [undefined, undefined]], + ['README.md', [undefined, 7]], + ['.github/workflows/validation.yml', [undefined, 7]], ]) { for (let index = 0; index < scripts.length; index++) { assert.equal(await check({}, {}, [filename], scripts[index]), expected[index]); } } })().catch((error) => { console.error(error); process.exitCode = 1; }); +""" + subprocess.run(["node", "-e", script], cwd=REPO_ROOT, check=True) + + def test_follow_up_checks_are_queued_and_report_actual_results(self) -> None: + script = r""" +const assert = require('assert').strict; +const fs = require('fs'); +const YAML = require('yaml'); +const report = require('./build/reportPullRequestCheck.js'); +const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor; +const load = (name) => YAML.parse(fs.readFileSync(`.github/workflows/${name}`, 'utf8')); +const queue = load('pr_follow_up_checks.yml'); +assert.deepEqual(queue.on.pull_request_target, { + branches: ['main'], types: ['opened', 'synchronize', 'reopened'], +}); +assert.deepEqual(queue.on.workflow_run, { workflows: ['Validation'], types: ['completed'] }); +assert.equal(queue.permissions.checks, 'write'); +assert.ok(queue.jobs.report.if.includes("github.event.workflow_run.conclusion != 'success'")); +assert.deepEqual(queue.jobs.report.strategy.matrix.check, ['mypy_primer', 'Type checker benchmark']); +assert.equal(queue.jobs.report.concurrency['cancel-in-progress'], false); +assert.ok(queue.jobs.report.concurrency.group.includes('matrix.check')); +const workflows = [ + [load('mypy_primer_pr.yaml'), 'resolve', 'mypy_primer', 'RESOLVE_RESULT', 'PRIMER_RESULT'], + [load('typecheck_benchmark_trigger.yml'), 'trigger', 'benchmark', 'TRIGGER_RESULT', 'BENCHMARK_RESULT'], +]; +const context = { + repo: { owner: 'microsoft', repo: 'pyright' }, + serverUrl: 'https://github.com', runId: 123, +}; +let checks = []; +let nextId = 1; +const updates = []; +const github = { + rest: { checks: { + listForRef: 'checks', + create: async (parameters) => { + if (parameters.status === 'completed') { + assert.ok(parameters.conclusion); + assert.ok(Number.isFinite(Date.parse(parameters.completed_at))); + } + const check = { ...parameters, id: nextId++ }; + checks.unshift(check); + return { data: check }; + }, + update: async (parameters) => { + if (parameters.status === 'completed') { + assert.ok(parameters.conclusion); + assert.ok(Number.isFinite(Date.parse(parameters.completed_at))); + } + const check = checks.find((check) => check.id === parameters.check_run_id); + assert.ok(check); + Object.assign(check, parameters); + updates.push(parameters); + return { data: check }; + }, + } }, + paginate: async (method, parameters) => { + assert.equal(method, 'checks'); + return checks.filter((check) => + check.head_sha === parameters.ref && check.name === parameters.check_name + ); + }, +}; +const execute = async (script, eventContext = context) => + new AsyncFunction('github', 'context', 'core', 'require', script)( + github, eventContext, { notice: () => {} }, require + ); +(async () => { + const args = { + github, context, name: 'mypy_primer', headSha: 'a'.repeat(40), + status: 'queued', summary: 'Waiting for Validation to pass.', + }; + const id = await report(args); + assert.equal(checks[0].status, 'queued'); + assert.equal(checks[0].head_sha, args.headSha); + assert.equal(checks[0].details_url, 'https://github.com/microsoft/pyright/actions/runs/123'); + await report(args); + assert.equal(checks.length, 1); + assert.equal(await report({ ...args, status: 'in_progress' }), id); + assert.ok(Number.isFinite(Date.parse(checks[0].started_at))); + await report(args); + assert.equal(checks[0].status, 'in_progress'); + await report({ ...args, status: 'completed', conclusion: 'skipped' }); + assert.equal(checks[0].status, 'in_progress'); + const rerunId = await report({ ...args, status: 'in_progress' }); + assert.notEqual(rerunId, id); + await report({ ...args, checkId: id, conclusion: 'cancelled' }); + assert.equal(checks[0].status, 'in_progress'); + assert.equal(checks[1].conclusion, 'cancelled'); + await report({ ...args, checkId: rerunId, conclusion: 'success' }); + await report(args); + assert.equal(checks[0].conclusion, 'success'); + await report({ ...args, headSha: 'b'.repeat(40) }); + assert.equal(checks[0].status, 'queued'); + await report({ ...args, headSha: 'b'.repeat(40), status: 'completed', conclusion: 'skipped' }); + assert.equal(checks[0].conclusion, 'skipped'); + await report({ ...args, headSha: 'd'.repeat(40), status: 'completed', conclusion: 'skipped' }); + assert.equal(checks[0].conclusion, 'skipped'); + + for (const [workflow, resolveJob, workerJob, resolveEnv, workerEnv] of workflows) { + const resolver = workflow.jobs[resolveJob]; + const reporter = workflow.jobs.report; + assert.equal(resolver.permissions.checks, 'write'); + assert.equal(resolver.concurrency['cancel-in-progress'], false); + assert.ok(resolver.concurrency.group.includes('github.event.workflow_run.head_sha')); + assert.ok(resolver.outputs['check-id']); + assert.deepEqual(reporter.needs, [resolveJob, workerJob]); + assert.ok(reporter.if.includes('always()')); + assert.ok(reporter.if.includes(`needs.${resolveJob}.outputs.check-id != ''`)); + assert.deepEqual(reporter.permissions, { contents: 'read', checks: 'write' }); + for (const job of [resolver, reporter]) { + assert.equal(job.steps[0].with.ref, '${{ github.event.repository.default_branch }}'); + assert.equal(job.steps[0].with['persist-credentials'], false); + } + const step = reporter.steps[1]; + assert.equal(step.env.CHECK_ID, '${{ needs.' + resolveJob + '.outputs.check-id }}'); + assert.equal(step.env[resolveEnv], '${{ needs.' + resolveJob + '.result }}'); + assert.equal(step.env[workerEnv], '${{ needs.' + workerJob + '.result }}'); + for (const [resolveResult, workerResult, expected] of [ + ['success', 'success', 'success'], + ['success', 'failure', 'failure'], + ['failure', 'skipped', 'failure'], + ['success', 'cancelled', 'cancelled'], + ['cancelled', 'skipped', 'cancelled'], + ['success', 'skipped', 'skipped'], + ]) { + process.env.CHECK_ID = String(rerunId); + process.env[resolveEnv] = resolveResult; + process.env[workerEnv] = workerResult; + await execute(step.with.script); + assert.equal(updates.at(-1).conclusion, expected); + } + } + const primer = workflows[0][0].jobs.mypy_primer; + const benchmark = load('typecheck_benchmark_pr.yml').jobs.benchmark; + assert.deepEqual(primer.permissions, { contents: 'read' }); + assert.deepEqual(benchmark.permissions, { contents: 'read' }); + assert.equal(queue.jobs.report.steps[0].with.ref, '${{ github.event.repository.default_branch }}'); + assert.equal(queue.jobs.report.steps[0].with['persist-credentials'], false); + const queueScript = queue.jobs.report.steps[1].with.script; + assert.equal(queue.jobs.report.steps[1].env.CHECK_NAME, '${{ matrix.check }}'); + for (const name of queue.jobs.report.strategy.matrix.check) { + process.env.CHECK_NAME = name; + await execute(queueScript, { + ...context, eventName: 'pull_request_target', + payload: { pull_request: { head: { sha: 'c'.repeat(40) } } }, + }); + } + assert.deepEqual(checks.slice(0, 2).map((check) => check.status), ['queued', 'queued']); + assert.deepEqual(checks.slice(0, 2).map((check) => check.head_sha), ['c'.repeat(40), 'c'.repeat(40)]); + await execute(queueScript, { + ...context, eventName: 'workflow_run', + payload: { workflow_run: { + name: 'Validation', path: '.github/workflows/other.yml', + event: 'pull_request', conclusion: 'failure', + } }, + }); + assert.equal(checks[0].status, 'queued'); +})().catch((error) => { console.error(error); process.exitCode = 1; }); """ subprocess.run(["node", "-e", script], cwd=REPO_ROOT, check=True) diff --git a/build/reportPullRequestCheck.js b/build/reportPullRequestCheck.js new file mode 100644 index 000000000000..e5a331e0b50b --- /dev/null +++ b/build/reportPullRequestCheck.js @@ -0,0 +1,51 @@ +/* + * reportPullRequestCheck.js + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + */ + +module.exports = async ({ github, context, name, headSha, status, conclusion, summary, checkId }) => { + const externalId = `pyright-follow-up:${name}`; + const output = { title: name, summary }; + if (checkId) { + await github.rest.checks.update({ + ...context.repo, + check_run_id: Number(checkId), + status: 'completed', + conclusion, + completed_at: new Date().toISOString(), + output, + }); + return; + } + + const checks = await github.paginate(github.rest.checks.listForRef, { + ...context.repo, + ref: headSha, + check_name: name, + filter: 'latest', + per_page: 100, + }); + const existing = checks.find((check) => check.external_id === externalId); + // PR-open reporting can race with a fast Validation run. Never reset an active or finished check. + if (existing && (status === 'queued' || (status === 'completed' && existing.status !== 'queued'))) { + return; + } + const parameters = { + ...context.repo, + name, + external_id: externalId, + status, + ...(conclusion ? { conclusion } : {}), + ...(status === 'in_progress' ? { started_at: new Date().toISOString() } : {}), + ...(status === 'completed' ? { completed_at: new Date().toISOString() } : {}), + details_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, + output, + }; + // Each execution gets its own check so a cancelled older run cannot finish a newer run's check. + const { data } = + existing?.status === 'queued' + ? await github.rest.checks.update({ ...parameters, check_run_id: existing.id }) + : await github.rest.checks.create({ ...parameters, head_sha: headSha }); + return data.id; +}; diff --git a/build/resolveValidatedPullRequest.js b/build/resolveValidatedPullRequest.js index b0f1ab735cba..a6119c925edb 100644 --- a/build/resolveValidatedPullRequest.js +++ b/build/resolveValidatedPullRequest.js @@ -4,13 +4,13 @@ * Licensed under the MIT license. */ -module.exports = async ({ github, context, core }) => { +module.exports = async ({ github, context, core, requireSuccess = true }) => { const run = context.payload.workflow_run; if ( run.name !== 'Validation' || run.path !== '.github/workflows/validation.yml' || run.event !== 'pull_request' || - run.conclusion !== 'success' || + (requireSuccess && run.conclusion !== 'success') || run.repository.full_name !== `${context.repo.owner}/${context.repo.repo}` ) { core.notice('Only successful pull request Validation runs can start follow-up checks');