Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 45 additions & 10 deletions .github/workflows/mypy_primer_pr.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,17 @@ jobs:
github.event.workflow_run.event == 'pull_request' &&
github.event.workflow_run.conclusion == 'success' }}
runs-on: ubuntu-latest
concurrency:
group: pr-follow-up-mypy_primer-${{ github.event.workflow_run.head_sha }}
cancel-in-progress: false
permissions:
contents: read
pull-requests: read
checks: write
outputs:
pr-number: ${{ steps.resolve.outputs.pr-number }}
merge-sha: ${{ steps.resolve.outputs.merge-sha }}
check-id: ${{ steps.resolve.outputs.check-id }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
Expand All @@ -58,16 +66,13 @@ jobs:
const resolve = require('./build/resolveValidatedPullRequest.js')
const validated = await resolve({ github, context, core })
if (!validated) return
const { pullRequest, files } = validated
const paths = [
'packages/pyright/',
'packages/pyright-internal/src/',
'packages/pyright-internal/typeshed-fallback/',
]
if (!files.some((file) => paths.some((path) => file.filename.startsWith(path)))) {
core.notice('The pull request does not change primer inputs')
return
}
const { pullRequest } = validated
const report = require('./build/reportPullRequestCheck.js')
const checkId = await report({
github, context, name: 'mypy_primer', headSha: pullRequest.head.sha,
status: 'in_progress', summary: 'Running mypy_primer after successful Validation.',
})
core.setOutput('check-id', checkId)
if (!pullRequest.merge_commit_sha) {
core.setFailed('The pull request does not have a merge commit to test')
return
Expand Down Expand Up @@ -181,3 +186,33 @@ jobs:
with:
name: mypy_primer_diffs_pr_number
path: pr_number.txt

report:
name: Report mypy_primer check
needs: [resolve, mypy_primer]
if: ${{ always() && needs.resolve.outputs.check-id != '' }}
runs-on: ubuntu-latest
permissions:
contents: read
checks: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.event.repository.default_branch }}
persist-credentials: false
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
CHECK_ID: ${{ needs.resolve.outputs.check-id }}
RESOLVE_RESULT: ${{ needs.resolve.result }}
PRIMER_RESULT: ${{ needs.mypy_primer.result }}
with:
script: |
const report = require('./build/reportPullRequestCheck.js')
const results = [process.env.RESOLVE_RESULT, process.env.PRIMER_RESULT]
const conclusion = results.includes('failure') ? 'failure'
: results.includes('cancelled') ? 'cancelled'
: process.env.PRIMER_RESULT
await report({
github, context, name: 'mypy_primer', checkId: process.env.CHECK_ID,
conclusion, summary: `mypy_primer: ${conclusion}. See the workflow run for shard results.`,
})
58 changes: 58 additions & 0 deletions .github/workflows/pr_follow_up_checks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
name: Queue PR follow-up checks

on:
pull_request_target:
branches: [main]
types: [opened, synchronize, reopened]
workflow_run:
workflows: [Validation]
types: [completed]

permissions:
contents: read
pull-requests: read
checks: write

jobs:
report:
if: >-
${{ github.repository == 'microsoft/pyright' &&
(github.event_name == 'pull_request_target' ||
(github.event.workflow_run.event == 'pull_request' &&
github.event.workflow_run.conclusion != 'success')) }}
runs-on: ubuntu-latest
strategy:
matrix:
check: [mypy_primer, Type checker benchmark]
concurrency:
group: pr-follow-up-${{ matrix.check }}-${{ github.event.pull_request.head.sha || github.event.workflow_run.head_sha }}
cancel-in-progress: false
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.event.repository.default_branch }}
persist-credentials: false
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
CHECK_NAME: ${{ matrix.check }}
with:
script: |
const report = require('./build/reportPullRequestCheck.js')
let headSha
const failedValidation = context.eventName === 'workflow_run'
if (failedValidation) {
const resolve = require('./build/resolveValidatedPullRequest.js')
const validated = await resolve({ github, context, core, requireSuccess: false })
if (!validated) return
headSha = validated.pullRequest.head.sha
} else {
headSha = context.payload.pull_request.head.sha
}
await report({
github, context, name: process.env.CHECK_NAME, headSha,
status: failedValidation ? 'completed' : 'queued',
conclusion: failedValidation ? 'skipped' : undefined,
summary: failedValidation
? 'Not run because Validation did not pass.'
: 'Waiting for Validation to pass.',
})
47 changes: 47 additions & 0 deletions .github/workflows/typecheck_benchmark_trigger.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,17 @@ jobs:
actions: read
contents: read
pull-requests: read
checks: write
runs-on: ubuntu-latest
concurrency:
group: pr-follow-up-Type checker benchmark-${{ github.event.workflow_run.head_sha || github.event.issue.number }}
cancel-in-progress: false
outputs:
pr-number: ${{ steps.resolve.outputs.pr-number }}
head-sha: ${{ steps.resolve.outputs.head-sha }}
base-sha: ${{ steps.resolve.outputs.base-sha }}
merge-sha: ${{ steps.resolve.outputs.merge-sha }}
check-id: ${{ steps.resolve.outputs.check-id }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
Expand All @@ -52,6 +57,12 @@ jobs:
const { pullRequest, files } = validated
issueNumber = pullRequest.number
if (!files.some((file) => file.filename.startsWith('packages/pyright-internal/src/analyzer/'))) {
const report = require('./build/reportPullRequestCheck.js')
await report({
github, context, name: 'Type checker benchmark', headSha: pullRequest.head.sha,
status: 'completed', conclusion: 'skipped',
summary: 'Not run because the pull request does not change analyzer sources.',
})
core.notice('The pull request does not change analyzer sources')
return
}
Expand Down Expand Up @@ -105,6 +116,12 @@ jobs:
return
}
}
const report = require('./build/reportPullRequestCheck.js')
const checkId = await report({
github, context, name: 'Type checker benchmark', headSha: pullRequest.data.head.sha,
status: 'in_progress', summary: 'Running analyzer benchmarks after successful Validation.',
})
core.setOutput('check-id', checkId)
let candidateRef
if (pullRequest.data.merged) {
candidateRef = pullRequest.data.merge_commit_sha
Expand Down Expand Up @@ -165,3 +182,33 @@ jobs:
head_sha: ${{ needs.trigger.outputs.head-sha }}
base_sha: ${{ needs.trigger.outputs.base-sha }}
merge_sha: ${{ needs.trigger.outputs.merge-sha }}

report:
name: Report benchmark check
needs: [trigger, benchmark]
if: ${{ always() && needs.trigger.outputs.check-id != '' }}
runs-on: ubuntu-latest
permissions:
contents: read
checks: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.event.repository.default_branch }}
persist-credentials: false
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
CHECK_ID: ${{ needs.trigger.outputs.check-id }}
TRIGGER_RESULT: ${{ needs.trigger.result }}
BENCHMARK_RESULT: ${{ needs.benchmark.result }}
with:
script: |
const report = require('./build/reportPullRequestCheck.js')
const results = [process.env.TRIGGER_RESULT, process.env.BENCHMARK_RESULT]
const conclusion = results.includes('failure') ? 'failure'
: results.includes('cancelled') ? 'cancelled'
: process.env.BENCHMARK_RESULT
await report({
github, context, name: 'Type checker benchmark', checkId: process.env.CHECK_ID,
conclusion, summary: `Type checker benchmark: ${conclusion}. See the workflow run for results.`,
})
8 changes: 7 additions & 1 deletion build/benchmark/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -119,11 +119,17 @@ result contract, and the comparator rejects mismatched environments.

Pull requests that change `packages/pyright-internal/src/analyzer/` automatically run the hosted
benchmark only after `Validation` succeeds, including all test platforms, type checking, style, and
the build. `mypy_primer` also waits for successful Validation and retains its own path filters. For
the build. `mypy_primer` also waits for successful Validation and runs on every pull request. For
external contributors, both wait for a maintainer to approve workflows to run before Validation can
start. Subsequent commits require successful Validation for the new head before either check runs.
The trusted triggers run from the base repository and reject outdated PR heads.

Both follow-up checks appear on the PR head as queued while Validation runs, then report their
execution results with links to the workflow runs. If Validation does not pass, both are skipped;
the benchmark is also skipped for PRs without analyzer changes. Check reporting does not change
branch protection. Only trusted reporting jobs receive permission to write checks; jobs executing
the candidate code keep read-only repository access.

To run the benchmark again, or to run it for another pull request, a maintainer can comment exactly
`/benchmark` on an open or merged pull request. The command must be the entire comment. The trusted
command workflow checks that the commenter has `write`, `maintain`, or `admin` repository permission
Expand Down
Loading
Loading