Repository navigation
136 lines (132 loc) · 5.64 KB
/
Copy pathperf-regression.yml
File metadata and controls
136 lines (132 loc) · 5.64 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
name: Performance regression
# Same-run A/B perf gate on the CleanBench T5 workload: HEAD is measured against
# a base commit on the same runner, interleaved, and fails only when it is >20%
# slower or uses >15% more peak memory than base AND a confirmation re-run
# reproduces the breach. No absolute numbers from other machines are compared.
#
# pull_request PR head vs the PR base commit (blocking)
# daily main vs main as of ~26 hours earlier (blocking, alert issue)
# weekly main vs the latest release tag (report only)
# manual any base_ref; accept_regression=true reports without failing
on:
# schedule: disabled in favor of ci-verification-gates.yml
# pull_request: disabled in favor of ci-verification-gates.yml
workflow_dispatch:
inputs:
base_ref:
description: "Base ref to compare against (default: main as of 26h ago)"
type: string
default: ""
accept_regression:
description: "Report only; do not fail on a regression"
type: boolean
default: false
# Cancel superseded runs only for pull requests. Scheduled and manual runs are
# grouped per commit so they never cancel or replace each other's queued runs.
concurrency:
group: perf-regression-${{ github.event_name }}-${{ github.event_name == 'pull_request' && github.ref || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
permissions:
contents: read
jobs:
perf:
runs-on: ubuntu-latest
timeout-minutes: 45
permissions:
contents: read
issues: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0 # the base commit and release tags must be resolvable
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.12"
cache: pip
- name: Install
run: |
python -m pip install -U pip
pip install -c constraints/ci.txt -e ".[bench]"
- name: Resolve base commit
id: base
env:
EVENT: ${{ github.event_name }}
SCHEDULE: ${{ github.event.schedule }}
PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
INPUT_BASE_REF: ${{ inputs.base_ref }}
INPUT_ACCEPT: ${{ inputs.accept_regression }}
run: |
set -euo pipefail
mode=gate
if [ "$EVENT" = "pull_request" ]; then
ref="$PR_BASE_SHA"
elif [ "$EVENT" = "workflow_dispatch" ] && [ -n "$INPUT_BASE_REF" ]; then
ref="$INPUT_BASE_REF"
elif [ "$SCHEDULE" = "30 5 * * 1" ]; then
ref="$(git describe --tags --abbrev=0 --match 'v*' origin/main)"
mode=report
else
ref="$(git rev-list -1 --before='26 hours ago' origin/main)"
fi
if [ "$INPUT_ACCEPT" = "true" ]; then mode=report; fi
sha="$(git rev-parse --verify "${ref}^{commit}")"
{
echo "ref=$ref"
echo "sha=$sha"
echo "mode=$mode"
} >> "$GITHUB_OUTPUT"
if [ "$sha" = "$(git rev-parse HEAD)" ]; then
echo "skip=true" >> "$GITHUB_OUTPUT"
echo "Base \`$ref\` is HEAD; nothing to compare." >> "$GITHUB_STEP_SUMMARY"
else
echo "skip=false" >> "$GITHUB_OUTPUT"
git worktree add --detach "$RUNNER_TEMP/perf-base" "$sha"
fi
- name: Same-run A/B perf gate
if: steps.base.outputs.skip != 'true'
env:
BASE_REF: ${{ steps.base.outputs.ref }}
BASE_SHA: ${{ steps.base.outputs.sha }}
MODE: ${{ steps.base.outputs.mode }}
run: |
GATE=""
if [ "$MODE" = "gate" ]; then GATE="--check-gates"; fi
python -m benchmarks.cleanbench.ab \
--base-src "$RUNNER_TEMP/perf-base/src" \
--head-src src \
--base-label "$BASE_REF (${BASE_SHA:0:7})" \
--head-label "HEAD (${GITHUB_SHA:0:7})" \
--output benchmarks/cleanbench/results/latest.ab.json \
--summary "$GITHUB_STEP_SUMMARY" \
$GATE
- name: Upload results
if: always() && steps.base.outputs.skip == 'false'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: perf-ab-results
path: benchmarks/cleanbench/results/latest.ab.json
retention-days: 30
- name: Open/refresh alert issue on failure
if: failure() && github.event_name == 'schedule'
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const title = "nightly perf-regression gate failing";
const body = `The scheduled same-run A/B performance gate failed: main ran ` +
`>20% slower or used >15% more peak memory than main from ~26 hours ` +
`earlier, reproduced on a confirmation run (or the job errored).\n\n` +
`Run: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}` +
`/actions/runs/${context.runId}`;
const open = await github.rest.issues.listForRepo({
...context.repo, state: "open", labels: "nightly-failure",
});
const existing = open.data.find(i => i.title === title);
if (existing) {
await github.rest.issues.createComment({
...context.repo, issue_number: existing.number, body,
});
} else {
await github.rest.issues.create({
...context.repo, title, body, labels: ["nightly-failure"],
});
}