Repository navigation
398 lines (381 loc) · 18.1 KB
/
Copy pathlocal-backends.yml
File metadata and controls
398 lines (381 loc) · 18.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
name: Local backends
# The public tier: the unit suite and the backend-agnostic integration suite under the built-in
# local defaults (SQLite, filesystem objects, a local registry, Docker sandboxes), with no secrets
# and no external services. Tests whose assertions are about one particular backend live with
# that backend's own suite, not here.
on:
pull_request:
push:
branches: [main]
permissions:
contents: read
# One group per pull request, so a newer push cancels the superseded run; one group per commit on
# main, so no push to main is ever cancelled or queued behind another (a group holds at most one
# pending run, and the version bump lands seconds after every merge).
concurrency:
group: local-backends-${{ github.event_name == 'pull_request' && github.ref || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
env:
AGENT_ENV_ENVIRONMENT: dev
COMPUTER_IMAGE_MIRROR: ghcr.io/scaleapi/agentenv-ci-computers
PIP_DISABLE_PIP_VERSION_CHECK: "1"
jobs:
# The base computer's egress proxy (src/agent_env/computers/artifacts/ubuntu-24.04/egress/): gofmt, go vet and its
# tests under the race detector, in the Go image its Dockerfile builds with, so a failing Go test fails here rather
# than as a broken base image in the slow tier.
egress:
name: egress
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
# ECR Public's anonymous pulls fail here for hours at a time.
- name: Pull the Go image from mirror.gcr.io
run: python3 .github/scripts/mirror_base_images.py src/agent_env/computers/artifacts/ubuntu-24.04/Dockerfile
- name: gofmt, go vet and go test -race
run: make egress-test
# The unit tier on two runners, AGENTENV_TEST_SHARD picking each one's share by the times in
# tst/integration/slow_durations.json (tst/util/shards.py); unit passes when both shards do. The
# lockfile and lint checks run on the first shard only.
unit-shard:
name: unit (${{ matrix.shard }}/2)
strategy:
fail-fast: false
matrix:
shard: [1, 2]
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Install from uv.lock
run: |
python -m pip install --quiet -r .github/ci-requirements.txt
uv sync --locked --extra dev --python "$(command -v python)"
echo "$PWD/.venv/bin" >> "$GITHUB_PATH"
- name: Lockfile resolves from PyPI only
if: matrix.shard == 1
run: |
if grep 'source = { registry = ' uv.lock | grep -v 'registry = "https://pypi.org/simple"' | grep -q .; then echo "uv.lock has a registry source other than PyPI"; exit 1; fi
grep -A2 '^name = "agentenv-framework-protocol"$' uv.lock | grep -q 'source = { editable = "packages/agentenv-protocol" }'
- name: Lockfile is current, before and after a release bump
if: matrix.shard == 1
run: |
uv lock --check
uv run --no-project --with tomlkit python scripts/bump_version.py
uv lock --check
git checkout -- pyproject.toml packages/agentenv-protocol/pyproject.toml uv.lock
- name: Lint
if: matrix.shard == 1
run: .venv/bin/ruff check
# worksteal: a worker that runs out of tests takes queued ones from the others, so no worker idles at the end.
- name: Unit tier
env:
AGENTENV_TEST_SHARD: ${{ matrix.shard }}/2
run: |
.venv/bin/python -m pytest tst/unit packages/agentenv-protocol/tests -n auto --dist worksteal -q --timeout=300 \
--durations=30 --junitxml=junit-unit.xml
# The tests' times, for .github/scripts/slow_test_durations.py to rebalance the shards with.
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
if: always()
with:
name: junit-unit-${{ matrix.shard }}
path: junit-unit.xml
retention-days: 14
# One check under the name the main ruleset requires: it passes when both unit shards did.
unit:
if: always()
needs: unit-shard
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Every shard passed
run: test "${{ needs.unit-shard.result }}" = success
ui:
runs-on: ubuntu-latest
timeout-minutes: 15
defaults:
run:
working-directory: src/agent_env/explorer/ui
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: "22"
cache: npm
cache-dependency-path: src/agent_env/explorer/ui/package-lock.json
- run: npm ci
- name: Typecheck
run: npm run typecheck
- name: Lint
run: npm run lint
- name: Smoke suite
run: npm run test:smoke
- name: Static export
run: npm run build:static
integration-local:
runs-on: ubuntu-latest
timeout-minutes: 45
services:
registry:
image: mirror.gcr.io/library/registry:2
ports:
- 5000:5000
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Install from uv.lock
run: |
python -m pip install --quiet -r .github/ci-requirements.txt
uv sync --locked --extra dev --python "$(command -v python)"
echo "$PWD/.venv/bin" >> "$GITHUB_PATH"
# loadscope keeps a module's or class's tests on one worker, so its fixtures run once, as make int-test-fast does.
- name: Integration tier on the local defaults
run: |
.venv/bin/python -m pytest -m 'not int_test_slow' -n auto --dist loadscope -rA --timeout=900 \
--junitxml=junit-integration.xml tst/integration
- name: Skips must be declared capability gaps
if: always()
run: |
.venv/bin/python .github/scripts/check_skip_policy.py junit-integration.xml \
--allow model_endpoint_configured remote_sandbox \
--no-skips-under tst/integration/store/
# The slow tier: builds the env images and the echo A2A agent (tst/data/a2a_agent) and deploys
# them on the runner's Docker. The tests that need a real model, a remote sandbox, a registered
# default A2A agent or MCP server sources from outside this repository skip with the capability reason.
# Each runner runs its files on two workers, a file's tests on one, and AGENTENV_TEST_SHARD picks
# each runner's share of the files, balanced by tst/integration/slow_durations.json
# (tst/util/shards.py). The gateway tests are a check of their own, split the same way. Each check passes
# when all of its shards do (the integration-local-slow and integration-local-gateway jobs below).
integration-local-slow-shard:
name: integration-local-slow (${{ matrix.shard }}/12)
strategy:
fail-fast: false
matrix:
shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
runs-on: ubuntu-latest
timeout-minutes: 40
permissions:
contents: read
packages: write
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
# The tests run with packages: write; only the share step, on main, gets the token.
with:
persist-credentials: false
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Install from uv.lock
run: |
python -m pip install --quiet -r .github/ci-requirements.txt
uv sync --locked --extra dev --python "$(command -v python)"
echo "$PWD/.venv/bin" >> "$GITHUB_PATH"
# ECR Public's anonymous pulls fail here for hours at a time. This can restart dockerd, so the registry
# starts afterwards.
- name: Pull the base images from mirror.gcr.io
run: python3 .github/scripts/mirror_base_images.py src tst
# The computer tests share the package locks a compose resolves, and the files they name, in the packages folder
# (tst/util/package_cache.py). An image's key includes its lock, which the composer otherwise resolves from
# Ubuntu's archive, PyPI and download URLs, so the folder is restored from what main saved latest this week.
- name: Week of the packages' cache
id: week
run: echo "week=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT"
- name: Restore the packages
id: packages
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ${{ runner.temp }}/packages
key: packages-${{ matrix.shard }}-${{ steps.week.outputs.week }}
restore-keys: packages-${{ matrix.shard }}-${{ steps.week.outputs.week }}-
- name: Start the local registry
run: |
docker run --detach --name registry --publish 5000:5000 mirror.gcr.io/library/registry:2
timeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null; do sleep 1; done'
# x64 hosted runners have /dev/kvm; the disk form's tests boot under it, and skip without it, which fails below.
- name: Let the local provider boot disks with KVM
run: sudo chmod 666 /dev/kvm
- name: Slow integration tier on the local defaults
id: tests
env:
AGENTENV_TEST_SHARD: ${{ matrix.shard }}/12
AGENTENV_TEST_PACKAGES: ${{ runner.temp }}/packages
AGENT_ENV_COMPUTER_IMAGE_MIRROR: ${{ env.COMPUTER_IMAGE_MIRROR }}
run: |
.venv/bin/python -m pytest -m int_test_slow -n 2 --dist loadfile -rA --timeout=1200 \
--junitxml=junit-integration-slow.xml tst/integration --ignore=tst/integration/env/gateway/gateway_test.py
# Only main saves the packages, once its tests pass, so pull requests reuse main's without filling the
# repository's cache. The entry is named by the locks the folder holds, so a run that resolved a new lock saves a
# new one.
- name: Name the packages' cache entry
id: lockkey
if: always()
run: |
locks="$RUNNER_TEMP/packages/agentenv-packages/locks"
if [ -z "$(ls "$locks" 2>/dev/null)" ]; then echo "packages=false" >> "$GITHUB_OUTPUT"; exit 0; fi
key=$(ls "$locks" | sort | sha256sum | cut -c1-16)
echo "key=packages-${{ matrix.shard }}-${{ steps.week.outputs.week }}-$key" >> "$GITHUB_OUTPUT"
echo "packages=true" >> "$GITHUB_OUTPUT"
- name: Save the packages
if: >-
always() && github.event_name == 'push' && steps.tests.outcome == 'success'
&& steps.lockkey.outputs.packages == 'true' && steps.lockkey.outputs.key != steps.packages.outputs.cache-matched-key
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ${{ runner.temp }}/packages
key: ${{ steps.lockkey.outputs.key }}
- name: Share the computer images
continue-on-error: true
if: >-
always() && steps.tests.outcome == 'success' && github.event_name == 'push'
env:
GITHUB_TOKEN: ${{ github.token }}
run: .venv/bin/python .github/scripts/share_computer_images.py "$COMPUTER_IMAGE_MIRROR"
- name: Skips must be declared capability gaps
if: always()
run: |
.venv/bin/python .github/scripts/check_skip_policy.py junit-integration-slow.xml \
--allow model_endpoint_configured remote_sandbox default_a2a_agent mcp_server_sources windows_media \
omarchy_image
# The tests' times, for .github/scripts/slow_test_durations.py to rebalance the shards with.
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
if: always()
with:
name: junit-integration-local-slow-${{ matrix.shard }}
path: junit-integration-slow.xml
retention-days: 14
integration-local-gateway-shard:
name: integration-local-gateway (${{ matrix.shard }}/5)
strategy:
fail-fast: false
matrix:
shard: [1, 2, 3, 4, 5]
runs-on: ubuntu-latest
timeout-minutes: 40
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Install from uv.lock
run: |
python -m pip install --quiet -r .github/ci-requirements.txt
uv sync --locked --extra dev --python "$(command -v python)"
echo "$PWD/.venv/bin" >> "$GITHUB_PATH"
- name: Pull the base images from mirror.gcr.io
run: python3 .github/scripts/mirror_base_images.py src tst
- name: Start the local registry
run: |
docker run --detach --name registry --publish 5000:5000 mirror.gcr.io/library/registry:2
timeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null; do sleep 1; done'
- name: Slow integration tier on the local defaults
env:
AGENTENV_TEST_SHARD: ${{ matrix.shard }}/5
run: |
.venv/bin/python -m pytest -m int_test_slow -rA --timeout=1200 --junitxml=junit-integration-slow.xml \
tst/integration/env/gateway/gateway_test.py
- name: Skips must be declared capability gaps
if: always()
run: |
.venv/bin/python .github/scripts/check_skip_policy.py junit-integration-slow.xml \
--allow model_endpoint_configured remote_sandbox default_a2a_agent mcp_server_sources
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
if: always()
with:
name: junit-integration-local-gateway-${{ matrix.shard }}
path: junit-integration-slow.xml
retention-days: 14
# One check per tier under its own name (the main ruleset requires integration-local-slow): each passes
# when every shard of its tier passed.
integration-local-slow:
if: always()
needs: integration-local-slow-shard
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Every shard passed
run: test "${{ needs.integration-local-slow-shard.result }}" = success
integration-local-gateway:
if: always()
needs: integration-local-gateway-shard
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Every shard passed
run: test "${{ needs.integration-local-gateway-shard.result }}" = success
# Whether this change touches the code the installer tier covers; the tier costs minutes, so
# it runs only then. A skipped job reports as passed to a required check.
changes:
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
plugins: ${{ steps.diff.outputs.plugins }}
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
with:
fetch-depth: 0
- id: diff
env:
# A pull request's merge ref sits on the current base, which base.sha lags once the base
# moves; diffing against base.sha would count the base's own later commits as changes.
BASE: ${{ github.event_name == 'pull_request' && 'HEAD^1' || github.event.before }}
run: |
paths='^(src/agent_env/cli/(__init__|__main__|_installers|_plugin_changes|plugin)\.py|src/agent_env/(plugins|config)/|src/agent_env/py\.typed|packages/agentenv-protocol/|tst/installer/|tst/util/capabilities\.py|pyproject\.toml|uv\.lock|Makefile|\.github/(ci-requirements\.txt|workflows/local-backends\.yml|scripts/check_skip_policy\.py))'
if ! changed=$(git diff --name-only "$BASE" HEAD 2>/dev/null); then
echo "plugins=true" >> "$GITHUB_OUTPUT"
elif grep -qE "$paths" <<< "$changed"; then
echo "plugins=true" >> "$GITHUB_OUTPUT"
else
echo "plugins=false" >> "$GITHUB_OUTPUT"
fi
# `plugin add` and `remove` through the real pip, uv and pipx, offline against wheels built from
# this checkout; the dependencies are downloaded from PyPI once, before the tests go offline. It runs on
# two runners, split the same way as the unit tier.
installer-shard:
name: installer (${{ matrix.shard }}/2)
needs: changes
if: needs.changes.outputs.plugins == 'true'
strategy:
fail-fast: false
matrix:
shard: [1, 2]
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Install from uv.lock
run: |
python -m pip install --quiet -r .github/ci-requirements.txt pipx
uv sync --locked --extra dev --python "$(command -v python)"
echo "$PWD/.venv/bin" >> "$GITHUB_PATH"
- name: Installer tier
env:
AGENTENV_TEST_SHARD: ${{ matrix.shard }}/2
run: .venv/bin/python -m pytest tst/installer -n auto --dist worksteal -rA --timeout=900 --junitxml=junit-installer.xml
- name: Skips must be declared capability gaps
if: always()
run: .venv/bin/python .github/scripts/check_skip_policy.py junit-installer.xml
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
if: always()
with:
name: junit-installer-${{ matrix.shard }}
path: junit-installer.xml
retention-days: 14
# Passes when both installer shards did, or when the change touches nothing the tier covers.
installer:
if: always()
needs: [changes, installer-shard]
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Every shard passed, or the tier had nothing to check
run: |
test "${{ needs.changes.result }}" = success
test "${{ needs.installer-shard.result }}" = success -o "${{ needs.installer-shard.result }}" = skipped