-
Notifications
You must be signed in to change notification settings - Fork 6
1344 lines (1202 loc) · 60 KB
/
Copy pathci.yml
File metadata and controls
1344 lines (1202 loc) · 60 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
name: CI
on:
push:
branches: [master, feature/holographic-memory]
workflow_dispatch:
inputs:
run_os:
description: "Also run macOS and Windows matrices"
type: boolean
default: false
run_hosts:
description: "Also run stock host integrations"
type: boolean
default: false
run_perf:
description: "Also run hotpath parity"
type: boolean
default: false
run_full_nextest:
description: "Also run the full Linux search and transport selections"
type: boolean
default: false
permissions:
contents: read
# One run per ref, and a newer push cancels the run in flight, master
# included. The newest master commit contains every merge before it, so its
# run is the verdict on what ships; a superseded run proves nothing the newer
# one does not. One master push spends 16 of the account's 20 hosted jobs, and
# keeping superseded master runs (keyed per SHA) serialized a merge burst at
# ~27 min per merge and left release-please waiting 80-93 min for a runner.
# To attribute a break inside a burst, bisect or dispatch this workflow on a
# branch at the suspect commit.
concurrency:
group: ci-${{ github.ref }}
cancel-in-progress: true
env:
CARGO_TERM_COLOR: always
CARGO_INCREMENTAL: "0"
CARGO_PROFILE_DEV_DEBUG: "0"
CARGO_PROFILE_TEST_DEBUG: "0"
AST_GREP_VERSION: "0.44.0"
jobs:
# Manual dispatch is the only PR-head entry point. GitHub queues a
# pull_request workflow before evaluating job conditions, and automation
# applied every opt-in label to 173 PRs; only removing the trigger prevents
# that volume from occupying the account FIFO. A dispatch runs the Linux
# lane and may add OS, host, perf, or full-workspace work through explicit
# inputs. A master push runs the Linux lane only so release jobs retain
# runner slots.
scope-gate:
name: Scope gate
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
run-heavy: ${{ steps.decide.outputs.run-heavy }}
run-os: ${{ steps.decide.outputs.run-os }}
run-hosts: ${{ steps.decide.outputs.run-hosts }}
run-perf: ${{ steps.decide.outputs.run-perf }}
linux-partitions: ${{ steps.linux-partitions.outputs.matrix }}
windows-partitions: ${{ steps.windows-partitions.outputs.matrix }}
macos-groups: ${{ steps.macos-groups.outputs.matrix }}
steps:
- name: Decide what this run may spend
id: decide
env:
EVENT: ${{ github.event_name }}
RUN_OS: ${{ inputs.run_os || false }}
RUN_HOSTS: ${{ inputs.run_hosts || false }}
RUN_PERF: ${{ inputs.run_perf || false }}
run: |
heavy=true
os=false
hosts=false
perf=false
if [[ $EVENT == workflow_dispatch ]]; then
[[ $RUN_OS == true ]] && os=true
[[ $RUN_HOSTS == true ]] && hosts=true
[[ $RUN_PERF == true ]] && perf=true
fi
{
echo "run-heavy=$heavy"
echo "run-os=$os"
echo "run-hosts=$hosts"
echo "run-perf=$perf"
} >>"$GITHUB_OUTPUT"
echo "run-heavy=$heavy run-os=$os run-hosts=$hosts run-perf=$perf (event=$EVENT)"
# The Linux, Windows and macOS test matrices come from the same manifest
# the partition jobs select their targets from, so a partition cannot
# exist without a job or a job without a partition.
- uses: actions/checkout@v7
with:
sparse-checkout: |
.github/linux-test-partitions.json
scripts
sparse-checkout-cone-mode: false
- name: Derive the Linux test matrix
id: linux-partitions
run: echo "matrix=$(python3 scripts/linux-test-partitions.py matrix)" >> "$GITHUB_OUTPUT"
- name: Derive the Windows test matrix
id: windows-partitions
run: echo "matrix=$(python3 scripts/linux-test-partitions.py windows-matrix)" >> "$GITHUB_OUTPUT"
- name: Derive the macOS test matrix
id: macos-groups
run: echo "matrix=$(python3 scripts/linux-test-partitions.py macos-matrix)" >> "$GITHUB_OUTPUT"
# One dev-profile `tracedecay` binary for the explicitly requested stock
# host integrations. Ordinary Linux validation needs no CLI artifact, so
# this build is absent unless `run_hosts=true`.
debug-cli:
name: Build debug CLI
needs: scope-gate
if: ${{ needs.scope-gate.outputs.run-hosts == 'true' }}
runs-on: ubuntu-24.04-arm
timeout-minutes: 45
steps:
- uses: actions/checkout@v7
- uses: dtolnay/rust-toolchain@stable
- uses: ./.github/actions/setup-linux-mold
- uses: ./.github/actions/setup-pnpm
# This is the workflow's only dev-profile build, so `ci-dev-*` has one
# writer. rust-cache stores dependency artifacts only. The repository
# cache is capped at 10 GB and was full of per-job copies of the same
# dependencies (`pr-dogfood` and `dashboard` each built this graph too),
# which evicted the test lanes' caches between runs and made every test
# run a cold build. Only the lockfile-keyed dependency cache is kept:
# the workspace crates are exactly what a push changes, so caching
# their outputs per run re-saved ~1 GB per push for a restore that hit
# nothing.
- name: Cache Rust build outputs
uses: Swatinem/rust-cache@v2
with:
shared-key: ci-dev-${{ runner.os }}-${{ runner.arch }}
cache-on-failure: true
- name: Build tracedecay binary
run: cargo build -p tracedecay-cli --bin tracedecay --locked
- name: Publish the binary for the host integration jobs
uses: actions/upload-artifact@v4
with:
name: debug-tracedecay-cli
path: target/debug/tracedecay
if-no-files-found: error
retention-days: 1
benchmark-harness:
name: Benchmark harness self-tests
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@v7
# The runtime benchmark harness produces the evidence PRs cite; its own
# unit tests only count if something runs them.
- name: Test the runtime benchmark harness
working-directory: benchmark_data/runtime
run: python3 -m unittest discover -s tests -p 'test_*.py'
# The cheap repository gates in one job: commit lint, cache lineage, the
# release guards, and rustfmt. Each used to hold its own ubuntu-latest
# runner for a couple of minutes of work; on 20 free slots that is three
# queue entries per push doing what one does.
gates:
name: Repository gates
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v7
with:
fetch-depth: 0
- uses: actions/setup-node@v4
with:
node-version: 22
- uses: ./.github/actions/setup-pnpm
- name: Reject tracked ignored files
run: scripts/check-release-pr-integrity.sh HEAD HEAD
- name: Test bounded commit range linting
run: python3 scripts/test-lint-commit-range.py
- name: Test worktree GC classification and build reclamation
run: python3 scripts/test-worktree-gc.py
- name: Check Rust cache key lineage and the superseded-entry selection
run: |
python3 scripts/test-prune-superseded-actions-caches.py
python3 scripts/test-rust-cache-lineage.py
python3 scripts/check-rust-cache-lineage.py
# Packager unit tests. They do not need a release binary, so they run
# once here instead of once per release matrix target.
- name: Test release packagers
run: |
python3 scripts/test-check-packaged-lsp-bridge.py
python3 scripts/test-build-mcpb.py
python3 scripts/test-resolve-installed-binary.py
python3 scripts/test-check-packaged-mcp-stdio.py
python3 scripts/test-check-distribution-feature-wiring.py
python3 scripts/test-resolve-release-source-profile.py
python3 scripts/test-package-release-archive.py
python3 scripts/test-plan-release-recovery.py
python3 scripts/test-check-release-artifacts.py
python3 scripts/test-neutralize-release-pr-closing-keywords.py
bash scripts/test-check-distribution-clean-source.sh
bash scripts/test-check-distribution-snapshot.sh
bash scripts/test-check-distribution-reuse.sh
bash tests/macos_stable_codesign_test.sh
# Release guards: drift, safety, installer, PR integrity, bundle check.
# Pull requests only; the release PR itself is exempt because it is
# the one change allowed to move the version authorities.
- name: Check the release guards
if: github.event_name == 'workflow_dispatch'
run: |
bash tests/release_drift_check_test.sh
bash tests/release_safety_test.sh
bash tests/install_script_test.sh
bash tests/release_pr_integrity_test.sh
bash tests/dashboard_bundle_check_test.sh
# Judge formatting with the toolchain `rust-toolchain.toml` pins, not
# whatever `stable` is on the runner: rustfmt's output changes between
# releases (1.98.1 breaks a chain 1.97.1 keeps inline), and
# `dtolnay/rust-toolchain@stable` exports `RUSTUP_TOOLCHAIN`, which
# overrides the file. Installing through rustup with no override keeps
# the pin in one place.
- name: Install the repository's pinned toolchain
run: rustup show active-toolchain || rustup toolchain install
- name: Check formatting
run: cargo fmt --all -- --check
- name: Test and check dev skill host copies
run: |
python3 scripts/test-check-dev-skill-mirrors.py
python3 scripts/check-dev-skill-mirrors.py check
# Dispatch admits an integration branch. Judge the commits a merge onto
# the default branch would introduce, with the same linter a push uses.
# A push still uses the before SHA, so published history is not rejudged.
- name: Validate commit messages
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
env:
EVENT_NAME: ${{ github.event_name }}
BEFORE_SHA: ${{ github.event.before }}
HEAD_SHA: ${{ github.sha }}
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
run: scripts/lint-ci-commits.sh
macos-test-partition:
name: Test macOS ${{ matrix.group }}
needs: scope-gate
if: ${{ needs.scope-gate.outputs.run-os == 'true' }}
runs-on: macos-14
# Each group's budget is set in the manifest beside the group, with the
# measurement it rests on.
timeout-minutes: ${{ matrix.timeout }}
# Re-signs the product after release strip (see .cargo/config.toml).
env:
CARGO_BUILD_RUSTC_WORKSPACE_WRAPPER: ${{ github.workspace }}/scripts/macos-rustc-wrapper.sh
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.scope-gate.outputs.macos-groups) }}
steps:
# Full history: the search-quality workload fixture pins a
# `source_repository_commit`, and `validate_source_bindings` resolves that
# object out of this checkout to prove the checked-in corpus really is the
# product source at that commit. A depth-1 checkout carries only the tip,
# so every candidate_output/report test fails with "resolve fixture source
# commit: An object with id ... could not be found".
- uses: actions/checkout@v7
with:
fetch-depth: 0
# `rust-analyzer` is a test dependency of this lane, not a convenience.
# `runtime_surface_acceptance::production_lsp_negotiates_and_projects_\
# canonical_context` asserts that a routed analyzer negotiates the
# standard methods the client declared, and the upstream `initialize`
# response is the only authority for those. Without the component the
# runner still has rustup's `rust-analyzer` proxy on PATH, so the daemon
# routes the language and only the spawn fails. The test would then
# measure a missing component instead of the negotiation contract. This
# action exports `RUSTUP_TOOLCHAIN`, so the component has to be installed
# here rather than through `rust-toolchain.toml`, which it overrides.
- uses: dtolnay/rust-toolchain@stable
with:
components: rust-analyzer
- uses: actions/setup-node@v4
with:
node-version: 22
- uses: ./.github/actions/setup-pnpm
- uses: actions/setup-python@v6
with:
python-version: "3.12"
- name: Install ast-grep
uses: ./.github/actions/install-ast-grep
with:
version: ${{ env.AST_GREP_VERSION }}
- name: Install cargo-nextest
uses: taiki-e/install-action@nextest
# rust-cache carries the dependency graph, keyed by toolchain and
# lockfile; the workspace crates compile fresh every run. No per-run
# compiler store: the perf-profile test graph wrote 15.7 GB of outputs
# per build on macOS, which no store inside the repository's 10 GB
# cache budget can hold. One writer keeps the lane at one entry:
# `root-suites` resolves the widest root graph (the root crate with
# its dev-dependencies, the CLI and the evaluator), so its save covers
# `root-lib` and the `root-transport` group exactly (test-transport
# re-resolves 31 workspace crates but no dependency unit). The lower
# partitions unify the dependencies differently and recompile ~100-200
# units against it, which costs nothing measurable here: `runtime`
# compiled in 28.5-29.6 min with every one of its 604 dependency units
# built from source and in 29.5 with a full-match cache of its own
# (runs 36087768140 and 36095356282), because on three cores the
# serial workspace chain plus the test targets are the whole path and
# the dependency units fill the slots beside it. A lineage for them
# was tried and removed on that evidence. Crate sources come from the
# pnpm store that setup-pnpm restores, not from this cache.
- name: Cache Rust build outputs
uses: Swatinem/rust-cache@v2
with:
shared-key: ci-test-full-${{ runner.os }}
save-if: ${{ matrix.group == 'root-suites' }}
cache-on-failure: true
# Every group job re-proves the cover and the grouping before it
# compiles anything: a gap found here fails the lane rather than passing
# it with a target nobody ran.
- name: Check the partitions cover every test target
run: |
python3 scripts/test-linux-test-partitions.py
python3 scripts/linux-test-partitions.py check
# Each partition in turn, as `linux-test-partition` runs it: the
# executables its suites spawn are built into the same resolution first
# (`build-args`; nothing for a partition whose tests spawn nothing),
# then nextest under the `ci` policy and the perf cargo profile with
# `--no-tests=fail`. One target directory for the group, so a later
# partition's shared units are fingerprint hits; the manifest pairs
# partitions of one resolution where they exist (run 36051245663 had
# `root-transport` re-resolve 31 workspace crates behind
# `root-journeys`, 18 min on three cores, and `root-dashboard-api` as
# a lone `-p tracedecay` selection recompile 48 workspace crates and
# 108 dependency units, 24 min, because the CLI's clap graph changes
# `mio`'s features and so tokio's and every crate above it) and
# otherwise balances the sums of the measured partition costs, since
# a group's wall time is that sum. Every partition runs
# even after an earlier one fails, as nextest's `ci` policy runs every
# test, and the step fails if any did. Every nextest run rewrites the
# `ci` profile's junit.xml, so each partition's report is moved under
# its own name before the next runs. The per-partition wall times this
# prints are the measurements the manifest's macOS budgets wait for.
- name: Run the group's partitions
env:
PARTITIONS: ${{ matrix.partitions }}
run: |
set -uo pipefail
run_partition() {
local partition="$1" build selection
build="$(python3 scripts/linux-test-partitions.py build-args "$partition")" || return 1
selection="$(python3 scripts/linux-test-partitions.py cargo-args "$partition")" || return 1
if [ -n "$build" ]; then
eval cargo build --locked --profile perf "$build" || return 1
fi
eval cargo nextest run --profile ci --cargo-profile perf --locked "$selection" --no-tests=fail
}
read -ra partitions <<< "$PARTITIONS"
mkdir -p target/nextest/macos
failed=""
for partition in "${partitions[@]}"; do
echo "::group::Test partition ${partition}"
started=$SECONDS
if run_partition "$partition"; then
echo "${partition}: passed in $(( SECONDS - started )) s"
else
failed="${failed} ${partition}"
echo "${partition}: failed after $(( SECONDS - started )) s"
fi
[ ! -f target/nextest/ci/junit.xml ] || mv target/nextest/ci/junit.xml "target/nextest/macos/${partition}.xml"
echo "::endgroup::"
done
if [ -n "$failed" ]; then
echo "::error::Failed partitions:${failed}"
exit 1
fi
# `macos-test` folds the group reports back into one report.
- name: Upload macOS group nextest report
if: always()
uses: actions/upload-artifact@v4
with:
name: nextest-junit-macOS-${{ matrix.group }}
path: target/nextest/macos/
if-no-files-found: ignore
retention-days: 1
# The one `Test macOS` verdict, as when a single job carried the suite:
# every group must pass. It also folds the group reports back into the
# `nextest-junit-macOS` artifact the single job published, one directory
# per group holding a junit.xml per partition.
macos-test:
name: Test macOS
if: ${{ !cancelled() && needs.scope-gate.outputs.run-os == 'true' }}
needs: [macos-test-partition, scope-gate]
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Merge the group reports into the macOS nextest report
if: ${{ needs.macos-test-partition.result != 'skipped' }}
uses: actions/upload-artifact/merge@v4
with:
name: nextest-junit-macOS
pattern: nextest-junit-macOS-*
separate-directories: true
retention-days: 7
- name: Check macOS groups
if: ${{ !cancelled() }}
run: |
if [ "${{ needs.macos-test-partition.result }}" != "success" ]; then
echo "macOS group result: ${{ needs.macos-test-partition.result }}"
exit 1
fi
echo "All macOS groups passed."
# Linux compiles and runs the suite as parallel partitions of the test
# targets (.github/linux-test-partitions.json), one hosted 4-vCPU job each,
# so the lane's wall time is the slowest partition instead of the whole
# workspace. The single job that preceded it (run 34278967966, warm
# dependency cache, four compile slots) spent 55.5 min compiling: 32 min of
# workspace libraries beneath the root crate, then the root library
# (~8 min) and its library test target (~15.5 min), a serial chain that no
# job count shortens; the other 300-odd test units fit in the slots beside
# it. Tests then took 22.5 min, the dashboard's `test-transport` rebuild
# and run 11 min, and the hotpath parity rebuild ~16 min. Partitioned, the
# root chain is one job (`root-lib`: 19.7 min compile + 5.2 min tests, 25.9
# min in run 34296614024, the first partitioned run) while every other
# partition finishes in 17–27 min, so the lane's critical path drops from
# ~105 min to the slowest partition; the parity rebuild runs beside the
# partitions as `hotpath-parity` rather than after the root chain. Building
# the graph once and sharding only the test execution (the Windows lane's
# shape) keeps the whole 55 min compile plus an archive round trip on the
# path and then the dashboard rebuild and parity on the same job: ~88 min,
# or ~70 with both moved elsewhere. The partitions pay for the shorter path
# with duplicated compile. Each root partition rebuilds the library chain
# beneath the root crate, about 200 core-minutes per run across the seven
# jobs, which is free on hosted runners where only wall time and the
# 20-job concurrency cap count. A cold dependency cache adds ~6 min to every
# job alike (the dependencies compile at opt-level 0 in ~17 core-minutes).
# The floor no partitioning reaches is the root chain itself: library
# chain + root lib + root lib-test + its tests, the ~26 min above.
#
# The partitions are cargo selections, not nextest filters: `-p` plus
# `--lib` / `--test <name>` / `--bins` decide what compiles, and a filterset
# would compile everything and skip at run time. The three root partitions
# share one package selection (`tracedecay`, `tracedecay-cli`,
# `tracedecay-search-eval`) with the root fixture feature. Journeys also
# enable `tracedecay/search-eval`, the only link from the root crate to the
# evaluator library. It stays off every other partition: an unconditional
# dependency links the evaluator into every test target of the package.
# Every cargo invocation inside a job (the test build,
# the executables the suites spawn) is a cache hit against that job's
# resolution. `scripts/linux-test-partitions.py check` proves, from `cargo
# metadata`, that every test target in the workspace is selected by exactly
# one partition or listed under `not_run` with a reason, so a new crate or
# suite cannot fall out of the lane silently; `scope-gate` derives the
# matrix from the same manifest.
linux-test-partition:
name: Test Linux ${{ matrix.partition }}
needs: scope-gate
if: ${{ needs.scope-gate.outputs.run-heavy == 'true' }}
runs-on: ubuntu-24.04-arm
# Each partition's budget is its measured wall time plus headroom, set in
# the manifest beside the selection it covers.
timeout-minutes: ${{ matrix.timeout }}
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.scope-gate.outputs.linux-partitions) }}
steps:
# Full history: the search-quality workload fixture pins a
# `source_repository_commit`, and `validate_source_bindings` resolves that
# object out of this checkout to prove the checked-in corpus really is the
# product source at that commit. A depth-1 checkout carries only the tip,
# so every candidate_output/report test fails with "resolve fixture source
# commit: An object with id ... could not be found".
- uses: actions/checkout@v7
with:
fetch-depth: 0
# `rust-analyzer` is a test dependency of this lane, not a convenience.
# `runtime_surface_acceptance::production_lsp_negotiates_and_projects_\
# canonical_context` asserts that a routed analyzer negotiates the
# standard methods the client declared, and the upstream `initialize`
# response is the only authority for those. Without the component the
# runner still has rustup's `rust-analyzer` proxy on PATH, so the daemon
# routes the language and only the spawn fails. The test would then
# measure a missing component instead of the negotiation contract. This
# action exports `RUSTUP_TOOLCHAIN`, so the component has to be installed
# here rather than through `rust-toolchain.toml`, which it overrides.
- uses: dtolnay/rust-toolchain@stable
with:
components: rust-analyzer
- uses: ./.github/actions/setup-linux-mold
- uses: actions/setup-node@v4
with:
node-version: 22
- uses: ./.github/actions/setup-pnpm
- uses: actions/setup-python@v6
with:
python-version: "3.12"
- name: Install ast-grep
uses: ./.github/actions/install-ast-grep
with:
version: ${{ env.AST_GREP_VERSION }}
- name: Install cargo-nextest
uses: taiki-e/install-action@nextest
# One dependency cache for the lane, as before the split. rust-cache
# stores the dependency graph only, keyed by toolchain and lockfile; the
# workspace crates compile fresh every run. A single writer keeps the
# lane at one entry in the repository's 10 GB cache: `root-journeys`
# resolves the widest graph (the root crate with its dev-dependencies,
# the CLI and the evaluator), so its save covers the other root
# partitions exactly (`root-dashboard-api` selects the CLI package for
# that reason: alone, `-p tracedecay` resolves `mio` without the
# features the CLI's clap graph adds, so tokio and every workspace
# crate above it hash differently). The three lower partitions unify
# dependency features differently and each recompile ~150 dependency
# units (syn, serde_json, tokio, gix-*, grafeo-*, ...) against it every
# run, measured in run 36051245663: 4-6 min beside chains that end
# 4-12 min before `root-transport` does, cheaper than three more cache
# entries in the 10 GB budget (macOS, where the same recompile is 2.5x
# longer and on the lane's critical path, carries one). Crate sources
# come from the pnpm store that setup-pnpm restores, not from this cache.
- name: Cache Rust build outputs
uses: Swatinem/rust-cache@v2
with:
shared-key: ci-test-full-Linux-${{ runner.arch }}
save-if: ${{ matrix.partition == 'root-journeys' }}
cache-on-failure: true
# Every partition job re-proves the cover before it compiles anything:
# the manifest is consumed here, and a gap found here fails the lane
# rather than passing it with a target nobody ran.
- name: Check the partitions cover every test target
run: |
python3 scripts/test-linux-test-partitions.py
python3 scripts/linux-test-partitions.py check
- name: Resolve this partition's cargo selection
id: selection
run: |
{
echo "test=$(python3 scripts/linux-test-partitions.py cargo-args '${{ matrix.partition }}')"
echo "build=$(python3 scripts/linux-test-partitions.py build-args '${{ matrix.partition }}')"
} >> "$GITHUB_OUTPUT"
# Suites that spawn the CLI (`tests/common::tracedecay_bin`) or the
# search-eval evaluators (`search_eval_bin`) resolve them out of the
# profile directory, and cargo links a package's bins for a test build
# only when that package's own integration tests are selected. This
# builds the partition's selection plus those executables in one
# resolution: the test targets keep dev-dependencies in the graph, so
# the nextest build below is a cache hit, and the selection includes
# `tracedecay-search-eval` so the evaluator resolves
# `tracedecay-code-index` with its language tiers (a lone
# `-p tracedecay-search-eval` build yields one that cannot parse Rust).
# The `tracedecay-host-cli-fixture` example is the host fixture the CLI
# suites drive. Partitions whose tests spawn nothing skip this.
- name: Build the executables the suites spawn
if: ${{ steps.selection.outputs.build != '' }}
run: cargo build --locked --profile perf ${{ steps.selection.outputs.build }}
# nextest `ci` policy (.config/nextest.toml: fail-fast off, one retry
# that still fails flaky results, 8 threads, slow-timeout termination)
# and the perf cargo profile, as `cargo test-ci`; the selection replaces
# the alias's `--workspace`. `--no-tests=fail` so a selection that
# resolves to nothing cannot report green.
- name: Run tests
run: cargo nextest run --profile ci --cargo-profile perf --locked ${{ steps.selection.outputs.test }} --no-tests=fail
# `linux-test` folds the partition reports back into one report.
- name: Upload Linux partition nextest report
if: always()
uses: actions/upload-artifact@v4
with:
name: nextest-junit-Linux-${{ matrix.partition }}
path: target/nextest/ci/junit.xml
if-no-files-found: ignore
retention-days: 1
# The controlled-workload hotpath parity gate: `tracedecay-search-eval`'s
# `emit_controlled_workload_reports` example, built once with the hotpath
# profiler compiled out and once with `controlled-workload-hotpath`
# (`hotpath/hotpath`) on, must produce byte-identical durable results for
# the framed-log (private-fs) and cursor-parse (capture) workloads, from
# provably distinct executables. `scripts/build-controlled-workload-
# hotpath-helpers.py` builds the pair into target/controlled-workload-
# hotpath/, and the `#[ignore]`d test in the crate's library test target
# (`controlled_workloads::tests::hotpath_off_vs_on_durable_results_are_
# identical`) spawns them from there; `--run-ignored only` with
# `--no-tests=fail` is the only way it runs, and nothing else. No
# partition, artifact upload or junit merge consumes the executables.
#
# Its own job rather than a tail on the `root-lib` partition, where it ran
# after the root library's tests: the feature-on build recompiles every
# workspace crate beneath the evaluator (all 21 depend on `hotpath`, so
# the feature flip changes each one's metadata hash) plus the example,
# ~10 min on top of the ~26 min root chain, which made that job the lane's
# critical path. Here the graph beneath the evaluator compiles from the
# checkout in both modes. A serial chain domain → contracts →
# rusqlite-runtime → runtime-core → code-index → sessions → query →
# search-eval → example of ~9 min per mode with four compile slots (cargo
# `--timings`, -j4 on EPYC 7742 cores; the same pair of builds took 19.5
# min on a hosted runner in run 34138693824, when the single-job lane
# still built them as their own resolution). Then the library test target
# (24 s against the feature-off graph) and the test itself (under 1 s).
# With setup, ~24 min hosted: beside the partitions and no longer than the
# slowest of them (25.9 and 27.4 min in run 34296614024), so the `Test
# Linux` verdict no longer waits for it. The property does not depend on
# the platform (the Windows lane never provisioned it for that reason), so
# a Linux runner carries it.
#
# Not a job in hotpath-coverage.yml: that workflow is path-filtered to
# Rust inputs and never runs on `push`, while this is a gate on every pull
# request head, and its feature-on slices (storage, sessions) do not cover
# the query/code-index/runtime-core graph the evaluator needs; rust-cache
# keeps dependency artifacts only, so no other job's compiled workspace
# crates could be reused anyway. The evaluator alone (`-p
# tracedecay-search-eval`) is the selection: the root partitions' wider
# one would turn every language tier on in `tracedecay-code-extraction`
# and `-code-index` for two builds that parse no source, and carry the root
# fixture feature for a crate this job never compiles. No cache: under
# this selection ~160 of the 420 registry crates (`syn`, `serde_core`,
# `tokio` and their dependents) unify differently from the lane's
# `ci-test-full-Linux` graph and would miss it, and the rest compile in
# the slots the serial chain leaves idle (the cold build measured 8.8 min
# against 9.1 for the warm feature-on one), so a restore
# would buy about what it costs, while another lineage would compete for
# the 10 GB budget the test lanes' dependency caches are already evicted
# from.
hotpath-parity:
name: Hotpath parity
needs: scope-gate
if: ${{ needs.scope-gate.outputs.run-perf == 'true' }}
runs-on: ubuntu-24.04-arm
# 19.5 min for the two builds (hosted, above) + ~1.5 cold registry the
# chain does not hide + ~1.5 setup + 0.5 test target + 0.5 run ≈ 24 min,
# plus 25 % headroom, as for the partition budgets.
timeout-minutes: 30
steps:
- uses: actions/checkout@v7
- uses: dtolnay/rust-toolchain@stable
- uses: ./.github/actions/setup-linux-mold
- uses: ./.github/actions/setup-pnpm
- name: Install cargo-nextest
uses: taiki-e/install-action@nextest
# Both modes under the evaluator's own selection; the helper copies
# each result out of target/perf/examples immediately so the second
# resolution cannot replace the first.
- name: Build controlled-workload hotpath parity executables
run: python3 scripts/build-controlled-workload-hotpath-helpers.py --profile perf
# The library test target resolves the same feature-off graph as the
# first build above, so only the test target itself compiles here.
# nextest `ci` policy and the perf cargo profile, as the partitions.
- name: Verify controlled-workload hotpath parity
run: |
cargo nextest run --profile ci --cargo-profile perf --locked -p tracedecay-search-eval --lib \
--run-ignored only \
-E 'test(=controlled_workloads::tests::hotpath_off_vs_on_durable_results_are_identical)' \
--no-tests=fail
# The one `Test Linux` verdict, as when a single job carried the suite:
# every partition and the hotpath parity gate must pass. It also folds the
# partition reports back into the `nextest-junit-Linux` artifact the single
# job published; the parity verdict is its step's exit status, as it was
# inside the partition, and is not part of that report.
linux-test:
name: Test Linux
if: ${{ !cancelled() && needs.scope-gate.outputs.run-heavy == 'true' }}
needs: [linux-test-partition, hotpath-parity, scope-gate]
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
# Each partition's junit.xml keeps its name under its partition's
# directory; the partition artifacts themselves expire after a day.
- name: Merge the partition reports into the Linux nextest report
if: ${{ needs.linux-test-partition.result != 'skipped' }}
uses: actions/upload-artifact/merge@v4
with:
name: nextest-junit-Linux
pattern: nextest-junit-Linux-*
separate-directories: true
retention-days: 7
- name: Check Linux partitions and hotpath parity
if: ${{ !cancelled() }}
run: |
if [ "${{ needs.linux-test-partition.result }}" != "success" ]; then
echo "Linux partition result: ${{ needs.linux-test-partition.result }}"
exit 1
fi
if [ "${{ needs.hotpath-parity.result }}" != "success" ] && [ "${{ needs.hotpath-parity.result }}" != "skipped" ]; then
echo "Hotpath parity result: ${{ needs.hotpath-parity.result }}"
exit 1
fi
echo "All Linux partitions and the hotpath parity gate passed."
# Optional cross-test isolation proof. The partition lane cannot expose
# authorities leaked between targets that it runs in separate jobs. Keep
# the evaluator and transport resolutions separate: the transport surface
# includes a generation-scale cancellation journey whose bounded receipts
# assume it is not competing with the entire workspace. Each compatible
# selection still shares one nextest process across all of its targets.
# Keep this off ordinary pushes and run it after the normal Linux verdict so
# its extra compilation and eight concurrent tests do not crowd that lane.
linux-full-nextest:
name: Test Linux full selection isolation
needs: linux-test
if: ${{ !cancelled() && needs.linux-test.result != 'skipped' && github.event_name == 'workflow_dispatch' && inputs.run_full_nextest }}
runs-on: ubuntu-24.04-arm
steps:
- uses: actions/checkout@v7
with:
fetch-depth: 0
- name: Verify pinned Rust toolchain and analyzer
run: |
rustup show active-toolchain
rustup which rust-analyzer
rust-analyzer --version
- uses: ./.github/actions/setup-linux-mold
- uses: actions/setup-node@v4
with:
node-version: 22
- uses: ./.github/actions/setup-pnpm
- uses: actions/setup-python@v6
with:
python-version: "3.12"
- name: Install ast-grep
uses: ./.github/actions/install-ast-grep
with:
version: ${{ env.AST_GREP_VERSION }}
- name: Install cargo-nextest
uses: taiki-e/install-action@nextest
# Build the evaluator selection and the executables its journeys spawn
# with the same feature set and perf resolution nextest uses below.
- name: Build full evaluator selection and support executables
run: cargo build --workspace --bins --tests --example tracedecay-host-cli-fixture --locked --profile perf --features tracedecay/test-helpers,tracedecay/search-eval
- name: Run full evaluator selection
run: |
set +e
cargo test-ci --locked --no-tests=fail
status=$?
set -e
if [ -f target/nextest/ci/junit.xml ]; then
mv target/nextest/ci/junit.xml target/nextest/ci/full-evaluator.xml
fi
exit "$status"
# Transport-only modules live inside otherwise ordinary root targets, so
# run every root library, binary, and integration target in one process.
# The evaluator selection above already covers the remaining workspace
# crates without implicitly linking evaluator code into this resolution.
- name: Build full transport selection and support executables
id: transport_build
if: ${{ always() && !cancelled() }}
run: cargo build -p tracedecay -p tracedecay-cli --bins --tests --example tracedecay-host-cli-fixture --locked --profile perf --features tracedecay/test-helpers,tracedecay/test-transport,tracedecay-cli/test-transport
- name: Run full transport selection
if: ${{ always() && !cancelled() && steps.transport_build.outcome == 'success' }}
run: |
set +e
cargo nextest run -p tracedecay -p tracedecay-cli --bins --tests --profile ci --cargo-profile perf --locked --features tracedecay/test-helpers,tracedecay/test-transport,tracedecay-cli/test-transport --no-tests=fail
status=$?
set -e
if [ -f target/nextest/ci/junit.xml ]; then
mv target/nextest/ci/junit.xml target/nextest/ci/full-transport.xml
fi
exit "$status"
- name: Upload full-selection nextest reports
if: always()
uses: actions/upload-artifact@v4
with:
name: nextest-junit-Linux-full-selections
path: target/nextest/ci/full-*.xml
if-no-files-found: ignore
retention-days: 7
# Windows compiles and runs the suite as the same partitions as Linux, one
# hosted 4-vCPU `windows-latest` job each. The lane that preceded it built
# every test of the workspace in one job and sharded only the execution
# from a nextest archive: in run 36051245663 (warm dependency cache, full
# match) that build spent 27 min compiling the library chain beneath the
# root crate and then 67 min on the root crate's library and its ~40 test
# targets alone, 94 min of `cargo build` plus 5 of archive in a job that
# nothing could shorten, then the five shards ran 10-14 min each: 116 min
# for the lane's verdict, against 27 for Linux. As partitions each job pays
# the chain and its own selection's test targets, as `linux-test-partition`
# does, so the lane's wall time is the slowest partition: 66.5 min in run
# 36077289779 (`runtime`: 25 min of library chain with its dependency
# units beside it, 23 min of 31 test targets, 15 min of tests; the root
# partitions compile 35-38 min each). The manifest's
# `windows_timeout_minutes` are the budgets; the executables the suites
# spawn are built into the same resolution first, as on Linux.
windows-test-partition:
name: Test Windows ${{ matrix.partition }}
needs: scope-gate
if: ${{ needs.scope-gate.outputs.run-os == 'true' }}
runs-on: windows-latest
timeout-minutes: ${{ matrix.timeout }}
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.scope-gate.outputs.windows-partitions) }}
steps:
# Full history for the same reason as the Linux and macOS partition
# jobs: the pinned search-quality fixture commit must be resolvable here.
- uses: actions/checkout@v7
with:
fetch-depth: 0
- name: Tune Windows runner for build and test I/O
uses: ./.github/actions/tune-windows-runner
with:
redirect-temp: "true"
# Install only the toolchain pinned by rust-toolchain.toml, with the
# components it lists (`rust-analyzer` is a test dependency of the LSP
# negotiation acceptance). Installing `stable` as well makes
# Swatinem/rust-cache include an unused compiler in its environment
# key, fragmenting the cache whenever stable moves. The architecture
# tests exec `cargo metadata` at runtime under nextest, so the install
# has to be serial and up front: concurrent rustup shims racing to
# self-install corrupt each other's partial downloads.
- name: Install pinned toolchain
shell: pwsh
run: |
rustup toolchain install
rustup show active-toolchain
- name: Use lld-link linker
shell: pwsh
run: |
where.exe lld-link.exe
lld-link.exe --version
"CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER=lld-link.exe" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
# Node so build.rs can produce the embedded dashboard dist assets for
# the partitions that compile the CLI.
- uses: actions/setup-node@v4
with:
node-version: 22
- uses: ./.github/actions/setup-pnpm
- uses: actions/setup-python@v6
with:
python-version: "3.12"
- name: Install ast-grep
uses: ./.github/actions/install-ast-grep
with:
version: ${{ env.AST_GREP_VERSION }}
- name: Install cargo-nextest
uses: taiki-e/install-action@nextest
# One dependency cache for the lane, as on Linux: `root-journeys`
# resolves the widest graph, so its save covers the other root
# partitions; the lower partitions recompile the dependency units they
# unify differently, beside their shorter chains.
- name: Cache Windows Rust build outputs
uses: Swatinem/rust-cache@v2
with:
shared-key: ci-test-full-windows-msvc-lld
save-if: ${{ matrix.partition == 'root-journeys' }}
cache-on-failure: true
# Every partition job re-proves the cover before it compiles anything,
# as on Linux.
- name: Check the partitions cover every test target
shell: bash
run: |
python scripts/test-linux-test-partitions.py
python scripts/linux-test-partitions.py check
- name: Resolve this partition's cargo selection
id: selection
shell: bash
run: |
{
echo "test=$(python scripts/linux-test-partitions.py cargo-args '${{ matrix.partition }}')"
echo "build=$(python scripts/linux-test-partitions.py build-args '${{ matrix.partition }}')"
} >> "$GITHUB_OUTPUT"
# The CLI and evaluator executables the suites spawn, in the same
# resolution as the test build below (see `linux-test-partition`); the
# test resolvers find them beside the test executables under
# target/perf, so no override or preflight is needed.
- name: Build the executables the suites spawn
if: ${{ steps.selection.outputs.build != '' }}
shell: bash
run: cargo build --locked --profile perf ${{ steps.selection.outputs.build }}
# nextest `ci` policy and the perf cargo profile, as on Linux;
# `--test-threads num-cpus` as the Windows shards ran, since the
# Windows test groups in .config/nextest.toml are sized for it.
- name: Run tests
shell: bash
run: cargo nextest run --profile ci --cargo-profile perf --locked ${{ steps.selection.outputs.test }} --no-tests=fail --test-threads num-cpus --status-level slow
- name: Clean abandoned Windows test children
if: always()
shell: pwsh
run: |
$workspace = (Resolve-Path $env:GITHUB_WORKSPACE).Path
$all = Get-CimInstance Win32_Process
$liveProcessIds = [System.Collections.Generic.HashSet[uint32]]::new()
foreach ($process in $all) {
[void]$liveProcessIds.Add([uint32]$process.ProcessId)
}
$stale = foreach ($process in $all) {
if ($process.Name -ne "tracedecay.exe") {
continue
}
if (-not $process.CommandLine) {
continue
}
$inCurrentWorkspace = $process.CommandLine.IndexOf($workspace, [StringComparison]::OrdinalIgnoreCase) -ge 0
if (-not $inCurrentWorkspace) {
continue
}
if (-not $liveProcessIds.Contains([uint32]$process.ParentProcessId)) {
$process
}
}
foreach ($process in $stale) {
Write-Host "Stopping abandoned tracedecay child pid=$($process.ProcessId)"
Stop-Process -Id $process.ProcessId -Force -ErrorAction SilentlyContinue
}
# `windows-test` folds the partition reports back into one report.
- name: Upload Windows partition nextest report
if: always()
uses: actions/upload-artifact@v4
with:
name: nextest-junit-Windows-${{ matrix.partition }}
path: target/nextest/ci/junit.xml
if-no-files-found: ignore
retention-days: 1
# The one `Test Windows` verdict: every partition must pass. It also folds
# the partition reports into the `nextest-junit-Windows` artifact, one
# directory per partition, as `linux-test` does.
windows-test:
name: Test Windows
if: ${{ !cancelled() && needs.scope-gate.outputs.run-os == 'true' }}
needs: [windows-test-partition, scope-gate]