diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..8dafb5a --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,25 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + labels: + - 'benchmarks: skip' + - package-ecosystem: npm + directories: + - / + - /js/default + schedule: + interval: weekly + labels: + - 'benchmarks: skip' + - package-ecosystem: cargo + directories: + - / + - /crates/handler-rs + - /crates/hyperlight-dummy-guest + schedule: + interval: weekly + labels: + - 'benchmarks: skip' \ No newline at end of file diff --git a/.github/workflows/benchmark-policy.yml b/.github/workflows/benchmark-policy.yml index f26e9c6..1f011b1 100644 --- a/.github/workflows/benchmark-policy.yml +++ b/.github/workflows/benchmark-policy.yml @@ -23,7 +23,6 @@ jobs: steps: - uses: actions/checkout@v7 with: - ref: main persist-credentials: false - uses: actions/setup-node@v7 with: diff --git a/.github/workflows/benchmark-publication.yml b/.github/workflows/benchmark-publication.yml index f37baa5..7ab812b 100644 --- a/.github/workflows/benchmark-publication.yml +++ b/.github/workflows/benchmark-publication.yml @@ -9,6 +9,7 @@ on: permissions: contents: write + statuses: write actions: read pull-requests: read issues: read @@ -19,6 +20,7 @@ concurrency: jobs: publish: + if: github.event_name == 'push' || github.event.workflow_run.event == 'pull_request' runs-on: ubuntu-latest timeout-minutes: 15 steps: diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 38989da..2748c49 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,6 +1,8 @@ name: Benchmark on: + push: + branches: [main] pull_request: branches: [main] types: [opened, synchronize, reopened, labeled, unlabeled, edited, ready_for_review] @@ -11,7 +13,7 @@ permissions: issues: read concurrency: - group: benchmark-${{ github.event.pull_request.number }} + group: benchmark-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true env: @@ -19,6 +21,7 @@ env: jobs: eligibility: + if: github.event_name == 'pull_request' runs-on: ubuntu-latest timeout-minutes: 10 outputs: @@ -26,7 +29,6 @@ jobs: steps: - uses: actions/checkout@v7 with: - ref: main persist-credentials: false - uses: actions/setup-node@v7 with: @@ -40,7 +42,7 @@ jobs: # Record the commit and choose which runners and benchmarks to run. configure: needs: eligibility - if: needs.eligibility.outputs.mode != 'skip' + if: ${{ !cancelled() && (github.event_name == 'push' || (needs.eligibility.result == 'success' && needs.eligibility.outputs.mode != 'skip')) }} runs-on: ubuntu-latest outputs: platforms: ${{ steps.matrix.outputs.platforms }} @@ -60,9 +62,16 @@ jobs: key: npm-scripts-${{ runner.os }}-${{ hashFiles('package-lock.json') }} restore-keys: npm-scripts-${{ runner.os }}- - run: npm ci + - name: Save script npm cache + if: github.event_name == 'push' && steps.npm-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v6 + with: + path: ~/.npm + key: ${{ steps.npm-cache.outputs.cache-primary-key }} - id: matrix run: node scripts/configure-ci.ts - uses: actions/upload-artifact@v7 + if: github.event_name == 'pull_request' with: name: run-identity path: artifacts/run.json @@ -72,6 +81,7 @@ jobs: # Build guest inputs once on KVM for all measurement configurations. producer: needs: configure + if: ${{ !cancelled() && needs.configure.result == 'success' }} runs-on: [self-hosted, Linux, X64, '1ES.Pool=hld-kvm-amd'] timeout-minutes: 90 steps: @@ -99,6 +109,12 @@ jobs: - run: node scripts/build-harness.ts inputs # Build the embedded Hyperlight runtimes for reuse by platform jobs. - run: node scripts/build-harness.ts servers native + - name: Save guest npm cache + if: github.event_name == 'push' && steps.npm-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v6 + with: + path: ~/.npm + key: ${{ steps.npm-cache.outputs.cache-primary-key }} - uses: actions/upload-artifact@v7 with: name: guest-inputs @@ -123,6 +139,7 @@ jobs: # Prime each platform's Cargo cache before benchmark jobs fan out. prepare: needs: [configure, producer] + if: ${{ !cancelled() && needs.configure.result == 'success' && needs.producer.result == 'success' }} strategy: fail-fast: false matrix: ${{ fromJSON(needs.configure.outputs.platforms) }} @@ -154,6 +171,7 @@ jobs: # Build from the platform cache and run one runtime/strategy combination. measure: needs: [configure, prepare] + if: github.event_name == 'pull_request' strategy: fail-fast: false matrix: ${{ fromJSON(needs.configure.outputs.configurations) }} @@ -201,6 +219,7 @@ jobs: # Merge results only after every measurement job succeeds. collect: needs: measure + if: github.event_name == 'pull_request' runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 @@ -231,14 +250,13 @@ jobs: # Provide one required check that also reports failed, skipped, or cancelled stages. benchmark-status: name: Benchmark Status - if: always() + if: ${{ always() && github.event_name == 'pull_request' }} needs: [eligibility, configure, producer, prepare, measure, collect] runs-on: ubuntu-latest timeout-minutes: 5 steps: - uses: actions/checkout@v7 with: - ref: main persist-credentials: false - uses: actions/setup-node@v7 with: diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml index fe293e4..220b351 100644 --- a/.github/workflows/preview.yml +++ b/.github/workflows/preview.yml @@ -15,7 +15,7 @@ concurrency: jobs: revision: - if: ${{ github.event.pull_request.draft == false }} + if: ${{ github.event.pull_request.draft == false && github.event.pull_request.user.login != 'dependabot[bot]' }} runs-on: ubuntu-latest timeout-minutes: 10 outputs: @@ -36,18 +36,8 @@ jobs: env: GH_TOKEN: ${{ github.token }} - approve: - name: "Approve preview PR ${{ needs.revision.outputs.number }} at ${{ needs.revision.outputs.head }}" - needs: revision - runs-on: ubuntu-latest - environment: - name: preview-approval - url: "https://github.com/${{ needs.revision.outputs.repository }}/commit/${{ needs.revision.outputs.head }}" - steps: - - run: echo 'Preview revision approved' - build: - needs: [revision, approve] + needs: revision runs-on: ubuntu-latest timeout-minutes: 15 permissions: diff --git a/docs/README.md b/docs/README.md index ddb93f3..774cc01 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,50 +1,23 @@ # Development Guide -See the [project overview](../README.md) for an introduction. +See the [project overview](../README.md) and [result format](results.md). ## Development -Use Node.js 24 or later. +Requires Node.js 24 or later. ```sh npm ci npm run dev ``` -The dashboard loads published history from -`https://raw.githubusercontent.com/hyperlight-dev/hyperlight-bench/data/index.json`. -Local development uses that history too. Until the data branch is published, -the dashboard shows `Data unavailable`. +Development loads published history. Use `?demo=1` for synthetic data or set +`VITE_HISTORY_URL` in `.env.local` to select another index. -Open `http://localhost:5173/?demo=1` for synthetic data. -Demo view links retain the `demo=1` parameter. +## Local Benchmarks -Set `VITE_HISTORY_URL` in `.env.local` to select another history index: - -```dotenv -VITE_HISTORY_URL=/data/index.json -``` - -This example serves files from `public/data/`. An absolute HTTPS URL can point -to another host that allows cross-origin requests. Relative URLs resolve against -the page URL. Restart the dev server after changing the setting. Production -builds capture its value during `npm run build`. - -The index and bundles follow [the history format](results.md#history-index). -An empty index shows `No published runs yet`. Fetch or validation failures show -`Data unavailable` and log details in the browser console. Local benchmark -summaries are not loaded by this interface. - -## Benchmark Commands - -### Local End To End - -Requires Linux x86_64, Node.js 24+, Rust, just, and read/write access to -`/dev/kvm` or `/dev/mshv`. Install wasm-tools 1.259.0 or newer, -hyperlight-wasm-aot 0.15.0, cargo-hyperlight 0.1.14, -componentize-qjs-cli 0.3.0 with `--no-default-features`, and oha 1.9.0. -Use `cargo install @ --locked` for each tool. -Place the pinned tools on `PATH` before building. +Requires Linux x86_64, Rust, just, access to `/dev/kvm` or `/dev/mshv`, and the +tools pinned in [CI setup](../.github/actions/setup/action.yml) on `PATH`. ```sh npm ci @@ -54,403 +27,79 @@ npm run benchmark:local -- --platform kvm npm run dev:local ``` -Open the URL printed by Vite. The dashboard labels these results Local. -The collector runs all 18 runtimes and three lifecycle strategies sequentially. -Each configuration receives 60 seconds of load, requiring at least 54 minutes -per platform. Use `--platform mshv3` on an MSHV host. -Build and measure on the target host. Local collection does not contact Azure -metadata or GitHub and cannot publish to CI history. - -For a short verification run across all 54 configurations: - -```sh -npm run benchmark:local -- --platform kvm --smoke -``` - -Add `--runtime hyperlight-wasm-qjs` to run one runtime's three strategies. -Smoke mode sends one measured request per configuration with one connection -and a 120-second client timeout. It verifies execution and result loading, -not performance stability. The bundle records the request count and load settings. -JIT-heavy configurations can still take several minutes across the full matrix. - -Timed runs accept `--duration` and `--client-timeout` in seconds. -The default client timeout is unlimited. Requests still in flight are cancelled -when the load duration ends. `--client-timeout` enables a per-request limit. -`--concurrency` sets the connection count. For slower local hosts, use: - -```sh -npm run benchmark:local -- --platform kvm --duration 60 --concurrency 1 --client-timeout 120 -``` - -Timeouts and missing latency samples fail validation. All local load overrides -are recorded and keep the results separate from CI history. - -Raw output is retained under `results/local//`. A complete successful -run writes an immutable bundle under `public/local-data/runs/` and atomically -updates `public/local-data/index.json`. The site selects the latest successful -run. Failed runs retain their diagnostics and leave the index unchanged. -Both directories are ignored by Git. Local runs record CPU, memory, OS, and -tool metadata. SKU and pool are `local`, and Azure region is null. -Git revisions describe HEAD with uncommitted changes noted in the message. -Unversioned directories use zero SHAs and an explicit unversioned message. - -```sh -npm test -npm run build -``` - -These commands run local publication and Pages tests and compile the dashboard. -They require no GitHub credentials. Hosted approvals, runner permissions, and -deployment still require verification in GitHub Actions. - -### Individual Commands - -Run `just` to list commands and their descriptions. -Build commands require the tool versions pinned in `.github/actions/setup/action.yml`. -QuickJS requires `componentize-qjs-cli` 0.3.0 on the command's `PATH`. -The isolated local installation is not automatically selected yet. -Use `just setup-rust-wasm-toolchain` to install the Rust component toolchain. - -| Command | Scope | -| --- | --- | -| `just build-benchmark-artifacts` | All guest inputs and both server binaries | -| `just build-benchmark-inputs` | WIT, jco/QuickJS/Rust Wasm, native/Pulley AOT, and the native dummy guest | -| `just build-server-binaries` | Both release server binaries and build metadata | -| `just build-native-server [extra-features]` | Native release server, optionally with `time_phases` | -| `just build-pulley-server [extra-features]` | Pulley release server, optionally with `time_phases` | -| `just build-handler-wit` | Binary WIT interface | -| `just build-jco-wasm` | StarlingMonkey Wasm component | -| `just build-qjs-wasm` | QuickJS Wasm component | -| `just build-rust-wasm` | Rust Wasm component | -| `just compile-native-aot ` | Native AOT compilation | -| `just compile-pulley-aot ` | Pulley AOT compilation | -| `just build-hyperlight-dummy-guest` | Native Hyperlight baseline guest | - -Server builds invalidate only the `hyperlight-wasm` release package before -each flavor. Its build script embeds a guest from a shared path. Invalidation -ensures that path contains the requested native or Pulley runtime. -Other Cargo caches, cache keys, and permissions are retained. - -Guest selection and AOT target are separate commands. For example: - -```sh -just build-qjs-wasm -just compile-native-aot js/componentize-qjs/handler.wasm js/componentize-qjs/handler.aot -just compile-pulley-aot js/componentize-qjs/handler.wasm js/componentize-qjs/handler.pulley.aot -``` - -`just run-server ` starts a built server on port 3000. -Hyperlight Pulley runtime IDs select `artifacts/bin/pulley/http-bench`. -Other runtime IDs select `artifacts/bin/native/http-bench`. -Strategy arguments remain `new` (Renew), `reload` (Restore), and `reuse` (Reuse). -`just run-http-load [url] [duration] [output]` sends load to an existing server. -Its defaults are `http://127.0.0.1:3000`, `10s`, and `perf.json`. - -`just run-local-benchmark [timeout-ms] [timeout-check-interval-ms] [duration]` -builds both standard server binaries, starts the selected server, runs oha, and stops it. -Guest inputs must already exist. Defaults remain `1000`, `10`, and `60s`. -This local workflow produces root-level summaries and retains the imported load settings. -CI uses `scripts/benchmark.ts` for validated bundles and runner metadata. - -`just clean-build-artifacts` removes generated build outputs. -`just clean-local-benchmark-results` removes root-level performance and memory summaries. +Use `--platform mshv3` on MSHV. A full run takes at least 54 minutes per platform. +Add `--smoke` for one request per configuration or `--runtime ` to limit runtimes. +Raw output is in `results/local/`. Successful runs update `public/local-data/`. ## Validation ```sh +npm test npm run build -npm run validate -- --fixtures -npm run validate -- --complete run.json -npm run validate -- --policy policy.json run.json -node --test scripts/publication.test.ts ``` -The fixture command validates the run bundles and their dashboard projection. -JSON inputs may contain a single run bundle or a dashboard selection archive. - -Publication tests use temporary stores and simulated GitHub and Git responses. -They cover immutable writes, promotion retries, conflicts, stale provenance, -label changes, and archive retention after promotion failure. They make no -network requests or remote writes. Live workflow validation remains separate. - -See [the result contract](results.md) for fields and validation rules. - -## Local Publication Commands - -```sh -npm run results:store -- --directory ./data-store run.json -npm run results:publish -- --directory ./data-store publication.json -``` - -`results:store` validates a complete run against the shared publication policy -and stores its immutable bundle. It leaves the history index unchanged. -`results:publish` reads that stored bundle and the supplied merge provenance, -then writes a publication record and updates the history index. -Both accept `--policy policy.json` for a trusted policy captured for the measured revision. - -Promotion requires `run.pullRequest` in the stored bundle. PR benchmark runs -record this provenance. See [publication records](results.md#publication-records) -for the input format and trust requirements. - -These commands only write local files. They make no GitHub requests, commits, -pushes, or deployments. The separate CI publisher performs GitHub verification -and data-branch writes. Pages deployment manages preview cleanup. +Tests simulate GitHub. Validate permissions and deployment changes in Actions. ## Workflow Overview -Solid arrows show triggers and job order. Dotted arrows show artifacts and data. -Each PR workflow subscribes to its own set of `pull_request` events. +Arrows show execution and data paths. Cylinders store results. ```mermaid -flowchart TD - PR["PR events targeting main"] - Main["Push to main"] - - subgraph Policy["Benchmark Policy"] - Check["Read-only eligibility check"] - end - - subgraph Benchmark["Benchmark"] - Eligibility["Evaluate skip eligibility"] - Measure["Build and measure"] - Collect["Collect validated run bundle"] - Status["Benchmark Status"] - Eligibility -->|Required| Measure --> Collect --> Status - Eligibility -->|Approved skip| Status +flowchart TB + subgraph Before["Before merge
Outside contributors: maintainer selects Approve and run
Collaborators: automatic"] + direction TB + subgraph ResultPath["Benchmark results"] + direction LR + Benchmark["Run benchmark matrix"] --> Artifact[("Result artifact")] + Artifact --> Archive["Validate and store results
Runs code from main"] + Archive --> Results[("Stored results
data branch")] + end + subgraph PreviewPath["Website preview"] + direction LR + Preview["Record current PR head"] + Preview --> Build["Build PR website preview"] + Build --> PreviewSite["Deploy preview using main code"] + PreviewSite --> Live["Preview available
/previews/pr-N/"] + end + subgraph MergePath["Merge requirements"] + direction LR + Ready["Policy and benchmark checks pass
Branch is up to date with main"] + Ready --> Merge["User with merge permission
merges the PR"] + end + ResultPath ~~~ PreviewPath ~~~ MergePath end - subgraph Preview["PR Preview"] - Revision["Record open, non-draft PR head"] - Approval["preview-approval environment"] - Build["Build approved PR head"] - Revision --> Approval --> Build + subgraph After["After merge: code from main includes the merged changes"] + direction LR + Main["Push to main"] + Publish["Verify merged source
Add results to production history"] + ProductionSite["Deploy merged website
and remaining previews"] + Warm["Build binaries and update caches"] + Main --> Publish --> ProductionSite + Main --> Warm end - - Publish["Benchmark Publication: archive successful PR results, promote merged candidates"] - Data[("data branch: pending bundles and published history")] - Site["Pages: assemble from trusted main, verify, deploy"] - Live["Production site and /previews/pr-N/"] - - PR --> Policy - PR --> Benchmark - PR --> Preview - Main --> Publish - Main --> Site - - Benchmark -->|workflow_run: completed| Publish - Policy -->|workflow_run: completed| Site - Preview -->|workflow_run: completed| Site - Publish -->|workflow_run: completed| Site - - Collect -.->|Run artifact| Publish - Publish -.-> Data - Data -.-> Site - Build -.->|Approved preview artifact| Site - Site --> Live + MergePath --> After + ResultPath -->|Read stored results| After ``` -The policy workflow also receives PR closure events. Its check job is skipped -on closure, and workflow completion triggers Pages cleanup. -Publication accepts successful PR runs and checks merge -provenance before adding results to published history. - -## PR Eligibility And Publication - -`Benchmark Policy` uses `pull_request` with read-only permissions and runs -trusted `main` scripts on hosted runners. Its job check reports eligibility. -The job summary records the PR head, base, and benchmark decision. Benchmark -jobs also evaluate eligibility before starting the self-hosted stages. - -To skip benchmarks, a maintainer with write access must apply `benchmarks: skip`. -The allowlist covers `docs/`, `src/`, -`public/`, the README, license, and HTML entry point. Renames check -both paths. Cargo, JavaScript guests, shared contracts, dependencies, scripts, -and workflow changes require benchmarks. -After correcting a failed policy decision, rerun the Benchmark workflow if needed. - -The publisher reevaluates eligibility when archiving and promoting results. -An approved skip publishes no measurements. Removing the label requires a -complete benchmark run for the current candidate. - -`Benchmark Publication` runs trusted `main` scripts on hosted runners. Successful -PR attempts are checked against GitHub workflow and commit metadata. All jobs -must have a successful latest result through the selected attempt. Use -**Re-run failed jobs** to retain successful configurations, or rerun an individual -job and its dependents. Each retry replaces its configuration's result artifact. -Collection combines the artifacts from that workflow run. -A failed retry blocks publication even if an earlier -attempt succeeded. Source revisions and benchmark definitions must match. -Use **Re-run all jobs** when required artifacts have expired. -The executed Benchmark workflow must match the trusted workflow. A workflow -change can require archival recovery after maintainer review and merge. - -Validated bundles and trusted policies are retained on `data`, with a pending -pointer for each PR. A push to `main` resolves merged PRs through GitHub's -commit association API and promotes their archived results. Merge promotion -requires the current PR head and a merged -tree matching the measured candidate. It records the selected attempt and -updates history. It never launches a benchmark run. -PR website previews and Pages deployment use separate workflows. - -Publication jobs share a concurrency group, fetch the latest data branch, and -use ordinary pushes. Push conflicts fail. Rerun a failed archival job to retain -its artifact. Rerun the failed publication workflow to retry promotion. -Missing or expired artifacts require a new approved benchmark run. -Stale candidates and changed merged trees fail publication. -If a push commit list is unavailable or truncated, rerun the publication workflow -that archived each merged PR's results. Archival also promotes results for merged PRs. - -### Repository Requirements - -Before enabling PR execution: - -* Require `Benchmark Policy` and `Benchmark Status` and an up-to-date branch before merge. -* Benchmark jobs run PR code on self-hosted runners with no environment approval gate. -* Use isolated, disposable self-hosted runners with restricted credentials and network access. -* Protect `main` and the trusted workflow scripts. Restrict `data` writes to the publisher. -* Allow the trusted publication workflow to create and update `data`. - -PR caches remain scoped to GitHub's PR merge ref. The original producer and -platform cache keys, setup action, and designated writers are retained. -Trusted policy and publication jobs use hosted runners, install dependencies -with lifecycle scripts disabled, and do not restore PR caches or execute artifacts. -These repository settings cannot be enforced by the workflow files themselves. - -## Pages And PR Previews - -Configure GitHub Pages to deploy through **GitHub Actions**. Create the -`preview-approval` environment with required maintainer reviewers. Keep it free -of secrets. Enable prevention of self-review and disable administrator bypass -where available. The `github-pages` environment is used for final deployment. - -`PR Preview` uses `pull_request` with read-only permissions. It records the open -PR head and pauses at `preview-approval`. -The approval job names the full revision and links to its commit. A maintainer -uses **Review deployments**, then **Approve and deploy**. Each new head starts -a new approval. No label or comment is needed for preview approval. - -The approved head is built on a separate disposable hosted runner with read-only -permissions. PR code never runs in the approval or Pages deployment jobs. -The build artifact is named for the PR, head, and attempt and retained for 90 days. -Draft PRs have no preview. Rerun all jobs to rebuild an expired preview. - -`Pages` builds production from current `main` and discovers successful preview -builds for current open PR heads. It matches GitHub run metadata to the PR number, -head, and source repository. It verifies the trusted preview workflow and -the revision-specific approval job. A changed preview workflow requires fresh -approved builds. Closed, merged, draft, and outdated previews are omitted. - -One artifact contains production at `/` and previews at `/previews/pr-/`. -These paths are relative to the site's base URL, including a GitHub project path -when one is configured. Builds use relative asset and data URLs. - -Production receives only runs in the published index. Each preview receives -published history plus its current pending run, when available. -The pending run must match the PR head and base. Approved benchmark skips use -published history. Preview headers identify the PR revision and pending-data -status. A preview with no available runs shows an empty state. Different -benchmark definitions form separate selectable histories. Production defaults -to the group with the newest run. Previews default to the pending result's -group. Charts and downloads use only the selected compatible group. - -Assembly starts from a fresh directory. It rejects symlinks, hidden entries, -special files, and builds over 100 MiB or 10,000 files. Builds must reserve -`data/` and `previews/` for the assembler. Avoid placing those directories in -`public/` for deployed builds. - -Pages jobs share a concurrency group. They discover preview builds during -assembly, then recheck refs, PR revisions, eligibility, and selected builds -before deployment. Changed inputs fail the -job. Rerun Pages if a newer event has not already queued a replacement. An event -during deployment is reflected by the next serialized deployment. - -Main pushes and completion of preview, publication, or policy workflows refresh -Pages. PR lifecycle changes trigger the policy workflow. Its completion wakes -the trusted Pages workflow through `workflow_run`. Pages rereads GitHub state -and executes only `main` code. Rerun Pages to recover a failed deployment. -Closing a PR removes its path on the next successful deployment. Expired build -artifacts are omitted when Pages next assembles the site. - -Preview JavaScript shares the production origin. Environment approval is required -because preview paths are not a security boundary. These workflows have been -validated locally, not executed on GitHub. Environment rules and Pages settings -must be configured before enabling them. - -## CI Setup And Caching - -The setup action matches the source repository's action byte for byte. -It sets `CARGO_HOME=/opt/cargo`, configures `PATH`, repairs Cargo home ownership, -and reuses installed tools through the original version checks. - -The producer writes `kvm-amd-producer-v2`. Platform preparation writes -`kvm-amd-build-v2` or `mshv3-amd-build-v2` and skips compilation on an exact cache hit. -Benchmark jobs restore those platform caches with `BENCH_ONLY=true` and cache writes disabled. -Each benchmark job builds its selected server flavor before starting the server. -The collector invokes `oha` from the setup action's `PATH`. -Guest components and embedded runtime binaries are shared through artifacts. - -The producer retains the guest npm cache key. Configure, measurement, and -collection jobs restore the TypeScript npm cache. PR jobs do not save npm caches. - -## CI Failure Policy - -PR benchmark runs target `main` and record the -tested merge candidate. `Benchmark Status` requires successful eligibility and -all benchmark stages, including collection. An approved skip requires successful -eligibility and omits benchmark stages. Other failed, cancelled, skipped, and -missing stages fail this gate. - -Build commands propagate child-process failures and require nonempty artifacts. -Component generation removes its previous output before running. -Measurement removes previous summaries before launching the server. -Supervised server panics terminate the process with a failure exit code. -Benchmark failures print their reason in the job log and emit an error annotation. -Raw summaries and server logs remain available as job artifacts. - -Collection requires all 108 successful configurations and the expected machine SKUs. -Invalid metrics, missing results, and request errors reject the run. -The duration-limited workload permits up to 50 `aborted due to deadline` cancellations. -These represent in-flight requests at the measurement deadline and remain in the raw summary. -`fail-fast: false` lets other matrix jobs finish collecting diagnostics. Failed jobs retain their failed status. - -## Platform Support - -The benchmark targets Linux KVM and MSHV across 18 runtimes and three lifecycle strategies. -Windows/WHP support is deferred pending a reusable prepared AOT mapping API and runner validation. - -Hyperlight Wasm prepares a private file-backed AOT mapping once per worker. -Each sandbox context uses `load_module_by_mapping`. Renew reuses the worker's mapping. -The mapping remains alive until its worker and sandbox contexts are released. -The upstream API can fall back to copying if guest mapping fails. -The AOT loading method is recorded in the benchmark settings. - -## Handler Configuration - -The HTTP endpoint invokes a fixed guest request and ignores HTTP request bodies. -Benchmark version 2 excludes body collection and unused queue timestamps. -Phase timing counters are compiled only with `time_phases`. Shutdown writes -memory results and any enabled phase report before exiting. -Renew phase reports include context creation in `load` and destruction in `unload`. -Memory method version 2 takes a final RSS sample at shutdown and combines it -with the 500 ms samples. This covers runs shorter than the sampling interval. +## CI Rules -The server announces readiness after every worker completes initial setup. -Restore creates its initial context, and Reuse also loads it. Renew prepares -worker state and creates contexts per request. Startup fails if a worker fails -or the pool exceeds its 120-second startup deadline. The benchmark records -`workerReadiness: all-workers`. The collector still sends one readiness request -and performs no warmup. +* `benchmarks: skip` skips measurement and result publication. Dependabot applies it automatically. +* Preview builds run independently of benchmarks. Draft and Dependabot PRs have no preview. +* Production uses published history. Previews include matching pending results when available. +* Benchmark and preview workflow YAML must match `main` before publishers accept their artifacts. +* Merge requires `Benchmark Policy`, `Benchmark Status`, an up-to-date branch, and squash merging. Required PR review count is zero. Preview and publication success do not block merge. -Component paths and Hyperlight memory settings live in `crates/server/src/handlers/config.rs`. -`Runtime::start_pool` maps each CLI runtime to a backend configuration. -Hyperlight Wasm selects scratch memory through `PerStrategy` during context creation. -Each runtime can select its own memory settings. +Pages deploys a data snapshot. It refreshes on pushes to `main` and completion +of preview, policy, or publication workflows. Closing a PR removes its preview +on the next deployment. A direct push to `data` requires a Pages rerun. -`Handler::prepare_worker` reads or maps artifacts once per worker. -The worker pool owns the Renew, Restore, and Reuse loops. -Wasmtime compiles Wasm or deserializes native/Pulley AOT during context creation. -Its engine and pooling allocator are also created there. +## Security -To add a component variant, define its artifact paths, select its memory settings, -and add its runtime mapping and shared catalog entry. -A new execution backend implements `Handler`. \ No newline at end of file +* Outside contributors require **Approve and run**. Collaborators run automatically. Allowing preview execution also permits serving its JavaScript to visitors. +* PR jobs use read-only repository permissions. The separate `workflow_run` publishers execute `main` code with their own tokens. +* Benchmark Publication uses `contents: write` and `statuses: write`. It commits to `data` as `github-actions[bot]`. These permissions are repository-wide, so publisher code restricts writes. +* Pages uses `pages: write` and `id-token: write`. Configure Pages for GitHub Actions and restrict the `github-pages` environment to `main`. +* Publishers validate PR artifacts as untrusted input. They do not execute artifact scripts or restore PR caches. Workflow matching cannot prove measurements are genuine. +* PRs can change workflows and check scripts. Inspect those changes before allowing execution or merging. Self-hosted runners must be disposable and isolated from credentials and sensitive networks. +* Preview JavaScript shares production's origin and browser storage. Separate-origin hosting is required for browser isolation. \ No newline at end of file diff --git a/docs/results.md b/docs/results.md index e38f540..e7c1855 100644 --- a/docs/results.md +++ b/docs/results.md @@ -144,6 +144,10 @@ Use `{"schemaVersion":1,"runs":[]}` for an empty history. `historyRunPath` derives each bundle path relative to the index URL. Paths contain the recorded workflow ID and attempt, independent of chart labels. +Served index entries may include `displayCommit` with the final merge SHA, title, +and URL from the validated publication record. Publication records require the +merge commit message. Measured commit metadata remains in the immutable bundle. + `parseHistoryRun` requires a complete measured bundle whose identity matches the index entry. The dashboard sorts runs by creation time and run key. It rejects the history if any referenced bundle fails to load or validate. diff --git a/scripts/assemble-site.ts b/scripts/assemble-site.ts index a4fe64e..7337e43 100644 --- a/scripts/assemble-site.ts +++ b/scripts/assemble-site.ts @@ -1,7 +1,7 @@ import { cpSync, existsSync, lstatSync, mkdirSync, readdirSync, rmSync } from 'node:fs' import { resolve } from 'node:path' import { z } from 'zod' -import { historyIndexSchema, historyRunPath, parseHistoryRun, runKey, groupHistories } from '../shared/results.ts' +import { historyIndexSchema, historyRunPath, parseHistoryRun, runKey, groupHistories, validatePublication } from '../shared/results.ts' import type { RunBundle } from '../shared/results.ts' import { readJson, readOptionalJson, writeJson } from './result-store.ts' @@ -55,7 +55,20 @@ export function assembleSite(options: { const seen = new Set() function serveData(destination: string, bundles: RunBundle[], preview?: { number: number, head: string, pending: { id: string, attempt: number } | null }) { groupHistories(bundles) - const history = historyIndexSchema.parse({ schemaVersion: 1, runs: bundles.map(bundle => ({ id: bundle.run.id, attempt: bundle.run.attempt })), ...(preview ? { preview } : {}) }) + const runs = bundles.map(bundle => { + const entry = { id: bundle.run.id, attempt: bundle.run.attempt } + if (!index.runs.some(run => run.id === entry.id && run.attempt === entry.attempt)) return entry + const pr = bundle.run.pullRequest + const input = pr ? readOptionalJson(resolve(options.store, 'publications', `pr-${pr.number}.json`)) : undefined + if (input === undefined) return entry + const publication = validatePublication(bundle, input) + return { ...entry, displayCommit: { + sha: publication.merge.sha, + message: publication.merge.message.split('\n')[0], + url: `https://github.com/${publication.pullRequest.repository}/commit/${publication.merge.sha}`, + } } + }) + const history = historyIndexSchema.parse({ schemaVersion: 1, runs, ...(preview ? { preview } : {}) }) for (const bundle of bundles) writeJson(resolve(destination, 'data', historyRunPath(bundle.run)), bundle, true) writeJson(resolve(destination, 'data/index.json'), history, true) } diff --git a/scripts/configure-ci.ts b/scripts/configure-ci.ts index 2819257..24e544b 100644 --- a/scripts/configure-ci.ts +++ b/scripts/configure-ci.ts @@ -4,18 +4,23 @@ import { output } from './runner-metadata.ts' const event = JSON.parse(readFileSync(process.env.GITHUB_EVENT_PATH!, 'utf8')) const pullRequest = process.env.GITHUB_EVENT_NAME === 'pull_request' ? event.pull_request : undefined -if (!pullRequest || pullRequest.base.ref !== 'main') throw new Error('Benchmark requires a PR targeting main') +const warmCaches = process.env.GITHUB_EVENT_NAME === 'push' && event.ref === 'refs/heads/main' && !event.deleted +if (!warmCaches && (!pullRequest || pullRequest.base.ref !== 'main')) throw new Error('Expected a PR targeting main or a push to main') const repository = process.env.GITHUB_REPOSITORY const runId = process.env.GITHUB_RUN_ID const attempt = Number(process.env.GITHUB_RUN_ATTEMPT) if (!repository || !runId || !Number.isSafeInteger(attempt) || attempt < 1) throw new Error('GitHub workflow identity is required') -const sha = output('git', ['rev-parse', 'HEAD']) -const base = process.env.GITHUB_SERVER_URL ?? 'https://github.com' -const run = { - id: runId, attempt, createdAt: new Date().toISOString(), - workflow: { name: process.env.GITHUB_WORKFLOW ?? 'Benchmark', url: `${base}/${repository}/actions/runs/${runId}/attempts/${attempt}` }, - commit: { sha, tree: output('git', ['rev-parse', 'HEAD^{tree}']), message: output('git', ['show', '-s', '--format=%s', 'HEAD']), url: `${base}/${repository}/commit/${sha}` }, - pullRequest: { repository, number: pullRequest.number, head: pullRequest.head.sha, base: pullRequest.base.sha }, +if (pullRequest) { + const sha = output('git', ['rev-parse', 'HEAD']) + const base = process.env.GITHUB_SERVER_URL ?? 'https://github.com' + const run = { + id: runId, attempt, createdAt: new Date().toISOString(), + workflow: { name: process.env.GITHUB_WORKFLOW ?? 'Benchmark', url: `${base}/${repository}/actions/runs/${runId}/attempts/${attempt}` }, + commit: { sha, tree: output('git', ['rev-parse', 'HEAD^{tree}']), message: output('git', ['show', '-s', '--format=%s', 'HEAD']), url: `${base}/${repository}/commit/${sha}` }, + pullRequest: { repository, number: pullRequest.number, head: pullRequest.head.sha, base: pullRequest.base.sha }, + } + mkdirSync('artifacts', { recursive: true }) + writeFileSync('artifacts/run.json', `${JSON.stringify(run, null, 2)}\n`) } const platforms = Object.entries(runnerPools).map(([platform, runner]) => ({ platform, labels: ['self-hosted', 'Linux', 'X64', `1ES.Pool=${runner.pool}`] })) const configurations = benchmark.expectedConfigurations.map(configuration => ({ @@ -23,7 +28,5 @@ const configurations = benchmark.expectedConfigurations.map(configuration => ({ flavor: serverFlavor(configuration.runtimeId), labels: platforms.find(platform => platform.platform === configuration.platformId)!.labels, })) -mkdirSync('artifacts', { recursive: true }) -writeFileSync('artifacts/run.json', `${JSON.stringify(run, null, 2)}\n`) if (!process.env.GITHUB_OUTPUT) throw new Error('GITHUB_OUTPUT is required') appendFileSync(process.env.GITHUB_OUTPUT, `platforms=${JSON.stringify({ include: platforms })}\nconfigurations=${JSON.stringify({ include: configurations })}\n`) \ No newline at end of file diff --git a/scripts/github-pr.ts b/scripts/github-pr.ts index 2c9d4f4..47a510f 100644 --- a/scripts/github-pr.ts +++ b/scripts/github-pr.ts @@ -1,4 +1,5 @@ import { z } from 'zod' +import { readFileSync } from 'node:fs' const repository = process.env.GITHUB_REPOSITORY! if (!/^[\w.-]+\/[\w.-]+$/.test(repository)) throw new Error('GITHUB_REPOSITORY is required') @@ -6,12 +7,14 @@ const api = 'https://api.github.com' const token = process.env.GH_TOKEN if (!token) throw new Error('GH_TOKEN is required') -export async function github(path: string): Promise { +export async function github(path: string, body?: unknown): Promise { const response = await fetch(`${api}/repos/${repository}/${path}`, { - headers: { Accept: 'application/vnd.github+json', Authorization: `Bearer ${token}`, 'X-GitHub-Api-Version': '2022-11-28' }, + method: body === undefined ? 'GET' : 'POST', + body: body === undefined ? undefined : JSON.stringify(body), + headers: { Accept: 'application/vnd.github+json', Authorization: `Bearer ${token}`, 'X-GitHub-Api-Version': '2022-11-28', 'Content-Type': 'application/json' }, signal: AbortSignal.timeout(30000), }) - if (!response.ok) throw new Error(`GitHub GET ${path}: HTTP ${response.status}`) + if (!response.ok) throw new Error(`GitHub ${body === undefined ? 'GET' : 'POST'} ${path}: HTTP ${response.status}`) return response.json() } @@ -25,7 +28,14 @@ export async function pages(path: string): Promise { } } +export async function workflowTrusted(path: string, head: string): Promise { + const workflow = await github(`contents/${path}?ref=${head}`) + if (workflow.encoding !== 'base64') return false + return Buffer.from(workflow.content, 'base64').toString('utf8') === readFileSync(path, 'utf8') +} + const commit = z.string().regex(/^[a-f0-9]{40}$/) + export const decisionSchema = z.object({ schemaVersion: z.literal(1), repository: z.string(), @@ -37,11 +47,6 @@ export const decisionSchema = z.object({ approver: z.string().nullable(), }) -async function maintainer(login: string): Promise { - const result = await github(`collaborators/${encodeURIComponent(login)}/permission`) - return ['admin', 'maintain', 'write'].includes(result.permission) -} - export async function eligibility(number: number) { if (!Number.isSafeInteger(number) || number < 1) throw new Error('A PR number is required') const pr = await github(`pulls/${number}`) @@ -51,19 +56,8 @@ export async function eligibility(number: number) { mode: 'required', reason: 'Benchmark-affecting changes require a complete run.', approver: null, }) if (pr.labels.some((label: { name: string }) => label.name === 'benchmarks: skip')) { - const files = await pages(`pulls/${number}/files`) - if (files.length !== pr.changed_files || !files.length) throw new Error('Cannot verify the complete changed-file list') - const allowed = (path: string) => /^(docs\/|src\/|public\/)/.test(path) - || ['README.md', 'LICENSE', 'index.html'].includes(path) - if (files.some(file => !allowed(file.filename) || (file.previous_filename && !allowed(file.previous_filename)))) { - throw new Error('Skip label covers benchmark-sensitive files. Remove the label and run benchmarks.') - } - const events = await pages(`issues/${number}/events`) - const label = events.filter(event => event.event === 'labeled' && event.label?.name === 'benchmarks: skip').at(-1) - if (!label?.actor?.login || !await maintainer(label.actor.login)) throw new Error('Skip label must be applied by a maintainer') decision.mode = 'skip' - decision.reason = 'Maintainer applied benchmarks: skip and all changed files are eligible.' - decision.approver = label.actor.login + decision.reason = 'PR has the benchmarks: skip label.' } return { pr, decision } } \ No newline at end of file diff --git a/scripts/publication.test.ts b/scripts/publication.test.ts index 65ec43e..46abb7f 100644 --- a/scripts/publication.test.ts +++ b/scripts/publication.test.ts @@ -18,6 +18,33 @@ import { mergeBundles } from './merge-results.ts' const repository = 'example/benchmarks' const root = resolve(import.meta.dirname, '..') +test('main pushes configure cache warming without a measurement identity', async context => { + const temporary = mkdtempSync(resolve(tmpdir(), 'benchmark-cache-config-')) + context.after(() => rmSync(temporary, { recursive: true, force: true })) + const eventPath = resolve(temporary, 'event.json') + const outputPath = resolve(temporary, 'output') + const configure = () => spawnSync(process.execPath, [resolve(root, 'scripts/configure-ci.ts')], { + cwd: temporary, encoding: 'utf8', + env: { + ...process.env, GITHUB_EVENT_NAME: 'push', GITHUB_EVENT_PATH: eventPath, + GITHUB_OUTPUT: outputPath, GITHUB_REPOSITORY: repository, + GITHUB_RUN_ID: '12345', GITHUB_RUN_ATTEMPT: '1', + }, + }) + writeJson(eventPath, { ref: 'refs/heads/main', deleted: false }, false) + const result = configure() + assert.equal(result.status, 0, result.stderr) + const outputs = Object.fromEntries([...readFileSync(outputPath, 'utf8').matchAll(/^(\w+)=(.+)$/gm)] + .map(match => [match[1], JSON.parse(match[2]!)])) + assert.deepEqual(outputs.platforms.include.map((entry: { platform: string }) => entry.platform), ['kvm', 'mshv3']) + assert.equal(outputs.configurations.include.length, 108) + assert.equal(existsSync(resolve(temporary, 'artifacts')), false) + for (const event of [{ ref: 'refs/heads/feature' }, { ref: 'refs/heads/main', deleted: true }]) { + writeJson(eventPath, event, false) + assert.notEqual(configure().status, 0) + } +}) + test('partial retries retain successful measurements from the same source revision', () => { const bundle = fixture() const shards = bundle.measurements.map(measurement => { @@ -158,7 +185,7 @@ function publication(bundle = fixture()) { return { schemaVersion: 1, run: { id: bundle.run.id, attempt: bundle.run.attempt }, pullRequest: bundle.run.pullRequest, - merge: { sha: 'c'.repeat(40), tree: bundle.run.commit.tree, mergedAt: '2026-09-15T00:00:00Z' }, + merge: { sha: 'c'.repeat(40), tree: bundle.run.commit.tree, mergedAt: '2026-09-15T00:00:00Z', message: 'Benchmark results (#7)' }, } } @@ -185,8 +212,10 @@ test('benchmark definition changes retain selectable histories and pending previ const policy = { ...publicationPolicy, catalog: next.catalog, benchmark: next.benchmark } assert.throws(() => validateHistory([first, next])) storeRun(store, next, policy) - promoteRun(store, publication(next), policy) - promoteRun(store, publication(next), policy) + const record = publication(next) + const titled = { ...record, merge: { ...record.merge, message: 'Improve benchmarks (#10)\n\nCommit details' } } + promoteRun(store, titled, policy) + promoteRun(store, titled, policy) bundles.push(next) } assert.equal(readFileSync(resolve(store, historyRunPath(first.run)), 'utf8'), original) @@ -218,13 +247,19 @@ test('benchmark definition changes retain selectable histories and pending previ const latest = await loadHistory(url) assert.equal(latest!.histories!.length, 4) assert.deepEqual(latest!.runs.map(run => run.id), [runKey(bundles.at(-1)!)]) + assert.equal(latest!.runs[0]!.message, 'Improve benchmarks (#10)') + assert.equal(latest!.runs[0]!.commit, 'ccccccc') + assert.equal(latest!.runs[0]!.commitUrl, `https://github.com/${repository}/commit/${'c'.repeat(40)}`) + assert.deepEqual(latest!.runs[0]!.bundle, bundles.at(-1)) const older = await loadHistory(url, runKey(first)) assert.deepEqual(older!.runs.map(run => run.id), [runKey(first)]) + assert.equal(older!.runs[0]!.message, 'Benchmark results (#7)') assert.equal((await loadHistory(url, 'missing'))!.historyId, latest!.historyId) const previewUrl = new URL('https://fixture.test/previews/pr-7/data/index.json') const preview = await loadHistory(previewUrl) assert.equal(preview!.histories!.length, 5) assert.deepEqual(preview!.runs.map(run => run.id), [runKey(pending)]) + assert.equal(preview!.runs[0]!.commit, pending.run.commit.sha.slice(0, 7)) assert.deepEqual((await loadHistory(previewUrl, runKey(first)))!.runs.map(run => run.id), [runKey(first)]) }) @@ -302,16 +337,12 @@ test('label-only eligibility with simulated GitHub responses', async context => number: 7, base: { ref: 'main', repo: { full_name: repository }, sha: 'b'.repeat(40) }, head: { sha: 'a'.repeat(40) }, labels: [] as { name: string }[], changed_files: 1, } - let files: { filename: string, previous_filename?: string }[] = [{ filename: 'README.md' }] - let permission = 'write' const requests: string[] = [] context.mock.method(globalThis, 'fetch', async (url: string) => { const path = new URL(url).pathname.replace(`/repos/${repository}/`, '') requests.push(path) const responses: Record = { - 'pulls/7': pr, 'pulls/7/files': files, - 'issues/7/events': [{ event: 'labeled', label: { name: 'benchmarks: skip' }, actor: { login: 'maintainer' } }], - 'collaborators/maintainer/permission': { permission }, + 'pulls/7': pr, } assert.ok(path in responses, `Unexpected GitHub request: ${path}`) return Response.json(responses[path]) @@ -321,26 +352,109 @@ test('label-only eligibility with simulated GitHub responses', async context => pr.labels = [{ name: 'benchmarks: skip' }] const skipped = (await eligibility(7)).decision assert.equal(skipped.mode, 'skip') - assert.equal(skipped.approver, 'maintainer') + assert.equal(skipped.approver, null) pr.labels = [] assert.equal((await eligibility(7)).decision.mode, 'required') - assert.equal(requests.some(path => path.includes('comments')), false) + assert.deepEqual(requests, ['pulls/7', 'pulls/7', 'pulls/7']) }) - await context.test('unauthorized labels and sensitive renames fail', async () => { + await context.test('skipping does not inspect files or the label actor', async () => { pr.labels = [{ name: 'benchmarks: skip' }] - permission = 'read' - await assert.rejects(eligibility(7), /maintainer/) - permission = 'write' - files = [{ filename: 'README.md', previous_filename: 'Cargo.toml' }] - await assert.rejects(eligibility(7), /benchmark-sensitive/) - }) - await context.test('truncated changed-file lists fail', async () => { - files = [{ filename: 'README.md' }] pr.changed_files = 2 - await assert.rejects(eligibility(7), /complete changed-file/) + assert.equal((await eligibility(7)).decision.mode, 'skip') + assert.ok(requests.every(path => path === 'pulls/7')) }) }) +test('workflows must match trusted main', async context => { + process.env.GITHUB_REPOSITORY = repository + process.env.GH_TOKEN = 'fixture-token' + const { workflowTrusted } = await import('./github-pr.ts') + const temporary = mkdtempSync(resolve(tmpdir(), 'benchmark-workflow-trust-')) + const previousDirectory = process.cwd() + context.after(() => { + process.chdir(previousDirectory) + rmSync(temporary, { recursive: true, force: true }) + }) + process.chdir(temporary) + mkdirSync('.github/workflows', { recursive: true }) + const head = 'a'.repeat(40) + let content = 'trusted workflow' + context.mock.method(globalThis, 'fetch', async (url: string) => { + const path = new URL(url).pathname.replace(`/repos/${repository}/`, '') + assert.ok(path.startsWith('contents/'), `Unexpected GitHub request: ${path}`) + return Response.json({ encoding: 'base64', content: Buffer.from(content).toString('base64') }) + }) + for (const workflow of ['benchmark', 'preview']) { + const path = `.github/workflows/${workflow}.yml` + writeFileSync(path, 'trusted workflow') + content = 'trusted workflow' + assert.equal(await workflowTrusted(path, head), true) + content = 'changed workflow' + assert.equal(await workflowTrusted(path, head), false) + } +}) + +test('Pages verifies previews with revision and build jobs and excludes draft and Dependabot PRs', async context => { + const temporary = mkdtempSync(resolve(tmpdir(), 'benchmark-preview-policy-')) + const previousDirectory = process.cwd() + const previousArguments = process.argv + const previousEnvironment = { GITHUB_REPOSITORY: process.env.GITHUB_REPOSITORY, GH_TOKEN: process.env.GH_TOKEN } + context.after(() => { + process.chdir(previousDirectory) + process.argv = previousArguments + for (const [key, value] of Object.entries(previousEnvironment)) { + if (value === undefined) delete process.env[key] + else process.env[key] = value + } + rmSync(temporary, { recursive: true, force: true }) + }) + process.chdir(temporary) + process.argv = [process.execPath, resolve(root, 'scripts/site-ci.ts'), 'verify'] + process.env.GITHUB_REPOSITORY = repository + process.env.GH_TOKEN = 'fixture-token' + const pr = { + number: 7, draft: false, user: { login: 'contributor' }, labels: [], + head: { sha: 'a'.repeat(40), repo: { full_name: repository } }, + base: { sha: 'b'.repeat(40), ref: 'main', repo: { full_name: repository } }, + } + const workflow = readFileSync(resolve(root, '.github/workflows/preview.yml'), 'utf8') + mkdirSync('.github/workflows', { recursive: true }) + writeFileSync('.github/workflows/preview.yml', workflow) + const run = { + id: 123, run_attempt: 1, event: 'pull_request', path: '.github/workflows/preview.yml', + conclusion: 'success', head_sha: pr.head.sha, head_repository: pr.head.repo, + pull_requests: [{ number: pr.number }], + } + const artifact = { + id: 456, name: `preview-${pr.number}-${pr.head.sha}-1`, expired: false, + size_in_bytes: 1024, workflow_run: { id: run.id }, + } + writeJson(resolve(temporary, 'site-state.json'), { + main: pr.base.sha, data: null, + open: [{ number: pr.number, head: pr.head.sha, base: pr.base.sha }], + previews: [{ number: pr.number, head: pr.head.sha, base: pr.base.sha, + runId: run.id, attempt: run.run_attempt, artifactId: artifact.id, includePending: true }], + }, false) + context.mock.method(globalThis, 'fetch', async (url: string) => { + const path = new URL(url).pathname.replace(`/repos/${repository}/`, '') + const responses: Record = { + 'git/ref/heads/main': { object: { sha: pr.base.sha } }, + 'git/matching-refs/heads/data': [], + pulls: [pr, { ...pr, number: 8, user: { login: 'dependabot[bot]' } }, { ...pr, number: 9, draft: true }], + 'pulls/7': pr, + 'actions/runs/123': run, + 'actions/artifacts/456': artifact, + 'actions/runs/123/attempts/1/jobs': { jobs: [ + { name: 'revision', conclusion: 'success' }, { name: 'build', conclusion: 'success' }, + ] }, + 'contents/.github/workflows/preview.yml': { encoding: 'base64', content: Buffer.from(workflow).toString('base64') }, + } + assert.ok(path in responses, `Unexpected GitHub request: ${path}`) + return Response.json(responses[path]) + }) + await import('./site-ci.ts') +}) + test('CI archival and promotion with simulated GitHub and Git', async context => { const temporary = mkdtempSync(resolve(tmpdir(), 'benchmark-ci-publication-')) const previousDirectory = process.cwd() @@ -354,13 +468,16 @@ test('CI archival and promotion with simulated GitHub and Git', async context => process.env.GITHUB_REPOSITORY = repository process.env.GH_TOKEN = 'fixture-token' process.env.GITHUB_EVENT_NAME = 'workflow_run' + process.env.GITHUB_RUN_ID = '999' + process.env.GITHUB_RUN_ATTEMPT = '2' const eventPath = resolve(temporary, 'event.json') process.env.GITHUB_EVENT_PATH = eventPath const notification = { workflow_run: { - id: 12345, run_attempt: 1, event: 'pull_request', conclusion: 'success', pull_requests: [{ number: 7 }], + id: 12345, run_attempt: 1, event: 'pull_request', conclusion: 'success', head_sha: 'a'.repeat(40), pull_requests: [{ number: 7 }], } } writeJson(eventPath, notification, false) const workflow = readFileSync(resolve(root, '.github/workflows/benchmark.yml'), 'utf8') + let workflowContent = workflow const bundle = fixture() const pr = { number: 7, base: { ref: 'main', repo: { full_name: repository }, sha: bundle.run.pullRequest!.base }, @@ -377,9 +494,19 @@ test('CI archival and promotion with simulated GitHub and Git', async context => ...Array.from({ length: 108 }, (_, index) => `measure (${index})`), ].map(name => ({ name, conclusion: 'success' })) let retryJobs: typeof jobs = [] - context.mock.method(globalThis, 'fetch', async (url: string) => { + const statuses: any[] = [] + context.mock.method(globalThis, 'fetch', async (url: string, options?: RequestInit) => { const parsed = new URL(url) const path = parsed.pathname.replace(`/repos/${repository}/`, '') + if (path.startsWith('statuses/')) { + assert.equal(path, `statuses/${pr.head.sha}`) + assert.equal(options?.method, 'POST') + const status = JSON.parse(String(options?.body)) + assert.equal(status.context, 'Benchmark Publication') + assert.equal(status.target_url, `https://github.com/${repository}/actions/runs/999/attempts/2`) + statuses.push(status) + return Response.json(status) + } if (path.endsWith('/jobs')) { const page = Number(parsed.searchParams.get('page')) const response = structuredClone(path.includes('/attempts/2/') ? retryJobs : jobs) @@ -390,14 +517,11 @@ test('CI archival and promotion with simulated GitHub and Git', async context => 'pulls/7': pr, [`commits/${pr.merge_commit_sha}/pulls`]: [pr], [`commits/${'d'.repeat(40)}/pulls`]: [pr], - 'pulls/7/files': [{ filename: 'README.md' }], - 'issues/7/events': [{ event: 'labeled', label: { name: 'benchmarks: skip' }, actor: { login: 'maintainer' } }], - 'collaborators/maintainer/permission': { permission: 'write' }, 'actions/runs/12345/attempts/1': { id: 12345, event: 'pull_request', conclusion: 'success', path: '.github/workflows/benchmark.yml', head_sha: workflowHead, created_at: bundle.run.createdAt }, 'actions/runs/12345/attempts/2': { id: 12345, event: 'pull_request', conclusion: 'success', path: '.github/workflows/benchmark.yml', head_sha: workflowHead, created_at: bundle.run.createdAt }, - 'contents/.github/workflows/benchmark.yml': { encoding: 'base64', content: Buffer.from(workflow).toString('base64') }, + 'contents/.github/workflows/benchmark.yml': { encoding: 'base64', content: Buffer.from(workflowContent).toString('base64') }, [`git/commits/${bundle.run.commit.sha}`]: { tree: { sha: bundle.run.commit.tree }, parents: [{ sha: bundle.run.pullRequest!.base }, { sha: bundle.run.pullRequest!.head }] }, - [`git/commits/${pr.merge_commit_sha}`]: { tree: { sha: mergedTree } }, + [`git/commits/${pr.merge_commit_sha}`]: { tree: { sha: mergedTree }, message: 'Benchmark results (#7)' }, } assert.ok(path in responses, `Unexpected GitHub request: ${path}`) return Response.json(responses[path]) @@ -429,6 +553,7 @@ test('CI archival and promotion with simulated GitHub and Git', async context => assert.equal(downloads, 1) assert.deepEqual(readJson(resolve(temporary, 'data-store/index.json')), { schemaVersion: 1, runs: [] }) assert.ok(existsSync(resolve(temporary, 'data-store/policies/12345/1.json'))) + assert.deepEqual(statuses.map(status => status.state), ['pending', 'success']) }) await context.test('main push promotes the archived candidate through commit association', async () => { pr.merged = true @@ -471,6 +596,7 @@ test('CI archival and promotion with simulated GitHub and Git', async context => workflowHead = pr.head.sha failedJob = true await assert.rejects(main(), /successful eligibility/) + assert.equal(statuses.at(-1).state, 'failure') failedJob = false assert.equal(downloads, 0) assert.equal(pushes, 0) @@ -512,6 +638,59 @@ test('CI archival and promotion with simulated GitHub and Git', async context => assert.equal(pushes, 0) pr.labels = [] }) + await context.test('cache-warming completions do not archive results', async () => { + clearStore() + writeJson(eventPath, { workflow_run: { ...notification.workflow_run, event: 'push' } }, false) + try { + await main() + assert.equal(downloads, 0) + assert.equal(pushes, 0) + assert.equal(existsSync(resolve(temporary, 'data-store')), false) + } finally { + writeJson(eventPath, notification, false) + } + }) + await context.test('stale notifications cannot update the current PR status', async () => { + const count = statuses.length + writeJson(eventPath, { workflow_run: { ...notification.workflow_run, head_sha: 'e'.repeat(40) } }, false) + try { + await assert.rejects(main(), /stale/) + assert.equal(statuses.length, count) + } finally { + writeJson(eventPath, notification, false) + } + }) + await context.test('changed workflows cannot archive before matching main', async () => { + clearStore() + workflowContent = `${workflow}\n` + pr.merged = false + pr.state = 'open' + try { + await assert.rejects(main(), /workflow must match current main/) + assert.equal(downloads, 0) + assert.equal(pushes, 0) + assert.equal(statuses.at(-1).state, 'failure') + } finally { + workflowContent = workflow + pr.merged = true + pr.state = 'closed' + clearStore() + } + }) + await context.test('merge with no archive reports failure and requires manual archival', async () => { + clearStore() + process.env.GITHUB_EVENT_NAME = 'push' + writeJson(eventPath, { ref: 'refs/heads/main', after: pr.merge_commit_sha, commits: [{ id: pr.merge_commit_sha }] }, false) + try { + await assert.rejects(main(), /pending results are missing/) + assert.equal(downloads, 0) + assert.equal(pushes, 0) + assert.equal(statuses.at(-1).state, 'failure') + } finally { + process.env.GITHUB_EVENT_NAME = 'workflow_run' + writeJson(eventPath, notification, false) + } + }) await context.test('promotion failure after merge still retains the valid archive remotely', async () => { clearStore() mergedTree = 'f'.repeat(40) diff --git a/scripts/publish-ci.ts b/scripts/publish-ci.ts index bafaeb4..88085d4 100644 --- a/scripts/publish-ci.ts +++ b/scripts/publish-ci.ts @@ -4,7 +4,7 @@ import { resolve } from 'node:path' import { pathToFileURL } from 'node:url' import { publicationPolicy } from '../shared/catalog.ts' import { historyRunPath, validatePublication, validatePublishableRun } from '../shared/results.ts' -import { eligibility, github, pages } from './github-pr.ts' +import { eligibility, github, pages, workflowTrusted } from './github-pr.ts' import { readJson, readOptionalJson, writeJson } from './result-store.ts' import { promoteRun, storeRun } from './publish-results.ts' @@ -44,9 +44,8 @@ async function verifiedRun(runId: number, attempt: number) { if (run.event !== 'pull_request' || run.conclusion !== 'success' || run.path !== '.github/workflows/benchmark.yml') { throw new Error('Expected a successful PR Benchmark workflow attempt') } - const workflow = await github(`contents/.github/workflows/benchmark.yml?ref=${run.head_sha}`) - if (workflow.encoding !== 'base64' || Buffer.from(workflow.content, 'base64').toString('utf8') !== readFileSync('.github/workflows/benchmark.yml', 'utf8')) { - throw new Error('Benchmark workflow differs from the trusted workflow. Maintainer review and a fresh run are required.') + if (!await workflowTrusted('.github/workflows/benchmark.yml', run.head_sha)) { + throw new Error('Benchmark workflow must match current main. After merge, rerun this publication workflow.') } const latestJobs = new Map() for (let currentAttempt = attempt; currentAttempt >= 1; currentAttempt--) { @@ -107,11 +106,30 @@ async function promote(number: number) { schemaVersion: 1, run: { id: bundle.run.id, attempt: bundle.run.attempt }, pullRequest: bundle.run.pullRequest, - merge: { sha: pr.merge_commit_sha, tree: merged.tree.sha, mergedAt: pr.merged_at }, + merge: { sha: pr.merge_commit_sha, tree: merged.tree.sha, mergedAt: pr.merged_at, message: merged.message }, }) promoteRun(directory, publication, readJson(policyPath)) } +export async function withPublicationStatus(head: string, operation: () => Promise) { + const report = (state: string, description: string) => github(`statuses/${head}`, { + state, context: 'Benchmark Publication', description, + target_url: `https://github.com/${repository}/actions/runs/${process.env.GITHUB_RUN_ID}/attempts/${process.env.GITHUB_RUN_ATTEMPT}`, + }) + await report('pending', 'Validating benchmark publication') + try { + await operation() + await report('success', 'Benchmark publication completed') + } catch (error) { + try { + await report('failure', 'Benchmark publication failed. See workflow logs.') + } catch (statusError) { + console.error('Could not report publication failure:', statusError) + } + throw error + } +} + export async function main() { const event = JSON.parse(readFileSync(process.env.GITHUB_EVENT_PATH!, 'utf8')) if (process.env.GITHUB_EVENT_NAME === 'push') { @@ -130,8 +148,11 @@ export async function main() { if (!numbers.size) return openStore() for (const number of numbers) { - await promote(number) - pushStore() + const pr = await github(`pulls/${number}`) + await withPublicationStatus(pr.head.sha, async () => { + await promote(number) + pushStore() + }) } return } @@ -139,6 +160,12 @@ export async function main() { if (notification.event !== 'pull_request' || notification.conclusion !== 'success') return if (notification.pull_requests.length !== 1) throw new Error('Workflow must identify one PR') const number = notification.pull_requests[0].number + const current = await github(`pulls/${number}`) + if (current.head.sha !== notification.head_sha) throw new Error('Workflow is stale. A fresh benchmark run is required.') + await withPublicationStatus(notification.head_sha, () => archive(notification, number)) +} + +async function archive(notification: any, number: number) { const { pr, decision } = await eligibility(number) if (decision.mode === 'skip') { console.log(`PR ${number}: approved skip. No results archived.`) @@ -155,7 +182,7 @@ export async function main() { await verifyCandidate(bundle, pr) validatePublication(bundle, { schemaVersion: 1, run: { id: bundle.run.id, attempt: bundle.run.attempt }, pullRequest: bundle.run.pullRequest, - merge: { sha: bundle.run.commit.sha, tree: bundle.run.commit.tree, mergedAt: bundle.run.createdAt }, + merge: { sha: bundle.run.commit.sha, tree: bundle.run.commit.tree, mergedAt: bundle.run.createdAt, message: bundle.run.commit.message }, }) openStore() storeRun(directory, bundle) diff --git a/scripts/site-ci.ts b/scripts/site-ci.ts index 945e880..b03e8dc 100644 --- a/scripts/site-ci.ts +++ b/scripts/site-ci.ts @@ -1,8 +1,8 @@ import { spawnSync } from 'node:child_process' -import { existsSync, mkdirSync, readFileSync, rmSync } from 'node:fs' +import { existsSync, mkdirSync, rmSync } from 'node:fs' import { resolve } from 'node:path' import { assembleSite, previewBuildSchema } from './assemble-site.ts' -import { eligibility, github, pages } from './github-pr.ts' +import { eligibility, github, pages, workflowTrusted } from './github-pr.ts' import { canonicalJson } from '../shared/results.ts' import { readJson, writeJson } from './result-store.ts' @@ -20,7 +20,7 @@ async function inputs() { const main = await github('git/ref/heads/main') const refs = await github('git/matching-refs/heads/data') const data = refs.find((entry: { ref: string }) => entry.ref === 'refs/heads/data')?.object.sha ?? null - const prs = (await pages('pulls?state=open&base=main')).filter(pr => !pr.draft) + const prs = (await pages('pulls?state=open&base=main')).filter(pr => !pr.draft && pr.user?.login !== 'dependabot[bot]') return { prs, snapshot: { @@ -39,11 +39,9 @@ async function verifiedBuild(run: any, pr: any, artifact: any): Promise || !run.pull_requests.some((entry: { number: number }) => entry.number === pr.number)) return false if (!artifact || artifact.expired || artifact.name !== `preview-${pr.number}-${pr.head.sha}-${run.run_attempt}`) return false if (artifact.size_in_bytes > 100 * 1024 ** 2) throw new Error(`Preview artifact is too large: ${artifact.name}`) - const trustedWorkflow = readFileSync('.github/workflows/preview.yml', 'utf8') - const workflow = await github(`contents/.github/workflows/preview.yml?ref=${run.head_sha}`) - if (workflow.encoding !== 'base64' || Buffer.from(workflow.content, 'base64').toString('utf8') !== trustedWorkflow) return false + if (!await workflowTrusted('.github/workflows/preview.yml', run.head_sha)) return false const jobs = await github(`actions/runs/${run.id}/attempts/${run.run_attempt}/jobs?per_page=100`) - for (const jobName of ['revision', `Approve preview PR ${pr.number} at ${pr.head.sha}`, 'build']) { + for (const jobName of ['revision', 'build']) { const matches = jobs.jobs.filter((job: { name: string }) => job.name === jobName) if (matches.length !== 1 || matches[0].conclusion !== 'success') return false } @@ -90,7 +88,7 @@ async function verify(desired: SiteState) { const run = await github(`actions/runs/${preview.runId}`) const artifact = await github(`actions/artifacts/${preview.artifactId}`) if (run.run_attempt !== preview.attempt || artifact.workflow_run?.id !== preview.runId - || !await verifiedBuild(run, pr, artifact)) throw new Error(`PR ${preview.number}: approved preview build changed. Rerun Pages.`) + || !await verifiedBuild(run, pr, artifact)) throw new Error(`PR ${preview.number}: preview build changed. Rerun Pages.`) const { decision } = await eligibility(preview.number) if (decision.head !== preview.head || decision.base !== preview.base || (decision.mode === 'required') !== preview.includePending) throw new Error('Preview eligibility changed. Rerun Pages.') diff --git a/shared/results.ts b/shared/results.ts index 2293e0e..45f726b 100644 --- a/shared/results.ts +++ b/shared/results.ts @@ -122,7 +122,10 @@ export const runKey = (bundle: RunBundle) => `${bundle.run.id}.${bundle.run.atte export const historyIndexSchema = z.object({ schemaVersion: z.literal(1), - runs: z.array(z.object({ id: identifier, attempt: positiveInteger })), + runs: z.array(z.object({ + id: identifier, attempt: positiveInteger, + displayCommit: z.object({ sha, message: text, url: httpUrl }).optional(), + })), preview: z.object({ number: positiveInteger, head: sha, @@ -144,7 +147,7 @@ export const publicationSchema = z.object({ schemaVersion: z.literal(1), run: historyIndexSchema.shape.runs.element, pullRequest: pullRequestSchema, - merge: z.object({ sha, tree: sha, mergedAt: timestamp }), + merge: z.object({ sha, tree: sha, mergedAt: timestamp, message: text }), }) export function validatePublication(bundle: RunBundle, input: unknown) { diff --git a/src/data.ts b/src/data.ts index 8242bd6..fd66a55 100644 --- a/src/data.ts +++ b/src/data.ts @@ -75,8 +75,17 @@ export async function loadHistory(indexUrl: URL, selectedHistory?: string): Prom const containing = (key: string | undefined) => groups.find(group => group.some(bundle => runKey(bundle) === key)) const pending = index.preview?.pending const selected = containing(selectedHistory) ?? containing(pending ? `${pending.id}.${pending.attempt}` : undefined) ?? groups[0]! + const dataset = datasetFromBundles(selected) + for (const run of dataset.runs) { + const display = index.runs.find(entry => `${entry.id}.${entry.attempt}` === run.id)?.displayCommit + if (display) { + run.commit = display.sha.slice(0, 7) + run.message = display.message + run.commitUrl = display.url + } + } return { - ...datasetFromBundles(selected), + ...dataset, preview: index.preview, historyId: runKey(selected[0]!), histories: groups.map((group, index) => { diff --git a/src/main.ts b/src/main.ts index 7941ef2..fa8f27d 100644 --- a/src/main.ts +++ b/src/main.ts @@ -99,7 +99,7 @@ function renderDashboard(data: Dataset) {
-
COMMIT SNAPSHOT

+
COMMIT SNAPSHOT

RankRuntimePlatform% of largest selected value
Runner specifications
PlatformRunnerOperating systemAzure VM SKURunner poolRegionCPUvCPUsCoresThreads/coreMemory (GiB)
@@ -155,6 +155,12 @@ function renderDashboard(data: Dataset) { const commitLink = element('#commit-link') commitLink.hidden = !run.commitUrl if (run.commitUrl) commitLink.href = run.commitUrl + const prLink = element('#pr-link') + const pr = run.bundle.run.pullRequest + prLink.hidden = !pr + if (pr) { + prLink.href = `https://github.com/${pr.repository}/pull/${pr.number}` + } const runners = run.runners.filter(runner => selectedPlatforms.has(runner.platformId)) element('#runner-details').innerHTML = runners.length ? runners.map(runner => { const cells = [data.platforms.find(platform => platform.id === runner.platformId)!.label, runner.name, [runner.os.name, runner.os.version, runner.os.architecture].filter(Boolean).join(' '), runner.sku, runner.pool, runner.region, runner.cpu.model, runner.cpu.logicalProcessors, runner.cpu.cores, runner.cpu.threadsPerCore, runner.memoryBytes / 1024 ** 3]