From cee35eb8a570f11fd31537080e6c18bd685d1f63 Mon Sep 17 00:00:00 2001 From: Jackson Ferguson Date: Tue, 6 Oct 2026 11:07:05 -0700 Subject: [PATCH] ci(metrics): let a full manual Nightly record the rollback count A manual run that covers the whole matrix is as complete as a scheduled one, so it publishes. One narrowed to an OS or template is incomplete and now ends quietly instead of failing; a scheduled run still fails loudly. --- .github/workflows/rollback-metrics.yml | 19 +++++++++++++++---- docs/developer/testing.md | 2 +- tests/test_nightly.py | 7 +++++-- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/.github/workflows/rollback-metrics.yml b/.github/workflows/rollback-metrics.yml index 3a42f079..4165459f 100644 --- a/.github/workflows/rollback-metrics.yml +++ b/.github/workflows/rollback-metrics.yml @@ -5,7 +5,8 @@ name: Rollback Metrics # its rollback jobs injected and records them on gh-pages beside the # mutation score: a history file for the dashboard and a badge endpoint the # README and jacksonferguson.me read. Only a run in which every rollback job -# reported publishes; manual runs, which may narrow the jobs, never do. +# reported publishes. A manual Nightly that covers the whole matrix records +# too; one narrowed to an OS or template is incomplete, so it records nothing. on: workflow_run: workflows: ["Nightly"] @@ -27,7 +28,7 @@ jobs: timeout-minutes: 10 if: >- github.event.workflow_run.head_branch == 'main' && - github.event.workflow_run.event == 'schedule' && + contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event.workflow_run.event) && contains(fromJSON('["success", "failure"]'), github.event.workflow_run.conclusion) permissions: actions: read @@ -57,6 +58,7 @@ jobs: id: record env: COMMIT: ${{ github.event.workflow_run.head_sha }} + EVENT: ${{ github.event.workflow_run.event }} run: | # A night with no new commits skips its rollback jobs: nothing to record. if [[ ! -d results ]] || [[ -z "$(ls -A results)" ]]; then @@ -70,10 +72,19 @@ jobs: git -C pages-data config user.email '41898282+github-actions[bot]@users.noreply.github.com' for attempt in 1 2 3 4 5; do echo "Publishing rollback results (attempt $attempt/5)." - python3 scripts/rollback_report.py record \ + if ! python3 scripts/rollback_report.py record \ --results results --matrix "$matrix" --commit "$COMMIT" --date "$recorded_at" \ --history pages-data/metrics/rollback-history.json \ - --latest pages-data/metrics/rollback-latest.json + --latest pages-data/metrics/rollback-latest.json; then + # A manual run may be narrowed to one OS or template on purpose. + # A scheduled one never is, so an incomplete set there is a failure. + if [[ "$EVENT" == workflow_dispatch ]]; then + echo "::notice::Not every rollback job reported, so this manual run records nothing." + echo "published=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + exit 1 + fi git -C pages-data add metrics/rollback-history.json metrics/rollback-latest.json if ! git -C pages-data diff --cached --quiet; then git -C pages-data commit -m "Record rollback faults for ${COMMIT:0:7}" diff --git a/docs/developer/testing.md b/docs/developer/testing.md index f3696f65..65597d5a 100644 --- a/docs/developer/testing.md +++ b/docs/developer/testing.md @@ -104,7 +104,7 @@ uv run pytest tests/test_rollback.py --rollback-scope full --rollback-real all - Every test that injects a fault carries the `rollback_fault` marker, and `--rollback-report PATH` writes how many of them passed, and which failed, per scenario (`tests/rollback_report.py`). A retry of failed tests updates the same file, so it holds each fault's last outcome, and a run that is killed or interrupted writes nothing. Nightly's rollback jobs pass the option and upload the file as an artifact, even when the job fails. -The `Rollback Metrics` workflow (`rollback-metrics.yml`) runs after each scheduled Nightly, apart from it so a publishing failure can never fail the run a release requires. It adds the reports up with `scripts/rollback_report.py` and writes two files under `metrics/` on `gh-pages`: `rollback-history.json`, one entry per fully green run with the count for each operating system and template, and `rollback-latest.json`, a Shields endpoint that reads `rollback faults restored` with the total. The badge stays cyan like the others, so a failure shows in its text, as `N failing`, and never in its color; only green runs reach the history. A run publishes only when every job in the rollback matrix reported: a missing, empty, or foreign report means the result is unknown, so nothing changes. Manual Nightly runs, which may narrow the matrix, never publish. The count adds every operating system, so the same fault restored on three platforms counts three times. The dashboard graphs the history, and the Pages workflow deploys both files, after which the site is asked to rebuild as it is for the mutation score. +The `Rollback Metrics` workflow (`rollback-metrics.yml`) runs after each scheduled Nightly, apart from it so a publishing failure can never fail the run a release requires. It adds the reports up with `scripts/rollback_report.py` and writes two files under `metrics/` on `gh-pages`: `rollback-history.json`, one entry per fully green run with the count for each operating system and template, and `rollback-latest.json`, a Shields endpoint that reads `rollback faults restored` with the total. The badge stays cyan like the others, so a failure shows in its text, as `N failing`, and never in its color; only green runs reach the history. A run publishes only when every job in the rollback matrix reported: a missing, empty, or foreign report means the result is unknown, so nothing changes. A manual Nightly that covers the whole matrix publishes too; one narrowed to an OS or template is incomplete and records nothing. The count adds every operating system, so the same fault restored on three platforms counts three times. The dashboard graphs the history, and the Pages workflow deploys both files, after which the site is asked to rebuild as it is for the mutation score. ### Template Hooks Smoke Matrix (CI) diff --git a/tests/test_nightly.py b/tests/test_nightly.py index 5d86b7a5..e6940261 100644 --- a/tests/test_nightly.py +++ b/tests/test_nightly.py @@ -291,14 +291,14 @@ def test_every_rollback_job_uploads_its_fault_report_even_when_it_fails(): assert upload["if"] == "${{ !cancelled() }}" -def test_rollback_metrics_are_recorded_apart_from_nightly_and_only_for_scheduled_runs(): +def test_rollback_metrics_are_recorded_apart_from_nightly_for_scheduled_and_manual_runs(): """A publishing failure must not fail the run a release requires.""" workflow = yaml_workflow("rollback-metrics.yml") trigger = workflow["on"]["workflow_run"] assert trigger["workflows"] == ["Nightly"] assert "rollback-metrics.yml" not in (WORKFLOWS / "nightly.yml").read_text() record = workflow["jobs"]["record"] - assert "workflow_run.event == 'schedule'" in record["if"] + assert '"schedule", "workflow_dispatch"' in record["if"] assert "head_branch == 'main'" in record["if"] assert record["permissions"]["contents"] == "write" (download,) = [ @@ -306,6 +306,9 @@ def test_rollback_metrics_are_recorded_apart_from_nightly_and_only_for_scheduled ] assert download["with"]["pattern"] == "rollback-*" assert download["with"]["run-id"] == "${{ github.event.workflow_run.id }}" + (step,) = [s for s in record["steps"] if s.get("id") == "record"] + # Only a manual run may end quietly when the matrix was narrowed. + assert 'EVENT" == workflow_dispatch' in step["run"] assert workflow["jobs"]["publish-pages"]["uses"] == "./.github/workflows/pages.yml" assert workflow["jobs"]["refresh-site"]["needs"] == "publish-pages"