feat(metrics): track rollback faults restored on the dashboard and a badge

.github/workflows/nightly.yml

@@ -140,5 +140,17 @@ jobs:
           tests/test_rollback.py --rollback-scope full --rollback-real all
           --rollback-templates ${{ matrix.template }} -o faulthandler_timeout=300
           --rollback-shard ${{ matrix.slice }}
+          --rollback-report rollback-report/report.json
         retry: "true"
         flaky-artifact: flaky-tests-rollback-${{ matrix.os }}-${{ matrix.template }}-${{ strategy.job-index }}
+
+    # Rollback Metrics adds the reports up. A failing job reports too, so the
+    # badge can say how many faults failed.
+    - name: Upload the fault report
+      if: ${{ !cancelled() }}
+      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+      with:
+        name: ${{ matrix.artifact }}
+        path: rollback-report/report.json
+        if-no-files-found: ignore
+        retention-days: 7

.github/workflows/rollback-metrics.yml

@@ -0,0 +1,127 @@
+name: Rollback Metrics
+
+# Kept apart from Nightly so a publishing hiccup can never fail the run a
+# release requires. When a scheduled Nightly finishes, this adds up the faults
+# its rollback jobs injected and records them on gh-pages beside the
+# mutation score: a history file for the dashboard and a badge endpoint the
+# README and jacksonferguson.me read. Only a run in which every rollback job
+# reported publishes; manual runs, which may narrow the jobs, never do.
+on:
+  workflow_run:
+    workflows: ["Nightly"]
+    types:
+      - completed
+
+permissions:
+  contents: read
+
+# Keep records in order without cancelling the work already in progress.
+concurrency:
+  group: rollback-metrics
+  cancel-in-progress: false
+
+jobs:
+  record:
+    name: Record the rollback run
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    if: >-
+      github.event.workflow_run.head_branch == 'main' &&
+      github.event.workflow_run.event == 'schedule' &&
+      contains(fromJSON('["success", "failure"]'), github.event.workflow_run.conclusion)
+    permissions:
+      actions: read
+      contents: write
+    outputs:
+      published: ${{ steps.record.outputs.published }}
+
+    steps:
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+
+    - name: Download every rollback report
+      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+      with:
+        pattern: rollback-*
+        path: results
+        run-id: ${{ github.event.workflow_run.id }}
+        github-token: ${{ github.token }}
+
+    - name: Read the published data branch
+      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+      with:
+        ref: gh-pages
+        path: pages-data
+        fetch-depth: 0
+
+    - name: Append history and the latest count
+      id: record
+      env:
+        COMMIT: ${{ github.event.workflow_run.head_sha }}
+      run: |
+        # A night with no new commits skips its rollback jobs: nothing to record.
+        if [[ ! -d results ]] || [[ -z "$(ls -A results)" ]]; then
+          echo "No rollback reports in this run."
+          echo "published=false" >> "$GITHUB_OUTPUT"
+          exit 0
+        fi
+        matrix="$(python3 scripts/nightly_matrix.py)"
+        recorded_at="$(date -u +%Y-%m-%dT%H:%M:%S+00:00)"
+        git -C pages-data config user.name 'github-actions[bot]'
+        git -C pages-data config user.email '41898282+github-actions[bot]@users.noreply.github.com'
+        for attempt in 1 2 3 4 5; do
+          echo "Publishing rollback results (attempt $attempt/5)."
+          python3 scripts/rollback_report.py record \
+            --results results --matrix "$matrix" --commit "$COMMIT" --date "$recorded_at" \
+            --history pages-data/metrics/rollback-history.json \
+            --latest pages-data/metrics/rollback-latest.json
+          git -C pages-data add metrics/rollback-history.json metrics/rollback-latest.json
+          if ! git -C pages-data diff --cached --quiet; then
+            git -C pages-data commit -m "Record rollback faults for ${COMMIT:0:7}"
+          fi
+          if git -C pages-data push origin HEAD:gh-pages; then
+            echo "published=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          # Benchmarks, mutation scores, and releases also write here. Rebase
+          # non-conflicting edits; if history itself changed, regenerate it.
+          if ! git -C pages-data pull --rebase origin gh-pages; then
+            git -C pages-data rebase --abort
+            git -C pages-data reset --hard origin/gh-pages
+          fi
+        done
+        echo 'Could not publish rollback results after five attempts.' >&2
+        exit 1
+
+  publish-pages:
+    name: Publish updated rollback metrics
+    needs: record
+    if: needs.record.outputs.published == 'true'
+    permissions:
+      contents: read
+      pages: write
+      id-token: write
+    uses: ./.github/workflows/pages.yml
+
+  # jacksonferguson.me shows the latest count and fetches it only when it builds.
+  # Wait for the deploy, not the gh-pages push, so the build reads the new file.
+  refresh-site:
+    name: Refresh jacksonferguson.me
+    needs: publish-pages
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions: {}
+    steps:
+    - name: Ask the site to rebuild
+      env:
+        # A fine-grained token that can only send this dispatch to the site's repository.
+        GH_TOKEN: ${{ secrets.PORTFOLIO_DISPATCH_TOKEN }}
+      run: |
+        if [[ -z "$GH_TOKEN" ]]; then
+          echo '::error::Add the PORTFOLIO_DISPATCH_TOKEN secret: a fine-grained token with Contents read and write on JacksonFergusonDev/JacksonFergusonDev.github.io only.'
+          exit 1
+        fi
+        gh api repos/JacksonFergusonDev/JacksonFergusonDev.github.io/dispatches \
+          --method POST \
+          --field event_type=remote-assets-updated \
+          --field "client_payload[asset_source]=$GITHUB_REPOSITORY" \
+          --field "client_payload[ref]=refs/heads/main"

AGENTS.md

@@ -29,7 +29,7 @@ Execution is strictly split into two decoupled phases:
 - **Bytes-Over-Intent Rollback:** Rollback restores exact original bytes and POSIX file modes captured before first mutation. Created directories are removed only if empty. Symlinks and special filesystem nodes are rejected before transaction-managed mutation (`UnsupportedFilesystemNodeError`).
 - **Preview Trees Show Every File a Run Leaves:** `EnvironmentManifest.planned_files()` feeds the dry-run tree, the recipe editor's preview, the change review, and the dry-run JSON `entries`. It adds command outputs (a task's `owned_files`, outside `.git/`), the resolver footprint, and `protostar.lock` to the files Protostar writes. Declare every file a command creates in its task's `owned_files`: `check-snapshots` fails when a scaffold's tree differs from its dry-run plan.
 - **Installed Git Hooks Are Clone-Local:** `.git/hooks` is not project state. `init` installs hooks as a post-install task; `sync` converges them through `git_hooks.plan_hooks()` (install when a wanted hook is missing, remove only runner-generated hooks that can now only fail) and never counts them as pending, so `sync --check` passes in a fresh checkout. A failed install on `sync` is a diagnostic, not a rollback.
-- **Every Fault Rolls Back:** `tests/test_rollback.py` fails `init` and `sync` before, mid-way through, and after every write, directory, removal, and command, with an error and an interrupt, and requires the project and home directory to come back byte for byte. The commit is a site: an interrupt after it keeps the run and claims no rollback. A restore that fails is named in `RollbackFailedError` while every other path still comes back. Its cases come from the site lists in `tests/rollback_sites/`: a change that adds or moves a site fails until they are regenerated with `--snapshot-update`, so review sees it, and a write made around `TransactionAwareFS` fails the clean-run check. Only `sync`'s hook install may fail without a rollback. Pull requests run two scenarios; nightly runs every template and seed.
+- **Every Fault Rolls Back:** `tests/test_rollback.py` fails `init` and `sync` before, mid-way through, and after every write, directory, removal, and command, with an error and an interrupt, and requires the project and home directory to come back byte for byte. The commit is a site: an interrupt after it keeps the run and claims no rollback. A restore that fails is named in `RollbackFailedError` while every other path still comes back. Its cases come from the site lists in `tests/rollback_sites/`: a change that adds or moves a site fails until they are regenerated with `--snapshot-update`, so review sees it, and a write made around `TransactionAwareFS` fails the clean-run check. Only `sync`'s hook install may fail without a rollback. Pull requests run two scenarios; nightly runs every template and seed. Each fault test carries the `rollback_fault` marker, and nightly's `--rollback-report` counts them for the metrics dashboard and badge: a count is published only when every job reported, and a failure shows in the badge's text, never its color.
 - **Rollback Boundary:** Rollback reliably covers paths written directly through Protostar's transaction-aware filesystem layer. Subprocess side effects are strictly disclaimed unless explicitly pre-declared by a module (e.g., `git init` declaring the `.git/` tree, or `pre-commit` declaring specific hook files). Unbounded or externally managed state (like `.venv/` or global caches) falls outside the transacted boundary.
 
 ### 2. The Headless Core

README.md

@@ -15,6 +15,7 @@
 [![Release](https://img.shields.io/github/actions/workflow/status/jacksonfergusondev/protostar/release.yml?color=22d3ee&labelColor=0A0A0A&label=release)](https://github.com/jacksonfergusondev/protostar/actions/workflows/release.yml)
 [![Codecov](https://img.shields.io/codecov/c/github/JacksonFergusonDev/protostar?color=22d3ee&labelColor=0A0A0A&logo=codecov&logoColor=white)](https://codecov.io/gh/JacksonFergusonDev/protostar)
 [![Engine mutation score](https://img.shields.io/endpoint?url=https%3A%2F%2Fprotostar.jacksonferguson.me%2Fmetrics%2Fmutation-latest.json&label=engine%20mutation%20score&color=22d3ee&labelColor=0A0A0A)](https://protostar.jacksonferguson.me/metrics/)
+[![Rollback faults restored](https://img.shields.io/endpoint?url=https%3A%2F%2Fprotostar.jacksonferguson.me%2Fmetrics%2Frollback-latest.json&label=rollback%20faults%20restored&color=22d3ee&labelColor=0A0A0A)](https://protostar.jacksonferguson.me/metrics/)
 [![Python](https://img.shields.io/badge/python-3.12+-22d3ee?labelColor=0A0A0A&logo=python&logoColor=white)](https://www.python.org/downloads/)
 [![Documentation](https://img.shields.io/badge/docs-gh--pages-22d3ee?labelColor=0A0A0A&logo=github&logoColor=white)](https://protostar.jacksonferguson.me/)
 [![License](https://img.shields.io/badge/license-MIT-22d3ee?labelColor=0A0A0A)](LICENSE)
@@ -208,7 +209,7 @@ Written for people who already know Python tooling:
 
 Protostar edits other people's work, so it's built to be careful:
 
-- **It plans before it acts.** Every change is worked out in memory and shown to you first. If a step fails or you press Ctrl+C, every file it wrote is restored exactly as it was.
+- **It plans before it acts.** Every change is worked out in memory and shown to you first. If a step fails or you press Ctrl+C, every file it wrote is restored exactly as it was, and a nightly run proves it by failing a real run at every step.
 - **The engine is separate from the interface.** The same core runs behind the interactive editor, the command line, and coding agents, which is why every command can also answer in JSON.
 - **It's tested thoroughly.** Over 3,500 tests, including runs of the real tools, pass on Linux, macOS, and Windows, with strict type checking and at least 85% coverage.
 - **Its docs are checked against its code.** Before every push, a script confirms that the commands, errors, exit codes, and file paths the docs mention still match the code.

docs/developer/testing.md

@@ -100,6 +100,12 @@ uv run pytest tests/test_rollback.py --rollback-scope full --rollback-real all -
 
 `just test-rollback` runs the full scope with faked commands. The harness skips `fsync`: rollback restores what is on disk, and durability through a power cut isn't under test.
 
+#### Counting the Faults
+
+Every test that injects a fault carries the `rollback_fault` marker, and `--rollback-report PATH` writes how many of them passed, and which failed, per scenario (`tests/rollback_report.py`). A retry of failed tests updates the same file, so it holds each fault's last outcome, and a run that is killed or interrupted writes nothing. Nightly's rollback jobs pass the option and upload the file as an artifact, even when the job fails.
+
+The `Rollback Metrics` workflow (`rollback-metrics.yml`) runs after each scheduled Nightly, apart from it so a publishing failure can never fail the run a release requires. It adds the reports up with `scripts/rollback_report.py` and writes two files under `metrics/` on `gh-pages`: `rollback-history.json`, one entry per fully green run with the count for each operating system and template, and `rollback-latest.json`, a Shields endpoint that reads `rollback faults restored` with the total. The badge stays cyan like the others, so a failure shows in its text, as `N failing`, and never in its color; only green runs reach the history. A run publishes only when every job in the rollback matrix reported: a missing, empty, or foreign report means the result is unknown, so nothing changes. Manual Nightly runs, which may narrow the matrix, never publish. The count adds every operating system, so the same fault restored on three platforms counts three times. The dashboard graphs the history, and the Pages workflow deploys both files, after which the site is asked to rebuild as it is for the mutation score.
+
 ### Template Hooks Smoke Matrix (CI)
 
 End-to-end template validation runs through the `template-smoke` action (`.github/actions/template-smoke`). It scaffolds each built-in template it is given in turn, verifies that `prek` hooks are installed, runs a canary check ensuring non-conventional commit messages are rejected, and asserts that the first commit triggers and cleanly passes all pre-commit hooks. A `template-hooks-smoke` job runs it for a list of templates, and a test job can run it after the suite through its matrix entry's `smoke` list, which is how macOS smoke-tests templates without a runner of its own.

metrics/index.html

@@ -6,7 +6,7 @@
   <meta name="viewport" content="width=device-width, initial-scale=1">
   <title>Protostar · Metrics</title>
   <meta name="description"
-    content="Startup performance and mutation scores for Protostar, recorded from CI.">
+    content="Startup performance, mutation scores, and rollback fault counts for Protostar, recorded from CI.">
   <link rel="icon" type="image/svg+xml" href="favicon.svg">
   <link rel="alternate icon" type="image/png" href="favicon.png">
   <link rel="stylesheet" href="house/css/fonts.css">
@@ -175,9 +175,57 @@ <h2 id="mutations-title">Can the tests catch a bug?</h2>
       </div>
     </section>
 
+    <section class="history-section" id="rollback" aria-labelledby="rollback-title">
+      <div class="page-width">
+        <div class="hs-section-marker"><span>04 / ROLLBACK</span></div>
+        <h2 id="rollback-title">Does a failed run leave nothing behind?</h2>
+        <p class="section-note">Every night, Protostar fails a real <code>init</code> or <code>sync</code> at each
+          write, directory, removal, and command of every built-in template, with an error and with an interrupt,
+          before, during, and after the step. A fault counts as restored when the project and home directory come back
+          byte for byte. The count covers every operating system the nightly run uses.</p>
+        <p class="section-note">The overall line adds every job&rsquo;s count, so it rises when a template, a step,
+          or a platform is added. Dashed vertical markers show a change in the set of jobs. Toggle operating
+          systems and templates in the legend to inspect them.</p>
+        <p id="rollback-status" class="status" role="status">Loading rollback history…</p>
+        <p id="rollback-failing" class="status" role="status"></p>
+        <div id="rollback-results" hidden>
+          <p id="rollback-latest" class="metric-measurement"></p>
+          <div class="chart-panel">
+            <div class="chart-header">
+              <span class="chart-caption">Faults restored / count</span>
+            </div>
+            <div id="chart-rollback" class="chart-canvas" role="img"
+              aria-label="Faults restored per nightly run, overall and by operating system and template. Exact values and changes of scope are in the tables below."></div>
+          </div>
+          <details class="measurements" open>
+            <summary>Latest faults by template</summary>
+            <div class="table-scroll" tabindex="0" role="region" aria-label="Latest rollback faults by template and operating system">
+              <table>
+                <caption>Faults restored in the latest complete run.</caption>
+                <thead><tr id="rollback-os"><th scope="col">Template</th></tr></thead>
+                <tbody id="rollback-cells"></tbody>
+              </table>
+            </div>
+          </details>
+          <details class="measurements">
+            <summary>Recent runs and changes of scope<span class="table-range">Last 30 runs</span></summary>
+            <div class="table-scroll" tabindex="0" role="region" aria-label="Recent rollback runs">
+              <table>
+                <caption>Dates are shown in your local timezone. Use the JSON history for every run.</caption>
+                <thead><tr><th scope="col">Recorded</th><th scope="col">Faults restored</th><th scope="col">Commit</th><th scope="col">Jobs</th></tr></thead>
+                <tbody id="rollback-runs"></tbody>
+              </table>
+            </div>
+          </details>
+        </div>
+        <a class="text-link guide-link" href="rollback-history.json">Download rollback history<span class="hs-icon hs-icon-arrow-down" aria-hidden="true"></span></a>
+        <noscript><p class="status">JavaScript is required to display rollback counts. Download the history to read the raw counts.</p></noscript>
+      </div>
+    </section>
+
     <section class="history-section methodology" id="methodology" aria-labelledby="methodology-title">
       <div class="page-width">
-        <div class="hs-section-marker"><span>04 / METHODOLOGY</span></div>
+        <div class="hs-section-marker"><span>05 / METHODOLOGY</span></div>
         <h2 id="methodology-title">Read the trend, with context.</h2>
         <div class="methodology-grid">
           <div>
@@ -225,6 +273,7 @@ <h3>What the comparisons mean</h3>
   <script src="data.js" defer></script>
   <script type="module" src="dashboard.js"></script>
   <script type="module" src="mutation-dashboard.js"></script>
+  <script type="module" src="rollback-dashboard.js"></script>
 </body>
 
 </html>