diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index b3c5383a..8da1dada 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -140,5 +140,17 @@ jobs: tests/test_rollback.py --rollback-scope full --rollback-real all --rollback-templates ${{ matrix.template }} -o faulthandler_timeout=300 --rollback-shard ${{ matrix.slice }} + --rollback-report rollback-report/report.json retry: "true" flaky-artifact: flaky-tests-rollback-${{ matrix.os }}-${{ matrix.template }}-${{ strategy.job-index }} + + # Rollback Metrics adds the reports up. A failing job reports too, so the + # badge can say how many faults failed. + - name: Upload the fault report + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ matrix.artifact }} + path: rollback-report/report.json + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/rollback-metrics.yml b/.github/workflows/rollback-metrics.yml new file mode 100644 index 00000000..3a42f079 --- /dev/null +++ b/.github/workflows/rollback-metrics.yml @@ -0,0 +1,127 @@ +name: Rollback Metrics + +# Kept apart from Nightly so a publishing hiccup can never fail the run a +# release requires. When a scheduled Nightly finishes, this adds up the faults +# its rollback jobs injected and records them on gh-pages beside the +# mutation score: a history file for the dashboard and a badge endpoint the +# README and jacksonferguson.me read. Only a run in which every rollback job +# reported publishes; manual runs, which may narrow the jobs, never do. +on: + workflow_run: + workflows: ["Nightly"] + types: + - completed + +permissions: + contents: read + +# Keep records in order without cancelling the work already in progress. +concurrency: + group: rollback-metrics + cancel-in-progress: false + +jobs: + record: + name: Record the rollback run + runs-on: ubuntu-latest + timeout-minutes: 10 + if: >- + github.event.workflow_run.head_branch == 'main' && + github.event.workflow_run.event == 'schedule' && + contains(fromJSON('["success", "failure"]'), github.event.workflow_run.conclusion) + permissions: + actions: read + contents: write + outputs: + published: ${{ steps.record.outputs.published }} + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Download every rollback report + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: rollback-* + path: results + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + + - name: Read the published data branch + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: gh-pages + path: pages-data + fetch-depth: 0 + + - name: Append history and the latest count + id: record + env: + COMMIT: ${{ github.event.workflow_run.head_sha }} + run: | + # A night with no new commits skips its rollback jobs: nothing to record. + if [[ ! -d results ]] || [[ -z "$(ls -A results)" ]]; then + echo "No rollback reports in this run." + echo "published=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + matrix="$(python3 scripts/nightly_matrix.py)" + recorded_at="$(date -u +%Y-%m-%dT%H:%M:%S+00:00)" + git -C pages-data config user.name 'github-actions[bot]' + git -C pages-data config user.email '41898282+github-actions[bot]@users.noreply.github.com' + for attempt in 1 2 3 4 5; do + echo "Publishing rollback results (attempt $attempt/5)." + python3 scripts/rollback_report.py record \ + --results results --matrix "$matrix" --commit "$COMMIT" --date "$recorded_at" \ + --history pages-data/metrics/rollback-history.json \ + --latest pages-data/metrics/rollback-latest.json + git -C pages-data add metrics/rollback-history.json metrics/rollback-latest.json + if ! git -C pages-data diff --cached --quiet; then + git -C pages-data commit -m "Record rollback faults for ${COMMIT:0:7}" + fi + if git -C pages-data push origin HEAD:gh-pages; then + echo "published=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + # Benchmarks, mutation scores, and releases also write here. Rebase + # non-conflicting edits; if history itself changed, regenerate it. + if ! git -C pages-data pull --rebase origin gh-pages; then + git -C pages-data rebase --abort + git -C pages-data reset --hard origin/gh-pages + fi + done + echo 'Could not publish rollback results after five attempts.' >&2 + exit 1 + + publish-pages: + name: Publish updated rollback metrics + needs: record + if: needs.record.outputs.published == 'true' + permissions: + contents: read + pages: write + id-token: write + uses: ./.github/workflows/pages.yml + + # jacksonferguson.me shows the latest count and fetches it only when it builds. + # Wait for the deploy, not the gh-pages push, so the build reads the new file. + refresh-site: + name: Refresh jacksonferguson.me + needs: publish-pages + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: {} + steps: + - name: Ask the site to rebuild + env: + # A fine-grained token that can only send this dispatch to the site's repository. + GH_TOKEN: ${{ secrets.PORTFOLIO_DISPATCH_TOKEN }} + run: | + if [[ -z "$GH_TOKEN" ]]; then + echo '::error::Add the PORTFOLIO_DISPATCH_TOKEN secret: a fine-grained token with Contents read and write on JacksonFergusonDev/JacksonFergusonDev.github.io only.' + exit 1 + fi + gh api repos/JacksonFergusonDev/JacksonFergusonDev.github.io/dispatches \ + --method POST \ + --field event_type=remote-assets-updated \ + --field "client_payload[asset_source]=$GITHUB_REPOSITORY" \ + --field "client_payload[ref]=refs/heads/main" diff --git a/AGENTS.md b/AGENTS.md index 07a96b63..3d91bbf1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,7 +29,7 @@ Execution is strictly split into two decoupled phases: - **Bytes-Over-Intent Rollback:** Rollback restores exact original bytes and POSIX file modes captured before first mutation. Created directories are removed only if empty. Symlinks and special filesystem nodes are rejected before transaction-managed mutation (`UnsupportedFilesystemNodeError`). - **Preview Trees Show Every File a Run Leaves:** `EnvironmentManifest.planned_files()` feeds the dry-run tree, the recipe editor's preview, the change review, and the dry-run JSON `entries`. It adds command outputs (a task's `owned_files`, outside `.git/`), the resolver footprint, and `protostar.lock` to the files Protostar writes. Declare every file a command creates in its task's `owned_files`: `check-snapshots` fails when a scaffold's tree differs from its dry-run plan. - **Installed Git Hooks Are Clone-Local:** `.git/hooks` is not project state. `init` installs hooks as a post-install task; `sync` converges them through `git_hooks.plan_hooks()` (install when a wanted hook is missing, remove only runner-generated hooks that can now only fail) and never counts them as pending, so `sync --check` passes in a fresh checkout. A failed install on `sync` is a diagnostic, not a rollback. -- **Every Fault Rolls Back:** `tests/test_rollback.py` fails `init` and `sync` before, mid-way through, and after every write, directory, removal, and command, with an error and an interrupt, and requires the project and home directory to come back byte for byte. The commit is a site: an interrupt after it keeps the run and claims no rollback. A restore that fails is named in `RollbackFailedError` while every other path still comes back. Its cases come from the site lists in `tests/rollback_sites/`: a change that adds or moves a site fails until they are regenerated with `--snapshot-update`, so review sees it, and a write made around `TransactionAwareFS` fails the clean-run check. Only `sync`'s hook install may fail without a rollback. Pull requests run two scenarios; nightly runs every template and seed. +- **Every Fault Rolls Back:** `tests/test_rollback.py` fails `init` and `sync` before, mid-way through, and after every write, directory, removal, and command, with an error and an interrupt, and requires the project and home directory to come back byte for byte. The commit is a site: an interrupt after it keeps the run and claims no rollback. A restore that fails is named in `RollbackFailedError` while every other path still comes back. Its cases come from the site lists in `tests/rollback_sites/`: a change that adds or moves a site fails until they are regenerated with `--snapshot-update`, so review sees it, and a write made around `TransactionAwareFS` fails the clean-run check. Only `sync`'s hook install may fail without a rollback. Pull requests run two scenarios; nightly runs every template and seed. Each fault test carries the `rollback_fault` marker, and nightly's `--rollback-report` counts them for the metrics dashboard and badge: a count is published only when every job reported, and a failure shows in the badge's text, never its color. - **Rollback Boundary:** Rollback reliably covers paths written directly through Protostar's transaction-aware filesystem layer. Subprocess side effects are strictly disclaimed unless explicitly pre-declared by a module (e.g., `git init` declaring the `.git/` tree, or `pre-commit` declaring specific hook files). Unbounded or externally managed state (like `.venv/` or global caches) falls outside the transacted boundary. ### 2. The Headless Core diff --git a/README.md b/README.md index 633180dd..b6c6c05d 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ [![Release](https://img.shields.io/github/actions/workflow/status/jacksonfergusondev/protostar/release.yml?color=22d3ee&labelColor=0A0A0A&label=release)](https://github.com/jacksonfergusondev/protostar/actions/workflows/release.yml) [![Codecov](https://img.shields.io/codecov/c/github/JacksonFergusonDev/protostar?color=22d3ee&labelColor=0A0A0A&logo=codecov&logoColor=white)](https://codecov.io/gh/JacksonFergusonDev/protostar) [![Engine mutation score](https://img.shields.io/endpoint?url=https%3A%2F%2Fprotostar.jacksonferguson.me%2Fmetrics%2Fmutation-latest.json&label=engine%20mutation%20score&color=22d3ee&labelColor=0A0A0A)](https://protostar.jacksonferguson.me/metrics/) +[![Rollback faults restored](https://img.shields.io/endpoint?url=https%3A%2F%2Fprotostar.jacksonferguson.me%2Fmetrics%2Frollback-latest.json&label=rollback%20faults%20restored&color=22d3ee&labelColor=0A0A0A)](https://protostar.jacksonferguson.me/metrics/) [![Python](https://img.shields.io/badge/python-3.12+-22d3ee?labelColor=0A0A0A&logo=python&logoColor=white)](https://www.python.org/downloads/) [![Documentation](https://img.shields.io/badge/docs-gh--pages-22d3ee?labelColor=0A0A0A&logo=github&logoColor=white)](https://protostar.jacksonferguson.me/) [![License](https://img.shields.io/badge/license-MIT-22d3ee?labelColor=0A0A0A)](LICENSE) @@ -208,7 +209,7 @@ Written for people who already know Python tooling: Protostar edits other people's work, so it's built to be careful: -- **It plans before it acts.** Every change is worked out in memory and shown to you first. If a step fails or you press Ctrl+C, every file it wrote is restored exactly as it was. +- **It plans before it acts.** Every change is worked out in memory and shown to you first. If a step fails or you press Ctrl+C, every file it wrote is restored exactly as it was, and a nightly run proves it by failing a real run at every step. - **The engine is separate from the interface.** The same core runs behind the interactive editor, the command line, and coding agents, which is why every command can also answer in JSON. - **It's tested thoroughly.** Over 3,500 tests, including runs of the real tools, pass on Linux, macOS, and Windows, with strict type checking and at least 85% coverage. - **Its docs are checked against its code.** Before every push, a script confirms that the commands, errors, exit codes, and file paths the docs mention still match the code. diff --git a/docs/developer/testing.md b/docs/developer/testing.md index e294ea07..f3696f65 100644 --- a/docs/developer/testing.md +++ b/docs/developer/testing.md @@ -100,6 +100,12 @@ uv run pytest tests/test_rollback.py --rollback-scope full --rollback-real all - `just test-rollback` runs the full scope with faked commands. The harness skips `fsync`: rollback restores what is on disk, and durability through a power cut isn't under test. +#### Counting the Faults + +Every test that injects a fault carries the `rollback_fault` marker, and `--rollback-report PATH` writes how many of them passed, and which failed, per scenario (`tests/rollback_report.py`). A retry of failed tests updates the same file, so it holds each fault's last outcome, and a run that is killed or interrupted writes nothing. Nightly's rollback jobs pass the option and upload the file as an artifact, even when the job fails. + +The `Rollback Metrics` workflow (`rollback-metrics.yml`) runs after each scheduled Nightly, apart from it so a publishing failure can never fail the run a release requires. It adds the reports up with `scripts/rollback_report.py` and writes two files under `metrics/` on `gh-pages`: `rollback-history.json`, one entry per fully green run with the count for each operating system and template, and `rollback-latest.json`, a Shields endpoint that reads `rollback faults restored` with the total. The badge stays cyan like the others, so a failure shows in its text, as `N failing`, and never in its color; only green runs reach the history. A run publishes only when every job in the rollback matrix reported: a missing, empty, or foreign report means the result is unknown, so nothing changes. Manual Nightly runs, which may narrow the matrix, never publish. The count adds every operating system, so the same fault restored on three platforms counts three times. The dashboard graphs the history, and the Pages workflow deploys both files, after which the site is asked to rebuild as it is for the mutation score. + ### Template Hooks Smoke Matrix (CI) End-to-end template validation runs through the `template-smoke` action (`.github/actions/template-smoke`). It scaffolds each built-in template it is given in turn, verifies that `prek` hooks are installed, runs a canary check ensuring non-conventional commit messages are rejected, and asserts that the first commit triggers and cleanly passes all pre-commit hooks. A `template-hooks-smoke` job runs it for a list of templates, and a test job can run it after the suite through its matrix entry's `smoke` list, which is how macOS smoke-tests templates without a runner of its own. diff --git a/metrics/index.html b/metrics/index.html index 9ef85de8..f280b262 100644 --- a/metrics/index.html +++ b/metrics/index.html @@ -6,7 +6,7 @@ Protostar · Metrics + content="Startup performance, mutation scores, and rollback fault counts for Protostar, recorded from CI."> @@ -175,9 +175,57 @@

Can the tests catch a bug?

+
+
+
04 / ROLLBACK
+

Does a failed run leave nothing behind?

+

Every night, Protostar fails a real init or sync at each + write, directory, removal, and command of every built-in template, with an error and with an interrupt, + before, during, and after the step. A fault counts as restored when the project and home directory come back + byte for byte. The count covers every operating system the nightly run uses.

+

The overall line adds every job’s count, so it rises when a template, a step, + or a platform is added. Dashed vertical markers show a change in the set of jobs. Toggle operating + systems and templates in the legend to inspect them.

+

Loading rollback history…

+

+ + Download rollback history + +
+
+
-
04 / METHODOLOGY
+
05 / METHODOLOGY

Read the trend, with context.

@@ -225,6 +273,7 @@

What the comparisons mean

+ diff --git a/metrics/rollback-dashboard.js b/metrics/rollback-dashboard.js new file mode 100644 index 00000000..c11f1288 --- /dev/null +++ b/metrics/rollback-dashboard.js @@ -0,0 +1,139 @@ +import { rollbackHistory, rollbackSeries } from './rollbacks.mjs'; +import { escapeHtml } from './metrics.mjs'; +import { loadCharts } from './charts.mjs'; + +const status = document.getElementById('rollback-status'); +const count = value => typeof value !== 'number' ? 'N/A' : value.toLocaleString('en-US'); +const scope = entry => entry.scopeChanged + ? `Scope changed: ${[...entry.added.map(name => `added ${name}`), ...entry.removed.map(name => `removed ${name}`)].join('; ')}` + : `${entry.cells.length} jobs`; + +(async function() { + const response = await fetch('rollback-history.json'); + if (response.status === 404) { + status.textContent = 'No complete rollback runs published yet.'; + return; + } + if (!response.ok) throw new Error('History could not load'); + const history = rollbackHistory(await response.json()); + if (!history.length) { + status.textContent = 'No complete rollback runs published yet.'; + return; + } + const latest = history.at(-1); + document.getElementById('rollback-latest').textContent = + `${count(latest.value)} faults restored · ${latest.shortSha} · ${new Date(latest.date).toLocaleString()}`; + + // The history keeps only green runs; the badge file says when the latest one failed. + try { + const badge = await fetch('rollback-latest.json'); + const message = badge.ok ? (await badge.json()).message : ''; + if (typeof message === 'string' && message.endsWith(' failing')) { + document.getElementById('rollback-failing').textContent = + `The most recent run had ${message}. The history below holds only runs in which every fault was restored.`; + } + } catch { /* The history alone is still true. */ } + + const systems = [...new Set(latest.cells.map(cell => cell.os))].sort(); + const templates = [...new Set(latest.cells.map(cell => cell.template))].sort(); + const header = document.getElementById('rollback-os'); + for (const system of systems) { + const cell = document.createElement('th'); + cell.scope = 'col'; + cell.textContent = system; + header.appendChild(cell); + } + const breakdown = document.getElementById('rollback-cells'); + for (const template of templates) { + const row = breakdown.insertRow(); + row.insertCell().textContent = template; + for (const system of systems) { + const cell = latest.cells.find(item => item.os === system && item.template === template); + row.insertCell().textContent = count(cell?.passed); + } + } + const runs = document.getElementById('rollback-runs'); + for (const entry of history.slice(-30).reverse()) { + const row = runs.insertRow(); + row.insertCell().textContent = new Date(entry.date).toLocaleString(); + row.insertCell().textContent = count(entry.value); + const link = document.createElement('a'); + link.href = entry.url; + link.textContent = entry.shortSha; + const icon = document.createElement('span'); + icon.className = 'hs-icon hs-icon-brand-github'; + icon.setAttribute('aria-hidden', 'true'); + link.prepend(icon); + row.insertCell().appendChild(link); + row.insertCell().textContent = scope(entry); + } + document.getElementById('rollback-results').hidden = false; + status.textContent = ''; + + try { + await loadCharts(); + await document.fonts.ready; + const tokens = getComputedStyle(document.documentElement); + const token = name => tokens.getPropertyValue(name).trim(); + const text = token('--text'), muted = token('--muted'), line = token('--line'); + const accent = token('--accent'), mono = token('--mono'); + const series = rollbackSeries(history); + series[0].itemStyle = { color: accent }; + series[0].markLine = { + symbol: 'none', silent: true, + lineStyle: { color: muted, type: 'dashed' }, + label: { formatter: 'Scope changed', color: muted, fontFamily: mono }, + data: history.flatMap((entry, index) => entry.scopeChanged ? [{ xAxis: index }] : []), + }; + const chart = echarts.init(document.getElementById('chart-rollback'), null, { renderer: 'canvas' }); + // Blend the house palette's hex tokens into canvas-compatible RGB colors. + const colors = series.slice(1).map((_, index) => { + const weight = (35 + (index % 5) * 13) / 100; + const channels = [1, 3, 5].map(offset => Math.round( + parseInt(accent.slice(offset, offset + 2), 16) * weight + + parseInt(text.slice(offset, offset + 2), 16) * (1 - weight))); + return `rgb(${channels.join(', ')})`; + }); + chart.setOption({ + animation: false, backgroundColor: 'transparent', + color: [accent, ...colors], + textStyle: { color: text, fontFamily: mono }, + legend: { type: 'scroll', top: 0, textStyle: { color: muted, fontSize: parseFloat(token('--fs-label')) }, + selected: Object.fromEntries(series.map(item => [item.name, item.name === 'Overall'])) }, + grid: { top: 55, right: 25, bottom: 65, left: 75 }, + tooltip: { + trigger: 'axis', confine: true, backgroundColor: token('--panel'), borderColor: line, + textStyle: { color: text, fontSize: parseFloat(token('--fs-ui')) }, + formatter(params) { + const entry = history[params[0]?.dataIndex]; + if (!entry) return ''; + return `
${escapeHtml(entry.shortSha)} · ${escapeHtml(new Date(entry.date).toLocaleString())}
` + + params.map(item => `${escapeHtml(item.seriesName)}: ${count(item.value)}`).join('
') + + `
${escapeHtml(scope(entry))}
`; + }, + }, + xAxis: { type: 'category', data: history.map(entry => entry.shortSha), + axisLine: { lineStyle: { color: line } }, axisTick: { show: false }, + axisLabel: { color: muted, fontFamily: mono } }, + yAxis: { type: 'value', min: 0, + axisLabel: { color: muted, fontFamily: mono }, + splitLine: { lineStyle: { color: line, type: 'dashed' } } }, + dataZoom: [ + { type: 'slider', bottom: 8, height: 24, startValue: Math.max(0, history.length - 100), + endValue: history.length - 1, borderColor: 'transparent', backgroundColor: token('--panel'), + fillerColor: line, handleStyle: { color: accent }, textStyle: { color: muted } }, + { type: 'inside', startValue: Math.max(0, history.length - 100), endValue: history.length - 1 }, + ], + series, + }); + chart.on('click', params => { + const entry = history[params.dataIndex]; + if (entry) window.open(entry.url, '_blank', 'noopener,noreferrer'); + }); + window.addEventListener('resize', () => chart.resize()); + } catch { + status.textContent = 'The rollback chart could not load. Counts and commit links remain available below.'; + } +})().catch(() => { + status.textContent = 'Rollback history could not load. Please try again later.'; +}); diff --git a/metrics/rollbacks.mjs b/metrics/rollbacks.mjs new file mode 100644 index 00000000..18ccc33a --- /dev/null +++ b/metrics/rollbacks.mjs @@ -0,0 +1,48 @@ +// A fault is one injected failure the project was restored from. Counts are +// kept per operating system and template, so the total stays correct when +// either set grows. +const cellKey = cell => `${cell.os}/${cell.template}`; + +export function rollbackTotal(cells) { + return cells.reduce((sum, cell) => sum + cell.passed, 0); +} + +export function rollbackHistory(entries) { + if (!Array.isArray(entries)) throw new Error('Rollback history must be an array'); + const history = entries.map(entry => { + if (!entry || !/^[a-f0-9]{40}$/i.test(entry.commit) || + typeof entry.date !== 'string' || !Number.isFinite(Date.parse(entry.date)) || + !Array.isArray(entry.cells) || !entry.cells.length || + entry.cells.some(cell => !cell || typeof cell.os !== 'string' || !cell.os.trim() || + typeof cell.template !== 'string' || !cell.template.trim() || + !Number.isSafeInteger(cell.passed) || cell.passed < 0) || + new Set(entry.cells.map(cellKey)).size !== entry.cells.length) { + throw new Error('Invalid rollback history entry'); + } + const cells = [...entry.cells].sort((a, b) => cellKey(a).localeCompare(cellKey(b))); + return { ...entry, cells, value: rollbackTotal(cells), shortSha: entry.commit.slice(0, 7), + url: `https://github.com/JacksonFergusonDev/protostar/commit/${entry.commit}` }; + }).sort((a, b) => Date.parse(a.date) - Date.parse(b.date)); + return history.map((entry, index) => { + const previous = history[index - 1]?.cells.map(cellKey) || []; + const current = entry.cells.map(cellKey); + return { ...entry, scopeChanged: index > 0 && JSON.stringify(previous) !== JSON.stringify(current), + added: current.filter(name => !previous.includes(name)), + removed: previous.filter(name => !current.includes(name)) }; + }); +} + +// The overall total, then each operating system and each template summed over the rest. +export function rollbackSeries(history) { + const total = (entry, field, name) => { + const rows = entry.cells.filter(cell => cell[field] === name); + return rows.length ? rollbackTotal(rows) : null; + }; + const names = field => [...new Set(history.flatMap(entry => entry.cells.map(cell => cell[field])))].sort(); + return [ + { name: 'Overall', type: 'line', data: history.map(entry => entry.value), lineStyle: { width: 3 } }, + ...['os', 'template'].flatMap(field => names(field).map(name => ({ + name, type: 'line', lineStyle: { width: 1, type: 'dashed' }, + data: history.map(entry => total(entry, field, name)) }))), + ].map(series => ({ ...series, connectNulls: false, smooth: false, symbolSize: 6 })); +} diff --git a/pyproject.toml b/pyproject.toml index 12f83512..2e18d2b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -169,7 +169,10 @@ addopts = "--strict-markers" testpaths = ["tests"] norecursedirs = ["snapshots", ".*", "build", "dist", "venv"] pythonpath = ["."] -markers = ["integration: runs real subprocesses in an isolated workspace"] +markers = [ + "integration: runs real subprocesses in an isolated workspace", + "rollback_fault: injects one fault into a run and requires rollback to restore the project", +] [tool.coverage.run] source = ["src/protostar"] diff --git a/scripts/nightly_matrix.py b/scripts/nightly_matrix.py index 69216568..1dcd8c0d 100644 --- a/scripts/nightly_matrix.py +++ b/scripts/nightly_matrix.py @@ -18,7 +18,10 @@ def rollback_matrix(os: str = "all", template: str = "all") -> dict[str, Any]: - """Returns the rollback jobs, each with its OS, template, and slice. + """Returns the rollback jobs, each with its OS, template, slice, and artifact. + + The artifact names the job's fault report, so the publisher can tell which + jobs reported. Args: os: One operating system, or ``all``. @@ -28,7 +31,12 @@ def rollback_matrix(os: str = "all", template: str = "all") -> dict[str, Any]: A ``strategy.matrix`` holding one ``include`` entry per job. """ jobs = [ - {"os": system, "template": name, "slice": f"{part}/{slices}"} + { + "os": system, + "template": name, + "slice": f"{part}/{slices}", + "artifact": f"rollback-{system}-{name}-{part}of{slices}", + } for system in OPERATING_SYSTEMS if os in ("all", system) for name in TEMPLATES diff --git a/scripts/prepare_pages.py b/scripts/prepare_pages.py index f18b48e3..782b5462 100644 --- a/scripts/prepare_pages.py +++ b/scripts/prepare_pages.py @@ -29,6 +29,8 @@ "charts.mjs", "mutations.mjs", "mutation-dashboard.js", + "rollbacks.mjs", + "rollback-dashboard.js", ) HOUSE_DIR = _repo_root / "docs" / "house" HEADER_CSS = _repo_root / "docs" / "stylesheets" / "site-header.css" @@ -264,7 +266,13 @@ def assemble_pages(source: Path, output: Path, config_path: Path) -> str: _write_redirects(tree, output / alias, target) metrics_files = [ source / "metrics" / filename - for filename in ("data.js", "mutation-history.json", "mutation-latest.json") + for filename in ( + "data.js", + "mutation-history.json", + "mutation-latest.json", + "rollback-history.json", + "rollback-latest.json", + ) if (source / "metrics" / filename).is_file() ] if metrics_files: diff --git a/scripts/rollback_report.py b/scripts/rollback_report.py new file mode 100644 index 00000000..bdffc4fa --- /dev/null +++ b/scripts/rollback_report.py @@ -0,0 +1,153 @@ +"""Records how many faults the nightly rollback jobs injected and restored. + +Each rollback job writes the faults it raised and how many the project +survived (``tests/rollback_report.py``). Once every job has reported, this +adds them up. Zero unrestored faults add a point to the history; any failure +only changes the badge, so the history stays a record of green runs. + +Run: + python3 scripts/rollback_report.py record --results DIR --matrix JSON \\ + --history PATH --latest PATH --commit SHA +""" + +from __future__ import annotations + +import argparse +import json +import re +from datetime import UTC, datetime +from pathlib import Path +from typing import TypedDict + +# Runs without the project installed (the workflow's publish job), so it +# depends on the standard library alone. +LABEL = "rollback faults restored" + + +class HistoryEntry(TypedDict): + """One fully green run, keeping the counts per operating system and template.""" + + commit: str + date: str + cells: list[dict[str, object]] + + +class RollbackReportError(Exception): + """The results are incomplete or inconsistent, so nothing is published.""" + + +def read_history(path: Path) -> list[HistoryEntry]: + """Read recorded runs, treating a missing history as the first run.""" + return json.loads(path.read_text(encoding="utf-8")) if path.exists() else [] + + +def combine( + results: Path, matrix: list[dict[str, str]] +) -> tuple[list[dict[str, object]], int]: + """Adds every job's counts into one row per operating system and template. + + Args: + results: A directory holding one folder per job, named by its artifact, + each with the ``report.json`` the job wrote. + matrix: The ``include`` entries of the rollback matrix. + + Returns: + The rows (``os``, ``template``, ``passed``), and how many faults failed. + + Raises: + RollbackReportError: A planned job has no report, an unplanned one has, + or a report holds no faults or another template's. + """ + expected = {entry["artifact"] for entry in matrix} + actual = {path.name for path in results.iterdir() if path.is_dir()} + if not expected or actual != expected: + raise RollbackReportError( + f"Expected artifacts {sorted(expected)}, got {sorted(actual)}." + ) + totals: dict[tuple[str, str], int] = {} + failed = 0 + for entry in matrix: + path = results / entry["artifact"] / "report.json" + if not path.is_file(): + raise RollbackReportError(f"Missing rollback report: {path}.") + scenarios = json.loads(path.read_text(encoding="utf-8"))["scenarios"] + if not scenarios: + raise RollbackReportError(f"No faults reported: {path}.") + if {row["template"] for row in scenarios.values()} != {entry["template"]}: + raise RollbackReportError(f"Faults of another template in {path}.") + key = (entry["os"].removesuffix("-latest"), entry["template"]) + totals[key] = totals.get(key, 0) + sum( + row["passed"] for row in scenarios.values() + ) + failed += sum(len(row["failed"]) for row in scenarios.values()) + rows = [ + {"os": system, "template": template, "passed": passed} + for (system, template), passed in sorted(totals.items()) + ] + return rows, failed + + +def badge(passed: int, failed: int) -> dict[str, object]: + """Returns the shields.io endpoint payload: the count, or how many failed.""" + return { + "schemaVersion": 1, + "label": LABEL, + "message": f"{failed:,} failing" if failed else f"{passed:,}", + "color": "22d3ee", + "labelColor": "0A0A0A", + } + + +def record(args: argparse.Namespace) -> None: + """Writes the badge for a complete run, and its history point if none failed.""" + if not re.fullmatch(r"[0-9a-f]{40}", args.commit): + raise SystemExit("--commit must be a full Git commit hash.") + date = datetime.fromisoformat(args.date) + offset = date.utcoffset() + if offset is None or offset.total_seconds() != 0: + raise SystemExit("--date must be a UTC timestamp.") + try: + rows, failed = combine(args.results, json.loads(args.matrix)["include"]) + except RollbackReportError as error: + raise SystemExit(str(error)) from error + passed = sum(row["passed"] for row in rows) # type: ignore[misc] + entries = read_history(args.history) + # A retry of a recorded commit neither appends twice nor replaces a newer run. + if not failed and not any(item["commit"] == args.commit for item in entries): + if entries and datetime.fromisoformat(entries[-1]["date"]) > date: + raise SystemExit( + "Refusing to publish a run older than the latest recorded run." + ) + entries.append({"commit": args.commit, "date": args.date, "cells": rows}) + for path, payload in ( + (args.history, entries), + (args.latest, badge(passed, failed)), + ): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + print(f"{passed:,} faults restored, {failed:,} failing.") + + +def parse_args() -> argparse.Namespace: + """Parses command-line arguments.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + commands = parser.add_subparsers(dest="command", required=True) + command = commands.add_parser("record", help="Add up a complete run.") + command.add_argument("--results", type=Path, required=True) + command.add_argument("--matrix", required=True) + command.add_argument("--history", type=Path, required=True) + command.add_argument("--latest", type=Path, required=True) + command.add_argument("--commit", required=True) + command.add_argument("--date", default=datetime.now(UTC).isoformat()) + command.set_defaults(func=record) + return parser.parse_args() + + +def main() -> None: + """Runs the selected command.""" + args = parse_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/tests/conftest.py b/tests/conftest.py index 02c1ebd9..ae0641ec 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -54,6 +54,23 @@ def pytest_addoption(parser: pytest.Parser) -> None: "split them across jobs." ), ) + group.addoption( + "--rollback-report", + default="", + metavar="PATH", + help=( + "Write how many faults the run injected and restored, per " + "scenario, to this JSON file (nightly publishes it)." + ), + ) + + +def pytest_configure(config: pytest.Config) -> None: + """Reports the faults a rollback run injected, when asked to.""" + if report := config.getoption("--rollback-report"): + from tests.rollback_report import RollbackReport + + config.pluginmanager.register(RollbackReport(Path(report)), "rollback-report") def pytest_collection_modifyitems( diff --git a/tests/metrics_dashboard.test.mjs b/tests/metrics_dashboard.test.mjs index 6830e101..3f2f1254 100644 --- a/tests/metrics_dashboard.test.mjs +++ b/tests/metrics_dashboard.test.mjs @@ -238,3 +238,143 @@ test('a chart-library failure preserves mutation scores and the accessible table assert.equal(element('mutation-modules').rows[0][1].textContent, '100.0%'); assert.match(element('mutation-status').textContent, /Scores and commit links remain available/); }); + +// Rollback history is another raw-count dataset, kept per operating system and template. +const { rollbackTotal, rollbackHistory, rollbackSeries } = await import('../metrics/rollbacks.mjs'); +const rollbackCell = (os, template, passed) => ({ os, template, passed }); +const rollbackRun = (date, cells) => ({ commit: 'c'.repeat(40), date, cells }); + +test('rollback totals add every job and the series split them by system and template', () => { + const history = rollbackHistory([rollbackRun('2026-10-01T00:00:00Z', [ + rollbackCell('ubuntu', 'cli', 10), rollbackCell('ubuntu', 'lib', 5), + rollbackCell('windows', 'cli', 10)])]); + assert.equal(history[0].value, 25); + assert.equal(rollbackTotal([]), 0); + const series = rollbackSeries(history); + assert.deepEqual(series.map(line => line.name), ['Overall', 'ubuntu', 'windows', 'cli', 'lib']); + assert.deepEqual(series.map(line => line.data[0]), [25, 15, 10, 20, 5]); +}); + +test('rollback scope changes follow the set of jobs, not the counts', () => { + const history = rollbackHistory([ + rollbackRun('2026-10-03T00:00:00Z', [rollbackCell('ubuntu', 'cli', 12), rollbackCell('ubuntu', 'lib', 1)]), + rollbackRun('2026-10-01T00:00:00Z', [rollbackCell('ubuntu', 'cli', 10)]), + rollbackRun('2026-10-02T00:00:00Z', [rollbackCell('ubuntu', 'cli', 11)])]); + assert.deepEqual(history.map(entry => entry.scopeChanged), [false, false, true]); + assert.deepEqual(history[2].added, ['ubuntu/lib']); + assert.deepEqual(history[2].removed, []); + const series = rollbackSeries(history); + assert.deepEqual(series.find(line => line.name === 'lib').data, [null, null, 1]); + assert.equal(series[0].connectNulls, false); + assert.deepEqual(rollbackHistory([]), []); +}); + +test('rollback data rejects malformed dates, unsafe commits, duplicate jobs, and invalid counts', () => { + const valid = rollbackRun('2026-10-01T00:00:00Z', [rollbackCell('ubuntu', 'cli', 1)]); + for (const entry of [ + { ...valid, commit: 'javascript:alert(1)' }, { ...valid, date: 'invalid' }, + { ...valid, cells: [] }, { ...valid, cells: [rollbackCell('ubuntu', 'cli', -1)] }, + { ...valid, cells: [rollbackCell('ubuntu', 'cli', 1.5)] }, + { ...valid, cells: [rollbackCell('', 'cli', 1)] }, + { ...valid, cells: [rollbackCell('ubuntu', 'cli', 1), rollbackCell('ubuntu', 'cli', 2)] }, + ]) assert.throws(() => rollbackHistory([entry]), /Invalid/); + assert.throws(() => rollbackHistory({}), /array/); + assert.equal(rollbackHistory([valid])[0].url, + `https://github.com/JacksonFergusonDev/protostar/commit/${valid.commit}`); +}); + +async function renderRollbacks(responses, chartFailure = false) { + const elements = new Map(); + const charts = []; + const element = id => { + if (!elements.has(id)) elements.set(id, { + textContent: '', hidden: true, rows: [], children: [], + appendChild(child) { this.children.push(child); }, + insertRow() { + const cells = []; + this.rows.push(cells); + return { insertCell() { + const cell = { textContent: '', appendChild(child) { this.child = child; } }; + cells.push(cell); + return cell; + } }; + }, + }); + return elements.get(id); + }; + const source = await readFile(new URL('../metrics/rollback-dashboard.js', import.meta.url), 'utf8'); + await runInNewContext(source.replace(/^import[^\n]*\n/gm, ''), { + rollbackHistory, rollbackSeries, escapeHtml, + fetch: async url => responses[url], + loadCharts: async () => { if (chartFailure) throw new Error('CDN unavailable'); }, + window: { addEventListener() {} }, + document: { + documentElement: {}, fonts: { ready: Promise.resolve() }, getElementById: element, + createElement() { return { style: {}, setAttribute() {}, prepend() {}, remove() {} }; }, + body: { appendChild() {} }, + }, + getComputedStyle() { + return { color: '#22d3ee', getPropertyValue(name) { + return ({ '--text': '#e8edef', '--muted': '#939da6', '--line': '#252c31', + '--accent': '#22d3ee', '--mono': 'JetBrains Mono', '--panel': '#0e1114', + '--fs-ui': '14px', '--fs-label': '11px' })[name]; + } }; + }, + echarts: { init() { return { setOption(options) { charts.push(options); }, on() {} }; } }, + }); + return { element, charts }; +} + +const rollbackEntries = [ + rollbackRun('2026-10-01T00:00:00Z', [rollbackCell('ubuntu', 'cli', 1200), rollbackCell('windows', 'cli', 1200)]), + rollbackRun('2026-10-02T00:00:00Z', [rollbackCell('ubuntu', 'cli', 1300), rollbackCell('windows', 'cli', 1300), + rollbackCell('windows', 'lib', 800)])]; + +test('rollback panel renders counts, a per-template table, scope markers, and safe commit links', async () => { + const { element, charts } = await renderRollbacks({ + 'rollback-history.json': { ok: true, json: async () => rollbackEntries }, + 'rollback-latest.json': { ok: true, json: async () => ({ message: '3,400' }) } }); + assert.equal(element('rollback-status').textContent, ''); + assert.equal(element('rollback-failing').textContent, ''); + assert.equal(element('rollback-results').hidden, false); + assert.match(element('rollback-latest').textContent, /^3,400 faults restored/); + assert.deepEqual(element('rollback-os').children.map(cell => cell.textContent), ['ubuntu', 'windows']); + assert.deepEqual(element('rollback-cells').rows[0].map(cell => cell.textContent), ['cli', '1,300', '1,300']); + assert.deepEqual(element('rollback-cells').rows[1].map(cell => cell.textContent), ['lib', 'N/A', '800']); + assert.equal(element('rollback-runs').rows[0][2].child.href, + `https://github.com/JacksonFergusonDev/protostar/commit/${'c'.repeat(40)}`); + assert.match(element('rollback-runs').rows[0][3].textContent, /added windows\/lib/); + assert.equal(charts[0].series[0].markLine.data[0].xAxis, 1); + assert.equal(charts[0].legend.selected.Overall, true); + assert.equal(charts[0].legend.selected.cli, false); +}); + +test('rollback panel says when the latest run failed, without putting it in the history', async () => { + const { element } = await renderRollbacks({ + 'rollback-history.json': { ok: true, json: async () => rollbackEntries }, + 'rollback-latest.json': { ok: true, json: async () => ({ message: '2 failing' }) } }); + assert.match(element('rollback-failing').textContent, /most recent run had 2 failing/); + assert.equal(element('rollback-results').hidden, false); +}); + +test('rollback panel handles unpublished, empty, malformed, and unavailable history', async () => { + for (const response of [{ status: 404 }, { ok: true, json: async () => [] }]) { + const { element, charts } = await renderRollbacks({ 'rollback-history.json': response }); + assert.equal(element('rollback-status').textContent, 'No complete rollback runs published yet.'); + assert.equal(element('rollback-results').hidden, true); + assert.equal(charts.length, 0); + } + for (const response of [{ status: 500, ok: false }, { ok: true, json: async () => ({}) }]) { + const { element } = await renderRollbacks({ 'rollback-history.json': response }); + assert.match(element('rollback-status').textContent, /could not load/); + } +}); + +test('a missing badge file or a chart-library failure keeps the rollback counts', async () => { + const { element } = await renderRollbacks({ + 'rollback-history.json': { ok: true, json: async () => rollbackEntries }, + 'rollback-latest.json': { status: 404, ok: false } }, true); + assert.equal(element('rollback-failing').textContent, ''); + assert.equal(element('rollback-results').hidden, false); + assert.match(element('rollback-status').textContent, /Counts and commit links remain available/); +}); diff --git a/tests/rollback_report.py b/tests/rollback_report.py new file mode 100644 index 00000000..650e3dc8 --- /dev/null +++ b/tests/rollback_report.py @@ -0,0 +1,83 @@ +"""Counts the faults a rollback run injected, for the metrics dashboard. + +Every test marked ``rollback_fault`` raises one fault (an error or an +interrupt, around a write, directory, removal, command, or the commit, or +while a restore fails) and requires the project to come back as it was. With +``--rollback-report PATH`` this plugin writes how many of them passed, and +which failed, per scenario. A run that retries its failures (the nightly's +second pass) updates the same file: a fault that failed and then passed is +moved from the failed list to the passed count, so the report always holds each +fault's last outcome. + +A run that stops early (interrupted, or killed) writes nothing, so a missing +report means the result is unknown, never that it passed. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from tests.rollback_harness import SCENARIOS + +MARKER = "rollback_fault" +_TEMPLATE_BY_SCENARIO = {scenario.name: scenario.template for scenario in SCENARIOS} + + +def scenario_of(nodeid: str) -> str: + """Returns the scenario a fault test ran, from its parametrized id. + + Every fault test's id starts with its scenario's name: a case's id is + ``:::``, a failed restore's is + ``:``, and a scenario's own id is its name. + """ + return nodeid.split("[", 1)[1].rstrip("]").split(":", 1)[0] + + +class RollbackReport: + """Collects each fault test's outcome and writes the counts when the run ends.""" + + def __init__(self, path: Path) -> None: + self.path = path + self.failed: dict[str, bool] = {} + + def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: + """Records a fault test's outcome: failed in any phase, or passed its call.""" + if MARKER not in report.keywords or report.skipped: + return + if report.failed: + self.failed[report.nodeid] = True + elif report.when == "call": + self.failed.setdefault(report.nodeid, False) + + def pytest_sessionfinish(self, session: pytest.Session, exitstatus: int) -> None: + """Writes the counts of a run that finished, passing or failing.""" + if hasattr(session.config, "workerinput") or exitstatus not in ( + pytest.ExitCode.OK, + pytest.ExitCode.TESTS_FAILED, + ): + return + scenarios: dict[str, dict[str, Any]] = {} + if self.path.exists(): + scenarios = json.loads(self.path.read_text(encoding="utf-8"))["scenarios"] + for nodeid, failed in sorted(self.failed.items()): + name = scenario_of(nodeid) + entry = scenarios.setdefault( + name, + {"template": _TEMPLATE_BY_SCENARIO[name], "passed": 0, "failed": []}, + ) + if failed: + if nodeid not in entry["failed"]: + entry["failed"].append(nodeid) + else: + entry["passed"] += 1 + if nodeid in entry["failed"]: + entry["failed"].remove(nodeid) + self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.write_text( + json.dumps({"scenarios": dict(sorted(scenarios.items()))}, indent=2) + "\n", + encoding="utf-8", + ) diff --git a/tests/test_nightly.py b/tests/test_nightly.py index b22e5013..5d86b7a5 100644 --- a/tests/test_nightly.py +++ b/tests/test_nightly.py @@ -54,10 +54,12 @@ def expand(matrix: dict[str, Any]) -> list[Entry]: return combinations +def yaml_workflow(workflow: str) -> dict[str, Any]: + return YAML(typ="safe").load((WORKFLOWS / workflow).read_text(encoding="utf-8")) + + def jobs(workflow: str) -> dict[str, Any]: - return YAML(typ="safe").load((WORKFLOWS / workflow).read_text(encoding="utf-8"))[ - "jobs" - ] + return yaml_workflow(workflow)["jobs"] def job(workflow: str, name: str) -> dict[str, Any]: @@ -275,6 +277,39 @@ def test_a_manual_nightly_can_narrow_the_rollback_jobs(): assert args[args.index("--rollback-real") + 1] == "all" +def test_every_rollback_job_uploads_its_fault_report_even_when_it_fails(): + rollback = job("nightly.yml", "rollback") + (run,) = [ + s for s in rollback["steps"] if s.get("uses") == "./.github/actions/pytest" + ] + args = run["with"]["args"].split() + assert args[args.index("--rollback-report") + 1] == "rollback-report/report.json" + (upload,) = [s for s in rollback["steps"] if "upload-artifact" in s.get("uses", "")] + assert upload["name"] == "Upload the fault report" + assert upload["with"]["name"] == "${{ matrix.artifact }}" + assert upload["with"]["path"] == "rollback-report/report.json" + assert upload["if"] == "${{ !cancelled() }}" + + +def test_rollback_metrics_are_recorded_apart_from_nightly_and_only_for_scheduled_runs(): + """A publishing failure must not fail the run a release requires.""" + workflow = yaml_workflow("rollback-metrics.yml") + trigger = workflow["on"]["workflow_run"] + assert trigger["workflows"] == ["Nightly"] + assert "rollback-metrics.yml" not in (WORKFLOWS / "nightly.yml").read_text() + record = workflow["jobs"]["record"] + assert "workflow_run.event == 'schedule'" in record["if"] + assert "head_branch == 'main'" in record["if"] + assert record["permissions"]["contents"] == "write" + (download,) = [ + s for s in record["steps"] if "download-artifact" in s.get("uses", "") + ] + assert download["with"]["pattern"] == "rollback-*" + assert download["with"]["run-id"] == "${{ github.event.workflow_run.id }}" + assert workflow["jobs"]["publish-pages"]["uses"] == "./.github/workflows/pages.yml" + assert workflow["jobs"]["refresh-site"]["needs"] == "publish-pages" + + def test_a_release_smoke_tests_the_wheel_it_publishes_on_every_os(): smoke = job("release.yml", "smoke") assert smoke["needs"] == "build" diff --git a/tests/test_prepare_pages.py b/tests/test_prepare_pages.py index ebe52520..df0f0612 100644 --- a/tests/test_prepare_pages.py +++ b/tests/test_prepare_pages.py @@ -274,17 +274,25 @@ def test_the_old_dashboard_address_redirects_to_the_metrics_page( assert {path.name for path in (output / "benchmarks").iterdir()} == {"index.html"} +RECORDED_JSON = ( + "mutation-history.json", + "mutation-latest.json", + "rollback-history.json", + "rollback-latest.json", +) + + @pytest.mark.parametrize("performance", [False, True]) -def test_mutation_files_survive_pages_assembly_without_obsolete_assets( +def test_recorded_json_survives_pages_assembly_without_obsolete_assets( published_docs, performance ): source, output, config = published_docs if not performance: (source / "metrics/data.js").unlink() - for filename in ("mutation-history.json", "mutation-latest.json"): + for filename in RECORDED_JSON: _write(source / "metrics" / filename, '{"kept": true}\n') assemble_pages(source, output, config) - for filename in ("mutation-history.json", "mutation-latest.json"): + for filename in RECORDED_JSON: assert (output / "metrics" / filename).read_bytes() == ( source / "metrics" / filename ).read_bytes() diff --git a/tests/test_rollback.py b/tests/test_rollback.py index f218d197..184e92d2 100644 --- a/tests/test_rollback.py +++ b/tests/test_rollback.py @@ -360,6 +360,7 @@ def test_a_clean_run_passes_the_recorded_sites_and_journals_every_change( # ----------------------------------------------------------------- faults -- # +@pytest.mark.rollback_fault def test_rollback_restores_the_seed(case: Case, monkeypatch, seed): workspace = seed(case.scenario) before = workspace.capture() @@ -373,6 +374,7 @@ def clean() -> Workspace: _assert_handled(case, outcome, workspace, before, clean) +@pytest.mark.rollback_fault def test_a_failed_restore_is_reported_and_everything_else_restored( failed_restore: tuple[Case, str], monkeypatch, seed ): @@ -408,6 +410,7 @@ def test_a_failed_restore_is_reported_and_everything_else_restored( assert changed <= allowed, sorted(changed) +@pytest.mark.rollback_fault @pytest.mark.skipif(sys.platform == "win32", reason="Ctrl+C is a POSIX signal here") def test_rollback_finishes_through_a_second_interrupt( scoped_scenario: Scenario, monkeypatch, seed @@ -427,6 +430,7 @@ def test_rollback_finishes_through_a_second_interrupt( _assert_rolled_back(case, outcome, workspace, before) +@pytest.mark.rollback_fault def test_a_rolled_back_run_succeeds_when_retried(retried_case: Case, monkeypatch, seed): """Rollback leaves a project init can start over in, not just one that looks right.""" scenario = retried_case.scenario @@ -497,6 +501,7 @@ def test_real_commands_pass_the_recorded_sites( assert outcome.sites == scenario.recorded_sites() +@pytest.mark.rollback_fault @pytest.mark.integration @pytest.mark.usefixtures("real_commands") def test_real_commands_roll_back(real_case: Case, monkeypatch, seed): diff --git a/tests/test_rollback_report.py b/tests/test_rollback_report.py new file mode 100644 index 00000000..366160cf --- /dev/null +++ b/tests/test_rollback_report.py @@ -0,0 +1,249 @@ +"""The rollback fault count: what the plugin writes, and what is published.""" + +import argparse +import json +from types import SimpleNamespace + +import pytest + +from scripts.nightly_matrix import rollback_matrix +from scripts.rollback_report import badge, combine, record +from tests.rollback_report import RollbackReport, scenario_of + + +def report(nodeid, *, when="call", outcome="passed", marked=True): + return SimpleNamespace( + nodeid=nodeid, + when=when, + failed=outcome == "failed", + skipped=outcome == "skipped", + keywords={"rollback_fault": 1} if marked else {}, + ) + + +def finish(plugin, status=pytest.ExitCode.OK, worker=False): + config = SimpleNamespace(**({"workerinput": {}} if worker else {})) + plugin.pytest_sessionfinish(SimpleNamespace(config=config), status) + + +FAULT = "tests/test_rollback.py::test_rollback_restores_the_seed[cli-adopted:write:a#1:after:error]" +OTHER = "tests/test_rollback.py::test_a_failed_restore_is_reported[sync-cli-retooled:a.toml]" + + +@pytest.mark.parametrize( + ("nodeid", "scenario"), + [ + (FAULT, "cli-adopted"), + (OTHER, "sync-cli-retooled"), + ("tests/test_rollback.py::test_x[lib-empty]", "lib-empty"), + ], +) +def test_a_fault_belongs_to_the_scenario_its_id_starts_with(nodeid, scenario): + assert scenario_of(nodeid) == scenario + + +def test_counts_each_fault_once_per_scenario_with_its_template(tmp_path): + path = tmp_path / "out" / "report.json" + plugin = RollbackReport(path) + for nodeid in (FAULT, OTHER): + plugin.pytest_runtest_logreport(report(nodeid, when="setup")) + plugin.pytest_runtest_logreport(report(nodeid)) + plugin.pytest_runtest_logreport( + report("tests/other.py::t[cli-adopted]", marked=False) + ) + finish(plugin) + assert json.loads(path.read_text()) == { + "scenarios": { + "cli-adopted": {"template": "cli", "passed": 1, "failed": []}, + "sync-cli-retooled": {"template": "cli", "passed": 1, "failed": []}, + } + } + + +def test_a_failure_in_any_phase_fails_the_fault_and_a_skip_counts_for_nothing(tmp_path): + path = tmp_path / "report.json" + plugin = RollbackReport(path) + plugin.pytest_runtest_logreport(report(FAULT)) + plugin.pytest_runtest_logreport(report(FAULT, when="teardown", outcome="failed")) + plugin.pytest_runtest_logreport(report(OTHER, when="setup", outcome="skipped")) + finish(plugin, pytest.ExitCode.TESTS_FAILED) + assert json.loads(path.read_text())["scenarios"] == { + "cli-adopted": {"template": "cli", "passed": 0, "failed": [FAULT]} + } + + +def test_a_retry_replaces_a_failure_with_its_last_outcome(tmp_path): + path = tmp_path / "report.json" + first = RollbackReport(path) + first.pytest_runtest_logreport(report(FAULT, outcome="failed")) + first.pytest_runtest_logreport(report(OTHER)) + finish(first, pytest.ExitCode.TESTS_FAILED) + retry = RollbackReport(path) + retry.pytest_runtest_logreport(report(FAULT)) + finish(retry) + assert json.loads(path.read_text())["scenarios"] == { + "cli-adopted": {"template": "cli", "passed": 1, "failed": []}, + "sync-cli-retooled": {"template": "cli", "passed": 1, "failed": []}, + } + + +@pytest.mark.parametrize( + ("status", "worker"), + [ + (pytest.ExitCode.INTERRUPTED, False), + (pytest.ExitCode.INTERNAL_ERROR, False), + (pytest.ExitCode.OK, True), + ], +) +def test_nothing_is_written_when_the_result_is_unknown_or_the_worker_ends( + tmp_path, status, worker +): + path = tmp_path / "report.json" + plugin = RollbackReport(path) + plugin.pytest_runtest_logreport(report(FAULT)) + finish(plugin, status, worker) + assert not path.exists() + + +# ------------------------------------------------------------ publishing -- # + +MATRIX = rollback_matrix("all", "cli")["include"] +ONE = [entry for entry in MATRIX if entry["os"] != "windows-latest"] + + +def write_results(root, matrix, failed=(), passed=10): + for entry in matrix: + folder = root / entry["artifact"] + folder.mkdir(parents=True) + (folder / "report.json").write_text( + json.dumps( + { + "scenarios": { + f"{entry['template']}-adopted": { + "template": entry["template"], + "passed": passed, + "failed": list(failed), + } + } + } + ) + ) + + +@pytest.fixture +def run(tmp_path): + write_results(tmp_path / "results", ONE) + return argparse.Namespace( + results=tmp_path / "results", + matrix=json.dumps({"include": ONE}), + history=tmp_path / "pages/metrics/rollback-history.json", + latest=tmp_path / "pages/metrics/rollback-latest.json", + commit="a" * 40, + date="2026-10-06T05:30:00+00:00", + ) + + +def test_every_matrix_job_names_its_artifact(): + artifacts = [entry["artifact"] for entry in rollback_matrix()["include"]] + assert len(set(artifacts)) == len(artifacts) == 2 * 5 + 5 * 6 + assert "rollback-windows-latest-cli-3of6" in artifacts + assert "rollback-ubuntu-latest-ml-1of1" in artifacts + + +def test_a_green_run_adds_a_history_point_and_the_count_to_the_badge(run): + record(run) + assert json.loads(run.history.read_text()) == [ + { + "commit": "a" * 40, + "date": "2026-10-06T05:30:00+00:00", + "cells": [ + {"os": "macos", "template": "cli", "passed": 10}, + {"os": "ubuntu", "template": "cli", "passed": 10}, + ], + } + ] + assert json.loads(run.latest.read_text()) == { + "schemaVersion": 1, + "label": "rollback faults restored", + "message": "20", + "color": "22d3ee", + "labelColor": "0A0A0A", + } + + +def test_slices_of_one_cell_add_up(tmp_path): + matrix = [e for e in MATRIX if e["os"] == "windows-latest"] + write_results(tmp_path, matrix, passed=7) + assert combine(tmp_path, matrix) == ( + [{"os": "windows", "template": "cli", "passed": 42}], + 0, + ) + + +def test_a_failure_changes_the_badge_but_not_the_history(run): + write_json = run.history + write_json.parent.mkdir(parents=True) + write_json.write_text("[]") + for entry in ONE[:1]: + report_path = run.results / entry["artifact"] / "report.json" + data = json.loads(report_path.read_text()) + data["scenarios"]["cli-adopted"]["failed"] = ["a", "b"] + report_path.write_text(json.dumps(data)) + record(run) + assert json.loads(run.history.read_text()) == [] + assert json.loads(run.latest.read_text())["message"] == "2 failing" + + +def test_a_retried_commit_is_recorded_once(run): + record(run) + record(run) + assert len(json.loads(run.history.read_text())) == 1 + + +def test_an_older_run_is_refused(run): + record(run) + run.commit = "b" * 40 + run.date = "2026-10-05T05:30:00+00:00" + with pytest.raises(SystemExit, match="older"): + record(run) + + +@pytest.mark.parametrize( + "problem", ["missing", "extra", "empty", "foreign", "no-report"] +) +def test_an_incomplete_run_publishes_nothing(run, problem): + first = run.results / ONE[0]["artifact"] + if problem == "missing": + for child in first.iterdir(): + child.unlink() + first.rmdir() + elif problem == "extra": + (run.results / "rollback-other").mkdir() + elif problem == "empty": + (first / "report.json").write_text('{"scenarios": {}}') + elif problem == "foreign": + (first / "report.json").write_text( + '{"scenarios": {"x": {"template": "lib", "passed": 1, "failed": []}}}' + ) + else: + (first / "report.json").unlink() + with pytest.raises(SystemExit): + record(run) + assert not run.history.exists() + assert not run.latest.exists() + + +@pytest.mark.parametrize( + ("commit", "date"), + [("abc", "2026-10-06T05:30:00+00:00"), ("a" * 40, "2026-10-06T05:30:00-07:00")], +) +def test_a_bad_commit_or_local_time_is_refused(run, commit, date): + run.commit, run.date = commit, date + with pytest.raises(SystemExit): + record(run) + + +def test_the_badge_says_failing_in_words_and_keeps_the_house_colors(): + assert badge(5214, 0)["message"] == "5,214" + assert badge(5214, 1)["message"] == "1 failing" + assert badge(1, 0)["color"] == "22d3ee"