diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index cb01b305..9c73860a 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -52,7 +52,7 @@ jobs: tool: 'customSmallerIsBetter' output-file-path: benchmark-gh.json gh-pages-branch: 'gh-pages' - benchmark-data-dir-path: 'benchmarks' + benchmark-data-dir-path: 'metrics' github-token: ${{ secrets.GITHUB_TOKEN }} fail-on-alert: false alert-threshold: '200%' @@ -60,7 +60,7 @@ jobs: auto-push: ${{ github.event_name == 'push' }} publish-pages: - name: Publish updated benchmarks + name: Publish updated metrics needs: benchmark if: github.event_name == 'push' permissions: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d16f3d51..924b1b99 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -149,7 +149,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Test benchmark dashboard data handling - run: node --test tests/benchmark_metrics.test.mjs + run: node --test tests/metrics_dashboard.test.mjs - name: Install uv uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 @@ -180,7 +180,7 @@ jobs: tool: 'customSmallerIsBetter' output-file-path: benchmark-gh.json gh-pages-branch: 'gh-pages' - benchmark-data-dir-path: 'benchmarks' + benchmark-data-dir-path: 'metrics' github-token: ${{ secrets.GITHUB_TOKEN }} fail-on-alert: true alert-threshold: '250%' diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml index bfeda0e3..a0d83848 100644 --- a/.github/workflows/mutation.yml +++ b/.github/workflows/mutation.yml @@ -46,8 +46,8 @@ jobs: run: | if [[ "$EVENT" == "schedule" ]]; then git fetch origin gh-pages - if git cat-file -e origin/gh-pages:benchmarks/mutation-history.json; then - git show origin/gh-pages:benchmarks/mutation-history.json > mutation-history.json + if git cat-file -e origin/gh-pages:metrics/mutation-history.json; then + git show origin/gh-pages:metrics/mutation-history.json > mutation-history.json fi changed="$(python3 scripts/mutation_report.py changed --history mutation-history.json)" else @@ -246,9 +246,9 @@ jobs: echo "Publishing mutation results (attempt $attempt/5)." python3 scripts/mutation_report.py record \ --results results --matrix "$MATRIX" --commit "$COMMIT" --date "$recorded_at" \ - --history pages-data/benchmarks/mutation-history.json \ - --latest pages-data/benchmarks/mutation-latest.json - git -C pages-data add benchmarks/mutation-history.json benchmarks/mutation-latest.json + --history pages-data/metrics/mutation-history.json \ + --latest pages-data/metrics/mutation-latest.json + git -C pages-data add metrics/mutation-history.json metrics/mutation-latest.json if ! git -C pages-data diff --cached --quiet; then git -C pages-data commit -m "Record engine mutation score for ${COMMIT:0:7}" fi diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 40392b84..d5627079 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -28,7 +28,7 @@ jobs: fetch-depth: 0 persist-credentials: false - - name: Read current published versions and benchmarks + - name: Read current published versions and metrics uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: gh-pages @@ -44,15 +44,15 @@ jobs: - name: Install documentation dependencies run: uv sync --only-group docs --locked - - name: Assemble documentation, redirects, and benchmarks + - name: Assemble documentation, redirects, and metrics run: | latest_version="$(uv run --only-group docs python scripts/prepare_pages.py pages-source --print-latest)" git show "refs/tags/v${latest_version}:zensical.toml" > latest-zensical.toml uv run --only-group docs python scripts/prepare_pages.py pages-source \ --output published --config latest-zensical.toml - - name: Test benchmark dashboard data handling - run: node --test tests/benchmark_metrics.test.mjs + - name: Test metrics dashboard data handling + run: node --test tests/metrics_dashboard.test.mjs - name: Upload Pages artifact uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bb999386..b0a69f86 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -204,7 +204,7 @@ jobs: git push origin gh-pages publish-pages: - name: Publish documentation and benchmarks + name: Publish documentation and metrics needs: build-docs if: ${{ !cancelled() && needs.build-docs.result == 'success' }} permissions: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 24090f80..2e68e2c5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -83,12 +83,12 @@ repos: # --- Pre-push hooks --- # - - id: benchmark-dashboard - name: benchmark dashboard data handling - entry: node --test tests/benchmark_metrics.test.mjs + - id: metrics-dashboard + name: metrics dashboard data handling + entry: node --test tests/metrics_dashboard.test.mjs language: system pass_filenames: false - files: ^(benchmarks/|tests/benchmark_metrics\.test\.mjs$) + files: ^(metrics/|tests/metrics_dashboard\.test\.mjs$) stages: [pre-push] - id: zensical-build diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1b79aa90..8eb9cb73 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -195,11 +195,11 @@ just test ### Documentation Publishing -GitHub Pages must use **GitHub Actions** as its source (Settings → Pages → Build and deployment). The `gh-pages` branch stores released documentation and benchmark data; it is not itself the published site. Releases and benchmark updates invoke `.github/workflows/pages.yml`, which serializes site assembly and deployment together and reads the current branch contents after acquiring its publishing slot. +GitHub Pages must use **GitHub Actions** as its source (Settings → Pages → Build and deployment). The `gh-pages` branch stores released documentation and benchmark and mutation data; it is not itself the published site. Releases and benchmark updates invoke `.github/workflows/pages.yml`, which serializes site assembly and deployment together and reads the current branch contents after acquiring its publishing slot. -The publisher creates a clean artifact with `scripts/prepare_pages.py`. The release carrying `latest` in `versions.json` is served at the bare documentation URLs, and every release, that one included, keeps its versioned copy. Each versioned page the latest release still has names the bare URL as its canonical address, so search engines rank one URL across releases; pages the latest release dropped are marked `noindex`. Version aliases redirect to wherever their release is served, preserving query parameters and anchors, and the sitemap lists the bare URLs. Old root copies are excluded. Markdown and `llms.txt` come from the latest release's HTML and tagged navigation, with links to the bare Markdown files. Benchmark history is included in every deployment. The root also gets `robots.txt`, which welcomes search engines and AI search agents, turns away AI training crawlers, and names the sitemap. +The publisher creates a clean artifact with `scripts/prepare_pages.py`. The release carrying `latest` in `versions.json` is served at the bare documentation URLs, and every release, that one included, keeps its versioned copy. Each versioned page the latest release still has names the bare URL as its canonical address, so search engines rank one URL across releases; pages the latest release dropped are marked `noindex`. Version aliases redirect to wherever their release is served, preserving query parameters and anchors, and the sitemap lists the bare URLs. Old root copies are excluded. Markdown and `llms.txt` come from the latest release's HTML and tagged navigation, with links to the bare Markdown files. The metrics dashboard is included in every deployment, and the old `/benchmarks/` address redirects to it. The root also gets `robots.txt`, which welcomes search engines and AI search agents, turns away AI training crawlers, and names the sitemap. -To republish the current documentation and benchmark data without rebuilding a release or publishing to PyPI, run `gh workflow run pages.yml --ref main`. To rebuild a released documentation version, run `gh workflow run release.yml --ref main -f tag=vX.Y.Z`; this moves `latest` to that version and invokes the same publisher. Keep Pages in Actions mode so branch updates cannot replace the assembled artifact. +To republish the current documentation and metrics without rebuilding a release or publishing to PyPI, run `gh workflow run pages.yml --ref main`. To rebuild a released documentation version, run `gh workflow run release.yml --ref main -f tag=vX.Y.Z`; this moves `latest` to that version and invokes the same publisher. Keep Pages in Actions mode so branch updates cannot replace the assembled artifact. ### Isolated Manual Testing (Sandboxes) diff --git a/MUTATION_TRACKING_PLAN.md b/MUTATION_TRACKING_PLAN.md index 415e71b5..23c546bd 100644 --- a/MUTATION_TRACKING_PLAN.md +++ b/MUTATION_TRACKING_PLAN.md @@ -23,7 +23,7 @@ Widen mutation testing to the rest of the engine, then run it on a schedule, rec - **Scheduled, gated by change.** No run on every commit. A scheduled run first checks whether the mutated modules, `tests/`, `pyproject.toml`, `uv.lock`, `scripts/mutation_report.py`, or the mutation workflow changed since the last recorded commit, and stops if not. Quiet periods cost nothing, and a busy period gets one point per scheduled run. The manual trigger stays for development. - **Only complete runs publish.** A run publishes a score only if every module in the set succeeded. Manual runs on a subset never publish. - **Store raw counts per module.** Each history entry records the commit, the date, and each module's counts (killed, timeout, survived, and the rest `mutation_report.py` tracks), not just percentages. The overall score is computed from them, so it stays correct when the set grows, and the graph can mark when the scope changed. -- **Our own JSON, not github-action-benchmark.** That action writes a JS file built for speed regressions. Mutation results go in their own files under `benchmarks/` on `gh-pages`: a history file for the graph and a small latest-score file in shields.io's endpoint format, which the badge and the website both read. +- **Our own JSON, not github-action-benchmark.** That action writes a JS file built for speed regressions. Mutation results go in their own files under `metrics/` on `gh-pages`: a history file for the graph and a small latest-score file in shields.io's endpoint format, which the badge and the website both read. - **The website reads the score when it builds.** jacksonferguson.me (Astro, `~/Developer/JacksonFergusonDev.github.io`) already fetches remote files at build time through `config/remote-assets.json`, and its `deploy.yml` already rebuilds on `repository_dispatch` of type `remote-assets-updated`, plus weekly. Protostar sends that dispatch after the Pages deploy finishes, not just after the `gh-pages` push, so the site never fetches a stale file. - **Say what the score covers.** It covers the mutated modules, not the whole codebase. The dashboard says which modules, and the badge label doesn't imply full coverage. @@ -62,5 +62,5 @@ Added so far: | `reconciliation` (6 shards) | 2h30m wall, 8h40m runner | 36m wall, 1h55m runner | - Phase 3 (#441): Complete main-branch runs publish raw per-module history and the latest-score endpoint nightly at 02:23 UTC, gated by changed inputs. Pages carries both JSON files. -- Phase 4 (#442): The benchmarks dashboard graphs the overall score and each module from `mutation-history.json`, marking where the module set changed, and the README carries a shields.io endpoint badge reading `mutation-latest.json`. +- Phase 4 (#442): The metrics dashboard graphs the overall score and each module from `mutation-history.json`, marking where the module set changed, and the README carries a shields.io endpoint badge reading `mutation-latest.json`. - Phase 5 (#443, JacksonFergusonDev.github.io#7): jacksonferguson.me fetches `mutation-latest.json` at build time through a `json` remote asset type that checks the file's shape, and shows the score on the Protostar card, linked to the dashboard. After a published run's Pages deploy, the mutation workflow's `refresh-site` job sends the site a `remote-assets-updated` dispatch using `PORTFOLIO_DISPATCH_TOKEN`, a fine-grained token limited to the site's repository. The first complete run, started by hand, recorded 99.9% across 14 modules at `da9b757` on 2026-10-04. diff --git a/README.md b/README.md index d959ec6f..633180dd 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ [![CI](https://img.shields.io/github/actions/workflow/status/jacksonfergusondev/protostar/ci.yml?color=22d3ee&labelColor=0A0A0A&label=CI)](https://github.com/jacksonfergusondev/protostar/actions/workflows/ci.yml) [![Release](https://img.shields.io/github/actions/workflow/status/jacksonfergusondev/protostar/release.yml?color=22d3ee&labelColor=0A0A0A&label=release)](https://github.com/jacksonfergusondev/protostar/actions/workflows/release.yml) [![Codecov](https://img.shields.io/codecov/c/github/JacksonFergusonDev/protostar?color=22d3ee&labelColor=0A0A0A&logo=codecov&logoColor=white)](https://codecov.io/gh/JacksonFergusonDev/protostar) -[![Engine mutation score](https://img.shields.io/endpoint?url=https%3A%2F%2Fprotostar.jacksonferguson.me%2Fbenchmarks%2Fmutation-latest.json&label=engine%20mutation%20score&color=22d3ee&labelColor=0A0A0A)](https://protostar.jacksonferguson.me/benchmarks/) +[![Engine mutation score](https://img.shields.io/endpoint?url=https%3A%2F%2Fprotostar.jacksonferguson.me%2Fmetrics%2Fmutation-latest.json&label=engine%20mutation%20score&color=22d3ee&labelColor=0A0A0A)](https://protostar.jacksonferguson.me/metrics/) [![Python](https://img.shields.io/badge/python-3.12+-22d3ee?labelColor=0A0A0A&logo=python&logoColor=white)](https://www.python.org/downloads/) [![Documentation](https://img.shields.io/badge/docs-gh--pages-22d3ee?labelColor=0A0A0A&logo=github&logoColor=white)](https://protostar.jacksonferguson.me/) [![License](https://img.shields.io/badge/license-MIT-22d3ee?labelColor=0A0A0A)](LICENSE) @@ -212,7 +212,7 @@ Protostar edits other people's work, so it's built to be careful: - **The engine is separate from the interface.** The same core runs behind the interactive editor, the command line, and coding agents, which is why every command can also answer in JSON. - **It's tested thoroughly.** Over 3,500 tests, including runs of the real tools, pass on Linux, macOS, and Windows, with strict type checking and at least 85% coverage. - **Its docs are checked against its code.** Before every push, a script confirms that the commands, errors, exit codes, and file paths the docs mention still match the code. -- **Performance stays visible.** CI tracks help-command startup and the recipe editor's first frame, flagging large regressions; see the [CI performance history](https://protostar.jacksonferguson.me/benchmarks/). +- **Performance stays visible.** CI tracks help-command startup and the recipe editor's first frame, flagging large regressions; see the [metrics dashboard](https://protostar.jacksonferguson.me/metrics/). --- diff --git a/docs/developer/testing.md b/docs/developer/testing.md index cab6f32c..e294ea07 100644 --- a/docs/developer/testing.md +++ b/docs/developer/testing.md @@ -186,9 +186,9 @@ Pass Protostar arguments to run one command without entering a shell, such as `j ``` === "Benchmark Preview" - Opens the benchmark dashboard at `http://127.0.0.1:8765/benchmarks/`, independently of the documentation server on port 8000. It fetches the published measurements once; refresh the page to see source edits. Stop it with `Ctrl+C`, or choose another port with `just serve-benchmarks 8766`. + Opens the metrics dashboard at `http://127.0.0.1:8765/metrics/`, independently of the documentation server on port 8000. It fetches the published measurements once; refresh the page to see source edits. Stop it with `Ctrl+C`, or choose another port with `just serve-metrics 8766`. ```bash - just serve-benchmarks + just serve-metrics ``` !!! tip "Manual Execution" @@ -224,7 +224,7 @@ gh workflow run mutation.yml --ref main -f modules=all Local runs write each surviving edit as a diff to `mutants/survivors.md`. Each CI runner uploads raw counts, survivor names, and survivor diffs. The combined job adds shard counts into one result per module. The mutation score is `(killed + timeout) / (killed + timeout + survived + suspicious)`; mutants no test reaches are reported separately. Surviving mutants reduce the score but do not make the workflow fail: inspect their diffs and strengthen tests where a meaningful behavior change went unnoticed. -Only a successful complete run on `main` publishes. Failed or cancelled jobs and manual subset runs cannot update the public score. Publication checks every expected artifact and the configured module set, then appends the commit, UTC date, and raw per-module counts to `benchmarks/mutation-history.json` on `gh-pages`. `benchmarks/mutation-latest.json` is a Shields endpoint with the aggregate score and an explicit engine label. Retrying a recorded commit does not add another history point. Counts are retained so the dashboard can later show changes in the module set without averaging percentages. +Only a successful complete run on `main` publishes. Failed or cancelled jobs and manual subset runs cannot update the public score. Publication checks every expected artifact and the configured module set, then appends the commit, UTC date, and raw per-module counts to `metrics/mutation-history.json` on `gh-pages`. `metrics/mutation-latest.json` is a Shields endpoint with the aggregate score and an explicit engine label. Retrying a recorded commit does not add another history point. Counts are retained so the dashboard can later show changes in the module set without averaging percentages. Publication rebases and retries if the benchmark or release workflow updates `gh-pages` concurrently; a conflicting history update is regenerated against the new branch contents. The shared Pages workflow deploys both mutation JSON files alongside the performance history. The benchmark dashboard graphs the history, and the README badge reads the latest score. @@ -242,7 +242,7 @@ CI tracks help-command startup (`protostar help init`) and the recipe editor's f The recipe editor benchmark sets a hidden environment variable, `PROTOSTAR_BENCHMARK_RECIPE_EDITOR=1`. It treats the session as interactive and makes `protostar init` exit as soon as the recipe editor draws its first frame, without waiting on input. Hyperfine measures the whole process, including Python startup and shutdown, in the checked-out repository. -The [CI performance history](https://protostar.jacksonferguson.me/benchmarks/) records main-branch measurements on GitHub Actions Ubuntu runners with Python 3.14. Each recorded point is the mean of 90 executions after 30 warmups. These timings describe the CI environment, not local workstation latency; runner variability and changes to the runner image or project checkout also limit comparisons between commits. Look for sustained trends rather than treating a single increase as a confirmed regression. Older wizard measurements remain in the downloadable history but are excluded from the recipe-editor chart. +The [metrics dashboard](https://protostar.jacksonferguson.me/metrics/) records main-branch measurements on GitHub Actions Ubuntu runners with Python 3.14. Each recorded point is the mean of 90 executions after 30 warmups. These timings describe the CI environment, not local workstation latency; runner variability and changes to the runner image or project checkout also limit comparisons between commits. Look for sustained trends rather than treating a single increase as a confirmed regression. Older wizard measurements remain in the downloadable history but are excluded from the recipe-editor chart. The CI regression check uses 30 executions after 5 warmups and fails when a measurement exceeds 250% of the preceding recorded result for the same benchmark. Historical tracking alerts above 200% without failing the run. These are relative checks for large regressions, not an absolute latency budget. The dashboard's comparison with up to 100 preceding benchmark runs is a separate descriptive summary, not the baseline used by either check. @@ -250,10 +250,10 @@ The `justfile` includes predefined recipes using [Hyperfine](https://github.com/ Help and version requests load argument definitions and tool descriptions, but do not load project analysis, reviews, execution, or format engines. Command implementations load only after dispatch. A tool reads its document-backed signals when analysis asks for them, and loads its document generators when planning calls `build()`. `tests/test_cli_startup.py` checks this boundary in fresh interpreters, including JSON help; this catches unnecessary imports without a machine-dependent timing threshold. -The dashboard source lives in `benchmarks/` on `main`. Pages publishing combines that source with the recorded `benchmarks/data.js` from `gh-pages`; it never publishes dashboard code from the data branch. To test its data handling locally, use Node.js 18 or newer: +The dashboard source lives in `metrics/` on `main`. Pages publishing combines that source with the recorded `metrics/data.js` from `gh-pages`; it never publishes dashboard code from the data branch. To test its data handling locally, use Node.js 18 or newer: ```bash -node --test tests/benchmark_metrics.test.mjs +node --test tests/metrics_dashboard.test.mjs ``` === "Quick Benchmark" diff --git a/docs/stylesheets/site-footer.css b/docs/stylesheets/site-footer.css index e55a9dc3..89ddc3b0 100644 --- a/docs/stylesheets/site-footer.css +++ b/docs/stylesheets/site-footer.css @@ -1,4 +1,4 @@ -/* One footer for the documentation landing page and benchmark dashboard. */ +/* One footer for the documentation landing page and metrics dashboard. */ .ps-footer { background: var(--bg); color: var(--muted); font-family: var(--sans); font-size: var(--fs-ui); line-height: 1.6; } .ps-footer__inner { display: flex; align-items: center; justify-content: space-between; flex-wrap: wrap; gap: 16px; width: min(1160px, calc(100% - 64px)); margin-inline: auto; padding-block: 20px; } .ps-footer p { margin: 0; } diff --git a/docs/stylesheets/site-header.css b/docs/stylesheets/site-header.css index 05b535e1..4c7902d3 100644 --- a/docs/stylesheets/site-header.css +++ b/docs/stylesheets/site-header.css @@ -1,4 +1,4 @@ -/* Shared Protostar header: docs landing page and benchmark dashboard. +/* Shared Protostar header: docs landing page and metrics dashboard. Both pages use this component; page-specific links stay in their markup. */ :root { --ps-header: 80px; } .ps-bar { diff --git a/justfile b/justfile index 07081be2..0fb7f07f 100644 --- a/justfile +++ b/justfile @@ -230,9 +230,9 @@ serve: sync @printf "\n{{ blue }}=== Launching Zensical Server ==={{ nc }}\n" uv run zensical serve -o -# Preview benchmark history on port 8765, separately from the docs server -serve-benchmarks port="8765": - uv run python scripts/serve_benchmarks.py --port {{ port }} +# Preview the metrics dashboard on port 8765, separately from the docs server +serve-metrics port="8765": + uv run python scripts/serve_metrics.py --port {{ port }} # Refresh release inputs for review, then bump version, sync lockfile, commit, tag, and push bump part: diff --git a/benchmarks/charts.mjs b/metrics/charts.mjs similarity index 100% rename from benchmarks/charts.mjs rename to metrics/charts.mjs diff --git a/benchmarks/dashboard.js b/metrics/dashboard.js similarity index 100% rename from benchmarks/dashboard.js rename to metrics/dashboard.js diff --git a/benchmarks/index.html b/metrics/index.html similarity index 97% rename from benchmarks/index.html rename to metrics/index.html index 99768358..9ef85de8 100644 --- a/benchmarks/index.html +++ b/metrics/index.html @@ -4,9 +4,9 @@ - Protostar · Performance and mutation history + Protostar · Metrics + content="Startup performance and mutation scores for Protostar, recorded from CI."> @@ -25,7 +25,7 @@
Protostar - Performance and mutation history + Metrics