Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions .config/nextest.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
# cargo-nextest configuration. Shared by local runs and CI.
#
# Flakiness policy: no retries anywhere. A test that needs a retry is a bug
# in the test; fix the test. `slow-timeout` turns a hang into a failure with
# a name attached instead of a stalled job.

[profile.default]
slow-timeout = { period = "30s", terminate-after = 4 } # 2 min hard cap per test
fail-fast = false

[profile.ci]
slow-timeout = { period = "30s", terminate-after = 6 } # 3 min on slower runners
fail-fast = false
failure-output = "immediate-final"
status-level = "fail"
final-status-level = "slow"

[profile.ci.junit]
path = "junit.xml"

# Repeated-run profile for flake hunting: same policy, quieter output.
[profile.stress]
slow-timeout = { period = "30s", terminate-after = 4 }
fail-fast = false
status-level = "fail"
final-status-level = "fail"
129 changes: 129 additions & 0 deletions .github/scripts/coverage_summary.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
#!/usr/bin/env python3
"""Render a Markdown coverage table from `cargo llvm-cov report --json`.

Usage: coverage_summary.py HEAD.json [BASE.json]

Rows are grouped per workspace crate (derived from the source path), with the
line-coverage delta against BASE when given. Files under `target/` and
`examples/`/`tests/` fixtures are still counted (they are part of the
workspace) but examples are listed separately so library crates stay visible.
"""
import json
import os
import re
import sys
from collections import defaultdict

CRATE_RE = re.compile(
r"(?:^|/)(crates/rpc/bidirectional/[^/]+|crates/[^/]+/[^/]+|examples/[^/]+(?:/[^/]+)?|tests/playwright/fixtures/[^/]+)/"
)


def load(path):
if not path or not os.path.exists(path):
return None
with open(path) as f:
data = json.load(f)
return data.get("data", [None])[0]


def per_crate(report):
"""-> {crate: (covered_lines, total_lines)}"""
out = defaultdict(lambda: [0, 0])
if not report:
return out
for f in report.get("files", []):
m = CRATE_RE.search(f["filename"])
crate = m.group(1) if m else "(other)"
lines = f["summary"]["lines"]
out[crate][0] += lines["covered"]
out[crate][1] += lines["count"]
return out


def pct(cov, tot):
return 100.0 * cov / tot if tot else 0.0


def fmt_delta(d):
if d is None:
return ""
if abs(d) < 0.005:
return "±0.00"
return f"{d:+.2f}"


def main():
head = load(sys.argv[1])
base = load(sys.argv[2]) if len(sys.argv) > 2 else None
if head is None:
print("Coverage report unavailable.")
return

hc = per_crate(head)
bc = per_crate(base) if base else None

total_lines = head["totals"]["lines"]
total_pct = total_lines["percent"]
total_delta = None
if base:
total_delta = total_pct - base["totals"]["lines"]["percent"]

print("### Test coverage (lines)")
print()
headline = f"**Total: {total_pct:.2f}%**"
if total_delta is not None:
headline += f" ({fmt_delta(total_delta)} vs base)"
print(headline, f"— {total_lines['covered']}/{total_lines['count']} lines")
print()
cols = "| Crate | Lines | Coverage |" + (" Δ |" if base else "")
print(cols)
print("|---|---:|---:|" + ("---:|" if base else ""))

def rows(prefix):
for crate in sorted(k for k in hc if k.startswith(prefix)):
cov, tot = hc[crate]
p = pct(cov, tot)
line = f"| `{crate}` | {cov}/{tot} | {p:.2f}% |"
if base:
if crate in bc and bc[crate][1]:
d = p - pct(*bc[crate])
line += f" {fmt_delta(d)} |"
else:
line += " new |"
print(line)

rows("crates/")
if any(k.startswith(("examples/", "tests/")) for k in hc):
print("| **Examples and fixtures** | | |" + (" |" if base else ""))
rows("examples/")
rows("tests/")
if "(other)" in hc:
rows("(other)")

# Files whose coverage dropped the most, to make regressions actionable.
if base:
base_files = {f["filename"]: f["summary"]["lines"] for f in base.get("files", [])}
drops = []
for f in head.get("files", []):
b = base_files.get(f["filename"])
if not b or not b["count"]:
continue
d = f["summary"]["lines"]["percent"] - b["percent"]
if d <= -1.0:
drops.append((d, f["filename"], f["summary"]["lines"]["percent"]))
if drops:
print()
print("<details><summary>Files with coverage drops ≥ 1 point</summary>")
print()
print("| File | Coverage | Δ |")
print("|---|---:|---:|")
for d, name, p in sorted(drops)[:25]:
short = name.split("/rust-api-stack/", 1)[-1]
print(f"| `{short}` | {p:.2f}% | {fmt_delta(d)} |")
print()
print("</details>")


if __name__ == "__main__":
main()
86 changes: 69 additions & 17 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -161,13 +161,22 @@ jobs:
steps:
- uses: actions/checkout@v6.0.2
- uses: dtolnay/rust-toolchain@stable
- uses: taiki-e/install-action@nextest
- uses: Swatinem/rust-cache@v2
- name: Build tests
run: cargo test --workspace --all-targets --all-features --no-run --locked
- name: Run tests
run: cargo test --workspace --all-targets --all-features --locked
run: cargo nextest run --workspace --all-targets --all-features --locked --no-run
- name: Run tests (nextest, no retries)
run: cargo nextest run --workspace --all-targets --all-features --locked --profile ci
- name: Run doctests
run: cargo test --doc --workspace --all-features --locked
- name: Upload JUnit report
if: always()
uses: actions/upload-artifact@v7.0.1
with:
name: junit-tests
path: target/nextest/ci/junit.xml
if-no-files-found: ignore
retention-days: 14

feature-matrix:
name: Feature matrix
Expand Down Expand Up @@ -327,28 +336,71 @@ jobs:
coverage:
name: Coverage report
runs-on: ubuntu-latest
needs: [test]
permissions:
contents: read
pull-requests: write
steps:
- uses: actions/checkout@v6.0.2
with:
fetch-depth: 0
- uses: dtolnay/rust-toolchain@stable
with:
components: llvm-tools-preview
- uses: taiki-e/install-action@cargo-llvm-cov
- uses: taiki-e/install-action@v2
with:
tool: cargo-llvm-cov,nextest
- uses: Swatinem/rust-cache@v2
- name: Generate coverage (lcov)
run: cargo llvm-cov --workspace --all-targets --all-features --locked --lcov --output-path lcov.info
- name: Print summary
run: cargo llvm-cov report --summary-only

- name: Coverage for this ref
run: |
cargo llvm-cov nextest --workspace --all-targets --all-features --locked \
--lcov --output-path lcov.info
cargo llvm-cov report --json --output-path coverage-head.json
cargo llvm-cov report --summary-only | tee coverage-head.txt

- name: Coverage for base (PR only)
if: github.event_name == 'pull_request'
run: |
base=$(git merge-base "origin/${{ github.base_ref }}" HEAD)
git stash --include-untracked -q || true
git checkout -q "$base"
cargo llvm-cov nextest --workspace --all-targets --all-features --locked \
--json --output-path coverage-base.json || echo '{}' > coverage-base.json
git checkout -q -
git stash pop -q || true

- name: Build coverage summary
id: summary
shell: bash
run: |
python3 .github/scripts/coverage_summary.py \
coverage-head.json \
"${{ github.event_name == 'pull_request' && 'coverage-base.json' || '' }}" \
> coverage-summary.md
cat coverage-summary.md >> "$GITHUB_STEP_SUMMARY"

- name: Comment on PR
if: github.event_name == 'pull_request'
env:
GH_TOKEN: ${{ github.token }}
PR: ${{ github.event.pull_request.number }}
run: |
marker='<!-- ras-coverage-report -->'
body="$(printf '%s\n' "$marker"; cat coverage-summary.md)"
existing=$(gh api "repos/${{ github.repository }}/issues/$PR/comments" \
--jq ".[] | select(.body | startswith(\"$marker\")) | .id" | head -1)
if [ -n "$existing" ]; then
gh api -X PATCH "repos/${{ github.repository }}/issues/comments/$existing" -f body="$body" >/dev/null
else
gh pr comment "$PR" --body "$body"
fi

- name: Upload coverage artifact
uses: actions/upload-artifact@v7.0.1
with:
name: coverage-lcov
path: lcov.info
path: |
lcov.info
coverage-head.json
coverage-summary.md
retention-days: 30
# Optional: enable Codecov upload by adding a CODECOV_TOKEN secret and
# uncommenting. Without the token the run still succeeds and the lcov
# artifact above remains the source of truth.
# - uses: codecov/codecov-action@v4
# with:
# files: lcov.info
# fail_ci_if_error: false
50 changes: 50 additions & 0 deletions .github/workflows/stress.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
name: Flake detector

# Runs the whole suite several times at different parallelism levels. Any
# failure here is a real flake (retries are disabled everywhere), so it is
# a bug in the test, not noise. Weekly, plus on demand.

on:
schedule:
- cron: "17 3 * * 1"
workflow_dispatch:
inputs:
iterations:
description: Runs per parallelism level
default: "5"
required: false

env:
CARGO_TERM_COLOR: always
CARGO_INCREMENTAL: 0

jobs:
stress:
name: Repeated runs (threads=${{ matrix.threads }})
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
threads: [1, 4, 16]
steps:
- uses: actions/checkout@v6.0.2
- uses: dtolnay/rust-toolchain@stable
- uses: taiki-e/install-action@nextest
- uses: Swatinem/rust-cache@v2
- name: Build tests
run: cargo nextest run --workspace --all-targets --all-features --locked --no-run
- name: Run repeatedly
shell: bash
run: |
n="${{ github.event.inputs.iterations || '5' }}"
failed=0
for i in $(seq 1 "$n"); do
echo "::group::run $i/$n (threads=${{ matrix.threads }})"
if ! cargo nextest run --workspace --all-targets --all-features --locked \
--profile stress --test-threads "${{ matrix.threads }}"; then
failed=$((failed + 1))
fi
echo "::endgroup::"
done
echo "failed runs: $failed / $n" | tee -a "$GITHUB_STEP_SUMMARY"
exit "$failed"
Loading
Loading