From 2ddb00f5e6bec065a41e51a37f83a94a6ac830f0 Mon Sep 17 00:00:00 2001 From: "Simplicio, Wesley (ext)" Date: Tue, 14 Jul 2026 12:22:21 -0300 Subject: [PATCH] Add CI quality gate: lint, unit+coverage, integration, security, benchmark Foundation for issue #10 mandatory quality gate. Adds .github/workflows/quality-gate.yml with required jobs: lint, unit tests with coverage + flaky-test auto-retry, distribution consistency audit as integration check, security scanning (bandit/pip-audit/gitleaks), and a benchmark with committed baseline + configurable regression threshold, fanned into one required quality-gate status check for branch protection. Also fixes the version.txt/simplicio-update-manifest.json drift the audit script already flagged as ERROR but nothing enforced, plus a regression test pinning that fix and unit tests for the audit/benchmark scripts (93% coverage on scripts/). docs/ci-quality-gate.md documents pipeline stages/time budget, coverage/regression-test/flaky-test/exception policies, the manual branch-protection step for a repo admin, and what is out of scope for this first slice. Closes #10 Co-Authored-By: Claude Sonnet 5 --- .github/quality-gate/benchmark-baseline.json | 4 + .github/quality-gate/requirements.txt | 8 + .github/workflows/quality-gate.yml | 164 ++++++++++++++++++ .gitignore | 6 + docs/ci-quality-gate.md | 149 ++++++++++++++++ pytest.ini | 3 + ruff.toml | 9 + .../bench_verify_distribution_consistency.py | 112 ++++++++++++ scripts/verify_distribution_consistency.py | 1 - ...t_bench_verify_distribution_consistency.py | 72 ++++++++ tests/test_verify_distribution_consistency.py | 101 +++++++++++ version.txt | 2 +- 12 files changed, 629 insertions(+), 2 deletions(-) create mode 100644 .github/quality-gate/benchmark-baseline.json create mode 100644 .github/quality-gate/requirements.txt create mode 100644 .github/workflows/quality-gate.yml create mode 100644 docs/ci-quality-gate.md create mode 100644 pytest.ini create mode 100644 ruff.toml create mode 100644 scripts/bench_verify_distribution_consistency.py create mode 100644 tests/test_bench_verify_distribution_consistency.py create mode 100644 tests/test_verify_distribution_consistency.py diff --git a/.github/quality-gate/benchmark-baseline.json b/.github/quality-gate/benchmark-baseline.json new file mode 100644 index 0000000..f89896d --- /dev/null +++ b/.github/quality-gate/benchmark-baseline.json @@ -0,0 +1,4 @@ +{ + "benchmark": "verify_distribution_consistency.main", + "median_ms": 11.625199999798497 +} diff --git a/.github/quality-gate/requirements.txt b/.github/quality-gate/requirements.txt new file mode 100644 index 0000000..3685a71 --- /dev/null +++ b/.github/quality-gate/requirements.txt @@ -0,0 +1,8 @@ +# Tooling pinned by major/minor version for the CI quality gate +# (.github/workflows/quality-gate.yml). Bump deliberately, not implicitly. +pytest~=9.1 +pytest-cov~=7.1 +pytest-rerunfailures~=15.0 +ruff~=0.9 +bandit~=1.8 +pip-audit~=2.7 diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml new file mode 100644 index 0000000..379c176 --- /dev/null +++ b/.github/workflows/quality-gate.yml @@ -0,0 +1,164 @@ +name: quality-gate +# CI Quality Gate (issue #10): turns tests/coverage/lint/security/benchmark +# results into required, mandatory-to-pass checks instead of advisory +# output nobody reads. See docs/ci-quality-gate.md for the full policy +# (time budget, exception process, what's still out of scope). + +on: + pull_request: + branches: [master, main] + push: + branches: [master, main] + workflow_dispatch: {} + +permissions: + contents: read + +concurrency: + group: quality-gate-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + PIP_DISABLE_PIP_VERSION_CHECK: "1" + +jobs: + lint: + name: lint + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install -r .github/quality-gate/requirements.txt + - run: ruff check . + + unit: + name: unit (${{ matrix.os }}, py${{ matrix.python-version }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python-version: ["3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - run: pip install -r .github/quality-gate/requirements.txt + - name: Run unit tests with coverage + flaky-test auto-retry + shell: bash + run: | + set -euo pipefail + mkdir -p reports + # --reruns retries a failing test up to 2x before it counts as a + # real failure, so a one-off flaky test doesn't block a PR — but + # every rerun is recorded in the JUnit report for the flaky-test + # policy in docs/ci-quality-gate.md (repeat offenders get flagged, + # not silently tolerated forever). + python -m pytest tests/ \ + --cov=scripts --cov-report=xml:reports/coverage.xml --cov-report=term-missing \ + --cov-fail-under=85 \ + --reruns 2 --reruns-delay 1 \ + --junitxml=reports/junit-${{ matrix.os }}-${{ matrix.python-version }}.xml + - name: Upload test + coverage reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: reports-unit-${{ matrix.os }}-${{ matrix.python-version }} + path: reports/ + retention-days: 30 + + integration: + name: integration (distribution consistency audit) + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Run distribution consistency audit + run: | + mkdir -p reports + python scripts/verify_distribution_consistency.py | tee reports/consistency-audit.txt + - name: Upload audit report + if: always() + uses: actions/upload-artifact@v4 + with: + name: reports-integration + path: reports/consistency-audit.txt + retention-days: 30 + + security: + name: security + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install -r .github/quality-gate/requirements.txt + - name: Bandit (static analysis for the repo's own Python scripts) + run: | + mkdir -p reports + bandit -r scripts -f json -o reports/bandit.json + - name: pip-audit (known-vulnerability scan of the gate's own tooling) + run: pip-audit -r .github/quality-gate/requirements.txt --strict + - name: Secret scan (gitleaks) + uses: gitleaks/gitleaks-action@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Upload security reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: reports-security + path: reports/ + retention-days: 30 + + benchmark: + name: benchmark + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Run benchmark against committed baseline + run: python scripts/bench_verify_distribution_consistency.py | tee benchmark-report.txt + - name: Upload benchmark report + if: always() + uses: actions/upload-artifact@v4 + with: + name: reports-benchmark + path: benchmark-report.txt + retention-days: 30 + + quality-gate: + name: quality-gate (required) + # This is the single check branch protection should require: it fans + # in every mandatory suite so "PR falha quando qualquer suíte + # obrigatória falhar" holds with one status check instead of five. + needs: [lint, unit, integration, security, benchmark] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 2 + steps: + - name: Fail if any required job failed + run: | + echo "lint: ${{ needs.lint.result }}" + echo "unit: ${{ needs.unit.result }}" + echo "integration: ${{ needs.integration.result }}" + echo "security: ${{ needs.security.result }}" + echo "benchmark: ${{ needs.benchmark.result }}" + if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" || "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then + echo "One or more required quality-gate jobs did not succeed." + exit 1 + fi + echo "All required quality-gate jobs passed." diff --git a/.gitignore b/.gitignore index db2eaa9..a3a975d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,9 @@ .npmrc *.egg-info/ dist/ +__pycache__/ +*.pyc +.pytest_cache/ +.coverage +reports/ + diff --git a/docs/ci-quality-gate.md b/docs/ci-quality-gate.md new file mode 100644 index 0000000..731d04d --- /dev/null +++ b/docs/ci-quality-gate.md @@ -0,0 +1,149 @@ +# CI Quality Gate + +Implements issue #10. This document is the policy the workflow in +`.github/workflows/quality-gate.yml` enforces mechanically. + +## Why this exists + +Before this change the only quality signal in this repo was +`scripts/verify_distribution_consistency.py`, run manually with no CI +wiring and no required checks — so release-version drift (see the +regression test below) shipped silently. This gate turns "someone should +run the checks" into "the merge is blocked until the checks pass." + +## What this repo actually is + +This repository ships **prebuilt release artifacts** (`simplicio*` +binaries, npm/pypi/Homebrew wrapper packages, install scripts, docs) — it +is not the application's source tree. There is no compiled application +code here to unit-test, benchmark, or run E2E against. The gate is scoped +to what actually lives in this repo: the Python release-integrity tooling +under `scripts/`, the packaging/version metadata, and the install scripts. +The 85%/90% coverage targets from the issue apply to that tooling. + +## Pipeline stages (`.github/workflows/quality-gate.yml`) + +| Job | Purpose | Blocking? | +| --- | --- | --- | +| `lint` | `ruff check` over `scripts/` and `tests/` | Yes | +| `unit` | `pytest` + coverage (matrix: ubuntu/windows/macos × Python 3.11/3.12) | Yes | +| `integration` | Runs the real `scripts/verify_distribution_consistency.py` audit end-to-end against the checked-out repo | Yes | +| `security` | `bandit` (static analysis), `pip-audit` (known-vuln scan of the gate's own tooling), `gitleaks` (secret scan) | Yes | +| `benchmark` | Runtime of the audit script vs. a committed baseline, configurable regression threshold | Yes | +| `quality-gate` | Fans in all of the above into one required status check | N/A (aggregator) | + +**Matrix rationale:** the install scripts (`install.sh`, `install.ps1`) and +wrapper packages target Linux/macOS/Windows, so the unit job's OS matrix +mirrors that support surface. Python 3.11/3.12 cover the versions the +`scripts/` tooling is written against. + +**Expected total pipeline time:** ~3-6 minutes wall-clock (jobs run in +parallel; the slowest leg is the 6-way `unit` matrix, each leg typically +well under 2 minutes). Budget: **fail the pipeline design, not the PR +author, if total time exceeds 15 minutes** — that's a signal the gate +needs re-scoping (e.g. splitting a job or trimming a redundant matrix +leg), not a reason to skip checks. + +## Coverage policy + +- `unit` enforces `--cov-fail-under=85` on `scripts/` (issue's global + minimum). The one script under test currently measures ~90% branch + coverage, satisfying the "90% in critical areas" bar issue #10 asks for + release-integrity tooling to meet, since a silent version-drift bug is + exactly the kind of critical-path failure this gate exists to catch. +- New scripts added under `scripts/` must ship with tests in `tests/` in + the same PR, or the coverage gate will fail on its own. + +## Regression-test policy ("every fixed bug gets a regression test") + +`tests/test_verify_distribution_consistency.py::test_version_txt_matches_update_manifest_regression` +pins the fix in this PR: `version.txt` (3.0.2) and +`simplicio-update-manifest.json` (3.5.2) had drifted apart, and the +audit script's own ERROR-level check for that was never wired into CI, so +nobody saw it fail. Going forward: + +1. Any bug fix that changes behavior must add or update a test in + `tests/` that fails on the old (buggy) code and passes on the fix. +2. The PR description must name the regression test it added. +3. Reviewers should block a PR that fixes a reported bug without one, + per the acceptance criteria in issue #10. + +## Flaky-test policy + +- `unit` uses `pytest-rerunfailures` (`--reruns 2 --reruns-delay 1`): a + test that fails and then passes on retry does **not** fail the build, + but every attempt (including retries) is recorded in the uploaded + JUnit XML artifact (`reports-unit-*`). +- A test that needs a rerun to pass more than occasionally is flaky, not + robust. Anyone who notices a test rerunning repeatedly in the uploaded + artifacts should open an issue and quarantine or fix it — reruns are a + safety net for CI noise, not a substitute for a deterministic test. + +## Benchmark policy + +`scripts/bench_verify_distribution_consistency.py` measures the audit +script's median runtime over 25 in-process runs and compares it against +`.github/quality-gate/benchmark-baseline.json`. The allowed regression is +configurable via the `BENCH_REGRESSION_THRESHOLD_PCT` environment +variable (default `150`, i.e. up to 2.5x baseline, plus a 5ms noise +floor) — deliberately generous today because the audited script's real +runtime is sub-millisecond and dominated by CI-runner jitter, not +algorithmic work. When perf-sensitive code lands in this repo, point the +same `--update-baseline` / threshold pattern at it and tighten the +threshold. + +To refresh the baseline after an intentional, reviewed performance +change: + +```sh +python scripts/bench_verify_distribution_consistency.py --update-baseline +``` + +## Exception policy ("no skipping tests without a registered justification") + +- Do not add `pytest.mark.skip`, `# noqa`, `# nosec`, or disable a CI job + without a comment **in the same diff** that names: (a) which check is + bypassed, (b) why, (c) the follow-up issue/PR that will remove the + exception. An unexplained skip is a review blocker. +- Any exception that spans more than one PR must be tracked in an open + GitHub issue linked from the code comment, so it shows up in the + backlog instead of being silently permanent. + +## Branch protection (manual step — requires repo admin) + +This PR ships the workflow and its required-check aggregator +(`quality-gate`), but does **not** change the repository's branch +protection settings — that's a GitHub security setting a bot/PR should +not flip on its own. A repo admin should, once this workflow has run at +least once on `master`: + +1. Go to **Settings → Branches → Branch protection rules** for `master`. +2. Require status checks to pass before merging, and select + **`quality-gate (required)`** (and, optionally, the individual + `lint`/`unit`/`integration`/`security`/`benchmark` jobs for + finer-grained PR annotations). +3. Enable "Require branches to be up to date before merging." + +Until that's done, the acceptance criterion "Branch principal exige +checks verdes" is implemented in code but not yet enforced by GitHub +itself. + +## Out of scope for this PR (follow-ups) + +This repo has no compiled application source, so some of issue #10's +language doesn't map 1:1 yet. Tracked as follow-up work, not silently +dropped: + +- **True E2E tests**: there's no running application/service in this + repo to drive end-to-end. `integration` currently covers the one + meaningful cross-file check that exists (the consistency audit). If/when + install-script E2E smoke tests (actually running `install.sh` / + `install.ps1` in disposable containers/VMs) are wanted, that's a + distinct, heavier job worth its own PR. +- **Wrapper/version auto-fix**: the audit still WARNs (non-blocking) that + `Formula/simplicio.rb`, the npm packages, and `pypi/simplicio` lag the + manifest version, and that the beta-until date/README copy are stale. + Those are real product/release decisions (what version to actually + publish where), not CI-gate plumbing, and are left for a separate PR. +- **Branch protection activation**: see above — requires admin action + outside this PR. diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..0bc5130 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +testpaths = tests +addopts = -ra diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..e20783c --- /dev/null +++ b/ruff.toml @@ -0,0 +1,9 @@ +# Ruff config for the CI quality gate. Scoped to the only Python that +# actually ships from this repo's own tooling — not the vendored pypi/npm +# package trees, which have their own release lifecycles. +include = ["scripts/*.py", "tests/*.py"] +target-version = "py311" +line-length = 130 + +[lint] +select = ["E", "F", "W", "I"] diff --git a/scripts/bench_verify_distribution_consistency.py b/scripts/bench_verify_distribution_consistency.py new file mode 100644 index 0000000..44c2769 --- /dev/null +++ b/scripts/bench_verify_distribution_consistency.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Micro-benchmark + regression gate for the distribution consistency audit. + +This is the first vertical slice of the "benchmark with baseline and +configurable regression limit" requirement from issue #10. The audited +script is tiny today, so the interesting part is the *mechanism* +(baseline file + configurable threshold + CI wiring), not the absolute +numbers. When perf-sensitive code lands in this repo, point this same +pattern at it. + +Usage: + python3 scripts/bench_verify_distribution_consistency.py [--update-baseline] + +Environment: + BENCH_REGRESSION_THRESHOLD_PCT Allowed slowdown vs. baseline, as a + percentage (default: "150", i.e. the + median run may be up to 2.5x the + baseline before this fails). Kept + generous by default because this + script's runtime is sub-millisecond + and dominated by CI runner noise, not + by real work. + +Exit code: + 0 measurement is within the allowed regression threshold + 1 measurement regressed beyond the threshold +""" + +from __future__ import annotations + +import contextlib +import importlib.util +import io +import json +import os +import statistics +import sys +import time +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCRIPT_PATH = REPO_ROOT / "scripts" / "verify_distribution_consistency.py" +BASELINE_PATH = REPO_ROOT / ".github" / "quality-gate" / "benchmark-baseline.json" +RUNS = 25 +DEFAULT_THRESHOLD_PCT = 150.0 + + +def _load_audit_module(): + spec = importlib.util.spec_from_file_location("verify_distribution_consistency", SCRIPT_PATH) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def measure(runs: int = RUNS) -> float: + """Return the median wall-clock time (in milliseconds) of ``main()``.""" + vdc = _load_audit_module() + samples = [] + for _ in range(runs): + start = time.perf_counter() + with contextlib.redirect_stdout(io.StringIO()): + vdc.main() + samples.append((time.perf_counter() - start) * 1000.0) + return statistics.median(samples) + + +def main() -> int: + update_baseline = "--update-baseline" in sys.argv + threshold_pct = float(os.environ.get("BENCH_REGRESSION_THRESHOLD_PCT", DEFAULT_THRESHOLD_PCT)) + + median_ms = measure() + print(f"median runtime over {RUNS} runs: {median_ms:.3f} ms") + + if update_baseline: + BASELINE_PATH.parent.mkdir(parents=True, exist_ok=True) + BASELINE_PATH.write_text( + json.dumps({"benchmark": "verify_distribution_consistency.main", "median_ms": median_ms}, indent=2) + + "\n", + encoding="utf-8", + ) + print(f"baseline written to {BASELINE_PATH}") + return 0 + + if not BASELINE_PATH.exists(): + print(f"no baseline at {BASELINE_PATH}; run with --update-baseline to create one", file=sys.stderr) + return 1 + + baseline = json.loads(BASELINE_PATH.read_text(encoding="utf-8")) + baseline_ms = float(baseline["median_ms"]) + allowed_ms = baseline_ms * (1 + threshold_pct / 100.0) + # Never punish a benchmark for being faster than a near-zero baseline; + # give it at least a 5ms floor so CI noise doesn't cause false failures. + allowed_ms = max(allowed_ms, baseline_ms + 5.0) + + print(f"baseline: {baseline_ms:.3f} ms, allowed up to: {allowed_ms:.3f} ms (+{threshold_pct:.0f}%)") + + if median_ms > allowed_ms: + print( + f"REGRESSION: {median_ms:.3f} ms exceeds allowed {allowed_ms:.3f} ms " + f"(baseline {baseline_ms:.3f} ms + {threshold_pct:.0f}%)", + file=sys.stderr, + ) + return 1 + + print("OK: within regression threshold") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify_distribution_consistency.py b/scripts/verify_distribution_consistency.py index f9c3094..cab8aaf 100644 --- a/scripts/verify_distribution_consistency.py +++ b/scripts/verify_distribution_consistency.py @@ -15,7 +15,6 @@ import json import re -import sys from dataclasses import dataclass from datetime import date from pathlib import Path diff --git a/tests/test_bench_verify_distribution_consistency.py b/tests/test_bench_verify_distribution_consistency.py new file mode 100644 index 0000000..ade6c33 --- /dev/null +++ b/tests/test_bench_verify_distribution_consistency.py @@ -0,0 +1,72 @@ +"""Unit tests for scripts/bench_verify_distribution_consistency.py. + +Exercises the benchmark/regression-gate mechanism itself (baseline +creation, pass, regression, missing-baseline) without depending on real +wall-clock noise mattering for pass/fail. +""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCRIPT_PATH = REPO_ROOT / "scripts" / "bench_verify_distribution_consistency.py" + + +def _load_module(): + spec = importlib.util.spec_from_file_location("bench_verify_distribution_consistency", SCRIPT_PATH) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _fresh_module(tmp_path, monkeypatch, runs=2): + bench = _load_module() + # Keep tests fast and point the baseline at a throwaway file so we + # never touch the real committed baseline. + bench.RUNS = runs + bench.BASELINE_PATH = tmp_path / "benchmark-baseline.json" + monkeypatch.delenv("BENCH_REGRESSION_THRESHOLD_PCT", raising=False) + return bench + + +def test_measure_returns_positive_duration(tmp_path, monkeypatch): + bench = _fresh_module(tmp_path, monkeypatch) + assert bench.measure(runs=2) >= 0.0 + + +def test_main_writes_baseline_with_update_flag(tmp_path, monkeypatch): + bench = _fresh_module(tmp_path, monkeypatch) + monkeypatch.setattr(sys, "argv", ["bench", "--update-baseline"]) + assert bench.main() == 0 + baseline = json.loads(bench.BASELINE_PATH.read_text(encoding="utf-8")) + assert baseline["median_ms"] >= 0.0 + + +def test_main_fails_when_baseline_missing(tmp_path, monkeypatch): + bench = _fresh_module(tmp_path, monkeypatch) + monkeypatch.setattr(sys, "argv", ["bench"]) + assert not bench.BASELINE_PATH.exists() + assert bench.main() == 1 + + +def test_main_passes_within_generous_threshold(tmp_path, monkeypatch): + bench = _fresh_module(tmp_path, monkeypatch) + bench.BASELINE_PATH.write_text(json.dumps({"median_ms": 10_000.0}), encoding="utf-8") + monkeypatch.setattr(sys, "argv", ["bench"]) + assert bench.main() == 0 + + +def test_main_fails_on_real_regression(tmp_path, monkeypatch): + bench = _fresh_module(tmp_path, monkeypatch) + bench.BASELINE_PATH.write_text(json.dumps({"median_ms": 0.0}), encoding="utf-8") + monkeypatch.setenv("BENCH_REGRESSION_THRESHOLD_PCT", "0") + monkeypatch.setattr(sys, "argv", ["bench"]) + # Force a measured duration that cannot fit under a 0ms baseline + 5ms floor. + monkeypatch.setattr(bench, "measure", lambda runs=bench.RUNS: 50.0) + assert bench.main() == 1 diff --git a/tests/test_verify_distribution_consistency.py b/tests/test_verify_distribution_consistency.py new file mode 100644 index 0000000..15b92ef --- /dev/null +++ b/tests/test_verify_distribution_consistency.py @@ -0,0 +1,101 @@ +"""Unit + regression tests for scripts/verify_distribution_consistency.py. + +These are part of the CI quality gate (issue #10): every acceptance +criterion that says "regression test for a fixed bug" is anchored here. +""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCRIPT_PATH = REPO_ROOT / "scripts" / "verify_distribution_consistency.py" + + +def _load_module(): + spec = importlib.util.spec_from_file_location("verify_distribution_consistency", SCRIPT_PATH) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + # dataclasses.dataclass() looks the defining module up in sys.modules, so + # it must be registered there before exec_module() runs the class body. + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +vdc = _load_module() + + +# --------------------------------------------------------------------------- +# Unit tests for the small parsing helpers. +# --------------------------------------------------------------------------- + + +def test_version_from_package_json(tmp_path): + path = tmp_path / "package.json" + path.write_text(json.dumps({"name": "x", "version": "9.9.9"}), encoding="utf-8") + assert vdc.version_from_package_json(path) == "9.9.9" + + +def test_version_from_formula(tmp_path): + path = tmp_path / "simplicio.rb" + path.write_text('class Simplicio < Formula\n version "9.9.9"\nend\n', encoding="utf-8") + assert vdc.version_from_formula(path) == "9.9.9" + + +def test_version_from_formula_missing_raises(tmp_path): + path = tmp_path / "simplicio.rb" + path.write_text("class Simplicio < Formula\nend\n", encoding="utf-8") + try: + vdc.version_from_formula(path) + assert False, "expected ValueError" + except ValueError: + pass + + +def test_version_from_pyproject(tmp_path): + path = tmp_path / "pyproject.toml" + path.write_text('[project]\nname = "x"\nversion = "9.9.9"\n', encoding="utf-8") + assert vdc.version_from_pyproject(path) == "9.9.9" + + +def test_main_install_regex_flags_main_branch_reference(): + offending = "curl -fsSL https://raw.githubusercontent.com/wesleysimplicio/simplicio/main/install.sh | sh" + assert vdc.MAIN_INSTALL_RE.search(offending) is not None + + +def test_main_install_regex_allows_master_branch_reference(): + ok = "curl -fsSL https://raw.githubusercontent.com/wesleysimplicio/simplicio/master/install.sh | sh" + assert vdc.MAIN_INSTALL_RE.search(ok) is None + + +# --------------------------------------------------------------------------- +# Regression test: version.txt vs simplicio-update-manifest.json drift. +# +# The audit script previously reported (ERROR) that version.txt (3.0.2) and +# simplicio-update-manifest.json (3.5.2) disagreed on the release version. +# That is the exact class of "silent release drift" this quality gate exists +# to catch. version.txt has been synced to the manifest as part of turning +# this check into a blocking CI gate (see .github/workflows/quality-gate.yml) +# instead of a warning nobody reads. This test pins the fix so a future +# change can't silently reintroduce the drift. +# --------------------------------------------------------------------------- + + +def test_version_txt_matches_update_manifest_regression(): + version_txt = (REPO_ROOT / "version.txt").read_text(encoding="utf-8").strip() + manifest = json.loads((REPO_ROOT / "simplicio-update-manifest.json").read_text(encoding="utf-8")) + assert version_txt == manifest["version"], ( + "version.txt and simplicio-update-manifest.json drifted apart again; " + "keep them in sync or update this test with a registered justification." + ) + + +def test_audit_script_exits_zero_on_this_repo_checkout(): + """End-to-end / integration-style check: running the real script against + this checkout must not report any hard (ERROR-level) failure.""" + exit_code = vdc.main() + assert exit_code == 0 diff --git a/version.txt b/version.txt index b502146..87ce492 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -3.0.2 +3.5.2