From 3307ccfd2f1c8415da2428036e91082ba78bb5ee Mon Sep 17 00:00:00 2001 From: "Simplicio, Wesley (ext)" Date: Tue, 14 Jul 2026 12:19:05 -0300 Subject: [PATCH] test: add unit test suite for distribution-consistency checker Kick off the testing epic (issue #8) with a first fully-covered slice. Closes #8 --- .github/workflows/tests.yml | 41 ++++ .gitignore | 3 + CONTRIBUTING.md | 38 +++ README.md | 17 ++ docs/testing-strategy.md | 91 +++++++ pyproject.toml | 11 + requirements-dev.txt | 4 + tests/unit/conftest.py | 154 ++++++++++++ .../test_verify_distribution_consistency.py | 224 ++++++++++++++++++ 9 files changed, 583 insertions(+) create mode 100644 .github/workflows/tests.yml create mode 100644 CONTRIBUTING.md create mode 100644 docs/testing-strategy.md create mode 100644 pyproject.toml create mode 100644 requirements-dev.txt create mode 100644 tests/unit/conftest.py create mode 100644 tests/unit/test_verify_distribution_consistency.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..7201b0f --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,41 @@ +name: unit-tests + +# Runs the repo's Python unit test suite (tests/unit) against the tooling +# scripts under scripts/. This does NOT build, check, or test the Rust +# `simplicio` binaries — those are committed release artifacts in this +# distribution repo, not source built here. +on: + push: + branches: [master, main] + paths: + - "scripts/**" + - "tests/**" + - "pyproject.toml" + - "requirements-dev.txt" + - ".github/workflows/tests.yml" + pull_request: + paths: + - "scripts/**" + - "tests/**" + - "pyproject.toml" + - "requirements-dev.txt" + - ".github/workflows/tests.yml" + workflow_dispatch: {} + +permissions: + contents: read + +jobs: + unit-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install test dependencies + run: pip install -r requirements-dev.txt + - name: Run unit tests with coverage gate + run: > + python -m pytest tests/unit -v + --cov=scripts --cov-report=term-missing --cov-fail-under=85 diff --git a/.gitignore b/.gitignore index db2eaa9..43a20f3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ .npmrc *.egg-info/ dist/ +.pytest_cache/ +.coverage +__pycache__/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..c596905 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,38 @@ +# Contributing + +This repo is the public **distribution** repo for `simplicio`: committed +release binaries plus the packaging/tooling around them (npm/pypi/Homebrew +wrappers, install scripts, docs, release manifests). It does not contain the +agent's Rust source, so there is no `cargo test`/`cargo check`/`cargo build` +here. + +## Running tests + +Official test command for this repo's Python tooling +(`scripts/verify_distribution_consistency.py` today; more modules will be +added under `tests/unit` as the testing epic — issue #8 — progresses): + +```bash +pip install -r requirements-dev.txt +python -m pytest tests/unit -v --cov=scripts --cov-report=term-missing --cov-fail-under=85 +``` + +- Test layout: `tests/unit/`, one `test_.py` per `scripts/.py`, + shared fixtures/builders in `tests/unit/conftest.py`. +- Tests must be deterministic and isolated: no network access, no dependence + on the real wall clock (freeze `datetime.date` via `monkeypatch` if a + function reads it), and no reliance on execution order — each test builds + its own throwaway fixture tree under pytest's `tmp_path`. +- CI runs the same command via `.github/workflows/tests.yml` on every push/PR + touching `scripts/`, `tests/`, or the test config. + +See [docs/testing-strategy.md](docs/testing-strategy.md) for the full +rationale, current coverage, and what's left for the rest of the epic. + +## Making changes + +- Keep PRs scoped; this repo mixes release artifacts with tooling/docs, so + please don't bundle binary/version bumps with unrelated tooling changes. +- If you touch `scripts/verify_distribution_consistency.py`, add/update the + corresponding test(s) in `tests/unit/test_verify_distribution_consistency.py` + in the same PR. diff --git a/README.md b/README.md index fd1ad1c..13ac498 100644 --- a/README.md +++ b/README.md @@ -200,6 +200,23 @@ simplicio license status --- +## 🧪 Testing this repo's tooling + +This repo ships committed release binaries plus the packaging/tooling around +them (npm/pypi/Homebrew wrappers, install scripts, a distribution-consistency +checker). The official command to run that tooling's unit test suite: + +```bash +pip install -r requirements-dev.txt +python -m pytest tests/unit -v --cov=scripts --cov-report=term-missing --cov-fail-under=85 +``` + +See [docs/testing-strategy.md](docs/testing-strategy.md) for what's covered, +what's intentionally out of scope, and the plan for the rest of the testing +epic. See also [CONTRIBUTING.md](CONTRIBUTING.md). + +--- + ## 🌐 Ecosystem - [Website](https://simpleti.com.br/simplicio/#start) — full docs, benchmarks, install diff --git a/docs/testing-strategy.md b/docs/testing-strategy.md new file mode 100644 index 0000000..e885789 --- /dev/null +++ b/docs/testing-strategy.md @@ -0,0 +1,91 @@ +# Testing strategy (this repo) + +Tracks issue #8 ("[Quality Epic] Implantar estratégia completa de testes +automatizados no pacote principal"). This file documents what is tested here, +how, and what is intentionally out of scope, plus the plan for finishing the +epic across multiple PRs. + +## What this repo actually is + +`wesleysimplicio/simplicio` is the **public distribution repo**: committed +release binaries (`simplicio.exe`, `simplicio-linux-x64`, ...), package-manager +wrappers (`npm/`, `pypi/`, `Formula/`), install scripts (`install.sh`, +`install.ps1`), and docs/manifests describing a release. There is no Rust +source (`Cargo.toml`, `*.rs`) checked into this repo — the actual `simplicio` +agent source lives elsewhere. That means: + +- We do **not** run `cargo test` / `cargo check` / `cargo build` here — there + is nothing for cargo to build. +- The "pacote principal" testable from *this* repo is its own tooling: the + Python consistency-checker script and, longer-term, the npm installer + scripts and PowerShell/bash installers. + +## Scope of this PR + +- `scripts/verify_distribution_consistency.py` — the packaging/version-drift + auditor — now has a full unit test suite: + `tests/unit/test_verify_distribution_consistency.py` plus a fixture builder + in `tests/unit/conftest.py` (`RepoBuilder`) that assembles a throwaway fake + repo tree per test under pytest's `tmp_path`. +- Every finding branch in the script is covered: happy path, each `ERROR` + case (branch policy text missing, `/main/` references, version.txt/manifest + mismatch), each `WARN` case (wrapper version drift, ecosystem doc drift, + stale/unparseable beta date, contradictory beta copy), and the parser + helpers' error paths (malformed Formula/pyproject version strings). +- The suite is deterministic: `date.today()` is frozen via monkeypatch in + every test that touches the beta-date check, so no test depends on the + real wall clock. No test touches the network or mutates the real repo — + everything runs inside `tmp_path`. +- Coverage: `scripts/verify_distribution_consistency.py` is at 98% line + coverage from this suite (`--cov-fail-under=85` gate; see below), which + clears the 85%/90% acceptance thresholds for this module. +- CI: `.github/workflows/tests.yml` runs the suite (with the coverage gate) + on every push/PR that touches `scripts/`, `tests/`, or the test config, + independent of the existing binary-release workflow. + +## Running and debugging the suite + +```bash +pip install -r requirements-dev.txt +python -m pytest tests/unit -v --cov=scripts --cov-report=term-missing --cov-fail-under=85 +``` + +- Run a single test: `python -m pytest tests/unit -k stale_beta_until -v` +- Drop into a debugger on failure: `python -m pytest tests/unit --pdb` +- The suite never depends on execution order — each test builds its own + `tmp_path` repo tree via the `repo` fixture, so tests may run in any order + or in parallel (`pytest -p xdist -n auto`, once `pytest-xdist` is added). + +## Known pre-existing drift (not fixed by this PR) + +Running `python scripts/verify_distribution_consistency.py` against the real +repo currently reports real ERROR/WARN findings (e.g. `version.txt` vs. +`simplicio-update-manifest.json` mismatch, stale `beta_until`). Fixing that +drift is a separate, unrelated content/release-hygiene fix, not a testing +concern — this PR verifies the *checker* is correct, it does not silence or +paper over what the checker (correctly) flags. The CI job added here is +scoped to `scripts/`/`tests/` changes rather than every push, specifically so +it doesn't block unrelated work on this pre-existing, already-flagged drift. + +## Remaining work for this epic (tracked for follow-up PRs) + +This issue is explicitly an epic ("cobertura mínima de 85%/90% no projeto +inteiro, branch coverage monitorada, todos os módulos críticos"), which is +larger than one PR. Suggested follow-ups, roughly in priority order: + +1. **npm installers** (`npm/simplicio/install.js`, + `npm/simplicio-installer/install.js`, `npm/simplicio-unscoped/install.js`): + add a Node test runner (`node --test` or `vitest`) with fakes for + filesystem/network calls (these currently hit the real npm registry / + GitHub releases — needs dependency injection or mocking before they're + testable in isolation). +2. **Install scripts** (`install.sh`, `install.ps1`): add `bats`/Pester-based + tests against a faked download/checksum step. +3. **Branch coverage**: wire `coverage.py`'s branch mode + (`--cov-branch`) once module #1 above exists, so the 85%/90% target is + measured project-wide rather than per-module. +4. Add a repo-wide coverage roll-up job once (1) and (2) land, and only then + close out issue #8 — right now this PR closes the *initial slice* + (convention, fixtures, first fully-covered module, CI wiring, docs) per + the issue's own step 4 ("Adicionar testes por módulo começando pelos + componentes de maior risco"), not the entire epic. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..ea9b5ed --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,11 @@ +[tool.pytest.ini_options] +testpaths = ["tests/unit"] +addopts = "-ra" + +[tool.coverage.run] +source = ["scripts"] +omit = ["scripts/__pycache__/*"] + +[tool.coverage.report] +show_missing = true +skip_empty = true diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..fbb6813 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,4 @@ +# Dev/test-only dependencies for this distribution repo's Python tooling +# (scripts/*.py). Not required to install, build, or ship simplicio itself. +pytest>=8.0 +pytest-cov>=5.0 diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py new file mode 100644 index 0000000..9913b5a --- /dev/null +++ b/tests/unit/conftest.py @@ -0,0 +1,154 @@ +"""Shared fixtures for the `simplicio` distribution-repo unit test suite. + +These tests are fully isolated: every test builds its own throwaway repo +tree under ``tmp_path`` (pytest's built-in fixture), so nothing here reads +or writes real repository files, touches the network, or depends on wall +clock time unless a test explicitly freezes it. +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any + +import pytest + +SCRIPTS_DIR = Path(__file__).resolve().parents[2] / "scripts" +if str(SCRIPTS_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPTS_DIR)) + +DEFAULT_VERSION = "3.5.2" + + +def _write(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def _write_json(path: Path, data: dict) -> None: + _write(path, json.dumps(data)) + + +class RepoBuilder: + """Builds a minimal, valid fake copy of the distribution repo layout. + + Call the individual `with_*` helpers to mutate a single file away from + the "everything consistent" baseline for a given test scenario, then + call `root` (or just use the returned Path) to point the module at it. + """ + + def __init__(self, root: Path): + self.root = root + self.version = DEFAULT_VERSION + + def build(self) -> "RepoBuilder": + root = self.root + v = self.version + + _write(root / "VERSION.md", "# Version policy\n\nUse `master` branch only for installs.\n") + _write(root / "version.txt", v) + _write_json( + root / "simplicio-update-manifest.json", + { + "version": v, + "entitlement": {"beta_until": "2099-01-01"}, + }, + ) + + install_body = ( + "Install:\n" + "curl -fsSL https://raw.githubusercontent.com/wesleysimplicio/simplicio/master/install.sh | sh\n" + ) + _write(root / "README.md", install_body) + _write(root / "INSTALL.md", install_body) + _write(root / "install.sh", install_body) + _write(root / "install.ps1", install_body.replace("install.sh", "install.ps1")) + _write(root / ".github/workflows/release.yml", "name: release\n") + _write(root / "pypi/simplicio/simplicio/__main__.py", "# entrypoint\n") + (root / "READMEs").mkdir(parents=True, exist_ok=True) + + _write(root / "Formula/simplicio.rb", f'class Simplicio < Formula\n version "{v}"\nend\n') + _write_json(root / "npm/simplicio/package.json", {"version": v}) + _write_json(root / "npm/simplicio-installer/package.json", {"version": v}) + _write_json(root / "npm/simplicio-unscoped/package.json", {"version": v}) + _write(root / "pypi/simplicio/pyproject.toml", f'[project]\nname = "simplicio"\nversion = "{v}"\n') + + _write(root / "SIMPLICIO_ECOSYSTEM.md", f"## Versão atual\n{v}\n") + + return self + + # -- scenario mutators ------------------------------------------------- + + def with_version_txt(self, value: str) -> "RepoBuilder": + _write(self.root / "version.txt", value) + return self + + def with_manifest(self, **overrides: Any) -> "RepoBuilder": + path = self.root / "simplicio-update-manifest.json" + data = json.loads(path.read_text(encoding="utf-8")) + data.update(overrides) + _write_json(path, data) + return self + + def with_beta_until(self, value) -> "RepoBuilder": + path = self.root / "simplicio-update-manifest.json" + data = json.loads(path.read_text(encoding="utf-8")) + data.setdefault("entitlement", {})["beta_until"] = value + _write_json(path, data) + return self + + def with_version_md_missing_master_clause(self) -> "RepoBuilder": + _write(self.root / "VERSION.md", "# Version policy\n\nNo canonical branch declared.\n") + return self + + def with_main_branch_reference(self, filename: str = "README.md") -> "RepoBuilder": + _write( + self.root / filename, + "Install:\ncurl -fsSL https://raw.githubusercontent.com/wesleysimplicio/simplicio/main/install.sh | sh\n", + ) + return self + + def with_formula_version(self, value: str) -> "RepoBuilder": + _write(self.root / "Formula/simplicio.rb", f'class Simplicio < Formula\n version "{value}"\nend\n') + return self + + def with_formula_unparseable(self) -> "RepoBuilder": + _write(self.root / "Formula/simplicio.rb", "class Simplicio < Formula\nend\n") + return self + + def with_npm_version(self, package: str, value: str) -> "RepoBuilder": + _write_json(self.root / f"npm/{package}/package.json", {"version": value}) + return self + + def with_pyproject_version(self, value: str) -> "RepoBuilder": + _write(self.root / "pypi/simplicio/pyproject.toml", f'[project]\nname = "simplicio"\nversion = "{value}"\n') + return self + + def with_pyproject_unparseable(self) -> "RepoBuilder": + _write(self.root / "pypi/simplicio/pyproject.toml", '[project]\nname = "simplicio"\n') + return self + + def with_ecosystem_version(self, value: str) -> "RepoBuilder": + _write(self.root / "SIMPLICIO_ECOSYSTEM.md", f"## Versão atual\n{value}\n") + return self + + def with_readme_beta_no_end_date(self) -> "RepoBuilder": + path = self.root / "README.md" + text = path.read_text(encoding="utf-8") + _write(path, text + "\nWe are in public beta with no end date.\n") + return self + + +@pytest.fixture +def repo(tmp_path: Path) -> RepoBuilder: + """A consistent, passing fake repo tree ready for scenario mutation.""" + return RepoBuilder(tmp_path).build() + + +@pytest.fixture +def verify_module(): + """Import (or reuse) the module under test, isolated from sys.path pollution.""" + import verify_distribution_consistency as module + + return module diff --git a/tests/unit/test_verify_distribution_consistency.py b/tests/unit/test_verify_distribution_consistency.py new file mode 100644 index 0000000..a49ba50 --- /dev/null +++ b/tests/unit/test_verify_distribution_consistency.py @@ -0,0 +1,224 @@ +"""Unit tests for scripts/verify_distribution_consistency.py. + +Coverage intent (see issue #8 acceptance criteria): +- happy path (all sources agree) exits 0 with no ERROR findings +- each ERROR-producing drift is exercised in isolation +- each WARN-producing drift is exercised in isolation +- parser helpers raise on malformed input (error/edge cases) +- the beta-date check is deterministic: real wall-clock time is never + read directly by a test — we freeze `date.today()` via monkeypatch. + +No test touches the network, a real clock, or global state outside of its +own tmp_path repo tree, satisfying the "no test depends on network, real +clock, or unisolated global state" acceptance criterion. +""" +from __future__ import annotations + +import datetime as real_datetime +import json + +import pytest + + +def run_main(module, repo, capsys): + module.ROOT = repo.root + exit_code = module.main() + captured = capsys.readouterr().out + return exit_code, captured + + +def freeze_today(monkeypatch, module, iso_date: str) -> None: + frozen = real_datetime.date.fromisoformat(iso_date) + + class _FrozenDate(real_datetime.date): + @classmethod + def today(cls): + return frozen + + monkeypatch.setattr(module, "date", _FrozenDate) + + +# --------------------------------------------------------------------------- +# Happy path +# --------------------------------------------------------------------------- + + +def test_happy_path_all_consistent_exits_zero(verify_module, repo, monkeypatch, capsys): + freeze_today(monkeypatch, verify_module, "2020-01-01") # well before beta_until + exit_code, out = run_main(verify_module, repo, capsys) + + assert exit_code == 0 + assert "[ERROR]" not in out + assert "0 error(s)" in out + assert "all public install references use the canonical `master` branch" in out + assert "release version sources agree on 3.5.2" in out + assert "wrapper/package versions match the release manifest" in out + + +# --------------------------------------------------------------------------- +# ERROR cases +# --------------------------------------------------------------------------- + + +def test_version_md_missing_master_clause_is_error(verify_module, repo, monkeypatch, capsys): + repo.with_version_md_missing_master_clause() + freeze_today(monkeypatch, verify_module, "2020-01-01") + + exit_code, out = run_main(verify_module, repo, capsys) + + assert exit_code == 1 + assert "no longer declares `master` as canonical branch" in out + + +@pytest.mark.parametrize( + "filename", ["README.md", "INSTALL.md", "install.sh", "install.ps1"] +) +def test_main_branch_reference_is_error(verify_module, repo, monkeypatch, capsys, filename): + repo.with_main_branch_reference(filename) + freeze_today(monkeypatch, verify_module, "2020-01-01") + + exit_code, out = run_main(verify_module, repo, capsys) + + assert exit_code == 1 + assert "install references still point at `/main/`" in out + assert filename in out + + +def test_version_txt_manifest_mismatch_is_error(verify_module, repo, monkeypatch, capsys): + repo.with_version_txt("9.9.9") + freeze_today(monkeypatch, verify_module, "2020-01-01") + + exit_code, out = run_main(verify_module, repo, capsys) + + assert exit_code == 1 + assert "version mismatch: version.txt=9.9.9 but simplicio-update-manifest.json=3.5.2" in out + + +# --------------------------------------------------------------------------- +# WARN cases (non-fatal drift) +# --------------------------------------------------------------------------- + + +def test_wrapper_version_drift_is_warning_not_error(verify_module, repo, monkeypatch, capsys): + repo.with_formula_version("1.0.0") + freeze_today(monkeypatch, verify_module, "2020-01-01") + + exit_code, out = run_main(verify_module, repo, capsys) + + assert exit_code == 0 # warnings alone must not fail the build + assert "wrapper/package versions lag manifest 3.5.2" in out + assert "Formula/simplicio.rb=1.0.0" in out + + +def test_ecosystem_doc_version_drift_is_warning(verify_module, repo, monkeypatch, capsys): + repo.with_ecosystem_version("1.0.0 (package.json)") + freeze_today(monkeypatch, verify_module, "2020-01-01") + + exit_code, out = run_main(verify_module, repo, capsys) + + assert exit_code == 0 + assert "SIMPLICIO_ECOSYSTEM.md advertises" in out + + +def test_stale_beta_until_is_warning(verify_module, repo, monkeypatch, capsys): + repo.with_beta_until("2020-06-30") + freeze_today(monkeypatch, verify_module, "2020-07-14") + + exit_code, out = run_main(verify_module, repo, capsys) + + assert exit_code == 0 + assert "public-beta date is stale" in out + + +def test_beta_until_not_yet_stale_produces_no_warning(verify_module, repo, monkeypatch, capsys): + repo.with_beta_until("2099-06-30") + freeze_today(monkeypatch, verify_module, "2020-07-14") + + exit_code, out = run_main(verify_module, repo, capsys) + + assert exit_code == 0 + assert "public-beta date is stale" not in out + + +def test_unparseable_beta_until_is_warning(verify_module, repo, monkeypatch, capsys): + repo.with_beta_until("not-a-date") + freeze_today(monkeypatch, verify_module, "2020-01-01") + + exit_code, out = run_main(verify_module, repo, capsys) + + assert exit_code == 0 + assert "could not parse beta_until date: not-a-date" in out + + +def test_readme_beta_no_end_date_contradicts_manifest(verify_module, repo, monkeypatch, capsys): + repo.with_beta_until("2099-06-30") + repo.with_readme_beta_no_end_date() + freeze_today(monkeypatch, verify_module, "2020-01-01") + + exit_code, out = run_main(verify_module, repo, capsys) + + assert exit_code == 0 + assert "public beta with no end date" in out.lower() + + +def test_no_beta_no_end_date_warning_when_no_beta_until(verify_module, repo, monkeypatch, capsys): + repo.with_manifest(entitlement={}) + repo.with_readme_beta_no_end_date() + freeze_today(monkeypatch, verify_module, "2020-01-01") + + exit_code, out = run_main(verify_module, repo, capsys) + + # No beta_until at all means nothing to contradict. + assert exit_code == 0 + assert "README says 'public beta with no end date'" not in out + + +# --------------------------------------------------------------------------- +# Parser helper edge/error cases +# --------------------------------------------------------------------------- + + +def test_version_from_formula_parses_double_quoted_version(verify_module, tmp_path): + path = tmp_path / "simplicio.rb" + path.write_text('class Foo < Formula\n version "1.2.3"\nend\n', encoding="utf-8") + + assert verify_module.version_from_formula(path) == "1.2.3" + + +def test_version_from_formula_raises_when_unparseable(verify_module, tmp_path): + path = tmp_path / "simplicio.rb" + path.write_text("class Foo < Formula\nend\n", encoding="utf-8") + + with pytest.raises(ValueError, match="could not parse formula version"): + verify_module.version_from_formula(path) + + +def test_version_from_pyproject_parses_version(verify_module, tmp_path): + path = tmp_path / "pyproject.toml" + path.write_text('[project]\nversion = "4.5.6"\n', encoding="utf-8") + + assert verify_module.version_from_pyproject(path) == "4.5.6" + + +def test_version_from_pyproject_raises_when_unparseable(verify_module, tmp_path): + path = tmp_path / "pyproject.toml" + path.write_text('[project]\nname = "x"\n', encoding="utf-8") + + with pytest.raises(ValueError, match="could not parse pyproject version"): + verify_module.version_from_pyproject(path) + + +def test_version_from_package_json_reads_version_field(verify_module, tmp_path): + path = tmp_path / "package.json" + path.write_text(json.dumps({"version": "7.8.9", "name": "x"}), encoding="utf-8") + + assert verify_module.version_from_package_json(path) == "7.8.9" + + +def test_rel_returns_path_relative_to_root(verify_module, tmp_path): + verify_module.ROOT = tmp_path + nested = tmp_path / "a" / "b.txt" + nested.parent.mkdir(parents=True) + nested.write_text("x", encoding="utf-8") + + assert verify_module.rel(nested) == str(nested.relative_to(tmp_path))