From b581c2d1c4c85dd3df611ab9d9c34f160f7261ed Mon Sep 17 00:00:00 2001 From: "Simplicio, Wesley (ext)" Date: Tue, 14 Jul 2026 12:36:13 -0300 Subject: [PATCH] test: add real E2E integration suite across the ecosystem (issue #9) Adds scripts/e2e_ecosystem.py, exercising the already-compiled simplicio binary end to end in an ephemeral temp repo: clean install/init, valid vs corrupted config loading, project mapping, a full mechanical-edit task (input to final result), error/retry-exhaustion propagation, state persistence/resume across process runs, and compatibility with the published npm/PyPI/Homebrew packages (reusing verify_distribution_consistency.py). Wires it into .github/workflows/e2e.yml (PR + push + workflow_dispatch) so it is reproducible in CI, and documents commands/evidence/known findings in docs/e2e-integration.md. No cargo build/check/test is invoked anywhere. One real drift was found and intentionally left failing as evidence the gate works: version.txt/wrapper packages lag simplicio-update-manifest.json (3.0.2 vs 3.5.2) - documented as a follow-up in docs/e2e-integration.md. Closes #9 --- .github/workflows/e2e.yml | 63 +++++++ docs/e2e-integration.md | 103 ++++++++++++ scripts/e2e_ecosystem.py | 338 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 504 insertions(+) create mode 100644 .github/workflows/e2e.yml create mode 100644 docs/e2e-integration.md create mode 100644 scripts/e2e_ecosystem.py diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..9bced02 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,63 @@ +name: e2e-ecosystem + +# Real integration/E2E checks against the already-compiled `simplicio` +# binary committed in this repo (see docs/e2e-integration.md). Deliberately +# does NOT invoke cargo build/test/check — this repo only distributes the +# runtime, it doesn't build it. +on: + pull_request: + paths: + - simplicio.exe + - simplicio + - simplicio-windows-x64.exe + - simplicio-darwin-x64 + - simplicio-linux-x64 + - simplicio-macos-arm64 + - simplicio-update-manifest.json + - version.txt + - "npm/**" + - "pypi/**" + - "Formula/**" + - "scripts/e2e_ecosystem.py" + - "scripts/verify_distribution_consistency.py" + - ".github/workflows/e2e.yml" + push: + branches: [master, main] + paths: + - simplicio.exe + - simplicio-windows-x64.exe + - simplicio-update-manifest.json + - version.txt + - "npm/**" + - "pypi/**" + - "Formula/**" + - "scripts/e2e_ecosystem.py" + - "scripts/verify_distribution_consistency.py" + workflow_dispatch: {} + +permissions: + contents: read + +jobs: + e2e: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + + - name: Run ecosystem E2E integration tests + id: e2e + shell: bash + run: | + python scripts/e2e_ecosystem.py --json | tee e2e-output.txt + continue-on-error: true + + - name: Upload E2E evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-ecosystem-evidence + path: e2e-output.txt + + - name: Fail the job if any scenario failed + if: steps.e2e.outcome == 'failure' + run: exit 1 diff --git a/docs/e2e-integration.md b/docs/e2e-integration.md new file mode 100644 index 0000000..436b639 --- /dev/null +++ b/docs/e2e-integration.md @@ -0,0 +1,103 @@ +# Integration / E2E — validating real flows across the Simplicio ecosystem + +> Implements the minimum scope of issue #9 for this repo: +> `wesleysimplicio/simplicio` is the **public distribution repo** (compiled +> runtime binary + npm/pip/brew wrappers). The runtime source, mapper, +> dev-cli, loop and agent components live in sibling repos and are consumed +> here only as optional PATH adapters (`simplicio version --json` → +> `components`). This repo does not build the Rust runtime (no +> `cargo build`/`cargo test` here — see `INSTALL.md` "Building from Source" +> for that flow in `simplicio-runtime`), so the tests below exercise the +> **already-compiled, committed binary** end to end, exactly as a real user +> or CI consumer would. + +## What it validates + +`scripts/e2e_ecosystem.py` runs each of the following against a disposable, +ephemeral repo (`tempfile.mkdtemp`, deleted at the end — never against a +developer's real `~/.simplicio` state): + +| # | Cenário mínimo (issue #9) | How it's exercised | +|---|----------------------------|---------------------| +| 1 | Instalação limpa e inicialização | `simplicio setup --repo ` on a brand-new repo; asserts `.simplicio/config.json` is created with `schema=simplicio.config/v1` | +| 2 | Config válida / inválida | `simplicio doctor --repo --json` against the generated config, then again after corrupting `config.json` — asserts the runtime degrades with a diagnostic instead of crashing | +| 3 | Descoberta/mapeamento de projeto | `simplicio runtime map --repo --json`, asserts `simplicio.map-result/v1` | +| 4 | Execução de uma tarefa completa | `simplicio edit '{"file":...}' --json` — a real mechanical edit; asserts the file content actually changed on disk (input → final result) | +| 5 | Propagação de erro/timeout/cancelamento | An edit with a pattern that doesn't exist (asserts non-zero exit + clear message), and `simplicio recover "exit 3" --attempts 2` (asserts retry-exhaustion is reported clearly, non-zero exit) | +| 6 | Persistência e retomada após falha | Runs `doctor` as a second, independent process against the same repo and asserts `.simplicio/` state (config, memory, runs, checkpoints) is unchanged/resumed, not re-initialized | +| 7 | Compatibilidade com dependências publicadas | `simplicio version --json` compatibility matrix (`components[*].minimum_version`), plus re-running the existing `scripts/verify_distribution_consistency.py` packaging-contract gate | + +Each scenario prints `[PASS]`/`[FAIL]` with a one-line diagnostic. The +process exits `0` only if every scenario passed, `1` if any failed, and `2` +if no usable binary exists for the current OS/arch (an environment gap, not +a test failure). + +## Running it + +```bash +# Windows (repo ships simplicio-windows-x64.exe / simplicio.exe) +python scripts/e2e_ecosystem.py + +# macOS / Linux (repo ships simplicio-darwin-x64 / simplicio-macos-arm64 / simplicio-linux-x64) +python3 scripts/e2e_ecosystem.py + +# Explicit binary + machine-readable summary +python scripts/e2e_ecosystem.py --binary ./simplicio.exe --json +``` + +No network access, credentials, or `cargo` toolchain required — it only +invokes the binary already checked into the repo root. + +## Evidence captured on 2026-07-14 (Windows, simplicio-windows-x64.exe v1.6.4) + +``` +Using binary: simplicio-windows-x64.exe +Platform: Windows AMD64 + +[PASS] 1. clean install & initialization (simplicio setup) — exit=0, config.json exists=True, schema=simplicio.config/v1 +[PASS] 2a. doctor loads a valid repo config — exit=0 +[PASS] 2b. doctor handles a corrupted config without crashing — exit=0, crashed=False +[PASS] 3. project discovery/mapping (simplicio runtime map) — exit=0 +[PASS] 4. end-to-end task execution (simplicio edit, input->result) — exit=0, before="console.log('hi')", after="console.log('bye')" +[PASS] 5a. edit failure propagates non-zero exit + clear diagnostic — exit=1, message='simplicio: operation 0 (replace): pattern not found: "DOES_NOT_EXIST_IN_FILE"' +[PASS] 5b. retry exhaustion (simplicio recover) reports cancellation clearly — exit=1 +[PASS] 6. state persists across independent process runs (.simplicio/) — state_dir_entries=[...] +[PASS] 7a. release manifest declares component compatibility matrix — components=['dev_cli', 'mapper', 'prompt', 'sprint'] +[FAIL] 7b. published package/manifest versions agree (verify_distribution_consistency.py) — summary: 1 error(s), 4 warning(s), 1 ok + +summary: 9/10 scenarios passed +``` + +**Known finding (not introduced by this change):** scenario 7b currently +fails for a real reason — `version.txt` (3.0.2) is stale relative to +`simplicio-update-manifest.json` (3.5.2, published in commit `e24b426`), and +the npm/PyPI/Homebrew wrapper versions lag both. This is exactly the +contract-incompatibility failure mode issue #9 asks the pipeline to catch +("Contratos incompatíveis fazem o pipeline falhar"), so scenario 7b is +intentionally left failing rather than papered over — see "Next steps" +below. + +## CI + +`.github/workflows/e2e.yml` runs the same script on `windows-latest` for +pull requests and pushes touching the binaries, wrappers, manifest, or the +test script itself, plus `workflow_dispatch` for on-demand runs. The step +uploads full stdout/stderr as a build artifact so a failing run always has +attached diagnostics. + +## Next steps (out of scope for this PR) + +- Fix the real version drift found by 7b (`version.txt`, wrapper + `package.json`/`pyproject.toml`/`Formula/simplicio.rb`, and + `SIMPLICIO_ECOSYSTEM.md` vs. `simplicio-update-manifest.json`) — separate + from the testing work in this issue. +- Add macOS/Linux CI legs once hosted runners for this repo have `git` + available in the default image path used by the ephemeral repo bootstrap + (currently only exercised locally on those OSes). +- Extend scenario 5 with an actual long-running/cancelable task once the + runtime exposes a stable cancellation-token flow via the CLI (today's + `recover` covers retry-exhaustion but not mid-flight cancellation). +- Wire `simplicio contracts smoke` / `simplicio runtime smoke` (present in + the compiled binary) into this suite once their expected fixture layout + (`docs/SIMPLICIO_OPERATIONAL_MANUAL.md`, `examples/EXAMPLES.md`) is + defined for a minimal ephemeral repo, instead of only the real one. diff --git a/scripts/e2e_ecosystem.py b/scripts/e2e_ecosystem.py new file mode 100644 index 0000000..b5eda8f --- /dev/null +++ b/scripts/e2e_ecosystem.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python3 +"""Real end-to-end integration tests for the Simplicio ecosystem (issue #9). + +This repo ships the compiled `simplicio` runtime binary plus the npm/pip/brew +wrappers around it. The actual mapper/dev-cli/prompt/sprint/agent components +live in sibling repos and are consumed here only as optional PATH adapters +(see `simplicio version --json` -> `components`). This script therefore +exercises the *real*, already-built binary end-to-end against an ephemeral, +disposable repo — no `cargo build`/`cargo test` is invoked (compiling the +runtime is out of scope for this packaging repo and explicitly disallowed by +its contribution workflow). + +Scenarios covered (see issue #9 "Cenarios minimos"): + 1. Clean install & initialization -> `simplicio setup` + 2. Valid vs. invalid configuration loading -> `simplicio doctor` + 3. Project discovery / mapping -> `simplicio runtime map` + 4. A complete task end to end (input->result) -> `simplicio edit` + 5. Error / retry-exhaustion propagation -> `simplicio edit` + `simplicio recover` + 6. Persistence & resume across process runs -> `.simplicio/` state directory + 7. Compatibility with published dependencies -> `simplicio version` + the + existing `verify_distribution_consistency.py` packaging contract check + +Every check runs against a throwaway temp directory (tempfile.mkdtemp), +never against the developer's real ~/.simplicio state for the *repo-scoped* +assertions. Nothing here talks to the network or requires credentials. + +Usage: + python scripts/e2e_ecosystem.py [--binary PATH] [--json] + +Exit code: + 0 - all scenarios passed + 1 - at least one scenario failed (diagnostics printed for each) + 2 - no usable simplicio binary found for this platform (environment gap, + not a test failure) +""" + +from __future__ import annotations + +import argparse +import json +import platform +import shutil +import subprocess +import sys +import tempfile +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable + +ROOT = Path(__file__).resolve().parents[1] + + +@dataclass +class Result: + name: str + passed: bool + detail: str = "" + + +@dataclass +class Report: + results: list[Result] = field(default_factory=list) + + def add(self, name: str, passed: bool, detail: str = "") -> None: + self.results.append(Result(name, passed, detail)) + status = "PASS" if passed else "FAIL" + print(f"[{status}] {name}" + (f" — {detail}" if detail else "")) + + @property + def ok(self) -> bool: + return all(r.passed for r in self.results) + + +def pick_binary(explicit: str | None) -> Path | None: + if explicit: + p = Path(explicit) + return p if p.exists() else None + + system = platform.system().lower() + candidates: list[str] + if system == "windows": + candidates = ["simplicio-windows-x64.exe", "simplicio.exe"] + elif system == "darwin": + machine = platform.machine().lower() + if "arm" in machine or "aarch64" in machine: + candidates = ["simplicio-macos-arm64", "simplicio"] + else: + candidates = ["simplicio-darwin-x64", "simplicio"] + else: + candidates = ["simplicio-linux-x64", "simplicio"] + + for name in candidates: + p = ROOT / name + if p.exists(): + return p + return None + + +def run(binary: Path, args: list[str], cwd: Path, input_text: str | None = None, + timeout: int = 30) -> subprocess.CompletedProcess: + return subprocess.run( + [str(binary), *args], + cwd=cwd, + input=input_text, + capture_output=True, + text=True, + timeout=timeout, + ) + + +def scenario_clean_install(binary: Path, report: Report) -> Path: + """1. Clean install & initialization in a brand-new ephemeral repo.""" + tmp = Path(tempfile.mkdtemp(prefix="simplicio-e2e-init-")) + subprocess.run(["git", "init", "-q"], cwd=tmp, check=False) + (tmp / "main.js").write_text("console.log('hi')\n", encoding="utf-8") + + proc = run(binary, ["setup", "--repo", "."], cwd=tmp, input_text="") + config_path = tmp / ".simplicio" / "config.json" + ok = proc.returncode == 0 and config_path.exists() + detail = f"exit={proc.returncode}, config.json exists={config_path.exists()}" + if ok: + try: + data = json.loads(config_path.read_text(encoding="utf-8")) + ok = data.get("schema") == "simplicio.config/v1" + detail += f", schema={data.get('schema')}" + except json.JSONDecodeError as exc: + ok = False + detail += f", config.json unparsable: {exc}" + report.add("1. clean install & initialization (simplicio setup)", ok, detail) + return tmp + + +def scenario_config_loading(binary: Path, repo: Path, report: Report) -> None: + """2. Valid config loads cleanly; corrupted config degrades, doesn't crash.""" + config_path = repo / ".simplicio" / "config.json" + good_backup = config_path.read_text(encoding="utf-8") + + proc_valid = run(binary, ["doctor", "--repo", ".", "--json"], cwd=repo) + valid_ok = proc_valid.returncode == 0 and '"overall_status"' in proc_valid.stdout + report.add( + "2a. doctor loads a valid repo config", + valid_ok, + f"exit={proc_valid.returncode}", + ) + + config_path.write_text("{ this is not valid json ####", encoding="utf-8") + proc_invalid = run(binary, ["doctor", "--repo", ".", "--json"], cwd=repo) + # A corrupted config must not crash the runtime (no panic/traceback) and + # must still produce a diagnostic the caller can act on. + crashed = proc_invalid.returncode not in (0, 1) or "panic" in proc_invalid.stderr.lower() + invalid_ok = not crashed and (proc_invalid.stdout.strip() or proc_invalid.stderr.strip()) + report.add( + "2b. doctor handles a corrupted config without crashing", + invalid_ok, + f"exit={proc_invalid.returncode}, crashed={crashed}", + ) + + config_path.write_text(good_backup, encoding="utf-8") + + +def scenario_project_mapping(binary: Path, repo: Path, report: Report) -> None: + """3. Project discovery / mapping of the ephemeral repo.""" + proc = run(binary, ["runtime", "map", "--repo", ".", "--json"], cwd=repo, timeout=90) + ok = proc.returncode == 0 and "simplicio.map-result/v1" in proc.stdout + report.add( + "3. project discovery/mapping (simplicio runtime map)", + ok, + f"exit={proc.returncode}", + ) + + +def scenario_full_task(binary: Path, repo: Path, report: Report) -> None: + """4. A complete task, input to final result: a real mechanical edit.""" + target = repo / "main.js" + before = target.read_text(encoding="utf-8") + plan = json.dumps({ + "file": "main.js", + "operations": [{"op": "replace", "find": "hi", "with": "bye"}], + }) + proc = run(binary, ["edit", plan, "--json"], cwd=repo) + after = target.read_text(encoding="utf-8") if target.exists() else "" + ok = ( + proc.returncode == 0 + and '"status":"ok"' in proc.stdout + and before != after + and "bye" in after + ) + report.add( + "4. end-to-end task execution (simplicio edit, input->result)", + ok, + f"exit={proc.returncode}, before={before.strip()!r}, after={after.strip()!r}", + ) + + +def scenario_error_propagation(binary: Path, repo: Path, report: Report) -> None: + """5. Errors and retry-exhaustion propagate with a clear diagnostic.""" + bad_plan = json.dumps({ + "file": "main.js", + "operations": [{"op": "replace", "find": "DOES_NOT_EXIST_IN_FILE", "with": "x"}], + }) + proc = run(binary, ["edit", bad_plan, "--json"], cwd=repo) + edit_ok = proc.returncode != 0 and "pattern not found" in (proc.stdout + proc.stderr) + report.add( + "5a. edit failure propagates non-zero exit + clear diagnostic", + edit_ok, + f"exit={proc.returncode}, message={(proc.stdout + proc.stderr).strip()[:120]!r}", + ) + + proc_recover = run(binary, ["recover", "exit 3", "--attempts", "2", "--json"], cwd=repo) + recover_ok = ( + proc_recover.returncode != 0 + and "exhausted 2 attempts" in (proc_recover.stdout + proc_recover.stderr) + ) + report.add( + "5b. retry exhaustion (simplicio recover) reports cancellation clearly", + recover_ok, + f"exit={proc_recover.returncode}", + ) + + +def scenario_persistence(binary: Path, repo: Path, report: Report) -> None: + """6. State persists in .simplicio/ and is resumed by a fresh process.""" + state_dir = repo / ".simplicio" + before_files = sorted(p.name for p in state_dir.iterdir()) if state_dir.exists() else [] + + # A brand-new, independent process invocation must see the same state + # (not re-run first-run setup) — this is the "resume after failure" bar. + proc = run(binary, ["doctor", "--repo", ".", "--json"], cwd=repo) + after_files = sorted(p.name for p in state_dir.iterdir()) if state_dir.exists() else [] + + ok = ( + proc.returncode == 0 + and state_dir.exists() + and "config.json" in after_files + and before_files == after_files + ) + report.add( + "6. state persists across independent process runs (.simplicio/)", + ok, + f"state_dir_entries={after_files}", + ) + + +def scenario_compat(binary: Path, repo: Path, report: Report) -> None: + """7. Compatibility matrix + published-package version consistency.""" + proc = run(binary, ["version", "--json"], cwd=repo) + ok = proc.returncode == 0 + components: dict = {} + if ok: + try: + data = json.loads(proc.stdout) + components = data.get("components", {}) + ok = data.get("schema") == "simplicio.release-manifest/v1" and bool(components) + except json.JSONDecodeError as exc: + ok = False + proc_detail = f"unparsable JSON: {exc}" + else: + proc_detail = f"components={sorted(components)}" + else: + proc_detail = f"exit={proc.returncode}" + report.add("7a. release manifest declares component compatibility matrix", ok, proc_detail) + + # Reuse the existing packaging-consistency contract check: if published + # wrapper/package versions drift from the signed manifest, this must fail. + consistency_script = ROOT / "scripts" / "verify_distribution_consistency.py" + proc2 = subprocess.run( + [sys.executable, str(consistency_script)], + cwd=ROOT, + capture_output=True, + text=True, + timeout=30, + ) + report.add( + "7b. published package/manifest versions agree (verify_distribution_consistency.py)", + proc2.returncode == 0, + proc2.stdout.strip().splitlines()[-1] if proc2.stdout.strip() else f"exit={proc2.returncode}", + ) + + +SCENARIOS: list[Callable[[Path, Path, Report], None]] = [ + scenario_config_loading, + scenario_project_mapping, + scenario_full_task, + scenario_error_propagation, + scenario_persistence, + scenario_compat, +] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--binary", help="explicit path to a simplicio binary to test") + parser.add_argument("--json", action="store_true", help="emit a JSON summary on stdout") + args = parser.parse_args() + + binary = pick_binary(args.binary) + if binary is None: + print( + "No usable simplicio binary found for this platform in the repo root " + f"({platform.system()}/{platform.machine()}). This is an environment " + "gap, not a test failure — skipping.", + file=sys.stderr, + ) + return 2 + + print(f"Using binary: {binary}") + print(f"Platform: {platform.system()} {platform.machine()}") + print() + + report = Report() + repo = scenario_clean_install(binary, report) + try: + for scenario in SCENARIOS: + try: + scenario(binary, repo, report) + except subprocess.TimeoutExpired as exc: + report.add(f"{scenario.__name__} (timeout)", False, str(exc)) + except Exception as exc: # noqa: BLE001 - surface any unexpected failure as a finding + report.add(f"{scenario.__name__} (unexpected error)", False, repr(exc)) + finally: + shutil.rmtree(repo, ignore_errors=True) + + print() + passed = sum(1 for r in report.results if r.passed) + total = len(report.results) + print(f"summary: {passed}/{total} scenarios passed") + + if args.json: + print(json.dumps({ + "binary": str(binary), + "results": [r.__dict__ for r in report.results], + })) + + return 0 if report.ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main())