From 3d299b3771f27e6f961f2416ee513fc7ca20dc52 Mon Sep 17 00:00:00 2001 From: Roxana del Toro Date: Thu, 30 Jul 2026 18:55:31 -0700 Subject: [PATCH 1/5] Update template to be bare on `install` and work with Poetry and pip. - Work with poetry, pip, or uv - Rename directory to satisfy Poetry: 'harness' - Edits pyproject.toml to remove 'harness' from tool paths - Updates ci.yml to a generalized version across 3 OS matrix - Update markdown files - Move 'prefereneces/' to repo root-level - Move 'prefereneces/tests' to repo tests - Move Hypothesis properties tests to repo tests - Update pyproject.toml to include moved 'preferences' - Updates to cli.py to delete files on `install` - Do not use forbidden patterns ANYWHERE - Removed fake_harness from tests - test_gate.py and test_cli.py refactored - Record env specific path (uv, pip, poetry) --- .githooks/_resolve | 23 + .githooks/pre-commit | 3 +- .githooks/pre-push | 3 +- .githooks/prepare-commit-msg | 5 +- .github/workflows/ci.yml | 37 +- .gitignore | 7 +- CONTRIBUTING.md | 6 +- README.md | 183 ++- README.template.md | 228 +++ harness/cli.py | 269 +++- harness/gate.py | 84 +- harness/ralph.ps1 | 39 +- harness/tests/conftest.py | 145 +- harness/tests/test_cli.py | 1188 ++++++++------- harness/tests/test_gate.py | 1275 +++++++---------- harness/tests/test_integration.py | 162 --- harness/tests/test_properties.py | 415 ++---- harness/tests/test_ralph.py | 7 +- harness/tests/test_ralph_ps1.py | 148 ++ {src/preferences => preferences}/__init__.py | 0 .../preferences.js | 0 .../preferences.py | 2 +- pyproject.toml | 147 +- requirements.txt | 11 + src/preferences/tests/__init__.py | 0 tests/.gitkeep | 0 .../preferences}/test_preferences.py | 3 +- .../test_preferences_properties.py | 247 ++++ 28 files changed, 2389 insertions(+), 2248 deletions(-) create mode 100644 .githooks/_resolve create mode 100644 README.template.md delete mode 100644 harness/tests/test_integration.py create mode 100644 harness/tests/test_ralph_ps1.py rename {src/preferences => preferences}/__init__.py (100%) rename {src/preferences => preferences}/preferences.js (100%) rename {src/preferences => preferences}/preferences.py (99%) create mode 100644 requirements.txt delete mode 100644 src/preferences/tests/__init__.py delete mode 100644 tests/.gitkeep rename {src/preferences/tests => tests/preferences}/test_preferences.py (99%) create mode 100644 tests/preferences/test_preferences_properties.py diff --git a/.githooks/_resolve b/.githooks/_resolve new file mode 100644 index 0000000..db6272d --- /dev/null +++ b/.githooks/_resolve @@ -0,0 +1,23 @@ +# Used by the git hooks so they can run checks. +# e.g. +# . "$(dirname "$0")/_resolve" +# exec "$HARNESS" preflight +# +# Loads the harness executable. +# The harness executable was created in cli.py at `def record_harness`` called with `harness install`. +# +# $HARNESS is the path recorded by `record_harness`. +# +# Then, git hook does not require `uv` or `pip` or any specific environment layout. + +recorded="${GIT_DIR:-$(git rev-parse --absolute-git-dir)}/harness-path" +if [ ! -r "$recorded" ]; then + echo "loopgate: hooks are not installed. Run 'harness install' in this repo." >&2 + exit 1 +fi + +HARNESS="$(cat "$recorded")" +if [ ! -x "$HARNESS" ]; then + echo "loopgate: recorded harness '$HARNESS' is gone. Re-run 'harness install'." >&2 + exit 1 +fi diff --git a/.githooks/pre-commit b/.githooks/pre-commit index f052752..4b7d509 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -3,4 +3,5 @@ # Bypassing this hook is forbidden by AGENTS.md. set -eu -.venv/bin/harness preflight +. "$(dirname "$0")/_resolve" +exec "$HARNESS" preflight diff --git a/.githooks/pre-push b/.githooks/pre-push index bccaeda..52ba4ef 100755 --- a/.githooks/pre-push +++ b/.githooks/pre-push @@ -6,4 +6,5 @@ # Bypassing this hook is forbidden by AGENTS.md. set -eu -.venv/bin/harness gate +. "$(dirname "$0")/_resolve" +exec "$HARNESS" gate diff --git a/.githooks/prepare-commit-msg b/.githooks/prepare-commit-msg index a7ba035..9b6fe6f 100755 --- a/.githooks/prepare-commit-msg +++ b/.githooks/prepare-commit-msg @@ -1,4 +1,7 @@ #!/bin/sh +# Agent containment on the proposed commit message. Runs even under --no-verify. +# Bypassing this hook is forbidden by AGENTS.md. set -eu -.venv/bin/python -c 'import sys; from harness.gate import prepare_commit_msg; raise SystemExit(prepare_commit_msg(sys.argv))' "$@" +. "$(dirname "$0")/_resolve" +exec "$HARNESS" prepare-commit-msg "$@" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 20351ff..8ca30cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,8 +1,6 @@ name: gate -# The unbypassable backstop. -# hook, but this re-runs the same checks on every PR and push, so nothing merges -# without passing them. Branch protection makes this required. +# Unbypassable backstop. Re-runs the same local checks on every PR and push so nothing merges# without passing all. Branch protection makes this required. on: pull_request: @@ -25,19 +23,25 @@ concurrency: jobs: dependency-review: runs-on: ubuntu-latest - if: github.event_name == 'pull_request' timeout-minutes: 10 steps: - uses: actions/checkout@v6 - uses: actions/dependency-review-action@v5 with: - fail-on-severity: moderate + # On push there is no PR base/head, so diff the pushed commit range. + base-ref: ${{ github.event.pull_request.base.sha || github.event.before }} + head-ref: ${{ github.event.pull_request.head.sha || github.event.after }} + fail-on-severity: low fail-on-scopes: runtime, development vulnerability-check: true warn-only: false gate: - runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} timeout-minutes: 45 steps: - uses: actions/checkout@v6 @@ -51,24 +55,3 @@ jobs: - name: Harness Checks run: uv run --no-sync harness gate - - # The full gate above runs once on Linux. - # This job runs on Windows we can't run. It only proves install + collection work. - # A Windows dev should add a real ralph.ps1 test. - loop-runner: - strategy: - fail-fast: false - matrix: - os: [macos-latest, windows-latest] - runs-on: ${{ matrix.os }} - timeout-minutes: 15 - steps: - - uses: actions/checkout@v6 - - name: Install uv - uses: astral-sh/setup-uv@v6 - with: - enable-cache: true - - name: uv Sync Dependencies - run: uv sync - - name: Loop-runner tests (ralph.sh on POSIX; skips on Windows) - run: uv run --no-sync pytest harness/tests/test_ralph.py -q diff --git a/.gitignore b/.gitignore index 784c677..4065523 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ __pycache__/ .ruff_cache/ .hypothesis/ .coverage -dist/ .DS_Store .claude .codex @@ -13,8 +12,7 @@ dist/ docs/launch.md test-results/ **/node_modules/ -.python-version -uv.lock +harness-path .env .env.* @@ -28,3 +26,6 @@ scratchpad/* !scratchpad/runs/ scratchpad/runs/* !scratchpad/runs/.gitkeep + +# kept at the last line for easy deletion (real projects need lockfile) +uv.lock diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 319220f..32620cb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,7 +32,7 @@ Humans edit - `pyproject.toml`: Python and tool configuration. - `harness/`: CLI, gate, loop runner, and harness tests. Wraps the loop that starts agents. -- `src/preferences` can contain user-specific preferences. A live Python example is already there. +- `preferences/` contains user-specific preferences. A live Python example is already there. - `docs/PROMPT.md`: tells agents how to operate headless in the repo (mechanics) - [A javascript example](harness/js-scaffold) lives in its own directory within [`harness/`](harness) and can be expanded on. @@ -62,7 +62,7 @@ Changes to the [`harness/`](harness) itself should preserve the core contract: * ### Where tests live -The harness's own tests live in [`harness/tests/`](harness/tests), including the Hypothesis property tests in [`test_properties.py`](harness/tests/test_properties.py). +The harness's own tests live in [`harness/tests/`](harness/tests). Hypothesis coverage for the gate and preferences lives in [`test_properties.py`](tests/preferences/test_properties.py). The full suite runs as part of `harness gate` (at 100% coverage). To run only the harness tests while iterating: @@ -80,7 +80,7 @@ The full gate before a pull request [`harness/gate.py line 177`](harness/gate.py ```sh harness gate -# or, to mimic what an agent will see: +# To mimic what an agent will see add the env var: RALPH_LOOP=1 harness gate ``` diff --git a/README.md b/README.md index cc27b23..0193761 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,8 @@ Blue infinity loop

L∞pGate

-

A coding-agent loop harness for Claude, Codex, Copilot, or any CLI agent. A dumb Ralph loop runner tells an agent to "Go!" and hands it a PROMPT. Agents can edit. Gates decide what lands. You set the plan in motion. The loops eat the prompt, and each agent iteration must update specs and commit through guardrails.

+

Run coding agents strictly andonly accept changes that pass your quality gates.

+

A coding-agent loop harness for Claude, Codex, Copilot, or any CLI agent. A dumb Ralph loop runner tells an agent to "Go!" and hands it a PROMPT. Agents can edit. Gates decide what lands. You set the plan in motion. The loops eat the prompt, and each agent iteration must update specs and commit through guardrails.

![Python](https://img.shields.io/badge/Python-3.11+-3776AB?logo=python&logoColor=white) ![Status](https://img.shields.io/badge/github-repo-blue?logo=github) @@ -21,13 +22,12 @@ --- -## TL;DR: Getting Started. +## TL;DR -1. `gh repo create my-app --template /loopgate_harness --private --clone` **or** ['Use This Template'](https://github.com/new?template_name=loopgate_harness&template_owner=rxdt) -2. `uv run harness install ` -3. Write your project goal in [docs/plan.md](docs/plan.md) -4. `harness run [max_iterations] [max_minutes]` -5. Not what you wanted? Refine [`docs/plan.md`](docs/plan.md) / [`docs/PROMPT.md`](docs/PROMPT.md) and re-run +1. `gh repo create / --template rxdt/loopgate_harness --private --clone && cd && uv run harness install && source .venv/bin/activate` +2. `harness run codex` + +**Requirements**: `pip`, `uv`, or `poetry`. Python 3.11. --- @@ -40,7 +40,7 @@ - **Built-in stack**: Ruff, Pyright, Pylint, Semgrep, Complexipy, Hypothesis, 100% coverage β˜‘β˜‘β˜‘ - **Progressive**: Preflight vs full gate split πŸ†— - **Forbidden-path containment**: Don't touch that!-configurable πŸ›‘ -- **Installable project template**: `harness install loopgate` gets the repo ready ▢️ +- **Installable project template**: `harness install ` gets the repo ready ▢️ - **No-rot**: Fresh-context agent iterations to reduce context rot πŸ”„ - **Simple**: One command setup gets you git hooks and everything else - **No-waste**: Timeouts and time-limits for all loops ⏸ @@ -51,30 +51,59 @@ ## Details > [!IMPORTANT] -> Default configurations In [`pyproject.toml`](pyproject.toml) Update tool settings, add agent calls, remove or include checks... or leave as it. +> Default configurations In [`pyproject.toml`](pyproject.toml) Update tool settings, add agent calls, remove or include checks... or leave as is. `docs/PROMPT.md` tells each agent to pick a `spec` and build. `docs/specs/` say _what_ to build. The agent decides _what next_. You keep `docs/plan.md` current, and specs get rewritten from it (agent is told in `docs/PROMPT.md` to update the specs). Each iteration the agent updates its spec and `PROJECT_STATUS`. Ideas from [ghuntley](https://github.com/ghuntley), How to Ralph Wiggum. > [!TIP] -> If you don't like _ANYTHING_ in this framework, remove it. +> If you don't like _ANYTHING_ in this framework, update it. + +### Start a project + +1. `gh repo create my-app-name --template /loopgate_harness --private --clone` **or** + ['Use This Template'](https://github.com/new?template_name=loopgate_harness&template_owner=rxdt) +2. Source your environment (if applicable) +3. From the root, run `harness install ` to name the project, install dependencies, set up the git hooks, and delete excess files. Install dependencies e.g. `uv sync && source .venv/bin/activate && harness install [max_iterations] [max_minutes]` +9. Not what you wanted? Refine [`docs/plan.md`](docs/plan.md) / [`docs/PROMPT.md`](docs/PROMPT.md) and re-run +10. Strict Ruff rules, type-checking Pyright, Complexipy, and Pytest coverage are set in [`pyproject.toml`](pyproject.toml). +11. Your coding quirks go in [`preferences/preferences.py`](preferences/preferences.py). +12. Loop!: -## Start a project +```sh +harness run [max_iterations] [max_minutes] # agent: claude/codex/agy/copilot. ralph loop runner injects prompt +``` -1. From inside the checkout, run `harness install ` to name the project, installs dependencies, and set up the 3 git hook. -2. Write your grand vision into `docs/plan.md`. -3. Optionally add the first spec in `docs/specs/`, or have an agent draft the first specs. -4. Put product code under `src/` and list new source directories in `pyproject.toml [tool.coverage.run]`. -5. Strict Ruff rules, type checking, pyright, complexipy, and pytest coverage are set in `pyproject.toml`. -6. Your coding quirks go in [`src/preferences/preferences.py`](src/preferences/preferences.py). -7. Run a loop: +### Works with `uv`, `poetry`, or `pip` ```sh -harness run [max_iterations] [max_minutes] # agent: claude/codex/agy/copilot. ralph loop runner adds prompt +uv sync +source .venv/bin/activate +harness install +harness gate +harness run + +poetry install +poetry run harness install +poetry run harness gate +poetry run harness run + +python -m venv .venv +source .venv/bin/activate +python -m pip install -r requirements.txt -e . +harness install +harness gate +harness run ``` ![L∞P architecture engine flow](.loops.svg) -## A L∞PS Loop +## A L∞Pgate Loop The repo is the only memory. Each iteration is a fresh-context agent. @@ -90,53 +119,36 @@ The repo is the only memory. Each iteration is a fresh-context agent. ![L∞PS Agents](.loops_agents.svg) -## Safety - -`harness run` launches an autonomous LLM worker with the configured permissions, e.g. -`--permission-mode acceptEdits` or `--sandbox danger-full-access`. - -The gate bounds what any **commit** may touch, but the worker itself is **not** sandboxed to this repo unless you set that config. Consider the balance: without access it cannot do much. With machine access it can wreak havoc. Under a permissive mode it can run arbitrary shell. You are authorizing real changes. Choose the worker and permission mode deliberately. - -#### The Gate: Tiered Checks - -⚑ `harness preflight` (pre-commit) β†’ fast checks. -Ruff lint + check format for everyone, _plus_ **containment** for the agents. Self-heals by un-staging forbidden files. - -βœ… `harness gate` (CI/PR pre-push). Local checks mirror CI β†’ ruff lint + format report-only, pyright, pylint, semgrep, complexipy, hypothesis, pytest @ 100% cov. - -Only humans can bypass triggered gates and commit by adding flag `--no-verify`. - -
- - ## Directory Layout ``` -harness/ the gate, loop runner, CLI, custom user checks (πŸ€– forbidden) - gate.py mirror the CI locally + preferences.py honored (πŸ€– forbidden) - tests/ the harness's own tests (πŸ€– forbidden) - test_properties.py hypothesis tests (πŸ€– forbidden) -.githooks/ pre-commit / pre-push gate hooks (πŸ€– forbidden) -.github/ CI that re-runs the gate (πŸ€– forbidden) +harness/ the gate, loop runner, CLI (πŸ€– forbidden directory) + gate.py mirror the CI locally + preferences.py honored + cli.py command-line entry point + tests/ the harness's own tests + js-scaffold javascript example to build upon +preferences/ user-defined preferences not covered by tools (πŸ€– forbidden directory) +tests/ + preferences/ (πŸ€– tests/preferences is forbidden directory) +.githooks/ pre-commit / pre-push gate hooks (πŸ€– forbidden directory) +.github/ CI that re-runs the gate (πŸ€– forbidden directory) pyproject.toml project + tooling config (πŸ€– forbidden) AGENTS.md rules for agents working in the repo (πŸ€– forbidden) docs/PROMPT.md the standing per-iteration instruction (human maintained) -docs/ PLAN, PROJECT_STATUS, PROMPT (human maintained plan.md) +docs/ PLAN, PROJECT_STATUS, PROMPT (human or agent maintained plan.md) scratchpad/ scratch dir agents can use for temp files (For the πŸ€– to play) -docs/specs/ WHAT to build, one PRIORITY-bannered file per track +docs/specs/ WHAT to build, one PRIORITY-bannered file per track (agent maintained) src/ your product/source code (add to coverage source) - preferences/ - preferences.py user-defined preferences not covered by tools (πŸ€– forbidden) ``` +[`pyproject.toml`](pyproject.toml) is the single source of harness configuration. Humans own it and [`preferences/`](preferences/); both are agent-protected. + If an agent edits a forbidden file, the file will be unstaged (not allowed to commit). A forbidden pattern by an agent (e.g. `# noqa` will also prevent their commit and force them to fix it.)
-[`pyproject.toml`](pyproject.toml) is the single source of harness configuration. Humans own all of it (`pyproject.toml` is agent-forbidden; `harness/preferences.py` is part of `harness/`). - A minimal `[tool.harness.gate]` snippet could look like: ```toml @@ -149,20 +161,6 @@ patterns = ["# noqa"] # banned in agent-authored diffs pytest = "uv sync pytest" # one check command, run by the local gate AND CI ``` -## ⚠️ Warnings. Read this before a first run. - -1. **This harness does not sandbox agents.** It _tries_ to harness bad code in loops via gates. Sandboxing agents will, e.g. prevent them from maintaining git, running Playwright, being seen as trustworthy by semgrep leading to cyclical failures, etc. - -2. **The gate is a guardrail, not a jail.** Agents are crafty, like people. They will find a way to complete a task at all costs. **Trust nothing and no one.** - -3. **Mind your usage limits.** `harness run` works agents to the cap set. You can easily burn through your tokens, context windows, and provider usage limits. **Workers continue running as long as there is work to do.** - -4. **`docs/PROMPT.md` tells the worker to push every iteration**. Protect `main` and run the loop on its own branch. - -5. **100% coverage does not mean good tests.** That is quantity, not quality. (Upcoming feature: mutation testing) - -6. **Note**: `semgrep --config auto` needs network for semgrep registry rules. - ## Commands Tool commands are defined once, in `[tool.harness.gate.checks]` in [pyproject.toml](pyproject.toml). The local gate and CI both derive them from there. @@ -180,13 +178,15 @@ harness run codex 2 20 harness run agy 3 10 harness run copilot 2 20 ``` -### Running with claude + +### Running with claude To run LoopGate with Claude : ```sh harness run claude 2 20 ``` + Note: The worker must be installed and authenticated separately.
@@ -196,10 +196,10 @@ Note: The worker must be installed and authenticated separately. - Edit rules at [pyproject.toml](pyproject.toml) for [ruff](https://docs.astral.sh/ruff/), [pylint](https://pypi.org/project/pylint/), [pydoclint](https://pypi.org/project/pydoclint/0.9.1/), [pyright](https://github.com/microsoft/pyright), [pytest](https://docs.pytest.org/en/stable/), [hypothesis](https://hypothesis.readthedocs.io/), [complexipy](https://github.com/rohaquinlop/complexipy) - Add forbidden files, directories, or patterns in `[tool.harness.gate]` at [pyproject.toml](pyproject.toml) -- Add Hypothesis tests in any test directory, examples at [test_properties.py](harness/tests/test_properties.py) -- [semgrep](https://docs.semgrep.dev/semgrep-ci/sample-ci-configs) has no repo config here. It uses registry configs plus Semgrep's built-in defaults which ignore tests. -- Edit `[tool.harness.gate.checks]` in [pyproject.toml](pyproject.toml). [ci.yml](.github/workflows/ci.yml) runs the same `harness gate`. -- Removing existing preferences or add your own preferences at [preferences.py](src/preferences/preferences.py). Current preferences: +- Add [Hypothesis](https://hypothesis.readthedocs.io/) tests in any test directory, examples at [test_properties.py](tests/preferences/test_properties.py). +- [semgrep](https://docs.semgrep.dev/semgrep-ci/sample-ci-configs) has no repo config here. It uses registry configs / Semgrep's built-in defaults which ignore tests. +- Update `[tool.harness.gate.checks]` in [pyproject.toml](pyproject.toml). [ci.yml](.github/workflows/ci.yml) runs those **same exact** `harness gate` checks. +- Add or remove coding preferences [preferences.py](preferences/preferences.py) that only agents in loops **must** respect. Current preferences: ```py function_argument_assignment_has_star # agents use non-specific `def fun(*)` @@ -209,7 +209,7 @@ dynamic_star_call # Calls to def fun(*items) breaks when you can't tell how man pointless_class # ensure classes are added for good reasons (carry state, values, methods) lazy_assert # enforce real assertions, stronger tests objects_injected_into_runtime_memory # finds calls that manipulate global state (dangerous, tricky) -lambda_found # abolish lambdas for agents to keep their code simpler +lambda_found # abolish lambdas, make agents keep their code simple lazy_any_type_hints # abolish type `Any` used to bypass strict type-checking chaotic_continue_statements # abolish unecessary nested continue statements, clean code complex_comprehension # no needlessly dense list/set/dict comprehensions, prefer linear code @@ -238,6 +238,10 @@ You don’t have to. The loop runner, Ralph, and the CLI take a prompt, launch a The included [`harness/js-scaffold`](harness/js-scaffold/package.json) is a simple JavaScript **example** to expand on. Go to [pyproject.toml line 75](pyproject.toml#L75). Update checks. Put `js` into list `[tool.harness].languages`. Remove `py` if unused. +- **Why not just a shell loop?** + +A shell loop only reruns an agent. LoopGate ensures fresh context, durable repo state, time and iteration limits, protected paths, and quality gates that stop bad changes _before_ they land. + ``` npm run --prefix harness/js-scaffold gate npm run --prefix harness/js-scaffold preflight @@ -254,7 +258,7 @@ npm run --prefix harness/js-scaffold preflight - There is NO worktree/branch creation by design. You can create branches/trees and run a loop in each, then merge _(if you really feel like managing that)_ - Agent duties can be contained to a part of the repo. e.g. Codex-1-frontend uses `docs/specs/frontend.md`, Claude-2-researcher `docs/specs/backend`... -### If you must be a ringleader +### If you want to run a graph **Recommendations for running several agents at once on one branch (no worktrees):** @@ -294,3 +298,38 @@ npm run --prefix harness/js-scaffold preflight
![diagram](.diagram.png) + +## Read this before a first run. + +1. **This harness does not sandbox agents.** It _tries_ to harness bad code in loops via gates. Sandboxing agents will, e.g. prevent them from maintaining git, running Playwright, being seen as trustworthy by semgrep leading to cyclical failures, etc. + +2. **The gate is a guardrail, not a jail.** Agents are crafty, like people. They will find a way to complete a task at all costs. **Trust nothing and no one.** + +3. **Mind your usage limits.** `harness run` works agents to the cap set. You can easily burn through your tokens, context windows, and provider usage limits. **Workers continue running as long as there is work to do.** + +4. **`docs/PROMPT.md` tells the worker to push or not**. + +5. Protect `main` and run the loop on its own branch. + +6. **100% coverage does not mean good tests.** That is quantity, not quality. (Upcoming feature: mutation testing) + +7. **Note**: `semgrep --config auto` needs network for semgrep registry rules. + +## Safety + +`harness run` launches an autonomous LLM worker with the configured permissions, e.g. +`--permission-mode acceptEdits` or `--sandbox danger-full-access`. + +The gate bounds what any **commit** may touch, but the worker itself is **not** sandboxed to this repo unless you set that config. Consider the balance: without access it cannot do much. With machine access it can wreak havoc. Under a permissive mode it can run arbitrary shell. You are authorizing real changes. Choose the worker and permission mode deliberately. + +#### The Gate: Tiered Checks + +⚑ `harness preflight` (pre-commit) β†’ fast checks. +Ruff lint + check format for everyone, _plus_ **containment** for the agents. Self-heals by un-staging forbidden files. + +βœ… `harness gate` (CI/PR pre-push). Local checks mirror CI β†’ ruff lint + format report-only, pyright, pylint, semgrep, complexipy, hypothesis, pytest @ 100% cov. + +Only humans can bypass triggered gates and commit by adding flag `--no-verify`. + +
+ diff --git a/README.template.md b/README.template.md new file mode 100644 index 0000000..9d29f1c --- /dev/null +++ b/README.template.md @@ -0,0 +1,228 @@ +## TL;DR: Getting Started. + +Now that you have the template locally: + +1. `uv sync` OR `poetry install` OR ` pip install -r requirements.txt`, then `harness install ` +2. Write your project goal in [docs/plan.md](docs/plan.md) +3. `harness run [max_iterations] [max_minutes]` +4. Not what you wanted? Refine [`docs/plan.md`](docs/plan.md) / [`docs/PROMPT.md`](docs/PROMPT.md) and re-run + +--- + +## Details + +> [!IMPORTANT] +> Default configurations In [`pyproject.toml`](pyproject.toml) Update tool settings, add agent calls, remove or include checks... or leave as is. +> If you don't like _ANYTHING_ in this framework, remove it. + +### Start a project + +```sh +uv sync +source .venv/bin/activate +harness install +harness gate +harness run > + +poetry install +poetry run harness install +poetry run harness gate +poetry run harness + +python -m venv .venv +source .venv/bin/activate +python -m pip install -r requirements.txt -e . +harness install +harness gate +harness run +``` + +1. From the root, run `harness install ` to name the project, install dependencies, set up the three git hooks, delete extraneous files. +2. Write your grand vision into `docs/plan.md`. + - Specs get rewritten from `docs/plan.md` and state of repo. + - Agent is told in `docs/PROMPT.md` to update the specs. + - `docs/specs/` tell an agent _what_ to build. + - Right now `docs/PROMPT.md` tells each agent to pick a `spec`. +3. Put your product code under `src/` . +4. Configurations plus strict Ruff rules, type checking, pyright, complexipy, and pytest coverage are set in [`pyproject.toml`](pyproject.toml). +5. Coding preferences not caught by tooling go in [`preferences/preferences.py`](preferences/preferences.py). +6. Any agent CLI that reads a prompt from stdin and can edit/commit works if they are set in `AGENTS` in [`pyproject.toml`](pyproject.toml). +7. The repo is the only memory. Each iteration is a fresh-context agent. +8. `harness run` launches an autonomous LLM worker with the configured permissions, +9. Run a loop `harness run [max_iterations] [max_minutes]`: + +- agent builds +- agent commits +- every git commit passes the fast preflight (lint, format, plus loop containment for the agent) +- every git push runs the full gate: lint, types, semgrep, tests, 100% coverage +- the loop stops at `max_iterations`, a nonzero worker exit, or a timeout +- Unspecified iterations/minutes β†’ default to 2 iterations Γ— 20 minutes each + +```sh +harness install # rewrite [project] name, uv sync, set core.hooksPath to .githooks +harness preflight # fast checks: preferences, ruff lint + format (plus loop containment) +harness gate # full pass: preferences, ruff, format, pyright, pylint, complexipy, semgrep, pytest @ 100% cov, hypothesis +RALPH_LOOP=1 harness gate # to run as if you are the agent in the loop +harness run [max_iterations] [max_minutes] [verbose] # claude/codex/agy/copilot, defaults: 2 20 True + +# AGENT CALLS +harness run claude 10 20 +harness run codex 2 20 +harness run agy 3 10 +harness run copilot 2 20 +``` + +Tool commands are defined in `[tool.harness.gate.checks]` in [pyproject.toml](pyproject.toml). The gate and CI both derive them from there. + +#### The Gate: Tiered Checks + +⚑ `harness preflight` (pre-commit) β†’ Ruff lint + check format for everyone, _plus_ **containment** for the agents. Self-heals by un-staging forbidden files. + +βœ… `harness gate` Local checks mirror CI β†’ ruff lint + format report-only, pyright, pylint, semgrep, complexipy, hypothesis, pytest @ 100% cov. + +πŸ€₯ `prepare-commit-msg` ensures an agent is not trying to commit empty -- must do work. + +> [!IMPORTANT] +> **Only humans can bypass triggered gates and commit by adding flag `--no-verify`.** + +
+ + +## Directory Layout + + + +``` +harness/ the gate, loop runner, CLI (πŸ€– forbidden directory) + gate.py mirror the CI locally + preferences.py honored + cli.py command-line entry point + tests/ the harness's own tests + js-scaffold javascript example to build upon +preferences/ user-defined preferences not covered by tools (πŸ€– forbidden directory) +tests/ + preferences/ (πŸ€– tests/preferences is forbidden directory) +.githooks/ pre-commit / pre-push gate hooks (πŸ€– forbidden directory) +.github/ CI that re-runs the gate (πŸ€– forbidden directory) +pyproject.toml project + tooling config (πŸ€– forbidden) +AGENTS.md rules for agents working in the repo (πŸ€– forbidden) +docs/PROMPT.md the standing per-iteration instruction (human maintained) +docs/ PLAN, PROJECT_STATUS, PROMPT (human or agent maintained plan.md) +scratchpad/ scratch dir agents can use for temp files (For the πŸ€– to play) +docs/specs/ WHAT to build, one PRIORITY-bannered file per track (agent maintained) +src/ your product/source code (add to coverage source) +``` + +[`pyproject.toml`](pyproject.toml) is the single source of harness configuration. Humans own it and [`preferences/`](preferences/); both are agent-protected. + +If an agent edits a forbidden file, the file will be unstaged (not allowed to commit). A forbidden pattern by an agent (e.g. `# noqa` will also prevent their commit and force them to fix it.) + +Review/edit current "[preferences.py](preferences/preferences.py)": + +```py +function_argument_assignment_has_star # agents use non-specific `def fun(*)` +function_argument_assignment_underscore_lead # agents love over-using underscore names `def _fun()` +hidden_signature_star_args # Complain when a function uses *args or **kwargs (it hides function signatures) +dynamic_star_call # Calls to def fun(*items) breaks when you can't tell how many arguments f is getting +pointless_class # ensure classes are added for good reasons (carry state, values, methods) +lazy_assert # enforce real assertions, stronger tests +objects_injected_into_runtime_memory # finds calls that manipulate global state (dangerous, tricky) +lambda_found # abolish lambdas for agents to keep their code simpler +lazy_any_type_hints # abolish type `Any` used to bypass strict type-checking +chaotic_continue_statements # abolish unecessary nested continue statements, clean code +complex_comprehension # no needlessly dense list/set/dict comprehensions, prefer linear code +``` + +## Read this before a first run. + +The gate bounds what any **commit** may touch, but the worker itself is **not** sandboxed to this repo unless you set that config. Consider the balance: without access it cannot do much. With machine access it can wreak havoc. Under a permissive mode it can run arbitrary shell. You are authorizing real changes. Choose the worker and permission mode deliberately. + +1. **This harness does not sandbox agents.** It _tries_ to harness bad code in loops via gates. Sandboxing agents will, e.g. prevent them from maintaining git, running Playwright, being seen as trustworthy by semgrep leading to cyclical failures, etc. + +2. **The gate is a guardrail, not a jail.** Agents are crafty, like people. They will find a way to complete a task at all costs. **Trust nothing and no one.** + +3. **Mind your usage limits.** `harness run` works agents to the cap set. You can easily burn through your tokens, context windows, and provider usage limits. **Workers continue running as long as there is work to do.** + +4. **`docs/PROMPT.md` tells the worker to push or not**. + +5. Protect `main` and run the loop on its own branch. + +6. **100% coverage does not mean good tests.** That is quantity, not quality. (Upcoming feature: mutation testing) + +7. **Note**: `semgrep --config auto` needs network for semgrep registry rules. + +
+ +
+ + +## Expanding your harness + +- Edit rules at [pyproject.toml](pyproject.toml) for [ruff](https://docs.astral.sh/ruff/), [pylint](https://pypi.org/project/pylint/), [pydoclint](https://pypi.org/project/pydoclint/0.9.1/), [pyright](https://github.com/microsoft/pyright), [pytest](https://docs.pytest.org/en/stable/), [hypothesis](https://hypothesis.readthedocs.io/), [complexipy](https://github.com/rohaquinlop/complexipy) +- Add forbidden files, directories, or patterns in `[tool.harness.gate]` at [pyproject.toml](pyproject.toml) +- Add [Hypothesis](https://hypothesis.readthedocs.io/) tests when generated cases improve coverage beyond example-based tests. `tests/` +- [semgrep](https://docs.semgrep.dev/semgrep-ci/sample-ci-configs) has no repo config here. It uses registry configs plus Semgrep's built-in defaults which ignore tests. +- Edit `[tool.harness.gate.checks]` in [pyproject.toml](pyproject.toml). [ci.yml](.github/workflows/ci.yml) runs the same `harness gate`. +- Remove or add preferences not caught by Ruff, Pylint, etc. at [preferences.py](preferences/preferences.py). + +
+ +
+ + +### FAQ + +- **What is the difference between a gate and a sandbox?** + +A **gate** is a workflow checkpoint that evaluates code and decides whether it is allowed to land in your commits. A **sandbox** is an isolated OS-level environment designed to prevent code from modifying your underlying machine. LoopGate uses gates to control your git history, but it does _not_ provide a secure OS sandbox. + +- **What if I don't want to build an app in Python?** + +You don’t have to. The loop runner, Ralph, and the CLI take a prompt, launch agents pointed at markdown files. LoopGate is language-agnostic at the agent-loop level, but the template is configured to be Python-specific at [pyproject.toml](pyproject.toml). Add your language and commands for your checks to run there. + +- **Javascript?** + +The included [`harness/js-scaffold`](harness/js-scaffold/package.json) is a simple JavaScript **example** to expand on. Go to [pyproject.toml line 75](pyproject.toml#L75). Update checks. Put `js` into list `[tool.harness].languages`. Remove `py` if unused. + +``` +npm run --prefix harness/js-scaffold gate +npm run --prefix harness/js-scaffold preflight +``` + +
+ +
+ + +## Coordination + +- Use `git log --oneline ..HEAD` to show what's unpushed. +- There is NO worktree/branch creation by design. You can create branches/trees and run a loop in each, then merge _(if you really feel like managing that)_ +- Agent duties can be contained to a part of the repo. e.g. Codex-1-frontend uses `docs/specs/frontend.md`, Claude-2-researcher `docs/specs/backend`... + +### If you want to run a graph + +**Recommendations for running several agents at once on one branch (no worktrees):** + +- **You (human):** seed each spec once with this exact line near the top: + + ``` + Spec claimed by agent: + ``` + +- **The agents:** paste this exact block into [PROMPT.md line 3](docs/PROMPT.md#L3): + + ``` + Other agents are working this repo. Before touching code, pick a spec whose claim line is + , replace it with your name, and commit that claim first. Own that spec's file and its + tests. Set the line back to on your last commit. + ``` + +- What fails when agents do not claim specs/work: agents all pick the top-priority spec, duplicate work, and leave a half-staged git index. +- What fails with too little time i.e. MAX_MINUTES too low: a worker dies mid-`gate` before it can commit. Give each iteration enough minutes to finish (the gate itself takes a while). One successful iteration needs ~2-3 min of pure overhead aside from 'real' work. + - A worker killed too soon leaves its spec claim STUCK: spec stays locked to its name. No other agent will take it until a human resets the line to ``. + - preflight on git commit: ~ a few seconds + - full gate on git push: ~20-48s + - push + cleanup: ~ few seconds - +- Do not rely on agent names for coordination: agents self-name inconsistently and can collide (e.g. two both call themselves the same thing). Names are for human blame/log-matching only; the claim line + committed code are what actually coordinate. + +
diff --git a/harness/cli.py b/harness/cli.py index 609d1f9..79b104c 100644 --- a/harness/cli.py +++ b/harness/cli.py @@ -14,13 +14,23 @@ import tomlkit import typer -from packaging.utils import canonicalize_name +from packaging.utils import canonicalize_name, is_normalized_name from rich import print as rprint from rich.console import Console from rich.json import JSON from rich.table import Table -from harness import gate as gate_module +from harness.gate import ( + AGENTS, + COMMIT_CHECKS, + FORBIDDEN, + REPO_ROOT, + gate_checks, + run_gate, + run_git, + run_preflight, +) +from harness.gate import prepare_commit_msg as commit_msg app = typer.Typer( name="loopgate", @@ -29,17 +39,49 @@ add_completion=False, rich_markup_mode="rich", ) -console = Console(force_terminal=True, stderr=True) +console = Console(force_terminal=True) +REPO_ROOT_STR = str(REPO_ROOT) + + +def setup_git_hooks(env_bin: Path, is_windows: bool) -> Path: + """Saves the installed `harness` executable's PATH for Git hooks to run. + + `harness install` calls setup_git_hooks after dependencies and git hooks are in. Because we have this in + pyproject.toml we create an executable: `[project.scripts] harness = "harness.cli:main"` + With an executable and recorded path, there's no dependance. A hook uses a path instead of needing + e.g. active `.venv` or calling `uv run...` + + Arguments: + env_bin: bin directory of the environment the dependency install just populated + is_windows: Operating System platform is Windows "win32" + + Returns: + The path of the file that records the harness command. + """ + rprint("\n[cyan2]Setting git hooks[/cyan2] with `git config core.hooksPath .githooks`:") + subprocess.run(("git", "config", "core.hooksPath", ".githooks"), cwd=REPO_ROOT_STR, check=True) + binary = env_bin / ("harness.exe" if is_windows else "harness") + recorded = Path(run_git(["rev-parse", "--absolute-git-dir"]).strip()).resolve() / "harness-path" + recorded.write_text(f"{binary.as_posix()}\n", encoding="utf-8", newline="\n") + typer.echo( + subprocess.run( + ("git", "config", "core.hooksPath"), cwd=REPO_ROOT_STR, capture_output=True, text=True, check=True + ).stdout.strip() + ) + if is_windows: + rprint("Windows is experimental. Reoprt issues at https://github.com/rxdt/loopgate_harness/issues") + else: + subprocess.run(("ls", "-l", ".githooks"), cwd=REPO_ROOT_STR, check=True) + return recorded -def run_worker(command: list[str], cwd: Path, log: Path, verbose: bool) -> int: +def run_worker(command: list[str], log: Path, verbose: bool) -> int: """Run the worker command, always saving stdout and optionally streaming it live. ralph.sh gets the prompt as a string to pass to the worker in the command Args: command: The worker argv to execute. - cwd: Working directory for the worker subprocess. log: File path that always receives the raw stdout. verbose: When True, also stream compacted output live to the terminal. @@ -48,8 +90,8 @@ def run_worker(command: list[str], cwd: Path, log: Path, verbose: bool) -> int: """ with log.open("w", encoding="utf-8") as handle: if not verbose: - return subprocess.run(command, cwd=str(cwd), stdout=handle, check=False).returncode - with subprocess.Popen(command, cwd=str(cwd), stdout=subprocess.PIPE, text=True) as process: + return subprocess.run(command, cwd=REPO_ROOT_STR, stdout=handle, check=False).returncode + with subprocess.Popen(command, cwd=REPO_ROOT_STR, stdout=subprocess.PIPE, text=True) as process: for line in process.stdout or (): handle.write(line) handle.flush() @@ -75,30 +117,18 @@ def check(name: str, command: Callable[[], dict[str, list[str]]]) -> dict[str, l typer.Exit: always β€” code 1 if anything failed, else code 0. """ results = command() - if os.environ.get("RALPH_LOOP"): - typer.secho( - json.dumps( - { - "Harness Summary": { - "PASSED": results["pass"], - "FAILED": results["fail"], - "result": "rejected by harness" if results["fail"] else f"ok: {name} pass", - } - }, - indent=0, - ) - ) - else: - table = Table(title="\nHarness Summary\n", title_style="bold grey74", box=None, padding=(0, 5)) - table.add_column("PASSED", style="bold dim white") - table.add_column("FAILED") - for passed in results["pass"]: - table.add_row(passed, "[green]βœ” PASSED[/]") - for fail in results["fail"]: - table.add_row(fail, "[bold red]βœ– FAILED[/]") - console.print(table, justify="center") - final = "\n[bold red]rejected by harness[/]" if results["fail"] else f"[green]ok: {name} pass[/]" - console.print(final, justify="center") + table = Table(title="\nHarness Summary\n", title_style="bold grey74", box=None, padding=(0, 5)) + table.add_column("RESULT") + table.add_column("CHECK", style="bold dim white") + for passed in results["pass"]: + table.add_row("[green]PASSED[/]", passed) + for fail in results["fail"]: + table.add_row("[bold red]FAILED[/]", fail) + for warn in results["warn"]: + table.add_row("[yellow]WARNED[/]", warn) + console.print(table, justify="center") + final = "\n[bold red]rejected by harness[/]" if results["fail"] else f"[green]ok: {name} pass[/]" + console.print(final, justify="center") raise typer.Exit(code=1 if results["fail"] else 0) @@ -106,24 +136,43 @@ def check(name: str, command: Callable[[], dict[str, list[str]]]) -> dict[str, l @app.command(help="Fast pre-commit checks (lint/format) plus agent containment") def preflight() -> None: """Dumb pass-through to the fast pre-commit gate.""" - check("preflight", gate_module.run_preflight) + check("preflight", run_preflight) @app.command(help="Pre-push checks match the CI gate exactly (lint, types, security, etc.)") def gate() -> None: """Dumb pass-through to the full pre-push gate; exit nonzero if anything fails.""" - check("gate", gate_module.run_gate) + check("gate", run_gate) + + +@app.command(hidden=True, help="Git prepare-commit-msg hook. Called by .githooks, not by people.") +def prepare_commit_msg( + args: Annotated[list[str] | None, typer.Argument(help="What git passes the hook")] = None, +) -> None: + """Dumb pass-through to prepare_commit_msg hook logic. Hidden git-only usage, not a human command. + + Args: + args: The hook's own arguments: message file, then optionally the source and its commit. + + Raises: + typer.Exit: the hook's status; git aborts the commit on 1. + """ + raise typer.Exit(code=commit_msg(["prepare-commit-msg", *(args or [])])) @app.command(help="Show harness configuration and capabilitie in pyproject.toml") def info() -> None: """Print everything the harness reads from [tool.harness] so nobody has to open pyproject.toml.""" - table = Table(title="\n[cyan2]Harness Configuration Settings[/]", box=None, padding=(0, 2)) + table = Table( + title="\n[cyan2]Basic Harness Configuration Settings[/]\n[dim cyan2]See pyproject.toml for more[/]", + box=None, + padding=(0, 2), + ) phases = ( - ("agents", gate_module.AGENTS), - ("preflight", gate_module.COMMIT_CHECKS), - ("gate", gate_module.gate), - ("forbidden", gate_module.FORBIDDEN), + ("agents", AGENTS), + ("preflight", COMMIT_CHECKS), + ("gate", gate_checks), + ("forbidden", FORBIDDEN), ) for title, checks in phases: table.add_row(f"[bold cyan]{title}[/]", "") @@ -135,64 +184,129 @@ def info() -> None: @app.command(help="Count agent run logs under scratchpad/runs") def status() -> None: """Count run logs and point at the newest one.""" - runs = Path.cwd() / "scratchpad" / "runs" + runs = REPO_ROOT / "scratchpad" / "runs" logs = sorted(runs.glob("*.jsonl")) if runs.is_dir() else [] typer.secho(f"{len(logs)} run log(s) in {runs}", fg=typer.colors.CYAN, bold=True) if logs: typer.secho(f"newest: {logs[-1]}", fg=typer.colors.GREEN, bold=True) -@app.command(help="Setup project: inject project name in pyproject, sync dependencies, set up githooks") +def cleanup(cwd: Path, name: str | None) -> bool: + """Cleans the new local repository of old loopgate things + Arguments: + cwd: the current working directory to leave a clean template in + name: the new name for the project + + Returns: + bool True if successful + """ + if not (cwd / "README.template.md").is_file(): + return False + (cwd / "README.template.md").replace(cwd / "README.md") + for file_name in ( + ".banner.svg", + ".diagram.png", + ".infin.png", + ".loops_agents.svg", + ".loops.svg", + ".github/workflows/publish.yml", + "CONTRIBUTING.md", + ): + (cwd / file_name).unlink(missing_ok=True) + for directory in (cwd / "dist", cwd / "harness" / "tests"): + if directory.exists(): + shutil.rmtree(directory) + gitignore = cwd / ".gitignore" + gitignore.write_text(gitignore.read_text(encoding="utf-8").removesuffix("\nuv.lock\n"), encoding="utf-8") + document = tomlkit.parse((cwd / "pyproject.toml").read_text(encoding="utf-8")) + project = document.setdefault("project", tomlkit.table()) + project.update({ + "name": canonicalize_name(name) if name and is_normalized_name(name) else "my-app-name", + "version": "0.0.0", + }) + tool = document.setdefault("tool", tomlkit.table()) + tool.setdefault("pyright", tomlkit.table()).update({"include": ["src", "preferences"]}) + tool.setdefault("pytest", tomlkit.table()).setdefault("ini_options", tomlkit.table()).update({ + "testpaths": ["tests"], + "pythonpath": ["src"], + }) + coverage = tool.setdefault("coverage", tomlkit.table()) + coverage.setdefault("run", tomlkit.table()).update({"source": ["src", "preferences"]}) + tool.setdefault("complexipy", tomlkit.table()).update({"paths": ["src", "preferences"]}) + tool.setdefault("ruff", tomlkit.table()).setdefault("exclude", tomlkit.array()).append("harness") + tool.setdefault("pylint", tomlkit.table()).setdefault("main", tomlkit.table()).setdefault( + "ignore", tomlkit.array() + ).append("harness") + rprint(f"\n[cyan2]project name[/cyan2] '{project['name']}' set in `pyproject.toml`") + (cwd / "pyproject.toml").write_text(tomlkit.dumps(document), encoding="utf-8") + return True + + +@app.command( + help="Only run this if setting up a project from the template cloned from Github at project root: injects" + " project name in pyproject.toml, syncs dependencies, adds githooks, DELETES unecessary files!" +) def install(name: Annotated[str | None, typer.Argument(help="Set up project for loops")] = None) -> None: - """Injects NAME (PEP 503) into pyproject, sync deps, and activate the git hooks. + """Used by template cloned from Github. Not used by library from PyPi. + Injects NAME (PEP 503) into pyproject, syncs dependencies, and activates the git hooks. Args: name: Optional project name, canonicalized to a PEP 503 form before being written. If name is given, will overwrite existing name in pyproject.toml. When ommitted, project name is left untouched. """ - cwd = Path.cwd() - document = tomlkit.parse((cwd / "pyproject.toml").read_text(encoding="utf-8")) - # Set the requested name (if any); default a missing version to 0.0.0 but never clobber an existing one. - project = document.setdefault("project", tomlkit.table()) - if name: - project["name"] = canonicalize_name(name, validate=True) - rprint(f"\n[cyan2]project name[/cyan2] '{project['name']}' set in `pyproject.toml`") - if not project.get("version"): - project["version"] = "0.0.0" - (cwd / "pyproject.toml").write_text(tomlkit.dumps(document), encoding="utf-8") - rprint("\n[cyan2]installing dependencies[/cyan2] with `uv sync`, then setting git hooks:") - subprocess.run(("uv", "sync"), cwd=str(cwd), check=True) - rprint("\n[cyan2]setting git hooks[/cyan2] with `git config core.hooksPath .githooks`:") - subprocess.run(("git", "config", "core.hooksPath", ".githooks"), cwd=str(cwd), check=True) - typer.echo( + rprint("\n[cyan2]installing dependencies[/cyan2]") + is_windows = sys.platform == "win32" + # Record the env the manager just filled that holds the harness executable + if (REPO_ROOT / "uv.lock").is_file(): + subprocess.run(("uv", "sync"), cwd=REPO_ROOT_STR, check=True) + env_bin = REPO_ROOT / ".venv" / ("Scripts" if is_windows else "bin") + elif (REPO_ROOT / "poetry.lock").is_file(): + subprocess.run(("poetry", "install"), cwd=REPO_ROOT_STR, check=True) + poetry_env = subprocess.run( + ("poetry", "env", "info", "--executable"), capture_output=True, text=True, check=True + ) + env_bin = Path(poetry_env.stdout.strip()).parent + name = "harness" + else: subprocess.run( - ("git", "config", "core.hooksPath"), cwd=str(cwd), capture_output=True, text=True, check=True - ).stdout.strip() - ) - subprocess.run(("ls", "-l", ".githooks"), cwd=str(cwd), check=True) + [sys.executable, "-m", "pip", "install", "-r", "requirements.txt", "-e", "."], + cwd=REPO_ROOT_STR, + check=True, + ) + env_bin = Path(sys.executable).parent + cleanup(REPO_ROOT, name) + recorded = setup_git_hooks(env_bin, is_windows) + if not is_windows: + check_for_timeout_and_prompt(env_bin) + rprint(f"\nRecorded in {recorded} is the path to executable {env_bin}") + rprint("\n[red]COMMIT UNSTAGED CHANGES[/red]") + - # Warn (and offer to install) when `harness run` lacks a timeout tool. Linux ships `timeout`; macOS - # needs `gtimeout` from coreutils. Windows uses ralph.ps1 (no timeout tool), so this is a no-op there. - if not (sys.platform == "win32" or shutil.which("timeout") or shutil.which("gtimeout")): - rprint("\n[yellow]macOS harness needs timeout/gtimeout from coreutils[/yellow]") +def check_for_timeout_and_prompt(env_bin: Path) -> None: + """Offer install when macOS lacks a timeout tool. Linux has `timeout`, macOS needs coreutils.gtimeout. + Args: + env_bin: Path to the harness executable + """ + if not (shutil.which("timeout") or shutil.which("gtimeout")): + rprint("\n[yellow]macOS harness needs timeout/gtimeout from coreutils to loop[/yellow]") if not shutil.which("brew"): - rprint("no Homebrew https://brew.sh then run `brew install coreutils`, or `sudo port install`") - elif typer.confirm("Allow install now with `brew install coreutils`?"): + rprint("Get Homebrew https://brew.sh then run `brew install coreutils` or `sudo port install`") + elif typer.confirm("[magenta]Install now `brew install coreutils`?[/magenta]"): subprocess.run(("brew", "install", "coreutils"), check=False) else: - rprint("[yellow]skipped[/yellow] β€” run `brew install coreutils` before `harness run`.") - + rprint("[yellow]skipped[/yellow]: run `brew install coreutils` before `harness run`.") rprint( - "\nActivate env by running command [turquoise2]`source .venv/bin/activate`[/turquoise2] " - "to use the [green]`harness`[/green] command.\n" + "\nIf timeout or gtimeout is installed, you can run loops after activating the environment" + f"\nActivate env with [turquoise2]`source {env_bin / 'activate'}`[/turquoise2] " + "to use the [green]`harness`[/green] commands.\n" "\n[turquoise2]python:[/turquoise2] project supports >=3.11" - "\nPIN NEWER local Python with [turquoise2]`uv python pin 3.13 && uv sync`[/turquoise2]" + "\nOptionally pin newer local Python with [turquoise2]`uv python pin 3.13 && uv sync`[/turquoise2]" ) @app.command( help="Run one harnessed ralph loop with , e.g. harness run claude 3 20.\n\n" - f"Integrated agents (from tool.harness.agents): {', '.join(gate_module.AGENTS)}" + f"Integrated agents (from tool.harness.agents): {', '.join(AGENTS)}" ) def run( agent: str, @@ -214,7 +328,7 @@ def run( typer.Exit: code 2 for an unknown agent or non-positive counts, else the worker's exit code. """ agent = agent.casefold() - if agent not in gate_module.AGENTS: + if agent not in AGENTS: typer.secho(f"Unknown agent name '{agent}'", err=True, fg=typer.colors.MAGENTA, bold=True) raise typer.Exit(code=2) if num_iterations < 1 or max_minutes < 1: @@ -229,21 +343,20 @@ def run( # Hand the agent a fixed identity to use in claims and commits prompt = (cwd / "docs" / "PROMPT.md").read_text(encoding="utf-8").rstrip("\n") os.environ["RALPH_PROMPT"] = f"Your agent id is `{worker_id}`\n\n{prompt}" - # each log file is one run / ralph invocation, not one iteration - log = runs / f"{worker_id}.jsonl" + log = runs / f"{worker_id}.jsonl" # each log file is one run / ralph invocation, not one iteration loop_dir = Path(__file__).resolve().parent - # Windows has no POSIX shell/timeout so run PowerShell twin, ralph.sh otherwise + # Windows has no POSIX shell/timeout so run PowerShell ralph.ps1 twin launcher = ( ["powershell.exe", "-NoProfile", "-File", str(loop_dir / "ralph.ps1")] if sys.platform == "win32" # support windows else [str(loop_dir / "ralph.sh")] ) - agent_argv = [tok.replace("{log_path}", str(log)) for tok in gate_module.AGENTS[agent]] + agent_argv = [tok.replace("{log_path}", str(log)) for tok in AGENTS[agent]] if model: agent_argv[agent_argv.index("--model") + 1] = model command = [*launcher, str(num_iterations), str(max_minutes), *agent_argv] typer.echo(f"harness: {' '.join(command)} -> {log}", err=True) - raise typer.Exit(code=run_worker(command, cwd, log, verbose)) + raise typer.Exit(code=run_worker(command, log, verbose)) def main(argv: list[str] | None = None) -> None: diff --git a/harness/gate.py b/harness/gate.py index f70ecd4..ff7abda 100644 --- a/harness/gate.py +++ b/harness/gate.py @@ -25,39 +25,43 @@ except ImportError: # humans do what they want with preferences.py prefs = None -REPO_ROOT = Path(__file__).resolve().parents[1] + +def run_git(args: list[str], repo: Path | None = None, check: bool = True) -> str: + """Run a git command in the repo and return its stdout. + + Arguments: + args: Git subcommand and its arguments + repo: the repository directory to run the git command from; defaults to REPO_ROOT + check: If check is True and the exit code was non-zero, it raises a CalledProcessError which has + returncode attribute, and output attribute + + Returns: + The command's raw stdout string (callers will .splitlines() as needed) + """ + target = REPO_ROOT if repo is None else repo + command = ["git", "-C", str(target), *args] + git_env = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} + result = subprocess.run(command, capture_output=True, text=True, check=check, env=git_env) + return result.stdout + + +# GATE AND PREFLIGHT RUN FROM PROJECT LEVEL DIRECTORY +REPO_ROOT = Path(run_git(["rev-parse", "--show-toplevel"], repo=Path.cwd()).strip()).resolve() + raw_toml = tomllib.loads((REPO_ROOT / "pyproject.toml").read_bytes().decode()) HARNESS = raw_toml.get("tool", {}).get("harness", {}) languages = HARNESS.get("languages", {}) -gate = HARNESS.get("gate", {}) +gate_checks = HARNESS.get("gate", {}) AGENTS = HARNESS.get("agents", {}) COMMIT_CHECKS = HARNESS.get("preflight", {}) -FULL_CHECKS = COMMIT_CHECKS | gate +FULL_CHECKS = COMMIT_CHECKS | gate_checks FORBIDDEN = HARNESS.get("FORBIDDEN", {}) FORBIDDEN_FILES = FORBIDDEN.get("FILES", []) FORBIDDEN_DIRS = tuple(FORBIDDEN.get("DIRS", [])) FORBIDDEN_PATTERNS = FORBIDDEN.get("PATTERNS", []) -def run_git(args: list[str], check: bool = True) -> str: - """Run a git command in the repo and return its stdout. - - Args: - args: Git subcommand and its arguments. - check: If check is True and the exit code was non-zero, it raises a - CalledProcessError which has returncode attribute, and output attribute - - Returns: - The command's raw stdout string (callers .splitlines() as needed). - """ - command = ["git", "-C", str(REPO_ROOT)] - command.extend(args) - git_env = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} - result = subprocess.run(command, capture_output=True, text=True, check=check, env=git_env) - return result.stdout - - def colorize(name: str, command: str) -> None: """Rich consosle printing to signpost checks. @@ -66,8 +70,7 @@ def colorize(name: str, command: str) -> None: command: The command string printed beneath the header. """ if os.environ.get("RALPH_LOOP"): # loop agents get plain text (no ANSI) - typer.echo(f"PHASE: {name.upper()}") - typer.echo(command) + typer.echo(f"PHASE: {name.upper()}\n{command}") else: console.rule(f"[bold cyan] PHASE: {name.upper()}[/]", style="blink cyan on grey15") console.print(f"[dim italic]{command}[/dim italic]\n", justify="center") @@ -82,7 +85,7 @@ def run_checks(checks: dict[str, list[str]]) -> dict[str, list[str]]: Returns: { "pass": [...], "warn": [...], "fail": [ problems ] } bucketing each check name by exit code. - if anything is in "fail", a commit is not allowed + If anything is in "fail", a commit is not allowed. """ clean_env = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} if not os.environ.get("RALPH_LOOP"): @@ -93,8 +96,12 @@ def run_checks(checks: dict[str, list[str]]) -> dict[str, list[str]]: sys.stdout.flush() with subprocess.Popen(command, cwd=REPO_ROOT, env=clean_env) as process: exit_code = process.wait() - key = "warn" if "format" in name else ("pass" if exit_code == 0 else "fail") - results[key].append(name) + if exit_code == 0: + results["pass"].append(name) + elif "format" in name: + results["warn"].append(name) + else: + results["fail"].append(name) if os.environ.get("RALPH_LOOP"): results["fail"].extend(run_non_human_checks()) @@ -113,7 +120,8 @@ def run_non_human_checks() -> list[str]: if not staged: colorize("EMPTY COMMIT", "nothing staged: do real work, do not commit empty") return problems # yell, but don't block - forbidden = [ + problems.extend(check_for_bad_patterns()) + forbidden: list[str] = [ path for path in staged if path.casefold() in FORBIDDEN_FILES or path.casefold().startswith(FORBIDDEN_DIRS) @@ -121,8 +129,7 @@ def run_non_human_checks() -> list[str]: if forbidden: run_git(["reset", "-q", "HEAD", "--", *forbidden]) colorize("EJECTED", f"kept forbidden paths out of the commit: {', '.join(forbidden)}") - problems.extend(check_for_bad_patterns()) - return problems + return ["problems:\n" + "\n".join(problems)] if problems else [] def check_for_bad_patterns() -> list[str]: @@ -134,15 +141,16 @@ def check_for_bad_patterns() -> list[str]: Returns: The banned-pattern hits plus any preference violations found in the staged files. """ - diff_args = ["diff", "--cached", "--unified=0", "--", ".", ":(exclude)*.md"] - staged_lines = run_git(diff_args).splitlines() colorize("BANNED PATTERNS CHECK", "checking for banned patterns in staged files") - problems = [ - f"'{pattern}' line: {line[1:].strip()}" - for line in staged_lines - for pattern in FORBIDDEN_PATTERNS - if line.startswith("+") and not line.startswith("+++") and pattern.casefold() in line.casefold() - ] + diff_args = ["diff", "--cached", "--unified=0", "--output-indicator-new=a", "--", ".", ":(exclude)*.md"] + staged_lines = run_git(diff_args).splitlines() + problems: list[str] = [] + for line in staged_lines: + if line.startswith("a"): + for pattern in FORBIDDEN_PATTERNS: + pattern_and_bare_line = f"'{pattern}' line: {line[1:].strip()}" + if pattern.casefold() in line.casefold(): + problems.append(pattern_and_bare_line) problems.extend(filter(None, check_for_preferences())) return problems @@ -154,10 +162,10 @@ def check_for_preferences() -> list[str]: Returns: The banned-pattern hits plus any preference violations found in the staged files. """ + colorize("USER PREFERENCES", "checking that user's preferences are respected") if "py" in languages: staged = run_git(["diff", "--cached", "--name-only", "--diff-filter=d", "--", "*.py"]).splitlines() if staged and prefs: - colorize("USER PREFERENCES", "checking that user's preferences are respected") return [prefs(path, run_git(["show", f":{path}"])) for path in staged] return [] diff --git a/harness/ralph.ps1 b/harness/ralph.ps1 index 31a5b59..617adf1 100644 --- a/harness/ralph.ps1 +++ b/harness/ralph.ps1 @@ -2,23 +2,36 @@ # Keep Ralph Dumb: start the worker, give it the prompt, print a line, repeat. Nothing else. # Windows has no POSIX `timeout`, so this uses Wait-Process + taskkill /T to bound each iteration. # -# Usage: powershell -File ralph.ps1 [max_iterations] [max_minutes_per_iteration] -param([Parameter(ValueFromRemainingArguments = $true)] [string[]] $Args) +# Usage: pwsh -File ralph.ps1 [max_iterations] [max_minutes_per_iteration] $ErrorActionPreference = "Stop" +[Console]::InputEncoding = [System.Text.UTF8Encoding]::new($false) $env:RALPH_LOOP = "1" # mark loop commits so the gate applies containment to the worker +function ConvertTo-WindowsArgument([string]$argument) { + $escaped = [regex]::Replace($argument, '(\\*)"', '$1$1\"') + return '"' + $escaped + [regex]::Match($argument, '(\\*)$').Groups[1].Value + '"' +} + $maxIterations = 2 -$maxMinutes = 20 -$rest = @($Args) -if ($rest.Count -gt 0 -and $rest[0] -match '^\d+$') { $maxIterations = [int]$rest[0]; $rest = $rest[1..($rest.Count - 1)] } -if ($rest.Count -gt 0 -and $rest[0] -match '^\d+$') { $maxMinutes = [int]$rest[0]; $rest = $rest[1..($rest.Count - 1)] } +$maxMinutes = [double]20 +$rest = @($args) +if ($rest.Count -gt 0 -and $rest[0] -match '^\d+$') { + $maxIterations = [int]$rest[0] + $rest = @($rest | Select-Object -Skip 1) +} +if ($rest.Count -gt 0 -and $rest[0] -match '^\d+(?:\.\d+)?$') { + $maxMinutes = [double]$rest[0] + $rest = @($rest | Select-Object -Skip 1) +} if ($rest.Count -lt 1) { - Write-Error "defaults: max_iterations=$maxIterations max_minutes_per_iteration=$maxMinutes"; exit 2 + [Console]::Error.WriteLine("defaults: max_iterations=$maxIterations max_minutes_per_iteration=$maxMinutes") + exit 2 } -if ($maxIterations -lt 1 -or $maxMinutes -lt 1) { - Write-Error "ralph: max_iterations and max_minutes must be >= 1"; exit 2 +if ($maxIterations -lt 1 -or $maxMinutes -le 0) { + [Console]::Error.WriteLine("ralph: max_iterations must be >= 1 and max_minutes must be > 0") + exit 2 } for ($i = 1; $i -le $maxIterations; $i++) { @@ -26,7 +39,9 @@ for ($i = 1; $i -le $maxIterations; $i++) { $stdin = "$($env:RALPH_PROMPT)`n`nRALPH_ITERATION=$i/$maxIterations`n" $psi = [System.Diagnostics.ProcessStartInfo]::new() $psi.FileName = $rest[0] - foreach ($a in $rest[1..($rest.Count - 1)]) { $psi.ArgumentList.Add($a) } + if ($rest.Count -gt 1) { + $psi.Arguments = (($rest[1..($rest.Count - 1)] | ForEach-Object { ConvertTo-WindowsArgument $_ }) -join ' ') + } $psi.RedirectStandardInput = $true $psi.UseShellExecute = $false $proc = [System.Diagnostics.Process]::Start($psi) @@ -34,8 +49,10 @@ for ($i = 1; $i -le $maxIterations; $i++) { $proc.StandardInput.Write($stdin); $proc.StandardInput.Close() # Bound the run. Like ralph.sh's `set -e` + timeout: a timeout or a nonzero worker exit stops the # loop and propagates failure, so `harness run` never reports success for a failed iteration. - if (-not $proc.WaitForExit($maxMinutes * 60 * 1000)) { + $timeoutMilliseconds = [int][Math]::Ceiling($maxMinutes * 60 * 1000) + if (-not $proc.WaitForExit($timeoutMilliseconds)) { taskkill.exe /F /T /PID $proc.Id | Out-Null + $proc.WaitForExit() exit 124 # match GNU timeout's exit code } if ($proc.ExitCode -ne 0) { diff --git a/harness/tests/conftest.py b/harness/tests/conftest.py index 7bd4419..e1b454a 100644 --- a/harness/tests/conftest.py +++ b/harness/tests/conftest.py @@ -2,24 +2,18 @@ from __future__ import annotations -import os import subprocess import sys from pathlib import Path +from subprocess import PIPE from typing import Self import pytest -from harness import gate +from harness import cli, gate REPO_ROOT = Path(__file__).resolve().parents[2] - - -def run_cmd(args: list[str], cwd: Path) -> str: - """Run a command in a directory with hook-safe env, failing the test on error.""" - env = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} - result = subprocess.run(args, cwd=cwd, check=True, capture_output=True, text=True, env=env) - return result.stdout +collect_ignore = ["test_ralph.py"] if sys.platform == "win32" else ["test_ralph_ps1.py"] class FakePopen: @@ -41,130 +35,47 @@ def wait(self) -> int: def fake_popen( monkeypatch: pytest.MonkeyPatch, fails: list[list[str]] | None = None ) -> list[tuple[list[str], Path, dict[str, str]]]: - """Fake the Popen seam run_checks uses to spawn each check so no real tool runs. + """Stand in for the external tool run_checks spawns, so no real linter or test runner runs. - Every faked check reports exit 0 (pass) unless its exact argv is in fails, which reports exit 1. - Every launch is recorded (command, cwd, env) so dispatch tests can assert what run_checks ran. + Git is never faked. run_git reaches Popen through subprocess.run, so a git command is handed + straight to the real Popen and the real gate.run_git keeps working against the temp repo the + test points REPO_ROOT at. Only the checks around it are stand-ins. - This replaces the whole subprocess.Popen used by run_checks. run_git reaches Popen too (via - subprocess.run), so a test must do its real git β€” staging, reading the index β€” before calling - this, and stay off the RALPH_LOOP containment path that would run git after the fake is in place. + Every faked check reports exit 0 (pass) unless its exact argv is in fails, which reports exit 1. + Every faked launch is recorded (command, cwd, env) so dispatch tests can assert what run_checks ran. Returns: The live list of recorded launches. """ failing = fails or [] calls: list[tuple[list[str], Path, dict[str, str]]] = [] - - def spawn(command: list[str], *, cwd: Path, env: dict[str, str]) -> FakePopen: - calls.append((command, cwd, env)) + real_popen = gate.subprocess.Popen + + def spawn( + command: list[str], *, cwd: Path | None = None, env: dict[str, str] | None = None, **piping: object + ) -> subprocess.Popen[str] | FakePopen: + del piping # run_git's capture settings, rebuilt below rather than forwarded + if command[:1] == ["git"]: + return real_popen(command, env=env, stdout=PIPE, stderr=PIPE, text=True) + calls.append((command, cwd or REPO_ROOT, env or {})) return FakePopen(1 if command in failing else 0) monkeypatch.setattr(gate.subprocess, "Popen", spawn) return calls -@pytest.fixture -def fake_hook_repo(tmp_path: Path) -> Path: - """A git repo wired to the tracked hooks and a fake harness executable.""" - run_cmd(["git", "init", "-q"], tmp_path) - run_cmd(["git", "config", "user.email", "harness@test.local"], tmp_path) - run_cmd(["git", "config", "user.name", "harness-test"], tmp_path) - hooks = tmp_path / ".githooks" - hooks.mkdir() - for hook in ("pre-commit", "pre-push"): - target = hooks / hook - target.write_text((REPO_ROOT / ".githooks" / hook).read_text(encoding="utf-8"), encoding="utf-8") - target.chmod(0o755) - bin_dir = tmp_path / ".venv" / "bin" - bin_dir.mkdir(parents=True) - harness = bin_dir / "harness" - harness.write_text( - "#!/bin/sh\n" - "printf '%s\\n' \"$@\" > harness.args\n" - "printf '%s\\n' \"${RALPH_LOOP:-}\" > harness.loop\n" - "if test -f harness.exit; then\n" - ' exit "$(cat harness.exit)"\n' - "fi\n" - "exit 0\n", - encoding="utf-8", - ) - harness.chmod(0o755) - run_cmd(["git", "config", "core.hooksPath", ".githooks"], tmp_path) - (tmp_path / "seed.py").write_text("x = 1\n", encoding="utf-8") - run_cmd(["git", "add", "seed.py", ".githooks"], tmp_path) - run_cmd(["git", "commit", "-q", "-m", "seed", "--no-verify"], tmp_path) - return tmp_path - - @pytest.fixture def git_repo(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: - """Provide a git repo with an identity, the tracked git hooks, and a clean initial commit. - - Points gate's git calls at this repo (run_git runs `git -C gate.REPO_ROOT`), so containment - tests stage and read the throwaway repo instead of the real one. - """ - run_cmd(["git", "init", "-q"], tmp_path) - run_cmd(["git", "config", "user.email", "harness@test.local"], tmp_path) - run_cmd(["git", "config", "user.name", "harness-test"], tmp_path) - hooks = tmp_path / ".githooks" - hooks.mkdir() - for hook in ("pre-commit", "pre-push"): - (hooks / hook).write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") - (hooks / hook).chmod(0o755) - (tmp_path / "README.md").write_text("seed\n", encoding="utf-8") - run_cmd(["git", "add", "README.md", ".githooks"], tmp_path) - run_cmd(["git", "commit", "-q", "-m", "seed"], tmp_path) - monkeypatch.setattr(gate, "REPO_ROOT", tmp_path) - return tmp_path - - -@pytest.fixture -def tiny_fake_repo(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: - """A git repo with identity, tracked hooks, fake harness, and a clean seed commit. - - Points gate's git calls at this repo (run_git runs `git -C gate.REPO_ROOT`), so - prepare_commit_msg's internal zero-arg run_git calls target this fake repo. - """ - run_cmd(["git", "init", "-q"], tmp_path) - run_cmd(["git", "config", "user.email", "harness@test.local"], tmp_path) - run_cmd(["git", "config", "user.name", "harness-test"], tmp_path) - hooks = tmp_path / ".githooks" - hooks.mkdir() - for hook in ("pre-commit", "pre-push", "prepare-commit-msg"): - target = hooks / hook - target.write_text((REPO_ROOT / ".githooks" / hook).read_text(encoding="utf-8"), encoding="utf-8") - target.chmod(0o755) - bin_dir = tmp_path / ".venv" / "bin" - bin_dir.mkdir(parents=True) - harness = bin_dir / "harness" - harness.write_text( - "#!/bin/sh\n" - "printf '%s\\n' \"$@\" > harness.args\n" - "printf '%s\\n' \"${RALPH_LOOP:-}\" > harness.loop\n" - "if test -f harness.exit; then\n" - ' exit "$(cat harness.exit)"\n' - "fi\n" - "exit 0\n", - encoding="utf-8", - ) - harness.chmod(0o755) - # sitecustomize runs at interpreter startup: point the real gate's fixed REPO_ROOT at the - # invocation cwd (this fake repo) so the tracked prepare-commit-msg hook's internal run_git - # (git -C gate.REPO_ROOT) reads this repo's index instead of the real project's. - (bin_dir / "sitecustomize.py").write_text( - "import os\nfrom harness import gate\n\ngate.REPO_ROOT = os.getcwd()\n", - encoding="utf-8", - ) - python = bin_dir / "python" - python.write_text( - f"#!/bin/sh\nPYTHONPATH='{bin_dir}:{REPO_ROOT}' exec '{sys.executable}' \"$@\"\n", - encoding="utf-8", - ) - python.chmod(0o755) - run_cmd(["git", "config", "core.hooksPath", ".githooks"], tmp_path) + """Provide a seeded Git repository and point production commands at it.""" + gate.run_git(["init", "-q"], tmp_path) + gate.run_git(["config", "user.email", "harness@test.local"], tmp_path) + gate.run_git(["config", "user.name", "harness-test"], tmp_path) + (tmp_path / ".githooks").mkdir() (tmp_path / "README.md").write_text("seed\n", encoding="utf-8") - run_cmd(["git", "add", "README.md", ".githooks"], tmp_path) - run_cmd(["git", "commit", "-q", "-m", "seed", "--no-verify"], tmp_path) + (tmp_path / "README.template.md").write_text("seed\n", encoding="utf-8") + (tmp_path / ".gitignore").write_text("existing\n", encoding="utf-8") + gate.run_git(["add", ".gitignore", "README.md", "README.template.md"], tmp_path) + gate.run_git(["commit", "-q", "-m", "seed"], tmp_path) + monkeypatch.setattr(cli, "REPO_ROOT", tmp_path) monkeypatch.setattr(gate, "REPO_ROOT", tmp_path) return tmp_path diff --git a/harness/tests/test_cli.py b/harness/tests/test_cli.py index 11c4368..ad4d753 100644 --- a/harness/tests/test_cli.py +++ b/harness/tests/test_cli.py @@ -1,308 +1,264 @@ -"""Tests for the ralph CLI (harness.cli). Commands drive the real Typer app; only the external -toolchain (gate checks, uv sync, the worker subprocess) is stubbed at the boundary. +"""Tests for the harness CLI (harness.cli). Commands drive the real Typer app against a temp git repo; +only the external toolchain (gate checks, package managers, the worker subprocess) is stubbed. """ from __future__ import annotations import io -import json import os +import shutil import subprocess +import sys import tomllib from datetime import UTC, datetime from pathlib import Path from types import SimpleNamespace from typing import TYPE_CHECKING +from unittest.mock import Mock import pytest -import typer -from packaging.utils import InvalidName +from click import unstyle from typer.testing import CliRunner from harness import cli, gate -from harness.tests.conftest import run_cmd +from harness.gate import AGENTS, FORBIDDEN_DIRS, FORBIDDEN_FILES, FORBIDDEN_PATTERNS, FULL_CHECKS +from harness.tests.conftest import REPO_ROOT, fake_popen if TYPE_CHECKING: from collections.abc import Callable - from typing import Self runner = CliRunner() -REPO_ROOT = Path(__file__).resolve().parents[2] -def returns(fail: list[str], passed: list[str] | None = None) -> Callable[[], dict[str, list[str]]]: - """Build a typed stand-in for gate.run_preflight / gate.run_gate that returns fixed results. +def stub_toolchain( + real: Callable[..., subprocess.CompletedProcess[str]], + calls: list[tuple[str, ...]], + poetry_python: str = "", +) -> Callable[..., subprocess.CompletedProcess[str]]: + """Record every launched command, running git for real and reporting a clean exit for the rest. - `fail` is the list of check names that fail; passing an empty list means a clean gate. - `pass` is the list of checks that pass (defaults to a single 'lint' so the summary always - renders at least one PASSED row). + `poetry_python` is what `poetry env info --executable` reports, the way the real Poetry does. """ - def check() -> dict[str, list[str]]: - return {"pass": passed if passed is not None else ["lint"], "fail": fail, "warn": []} - - return check - - -def stub_toolchain(real: Callable[..., object], calls: list[tuple[str, ...]]) -> Callable[..., object]: - """Run git for real, stub everything else (uv sync) with a clean exit.""" - - def fake(args: tuple[str, ...] | list[str], **kwargs: object) -> object: + def fake(args: tuple[str, ...] | list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: calls.append(tuple(args)) if tuple(args)[:1] == ("git",): return real(args, **kwargs) - completed: subprocess.CompletedProcess[str] = subprocess.CompletedProcess(list(args), 0) - return completed + reported = f"{poetry_python}\n" if tuple(args)[:2] == ("poetry", "env") else "" + return subprocess.CompletedProcess(list(args), 0, reported) return fake -def fake_agent(captured: dict[str, list[list[str]]], code: int = 0) -> Callable[..., object]: - """Stand in for the worker: record the launched command and write canned jsonl to its stdout.""" +def fake_agent(captured: list[list[str]], code: int = 0) -> Callable[..., subprocess.CompletedProcess[str]]: + """Stand in for the worker: record the launched command and write one jsonl line to its stdout.""" - def fake(command: list[str], *, stdout: io.TextIOBase | None = None, **kwargs: object) -> object: + def fake( + command: list[str], *, stdout: io.TextIOBase | None = None, **kwargs: object + ) -> subprocess.CompletedProcess[str]: del kwargs - captured.setdefault("commands", []).append(list(command)) + captured.append(list(command)) if stdout is not None: - stdout.write('{"type":"result","result":"ok"}\n') # the "agent" emits one line - completed: subprocess.CompletedProcess[str] = subprocess.CompletedProcess(list(command), code) - return completed + stdout.write('{"type":"result","result":"ok"}\n') + return subprocess.CompletedProcess(list(command), code) return fake -def write_log(repo: Path, name: str) -> None: - """Drop a run receipt under scratchpad/runs.""" - runs = repo / "scratchpad" / "runs" - runs.mkdir(parents=True, exist_ok=True) - (runs / name).write_text("{}\n", encoding="utf-8") +def which_finds(*tools: str) -> Callable[[str], str | None]: + """A shutil.which stand-in that finds only the named tools on PATH.""" + def which(name: str) -> str | None: + return f"/usr/bin/{name}" if name in tools else None -def write_executable(path: Path, text: str) -> None: - """Write an executable script for CLI integration tests.""" - path.write_text(text, encoding="utf-8") - path.chmod(0o755) + return which -def seed_prompt(cwd: Path) -> None: - """Create docs/PROMPT.md so `run` (which reads it into RALPH_PROMPT) has a prompt to pass.""" - (cwd / "docs").mkdir(parents=True, exist_ok=True) - (cwd / "docs" / "PROMPT.md").write_text("do the most important thing\n", encoding="utf-8") +def normalized_path(path: str | Path) -> str: + """Normalize recorded executable paths for comparisons across operating systems.""" + return os.path.normcase(os.path.normpath(str(path))) -def frozen_now(tz: object | None = None) -> datetime: - """Fixed clock for cli.run's dated log dir: 2099-01-02 UTC -> "20990102".""" - del tz - return datetime(2099, 1, 2, tzinfo=UTC) +def harness_executable(env_bin: Path) -> Path: + """Return the installed console-script path for the current platform.""" + return env_bin / ("harness.exe" if sys.platform == "win32" else "harness") def freeze_run_day(monkeypatch: pytest.MonkeyPatch) -> None: - """Pin cli.run's dated log dir so path assertions cannot race midnight. The dated dir is 20990102.""" - monkeypatch.setattr(cli, "datetime", SimpleNamespace(now=frozen_now)) + """Pin cli.run's dated receipt dir to 20990102 so path assertions cannot race midnight.""" + + def now(tz: object) -> datetime: + del tz + return datetime(2099, 1, 2, tzinfo=UTC) + + monkeypatch.setattr(cli, "datetime", SimpleNamespace(now=now)) -# --------------------------------------------------------------------------- entry point +def write_executable(path: Path, text: str) -> None: + """Write an executable script for the end-to-end loop test.""" + path.write_text(text, encoding="utf-8") + path.chmod(0o755) -def test_main_propagates_exit_code() -> None: - """The console-script entry point lets typer.Exit reach the shell.""" +def test_entry_point_propagates_exit_codes_and_rejects_unknown_commands( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The console script lets typer.Exit reach the shell; unknown or missing commands are usage errors.""" with pytest.raises(SystemExit) as exit_info: cli.main(["--help"]) - assert exit_info.value.code == 0 - -def test_unknown_command_is_usage_error() -> None: - """An unknown command and no command both exit 2.""" + assert exit_info.value.code == 0 assert runner.invoke(cli.app, ["bogus"]).exit_code == 2 assert runner.invoke(cli.app, []).exit_code == 2 + fake_popen(monkeypatch, fails=[gate.COMMIT_CHECKS["lint"], gate.COMMIT_CHECKS["format"]]) + rejected = runner.invoke(cli.app, ["preflight"]) + summary = " ".join(unstyle(rejected.stdout).split()) -def test_completion_options_are_not_exposed() -> None: - """The harness help stays focused on harness commands, not shell completion plumbing.""" - result = runner.invoke(cli.app, ["--help"]) - assert result.exit_code == 0 - assert "--install-completion" not in result.output - assert "--show-completion" not in result.output - - -def test_git_hooks_call_commands_that_exist() -> None: - """The git hooks must invoke harness commands that are actually registered.""" - for hook in (".githooks/pre-commit", ".githooks/pre-push"): - text = (REPO_ROOT / hook).read_text(encoding="utf-8") - called = [ - tokens[index + 1] - for tokens in (line.split() for line in text.splitlines()) - for index, token in enumerate(tokens) - if token.endswith("harness") and index + 1 < len(tokens) - ] - assert called, f"{hook} does not invoke harness" - for command in called: - assert runner.invoke(cli.app, [command, "--help"]).exit_code == 0 - - -def test_run_exposes_verbose_as_positional_without_disable_flag() -> None: - """Run accepts positional verbose and does not expose a --no-verbose CLI flag.""" - result = runner.invoke(cli.app, ["run", "--help"]) - assert result.exit_code == 0 - assert "verbose" in result.output - assert "--verbose" not in result.output - assert "--no-verbose" not in result.output + assert rejected.exit_code == 1 + assert "FAILED lint" in summary + assert "WARNED format" in summary + assert "rejected by harness" in summary -# --------------------------------------------------------------------------- preflight / gate +def test_help_and_info_surface_every_check_agent_and_containment_rule( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Nobody has to open pyproject.toml: info renders both phases with their argv, the containment + lists and the agents, while help offers the human commands and hides the git-only plumbing. + """ + monkeypatch.setattr(cli.console, "width", 40) + info = runner.invoke(cli.app, ["info"]) + assert info.exit_code == 0 + flat = " ".join(unstyle(info.output).split()) + for phase in ("preflight", "gate"): + assert phase in flat + for name, command in FULL_CHECKS.items(): + assert name in flat + assert command[0] in flat + for pattern in FORBIDDEN_PATTERNS: + assert pattern in flat + for path in (*FORBIDDEN_DIRS, *FORBIDDEN_FILES): + assert path in flat + for agent in AGENTS: + assert agent in flat -def test_preflight_passes_when_gate_clean(monkeypatch: pytest.MonkeyPatch) -> None: - """A human run of a clean preflight renders the Rich summary (styled) and exits 0.""" - monkeypatch.delenv("RALPH_LOOP", raising=False) - monkeypatch.setattr(gate, "run_preflight", returns([], passed=["lint"])) - result = runner.invoke(cli.app, ["preflight"]) - assert result.exit_code == 0 - assert "\x1b[" in result.stderr # humans get styled output - assert "Harness Summary" in result.stderr - assert "lint" in result.stderr - assert "ok: preflight pass" in result.stderr - assert "rejected by harness" not in result.stderr + run_help = runner.invoke(cli.app, ["run", "--help"]) + assert run_help.exit_code == 0 + for agent in AGENTS: + assert agent in run_help.output + assert "verbose" in run_help.output + assert "--verbose" not in run_help.output + assert "--no-verbose" not in run_help.output -def test_preflight_rejects_and_names_the_fail_check(monkeypatch: pytest.MonkeyPatch) -> None: - """A human run with a failing preflight names the check and rejects, styled, exit 1.""" - monkeypatch.delenv("RALPH_LOOP", raising=False) - monkeypatch.setattr(gate, "run_preflight", returns(["lint"])) - result = runner.invoke(cli.app, ["preflight"]) - assert result.exit_code == 1 - assert "\x1b[" in result.stderr - assert "lint" in result.stderr - assert "rejected by harness" in result.stderr + root_help = runner.invoke(cli.app, ["--help"]) + assert root_help.exit_code == 0 + assert "preflight" in root_help.output + assert "prepare-commit-msg" not in root_help.output + assert "--install-completion" not in root_help.output + assert "--show-completion" not in root_help.output -def test_gate_passes_when_checks_clean(monkeypatch: pytest.MonkeyPatch) -> None: - """A human run of a clean gate exits 0 and does not reject.""" - monkeypatch.delenv("RALPH_LOOP", raising=False) - monkeypatch.setattr(gate, "run_gate", returns([])) - result = runner.invoke(cli.app, ["gate"]) - assert result.exit_code == 0 - assert "rejected by harness" not in result.stderr - assert "ok: gate pass" in result.stderr +def test_every_supported_agent_has_a_nonempty_command() -> None: + """Every advertised agent has a usable argv preset rather than a missing or blank command.""" + agents: dict[str, list[str]] = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8"))[ + "tool" + ]["harness"]["agents"] -def test_gate_rejects_on_failure(monkeypatch: pytest.MonkeyPatch) -> None: - """A human run with a failing gate names the check and rejects, exit 1.""" - monkeypatch.delenv("RALPH_LOOP", raising=False) - monkeypatch.setattr(gate, "run_gate", returns(["types"])) - result = runner.invoke(cli.app, ["gate"]) - assert result.exit_code == 1 - assert "types" in result.stderr - assert "rejected by harness" in result.stderr + assert agents == AGENTS + assert set(agents) == {"claude", "codex", "agy", "copilot"} + assert all(isinstance(command, list) and bool(command) for command in agents.values()) + assert all( + isinstance(argument, str) and bool(argument) for command in agents.values() for argument in command + ) -def test_agent_gate_summary_is_plain_json(monkeypatch: pytest.MonkeyPatch) -> None: - """Under RALPH_LOOP the summary is plain (no ANSI) JSON carrying the same pass/fail info.""" +def test_preflight_summary_names_every_check_for_agents( + monkeypatch: pytest.MonkeyPatch, git_repo: Path +) -> None: + """The preflight summary names configured checks and the separate preferences containment result.""" monkeypatch.setenv("RALPH_LOOP", "1") - monkeypatch.setattr(gate, "run_gate", returns(["types"], passed=["lint"])) - result = runner.invoke(cli.app, ["gate"]) - assert result.exit_code == 1 - payload = json.loads(result.stdout)["Harness Summary"] - assert payload == {"PASSED": ["lint"], "FAILED": ["types"], "result": "rejected by harness"} - assert "\x1b[" not in result.stdout # agents get no styled output + source = git_repo / "src" / "mod.py" + source.parent.mkdir() + source.write_text("value = 1\n", encoding="utf-8") + gate.run_git(["add", "src/mod.py"], git_repo) + fake_popen(monkeypatch) + result = runner.invoke(cli.app, ["preflight"]) + assert result.exit_code == 0 + output = " ".join(unstyle(result.stdout).split()) + assert output == ( + "PHASE: LINT ruff check --no-cache --show-fixes . PHASE: PYLINT pylint . " + "PHASE: FORMAT ruff format --no-cache --check PHASE: COMPLEXIPY complexipy . " + "PHASE: BANNED PATTERNS CHECK checking for banned patterns in staged files " + "PHASE: USER PREFERENCES checking that user's preferences are respected " + "Harness Summary RESULT CHECK PASSED lint PASSED pylint PASSED format PASSED complexipy " + "ok: preflight pass" + ) + for name, command in gate.COMMIT_CHECKS.items(): + assert name in output + assert " ".join(command) in output -def test_agent_gate_summary_reports_pass(monkeypatch: pytest.MonkeyPatch) -> None: - """Under RALPH_LOOP a clean gate emits plain JSON with the pass result and exits 0.""" +def test_gate_summary_names_every_check_for_agents(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: + """The gate summary names configured checks and the separate preferences containment result.""" monkeypatch.setenv("RALPH_LOOP", "1") - monkeypatch.setattr(gate, "run_gate", returns([], passed=["lint"])) + source = git_repo / "src" / "mod.py" + source.parent.mkdir() + source.write_text("value = 1\n", encoding="utf-8") + gate.run_git(["add", "src/mod.py"], git_repo) + fake_popen(monkeypatch) result = runner.invoke(cli.app, ["gate"]) assert result.exit_code == 0 - payload = json.loads(result.stdout)["Harness Summary"] - assert payload == {"PASSED": ["lint"], "FAILED": [], "result": "ok: gate pass"} - assert "\x1b[" not in result.stdout + output = " ".join(unstyle(result.stdout).split()) + assert output == ( + "PHASE: LINT ruff check --no-cache --show-fixes . PHASE: PYLINT pylint . " + "PHASE: FORMAT ruff format --no-cache --check " + "PHASE: COMPLEXIPY complexipy . " + "PHASE: SECURITY semgrep scan --error --config auto --config p/secrets --exclude-rule " + "yaml.github-actions.security.github-actions-mutable-action-tag.github-actions-mutable-action-tag . " + "PHASE: TYPES pyright --outputjson " + "PHASE: PYTEST pytest -p no:cacheprovider -n auto --cov " + "--cov-report=term-missing --cov-fail-under=100 " + "PHASE: BANNED PATTERNS CHECK checking for banned patterns in staged files " + "PHASE: USER PREFERENCES checking that user's preferences are respected " + "Harness Summary RESULT CHECK PASSED lint PASSED pylint PASSED format PASSED complexipy " + "PASSED security PASSED types PASSED pytest ok: gate pass" + ) + for name, command in gate.FULL_CHECKS.items(): + assert name in output + assert " ".join(command) in output -def test_verify_passes_when_gate_clean(monkeypatch: pytest.MonkeyPatch) -> None: - """Verify is gone, so it cannot pass through to run_gate.""" - monkeypatch.setattr(gate, "run_gate", pytest.fail) - result = runner.invoke(cli.app, ["verify"]) - assert result.exit_code == 2 - assert "No such command 'verify'" in result.output - assert "ok: verify pass" not in result.output +def test_status_counts_run_receipts_and_names_the_newest(git_repo: Path) -> None: + """Status reports zero without crashing, then counts the receipts and points at the last one.""" + empty = runner.invoke(cli.app, ["status"]) + assert empty.exit_code == 0 + assert "0 run log(s)" in empty.stdout -def test_verify_rejects_on_failure(monkeypatch: pytest.MonkeyPatch) -> None: - """Verify is gone, so even a failing gate stub is never called.""" - monkeypatch.setattr(gate, "run_gate", pytest.fail) - result = runner.invoke(cli.app, ["verify"]) - assert result.exit_code == 2 - assert "No such command 'verify'" in result.output - assert "gate: security fail" not in result.output + runs = git_repo / "scratchpad" / "runs" + runs.mkdir(parents=True) + (runs / "0001-claude.jsonl").write_text("{}\n", encoding="utf-8") + (runs / "0002-codex.jsonl").write_text("{}\n", encoding="utf-8") + counted = runner.invoke(cli.app, ["status"]) -# --------------------------------------------------------------------------- info + assert counted.exit_code == 0 + assert "2 run log(s)" in counted.stdout + assert "newest: " in counted.stdout + assert "0002-codex.jsonl" in counted.stdout -def test_info_prints_all_harness_config() -> None: - """Info surfaces every [tool.harness] section so nobody has to open pyproject.toml: both check - phases with their argv, the containment lists, and the integrated agents. +def test_installing_the_template_cleans_the_repo_sets_hooks_and_reruns_cleanly( + monkeypatch: pytest.MonkeyPatch, git_repo: Path +) -> None: + """Install turns a freshly cloned template into the user's own project: it names the project, starts + it at v0, scopes the project checks away from the embedded harness, deletes what the template + shipped for itself, syncs dependencies and activates the git hooks. Running it again is harmless. """ - result = runner.invoke(cli.app, ["info"]) - assert result.exit_code == 0 - flat = " ".join(result.output.split()) # collapse Rich's line-wrapping so long entries stay whole - for phase in ("preflight", "gate"): - assert phase in flat - for name, command in (gate.COMMIT_CHECKS | gate.gate).items(): - assert name in flat - assert command[0] in flat # the argv is rendered, not just the check name - for pattern in gate.FORBIDDEN_PATTERNS: - assert pattern in flat - for path in (*gate.FORBIDDEN_DIRS, *gate.FORBIDDEN_FILES): - assert path in flat - for agent in gate.AGENTS: - assert agent in flat - - -def test_run_help_lists_integrated_agents() -> None: - """`run --help` names every integrated agent so callers see the choices without reading the toml.""" - result = runner.invoke(cli.app, ["run", "--help"]) - assert result.exit_code == 0 - for agent in gate.AGENTS: - assert agent in result.output - - -# --------------------------------------------------------------------------- status - - -def test_status_reports_zero_when_empty(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - """No logs β†’ reports 0, no crash.""" - monkeypatch.chdir(tmp_path) - seed_prompt(tmp_path) - result = runner.invoke(cli.app, ["status"]) - assert result.exit_code == 0 - assert "0 run log(s)" in result.stdout - - -def test_status_counts_logs_and_names_newest(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - """Status counts the *.jsonl logs and points at the newest (last sorted).""" - monkeypatch.chdir(tmp_path) - seed_prompt(tmp_path) - write_log(tmp_path, "0001-claude.jsonl") - write_log(tmp_path, "0002-codex.jsonl") - result = runner.invoke(cli.app, ["status"]) - assert result.exit_code == 0 - assert "2 run log(s)" in result.stdout - assert "newest: " in result.stdout - assert "0002-codex.jsonl" in result.stdout - - -def test_cli_does_not_shadow_builtin_print() -> None: - """CLI output uses Typer helpers, so stderr handling and lint stay clean.""" - assert "print" not in cli.__dict__ - - -# --------------------------------------------------------------------------- install - - -def test_install_renames_syncs_and_sets_hooks(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: - """Install sets the name (PEP 503), preserves an existing version + metadata, syncs, sets hooks.""" - monkeypatch.chdir(git_repo) (git_repo / "pyproject.toml").write_text( "[project]\n" 'name = "old-name"\n' @@ -311,440 +267,480 @@ def test_install_renames_syncs_and_sets_hooks(monkeypatch: pytest.MonkeyPatch, g 'authors = [{ name = "someone" }]\n' 'requires-python = ">=3.11"\n' "\n[project.scripts]\n" - 'harness = "harness.cli:main"\n', + 'harness = "harness.cli:main"\n' + "\n[tool.pyright]\n" + 'typeCheckingMode = "strict"\n' + 'include = ["src", "harness"]\n' + "\n[tool.pytest.ini_options]\n" + 'addopts = ["-ra"]\n' + 'testpaths = ["tests", "harness"]\n' + 'pythonpath = ["src", "harness"]\n' + "\n[tool.coverage]\n" + 'run.source = ["src", "harness"]\n' + "report.fail_under = 100\n" + "\n[tool.complexipy]\n" + 'paths = ["src", "harness"]\n' + "max-complexity-allowed = 10\n" + "\n[tool.ruff]\n" + 'exclude = [".git"]\n' + "\n[tool.pylint.main]\n" + 'ignore = [".git"]\n', encoding="utf-8", ) + (git_repo / "uv.lock").touch() + template_files = (".banner.svg", ".diagram.png", ".infin.png", ".loops_agents.svg", ".loops.svg") + for file_name in template_files: + (git_repo / file_name).touch() + (git_repo / ".github" / "workflows").mkdir(parents=True) + (git_repo / ".github" / "workflows" / "publish.yml").touch() + (git_repo / "CONTRIBUTING.md").touch() + (git_repo / "dist").mkdir() + (git_repo / "dist" / "stale.whl").touch() + for directory in ("harness/tests", "preferences", "tests/preferences"): + (git_repo / directory).mkdir(parents=True) + monkeypatch.setattr(cli.shutil, "which", which_finds("timeout")) + monkeypatch.setattr(cli, "REPO_ROOT_STR", str(git_repo)) calls: list[tuple[str, ...]] = [] monkeypatch.setattr(subprocess, "run", stub_toolchain(subprocess.run, calls)) - result = runner.invoke(cli.app, ["install", "My_Cool.Project"]) - assert result.exit_code == 0 - assert ("uv", "sync") in calls - assert ("git", "config", "core.hooksPath", ".githooks") in calls - assert ("git", "config", "core.hooksPath") in calls - assert ("ls", "-l", ".githooks") in calls - with (git_repo / "pyproject.toml").open("rb") as handle: - project = tomllib.load(handle)["project"] - assert project["name"] == "my-cool-project" # the requested name is set - assert project["version"] == "2.3.4" # existing version is preserved, never clobbered - # Other metadata is left untouched (not clobbered). - assert project["description"] == "the user's own project" - assert project["authors"] == [{"name": "someone"}] - assert project["requires-python"] == ">=3.11" - assert project["scripts"] == {"harness": "harness.cli:main"} - monkeypatch.undo() - assert run_cmd(["git", "config", "core.hooksPath"], git_repo).strip() == ".githooks" - - -def test_install_defaults_version_when_absent(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: - """With no version in pyproject, install sets a starter 0.0.0 (only defaults, never clobbers).""" - monkeypatch.chdir(git_repo) - (git_repo / "pyproject.toml").write_text('[project]\nname = "old-name"\n', encoding="utf-8") - calls: list[tuple[str, ...]] = [] - monkeypatch.setattr(subprocess, "run", stub_toolchain(subprocess.run, calls)) - result = runner.invoke(cli.app, ["install", "fresh-project"]) - assert result.exit_code == 0 - with (git_repo / "pyproject.toml").open("rb") as handle: - project = tomllib.load(handle)["project"] - assert project["name"] == "fresh-project" - assert project["version"] == "0.0.0" # defaulted because none existed - - -def test_install_rejects_invalid_name(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: - """A name that can't be canonicalized raises InvalidName before any sync.""" - monkeypatch.chdir(git_repo) - (git_repo / "pyproject.toml").write_text('[project]\nname = "ok"\n', encoding="utf-8") - calls: list[tuple[str, ...]] = [] - monkeypatch.setattr(subprocess, "run", stub_toolchain(subprocess.run, calls)) - result = runner.invoke(cli.app, ["install", 'bad"name']) - assert isinstance(result.exception, InvalidName) - assert ("uv", "sync") not in calls + result = runner.invoke(cli.app, ["install", "fresh-project"]) -def test_install_without_name_keeps_existing_name(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: - """The name argument is optional: bare `install` preserves the existing project name and the rest of - install (defaults version, syncs, sets hooks) still runs. - """ - monkeypatch.chdir(git_repo) - (git_repo / "pyproject.toml").write_text('[project]\nname = "keep-me"\n', encoding="utf-8") - calls: list[tuple[str, ...]] = [] - monkeypatch.setattr(subprocess, "run", stub_toolchain(subprocess.run, calls)) - result = runner.invoke(cli.app, ["install"]) assert result.exit_code == 0 - assert ("uv", "sync") in calls # install still runs its steps without a name with (git_repo / "pyproject.toml").open("rb") as handle: - project = tomllib.load(handle)["project"] - assert project["name"] == "keep-me" # existing name untouched - assert project["version"] == "0.0.0" # version still defaulted - - -def test_install_without_name_on_nameless_pyproject(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: - """Bare `install` on a pyproject that has no `name` key must still complete: the confirmation line - only echoes the name, so a missing name must not abort install (uv sync / hooks must still run). + document = tomllib.load(handle) + assert document["project"] == { + "name": "fresh-project", + "version": "0.0.0", + "description": "the user's own project", + "authors": [{"name": "someone"}], + "requires-python": ">=3.11", + "scripts": {"harness": "harness.cli:main"}, + } + assert document["tool"]["pyright"] == {"typeCheckingMode": "strict", "include": ["src", "preferences"]} + assert document["tool"]["pytest"]["ini_options"] == { + "addopts": ["-ra"], + "testpaths": ["tests"], + "pythonpath": ["src"], + } + assert document["tool"]["coverage"] == { + "run": {"source": ["src", "preferences"]}, + "report": {"fail_under": 100}, + } + assert document["tool"]["complexipy"] == {"paths": ["src", "preferences"], "max-complexity-allowed": 10} + assert document["tool"]["ruff"]["exclude"] == [".git", "harness"] + assert document["tool"]["pylint"]["main"]["ignore"] == [".git", "harness"] + assert (git_repo / "README.md").read_text(encoding="utf-8") == "seed\n" + assert not (git_repo / "README.template.md").exists() + assert all(not (git_repo / name).exists() for name in template_files) + assert not (git_repo / ".github" / "workflows" / "publish.yml").exists() + assert not (git_repo / "CONTRIBUTING.md").exists() + assert not (git_repo / "dist").exists() + assert not (git_repo / "harness" / "tests").exists() + assert (git_repo / "preferences").is_dir() + assert (git_repo / "tests" / "preferences").is_dir() + assert ("uv", "sync") in calls + recorded_harness = (git_repo / ".git" / "harness-path").read_text(encoding="utf-8").strip() + env_bin = git_repo / ".venv" / ("Scripts" if sys.platform == "win32" else "bin") + assert normalized_path(recorded_harness) == normalized_path(harness_executable(env_bin)) + assert gate.run_git(["config", "core.hooksPath"], git_repo).strip() == ".githooks" + + again = runner.invoke(cli.app, ["install"]) + + assert again.exit_code == 0 + assert (git_repo / "README.md").read_text(encoding="utf-8") == "seed\n" + + +@pytest.mark.parametrize( + ("lockfile", "manager"), + [ + pytest.param("uv.lock", "uv", id="uv-lockfile"), + pytest.param("poetry.lock", "poetry", id="poetry-lockfile"), + pytest.param(None, "pip", id="no-lockfile"), + ], +) +def test_install_picks_the_package_manager_from_the_lockfile( + lockfile: str | None, manager: str, monkeypatch: pytest.MonkeyPatch, git_repo: Path +) -> None: + """The lockfile picks the package manager, and the hooks record the harness of the environment + that manager filled, which is not the interpreter running install unless pip did the work. """ - monkeypatch.chdir(git_repo) - (git_repo / "pyproject.toml").write_text('[project]\nversion = "1.0"\n', encoding="utf-8") # no name key + scripts = "Scripts" if sys.platform == "win32" else "bin" + python_name = "python.exe" if sys.platform == "win32" else "python" + interpreter = git_repo / ".pyenv" / scripts + poetry_bin = git_repo / ".poetry" / "virtualenvs" / "project" / scripts + monkeypatch.setattr(cli.sys, "executable", str(interpreter / python_name)) + monkeypatch.setattr(cli.shutil, "which", which_finds("timeout")) + monkeypatch.setattr(cli, "REPO_ROOT_STR", str(git_repo)) + (git_repo / "pyproject.toml").write_text('[project]\nname = "x"\n', encoding="utf-8") + if lockfile: + (git_repo / lockfile).touch() calls: list[tuple[str, ...]] = [] - monkeypatch.setattr(subprocess, "run", stub_toolchain(subprocess.run, calls)) - result = runner.invoke(cli.app, ["install"]) - assert result.exit_code == 0 # does not crash on the missing name - assert ("uv", "sync") in calls # install proceeds past the (name-less) confirmation line - with (git_repo / "pyproject.toml").open("rb") as handle: - assert "name" not in tomllib.load(handle)["project"] # bare install never invents a name - - -# --------------------------------------------------------------------------- run - - -def test_run_rejects_unknown_agent(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - """An agent not in AGENTS exits 2 with a helpful message β€” before launching anything.""" - monkeypatch.chdir(tmp_path) - seed_prompt(tmp_path) - monkeypatch.setattr(subprocess, "run", pytest.fail) - result = runner.invoke(cli.app, ["run", "bogus"]) - assert result.exit_code == 2 - assert result.stderr.strip() == "Unknown agent name 'bogus'" - assert not (tmp_path / "scratchpad").exists() + monkeypatch.setattr( + subprocess, "run", stub_toolchain(subprocess.run, calls, str(poetry_bin / python_name)) + ) + assert runner.invoke(cli.app, ["install"]).exit_code == 0 -def test_run_builds_ralph_command_and_writes_sequential_log( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path + managers = { + "uv": ("uv", "sync"), + "poetry": ("poetry", "install"), + "pip": (str(interpreter / python_name), "-m", "pip", "install", "-r", "requirements.txt", "-e", "."), + } + recorded = { + "uv": harness_executable(git_repo / ".venv" / scripts), + "poetry": harness_executable(poetry_bin), + "pip": harness_executable(interpreter), + } + assert [call for call in calls if call in managers.values()] == [managers[manager]] + installed = (git_repo / ".git" / "harness-path").read_text(encoding="utf-8").strip() + assert normalized_path(installed) == normalized_path(recorded[manager]) + + +@pytest.mark.parametrize( + ("on_path", "answer", "outcome"), + [ + pytest.param(("timeout",), None, (False, ""), id="timeout-present"), + pytest.param(("gtimeout",), None, (False, ""), id="gtimeout-present"), + pytest.param((), None, (False, "brew.sh"), id="no-timeout-no-homebrew"), + pytest.param(("brew",), True, (True, ""), id="confirmed"), + pytest.param(("brew",), False, (False, "skipped"), id="declined"), + ], +) +def test_install_offers_coreutils_only_when_no_timeout_tool_exists( + on_path: tuple[str, ...], + answer: bool | None, + outcome: tuple[bool, str], + monkeypatch: pytest.MonkeyPatch, + git_repo: Path, ) -> None: - """Run fires ralph.sh with the preset and the worker writes the dated NNNN.jsonl receipt.""" - monkeypatch.chdir(tmp_path) - seed_prompt(tmp_path) - freeze_run_day(monkeypatch) # pin the dated log dir to 20990102 so the path assertion can't race midnight - captured: dict[str, list[list[str]]] = {} - monkeypatch.setattr(subprocess, "run", fake_agent(captured)) - result = runner.invoke(cli.app, ["run", "claude", "1", "2", "False"]) - assert result.exit_code == 0 - command = captured["commands"][0] - assert command[0].endswith("ralph.sh") - assert command[1:3] == ["1", "2"] - assert command[3:] == list(gate.AGENTS["claude"]) # preset expanded - assert (tmp_path / "scratchpad" / "runs").is_dir() # run creates the log dir - log = tmp_path / "scratchpad" / "runs" / "20990102" / "claude" / "0001.jsonl" - assert log.read_text(encoding="utf-8") == '{"type":"result","result":"ok"}\n' - - -def test_run_model_option_replaces_preset_model(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - """`--model X` swaps the value after the preset's `--model` in place, so exactly one --model X is sent - (no duplicate flag) and no other preset arg changes. + """macOS needs coreutils to time out a loop iteration, so install probes for it and offers the + install only when Homebrew can do it. It never prompts when a timeout tool is already there. """ - monkeypatch.chdir(tmp_path) - seed_prompt(tmp_path) - captured: dict[str, list[list[str]]] = {} - monkeypatch.setattr(subprocess, "run", fake_agent(captured)) - result = runner.invoke(cli.app, ["run", "claude", "1", "2", "False", "--model", "haiku"]) - assert result.exit_code == 0 - agent_argv = captured["commands"][0][3:] # drop launcher + iterations + minutes - expected = list(gate.AGENTS["claude"]) - expected[expected.index("--model") + 1] = "haiku" # in-place swap, not an appended second flag - assert agent_argv == expected - assert agent_argv.count("--model") == 1 # replaced, never duplicated - - -def test_agent_presets_are_registered() -> None: - """Every supported agent has one nonempty command list registered in the CLI.""" - agents: dict[str, list[str]] = gate.AGENTS # TOML: each preset is a name -> argv-string list - assert set(agents) == {"claude", "codex", "agy", "copilot"} - for command in agents.values(): - assert command # nonempty command - assert all(part for part in command) # no empty argv entries - - -def test_run_claude_executes_real_loop_twice_with_prompt( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - """The Claude preset runs through ralph.sh and receives the prompt each iteration.""" - bin_dir = tmp_path / "bin" - bin_dir.mkdir() - write_executable(bin_dir / "gtimeout", '#!/bin/sh\nshift\nexec "$@"\n') - write_executable( - bin_dir / "claude", - ( - "#!/bin/sh\n" - "count=$(cat claude-count 2>/dev/null || printf 0)\n" - "count=$((count + 1))\n" - 'printf "%s" "$count" > claude-count\n' - 'printf "%s\\n" "$@" >> claude-args.txt\n' - 'cat > "prompt-$count.txt"\n' - 'printf \'{ "type" : "result", "result" : "ok" }\\n\'\n' - ), - ) - monkeypatch.chdir(tmp_path) - monkeypatch.setenv("PATH", f"{bin_dir}{os.pathsep}{os.environ['PATH']}") - freeze_run_day(monkeypatch) # pin the dated log dir to 20990102 - (tmp_path / "docs").mkdir() - (tmp_path / "docs" / "PROMPT.md").write_text("build from specs\n", encoding="utf-8") - - result = runner.invoke(cli.app, ["run", "claude", "2", "1"]) - - assert result.exit_code == 0 - assert (tmp_path / "claude-count").read_text(encoding="utf-8") == "2" - identity = "Your agent id is `0001`\n\n" # worker_id is NNNN, no agent suffix - assert (tmp_path / "prompt-1.txt").read_text(encoding="utf-8") == ( - f"{identity}build from specs\n\nRALPH_ITERATION=1/2\n" - ) - assert (tmp_path / "prompt-2.txt").read_text(encoding="utf-8") == ( - f"{identity}build from specs\n\nRALPH_ITERATION=2/2\n" - ) - claude_args = list(gate.AGENTS["claude"][1:]) - expected_args = claude_args.copy() - expected_args.extend(claude_args) - assert (tmp_path / "claude-args.txt").read_text(encoding="utf-8").splitlines() == expected_args - assert (tmp_path / "scratchpad" / "runs" / "20990102" / "claude" / "0001.jsonl").read_text( - encoding="utf-8" - ) == '{ "type" : "result", "result" : "ok" }\n{ "type" : "result", "result" : "ok" }\n' - - -def capture_run_worker(seen: list[list[str]]) -> Callable[..., int]: - """A run_worker stand-in that records the command run built and reports a clean exit.""" - - def worker(command: list[str], cwd: Path, log: Path, verbose: bool) -> int: - del cwd, log, verbose - seen.append(command) - return 0 - - return worker + installs_coreutils, hint = outcome + prompts: list[str] = [] + def confirm(prompt: str) -> bool: + prompts.append(prompt) + return bool(answer) -def test_run_uses_shell_script_off_windows(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - """Off Windows, run launches ralph.sh directly, with the counts then the agent argv.""" - seed_prompt(tmp_path) - monkeypatch.chdir(tmp_path) monkeypatch.setattr(cli.sys, "platform", "darwin") - seen: list[list[str]] = [] - monkeypatch.setattr(cli, "run_worker", capture_run_worker(seen)) - runner.invoke(cli.app, ["run", "codex", "3", "8"]) - assert seen[0][0].endswith("ralph.sh") - assert seen[0][1:3] == ["3", "8"] - - -def test_run_uses_powershell_on_windows(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - """On Windows, run launches ralph.ps1 through powershell, keeping ralph.sh untouched.""" - seed_prompt(tmp_path) - monkeypatch.chdir(tmp_path) - monkeypatch.setattr(cli.sys, "platform", "win32") - seen: list[list[str]] = [] - monkeypatch.setattr(cli, "run_worker", capture_run_worker(seen)) - runner.invoke(cli.app, ["run", "claude", "2", "5"]) - assert seen[0][0] == "powershell.exe" - assert any(part.endswith("ralph.ps1") for part in seen[0]) - assert seen[0][seen[0].index("2") : seen[0].index("2") + 2] == ["2", "5"] - - -def which_only(present: str) -> Callable[[str], str | None]: - """A shutil.which stand-in that finds only the named tool on PATH.""" - - def which(name: str) -> str | None: - return f"/usr/bin/{name}" if name == present else None - - return which - - -def which_none(name: str) -> None: - """A shutil.which stand-in where nothing is on PATH.""" - del name - - -def say_yes(prompt: str) -> bool: - """A typer.confirm stand-in that always confirms.""" - del prompt - return True - - -def say_no(prompt: str) -> bool: - """A typer.confirm stand-in that always declines.""" - del prompt - return False - - -def install_in(monkeypatch: pytest.MonkeyPatch, repo: Path, calls: list[tuple[str, ...]]) -> None: - """Run `install` in `repo` with a seeded pyproject, recording every subprocess call into `calls`. - - The timeout-tool step is inlined at the tail of install, so these tests exercise it through install. - """ - monkeypatch.chdir(repo) - (repo / "pyproject.toml").write_text('[project]\nname = "x"\n', encoding="utf-8") + monkeypatch.setattr(cli.shutil, "which", which_finds(*on_path)) + monkeypatch.setattr(cli.typer, "confirm", confirm) + monkeypatch.setattr(cli, "REPO_ROOT_STR", str(git_repo)) + (git_repo / "pyproject.toml").write_text('[project]\nname = "x"\n', encoding="utf-8") + calls: list[tuple[str, ...]] = [] monkeypatch.setattr(subprocess, "run", stub_toolchain(subprocess.run, calls)) - assert runner.invoke(cli.app, ["install"]).exit_code == 0 + result = runner.invoke(cli.app, ["install"]) -def test_install_timeout_skips_when_present(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: - """With a timeout tool on PATH, install neither prompts nor installs coreutils.""" - monkeypatch.setattr(cli.sys, "platform", "darwin") - monkeypatch.setattr(cli.shutil, "which", which_only("timeout")) - monkeypatch.setattr(cli.typer, "confirm", pytest.fail) # must not prompt - calls: list[tuple[str, ...]] = [] - install_in(monkeypatch, git_repo, calls) - assert ("brew", "install", "coreutils") not in calls + assert result.exit_code == 0 + assert (("brew", "install", "coreutils") in calls) is installs_coreutils + assert prompts == ([] if answer is None else ["[magenta]Install now `brew install coreutils`?[/magenta]"]) + assert hint in result.stdout -def test_install_timeout_skips_on_windows(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: - """On Windows the ps1 path handles timing, so no coreutils probe or prompt runs.""" +def test_windows_skips_posix_steps_and_launches_the_powershell_twin( + monkeypatch: pytest.MonkeyPatch, git_repo: Path +) -> None: + """Windows has no POSIX shell, ls or coreutils, so install records harness.exe and warns that the + support is experimental, and a run goes through PowerShell instead of ralph.sh. + """ + monkeypatch.chdir(git_repo) monkeypatch.setattr(cli.sys, "platform", "win32") - monkeypatch.setattr(cli.shutil, "which", pytest.fail) # must not even probe PATH for timeout tools - monkeypatch.setattr(cli.typer, "confirm", pytest.fail) - install_in(monkeypatch, git_repo, []) - - -def test_install_timeout_installs_when_confirmed(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: - """Missing tool + brew present + user confirms -> install shells out to `brew install coreutils`.""" - monkeypatch.setattr(cli.sys, "platform", "darwin") - monkeypatch.setattr(cli.shutil, "which", which_only("brew")) - monkeypatch.setattr(cli.typer, "confirm", say_yes) + monkeypatch.setattr(cli.shutil, "which", which_finds("uv")) + monkeypatch.setattr(cli, "REPO_ROOT_STR", str(git_repo)) + (git_repo / "pyproject.toml").write_text('[project]\nname = "x"\n', encoding="utf-8") + (git_repo / "uv.lock").touch() calls: list[tuple[str, ...]] = [] - install_in(monkeypatch, git_repo, calls) - assert ("brew", "install", "coreutils") in calls + monkeypatch.setattr(subprocess, "run", stub_toolchain(subprocess.run, calls)) + installed = runner.invoke(cli.app, ["install"]) -def test_install_timeout_points_to_homebrew(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: - """No timeout tool and no Homebrew -> install never prompts or installs, just points at brew.sh.""" - monkeypatch.setattr(cli.sys, "platform", "darwin") - monkeypatch.setattr(cli.shutil, "which", which_none) # no timeout, no brew - monkeypatch.setattr(cli.typer, "confirm", pytest.fail) # cannot confirm without brew - calls: list[tuple[str, ...]] = [] - install_in(monkeypatch, git_repo, calls) + assert installed.exit_code == 0 + assert ("ls", "-l", ".githooks") not in calls assert ("brew", "install", "coreutils") not in calls + assert "source .venv/bin/activate" not in installed.stdout + assert "Windows is experimental. Reoprt issues" in installed.stdout + assert "https://github.com/rxdt/loopgate_harness/issues" in installed.stdout + assert (git_repo / ".git" / "harness-path").read_text(encoding="utf-8") == ( + f"{(git_repo / '.venv' / 'Scripts' / 'harness.exe').as_posix()}\n" + ) + launched: list[list[str]] = [] -def test_install_timeout_skips_install_when_declined(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: - """Missing tool, brew present, user declines -> nothing is installed, just a hint.""" - monkeypatch.setattr(cli.sys, "platform", "darwin") - monkeypatch.setattr(cli.shutil, "which", which_only("brew")) - monkeypatch.setattr(cli.typer, "confirm", say_no) - calls: list[tuple[str, ...]] = [] - install_in(monkeypatch, git_repo, calls) - assert ("brew", "install", "coreutils") not in calls + def capture_worker(command: list[str], log: Path, verbose: bool) -> int: + del log, verbose + launched.append(command) + return 0 + monkeypatch.setattr(cli, "run_worker", capture_worker) + (git_repo / "docs").mkdir() + (git_repo / "docs" / "PROMPT.md").write_text("do the most important thing\n", encoding="utf-8") -def test_run_log_sequence_increments_past_existing(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - """The receipt number is max(existing NNNN in that dated/agent dir) + 1, so a prior run is never lost.""" - monkeypatch.chdir(tmp_path) - seed_prompt(tmp_path) - freeze_run_day(monkeypatch) # dated dir is 20990102 - claude_dir = tmp_path / "scratchpad" / "runs" / "20990102" / "claude" - claude_dir.mkdir(parents=True) - (claude_dir / "0007.jsonl").write_text("{}\n", encoding="utf-8") # prior run in this dated/agent dir - monkeypatch.setattr(subprocess, "run", fake_agent({})) - assert runner.invoke(cli.app, ["run", "claude", "2", "20", "False"]).exit_code == 0 - assert (claude_dir / "0008.jsonl").exists() # max(0007)+1 - assert (claude_dir / "0007.jsonl").read_text(encoding="utf-8") == "{}\n" # prior run untouched + assert runner.invoke(cli.app, ["run", "claude", "2", "5"]).exit_code == 0 + assert launched[0][:3] == ["powershell.exe", "-NoProfile", "-File"] + assert launched[0][3].endswith("ralph.ps1") + assert launched[0][4:6] == ["2", "5"] + assert launched[0][6:] == list(AGENTS["claude"]) -@pytest.mark.parametrize("args", [["claude", "0", "1"], ["claude", "1", "0"]]) -def test_run_rejects_nonpositive_limits_before_creating_log( - args: list[str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path +def test_windows_run_uses_powershell_without_path_lookup( + monkeypatch: pytest.MonkeyPatch, git_repo: Path ) -> None: - """Nonpositive loop limits fail in the CLI before any run receipt is opened.""" - monkeypatch.chdir(tmp_path) - seed_prompt(tmp_path) - monkeypatch.setattr(subprocess, "run", pytest.fail) - result = runner.invoke(cli.app, ["run", *args]) - assert result.exit_code == 2 - assert "num_iterations and max_minutes must be >= 1" in result.stderr - assert not (tmp_path / "scratchpad").exists() - - -@pytest.mark.parametrize("code", [0, 1, 2, 124]) -def test_run_propagates_worker_exit_code(code: int, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - """ralph.sh's exit code (success, abort, usage, timeout) reaches the shell.""" - monkeypatch.chdir(tmp_path) - seed_prompt(tmp_path) - monkeypatch.setattr(subprocess, "run", fake_agent({}, code)) - assert runner.invoke(cli.app, ["run", "codex", "2", "20", "False"]).exit_code == code - + """Windows relies on its system PowerShell command without resolving a separate executable.""" + monkeypatch.chdir(git_repo) + monkeypatch.setattr(cli.sys, "platform", "win32") + path_lookup = Mock(return_value=None) + worker = Mock(return_value=0) + monkeypatch.setattr(cli.shutil, "which", path_lookup) + monkeypatch.setattr(cli, "run_worker", worker) + (git_repo / "docs").mkdir() + (git_repo / "docs" / "PROMPT.md").write_text("do the most important thing\n", encoding="utf-8") -class FakeProcess: - """Stand in for the worker subprocess: replays canned stdout lines and a fixed exit code.""" + result = runner.invoke(cli.app, ["run", "claude"]) - def __init__(self, lines: list[str], code: int) -> None: - self.stdout = iter(lines) - self.code = code + assert result.exit_code == 0 + path_lookup.assert_not_called() + worker.assert_called_once() + command, log, verbose = worker.call_args.args + assert command[:3] == ["powershell.exe", "-NoProfile", "-File"] + assert log.parent.is_dir() + assert verbose is True + + +@pytest.mark.parametrize( + ("initial_name", "requested_name", "expected_name"), + [ + pytest.param("old-name", "fresh-project", "fresh-project", id="normalized-explicit-name"), + pytest.param("old-name", "I_build.Things!", "my-app-name", id="non-normalized-name"), + pytest.param("old-name", '*bad"-name_!/ ', "my-app-name", id="invalid-name"), + pytest.param("old-name", None, "my-app-name", id="omitted-name"), + pytest.param(None, None, "my-app-name", id="omitted-name-with-nameless-project"), + ], +) +def test_cleanup_applies_the_project_name_rules( + tmp_path: Path, initial_name: str | None, requested_name: str | None, expected_name: str +) -> None: + """Cleanup starts the project at v0 and only accepts a name that is already PEP 503 normalized.""" + (tmp_path / "README.md").write_text("old\n", encoding="utf-8") + (tmp_path / "README.template.md").write_text("seed\n", encoding="utf-8") + (tmp_path / ".gitignore").write_text("existing\n\nuv.lock\n", encoding="utf-8") + project_toml = "[project]\n" + if initial_name is not None: + project_toml += f'name = "{initial_name}"\n' + (tmp_path / "pyproject.toml").write_text(project_toml, encoding="utf-8") + + assert cli.cleanup(tmp_path, requested_name) is True + + with (tmp_path / "pyproject.toml").open("rb") as handle: + project = tomllib.load(handle)["project"] + assert project["name"] == expected_name + assert project["version"] == "0.0.0" + assert not (tmp_path / "README.template.md").exists() + assert (tmp_path / ".gitignore").read_text(encoding="utf-8") == "existing\n" + + +@pytest.mark.parametrize("hook", ["pre-commit", "pre-push", "prepare-commit-msg"]) +def test_tracked_hooks_call_registered_commands_without_venv_paths(hook: str) -> None: + """Each hook invokes a harness command that exists and assumes no POSIX virtualenv layout.""" + text = (REPO_ROOT / ".githooks" / hook).read_text(encoding="utf-8") + called = [ + words[index + 1] + for words in (line.split() for line in text.splitlines()) + for index, word in enumerate(words) + if word == '"$HARNESS"' and index + 1 < len(words) + ] + + assert called, f"{hook} does not invoke harness" + for command in called: + assert runner.invoke(cli.app, [command, "--help"]).exit_code == 0 + assert ".venv/bin/harness" not in text + assert ".venv/bin/python" not in text + assert "uv" not in text + + +@pytest.mark.parametrize( + ("recorded", "message"), + [ + pytest.param(None, "hooks are not installed. Run 'harness install' in this repo.", id="never-ran"), + pytest.param("missing-harness", "is gone. Re-run 'harness install'.", id="stale-record"), + ], +) +def test_hooks_name_the_fix_when_the_recorded_harness_is_missing_or_stale( + recorded: str | None, message: str, git_repo: Path +) -> None: + """A repo without the install record, or one whose environment was rebuilt, gets instructions.""" + shutil.copytree(REPO_ROOT / ".githooks", git_repo / ".githooks", dirs_exist_ok=True) + if recorded: + (git_repo / ".git" / "harness-path").write_text(f"{git_repo / recorded}\n", encoding="utf-8") + env = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} + gate.run_git(["config", "core.hooksPath", ".githooks"], git_repo) + + result = subprocess.run( + ["git", "commit", "--allow-empty", "-m", "exercise pre-commit"], + cwd=git_repo, + capture_output=True, + text=True, + check=False, + env=env, + ) - def __enter__(self) -> Self: - return self + assert result.returncode == 1 + assert message in result.stderr - def __exit__(self, *exc: object) -> bool: - del exc - return False - def wait(self) -> int: - return self.code +@pytest.mark.parametrize( + ("arguments", "code"), + [ + pytest.param([".git/COMMIT_EDITMSG", "merge"], 1, id="blocked-merge"), + pytest.param([], 0, id="no-arguments"), + ], +) +def test_prepare_commit_msg_forwards_gits_own_arguments( + arguments: list[str], code: int, monkeypatch: pytest.MonkeyPatch +) -> None: + """The hook command hands git's own arguments to the gate logic and exits with its status.""" + seen: list[list[str]] = [] + def commit_msg(argv: list[str]) -> int: + seen.append(argv) + return code -def fake_popen(lines: list[str], code: int = 0) -> Callable[..., FakeProcess]: - """Stand in for subprocess.Popen: yield canned worker stdout lines, then exit with code.""" + monkeypatch.setattr(cli, "commit_msg", commit_msg) - def make(command: list[str], **kwargs: object) -> FakeProcess: - del command, kwargs - return FakeProcess(lines, code) + result = runner.invoke(cli.app, ["prepare-commit-msg", *arguments]) - return make + assert result.exit_code == code + assert seen == [["prepare-commit-msg", *arguments]] -def test_run_worker_streams_and_logs_json_and_invalid_lines( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path +def test_a_harnessed_run_writes_numbered_receipts_and_propagates_exit_codes( + monkeypatch: pytest.MonkeyPatch, git_repo: Path ) -> None: - """Verbose streaming writes the raw stdout to the log, JSON and non-JSON alike, and does - not crash on a non-JSON line. Terminal coloring is a human cosmetic, so it is not asserted here. + """A day of runs replayed: bad arguments are refused before anything is created, each accepted run + launches ralph.sh with the agent's preset and lands its own numbered receipt beside the earlier + ones, an overridden model replaces the preset's, and the worker's exit code reaches the shell. """ - monkeypatch.setattr(cli.subprocess, "Popen", fake_popen(['{ "type" : "result" }\n', "not json\n"])) + monkeypatch.chdir(git_repo) + monkeypatch.setattr(cli.sys, "platform", "linux") + monkeypatch.setenv("RALPH_PROMPT", "") + freeze_run_day(monkeypatch) + (git_repo / "docs").mkdir() + (git_repo / "docs" / "PROMPT.md").write_text("do the most important thing\n", encoding="utf-8") + launched: list[list[str]] = [] + monkeypatch.setattr(subprocess, "run", fake_agent(launched)) + + unknown = runner.invoke(cli.app, ["run", "bogus"]) + + assert unknown.exit_code == 2 + assert unknown.stderr.strip() == "Unknown agent name 'bogus'" + for limits in (["0", "1"], ["1", "0"]): + refused = runner.invoke(cli.app, ["run", "claude", *limits]) + assert refused.exit_code == 2 + assert "num_iterations and max_minutes must be >= 1" in refused.stderr + assert not (git_repo / "scratchpad").exists() + assert launched == [] + + first = runner.invoke(cli.app, ["run", "claude", "1", "2", "False"]) + + assert first.exit_code == 0 + assert not first.stdout + assert launched[0][0].endswith("ralph.sh") + assert launched[0][1:3] == ["1", "2"] + assert launched[0][3:] == list(AGENTS["claude"]) + receipts = git_repo / "scratchpad" / "runs" / "20990102" / "claude" + assert (receipts / "0001.jsonl").read_text(encoding="utf-8") == '{"type":"result","result":"ok"}\n' + assert os.environ["RALPH_PROMPT"] == "Your agent id is `0001`\n\ndo the most important thing" + + second = runner.invoke(cli.app, ["run", "claude", "1", "2", "False", "--model", "haiku"]) + + assert second.exit_code == 0 + swapped = list(AGENTS["claude"]) + swapped[swapped.index("--model") + 1] = "haiku" + assert launched[1][3:] == swapped + assert launched[1].count("--model") == 1 + assert (receipts / "0002.jsonl").exists() + + (receipts / "0007.jsonl").write_text("{}\n", encoding="utf-8") + monkeypatch.setattr(subprocess, "run", fake_agent(launched, 124)) + + timed_out = runner.invoke(cli.app, ["run", "claude", "2", "20", "False"]) + + assert timed_out.exit_code == 124 + assert (receipts / "0008.jsonl").exists() + assert (receipts / "0007.jsonl").read_text(encoding="utf-8") == "{}\n" + + +def test_run_worker_logs_every_line_and_streams_only_when_verbose( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """The receipt always gets the worker's raw stdout; verbose also renders each line live, JSON or + not, without crashing on a line that is not JSON. Terminal coloring is cosmetic and not asserted. + """ + monkeypatch.setattr(cli, "REPO_ROOT_STR", str(tmp_path)) log = tmp_path / "out.jsonl" + streaming_worker = [sys.executable, "-c", 'print(\'{ "type" : "result" }\'); print("not json")'] + + assert cli.run_worker(streaming_worker, log, verbose=True) == 0 - assert cli.run_worker(["worker"], tmp_path, log, verbose=True) == 0 + streamed = capsys.readouterr().out + assert '"type"' in streamed + assert '"result"' in streamed + assert "not json" in streamed assert log.read_text(encoding="utf-8") == '{ "type" : "result" }\nnot json\n' + failing_worker = [sys.executable, "-c", 'print("worker output"); raise SystemExit(3)'] -def test_run_worker_compacts_valid_json_in_process_without_subprocess( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] -) -> None: - """Verbose streaming renders valid JSON in-process (no per-line subprocess) and logs the raw line - verbatim. Rich's coloring/spacing of the streamed copy is TTY-dependent, so it is not asserted; - only the tokens' presence (which survive any coloring) and the untouched log are checked. - """ - monkeypatch.setattr(cli.subprocess, "run", pytest.fail) # any per-line subprocess fails the test - monkeypatch.setattr(cli.subprocess, "Popen", fake_popen(['{ "type" : "result" }\n'])) - log = tmp_path / "out.jsonl" - assert cli.run_worker(["worker"], tmp_path, log, verbose=True) == 0 - out = capsys.readouterr().out - assert '"type"' in out # the JSON tokens reach the terminal (coloring, if any, wraps each one) - assert '"result"' in out - assert log.read_text(encoding="utf-8") == '{ "type" : "result" }\n' # raw stdout logged verbatim + assert cli.run_worker(failing_worker, log, verbose=False) == 3 + assert not capsys.readouterr().out + assert log.read_text(encoding="utf-8") == "worker output\n" -def test_run_worker_passes_non_json_through_verbatim( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] -) -> None: - """A non-JSON streamed line is passed through unchanged and never crashes the renderer.""" - monkeypatch.setattr(cli.subprocess, "run", pytest.fail) - monkeypatch.setattr(cli.subprocess, "Popen", fake_popen(["not json\n"])) - log = tmp_path / "out.jsonl" - assert cli.run_worker(["worker"], tmp_path, log, verbose=True) == 0 - assert "not json" in capsys.readouterr().out +def test_claude_preset_runs_two_real_loop_iterations(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: + """The platform runner loops twice and preserves Claude's trailing -p argument.""" + bin_dir = git_repo / "bin" + bin_dir.mkdir() + write_executable(bin_dir / "gtimeout", '#!/bin/sh\nshift\nexec "$@"\n') + worker = git_repo / "claude_worker.py" + worker.write_text( + "from pathlib import Path\n" + "import json\n" + "import sys\n" + "count_path = Path('claude-count')\n" + "count = int(count_path.read_text() if count_path.exists() else '0') + 1\n" + "count_path.write_text(str(count))\n" + "with Path('claude-args.txt').open('a', encoding='utf-8') as handle:\n" + " handle.write('\\n'.join(sys.argv[1:]) + '\\n')\n" + "Path(f'prompt-{count}.txt').write_text(sys.stdin.read(), encoding='utf-8')\n" + "print(json.dumps({'type': 'result', 'result': 'ok'}))\n", + encoding="utf-8", + ) + preset = [sys.executable, str(worker), "--model", "opus", "-p"] + monkeypatch.setitem(cli.AGENTS, "claude", preset) + monkeypatch.chdir(git_repo) + monkeypatch.setenv("PATH", f"{bin_dir}{os.pathsep}{os.environ['PATH']}") + monkeypatch.setattr(cli, "REPO_ROOT_STR", str(git_repo)) + freeze_run_day(monkeypatch) + (git_repo / "docs").mkdir() + (git_repo / "docs" / "PROMPT.md").write_text("build from specs\n", encoding="utf-8") + + result = runner.invoke(cli.app, ["run", "claude", "2", "1"]) -def test_run_accepts_positional_verbose_false(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - """A fourth positional False disables live terminal streaming.""" - monkeypatch.chdir(tmp_path) - seed_prompt(tmp_path) - captured: dict[str, list[list[str]]] = {} - monkeypatch.setattr(subprocess, "run", fake_agent(captured)) - result = runner.invoke(cli.app, ["run", "claude", "1", "2", "False"]) assert result.exit_code == 0 - assert not result.stdout - - -def test_run_accepts_python_verbose_false(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - """Calling run(..., verbose=False) keeps output in the receipt only.""" - monkeypatch.chdir(tmp_path) - seed_prompt(tmp_path) - freeze_run_day(monkeypatch) # dated dir is 20990102 - captured: dict[str, list[list[str]]] = {} - monkeypatch.setattr(subprocess, "run", fake_agent(captured)) - with pytest.raises(typer.Exit) as exit_info: - cli.run("claude", 2, 20, verbose=False) - assert exit_info.value.exit_code == 0 - assert (tmp_path / "scratchpad" / "runs" / "20990102" / "claude" / "0001.jsonl").read_text( + assert (git_repo / "claude-count").read_text(encoding="utf-8") == "2" + identity = "Your agent id is `0001`\n\n" + assert (git_repo / "prompt-1.txt").read_text(encoding="utf-8") == ( + f"{identity}build from specs\n\nRALPH_ITERATION=1/2\n" + ) + assert (git_repo / "prompt-2.txt").read_text(encoding="utf-8") == ( + f"{identity}build from specs\n\nRALPH_ITERATION=2/2\n" + ) + preset_args = preset[2:] + assert (git_repo / "claude-args.txt").read_text(encoding="utf-8").splitlines() == [ + *preset_args, + *preset_args, + ] + assert (git_repo / "scratchpad" / "runs" / "20990102" / "claude" / "0001.jsonl").read_text( encoding="utf-8" - ) == '{"type":"result","result":"ok"}\n' + ) == '{"type": "result", "result": "ok"}\n{"type": "result", "result": "ok"}\n' diff --git a/harness/tests/test_gate.py b/harness/tests/test_gate.py index a165752..06eeb4f 100644 --- a/harness/tests/test_gate.py +++ b/harness/tests/test_gate.py @@ -3,18 +3,19 @@ from __future__ import annotations import importlib +import json import os +import shutil import subprocess import sys import tomllib from pathlib import Path +from unittest.mock import Mock, call import pytest from harness import gate -from harness.tests.conftest import fake_popen, run_cmd - -REPO_ROOT = Path(__file__).resolve().parents[2] +from harness.tests.conftest import REPO_ROOT, fake_popen def stage(repo: Path, name: str, content: str) -> None: @@ -22,839 +23,657 @@ def stage(repo: Path, name: str, content: str) -> None: target = repo / name target.parent.mkdir(parents=True, exist_ok=True) target.write_text(content, encoding="utf-8") - run_cmd(["git", "add", name], repo) - - -def staged() -> list[str]: - """Paths currently in the index, via the gate's own git helper (run_git returns raw stdout).""" - return gate.run_git(["diff", "--cached", "--name-only"]).splitlines() - - -def containment_fail() -> list[str]: - """Run only the loop-containment checks against the staged index.""" - return gate.run_non_human_checks() - - -# --------------------------------------------------------------------------- run_git - - -def test_run_git_returns_stdout(git_repo: Path) -> None: - """run_git runs git in the repo and returns its raw stdout string (callers .splitlines()).""" - stage(git_repo, "pkg/a.py", "x = 1\n") - assert gate.run_git(["diff", "--cached", "--name-only"]) == "pkg/a.py\n" - - -def test_run_git_ignores_poisoned_hook_env(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: - """A poisoned GIT_DIR a hook exported does not redirect the gate's git calls: run_git strips GIT_*, - so it still runs against the real repo. (Without stripping, git would honor the bogus GIT_DIR and - fail β€” this asserts the strip is load-bearing, not just that staging happens to work.) - """ - monkeypatch.setenv("GIT_DIR", str(git_repo / "does-not-exist" / ".git")) - stage(git_repo, "pkg/a.py", "x = 1\n") - assert staged() == ["pkg/a.py"] # real index read despite the poisoned GIT_DIR - - -# --------------------------------------------------------------------------- prepare-commit-msg - - -def test_prepare_commit_msg_noops_without_loop( - monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], tiny_fake_repo: Path -) -> None: - """Human mode is untouched.""" - message = tiny_fake_repo / ".git" / "COMMIT_EDITMSG" - message.write_text("", encoding="utf-8") - monkeypatch.delenv("RALPH_LOOP", raising=False) - monkeypatch.chdir(tiny_fake_repo) - assert gate.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG", "message"]) == 0 - assert not capsys.readouterr().out - + gate.run_git(["add", name], repo) + + +def wipe_history(repo: Path) -> None: + """Rewind the repo to having no commits at all.""" + gate.run_git(["checkout", "--orphan", "initial"], repo) + gate.run_git(["rm", "-qr", "--cached", "."], repo) + + +def stage_a_bad_iteration(repo: Path) -> None: + """Stage everything an agent might try in one iteration, honest work and cheating alike.""" + stage(repo, "pyproject.toml", "[tool.harness]\n") + gate.run_git(["commit", "-q", "-m", "add config"], repo) + gate.run_git(["rm", "-q", "pyproject.toml"], repo) + stage(repo, "src/feature.py", "value = 2\n") + stage(repo, "harness/gate.py", "FORBIDDEN_PATTERNS = []\n") + stage(repo, "harness/evil.py", "_ejected = 1 # noqa\n") + stage(repo, "harness/tests/test_gate.py", "def test_x() -> None:\n pass\n") + stage(repo, "PyProject.TOML", "[tool.harness]\n") + stage(repo, ".github/workflows/ci.yml", "jobs:\n gate:\n steps: []\n") + stage(repo, ".githooks/pre-commit", "#!/bin/sh\nexit 0\n") + stage(repo, "src/sloppy.py", "import os # noqa\n") + stage(repo, "release.sh", "git commit --no-verify -m ship\n") + stage(repo, "src/named.py", "_bad = 1\n") + stage(repo, "src/clean.py", "good = 1\n") + (repo / "src" / "clean.py").write_text("_never_staged = 1\n", encoding="utf-8") + + +def git_process(repo: Path, *args: str, loop: bool = True) -> subprocess.CompletedProcess[str]: + """Run Git in a disposable repo without inherited Git state.""" + env = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} + if not loop: + env.pop("RALPH_LOOP", None) + return subprocess.run(["git", *args], cwd=repo, capture_output=True, text=True, check=False, env=env) + + +def get_logged_calls_and_clear(repo: Path) -> list[object]: + """Read and clear complete calls recorded by the temporary harness.""" + log = repo / "harness.calls" + calls = [json.loads(line) for line in log.read_text(encoding="utf-8").splitlines()] + log.write_text("", encoding="utf-8") + return calls + + +@pytest.fixture +def real_hook_repo(request: pytest.FixtureRequest, git_repo: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Wire selected tracked hooks to a recorded executable in the disposable repository.""" + hooks = git_repo / ".active-hooks" + hooks.mkdir() + for name in (*request.param, "_resolve"): + shutil.copy2(REPO_ROOT / ".githooks" / name, hooks / name) + gate.run_git(["config", "core.hooksPath", ".active-hooks"], git_repo) + + executable = git_repo / "recorded-harness" + executable.write_text( + f"""#!{Path(sys.executable).as_posix()} +import json +import os +import sys +from pathlib import Path -def test_prepare_commit_msg_allows_loop_staged_file( - monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], tiny_fake_repo: Path -) -> None: - """A loop commit with a real index change passes.""" - message = tiny_fake_repo / ".git" / "COMMIT_EDITMSG" - message.write_text("real work\n", encoding="utf-8") - stage(tiny_fake_repo, "feature.py", "y = 2\n") +repo = Path.cwd() +arguments = sys.argv[1:] +command = arguments[0] if arguments else "" +recorded = arguments.copy() +if command == "prepare-commit-msg" and len(recorded) > 1: + recorded[1] = Path(recorded[1]).name +with (repo / "harness.calls").open("a", encoding="utf-8") as handle: + handle.write(json.dumps({{"arguments": recorded, "RALPH_LOOP": os.environ.get("RALPH_LOOP")}}) + "\\n") +real_file = repo / "harness.real" +real_commands = real_file.read_text(encoding="utf-8").splitlines() if real_file.exists() else [] +if command == "prepare-commit-msg" or command in real_commands: + os.chdir({str(REPO_ROOT)!r}) + from harness import cli, gate + os.chdir(repo) + gate.REPO_ROOT = repo + if command == "preflight": + gate.COMMIT_CHECKS = {{}} + cli.main(arguments) +status_file = repo / "harness.exit" +raise SystemExit(int(status_file.read_text(encoding="utf-8")) if status_file.exists() else 0) +""", + encoding="utf-8", + ) + executable.chmod(0o755) + (git_repo / ".git" / "harness-path").write_text(f"{executable}\n", encoding="utf-8") monkeypatch.setenv("RALPH_LOOP", "1") - monkeypatch.chdir(tiny_fake_repo) - assert gate.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG", "message"]) == 0 - assert not capsys.readouterr().out + return git_repo -def test_prepare_commit_msg_rejects_loop_empty_tree( - monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], tiny_fake_repo: Path +@pytest.mark.parametrize("real_hook_repo", [("pre-commit",)], indirect=True) +@pytest.mark.parametrize( + ("exit_code", "lands"), [pytest.param(0, True, id="passing"), pytest.param(1, False, id="blocking")] +) +def test_pre_commit_hook_dispatches_preflight_and_controls_commit( + exit_code: int, lands: bool, real_hook_repo: Path ) -> None: - """A loop commit whose index tree equals HEAD is blocked.""" - message = tiny_fake_repo / ".git" / "COMMIT_EDITMSG" - message.write_text("empty\n", encoding="utf-8") - monkeypatch.setenv("RALPH_LOOP", "1") - monkeypatch.chdir(tiny_fake_repo) - assert gate.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG", "message"]) == 1 - assert capsys.readouterr().out == ( - "\n[COMMIT BLOCKED]:\n" - "Empty-tree commit detected. Stage real work and don't use --allow-empty. Lazy.\n\n" + """The tracked pre-commit hook runs the recorded preflight and owns the commit verdict.""" + stage(real_hook_repo, "feature.py", "value = 1\n") + (real_hook_repo / "harness.exit").write_text(str(exit_code), encoding="utf-8") + before = gate.run_git(["rev-parse", "HEAD"], real_hook_repo).strip() + result = git_process(real_hook_repo, "commit", "-q", "-m", "exercise pre-commit") + after = gate.run_git(["rev-parse", "HEAD"], real_hook_repo).strip() + assert ( + result.returncode == 0, + get_logged_calls_and_clear(real_hook_repo), + after != before, + gate.run_git(["show", "--name-only", "--format=", "HEAD"], real_hook_repo).splitlines(), + ) == ( + lands, + [{"arguments": ["preflight"], "RALPH_LOOP": "1"}], + lands, + ["feature.py"] if lands else [".gitignore", "README.md", "README.template.md"], ) -def test_prepare_commit_msg_rejects_blank_loop_message( - monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], tiny_fake_repo: Path -) -> None: - """A blank or comment-only loop commit message is blocked even with staged work.""" - message = tiny_fake_repo / ".git" / "COMMIT_EDITMSG" - message.write_text("# generated comment only\n\n", encoding="utf-8") - stage(tiny_fake_repo, "feature.py", "y = 2\n") - monkeypatch.setenv("RALPH_LOOP", "1") - monkeypatch.chdir(tiny_fake_repo) - assert gate.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG", "message"]) == 1 - assert capsys.readouterr().out == ( - "\n[COMMIT BLOCKED]:\nCommit message is blank. Provide an informative message with your agent ID.\n\n" +@pytest.mark.parametrize("real_hook_repo", [("pre-commit", "pre-push")], indirect=True) +def test_pre_push_hook_dispatches_gate_and_blocks_push(real_hook_repo: Path) -> None: + """The tracked pre-push hook invokes gate and prevents a local remote ref update on failure.""" + stage(real_hook_repo, "pushable.py", "value = 1\n") + commit = git_process(real_hook_repo, "commit", "-q", "-m", "pushable work") + assert commit.returncode == 0, commit.stderr + get_logged_calls_and_clear(real_hook_repo) + + remote = real_hook_repo.parent / "origin.git" + assert git_process(real_hook_repo, "init", "--bare", "-q", str(remote)).returncode == 0 + gate.run_git(["remote", "add", "origin", str(remote)], real_hook_repo) + (real_hook_repo / "harness.exit").write_text("1", encoding="utf-8") + push = git_process(real_hook_repo, "push", "-q", "origin", "HEAD:main") + remote_ref = git_process( + real_hook_repo, "--git-dir", str(remote), "rev-parse", "--verify", "refs/heads/main" ) - -def test_prepare_commit_msg_allows_initial_staged_commit( - monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], tiny_fake_repo: Path -) -> None: - """An unborn repo with staged files is real work.""" - message = tiny_fake_repo / ".git" / "COMMIT_EDITMSG" - message.write_text("initial work\n", encoding="utf-8") - run_cmd(["git", "checkout", "--orphan", "initial"], tiny_fake_repo) - run_cmd(["git", "rm", "-qr", "--cached", "."], tiny_fake_repo) - stage(tiny_fake_repo, "first.py", "x = 1\n") - monkeypatch.setenv("RALPH_LOOP", "1") - monkeypatch.chdir(tiny_fake_repo) - assert gate.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG", "message"]) == 0 - assert not capsys.readouterr().out - - -def test_prepare_commit_msg_rejects_initial_empty_commit( - monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], tiny_fake_repo: Path -) -> None: - """An unborn repo with an empty index is still an empty-tree loop commit.""" - message = tiny_fake_repo / ".git" / "COMMIT_EDITMSG" - message.write_text("empty initial\n", encoding="utf-8") - run_cmd(["git", "checkout", "--orphan", "initial"], tiny_fake_repo) - run_cmd(["git", "rm", "-qr", "--cached", "."], tiny_fake_repo) - monkeypatch.setenv("RALPH_LOOP", "1") - monkeypatch.chdir(tiny_fake_repo) - assert gate.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG", "message"]) == 1 - assert capsys.readouterr().out == ( - "\n[COMMIT BLOCKED]:\n" - "Empty-tree commit detected. Stage real work and don't use --allow-empty. Lazy.\n\n" + assert (push.returncode != 0, get_logged_calls_and_clear(real_hook_repo), remote_ref.returncode == 0) == ( + True, + [{"arguments": ["gate"], "RALPH_LOOP": "1"}], + False, ) -@pytest.mark.parametrize("source", ["merge", "squash", "rebase", "reset", "clean", "filter-branch"]) -def test_prepare_commit_msg_rejects_dangerous_loop_sources( - source: str, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], tiny_fake_repo: Path +@pytest.mark.parametrize("real_hook_repo", [("prepare-commit-msg",)], indirect=True) +def test_prepare_commit_msg_hook_rejects_empty_agent_then_accepts_staged_work( + real_hook_repo: Path, ) -> None: - """Dangerous loop commit sources are blocked.""" - message = tiny_fake_repo / ".git" / "COMMIT_EDITMSG" - message.write_text("real work\n", encoding="utf-8") - stage(tiny_fake_repo, "feature.py", "y = 2\n") - monkeypatch.setenv("RALPH_LOOP", "1") - monkeypatch.chdir(tiny_fake_repo) - assert gate.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG", source]) == 1 - assert capsys.readouterr().out == ( - f"\n[COMMIT BLOCKED]:\nYou cannot use that git command `{source}`.\n\n" + """The hook rejects an empty agent commit, then accepts the agent's staged work.""" + before = gate.run_git(["rev-parse", "HEAD"], real_hook_repo).strip() + agent_empty = git_process( + real_hook_repo, "commit", "--allow-empty", "--no-verify", "-q", "-m", "agent empty" ) + assert agent_empty.returncode != 0 + assert get_logged_calls_and_clear(real_hook_repo) == [ + {"arguments": ["prepare-commit-msg", "COMMIT_EDITMSG", "message"], "RALPH_LOOP": "1"} + ] + assert gate.run_git(["rev-parse", "HEAD"], real_hook_repo).strip() == before + assert "Empty-tree commit detected" in agent_empty.stdout + agent_empty.stderr + + stage(real_hook_repo, "feature.py", "value = 1\n") + agent_work = git_process(real_hook_repo, "commit", "-q", "-m", "agent work") + assert agent_work.returncode == 0 + assert get_logged_calls_and_clear(real_hook_repo) == [ + {"arguments": ["prepare-commit-msg", "COMMIT_EDITMSG", "message"], "RALPH_LOOP": "1"} + ] + assert gate.run_git(["show", "--name-only", "--format=", "HEAD"], real_hook_repo).splitlines() == [ + "feature.py" + ] -def test_prepare_commit_msg_allows_loop_commit_source( - monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], tiny_fake_repo: Path -) -> None: - """Source `commit` is allowed for amend/reuse-message flows.""" - message = tiny_fake_repo / ".git" / "COMMIT_EDITMSG" - message.write_text("real work\n", encoding="utf-8") - stage(tiny_fake_repo, "feature.py", "y = 2\n") - monkeypatch.setenv("RALPH_LOOP", "1") - monkeypatch.chdir(tiny_fake_repo) - assert gate.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG", "commit"]) == 0 - assert not capsys.readouterr().out - - -def test_prepare_commit_msg_hook_rejects_loop_empty_no_verify(tiny_fake_repo: Path) -> None: - """prepare-commit-msg still runs under --no-verify and blocks loop empty-tree commits.""" - before = run_cmd(["git", "rev-parse", "HEAD"], tiny_fake_repo) - env = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} - env["RALPH_LOOP"] = "1" - result = subprocess.run( - ["git", "commit", "--allow-empty", "--no-verify", "-m", "empty"], - cwd=tiny_fake_repo, - capture_output=True, - text=True, - check=False, - env=env, +@pytest.mark.parametrize("real_hook_repo", [("prepare-commit-msg",)], indirect=True) +def test_prepare_commit_msg_hook_allows_human_empty_commit(real_hook_repo: Path) -> None: + """The hook does not apply agent containment to a human's empty commit.""" + human_empty = git_process( + real_hook_repo, + "commit", + "--allow-empty", + "--no-verify", + "-q", + "-m", + "human empty", + loop=False, ) - assert result.returncode != 0 - assert "\n[COMMIT BLOCKED]:\nEmpty-tree commit detected." in result.stderr - assert run_cmd(["git", "rev-parse", "HEAD"], tiny_fake_repo) == before - -def test_prepare_commit_msg_hook_allows_human_empty_no_verify(tiny_fake_repo: Path) -> None: - """Humans keep the same empty-commit behavior.""" - before = run_cmd(["git", "rev-parse", "HEAD"], tiny_fake_repo) - env = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} - env.pop("RALPH_LOOP", None) - result = subprocess.run( - ["git", "commit", "--allow-empty", "--no-verify", "-m", "empty"], - cwd=tiny_fake_repo, - capture_output=True, - text=True, - check=False, - env=env, - ) - assert result.returncode == 0, result.stderr - assert run_cmd(["git", "rev-parse", "HEAD"], tiny_fake_repo) != before + assert human_empty.returncode == 0 + assert get_logged_calls_and_clear(real_hook_repo) == [ + {"arguments": ["prepare-commit-msg", "COMMIT_EDITMSG", "message"], "RALPH_LOOP": None} + ] + assert gate.run_git(["show", "--name-only", "--format=", "HEAD"], real_hook_repo).splitlines() == [] -def test_prepare_commit_msg_hook_allows_loop_staged_commit(tiny_fake_repo: Path) -> None: - """Loop commits with staged work still land.""" - stage(tiny_fake_repo, "feature.py", "y = 2\n") - env = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} - env["RALPH_LOOP"] = "1" - result = subprocess.run( - ["git", "commit", "-m", "real work"], - cwd=tiny_fake_repo, - capture_output=True, - text=True, - check=False, - env=env, - ) - assert result.returncode == 0, result.stderr +@pytest.mark.parametrize("real_hook_repo", [("pre-commit", "prepare-commit-msg")], indirect=True) +def test_agent_iteration_is_contained_and_rejected( + real_hook_repo: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """Real commits reject blank, bad, forbidden, and empty attempts before landing only good work.""" + preflight = {"arguments": ["preflight"], "RALPH_LOOP": "1"} + prepare = {"arguments": ["prepare-commit-msg", "COMMIT_EDITMSG", "message"], "RALPH_LOOP": "1"} + stage_a_bad_iteration(real_hook_repo) + get_logged_calls_and_clear(real_hook_repo) + (real_hook_repo / "harness.real").write_text("preflight\n", encoding="utf-8") + initial_head = gate.run_git(["rev-parse", "HEAD"], real_hook_repo).strip() + + blank = git_process(real_hook_repo, "commit", "-q", "--no-verify", "--allow-empty-message", "-m", "") + assert ( + blank.returncode != 0, + "Commit message is blank" in blank.stdout + blank.stderr, + gate.run_git(["rev-parse", "HEAD"], real_hook_repo).strip(), + get_logged_calls_and_clear(real_hook_repo), + ) == (True, True, initial_head, [prepare]) + message_file = real_hook_repo / ".git" / "COMMIT_EDITMSG" + message_file.write_text("# generated comment only\n", encoding="utf-8") + assert gate.prepare_commit_msg(["prepare-commit-msg", str(message_file), "message"]) == 1 + assert "Commit message is blank" in capsys.readouterr().out + + bad = git_process(real_hook_repo, "commit", "-q", "-m", "bad and forbidden work") assert ( - "feature.py" in run_cmd(["git", "show", "--name-only", "--format=", "HEAD"], tiny_fake_repo).split() + bad.returncode != 0, + gate.run_git(["rev-parse", "HEAD"], real_hook_repo).strip(), + gate.run_git(["diff", "--cached", "--name-only"], real_hook_repo).splitlines(), + get_logged_calls_and_clear(real_hook_repo), + [value in bad.stdout + bad.stderr for value in ("# noqa", "--no-verify", "_bad")], + ) == ( + True, + initial_head, + ["release.sh", "src/clean.py", "src/feature.py", "src/named.py", "src/sloppy.py"], + [preflight], + [True, True, True], ) + assert (real_hook_repo / "harness" / "gate.py").exists() - -@pytest.mark.parametrize("source", ["merge", "squash"]) -def test_prepare_commit_msg_hook_dispatch_blocks_loop_merge_and_squash_sources( - tiny_fake_repo: Path, source: str -) -> None: - """Running the tracked prepare-commit-msg hook directly blocks merge and squash commits in loop mode.""" - stage(tiny_fake_repo, "feature.py", "y = 2\n") - env = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} - env["RALPH_LOOP"] = "1" - result = subprocess.run( - [".githooks/prepare-commit-msg", ".git/COMMIT_EDITMSG", source], - cwd=tiny_fake_repo, - capture_output=True, - text=True, - check=False, - env=env, - ) - assert result.returncode != 0 - assert f"You cannot use that git command `{source}`." in result.stdout + gate.run_git(["reset", "-q", "HEAD", "--", "release.sh", "src/named.py", "src/sloppy.py"], real_hook_repo) + good = git_process(real_hook_repo, "commit", "-q", "-m", "good work") + good_head = gate.run_git(["rev-parse", "HEAD"], real_hook_repo).strip() + assert ( + good.returncode, + good_head != initial_head, + gate.run_git(["show", "--name-only", "--format=", "HEAD"], real_hook_repo).splitlines(), + get_logged_calls_and_clear(real_hook_repo), + ) == (0, True, ["src/clean.py", "src/feature.py"], [preflight, prepare]) + assert gate.run_git(["show", "HEAD:src/clean.py"], real_hook_repo) == "good = 1\n" + + stage(real_hook_repo, "harness/again.py", "value = 1\n") + forbidden = git_process(real_hook_repo, "commit", "-q", "-m", "forbidden only") + assert ( + forbidden.returncode != 0, + "Empty-tree commit detected" in forbidden.stdout + forbidden.stderr, + gate.run_git(["rev-parse", "HEAD"], real_hook_repo).strip(), + gate.run_git(["diff", "--cached", "--name-only"], real_hook_repo).splitlines(), + get_logged_calls_and_clear(real_hook_repo), + ) == (True, True, good_head, [], [preflight, prepare]) + + empty = git_process(real_hook_repo, "commit", "-q", "--allow-empty", "--no-verify", "-m", "empty work") + assert ( + empty.returncode != 0, + "Empty-tree commit detected" in empty.stdout + empty.stderr, + gate.run_git(["rev-parse", "HEAD"], real_hook_repo).strip(), + get_logged_calls_and_clear(real_hook_repo), + ) == (True, True, good_head, [prepare]) -# --------------------------------------------------------------------------- tool dispatch +def test_agent_iteration_that_does_the_work_lands( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], git_repo: Path +) -> None: + """An honest iteration passes every stage, on an established repo and on a brand new one.""" + monkeypatch.setenv("RALPH_LOOP", "1") + monkeypatch.chdir(git_repo) + stage(git_repo, "src/feature.py", "value = 2\n") + stage(git_repo, "docs/notes.md", "Run with `# noqa` to silence the linter.\n") + (git_repo / ".git" / "COMMIT_EDITMSG").write_text("add the feature\n", encoding="utf-8") + assert gate.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG", "message"]) == 0 + (git_repo / ".git" / "COMMIT_EDITMSG").write_text("add the feature\n", encoding="utf-8") + assert gate.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG", "commit"]) == 0 + assert gate.run_non_human_checks() == [] + assert gate.run_git(["diff", "--cached", "--name-only"]).splitlines() == [ + "docs/notes.md", + "src/feature.py", + ] -def test_run_checks_reports_fully(monkeypatch: pytest.MonkeyPatch) -> None: - """Each check is recorded by name under 'pass' or 'fail' from the tool's exit code. + gate.run_git(["commit", "-q", "-m", "add the feature"], git_repo) + wipe_history(git_repo) + stage(git_repo, "first.py", "x = 1\n") + (git_repo / ".git" / "COMMIT_EDITMSG").write_text("first commit\n", encoding="utf-8") + assert gate.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG", "message"]) == 0 + assert "[COMMIT BLOCKED]" not in capsys.readouterr().out - Fakes the Popen seam (the external tool) so the real header + bucketing run. - """ - monkeypatch.delenv("RALPH_LOOP", raising=False) # bucketing-only: skip the loop containment git path - fake_popen(monkeypatch, fails=[["boom"]]) - captured = gate.run_checks({"boom check": ["boom"], "fine check": ["fine"]}) - assert captured == {"pass": ["fine check"], "fail": ["boom check"], "warn": []} + calls = fake_popen(monkeypatch) + assert gate.run_gate()["fail"] == [] + assert [launch[0] for launch in calls] == list(gate.FULL_CHECKS.values()) -def test_run_checks_messages_what_happened(monkeypatch: pytest.MonkeyPatch) -> None: - """A passing check is recorded under 'pass' with nothing in 'fail'.""" - monkeypatch.delenv("RALPH_LOOP", raising=False) # bucketing-only: skip the loop containment git path - fake_popen(monkeypatch) - assert gate.run_checks({"ok": ["tool"]}) == {"pass": ["ok"], "fail": [], "warn": []} +@pytest.mark.parametrize( + ("source", "refusal"), + [ + ("message", ""), + ("merge", "You cannot use that git command `merge`.\n"), + ("squash", "You cannot use that git command `squash`.\n"), + ("rebase", "You cannot use that git command `rebase`.\n"), + ("reset", "You cannot use that git command `reset`.\n"), + ("clean", "You cannot use that git command `clean`.\n"), + ("filter-branch", "You cannot use that git command `filter-branch`.\n"), + ], +) +def test_agent_cannot_commit_an_empty_iteration( + source: str, + refusal: str, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + git_repo: Path, +) -> None: + """Nothing staged is nothing done, and rewriting history is not a way to produce work.""" + monkeypatch.setenv("RALPH_LOOP", "1") + monkeypatch.chdir(git_repo) + empty = "Empty-tree commit detected. Stage real work and don't use --allow-empty. Lazy.\n" + (git_repo / ".git" / "COMMIT_EDITMSG").write_text("did nothing\n", encoding="utf-8") + assert gate.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG", source]) == 1 + assert capsys.readouterr().out == f"\n[COMMIT BLOCKED]:\n{refusal}{empty}\n" -def test_run_checks_records_a_failing_check_by_name(monkeypatch: pytest.MonkeyPatch) -> None: - """A failing check lands under 'fail' by its name, with nothing in 'pass'.""" - monkeypatch.delenv("RALPH_LOOP", raising=False) # bucketing-only: skip the loop containment git path - fake_popen(monkeypatch, fails=[["tool"]]) - captured = gate.run_checks({"random_check": ["tool"]}) - assert captured == {"pass": [], "fail": ["random_check"], "warn": []} + wipe_history(git_repo) + (git_repo / ".git" / "COMMIT_EDITMSG").write_text("did nothing\n", encoding="utf-8") + assert gate.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG", "message"]) == 1 + assert capsys.readouterr().out == f"\n[COMMIT BLOCKED]:\n{empty}\n" + assert gate.run_non_human_checks() == [] -def test_run_checks_streams_command_output_live( - monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str] +def test_human_running_the_same_commands_is_not_policed( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], git_repo: Path ) -> None: - """The real Popen seam streams the child process output and buckets a zero exit as a pass.""" - monkeypatch.delenv("RALPH_LOOP", raising=False) # dispatch-only: skip the loop containment git path - result = gate.run_checks({"echo": ["/bin/sh", "-c", "printf 'hello from the check\\n'"]}) - assert result == {"pass": ["echo"], "fail": [], "warn": []} - assert "hello from the check" in capfd.readouterr().out + """The same iteration outside the loop keeps every edit and blocks nothing.""" + monkeypatch.delenv("RALPH_LOOP", raising=False) + monkeypatch.chdir(git_repo) + recorder = Mock(return_value="unexpected preference call") + monkeypatch.setattr(gate, "prefs", recorder) + stage_a_bad_iteration(git_repo) + before = gate.run_git(["diff", "--cached", "--name-only"]).splitlines() + (git_repo / ".git" / "COMMIT_EDITMSG").write_text("", encoding="utf-8") + assert gate.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG", "message"]) == 0 + assert not capsys.readouterr().out -def test_run_checks_buckets_nonzero_exit_as_fail(monkeypatch: pytest.MonkeyPatch) -> None: - """The real Popen seam reads the child's nonzero status and buckets that check under 'fail'.""" - monkeypatch.delenv("RALPH_LOOP", raising=False) # dispatch-only: skip the loop containment git path - result = gate.run_checks({"boom": ["/bin/sh", "-c", "exit 7"]}) - assert result == {"pass": [], "fail": ["boom"], "warn": []} + calls = fake_popen(monkeypatch) + assert gate.run_preflight()["fail"] == [] + recorder.assert_not_called() + assert gate.run_git(["diff", "--cached", "--name-only"]).splitlines() == before + assert "harness/gate.py" in before + assert all(env["FORCE_COLOR"] == "1" for _, _, env in calls) + assert not [key for _, _, env in calls for key in env if key.startswith("GIT_")] -def test_run_checks_prints_phase_header_then_spawns( - monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +@pytest.mark.parametrize( + "forbidden_path", + [ + *( + pytest.param(f"{directory}blocked.txt", id=f"dir-{directory}") + for directory in gate.FORBIDDEN_DIRS + ), + *(pytest.param(path, id=f"file-{path}") for path in gate.FORBIDDEN_FILES), + ], +) +def test_every_configured_forbidden_path_is_ejected( + forbidden_path: str, monkeypatch: pytest.MonkeyPatch, git_repo: Path ) -> None: - """run_checks prints our PHASE header for each check even when the tool itself is faked.""" - monkeypatch.delenv("RALPH_LOOP", raising=False) # dispatch-only: skip the loop containment git path - fake_popen(monkeypatch) - result = gate.run_checks({"ruff lint": ["tool"]}) - assert result == {"pass": ["ruff lint"], "fail": [], "warn": []} - assert "PHASE: RUFF LINT" in capsys.readouterr().out - + """Every forbidden directory and exact file configured in pyproject is removed from the index.""" + monkeypatch.setenv("RALPH_LOOP", "1") + stage(git_repo, forbidden_path, "blocked\n") -def test_preflight_appends_containment_only_under_loop(monkeypatch: pytest.MonkeyPatch) -> None: - """Containment runs at pre-commit (run_preflight) only under RALPH_LOOP, never for a human.""" + assert gate.run_git(["diff", "--cached", "--name-only"]).splitlines() == [forbidden_path] + assert gate.run_non_human_checks() == [] + assert gate.run_git(["diff", "--cached", "--name-only"]).splitlines() == [] - def fake_containment() -> list[str]: - return ["containment problem"] - fake_popen(monkeypatch) - monkeypatch.setattr(gate, "run_non_human_checks", fake_containment) - monkeypatch.delenv("RALPH_LOOP", raising=False) - assert gate.run_preflight()["fail"] == [] # human: no containment +def test_every_configured_check_can_block_the_gate(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: + """Each configured check takes its turn failing; all of them run and the failing one blocks.""" monkeypatch.setenv("RALPH_LOOP", "1") - assert "containment problem" in gate.run_preflight()["fail"] # agent: containment appended - - -def test_lint_command_keeps_show_fixes_flag() -> None: - """The lint command asks ruff to show applied and suggested fixes.""" - assert gate.COMMIT_CHECKS["lint"] == [ - "uv", - "run", - "--no-cache", - "--no-sync", - "ruff", - "check", - "--show-fixes", - ".", - ] + stage(git_repo, "src/mod.py", "value = 1\n") + fake_popen(monkeypatch, fails=list(gate.FULL_CHECKS.values())) - -def test_full_gate_runs_every_preflight_and_gate_check_from_pyproject() -> None: - """The full gate runs the preflight + gate checks declared in pyproject.toml: at least 7 in total, - and each FULL_CHECKS name matches a key under [tool.harness.preflight] or [tool.harness.gate]. - """ - raw_toml = tomllib.loads((REPO_ROOT / "pyproject.toml").read_bytes().decode())["tool"]["harness"] - preflight, gate_checks = raw_toml.get("preflight"), raw_toml.get("gate") - assert len(preflight) >= 4 - assert len(gate_checks) >= 3 - assert set(gate.FULL_CHECKS) == set(preflight) | set(gate_checks) - - -def test_gate_tolerates_fully_deleted_harness_config(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: - """When a user deletes every [tool.harness.*] section, the loader's `.get(..., {})` defaults collapse - each constant to empty (the loader parsing a POPULATED config is already covered by the other tests). - This pins the CONSUMER side: with everything empty, running the whole `harness gate` under - RALPH_LOOP=1 (checks + containment) runs zero checks and ejects/flags nothing, so a clean staged - commit passes instead of crashing. - """ - monkeypatch.setenv("RALPH_LOOP", "1") - deleted: dict[str, dict[str, list[str]]] = {} # a pyproject with [tool.harness] removed parses to this - assert deleted.get("preflight", {}) | deleted.get("gate", {}) == {} # deletion -> empty via .get default - monkeypatch.setattr(gate, "FULL_CHECKS", {}) - monkeypatch.setattr(gate, "FORBIDDEN_FILES", []) - monkeypatch.setattr(gate, "FORBIDDEN_DIRS", ()) - monkeypatch.setattr(gate, "FORBIDDEN_PATTERNS", []) - stage( - git_repo, "src/feature.py", "def g(*args):\n pass # noqa\n" - ) # banned pattern, no preference break assert gate.run_gate() == { "pass": [], - "fail": ["src/feature.py:1: '*args'/'**kwargs' hide the function signature, use explicit parameters"], - "warn": [], + "fail": ["lint", "pylint", "complexipy", "security", "types", "pytest"], + "warn": ["format"], } - assert "src/feature.py" in staged() # nothing ejected: no forbidden config to eject against -def test_gate_tolerates_partially_deleted_harness_config( - monkeypatch: pytest.MonkeyPatch, git_repo: Path +def test_gate_runs_exactly_what_pyproject_configures( + monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str], git_repo: Path ) -> None: - """The likelier user error: delete [tool.harness.gate] and FORBIDDEN but keep one preflight check. The - survivor still dispatches and the missing sections default to empty, so the gate runs exactly the - remaining check and containment (real git on the fixture repo) ejects nothing β€” a smaller gate, not a - crash. - """ - monkeypatch.setenv("RALPH_LOOP", "1") - # Only preflight.lint survived; a real no-op command so containment's real git can run alongside it. - monkeypatch.setattr(gate, "FULL_CHECKS", {"lint": ["/bin/sh", "-c", "exit 0"]}) - monkeypatch.setattr(gate, "FORBIDDEN_FILES", []) - monkeypatch.setattr(gate, "FORBIDDEN_DIRS", ()) - monkeypatch.setattr(gate, "FORBIDDEN_PATTERNS", []) - stage(git_repo, "harness/util.py", "value = 1\n") # would be ejected IF FORBIDDEN_DIRS still had it - result = gate.run_gate() - assert result == {"pass": ["lint"], "fail": [], "warn": []} # survivor ran; empty FORBIDDEN flags nothing - assert "harness/util.py" in staged() # FORBIDDEN deleted -> nothing ejected, not a crash - - -def test_preflight_tolerates_deleted_format_check(monkeypatch: pytest.MonkeyPatch) -> None: - """Deleting `format` from [tool.harness.preflight] just drops a key; run_preflight iterates the - remaining checks and does not crash. Simulated by removing 'format' from COMMIT_CHECKS. - """ - without_format = {name: cmd for name, cmd in gate.COMMIT_CHECKS.items() if name != "format"} - monkeypatch.setattr(gate, "COMMIT_CHECKS", without_format) - fake_popen(monkeypatch) - result = gate.run_preflight() - assert result == {"pass": list(without_format), "fail": [], "warn": []} - - -def test_gate_runs_a_javascript_toolchain_config(monkeypatch: pytest.MonkeyPatch) -> None: - """A user can swap the whole [tool.harness] toolchain for JS commands (npm lint/format in preflight, - typecheck/test/build in gate) and the gate is agnostic: it spawns exactly the five configured checks, - in order, and buckets each by exit code β€” nothing here is Python-specific. - - Passes because fake Popen returns 0, not because js is configured yet. - """ - js_checks = { - "lint": ["npm", "run", "lint"], - "format": ["npm", "run", "format:check"], - "typecheck": ["npm", "run", "typecheck"], - "test": ["npm", "test"], - "build": ["npm", "run", "build"], - } - monkeypatch.setattr(gate, "FULL_CHECKS", js_checks) - calls = fake_popen(monkeypatch) - result = gate.run_gate() - assert [launch[0] for launch in calls] == list(js_checks.values()) - assert result == {"pass": ["lint", "typecheck", "test", "build"], "fail": [], "warn": ["format"]} - - -def test_types_check_uses_pyright_json_output() -> None: - """The types check runs pyright in JSON mode for stable machine-readable output.""" - assert gate.FULL_CHECKS["types"] == ["uv", "run", "--no-sync", "pyright", "--outputjson"] - - -def test_security_check_uses_semgrep_and_blocks_on_findings(monkeypatch: pytest.MonkeyPatch) -> None: - """The security check runs Semgrep with auto + secrets rules, and --error makes it BLOCKING: - a nonzero exit (Semgrep's signal for a finding under --error) buckets 'security' under 'fail', - not 'pass'. An advisory scan that reports but never blocks is worse than none. - """ - command = gate.FULL_CHECKS["security"] - assert command[:5] == ["uv", "run", "--no-sync", "semgrep", "scan"] - assert "--error" in command # exit nonzero on findings so the nonzero -> 'fail' rule below can bite - assert "--config" in command - assert "auto" in command - assert "p/secrets" in command - monkeypatch.delenv("RALPH_LOOP", raising=False) # dispatch-only: skip the loop containment git path - fake_popen(monkeypatch, fails=[command]) # semgrep --error exits nonzero on a finding - assert gate.run_checks({"security": command}) == {"pass": [], "fail": ["security"], "warn": []} - - -# --------------------------------------------------------------------- run_gate vs run_preflight routing - - -def test_gate_pytest_command_enforces_full_coverage_and_buckets_failures() -> None: - """The gate's pytest command keeps coverage reporting and the 100% coverage threshold.""" - pytest_command = gate.FULL_CHECKS["pytest"] - assert pytest_command[:5] == ["uv", "run", "--no-cache", "--no-sync", "pytest"] - assert "--cov" in pytest_command - assert "--cov-report=term-missing" in pytest_command - assert "--cov-fail-under=100" in pytest_command - - -def test_gate_buckets_a_failing_pytest_check(monkeypatch: pytest.MonkeyPatch) -> None: - """When the pytest command exits nonzero (e.g. a coverage gap), run_checks records it under 'fail'. - Faking the Popen seam proves the bucketing without a real, recursive pytest subprocess. - """ - monkeypatch.delenv("RALPH_LOOP", raising=False) # dispatch-only: skip the loop containment git path - fake_popen(monkeypatch, fails=[gate.FULL_CHECKS["pytest"]]) - result = gate.run_checks({"tests": gate.FULL_CHECKS["pytest"]}) - assert result["fail"] == ["tests"] - + """The gate dispatches whatever is configured, in order, and says so when nothing is.""" + raw_toml = tomllib.loads((REPO_ROOT / "pyproject.toml").read_bytes().decode())["tool"]["harness"] + assert raw_toml["preflight"] == gate.COMMIT_CHECKS + assert raw_toml["preflight"] | raw_toml["gate"] == gate.FULL_CHECKS + + live = gate.run_checks({ + "ruff lint": [sys.executable, "-c", "print('hello from the check')"], + "pyright types": [sys.executable, "-c", "raise SystemExit(7)"], + "ruff format": [sys.executable, "-c", "raise SystemExit(1)"], + }) + assert live == {"pass": ["ruff lint"], "fail": ["pyright types"], "warn": ["ruff format"]} + printed = capfd.readouterr().out + assert "hello from the check" in printed + assert "PHASE: RUFF LINT" in printed -def test_preflight_invokes_only_lint_end_to_end(monkeypatch: pytest.MonkeyPatch) -> None: - """Integrated routing: run_preflight runs only the commit checks.""" - monkeypatch.delenv("RALPH_LOOP", raising=False) calls = fake_popen(monkeypatch) - result = gate.run_preflight() - spawned = [launch[0] for launch in calls] - assert spawned == list(gate.COMMIT_CHECKS.values()) # preflight runs exactly the commit checks, in order - # format buckets into 'warn', not 'pass', so pass is the commit checks minus any format check. - assert result["pass"] == [name for name in gate.COMMIT_CHECKS if "format" not in name] - assert result["fail"] == [] - - -def test_run_gate_delegates_to_run_checks_with_full_checks(monkeypatch: pytest.MonkeyPatch) -> None: - """run_gate is a thin router: it runs exactly FULL_CHECKS on the given repo and returns that result. - - The real end-to-end behaviour of every gate check is proven in test_integration's single full-gate - test; here we only pin the routing (repo + FULL_CHECKS in, run_checks' result out) without paying - for real tools or risking the pytest check recursively collecting this suite. - - NO NESTED PYTEST: run_checks is stubbed with `spy`, so run_gate spawns nothing. This is the pattern - to copy for any new routing assertion β€” stub run_checks instead of adding a real-pytest spawn. - """ - seen: dict[str, object] = {} - - def spy(checks: dict[str, list[str]]) -> dict[str, list[str]]: - seen["checks"] = checks - return {"pass": ["types"], "fail": []} - - monkeypatch.setattr(gate, "run_checks", spy) - result = gate.run_gate() - assert seen == {"checks": gate.FULL_CHECKS} - assert result == {"pass": ["types"], "fail": []} - - -# --------------------------------------------------------------------------- containment (loop only) - - -def test_preflight_ejects_forbidden_file_under_loop(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: - """A staged forbidden FILE (exact-path set) is dropped from the index, kept in the tree.""" - monkeypatch.setenv("RALPH_LOOP", "1") - stage(git_repo, "pyproject.toml", "x = 1\n") - assert containment_fail() == [] # self-heals, not blocked - assert "pyproject.toml" not in staged() - assert (git_repo / "pyproject.toml").exists() # edit survives in the working tree - - -@pytest.mark.parametrize("path", ["harness/util.py", "tests/harness/x.py", ".github/ci.yml", ".githooks/x"]) -def test_preflight_ejects_forbidden_dir_under_loop( - path: str, monkeypatch: pytest.MonkeyPatch, git_repo: Path -) -> None: - """A staged file under any forbidden DIR (dir-set ancestor match) is dropped from the index.""" - monkeypatch.setenv("RALPH_LOOP", "1") - stage(git_repo, path, "value = 1\n") - assert containment_fail() == [] - assert path not in staged() + preflight = gate.run_preflight() + assert [launch[0] for launch in calls] == list(gate.COMMIT_CHECKS.values()) + assert all(cwd == gate.REPO_ROOT for _, cwd, _ in calls) + assert preflight == {"pass": ["lint", "pylint", "format", "complexipy"], "fail": [], "warn": []} + + preflight_output = capfd.readouterr().out + for name in gate.COMMIT_CHECKS: + assert preflight_output.count(f"PHASE: {name.upper()}") == 1 + assert "PHASE: COMMAND" not in preflight_output + + calls.clear() + full = gate.run_gate() + assert [launch[0] for launch in calls] == list(gate.FULL_CHECKS.values()) + assert full == { + "pass": ["lint", "pylint", "format", "complexipy", "security", "types", "pytest"], + "fail": [], + "warn": [], + } + gate_output = capfd.readouterr().out + for name in gate.FULL_CHECKS: + assert gate_output.count(f"PHASE: {name.upper()}") == 1 + assert "PHASE: COMMAND" not in gate_output + without_format = {name: cmd for name, cmd in gate.COMMIT_CHECKS.items() if name != "format"} + monkeypatch.setattr(gate, "COMMIT_CHECKS", without_format) + assert gate.run_preflight() == {"pass": list(without_format), "fail": [], "warn": []} -def test_preflight_keeps_legit_work_beside_forbidden(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: - """Only the forbidden path is dropped; the agent's own work still commits.""" + js_checks = {"lint": ["npm", "run", "lint"], "format": ["npm", "run", "format:check"]} + monkeypatch.setattr(gate, "FULL_CHECKS", js_checks) monkeypatch.setenv("RALPH_LOOP", "1") - stage(git_repo, "harness/util.py", "value = 1\n") - stage(git_repo, "src/feature.py", "y = 2\n") - assert containment_fail() == [] - after = staged() - assert "harness/util.py" not in after - assert "src/feature.py" in after + stage(git_repo, "src/app.js", "console.log('pass');\n") + assert gate.run_gate() == {"pass": ["lint", "format"], "fail": [], "warn": []} - -def test_ejected_forbidden_py_is_not_preference_checked( - monkeypatch: pytest.MonkeyPatch, git_repo: Path -) -> None: - """A forbidden .py that ALSO breaks a preference is ejected AND self-heals: because ejection - removes it from the judged set, its preference break must NOT land in fail (ejecting is exit-0). - Regression: the prefs loop once iterated the pre-eject staged list and blocked the commit. - """ - monkeypatch.setenv("RALPH_LOOP", "1") - stage(git_repo, "harness/evil.py", "_bad = 1\n") # forbidden DIR + underscore-name preference break - assert containment_fail() == [] # ejected, not judged: commit still succeeds - assert "harness/evil.py" not in staged() + monkeypatch.setattr(gate, "FULL_CHECKS", {}) + assert gate.run_gate() == {"pass": [], "fail": [], "warn": []} + stage(git_repo, "src/mod.py", "_bad = 1\nf = lambda: 0\n") + assert gate.run_gate() == { + "pass": [], + "fail": [ + ( + "problems:\nsrc/mod.py:1: Name '_bad' starts with underscore\nsrc/mod.py:2: Lambda found " + "hurting readability and adding complexity." + ) + ], + "warn": [], + } -def test_forbidden_file_match_is_case_insensitive(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: - """The forbidden-file set is matched case-insensitively, so a mixed-case protected filename is - still ejected (an agent can't smuggle it past by changing case). - """ - monkeypatch.setenv("RALPH_LOOP", "1") - stage(git_repo, "PyProject.TOML", "x = 1\n") # same file as pyproject.toml, different case - assert containment_fail() == [] - assert "PyProject.TOML" not in staged() # ejected despite the casing +def test_lint_command_keeps_required_flags() -> None: + """The fast lint command remains Ruff's fixing-aware repository-wide check.""" + command = gate.COMMIT_CHECKS["lint"] + assert command[:2] == ["ruff", "check"] + assert "--show-fixes" in command + assert command[-1] == "." -def test_banned_pattern_in_ejected_file_is_not_flagged( - monkeypatch: pytest.MonkeyPatch, git_repo: Path -) -> None: - """A banned pattern living in a forbidden file is not a failure: ejection happens BEFORE the - banned-pattern scan re-reads the staged diff, so the ejected file's noqa never reaches it. - """ - monkeypatch.setenv("RALPH_LOOP", "1") - stage(git_repo, "pyproject.toml", "x = 1 # noqa\n") # forbidden file that also holds a banned pattern - assert containment_fail() == [] # ejected before the scan; nothing to block - assert "pyproject.toml" not in staged() +def test_type_check_keeps_machine_readable_output() -> None: + """Pyright retains stable JSON output for callers that parse its diagnostics.""" + command = gate.FULL_CHECKS["types"] -def test_preflight_ejects_staged_deletion_of_forbidden( - monkeypatch: pytest.MonkeyPatch, git_repo: Path -) -> None: - """A staged DELETION of a forbidden file is undone, so the agent can't remove protected files.""" - stage(git_repo, "pyproject.toml", "x = 1\n") - run_cmd(["git", "commit", "-q", "-m", "add pyproject"], git_repo) - run_cmd(["git", "rm", "-q", "pyproject.toml"], git_repo) - monkeypatch.setenv("RALPH_LOOP", "1") - assert containment_fail() == [] - assert "pyproject.toml" not in staged() # the deletion was reset out of the index + assert command[0] == "pyright" + assert "--outputjson" in command -def test_preflight_skips_containment_without_loop(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: - """Without RALPH_LOOP, a human may stage forbidden paths: nothing is ejected.""" - monkeypatch.delenv("RALPH_LOOP", raising=False) - stage(git_repo, "harness/util.py", "value = 1\n") - assert "harness/util.py" in staged() # read the index before faking Popen (git can't run faked) - fake_popen(monkeypatch) - result = gate.run_preflight() - assert result["fail"] == [] # no-loop preflight runs only the faked checks; it has no eject path at all - - -@pytest.mark.parametrize("pattern", ["# noqa", "type: ignore", "--no-verify"]) -def test_preflight_flags_banned_pattern_under_loop( - pattern: str, monkeypatch: pytest.MonkeyPatch, git_repo: Path -) -> None: - """A banned escape-hatch in an added line is flagged (so the commit is rejected).""" - monkeypatch.setenv("RALPH_LOOP", "1") - stage(git_repo, "src/x.py", f"value = 1 # {pattern}\n") - assert any(f"'{pattern}' line:" in problem for problem in containment_fail()) +def test_security_scan_keeps_blocking_rules() -> None: + """Semgrep stays blocking, scans the repository, and includes code and secret rules.""" + command = gate.FULL_CHECKS["security"] + configs = [command[index + 1] for index, item in enumerate(command[:-1]) if item == "--config"] + assert command[:2] == ["semgrep", "scan"] + assert "--error" in command + assert configs == ["auto", "p/secrets"] + assert not any(item == "--exclude" or item.startswith("--exclude=") for item in command) + assert command[-1] == "." -def test_preflight_banned_pattern_is_case_insensitive( - monkeypatch: pytest.MonkeyPatch, git_repo: Path -) -> None: - """Mixed-case escape hatches are still caught.""" - monkeypatch.setenv("RALPH_LOOP", "1") - stage(git_repo, "src/x.py", "value = 1 # NoQA\n") - assert any("'# noqa' line:" in problem for problem in containment_fail()) +def test_pytest_gate_keeps_full_coverage_threshold() -> None: + """The configured test gate continues to require complete measured coverage.""" + command = gate.FULL_CHECKS["pytest"] -@pytest.mark.parametrize( - ("typed", "canonical"), - [("tS-ignoRe", "ts-ignore"), ("# Pylint:", "# pylint:"), ("PRAGMA: no cover", "pragma: no cover")], -) -def test_preflight_flags_weird_case_banned_patterns( - typed: str, canonical: str, monkeypatch: pytest.MonkeyPatch, git_repo: Path -) -> None: - """An added line carrying a banned pattern in odd mixed casing is still flagged: the pattern set is - hardcoded lowercase and the scan casefolds only the line, so the message uses that lowercase pattern. - """ - monkeypatch.setenv("RALPH_LOOP", "1") - stage(git_repo, "src/x.py", f"value = 1 # {typed}\n") - fail = containment_fail() - assert any(isinstance(p, str) and p.startswith(f"'{canonical}' line:") for p in fail) + assert {"--cov", "--cov-report=term-missing", "--cov-fail-under=100"} <= set(command) def test_preflight_flags_preferences_break_under_loop( monkeypatch: pytest.MonkeyPatch, git_repo: Path ) -> None: - """A staged Python file that breaks a preference (underscore name) is flagged.""" + """Preflight preserves every preference failure alongside a failing configured check.""" monkeypatch.setenv("RALPH_LOOP", "1") - stage(git_repo, "src/mod.py", "_bad = 1\n") - assert any("'_bad'" in problem for problem in containment_fail()) + monkeypatch.chdir(git_repo) + assert gate.prefs is not None + recorder = Mock(wraps=gate.prefs) + monkeypatch.setattr(gate, "prefs", recorder) + source = "def _bad(*args):\n transform = lambda item: item\n return transform(*args)\n" + stage(git_repo, "src/mod.py", source) + fake_popen(monkeypatch, fails=[gate.COMMIT_CHECKS["lint"]]) + result = gate.run_preflight() -def test_preflight_judges_staged_not_working_tree(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: - """Preferences judge the INDEX, not disk. Stage a clean file, then dirty the working tree with a - violation that is never staged: the commit is not blocked (only staged content counts). - """ - monkeypatch.setenv("RALPH_LOOP", "1") - stage(git_repo, "src/mod.py", "good = 1\n") # index: clean - (git_repo / "src/mod.py").write_text("_bad = 1\n", encoding="utf-8") # working tree only: violation - assert containment_fail() == [] + assert { + "preferences": recorder.call_args_list, + "result": result, + "staged_paths": gate.run_git(["diff", "--cached", "--name-only"], git_repo).splitlines(), + "staged_source": gate.run_git(["show", ":src/mod.py"], git_repo), + } == { + "preferences": [call("src/mod.py", source)], + "result": { + "pass": ["pylint", "format", "complexipy"], + "fail": [ + "lint", + ( + "problems:\n" + "src/mod.py:1: Name '_bad' starts with underscore\n" + "src/mod.py:1: '*args'/'**kwargs' hide the function signature, use explicit parameters\n" + "src/mod.py:2: Lambda found hurting readability and adding complexity.\n" + "src/mod.py:3: Dynamic '*' call hides positional arguments; pass explicit arguments" + ), + ], + "warn": [], + }, + "staged_paths": ["src/mod.py"], + "staged_source": source, + } def test_preflight_preferences_read_one_file_at_a_time( monkeypatch: pytest.MonkeyPatch, git_repo: Path ) -> None: - """Each prefs() call receives exactly one staged file's source, never several concatenated.""" + """Preferences receive each staged Python file and its index content separately.""" monkeypatch.setenv("RALPH_LOOP", "1") - sources: list[str] = [] - - def record(path: str, source: str) -> str: - del path - sources.append(source) - return "" - - monkeypatch.setattr(gate, "prefs", record) + monkeypatch.chdir(git_repo) + recorder = Mock(side_effect=["src/a.py:1: first violation", ""]) + monkeypatch.setattr(gate, "prefs", recorder) stage(git_repo, "src/a.py", "a = 1\n") stage(git_repo, "src/b.py", "b = 2\n") - containment_fail() - # each call gets exactly one file's staged source (git show preserves the trailing newline) - assert sorted(s.rstrip("\n") for s in sources) == ["a = 1", "b = 2"] + (git_repo / "src/a.py").write_text("_working_tree_only = 3\n", encoding="utf-8") + (git_repo / "src/b.py").write_text("_also_not_staged = 4\n", encoding="utf-8") + fake_popen(monkeypatch) + result = gate.run_preflight() -def test_preflight_skips_preferences_on_non_python_staged_file( - monkeypatch: pytest.MonkeyPatch, git_repo: Path -) -> None: - """A staged non-.py file is not preference-checked (only Python style is judged), so it never - lands in fail even with loop containment on. - """ - monkeypatch.setenv("RALPH_LOOP", "1") - stage(git_repo, "notes.txt", "_bad = 1\n") # underscore name, but not Python β€” must be ignored - assert containment_fail() == [] + assert { + "preferences": recorder.call_args_list, + "result": result, + "working_sources": [ + (git_repo / "src/a.py").read_text(encoding="utf-8"), + (git_repo / "src/b.py").read_text(encoding="utf-8"), + ], + } == { + "preferences": [call("src/a.py", "a = 1\n"), call("src/b.py", "b = 2\n")], + "result": { + "pass": list(gate.COMMIT_CHECKS), + "fail": ["problems:\nsrc/a.py:1: first violation"], + "warn": [], + }, + "working_sources": ["_working_tree_only = 3\n", "_also_not_staged = 4\n"], + } -def test_preflight_skips_preferences_for_staged_deletion( +def test_check_for_bad_patterns_appends_a_preference_violation( monkeypatch: pytest.MonkeyPatch, git_repo: Path ) -> None: - """A staged DELETION of a .py file is filtered out (--diff-filter=d) before any `git show :path`, - so preference checking skips it (nothing to judge) rather than crashing. - """ - stage(git_repo, "src/gone.py", "value = 1\n") - run_cmd(["git", "commit", "-q", "-m", "add gone"], git_repo) - run_cmd(["git", "rm", "-q", "src/gone.py"], git_repo) # staged deletion: no :path blob - monkeypatch.setenv("RALPH_LOOP", "1") - assert containment_fail() == [] - - -def test_preflight_tolerates_missing_preferences(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: - """If preferences.py was deleted (prefs is None), the Python style check is skipped, not crashed.""" + """A staged Python preference violation is included in the returned problems.""" monkeypatch.setenv("RALPH_LOOP", "1") - monkeypatch.setattr(gate, "prefs", None) - stage(git_repo, "src/mod.py", "_bad = 1\n") - assert containment_fail() == [] - - -def test_gate_imports_cleanly_without_preferences(monkeypatch: pytest.MonkeyPatch) -> None: - """If preferences.py is absent, gate still imports and prefs is None (the ImportError branch).""" - monkeypatch.setitem(sys.modules, "preferences.preferences", None) - importlib.reload(gate) - assert gate.prefs is None - monkeypatch.undo() - importlib.reload(gate) + monkeypatch.chdir(git_repo) assert gate.prefs is not None + recorder = Mock(wraps=gate.prefs) + monkeypatch.setattr(gate, "prefs", recorder) + source = "def _bad(*args):\n return 1 # noqa\n" + stage(git_repo, "src/mod.py", source) - -# ------------------------------------------------- check_for_bad_patterns (direct, no ejection wrapper) - - -def test_check_for_bad_patterns_flags_a_banned_pattern(git_repo: Path) -> None: - """Called directly, it returns a banned-pattern problem for a staged added line carrying one.""" - stage(git_repo, "src/x.py", "value = 1 # noqa\n") - problems = gate.check_for_bad_patterns() - assert any(problem.startswith("'# noqa' line:") for problem in problems) - - -def test_check_for_bad_patterns_ignores_markdown_prose(git_repo: Path) -> None: - """A banned token quoted in .md docs is prose, not a bypass, so it is excluded from the scan; - the same token in a non-.md file is still flagged (the anti-bypass net stays on code/config). - """ - stage(git_repo, "docs/notes.md", "Run with `# noqa` to silence the linter.\n") # prose: ignored - stage(git_repo, "run.sh", "grep --no-verify\n") # non-.md: still scanned problems = gate.check_for_bad_patterns() - assert not any("# noqa" in problem for problem in problems) # markdown excluded - assert any(problem.startswith("'--no-verify' line:") for problem in problems) # shell still caught - -def test_check_for_bad_patterns_appends_a_preference_violation(git_repo: Path) -> None: - """A staged .py file that breaks a preference contributes its violation to the returned problems.""" - stage(git_repo, "src/mod.py", "_bad = 1\n") # lone-underscore name trips a preference - problems = gate.check_for_bad_patterns() - assert any("'_bad'" in problem for problem in problems) - - -def test_check_for_bad_patterns_clean_staged_file_has_no_problems(git_repo: Path) -> None: - """A staged file with no banned patterns and no preference breaks yields an empty problem list.""" - stage(git_repo, "src/ok.py", "value = 1\n") - assert gate.check_for_bad_patterns() == [] - - -@pytest.mark.usefixtures("git_repo") # anchors gate.REPO_ROOT at the seeded fixture repo; not referenced -def test_check_for_bad_patterns_empty_index_returns_no_problems() -> None: - """With nothing staged the diff is empty, so both scans are skipped and no problems are returned.""" - assert not staged() # git_repo has only the seed commit; nothing staged - assert gate.check_for_bad_patterns() == [] # clean index: seed commit only, nothing staged - - -@pytest.mark.usefixtures("git_repo") # anchors gate.REPO_ROOT at the seeded fixture repo; not referenced -def test_empty_commit_does_not_block_under_loop(monkeypatch: pytest.MonkeyPatch) -> None: - """An empty commit (nothing staged) is not blocked: containment is skipped and no problems returned.""" - monkeypatch.setenv("RALPH_LOOP", "1") - assert gate.run_non_human_checks() == [] # seed commit only, nothing staged - - -def test_language_without_preferences_file_crashes_check_for_bad_patterns( - monkeypatch: pytest.MonkeyPatch, git_repo: Path -) -> None: - """Setting languages=['rb'] routes staged .rb files into the Python `ast`-based prefs, which cannot - parse Ruby: the preference walk raises SyntaxError. Pins that the prefs engine is Python-only. - """ - monkeypatch.setattr(gate, "languages", ["js"]) - stage(git_repo, "preferences.js", "console.log('pass');\n") # valid js, invalid py - gate.check_for_bad_patterns() - monkeypatch.setattr(gate, "languages", ["rb"]) - stage(git_repo, "app.rb", "def foo; end\n") # valid Ruby, invalid py - gate.check_for_bad_patterns() - - -# --------------------------------------------------------- spec tests (FAIL against the current bugs) + assert { + "preferences": recorder.call_args_list, + "problems": problems, + "staged_paths": gate.run_git(["diff", "--cached", "--name-only"], git_repo).splitlines(), + } == { + "preferences": [call("src/mod.py", source)], + "problems": [ + "'# noqa' line: return 1 # noqa", + ( + "src/mod.py:1: Name '_bad' starts with underscore\n" + "src/mod.py:1: '*args'/'**kwargs' hide the function signature, use explicit parameters" + ), + ], + "staged_paths": ["src/mod.py"], + } -def test_staged_noqa_produces_a_noqa_line_message_in_fail( - monkeypatch: pytest.MonkeyPatch, git_repo: Path +@pytest.mark.parametrize( + ("name", "content"), + [ + ("notes.txt", "_bad = 1\n"), + ("data.json", "{not: valid python (((\n"), + ("app.js", "console.log('pass');\n"), + ], +) +def test_preferences_only_ever_read_python( + name: str, content: str, monkeypatch: pytest.MonkeyPatch, git_repo: Path ) -> None: - """A staged `# noqa` must land in fail as the scan's own message `'# noqa' line: `. FAILS now: - `found.join(...)` discards its result, so the banned-pattern scan appends nothing. - """ + """Non-Python is never parsed as Python, whether by suffix, by deletion, or by project language.""" monkeypatch.setenv("RALPH_LOOP", "1") - stage(git_repo, "src/x.py", "value = 1 # noqa\n") - fail = containment_fail() - assert any(isinstance(p, str) and p.startswith("'# noqa' line:") for p in fail) + recorder = Mock(return_value="unexpected preference call") + monkeypatch.setattr(gate, "prefs", recorder) + stage(git_repo, name, content) + assert gate.run_non_human_checks() == [] + recorder.assert_not_called() + monkeypatch.setattr(gate, "languages", ["rb"]) + stage(git_repo, "app.rb", "def foo; end\n") + assert gate.check_for_bad_patterns() == [] + recorder.assert_not_called() -def test_reset_ejects_only_forbidden_keeping_legit_staged( - monkeypatch: pytest.MonkeyPatch, git_repo: Path -) -> None: - """Ejection resets ONLY forbidden paths; a legit file staged alongside stays in the index. FAILS - now: reset is passed every staged path, so the legit file is unstaged too. - """ - monkeypatch.setenv("RALPH_LOOP", "1") - stage(git_repo, "pyproject.toml", "x = 1\n") # forbidden - stage(git_repo, "src/feature.py", "y = 2\n") # legit - containment_fail() - after = staged() - assert "pyproject.toml" not in after # forbidden ejected - assert "src/feature.py" in after # legit work survives - - -def test_prefs_skips_non_python_invalid_source(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: - """A staged non-.py file is not fed to prefs/ast.parse. Its bytes are invalid Python, so if the - `.py` filter were missing the run would crash. FAILS now: no suffix filter, `ast.parse` raises. - """ - monkeypatch.setenv("RALPH_LOOP", "1") - stage(git_repo, "data.json", "{not: valid python (((\n") # invalid Python; must never reach prefs - assert containment_fail() == [] + monkeypatch.setattr(gate, "languages", ["py"]) + stage(git_repo, "src/gone.py", "value = 1\n") + gate.run_git(["commit", "-q", "-m", "add gone"], git_repo) + gate.run_git(["rm", "-q", "src/gone.py"], git_repo) + assert gate.run_non_human_checks() == [] + recorder.assert_not_called() -def test_ejected_forbidden_py_is_not_re_judged_by_prefs( +def test_deleting_preferences_disables_the_check_not_the_gate( monkeypatch: pytest.MonkeyPatch, git_repo: Path ) -> None: - """A forbidden .py that breaks a preference is ejected, so prefs must NOT re-judge it (ejecting is - exit-0). FAILS now: the prefs loop reads the post-eject staged list but has no forbidden filter, - and the ejected file is still on disk / in the diff path set, so its break lands in fail. - """ + """preferences.py is meant to be deletable, so the gate keeps running without it.""" monkeypatch.setenv("RALPH_LOOP", "1") - stage(git_repo, "harness/evil.py", "_bad = 1\n") # forbidden DIR + underscore-name break - fail = containment_fail() - assert fail == [] # ejected, not re-judged - assert "harness/evil.py" not in staged() - - -# ----------------------------------------- banned-pattern scan only matches added ('+', not '+++') lines -# git diff --cached --unified=0 emits, per hunk: `--- a/f`, `+++ b/f` (headers), `-old` (removed), -# `+new` (added). The scan (gate.run_preflight line 163) must flag ONLY the real added line ('+', -# excluding the '+++' file header); removed ('-') and header ('+++') lines carrying a banned pattern -# must be ignored. A '*'-prefixed line can never occur in unified diff output, so nothing starting -# with '*' is ever matched β€” proven here by the removed-line case (only '+' counts). - + stage(git_repo, "src/mod.py", "_bad = 1\n") + fake_popen(monkeypatch) -def test_banned_scan_flags_added_plus_line(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: - """An ADDED ('+') line carrying noqa is flagged. FAILS now: the scan's message is char-shredded.""" - monkeypatch.setenv("RALPH_LOOP", "1") - stage(git_repo, "src/x.py", "value = 1 # noqa\n") # a pure addition -> a '+' hunk line - fail = containment_fail() - assert any(isinstance(p, str) and p.startswith("'# noqa' line:") for p in fail) + assert gate.run_preflight() == { + "pass": list(gate.COMMIT_CHECKS), + "fail": ["problems:\nsrc/mod.py:1: Name '_bad' starts with underscore"], + "warn": [], + } + assert gate.run_gate() == { + "pass": list(gate.FULL_CHECKS), + "fail": ["problems:\nsrc/mod.py:1: Name '_bad' starts with underscore"], + "warn": [], + } + monkeypatch.setattr(gate, "prefs", None) + monkeypatch.setitem(sys.modules, "preferences.preferences", None) + importlib.reload(gate) + monkeypatch.setattr(gate, "REPO_ROOT", git_repo) + assert gate.prefs is None + assert gate.run_preflight() == {"pass": list(gate.COMMIT_CHECKS), "fail": [], "warn": []} + assert gate.run_gate() == {"pass": list(gate.FULL_CHECKS), "fail": [], "warn": []} -def test_banned_scan_ignores_plus_plus_plus_header_line( - monkeypatch: pytest.MonkeyPatch, git_repo: Path -) -> None: - """The '+++ b/' file-HEADER line is not an added code line: a banned pattern living only in - the path (a file literally named with 'noqa') must not be flagged by the header, since the scan - excludes lines starting with '+++'. - """ - monkeypatch.setenv("RALPH_LOOP", "1") - stage(git_repo, "src/noqa_helpers.py", "value = 1\n") # 'noqa' appears in the '+++ b/...' header - fail = containment_fail() - assert not any(isinstance(p, str) and p.startswith("'noqa' line:") for p in fail) - - -def test_banned_scan_ignores_removed_minus_line(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: - """A REMOVED ('-') line carrying a banned pattern is ignored: deleting a `# noqa` line is good, - not a violation. Also proves only '+' is matched (never '-', and never a '*' prefix). - """ - stage(git_repo, "src/x.py", "value = 1 # noqa\n") - run_cmd(["git", "commit", "-q", "-m", "seed noqa"], git_repo) # committed; not in the diff anymore - stage(git_repo, "src/x.py", "value = 1\n") # drops the escape-hatch line -> a removed ('-') hunk line - monkeypatch.setenv("RALPH_LOOP", "1") - fail = containment_fail() - assert not any(isinstance(p, str) and "noqa" in p for p in fail) # removed line is not flagged + monkeypatch.undo() + importlib.reload(gate) + assert gate.prefs is not None diff --git a/harness/tests/test_integration.py b/harness/tests/test_integration.py deleted file mode 100644 index 9455a40..0000000 --- a/harness/tests/test_integration.py +++ /dev/null @@ -1,162 +0,0 @@ -"""Integration tests for real git hooks with a fake harness, plus hermetic gate dispatch.""" - -from __future__ import annotations - -import os -import subprocess -from pathlib import Path - -import pytest - -from harness import gate -from harness.tests.conftest import fake_popen, run_cmd - - -def attempt_commit(repo: Path, message: str, loop: bool, no_verify: bool) -> subprocess.CompletedProcess[str]: - """Try a commit with optional RALPH_LOOP in the env and optional hook bypass.""" - env = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} - if loop: - env["RALPH_LOOP"] = "1" - args = ["git", "commit", "-q", "-m", message] - if no_verify: - args.append("--no-verify") - return subprocess.run(args, cwd=repo, capture_output=True, text=True, check=False, env=env) - - -def stage(repo: Path, name: str, content: str) -> None: - """Write a file inside the repo and stage it.""" - target = repo / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(content, encoding="utf-8") - run_cmd(["git", "add", name], repo) - - -def committed_files(repo: Path) -> list[str]: - """Paths in the most recent commit.""" - return run_cmd(["git", "show", "--name-only", "--format=", "HEAD"], repo).split() - - -def log(repo: Path) -> str: - """One-line git log of the repo.""" - return run_cmd(["git", "log", "--oneline"], repo) - - -def fake_harness_args(repo: Path) -> str: - """Return the fake harness argv recorded by the hook.""" - return (repo / "harness.args").read_text(encoding="utf-8") - - -def fake_harness_loop(repo: Path) -> str: - """Return the RALPH_LOOP value recorded by the fake harness.""" - return (repo / "harness.loop").read_text(encoding="utf-8") - - -def set_fake_harness_exit(repo: Path, code: int) -> None: - """Choose the fake harness process exit code for this repo.""" - (repo / "harness.exit").write_text(f"{code}\n", encoding="utf-8") - - -def push_head(repo: Path) -> subprocess.CompletedProcess[str]: - """Push HEAD to a local bare remote with hook-safe environment.""" - bare = repo.parent / "origin.git" - run_cmd(["git", "init", "--bare", "-q", str(bare)], repo) - run_cmd(["git", "remote", "add", "origin", str(bare)], repo) - env = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} - return subprocess.run( - ["git", "push", "-q", "origin", "HEAD:main"], - cwd=repo, - capture_output=True, - text=True, - check=False, - env=env, - ) - - -# --------------------------------------------------------------------------- pre-commit hook wiring - - -def test_clean_commit_invokes_preflight_and_lands(fake_hook_repo: Path) -> None: - """A passing fake harness lets the real pre-commit hook commit staged work.""" - stage(fake_hook_repo, "feature.py", "y = 2\n") - result = attempt_commit(fake_hook_repo, "clean work", loop=False, no_verify=False) - assert result.returncode == 0, result.stderr - assert fake_harness_args(fake_hook_repo) == "preflight\n" - assert "feature.py" in committed_files(fake_hook_repo) - - -def test_failing_preflight_blocks_the_commit(fake_hook_repo: Path) -> None: - """A nonzero fake harness exit makes the real pre-commit hook reject the commit.""" - set_fake_harness_exit(fake_hook_repo, 1) - stage(fake_hook_repo, "bad.py", "import os\ny = 2\n") - result = attempt_commit(fake_hook_repo, "blocked work", loop=False, no_verify=False) - assert result.returncode != 0 - assert fake_harness_args(fake_hook_repo) == "preflight\n" - assert "blocked work" not in log(fake_hook_repo) - - -def test_format_difference_does_not_block_when_preflight_passes(fake_hook_repo: Path) -> None: - """Hook wiring allows a commit whenever the harness preflight command exits cleanly.""" - stage(fake_hook_repo, "messy.py", "x=1\n") - result = attempt_commit(fake_hook_repo, "unformatted but clean", loop=False, no_verify=False) - assert result.returncode == 0, result.stderr - assert fake_harness_args(fake_hook_repo) == "preflight\n" - assert "messy.py" in committed_files(fake_hook_repo) - - -def test_loop_commit_passes_loop_env_to_preflight(fake_hook_repo: Path) -> None: - """A hook-run harness inherits RALPH_LOOP from the committing process.""" - stage(fake_hook_repo, "feature.py", "value = 1\n") - result = attempt_commit(fake_hook_repo, "loop work", loop=True, no_verify=False) - assert result.returncode == 0, result.stderr - assert fake_harness_args(fake_hook_repo) == "preflight\n" - assert fake_harness_loop(fake_hook_repo) == "1\n" - - -def test_no_verify_bypasses_the_hook(fake_hook_repo: Path) -> None: - """The git hook itself is skipped when git is asked not to run hooks.""" - stage(fake_hook_repo, "harness/evil.py", "value = 1\n") - result = attempt_commit(fake_hook_repo, "bypass", loop=True, no_verify=True) - assert result.returncode == 0 - assert not (fake_hook_repo / "harness.args").exists() - - -# --------------------------------------------------------------------------- full gate dispatch - - -def test_full_gate_end_to_end_and_pre_push_hook_blocks(monkeypatch: pytest.MonkeyPatch) -> None: - """Full gate dispatch runs the real header/bucket path and faults only the tool seam.""" - monkeypatch.delenv("RALPH_LOOP", raising=False) - calls = fake_popen(monkeypatch) - result = gate.run_checks(gate.FULL_CHECKS) - - assert [command for command, cwd, env in calls] == list(gate.FULL_CHECKS.values()) - assert all(cwd == gate.REPO_ROOT for command, cwd, env in calls) - assert all(env["FORCE_COLOR"] == "1" for command, cwd, env in calls) - assert set(result["pass"]) | set(result["fail"]) | set(result["warn"]) == set(gate.FULL_CHECKS) - assert result["fail"] == [] - assert len(result["pass"]) == 6 - assert "format" in result["warn"] - - -def test_format_report_stays_pass_when_runner_exits_nonzero(monkeypatch: pytest.MonkeyPatch) -> None: - """Format reports are informational, so a nonzero format check is still bucketed as pass.""" - monkeypatch.delenv("RALPH_LOOP", raising=False) # dispatch-only test: skip the containment git branch - fake_popen(monkeypatch, fails=[["fmt"]]) - result = gate.run_checks({"ruff lint": ["lint"], "ruff format (no fail)": ["fmt"]}) - assert result == {"pass": ["ruff lint"], "fail": [], "warn": ["ruff format (no fail)"]} - - -# --------------------------------------------------------------------------- pre-push hook wiring - - -def test_pre_push_hook_invokes_gate_and_blocks_on_fake_harness_failure(fake_hook_repo: Path) -> None: - """The real pre-push hook calls `.venv/bin/harness gate` and respects its nonzero exit.""" - stage(fake_hook_repo, "pushable.py", "value = 1\n") - commit = attempt_commit(fake_hook_repo, "push me", loop=False, no_verify=False) - assert commit.returncode == 0, commit.stderr - set_fake_harness_exit(fake_hook_repo, 1) - - push = push_head(fake_hook_repo) - - assert push.returncode != 0 - assert fake_harness_args(fake_hook_repo) == "gate\n" diff --git a/harness/tests/test_properties.py b/harness/tests/test_properties.py index 5f7902e..9dc8d1a 100644 --- a/harness/tests/test_properties.py +++ b/harness/tests/test_properties.py @@ -1,16 +1,12 @@ -"""Property-based tests for harness.gate and harness.preferences using Hypothesis. +"""Property-based tests for the banned-pattern scan and forbidden-path ejection in harness.gate. "With Hypothesis, you write tests which should pass for all inputs in whatever range you describe, and let -Hypothesis randomly choose which of those inputs to check - including edge cases you might not have thought -about." +Hypothesis randomly choose which of those inputs to check, including edge cases you might not have thought +about." TESTS THE CODE WITH A RANGE OF INPUTS. Hypothesis docs: https://hypothesis.readthedocs.io/ -This file keeps Hypothesis tests and the small example-based tests that share their helpers together. - -Covered behavior: - * banned-pattern scanning in gate.run_non_human_checks - * preferences.py checks for names, classes, comprehensions, and continue - * regression coverage for casefold-colliding forbidden paths in gate.py +Every test here stages real files in a temp repo and calls the real gate functions, which reach git +through the real gate.run_git. Nothing about git is stubbed. Hypothesis persistence: Do not set database=None by default. Local runs use Hypothesis's example database under .hypothesis/examples, so past failures are replayed first and users can debug them @@ -24,366 +20,115 @@ from __future__ import annotations -import importlib -import keyword -import string -import sys -from collections.abc import Callable -from unittest import mock +from collections.abc import Iterator +from pathlib import Path import pytest -from hypothesis import assume, example, given, settings -from hypothesis import strategies as st +from hypothesis import example, given, settings, strategies from harness import gate -# preferences.py is optional. Gate tests in this file should still run when it is absent. -preferences_violations: Callable[[str, str], str] | None -try: - from preferences.preferences import preferences_violations -except ImportError: - preferences_violations = None - -# Patterns in a deterministic order so hypothesis shrinks toward the first entry predictably. -IDENTIFIER_START = string.ascii_letters + "_" -IDENTIFIER_REST = IDENTIFIER_START + string.digits - - -@st.composite -def identifiers(draw: st.DrawFn) -> str: - """Draw a valid ASCII Python identifier that is not a keyword.""" - first = draw(st.sampled_from(IDENTIFIER_START)) - rest = draw(st.text(alphabet=IDENTIFIER_REST, max_size=24)) - name = first + rest - assume(not keyword.iskeyword(name)) - return name - - -IDENTIFIERS = identifiers() +def seed_repo(directory: Path) -> Path: + """Create a temp git repository with one commit and point gate's git calls at it.""" + gate.run_git(["init", "-q"], directory) + gate.run_git(["config", "user.email", "harness@test.local"], directory) + gate.run_git(["config", "user.name", "harness-test"], directory) + (directory / "README.md").write_text("seed\n", encoding="utf-8") + gate.run_git(["add", "README.md"], directory) + gate.run_git(["commit", "-q", "-m", "seed"], directory) + return directory -# ============================================================ banned-pattern scan (gate) helpers +@pytest.fixture(scope="module") +def scan_repo(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Path]: + """A temp repo shared by the generated examples, since @given cannot take a per-test fixture.""" + repo = seed_repo(tmp_path_factory.mktemp("banned-patterns")) + with pytest.MonkeyPatch.context() as patch: + patch.setattr(gate, "REPO_ROOT", repo) + yield repo -def scan_banned(diff: str, patterns: set[str] | None = None) -> list[str]: - """Drive run_non_human_checks' banned-pattern scan over a canned unified diff, no real git. - - Stubs run_git so the banned-pattern scan sees `diff` (returned for the `--unified=0` call). The - name-only ACMRD call returns one non-forbidden path so the empty-commit guard treats the index as - non-empty and nothing is ejected; every other git call returns "". prefs is forced to None so the - scan under test is the sole source of problems. - - Args: - diff: The `git diff --cached --unified=0` output the scan should read. - patterns: Optional override for FORBIDDEN_PATTERNS (to inject a mixed-case entry). - - Returns: - The problems list run_non_human_checks produced for that diff. - """ - - def fake_git(args: list[str]) -> str: - if "--unified=0" in args: - return diff - if "--name-only" in args and "--diff-filter=ACMRD" in args: - return "src/x.py\n" # a real staged file, so the empty-commit guard doesn't short-circuit - return "" - with ( - mock.patch.object(gate, "run_git", fake_git), - mock.patch.object(gate, "prefs", None), - mock.patch.object( - gate, "FORBIDDEN_PATTERNS", gate.FORBIDDEN_PATTERNS if patterns is None else patterns - ), - ): - return gate.run_non_human_checks() +def scan_staged(repo: Path, source: str) -> list[str]: + """Stage one file, run the real banned-pattern scan over the real index, then clear the index.""" + (repo / "x.py").write_text(source, encoding="utf-8") + gate.run_git(["add", "x.py"], repo) + problems = gate.check_for_bad_patterns() + gate.run_git(["reset", "-q"], repo) + return problems -@st.composite -def added_line_with_pattern(draw: st.DrawFn) -> tuple[str, str]: - """Draw a forbidden pattern and an added '+' diff line that embeds it in arbitrary casing. +@strategies.composite +def recased_pattern(draw: strategies.DrawFn) -> tuple[str, str]: + """Draw a forbidden pattern and the same pattern in arbitrary casing. Args: draw: Hypothesis draw callable. Returns: - (pattern, diff_line) where diff_line is a '+' added line whose casefold contains the pattern. + (pattern, recased) where recased differs from pattern only in the case of its letters. """ - pattern = draw(st.sampled_from(gate.FORBIDDEN_PATTERNS)) + pattern = draw(strategies.sampled_from(gate.FORBIDDEN_PATTERNS)) # Recase each alphabetic character independently; symbols (e.g. in '--no-verify') pass through. recased = "".join( - draw(st.sampled_from([char.lower(), char.upper()])) if char.isalpha() else char for char in pattern + draw(strategies.sampled_from([char.lower(), char.upper()])) if char.isalpha() else char + for char in pattern ) - return pattern, f"+value = 1 # {recased}" + return pattern, recased @settings(max_examples=50) -@given(case=added_line_with_pattern()) -@example(case=("# noqa", "+value = 1 # noqa")) # lowercase-alpha pattern -@example(case=("--no-verify", "+value = 1 # --NO-verify")) # symbol-heavy pattern -def test_banned_pattern_detected_across_arbitrary_line_casing(case: tuple[str, str]) -> None: - """Every forbidden pattern is flagged on a '+value = 1 # {pattern}' add line however that line is - cased. Generation earns its keep here: it recases each alpha char of each pattern independently, - covering casings no finite table would enumerate. It does NOT prove the pattern-side casefold fix - (all current patterns are lowercase, so recasing only the line never exercises it) -- that is - test_mixed_case_forbidden_entry_still_matches' job. +@given(case=recased_pattern()) +@example(case=("# noqa", "# noqa")) # lowercase-alpha pattern +@example(case=("--no-verify", "--NO-verify")) # symbol-heavy pattern +def test_banned_pattern_detected_across_arbitrary_line_casing(case: tuple[str, str], scan_repo: Path) -> None: + """An agent cannot smuggle an escape hatch past the scan by changing its capitalization. Every + forbidden pattern is caught on an added line however that line is cased. """ - pattern, diff_line = case - problems = scan_banned(diff_line) - assert any(p.startswith(f"'{pattern}' line:") for p in problems) - - -@pytest.mark.parametrize("diff_line", ["+++ b/hooksPath_helper.py", "-value = 1 # noqa"]) -def test_banned_pattern_ignores_header_and_removed_lines(diff_line: str) -> None: - """A pattern living only in a '+++' header or a removed '-' line is never flagged (only '+' adds).""" - assert scan_banned(diff_line) == [] + pattern, recased = case + problems = scan_staged(scan_repo, f"value = 1 # {recased}\n") + assert any(problem.startswith(f"'{pattern}' line:") for problem in problems) -@pytest.mark.parametrize("entry_casing", ["hookspath", "HooksPath", "HOOKSPATH"]) -def test_mixed_case_forbidden_entry_still_matches(entry_casing: str) -> None: - """A mixed-case ENTRY in FORBIDDEN_PATTERNS is still detected on an added line. This is the - regression guard for the source fix: with the old `pattern in line.casefold()`, a mixed-case entry - like 'HooksPath' can never be a substring of a casefolded line, so it silently never matches. It - FAILS on the pre-fix code (for the mixed-case entry) and PASSES once the comparison casefolds the - pattern too. The lowercase entry is the control that matched under the old code as well. +@pytest.mark.parametrize("pattern_in_toml", ["hookspath", "HooksPath", "HOOKSPATH"]) +def test_mixed_case_forbidden_entry_still_matches( + pattern_in_toml: str, monkeypatch: pytest.MonkeyPatch, git_repo: Path +) -> None: + """Editing pyproject.toml is how people add forbidden patterns, and they will not all type in + lowercase. A mixed-case entry has to match too, which it only does because the scan casefolds the + pattern as well as the line. """ - problems = scan_banned("+value = 1 # hookspath", patterns={entry_casing}) - assert any(p.startswith(f"'{entry_casing}' line:") for p in problems) - - -# ============================================================ AST style checks (preferences) - + monkeypatch.setattr(gate, "FORBIDDEN_PATTERNS", [pattern_in_toml]) + problems = scan_staged(git_repo, "value = 1 # hookspath\n") + assert any(problem.startswith(f"'{pattern_in_toml}' line:") for problem in problems) -def flags(source: str, needle: str) -> bool: - """Whether preferences_violations reports a message containing needle for source.""" - if preferences_violations is None: - pytest.skip("harness.preferences is optional and absent") - return needle in preferences_violations("m.py", source) - -# ------------------------------------------------------- preferences.py-absent fallback (optional module) - - -def test_module_tolerates_absent_preferences_on_import() -> None: - """When harness.preferences cannot be imported (a human deleted it), this module still loads and its - preferences_violations is None, so the gate tests here keep running. mock.patch.dict maps the module - name to None (the standard way to make `import` raise ImportError) and auto-restores it; reloading - this module under that patch exercises the ImportError fallback, then a final reload restores it. +def test_banned_pattern_ignores_the_diff_file_header(git_repo: Path) -> None: + """A file whose name contains a forbidden pattern puts that pattern in the diff's '+++ b/...' + header. The header is not code an agent added, so it must not be reported. """ - module = sys.modules[__name__] - try: - with mock.patch.dict(sys.modules, {"preferences.preferences": None}): - reloaded = importlib.reload(module) - assert reloaded.preferences_violations is None - finally: - importlib.reload(module) # restore the real preferences_violations for the rest of the suite + (git_repo / "noqa_helpers.py").write_text("value = 1\n", encoding="utf-8") + gate.run_git(["add", "noqa_helpers.py"], git_repo) + assert gate.check_for_bad_patterns() == [] -def test_flags_skips_when_preferences_absent() -> None: - """flags() skips (not crashes) when preferences_violations is None, so the AST-property tests skip as - a group when the optional module is gone. +def test_banned_pattern_ignores_a_removed_line(git_repo: Path) -> None: + """Deleting a line that carries an escape hatch is the fix, not the offense, so a removed '-' line + is never reported. """ - skipped = pytest.skip.Exception - with mock.patch.object(sys.modules[__name__], "preferences_violations", None), pytest.raises(skipped): - flags("x = 1\n", "anything") - - -# --------------------------------------------------------------------- underscore-lead identifier rule - - -@given(name=IDENTIFIERS) -def test_underscore_lead_flagged_iff_leading_underscore_not_dunder(name: str) -> None: - """An assignment target trips the underscore rule IFF it starts with '_' and does not end with '__'. - Covers the whole identifier domain, including dunders and lone '_', in one property. - """ - expected = name.startswith("_") and not name.endswith("__") - assert flags(f"{name} = 1\n", "starts with underscore") is expected - - -@given(name=IDENTIFIERS) -def test_underscore_rule_holds_for_function_and_argument_names(name: str) -> None: - """The same underscore rule applies to function names and argument names, not just assignments.""" - assume(not name.endswith("__")) # keep dunder methods/args (__init__ etc.) out of this slice - expected = name.startswith("_") - assert flags(f"def {name}():\n return 1\n", "starts with underscore") is expected - assert flags(f"def f({name}):\n return {name}\n", "starts with underscore") is expected - - -# ------------------------------------------------------------------------------- pointless-class rule - - -@st.composite -def class_source(draw: st.DrawFn) -> tuple[str, bool]: - """Draw a class definition varying base/decorator/keyword presence and method count. - - Args: - draw: Hypothesis draw callable. - - Returns: - (source, should_flag) where should_flag is the documented intent: trip IFF the class has no - base, no decorator, no keyword, and at most one method. - """ - has_base = draw(st.booleans()) - has_decorator = draw(st.booleans()) - has_keyword = draw(st.booleans()) - method_count = draw(st.integers(min_value=0, max_value=3)) - - decorator = "@deco\n" if has_decorator else "" - header_bits = (["Base"] if has_base else []) + (["metaclass=type"] if has_keyword else []) - header = f"({', '.join(header_bits)})" if header_bits else "" - body = "".join(f" def m{i}(self):\n return {i}\n" for i in range(method_count)) or " x = 1\n" - source = f"{decorator}class C{header}:\n{body}" - - should_flag = not has_base and not has_decorator and not has_keyword and method_count <= 1 - return source, should_flag - - -@given(case=class_source()) -def test_pointless_class_flagged_iff_plain_and_at_most_one_method(case: tuple[str, bool]) -> None: - """A class trips the pointless-class rule IFF it is plain (no base/decorator/keyword) with <= 1 - method. Any base, decorator, keyword, or a second method exempts it. - """ - source, should_flag = case - assert flags(source, "no base, decorator, or behavior") is should_flag - - -# ------------------------------------------------------------------------ complex-comprehension rule - - -@st.composite -def comprehension_source(draw: st.DrawFn) -> tuple[str, bool]: - """Draw a list comprehension with a chosen generator count and which generator (if any) filters. - - Args: - draw: Hypothesis draw callable. - - Returns: - (source, should_flag) where should_flag is the documented intent: trip IFF there is more than - one generator AND at least one generator carries an `if`. Crucially the filtered generator may - be a LATER one, exercising the check's early-return-on-first-match loop. - """ - generator_count = draw(st.integers(min_value=1, max_value=3)) - # -1 means "no if on any generator"; otherwise the index of the single generator that filters. - if_on = draw(st.integers(min_value=-1, max_value=generator_count - 1)) - - clauses: list[str] = [] - for index in range(generator_count): - clause = f"for v{index} in xs{index}" - if index == if_on: - clause += f" if v{index}" - clauses.append(clause) - source = f"[v0 {' '.join(clauses)}]\n" - - should_flag = generator_count > 1 and if_on != -1 - return source, should_flag - - -@given(case=comprehension_source()) -def test_complex_comprehension_flagged_iff_multi_generator_with_filter(case: tuple[str, bool]) -> None: - """A comprehension trips IFF it has multiple generators AND at least one has an `if` -- regardless of - WHICH generator carries the `if`. The later-generator case guards the check's early return, which - scans generators in order and returns on the first one that filters. - """ - source, should_flag = case - assert flags(source, "Overly complex comprehension") is should_flag - - -# --------------------------------------------------------------------------- chaotic-continue rule - - -@st.composite -def nested_continue_source(draw: st.DrawFn) -> str: - """Draw a `continue` wrapped in an outer `for` plus TWO-to-four more if/for blocks. - - The rule allows one `if` guard directly inside a loop (`for: if: continue`), so to always be - over-nested we stack at least two blocks below the outer loop. - - Args: - draw: Hypothesis draw callable. - - Returns: - Source whose `continue` sits at least two if/for blocks below its enclosing loop, so the - over-nesting rule always flags it. - """ - depth = draw(st.integers(min_value=2, max_value=4)) - blocks = draw(st.lists(st.sampled_from(["if cond", "for i in xs"]), min_size=depth, max_size=depth)) - - lines = ["for outer in items:"] # an outer loop the continue always belongs to - indent = " " - for block in blocks: - lines.append(f"{indent}{block}:") - indent += " " - lines.append(f"{indent}continue") - return "\n".join(lines) + "\n" - - -@given(source=nested_continue_source()) -def test_continue_nested_under_stacked_blocks_is_flagged(source: str) -> None: - """A `continue` stacked two or more if/for blocks below its enclosing loop is flagged as overly - nested, whatever mix of if/for those blocks are. - """ - assert flags(source, "Overly-nested 'continue'") - - -def test_single_if_guard_in_a_loop_is_not_flagged() -> None: - """The common, readable `for ...: if ...: continue` (one if guard in one loop) is NOT over-nested.""" - assert not flags("for i in items:\n if skip:\n continue\n", "Overly-nested 'continue'") - - -def test_shallow_continue_in_single_loop_is_not_flagged() -> None: - """Control (example, not property): a `continue` directly in one `for` -- parent For, grandparent - module -- is not overly nested, so it is not flagged. - """ - assert not flags("for x in items:\n continue\n", "Overly-nested 'continue'") - - -def test_continue_in_while_loop_is_flagged() -> None: - """Control (example): a `continue` anywhere inside a while loop is flagged (freeze risk), a separate - branch from the nested-if detection. - """ - assert flags("while cond:\n continue\n", "while loop banned") - - -# ============================================================ casefold path-collision ejection (gate) -# RED regression: fails until run_non_human_checks ejects EVERY casefold-colliding forbidden path. - - -def reset_paths_for(staged: list[str]) -> list[str] | None: - """Run run_non_human_checks over a canned staged list and capture the paths passed to `git reset`. - - Stubs run_git so `--name-only` returns the given staged paths and every other git call (the reset, - the unified diff, prefs) returns "". prefs is forced to None. Returns the path arguments of the - `reset -q HEAD --` call, or None if no reset was issued. Two colliding staged strings are injected - here rather than as real files (a case-insensitive filesystem could not hold both), so this exercises - the pure casefold-map logic. - - Args: - staged: The staged paths the ejection scan should see. - - Returns: - The list of paths git reset was asked to unstage, or None if ejection did not run. - """ - reset_args: list[str] | None = None - - def fake_git(args: list[str]) -> str: - nonlocal reset_args - if args[:1] == ["reset"]: - reset_args = args[args.index("--") + 1 :] - return "" - if "--name-only" in args and "--diff-filter=ACMRD" in args: - return "\n".join(staged) - return "" - - with mock.patch.object(gate, "run_git", fake_git), mock.patch.object(gate, "prefs", None): - gate.run_non_human_checks() - return reset_args + scan_staged(git_repo, "value = 1 # noqa\n") + gate.run_git(["add", "x.py"], git_repo) + gate.run_git(["commit", "-q", "-m", "seed noqa"], git_repo) + assert scan_staged(git_repo, "value = 1\n") == [] -def test_casefold_colliding_forbidden_paths_are_both_ejected() -> None: - """Two forbidden staged paths that differ only by case must BOTH be handed to git reset. Intended - behavior: neither slips past ejection. Fails hard while `{sf.casefold(): sf}` collapses them to one - key -- a real containment bug; this red test is the hand-off signal to fix run_non_human_checks. +def test_casefold_colliding_forbidden_paths_are_both_ejected(git_repo: Path) -> None: + """Two forbidden paths differing only in case must both be unstaged. A case-insensitive filesystem + cannot hold both as files, so they go into the index directly; neither may slip through. """ - colliding = ["harness/Gate.py", "harness/gate.py"] # both under the forbidden 'harness/' dir - reset = reset_paths_for(colliding) - assert reset is not None, "ejection did not run for forbidden paths" - assert set(reset) == set(colliding), f"only {reset} ejected; a casefold-colliding path slipped through" + colliding = ["harness/Gate.py", "harness/gate.py"] + blob = gate.run_git(["hash-object", "-w", "README.md"], git_repo).strip() + for path in colliding: + gate.run_git(["update-index", "--add", "--cacheinfo", f"100644,{blob},{path}"], git_repo) + gate.run_non_human_checks() + assert gate.run_git(["diff", "--cached", "--name-only"]).splitlines() == [] diff --git a/harness/tests/test_ralph.py b/harness/tests/test_ralph.py index 39f23cd..cc5923d 100644 --- a/harness/tests/test_ralph.py +++ b/harness/tests/test_ralph.py @@ -11,15 +11,10 @@ import os import shutil import subprocess -import sys from pathlib import Path -import pytest +from harness.tests.conftest import REPO_ROOT -# ralph.sh is the POSIX loop runner; Windows uses ralph.ps1 (see test_ralph_ps1.py). -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="POSIX ralph.sh; Windows uses ralph.ps1") - -REPO_ROOT = Path(__file__).resolve().parents[2] RALPH = REPO_ROOT / "harness" / "ralph.sh" diff --git a/harness/tests/test_ralph_ps1.py b/harness/tests/test_ralph_ps1.py new file mode 100644 index 0000000..2e16da1 --- /dev/null +++ b/harness/tests/test_ralph_ps1.py @@ -0,0 +1,148 @@ +"""Native Windows behavioral tests for the PowerShell Ralph runner.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from harness.tests.conftest import REPO_ROOT + +RALPH = REPO_ROOT / "harness" / "ralph.ps1" +POWERSHELL = shutil.which("powershell.exe") + + +def run_ralph( + tmp_path: Path, arguments: list[str], prompt: str = "do the most important thing" +) -> subprocess.CompletedProcess[str]: + """Run the PowerShell loop in a temporary working directory.""" + assert POWERSHELL is not None, "Windows CI requires Windows PowerShell" + env = os.environ.copy() + env["RALPH_PROMPT"] = prompt + return subprocess.run( + [POWERSHELL, "-NoProfile", "-File", str(RALPH), *arguments], + cwd=tmp_path, + capture_output=True, + text=True, + check=False, + env=env, + timeout=15, + ) + + +def write_worker(tmp_path: Path, source: str) -> Path: + """Write a Python worker used by a real PowerShell child process.""" + worker = tmp_path / "worker.py" + worker.write_text(source, encoding="utf-8") + return worker + + +def test_usage_fails_when_worker_command_is_missing(tmp_path: Path) -> None: + """The worker command is required after optional loop limits.""" + result = run_ralph(tmp_path, ["1", "1"]) + + assert result.returncode == 2 + assert "defaults: max_iterations=1 max_minutes_per_iteration=1" in result.stderr + + +@pytest.mark.parametrize(("iterations", "minutes"), [("0", "1"), ("1", "0")]) +def test_nonpositive_limits_are_rejected(tmp_path: Path, iterations: str, minutes: str) -> None: + """Zero cannot disable iteration or timeout bounds.""" + result = run_ralph(tmp_path, [iterations, minutes, sys.executable, "-c", "pass"]) + + assert result.returncode == 2 + assert "max_iterations must be >= 1 and max_minutes must be > 0" in result.stderr + + +def test_defaults_run_twice_and_pass_prompt_marker_and_environment(tmp_path: Path) -> None: + """Omitted limits use two iterations and pass the prompt, marker, and containment variable.""" + worker = write_worker( + tmp_path, + "from pathlib import Path\n" + "import os\n" + "import sys\n" + "count_path = Path('count.txt')\n" + "count = int(count_path.read_text() if count_path.exists() else '0') + 1\n" + "count_path.write_text(str(count))\n" + "Path(f'prompt-{count}.txt').write_text(sys.stdin.read(), encoding='utf-8')\n" + "Path(f'loop-{count}.txt').write_text(os.environ['RALPH_LOOP'], encoding='utf-8')\n", + ) + + result = run_ralph(tmp_path, [sys.executable, str(worker)]) + + assert result.returncode == 0 + assert (tmp_path / "count.txt").read_text(encoding="utf-8") == "2" + for iteration in (1, 2): + assert (tmp_path / f"prompt-{iteration}.txt").read_text(encoding="utf-8") == ( + f"do the most important thing\n\nRALPH_ITERATION={iteration}/2\n" + ) + assert (tmp_path / f"loop-{iteration}.txt").read_text(encoding="utf-8") == "1" + assert "completed 2 iteration(s)" in result.stderr + + +def test_explicit_one_iteration_completes(tmp_path: Path) -> None: + """An explicit one-iteration loop runs the worker once.""" + worker = write_worker(tmp_path, "import sys\nsys.stdin.read()\n") + + result = run_ralph(tmp_path, ["1", "1", sys.executable, str(worker)]) + + assert result.returncode == 0 + assert "iteration 1/1" in result.stderr + assert "completed 1 iteration(s)" in result.stderr + + +def test_worker_arguments_are_preserved_exactly(tmp_path: Path) -> None: + """Literal -p, model flags, and spaced values reach the worker as distinct unchanged arguments.""" + worker = write_worker( + tmp_path, + "from pathlib import Path\n" + "import json\n" + "import sys\n" + "Path('args.json').write_text(json.dumps(sys.argv[1:]), encoding='utf-8')\n" + "sys.stdin.read()\n", + ) + worker_args = ["-p", "--model", "claude opus", "value with spaces", 'quote"inside', "trailing\\"] + + result = run_ralph(tmp_path, ["1", "1", sys.executable, str(worker), *worker_args]) + + assert result.returncode == 0 + assert json.loads((tmp_path / "args.json").read_text(encoding="utf-8")) == worker_args + + +def test_command_without_additional_arguments_runs(tmp_path: Path) -> None: + """A worker executable with no argv tail does not trigger an invalid PowerShell array slice.""" + result = run_ralph(tmp_path, ["1", "1", "sort.exe"]) + + assert result.returncode == 0 + assert "completed 1 iteration(s)" in result.stderr + + +def test_nonzero_worker_exit_propagates_and_stops(tmp_path: Path) -> None: + """The first worker failure reaches the caller and prevents later iterations.""" + result = run_ralph( + tmp_path, + ["2", "1", sys.executable, "-c", "import sys; sys.stdin.read(); raise SystemExit(7)"], + ) + + assert result.returncode == 7 + assert "iteration 1/2" in result.stderr + assert "iteration 2/2" not in result.stderr + assert "completed" not in result.stderr + + +def test_fractional_timeout_returns_124_and_stops_process_tree(tmp_path: Path) -> None: + """A positive fractional minute gives a fast real timeout with GNU-compatible status 124.""" + result = run_ralph( + tmp_path, + ["2", "0.001", sys.executable, "-c", "import sys, time; sys.stdin.read(); time.sleep(30)"], + ) + + assert result.returncode == 124 + assert "iteration 1/2" in result.stderr + assert "iteration 2/2" not in result.stderr + assert "completed" not in result.stderr diff --git a/src/preferences/__init__.py b/preferences/__init__.py similarity index 100% rename from src/preferences/__init__.py rename to preferences/__init__.py diff --git a/src/preferences/preferences.js b/preferences/preferences.js similarity index 100% rename from src/preferences/preferences.js rename to preferences/preferences.js diff --git a/src/preferences/preferences.py b/preferences/preferences.py similarity index 99% rename from src/preferences/preferences.py rename to preferences/preferences.py index 22815c8..832f803 100644 --- a/src/preferences/preferences.py +++ b/preferences/preferences.py @@ -2,7 +2,7 @@ OPTIONAL for humans to use or edit! The functions below are examples to use. Or delete. -Agents in the loop cannot edit this file. It's in `FORBIDDEN_FILES` at `harness/gate.py`. +Agents in the loop cannot edit this file. It's in `FORBIDDEN_DIRS` at `harness/gate.py`. This module should reflect the repo owner's personal coding style hates. It's personal. e.g. indiscriminate __underscore_names, **star-unpacking, pointless classes, loops instead of Set math. diff --git a/pyproject.toml b/pyproject.toml index dc4001e..c0ffe1f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,35 +1,37 @@ [project] -name = "loopgate" -version = "0.0.0" +name = "harness" +version = "0.1.0" requires-python = ">=3.11" dependencies = [ "packaging", - "pydantic", "tomlkit", - "typer" + "typer", ] [dependency-groups] dev = [ + "complexipy", "hypothesis", "pylint", "pyright", "pytest", "pytest-cov", + "pytest-xdist", "ruff", "semgrep", - "complexipy" ] - [project.scripts] -harness = "harness.cli:main" +harness = "harness.cli:main" # creates the executable [build-system] requires = ["hatchling"] build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] -packages = ["harness", "src/preferences"] # src/ prefix stripped, so it imports as `preferences` -exclude = ["harness/tests"] # machinery tests don't ship; preferences/tests do +include = ["harness/**", "src/**"] +exclude = ["harness/**/tests/**"] + +[tool.poetry] +packages = [{ include = "harness" }] # ============================================================================== # Harness Configuration β€” single source of truth for agents and every check. @@ -91,28 +93,28 @@ languages = ["py"] # checks = ["bundle", "exec", "rspec"] [tool.harness.preflight] -lint = ["uv", "run", "--no-cache", "--no-sync", "ruff", "check", "--show-fixes", "."] -pylint = ["uv", "run", "--no-sync", "pylint", "src", "harness"] -format = ["uv", "run", "--no-sync", "ruff", "format", "--check"] -command = ["uv", "run", "--no-sync", "complexipy", "."] +lint = ["ruff", "check", "--no-cache", "--show-fixes", "."] +pylint = ["pylint", "."] +format = ["ruff", "format", "--no-cache", "--check"] +complexipy = ["complexipy", "."] [tool.harness.gate] security = [ - "uv", "run", "--no-sync", "semgrep", "scan", + "semgrep", "scan", "--error", # exit nonzero on findings so the gate blocks the commit, not just reports "--config", "auto", "--config", "p/secrets", "--exclude-rule", "yaml.github-actions.security.github-actions-mutable-action-tag.github-actions-mutable-action-tag", ".", ] -types = ["uv", "run", "--no-sync", "pyright", "--outputjson"] +types = ["pyright", "--outputjson"] pytest = [ - "uv", "run", "--no-cache", "--no-sync", "pytest", + "pytest", "-p", "no:cacheprovider", "-n", "auto", "--cov", "--cov-report=term-missing", "--cov-fail-under=100", ] [tool.harness.FORBIDDEN] -DIRS = ["harness/", "tests/harness/", ".githooks/", ".github/", "src/preferences/"] +DIRS = ["harness/", ".githooks/", ".github/", "preferences/", "tests/preferences/"] FILES = [ "agents.md", "pyproject.toml", @@ -153,7 +155,18 @@ PATTERNS = [ "fail_under", "cov-fail-under", "# pylint:", + "pytest.skip", "pytest.mark.xfail", + "pytest.mark.skipif", + "nosemgrep", + "# nosec", + "ruff: noqa", + "ruff: disable", + "ruff: ignore", + "fmt: off", + "fmt: skip", + "yapf: disable", + "complexipy: ignore" ] # ============================================================================== @@ -165,38 +178,37 @@ reportAssertAlwaysTrue = "error" # Catch asserts that are always true. reportUnusedImport = "none" # Handled by Ruff F401 reportUnusedVariable = "none" # Handled by Ruff F841 verboseOutput = true # JSON report output when Pyright is run -include = ["src", "harness"] # Type-check only code shipped by this template. +include = ["src", "preferences", "harness"] # Type-check shipped code plus repository preferences. # ============================================================================== # Pytest Configuration # ============================================================================== [tool.pytest.ini_options] minversion = "9.0" -addopts = ["-ra"] # print useful short reasons for outcomes -testpaths = ["tests", "harness/tests", "src/preferences/tests"] -pythonpath = ["src", "harness"] +addopts = ["-ra"] # Print useful short reasons for outcomes +testpaths = ["tests", "harness"] +pythonpath = ["src", "harness"] # Lets tests import Python code located in the named folders # ============================================================================== # Test Coverage Configuration # ============================================================================== [tool.coverage] -# Add every directory that ships code here to ensure 100% coverage -run.source = ["src", "harness"] -report.show_missing = true # Show exact missed lines when coverage fails. -report.skip_covered = false # Keep fully covered files visible in reports. -report.fail_under = 100 # Block merges unless all shipped code is covered. -skip_covered = false # Same behavior for older coverage config readers. +run.source = ["src", "preferences", "harness"] # ADD EVERY DIRECTORY THAT SHIPS CODE TO ENSURE 100% COVERAGE +report.show_missing = true # Show exact missed lines when coverage fails +report.skip_covered = false # Keep fully covered files visible in reports +report.fail_under = 100 # Block merges unless all shipped code is covered +skip_covered = false # Same behavior for older coverage config readers # ============================================================================== # Complexipy Configuration # ============================================================================== [tool.complexipy] -paths = ["src", "harness"] -exclude = ["**/tests/**"] # Test code can be more branching and example-heavy than production code. -max-complexity-allowed = 10 # Cognitive complexity cap catches deeply nested, hard-to-read flow. -no-ignore = true # Disallow `# complexipy: ignore`, fix complexity instead of suppressing it. +paths = ["src", "preferences", "harness"] +exclude = ["**/tests/**"] # Test code can be more branching and example-heavy than production code +max-complexity-allowed = 10 # Cognitive complexity cap catches deeply nested, hard-to-read flow +no-ignore = true # Disallow `# complexipy: ignore`, fix complexity instead of suppressing it report-ignored = true -failed = true # Show only functions over the threshold so gate output stays actionable. +failed = true # Show only functions over the threshold so gate output stays actionable sort = "file_name" quiet = false @@ -215,23 +227,24 @@ main.ignore = [ "scratchpad", "tests", "**/tests/**", - "harness/tests" + "harness/tests", + "worktrees" ] format.max-line-length = 110 # Keep Pylint line accounting aligned with Ruff E501 format.max-module-lines = 500 # Enable Pylint C0302 because Ruff has no equivalent -parameter_documentation.accept-no-param-doc = false # Require Args docs when Pylint docparams sees params. -reports.reports = "yes" # Print detailed Pylint reports so humans see why score changed. -reports.score = true # Keep Pylint's score visible as a coarse trend signal. -"messages control".enable = ["F", "I", "R0022", "missing-param-doc"] # Fatal/info, stale options, param docs. +parameter_documentation.accept-no-param-doc = false # Require Args docs when Pylint docparams sees params +reports.reports = "yes" # Print detailed Pylint reports so humans see why score changed +reports.score = true # Keep Pylint's score visible as a coarse trend signal +"messages control".enable = ["F", "I", "R0022", "missing-param-doc"] # Fatal/info, stale options, param docs # ============================================================================== # Unified Ruff Configuration # ============================================================================== [tool.ruff] target-version = "py311" -preview = true # Enable newer Ruff rules before they are fully stable. -line-length = 110 # Keep code readable without forcing narrow wrapping. -force-exclude = true # Respect excluded paths even when they are passed directly. +preview = true # Enable newer Ruff rules before they are fully stable +line-length = 110 # Keep code readable without forcing narrow wrapping +force-exclude = true # Respect excluded paths even when they are passed directly exclude = [ "*.md", ".git", @@ -251,8 +264,8 @@ fixable = ["ALL"] unfixable = [] # Allow Ruff to safely auto-fix what it can mccabe.max-complexity = 8 # Prevent structural bloat, path-count limit for functions pycodestyle.max-doc-length = 110 # Enforce W505 for comments and docstrings -pycodestyle.ignore-overlong-task-comments = true # Let TODO-style tracker lines exceed max doc length. -pydocstyle.convention = "google" # Interpret Args/Returns/Raises sections as Google-style docstrings. +pycodestyle.ignore-overlong-task-comments = true # Let TODO-style tracker lines exceed max doc length +pydocstyle.convention = "google" # Interpret Args/Returns/Raises sections as Google-style docstrings pydoclint.ignore-one-line-docstrings = true # Allow obvious one-line docstrings light select = [ "A", # flake8-builtins (prevents shadowing Python builtins) @@ -264,6 +277,16 @@ select = [ "C4", # flake8-comprehensions (.e.g unnecessary list comprehensions) "C90", # Enables McCabe cyclomatic complexity checks "COM", # flake8-commas. Trailing comma added. + "D102", # No missing docstring in public method + "D103", # No missing docstring in public function + "D202", # No blank line between a function docstring and the function body + "D300", # Use """ triple quotes for docstrings """ + "D417", # No missing arg description in docstring for {definition}: {name} + "D419", # No empty docstrings + "DOC", # Global. Validates parameter & return documentation + "DOC102", # Redundancy of "DOC". Documented parameter {id} must be in the function's signature + "DOC201", # Redundancy of "DOC". Structured docstrings must include Returns when code returns. + "DOC501", # Redundancy of "DOC". Raised exception {id} missing from docstring "DTZ", # flake8-datetimez (forbid naive datetime usage) "E", # pycodestyle: Catches objective syntax rule violations "EM", # flake8-errmsg (keep exception messages clean and reusable) @@ -291,28 +314,19 @@ select = [ "RUF", # Ruff-specific rules "S", # flake8-bandit (security checks: SQLi, hardcoded credentials) "SIM", # flake8-simplify (prefer simpler control flow and syntax) + "SLF001", # No start with single underscore outside defining module "T10", # flake8-debugger (forbid breakpoint/debugger calls) "T20", # flake8-print (forbids print statements; forces logging) "TID", # flake8-tidy-imports (forces clean absolute imports) "TRY", # tryceratops (exception handling hygiene) "UP", # Modern Python idioms "W", # Catches stylistic choices that harm readability - "private-member-access", # No start with single underscore outside defining module - "triple-single-quotes", # Use """ triple quotes for docstrings """ - "undocumented-public-method", # No missing docstring in public method - "undocumented-public-function", # No missing docstring in public function - "blank-line-after-function", # No blank line between a function docstring and the function body - "undocumented-param", # No missing arg description in docstring for {definition}: {name} - "empty-docstring", # No empty docstrings - "DOC", # Global. Validates parameter & return documentation - "docstring-extraneous-parameter", # Redundancy of "DOC". Documented parameter {id} must be in the function's signature - "docstring-missing-returns", # Redundancy of "DOC". Structured docstrings must include Returns when code returns. - "docstring-missing-exception", # Redundancy of "DOC". Raised exception {id} missing from docstring ] ignore = [ - "missing-trailing-comma", # Let Ruff format own trailing commas to avoid formatter conflicts - "magic-value-comparison", # Allow 'magic values" (unnamed numbers) - "start-process-with-partial-path", # Allow resolving git/uv from PATH + "COM812", # Let Ruff format own trailing commas to avoid formatter conflicts, missing-trailing-comma + "PLR2004", # Allow 'magic values" (unnamed numbers), start-process-with-partial-path + "S607", # Allow resolving git/uv from PATH, start-process-with-partial-path + "RUF201" # Do not complain no string rule-codes-in-selectors (too new since v.0.15.22) ] [tool.ruff.lint.pylint] # Prevent unreadable complexity and code max-nested-blocks = 4 # Stop indentation madness @@ -330,17 +344,18 @@ banned-api."typing.cast".msg = "Do not use cast(); fix the type at the boundary banned-api."re".msg = "Regex is slow. Use Python built-in string manipulation operations." [tool.ruff.lint.per-file-ignores] -"**/__init__.py" = ["undocumented-public-package"] # Package marker files do not need package docstrings. +"**/__init__.py" = ["D104"] # __init__.py Package marker files not required to have package docstrings "**/tests/**/*.py" = [ - "undocumented-public-module", # Test modules do not need noisy docstrings - "undocumented-public-class", # Test classes do not need noisy docstrings - "undocumented-public-method", # Test helper methods do not need noisy docstrings - "undocumented-public-function", # Test functions describe behavior through test names - "undocumented-public-package", # Test package boundaries do not need package docstrings - "docstring-missing-returns", # Tests explain behavior through assertions; no Returns section needed. - "assert", # Allow 'assert' statements (needed for pytest) + "D100", # Test modules do not need noisy docstrings, undocumented-public-module + "D101", # Test classes do not need noisy docstrings, undocumented-public-class + "D102", # Test helper methods do not need noisy docstrings, undocumented-public-method + "D103", # Test functions describe behavior through test names, undocumented-public-function + "D104", # Test package boundaries do not need package docstrings, undocumented-public-package + "DOC201", # Tests explain behavior through assertions, docstring-missing-returns + "S101", # Allow 'assert' statements (needed for pytest) + "PLR0915", # Tests may need larger setup functions ] "harness/**/*.py" = [ - "suspicious-subprocess-import", # Harness code intentionally imports subprocess to run local tools. - "subprocess-without-shell-equals-true" # Harness code intentionally executes trusted local tool commands without a shell. + "S404", # Harness code intentionally imports subprocess to run local tools. suspicious-subprocess-import + "S603", # Harness code intentionally executes trusted local tool commands without a shell. subprocess-without-shell-equals-true ] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..4268e2f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +complexipy +hypothesis +packaging +pylint +pyright +pytest +pytest-cov +ruff +semgrep +tomlkit +typer diff --git a/src/preferences/tests/__init__.py b/src/preferences/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/.gitkeep b/tests/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/src/preferences/tests/test_preferences.py b/tests/preferences/test_preferences.py similarity index 99% rename from src/preferences/tests/test_preferences.py rename to tests/preferences/test_preferences.py index 3d5e3ed..4b4fabd 100644 --- a/src/preferences/tests/test_preferences.py +++ b/tests/preferences/test_preferences.py @@ -1,4 +1,4 @@ -"""Tests for AST-based structural style checks (harness.preferences). +"""Tests for AST-based structural style checks (preferences.preferences). The preferences API is a registry of single-node `Check` functions (each takes one `ast.AST` node and returns a complaint string or None), plus `preferences_violations`, which walks a file @@ -13,7 +13,6 @@ import pytest -# preferences.py is optional β€” humans may delete it; skip its tests when it is gone. preferences = pytest.importorskip("preferences.preferences") preferences_violations = preferences.preferences_violations CHECKS = preferences.CHECKS diff --git a/tests/preferences/test_preferences_properties.py b/tests/preferences/test_preferences_properties.py new file mode 100644 index 0000000..3ca4705 --- /dev/null +++ b/tests/preferences/test_preferences_properties.py @@ -0,0 +1,247 @@ +"""Property-based tests for preferences.preferences.py using Hypothesis. + +"With Hypothesis, you write tests which should pass for all inputs in whatever range you describe, and let +Hypothesis randomly choose which of those inputs to check, including edge cases you might not have thought +about." TESTS THE CODE WITH A RANGE OF INPUTS. +Hypothesis docs: https://hypothesis.readthedocs.io/ + +Tests that preferences.py checks for names, classes, comprehensions, and continue + +Hypothesis persistence: Do not set database=None by default. Local runs use Hypothesis's example +database under .hypothesis/examples, so past failures are replayed first and users can debug them +quickly. CI automatically uses Hypothesis's built-in `ci` profile, which is stateless and deterministic. +If a generated input is important, save it as @example(...) or a normal regression test instead of +relying on the local database. The generated .hypothesis/ directory is gitignored. + +Test hygiene: keep strategies at module scope. Set max_examples only when a test needs a runtime cap. Do +not use function-scoped fixtures with @given; patch per-example state inside helper functions instead. +""" + +from __future__ import annotations + +import importlib +import keyword +import string +import sys +from collections.abc import Callable +from unittest import mock + +from hypothesis import assume, given, strategies + +# preferences.py are optional AST-style checks. Repo-owner can keep or delete. +preferences_violations: Callable[[str, str], str] | None +try: + from preferences.preferences import preferences_violations +except ImportError: + preferences_violations = None + +# Patterns in a deterministic order so hypothesis shrinks toward the first entry predictably. +IDENTIFIER_START = string.ascii_letters + "_" +IDENTIFIER_REST = IDENTIFIER_START + string.digits + + +@strategies.composite +def identifiers(draw: strategies.DrawFn) -> str: + """Draw a valid ASCII Python identifier that is not a keyword.""" + first = draw(strategies.sampled_from(IDENTIFIER_START)) + rest = draw(strategies.text(alphabet=IDENTIFIER_REST, max_size=24)) + name = first + rest + assume(not keyword.iskeyword(name)) + return name + + +IDENTIFIERS = identifiers() + + +def flags(source: str, needle: str) -> bool: + """Whether preferences_violations reports a message containing needle for source.""" + return needle in preferences_violations("m.py", source) if preferences_violations else False + + +# ------------------------------------------------------- preferences.py-absent fallback (optional module) + + +def test_module_tolerates_absent_preferences_on_import() -> None: + """When preferences.preferences.py can't be imported (e.g. human deleted), this module still loads and its + preferences_violations is None, so its gate tests here keep running. mock.patch.dict maps the module + name to None (the standard way to make `import` raise ImportError) and auto-restores it; reloading + this module under that patch exercises the ImportError fallback, then a final reload restores it. + """ + module = sys.modules[__name__] + try: + with mock.patch.dict(sys.modules, {"preferences.preferences": None}): + reloaded = importlib.reload(module) + assert reloaded.preferences_violations is None + finally: + importlib.reload(module) # restore the real preferences_violations for the rest of the suite + + +def test_flags_never_calls_preferences_when_it_is_absent() -> None: + """Deleting preferences.py must not crash flags(). It reports no violation instead. + + A variable named with a leading underscore breaks the underscore rule. The first assert confirms + the rule fires on it. The second feeds in that same variable with the module gone and gets no + violation back, which is only possible if preferences_violations was never called. + """ + source, rule = "_bad = 1\n", "starts with underscore" + assert flags(source, rule) is True + with mock.patch.object(sys.modules[__name__], "preferences_violations", None): + assert flags(source, rule) is False + + +# --------------------------------------------------------------------- underscore-lead identifier rule + + +@given(name=IDENTIFIERS) +def test_underscore_lead_flagged_iff_leading_underscore_not_dunder(name: str) -> None: + """An assignment target trips the underscore rule IFF it starts with '_' and does not end with '__'. + Covers the whole identifier domain, including dunders and lone '_', in one property. + """ + expected = name.startswith("_") and not name.endswith("__") + assert flags(f"{name} = 1\n", "starts with underscore") is expected + + +@given(name=IDENTIFIERS) +def test_underscore_rule_holds_for_function_and_argument_names(name: str) -> None: + """The same underscore rule applies to function names and argument names, not just assignments.""" + assume(not name.endswith("__")) # keep dunder methods/args (__init__ etc.) out of this slice + expected = name.startswith("_") + assert flags(f"def {name}():\n return 1\n", "starts with underscore") is expected + assert flags(f"def f({name}):\n return {name}\n", "starts with underscore") is expected + + +# ------------------------------------------------------------------------------- pointless-class rule + + +@strategies.composite +def class_source(draw: strategies.DrawFn) -> tuple[str, bool]: + """Draw a class definition varying base/decorator/keyword presence and method count. + + Args: + draw: Hypothesis draw callable. + + Returns: + (source, should_flag) where should_flag is the documented intent: trip IFF the class has no + base, no decorator, no keyword, and at most one method. + """ + has_base = draw(strategies.booleans()) + has_decorator = draw(strategies.booleans()) + has_keyword = draw(strategies.booleans()) + method_count = draw(strategies.integers(min_value=0, max_value=3)) + + decorator = "@deco\n" if has_decorator else "" + header_bits = (["Base"] if has_base else []) + (["metaclass=type"] if has_keyword else []) + header = f"({', '.join(header_bits)})" if header_bits else "" + body = "".join(f" def m{i}(self):\n return {i}\n" for i in range(method_count)) or " x = 1\n" + source = f"{decorator}class C{header}:\n{body}" + + should_flag = not has_base and not has_decorator and not has_keyword and method_count <= 1 + return source, should_flag + + +@given(case=class_source()) +def test_pointless_class_flagged_iff_plain_and_at_most_one_method(case: tuple[str, bool]) -> None: + """A class trips the pointless-class rule IFF it is plain (no base/decorator/keyword) with <= 1 + method. Any base, decorator, keyword, or a second method exempts it. + """ + source, should_flag = case + assert flags(source, "no base, decorator, or behavior") is should_flag + + +# ------------------------------------------------------------------------ complex-comprehension rule + + +@strategies.composite +def comprehension_source(draw: strategies.DrawFn) -> tuple[str, bool]: + """Draw a list comprehension with a chosen generator count and which generator (if any) filters. + + Args: + draw: Hypothesis draw callable. + + Returns: + (source, should_flag) where should_flag is the documented intent: trip IFF there is more than + one generator AND at least one generator carries an `if`. Crucially the filtered generator may + be a LATER one, exercising the check's early-return-on-first-match loop. + """ + generator_count = draw(strategies.integers(min_value=1, max_value=3)) + # -1 means "no if on any generator"; otherwise the index of the single generator that filters. + if_on = draw(strategies.integers(min_value=-1, max_value=generator_count - 1)) + + clauses: list[str] = [] + for index in range(generator_count): + clause = f"for v{index} in xs{index}" + if index == if_on: + clause += f" if v{index}" + clauses.append(clause) + source = f"[v0 {' '.join(clauses)}]\n" + + should_flag = generator_count > 1 and if_on != -1 + return source, should_flag + + +@given(case=comprehension_source()) +def test_complex_comprehension_flagged_iff_multi_generator_with_filter(case: tuple[str, bool]) -> None: + """A comprehension trips IFF it has multiple generators AND at least one has an `if` -- regardless of + WHICH generator carries the `if`. The later-generator case guards the check's early return, which + scans generators in order and returns on the first one that filters. + """ + source, should_flag = case + assert flags(source, "Overly complex comprehension") is should_flag + + +# --------------------------------------------------------------------------- chaotic-continue rule + + +@strategies.composite +def nested_continue_source(draw: strategies.DrawFn) -> str: + """Draw a `continue` wrapped in an outer `for` plus TWO-to-four more if/for blocks. + + The rule allows one `if` guard directly inside a loop (`for: if: continue`), so to always be + over-nested we stack at least two blocks below the outer loop. + + Args: + draw: Hypothesis draw callable. + + Returns: + Source whose `continue` sits at least two if/for blocks below its enclosing loop, so the + over-nesting rule always flags it. + """ + depth = draw(strategies.integers(min_value=2, max_value=4)) + blocks = draw( + strategies.lists(strategies.sampled_from(["if cond", "for i in xs"]), min_size=depth, max_size=depth) + ) + + lines = ["for outer in items:"] # an outer loop the continue always belongs to + indent = " " + for block in blocks: + lines.append(f"{indent}{block}:") + indent += " " + lines.append(f"{indent}continue") + return "\n".join(lines) + "\n" + + +@given(source=nested_continue_source()) +def test_continue_nested_under_stacked_blocks_is_flagged(source: str) -> None: + """A `continue` stacked two or more if/for blocks below its enclosing loop is flagged as overly + nested, whatever mix of if/for those blocks are. + """ + assert flags(source, "Overly-nested 'continue'") + + +def test_single_if_guard_in_a_loop_is_not_flagged() -> None: + """The common, readable `for ...: if ...: continue` (one if guard in one loop) is NOT over-nested.""" + assert not flags("for i in items:\n if skip:\n continue\n", "Overly-nested 'continue'") + + +def test_shallow_continue_in_single_loop_is_not_flagged() -> None: + """Control (example, not property): a `continue` directly in one `for` -- parent For, grandparent + module -- is not overly nested, so it is not flagged. + """ + assert not flags("for x in items:\n continue\n", "Overly-nested 'continue'") + + +def test_continue_in_while_loop_is_flagged() -> None: + """Control (example): a `continue` anywhere inside a while loop is flagged (freeze risk), a separate + branch from the nested-if detection. + """ + assert flags("while cond:\n continue\n", "while loop banned") From ea146dc3376bcf8c41a92d841195ee8ba2885b20 Mon Sep 17 00:00:00 2001 From: Roxana del Toro Date: Sun, 2 Aug 2026 07:40:46 -0700 Subject: [PATCH 2/5] Adds mutmut and pip-audit to pyproject.toml --- pyproject.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c0ffe1f..3e99b42 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,13 +11,15 @@ dependencies = [ dev = [ "complexipy", "hypothesis", + "mutmut", + "pip-audit", "pylint", "pyright", "pytest", "pytest-cov", "pytest-xdist", "ruff", - "semgrep", + "semgrep" ] [project.scripts] harness = "harness.cli:main" # creates the executable From e0dfd29be974f1e75bc54d096bb3d13c3af9c365 Mon Sep 17 00:00:00 2001 From: Roxana del Toro Date: Sun, 2 Aug 2026 10:34:33 -0700 Subject: [PATCH 3/5] Adds preferences test for when an AST parser doesn't report line number --- harness/gate.py | 2 +- tests/preferences/test_preferences.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/harness/gate.py b/harness/gate.py index ff7abda..968ae42 100644 --- a/harness/gate.py +++ b/harness/gate.py @@ -5,7 +5,7 @@ All containment lists and check commands come from [tool.harness] in pyproject.toml, read once at import into the constants below. A check is a (name, argv) pair; its `preflight`/`blocking` flags sort -it into the maps and sets this module runs on. Nothing is hardcoded here. +it into the maps and sets this module runs on. """ from __future__ import annotations diff --git a/tests/preferences/test_preferences.py b/tests/preferences/test_preferences.py index 4b4fabd..0d8e6be 100644 --- a/tests/preferences/test_preferences.py +++ b/tests/preferences/test_preferences.py @@ -10,6 +10,7 @@ import ast import inspect +from unittest.mock import Mock import pytest @@ -275,6 +276,22 @@ def test_preferences_violations_returns_grouped_str() -> None: assert not violations +def test_locationless_node_violation_reports_unknown_line(monkeypatch: pytest.MonkeyPatch) -> None: + """A `preferences` check reports '?' for missing lineno when its AST node has no parser-provided line. + + Args: + monkeypatch: Sets/restores the AST parser for the test. + """ + source = "value = lambda: None\n" + tree = ast.parse(source) + lambda_node = next(node for node in ast.walk(tree) if isinstance(node, ast.Lambda)) + del lambda_node.lineno + monkeypatch.setattr(ast, "parse", Mock(return_value=tree)) + + expected = "m.py:?: Lambda found hurting readability and adding complexity." + assert preferences_violations("m.py", source) == expected + + def test_clean_file_has_no_complaints() -> None: """A compliant module produces no complaints (the empty string).""" source = ( From 721a76f981500ea5bb75d110db2959140a32c2eb Mon Sep 17 00:00:00 2001 From: Roxana del Toro Date: Sun, 2 Aug 2026 11:09:32 -0700 Subject: [PATCH 4/5] Make universal hash for new repo, aka empty tree, a global constant in gate.py Adds proof that Mutmut will run in *this* directory Adds Mutmut ci.yml and test Make pylint ignore worktrees from root --- .githooks/_resolve | 2 +- .github/workflows/mutation.yml | 50 ++++ .gitignore | 3 + README.md | 9 +- harness/cli.py | 6 +- harness/gate.py | 7 +- harness/tests/test_cli.py | 2 +- harness/tests/test_gate.py | 21 +- mutation/check_mutmut.py | 105 ++++++++ pyproject.toml | 29 +- tests/mutation/mutmut-cicd-stats.json | 11 + tests/mutation/test_check_mutmut.py | 61 +++++ tests/preferences/test_preferences.py | 127 +++++++++ .../test_preferences_properties.py | 247 ------------------ 14 files changed, 402 insertions(+), 278 deletions(-) create mode 100644 .github/workflows/mutation.yml create mode 100644 mutation/check_mutmut.py create mode 100644 tests/mutation/mutmut-cicd-stats.json create mode 100644 tests/mutation/test_check_mutmut.py delete mode 100644 tests/preferences/test_preferences_properties.py diff --git a/.githooks/_resolve b/.githooks/_resolve index db6272d..f258b88 100644 --- a/.githooks/_resolve +++ b/.githooks/_resolve @@ -10,7 +10,7 @@ # # Then, git hook does not require `uv` or `pip` or any specific environment layout. -recorded="${GIT_DIR:-$(git rev-parse --absolute-git-dir)}/harness-path" +recorded="$(git rev-parse --path-format=absolute --git-common-dir)/harness-path" if [ ! -r "$recorded" ]; then echo "loopgate: hooks are not installed. Run 'harness install' in this repo." >&2 exit 1 diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml new file mode 100644 index 0000000..a393641 --- /dev/null +++ b/.github/workflows/mutation.yml @@ -0,0 +1,50 @@ +name: Mutation Testing + +on: + schedule: + - cron: "0 0 * * 0" # Weekly at midnight Sunday + workflow_dispatch: # adds "Run workflow" button + +env: + FORCE_COLOR: "1" + CLICOLOR_FORCE: "1" + TERM: "xterm-256color" + +permissions: + contents: read + +concurrency: + group: mutation-${{ github.ref }} + cancel-in-progress: true + +jobs: + mutate: + runs-on: ubuntu-latest + timeout-minutes: 60 + + steps: + - name: Checkout Code + uses: actions/checkout@v6 + + - name: Install uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + - name: uv Sync Dependencies + run: uv sync + + - name: Harness Checks + run: uv run --no-sync harness gate # need tests for Mutmut to target + + - name: Run Mutmut + run: mutmut run || true + + - name: Export CI/CD Stats + run: mutmut export-cicd-stats + + - name: Upload Mutmut JSON Report + uses: actions/upload-artifact@v4 + with: + name: mutmut-json-report + path: mutants/mutmut-cicd-stats.json diff --git a/.gitignore b/.gitignore index 4065523..16e3312 100644 --- a/.gitignore +++ b/.gitignore @@ -27,5 +27,8 @@ scratchpad/* scratchpad/runs/* !scratchpad/runs/.gitkeep +# mutmut's mutated copies of the source. Ignoring them is also what keeps ruff and semgrep out. +mutants/ + # kept at the last line for easy deletion (real projects need lockfile) uv.lock diff --git a/README.md b/README.md index 0193761..01100af 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ - **Worker-agnostic**: Claude, Codex, Copilot, Agy, or any prompt-reading CLI - **No lazy**: Agents work, _only if they pass the quality gates you set_ βœ… - **Repo-as-memory workflow**: specs/status/prompt are durable but code is king, leaving you free 😎 -- **Built-in stack**: Ruff, Pyright, Pylint, Semgrep, Complexipy, Hypothesis, 100% coverage β˜‘β˜‘β˜‘ +- **Built-in stack**: Ruff, Pyright, Pylint, Semgrep, Complexipy, Hypothesis, Mutmut, 100% coverage β˜‘β˜‘β˜‘ - **Progressive**: Preflight vs full gate split πŸ†— - **Forbidden-path containment**: Don't touch that!-configurable πŸ›‘ - **Installable project template**: `harness install ` gets the repo ready ▢️ @@ -187,16 +187,17 @@ To run LoopGate with Claude : harness run claude 2 20 ``` -Note: The worker must be installed and authenticated separately. +Note: The worker must be installed and authenticated separately. **Claude Code exports env vars into every shell it spawns. `RALPH_LOOP=1` can be set globally.**
## Expanding your harness -- Edit rules at [pyproject.toml](pyproject.toml) for [ruff](https://docs.astral.sh/ruff/), [pylint](https://pypi.org/project/pylint/), [pydoclint](https://pypi.org/project/pydoclint/0.9.1/), [pyright](https://github.com/microsoft/pyright), [pytest](https://docs.pytest.org/en/stable/), [hypothesis](https://hypothesis.readthedocs.io/), [complexipy](https://github.com/rohaquinlop/complexipy) +- Edit rules at [pyproject.toml](pyproject.toml) for [ruff](https://docs.astral.sh/ruff/), [pylint](https://pypi.org/project/pylint/), [pydoclint](https://pypi.org/project/pydoclint/0.9.1/), [pyright](https://github.com/microsoft/pyright), [pytest](https://docs.pytest.org/en/stable/), [hypothesis](https://hypothesis.readthedocs.io/), [complexipy](https://github.com/rohaquinlop/complexipy), [mutmut](https://mutmut.readthedocs.io/) - Add forbidden files, directories, or patterns in `[tool.harness.gate]` at [pyproject.toml](pyproject.toml) - Add [Hypothesis](https://hypothesis.readthedocs.io/) tests in any test directory, examples at [test_properties.py](tests/preferences/test_properties.py). +- Run [mutmut](https://mutmut.readthedocs.io/) by hand with `uv run mutmut run`, then `uv run mutmut browse`. A surviving mutant is a covered line no assertion checks. It is not a gate check: `mutmut run` exits 0 even with survivors. Its verdicts are cached against source hashes and ignore test edits, so re-run a single mutant by name once you have written a test for it. Examples at [test_with_mutations.py](tests/preferences/test_with_mutations.py). - [semgrep](https://docs.semgrep.dev/semgrep-ci/sample-ci-configs) has no repo config here. It uses registry configs / Semgrep's built-in defaults which ignore tests. - Update `[tool.harness.gate.checks]` in [pyproject.toml](pyproject.toml). [ci.yml](.github/workflows/ci.yml) runs those **same exact** `harness gate` checks. - Add or remove coding preferences [preferences.py](preferences/preferences.py) that only agents in loops **must** respect. Current preferences: @@ -311,7 +312,7 @@ npm run --prefix harness/js-scaffold preflight 5. Protect `main` and run the loop on its own branch. -6. **100% coverage does not mean good tests.** That is quantity, not quality. (Upcoming feature: mutation testing) +6. **100% coverage does not mean good tests.** That is quantity, not quality. Run `uv run mutmut run` to find covered lines that no assertion actually checks. 7. **Note**: `semgrep --config auto` needs network for semgrep registry rules. diff --git a/harness/cli.py b/harness/cli.py index 79b104c..abdcf18 100644 --- a/harness/cli.py +++ b/harness/cli.py @@ -61,7 +61,9 @@ def setup_git_hooks(env_bin: Path, is_windows: bool) -> Path: rprint("\n[cyan2]Setting git hooks[/cyan2] with `git config core.hooksPath .githooks`:") subprocess.run(("git", "config", "core.hooksPath", ".githooks"), cwd=REPO_ROOT_STR, check=True) binary = env_bin / ("harness.exe" if is_windows else "harness") - recorded = Path(run_git(["rev-parse", "--absolute-git-dir"]).strip()).resolve() / "harness-path" + recorded = ( + Path(run_git(["rev-parse", "--path-format=absolute", "--git-common-dir"]).strip()) / "harness-path" + ) recorded.write_text(f"{binary.as_posix()}\n", encoding="utf-8", newline="\n") typer.echo( subprocess.run( @@ -228,7 +230,7 @@ def cleanup(cwd: Path, name: str | None) -> bool: tool.setdefault("pyright", tomlkit.table()).update({"include": ["src", "preferences"]}) tool.setdefault("pytest", tomlkit.table()).setdefault("ini_options", tomlkit.table()).update({ "testpaths": ["tests"], - "pythonpath": ["src"], + "pythonpath": [".", "src"], }) coverage = tool.setdefault("coverage", tomlkit.table()) coverage.setdefault("run", tomlkit.table()).update({"source": ["src", "preferences"]}) diff --git a/harness/gate.py b/harness/gate.py index 968ae42..bdcaef4 100644 --- a/harness/gate.py +++ b/harness/gate.py @@ -25,6 +25,8 @@ except ImportError: # humans do what they want with preferences.py prefs = None +EMPTY_TREE = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" # universal empty tree hash + def run_git(args: list[str], repo: Path | None = None, check: bool = True) -> str: """Run a git command in the repo and return its stdout. @@ -199,13 +201,12 @@ def prepare_commit_msg(argv: list[str]) -> int: Returns: Status code integer 0 or 1 (git blocks commit on code 1) """ - if os.environ.get("RALPH_LOOP") != "1": + if not os.environ.get("RALPH_LOOP"): return 0 commit_msg_file: str = argv[1] if len(argv) > 1 else "" command = argv[2] if len(argv) > 2 else "" msg = "" - empty_tree = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" # universal empty tree hash - ref = "HEAD" if run_git(["rev-parse", "--verify", "HEAD"], check=False).strip() else empty_tree + ref = "HEAD" if run_git(["rev-parse", "--verify", "HEAD"], check=False).strip() else EMPTY_TREE if command in {"merge", "squash", "rebase", "reset", "clean", "filter-branch"}: msg = f"You cannot use that git command `{command}`.\n" if not run_git(["diff-index", "--cached", "--name-only", f"{ref}"]): diff --git a/harness/tests/test_cli.py b/harness/tests/test_cli.py index ad4d753..9d6f7fd 100644 --- a/harness/tests/test_cli.py +++ b/harness/tests/test_cli.py @@ -320,7 +320,7 @@ def test_installing_the_template_cleans_the_repo_sets_hooks_and_reruns_cleanly( assert document["tool"]["pytest"]["ini_options"] == { "addopts": ["-ra"], "testpaths": ["tests"], - "pythonpath": ["src"], + "pythonpath": [".", "src"], } assert document["tool"]["coverage"] == { "run": {"source": ["src", "preferences"]}, diff --git a/harness/tests/test_gate.py b/harness/tests/test_gate.py index 06eeb4f..5672220 100644 --- a/harness/tests/test_gate.py +++ b/harness/tests/test_gate.py @@ -164,9 +164,7 @@ def test_pre_push_hook_dispatches_gate_and_blocks_push(real_hook_repo: Path) -> @pytest.mark.parametrize("real_hook_repo", [("prepare-commit-msg",)], indirect=True) -def test_prepare_commit_msg_hook_rejects_empty_agent_then_accepts_staged_work( - real_hook_repo: Path, -) -> None: +def test_prepare_commit_msg_hook_rejects_empty_agent_then_accepts_staged_work(real_hook_repo: Path) -> None: """The hook rejects an empty agent commit, then accepts the agent's staged work.""" before = gate.run_git(["rev-parse", "HEAD"], real_hook_repo).strip() agent_empty = git_process( @@ -194,14 +192,7 @@ def test_prepare_commit_msg_hook_rejects_empty_agent_then_accepts_staged_work( def test_prepare_commit_msg_hook_allows_human_empty_commit(real_hook_repo: Path) -> None: """The hook does not apply agent containment to a human's empty commit.""" human_empty = git_process( - real_hook_repo, - "commit", - "--allow-empty", - "--no-verify", - "-q", - "-m", - "human empty", - loop=False, + real_hook_repo, "commit", "--allow-empty", "--no-verify", "-q", "-m", "human empty", loop=False ) assert human_empty.returncode == 0 @@ -377,14 +368,16 @@ def test_human_running_the_same_commands_is_not_policed( *( pytest.param(f"{directory}blocked.txt", id=f"dir-{directory}") for directory in gate.FORBIDDEN_DIRS + if directory != ".git/" ), *(pytest.param(path, id=f"file-{path}") for path in gate.FORBIDDEN_FILES), ], ) -def test_every_configured_forbidden_path_is_ejected( +def test_every_configured_forbidden_path_is_ejected_except_dot_git( forbidden_path: str, monkeypatch: pytest.MonkeyPatch, git_repo: Path ) -> None: - """Every forbidden directory and exact file configured in pyproject is removed from the index.""" + """Every forbidden directory and exact file configured in pyproject is removed from the index, except for + `.git` which never stages files (but is forbidden to be explicit to agents.)""" monkeypatch.setenv("RALPH_LOOP", "1") stage(git_repo, forbidden_path, "blocked\n") @@ -392,6 +385,8 @@ def test_every_configured_forbidden_path_is_ejected( assert gate.run_non_human_checks() == [] assert gate.run_git(["diff", "--cached", "--name-only"]).splitlines() == [] + assert ".git/" in gate.FORBIDDEN_DIRS + def test_every_configured_check_can_block_the_gate(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: """Each configured check takes its turn failing; all of them run and the failing one blocks.""" diff --git a/mutation/check_mutmut.py b/mutation/check_mutmut.py new file mode 100644 index 0000000..534d1ff --- /dev/null +++ b/mutation/check_mutmut.py @@ -0,0 +1,105 @@ +"""FAIL CI WHEN MUTMUT'S AGGREGATE EXPORT CONTAINS TEST GAPS. + +Mutation testing mutates the source code, e.g. a boolean flips, a constant shifts, a comparison widens. +A mutant that dies proves a test assertion was working on that line. A mutant that SURVIVES is a variant of +source code that shows tests execute (i.e. test coverage is met there) BUT the test is flimsy: +has weak assertions, overly-mock, happy path only, missing edge cases, no real assert in test, weak logic. + +i.e. Mutmut tests the tests. Mutmut breaks source then runs test suite. If test fails, the break was noticed, +a mutated source code variant is "killed." If every test passes after mutation the break went unnoticed, +mutant code variant "survived". + +Mutmut docs: https://mutmut.readthedocs.io/ + mutmut run # generate mutants and run the test suite against all source code + its mutants + mutmut browse # TUI over the survivors + mutmut results # plain-text summary + +Config in pyproject.toml [tool.mutmut] + +Each mutant variant has a name, e.g. tests.test_file.x_lazy_assert__mutmut_2. Mutmut records mutant as killed +or survived before you inspect, and re-uses variant until source code changes. + +CASE: +- best: 100% test coverage, run mutmut -> many mutations, none survive, nothing for you to kill +(tests are strongly sensitive to change) +- good: <100% coverage, run mutmut -> many mutations, none survive, you have to hunt some +(the tests that exist are good but you miss coverage) +- worst: 100% coverage, run mutmut -> many mutations, ALL survive + no easy kills +(there are many low quality tests) +- realistic: run mutmut, some mutants created, some survive, you find some to kill + +PROCESS: + +1. Run mutmut: mutations appear / are killed +2. Inspect: You look at surviving mutations +3. Test Update: You add or improve test assertions +4. Re-run: You ensure the mutant is killed + +5. Updating Source Code +- Dead Code +Sometimes a mutant survives because source code is redundant, e.g. If changing a line doesn't break a test, +ask if that code is needed. Maybe delete the useless code instead of writing tests. +- Surviving Mutants +You do not need to reach zero mutants, e.g. if changing source effects performance negatively, message output +would change, equivalent code swap e.g. i < 10 => i != 10 + +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import typer +from rich import print as rprint +from rich.console import Console +from rich.table import Table + +console = Console(force_terminal=True) + +JsonDocument = dict[str, object] | list[object] | str | int | float | bool | None + + +def analyze_mutmut_report(file_path: str = "mutants/mutmut-cicd-stats.json") -> float: + """Read the mutmut CI JSON results created each Sunday night. + + Arguments: + file_path (str): Default filepath to read mutmut run stats from. + + Returns: + mutation_score: float value, killed + timeout mutants as a% of all + + Raises: + JSONDecodeError: If the report does not contain valid JSON. + Exit: If the report file does not exist. + """ + if not Path(file_path).exists(): + rprint(rf"[red]Error: Mutmut JSON report not found at [\]'{file_path}'") + raise typer.Exit(code=1) + data: dict[str, int] = {} + with Path(file_path).open("r", encoding="utf-8") as fp: + try: + data = json.load(fp) + except json.JSONDecodeError: + rprint(rf"[red]JSONDecodeError [\]'{file_path}'") + raise + + mutation_score: float = 0.0 + total_mutants = data.get("total", 0) + skipped = data.get("skipped", 0) + tested_mutants = total_mutants - skipped + if tested_mutants > 0: + killed = data.get("killed", 0) + timeout = data.get("timeout", 0) + mutation_score = ((killed + timeout) / tested_mutants) * 100 + + table = Table(title="\n[cyan2]MUTMUT MUTATION RESULTS[/]\n", box=None, padding=(0, 2)) + for stat, result in data.items(): + table.add_row(f"[turquoise] {stat}[/]", f"[blue] {result}[/]") + table.add_row(f"[bold italic turquoise] MUTATION SCORE: [/][bold italic blue]{mutation_score}[/]") + console.print(table, justify="center") + return mutation_score + + +if __name__ == "__main__": + analyze_mutmut_report() diff --git a/pyproject.toml b/pyproject.toml index 3e99b42..27d863e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,7 +116,7 @@ pytest = [ ] [tool.harness.FORBIDDEN] -DIRS = ["harness/", ".githooks/", ".github/", "preferences/", "tests/preferences/"] +DIRS = ["harness/", ".githooks/", ".github/", "preferences/", "tests/preferences/", ".git/"] FILES = [ "agents.md", "pyproject.toml", @@ -168,7 +168,8 @@ PATTERNS = [ "fmt: off", "fmt: skip", "yapf: disable", - "complexipy: ignore" + "complexipy: ignore", + "pragma: no mutate" ] # ============================================================================== @@ -180,7 +181,7 @@ reportAssertAlwaysTrue = "error" # Catch asserts that are always true. reportUnusedImport = "none" # Handled by Ruff F401 reportUnusedVariable = "none" # Handled by Ruff F841 verboseOutput = true # JSON report output when Pyright is run -include = ["src", "preferences", "harness"] # Type-check shipped code plus repository preferences. +include = ["src", "preferences", "mutation", "harness"] # Type-check shipped code plus repository preferences. # ============================================================================== # Pytest Configuration @@ -189,23 +190,35 @@ include = ["src", "preferences", "harness"] # Type-check shipped code plus repos minversion = "9.0" addopts = ["-ra"] # Print useful short reasons for outcomes testpaths = ["tests", "harness"] -pythonpath = ["src", "harness"] # Lets tests import Python code located in the named folders +pythonpath = ["."] # Repo root: tests import root packages by full dotted path # ============================================================================== # Test Coverage Configuration # ============================================================================== [tool.coverage] -run.source = ["src", "preferences", "harness"] # ADD EVERY DIRECTORY THAT SHIPS CODE TO ENSURE 100% COVERAGE +run.source = ["src", "preferences", "mutation", "harness"] # ADD EVERY DIRECTORY THAT SHIPS CODE TO ENSURE 100% COVERAGE report.show_missing = true # Show exact missed lines when coverage fails report.skip_covered = false # Keep fully covered files visible in reports report.fail_under = 100 # Block merges unless all shipped code is covered skip_covered = false # Same behavior for older coverage config readers +# ============================================================================== +# Mutmut (Mutation Testing) Configuration +# ============================================================================== +[tool.mutmut] +max-children = 2 +source_paths = ["preferences/preferences.py", "harness/gate.py"] # what to mutate +also_copy = ["harness", ".githooks"] # not mutated paths, copied for imports +do_not_mutate = ["harness/tests/*"] # mutmut can't tell what's a test file +pytest_add_cli_args_test_selection = [ + "harness/tests/test_properties.py", "harness/tests/test_gate.py", +] + # ============================================================================== # Complexipy Configuration # ============================================================================== [tool.complexipy] -paths = ["src", "preferences", "harness"] +paths = ["src", "preferences", "mutation", "harness"] exclude = ["**/tests/**"] # Test code can be more branching and example-heavy than production code max-complexity-allowed = 10 # Cognitive complexity cap catches deeply nested, hard-to-read flow no-ignore = true # Disallow `# complexipy: ignore`, fix complexity instead of suppressing it @@ -230,7 +243,9 @@ main.ignore = [ "tests", "**/tests/**", "harness/tests", - "worktrees" + ".worktrees", + "worktrees", + "mutants" ] format.max-line-length = 110 # Keep Pylint line accounting aligned with Ruff E501 format.max-module-lines = 500 # Enable Pylint C0302 because Ruff has no equivalent diff --git a/tests/mutation/mutmut-cicd-stats.json b/tests/mutation/mutmut-cicd-stats.json new file mode 100644 index 0000000..7f7aad2 --- /dev/null +++ b/tests/mutation/mutmut-cicd-stats.json @@ -0,0 +1,11 @@ +{ + "killed": 132, + "survived": 0, + "total": 133, + "no_tests": 0, + "skipped": 0, + "suspicious": 0, + "timeout": 1, + "check_was_interrupted_by_user": 0, + "segfault": 0 +} diff --git a/tests/mutation/test_check_mutmut.py b/tests/mutation/test_check_mutmut.py new file mode 100644 index 0000000..9b1b79a --- /dev/null +++ b/tests/mutation/test_check_mutmut.py @@ -0,0 +1,61 @@ +"""Tests for the mutmut CI report checker.""" + +from __future__ import annotations + +import contextlib +import json +import runpy +from pathlib import Path + +import pytest +import typer +from click import unstyle + +from mutation.check_mutmut import analyze_mutmut_report + + +def test_report_with_timeout_passes_and_renders(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + """A timeout counts as detected and remains visible in the report.""" + report = tmp_path / "mutants" / "mutmut-cicd-stats.json" + report.parent.mkdir() + report.write_text( + Path(__file__).with_name("mutmut-cicd-stats.json").read_text(encoding="utf-8"), encoding="utf-8" + ) + + checker = Path(__file__).parents[2] / "mutation" / "check_mutmut.py" + with contextlib.chdir(tmp_path): + runpy.run_path(str(checker), run_name="__main__") + + output = " ".join(unstyle(capsys.readouterr().out).split()) + assert "MUTMUT MUTATION RESULTS" in output + assert "timeout 1" in output + assert "MUTATION SCORE: 100.0" in output + + +def test_actionable_result_fails(tmp_path: Path) -> None: + """A survived mutant is included in the report analysis.""" + data = json.loads(Path(__file__).with_name("mutmut-cicd-stats.json").read_text(encoding="utf-8")) + data["survived"] = 1 + data["total"] += 1 + report = tmp_path / "mutmut-cicd-stats.json" + report.write_text(json.dumps(data), encoding="utf-8") + mutation_score = analyze_mutmut_report(report) + assert mutation_score >= 60.0 + + +def test_missing_report_fails(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + """A missing export errors CI.""" + with pytest.raises(typer.Exit) as exc_info: + analyze_mutmut_report(tmp_path / "missing.json") + + assert exc_info.value.exit_code == 1 + assert "Error: Mutmut JSON report not found" in capsys.readouterr().out + + +def test_malformed_report_fails(tmp_path: Path) -> None: + """Malformed JSON is reported as malformed JSON.""" + report = tmp_path / "mutmut-cicd-stats.json" + report.write_text("{", encoding="utf-8") + + with pytest.raises(json.JSONDecodeError): + analyze_mutmut_report(report) diff --git a/tests/preferences/test_preferences.py b/tests/preferences/test_preferences.py index 0d8e6be..1121f07 100644 --- a/tests/preferences/test_preferences.py +++ b/tests/preferences/test_preferences.py @@ -10,9 +10,12 @@ import ast import inspect +import keyword +import string from unittest.mock import Mock import pytest +from hypothesis import assume, given, strategies preferences = pytest.importorskip("preferences.preferences") preferences_violations = preferences.preferences_violations @@ -355,3 +358,127 @@ def test_syntax_error_raises() -> None: """Unparseable source raises SyntaxError; preferences does not swallow it.""" with pytest.raises(SyntaxError): preferences_violations("m.py", "def broken(:\n") + + +# --------------------------------------------------------------------- generated behavior + + +IDENTIFIER_START = string.ascii_letters + "_" +IDENTIFIER_REST = IDENTIFIER_START + string.digits + + +@strategies.composite +def identifiers(draw: strategies.DrawFn) -> str: + """Draw a valid ASCII Python identifier that is not a keyword.""" + first = draw(strategies.sampled_from(IDENTIFIER_START)) + rest = draw(strategies.text(alphabet=IDENTIFIER_REST, max_size=24)) + name = first + rest + assume(not keyword.iskeyword(name)) + return name + + +def flags(source: str, needle: str) -> bool: + """Return whether the preferences output contains the expected text.""" + checker = preferences_violations + return needle in checker("m.py", source) if checker else False + + +@given(name=identifiers()) +def test_underscore_lead_flagged_iff_leading_underscore_not_dunder(name: str) -> None: + """Assignment targets are flagged exactly when they use a non-dunder leading underscore.""" + expected = name.startswith("_") and not name.endswith("__") + assert flags(f"{name} = 1\n", "starts with underscore") is expected + + +@given(name=identifiers()) +def test_underscore_rule_holds_for_function_and_argument_names(name: str) -> None: + """The underscore rule applies equally to function and argument names.""" + assume(not name.endswith("__")) + expected = name.startswith("_") + assert flags(f"def {name}():\n return 1\n", "starts with underscore") is expected + assert flags(f"def f({name}):\n return {name}\n", "starts with underscore") is expected + + +@strategies.composite +def class_source(draw: strategies.DrawFn) -> tuple[str, bool]: + """Draw a class and whether the pointless-class rule should flag it.""" + has_base = draw(strategies.booleans()) + has_decorator = draw(strategies.booleans()) + has_keyword = draw(strategies.booleans()) + method_count = draw(strategies.integers(min_value=0, max_value=3)) + + decorator = "@deco\n" if has_decorator else "" + header_bits = (["Base"] if has_base else []) + (["metaclass=type"] if has_keyword else []) + header = f"({', '.join(header_bits)})" if header_bits else "" + body = "".join(f" def m{i}(self):\n return {i}\n" for i in range(method_count)) or " x = 1\n" + source = f"{decorator}class C{header}:\n{body}" + should_flag = not has_base and not has_decorator and not has_keyword and method_count <= 1 + return source, should_flag + + +@given(case=class_source()) +def test_pointless_class_flagged_iff_plain_and_at_most_one_method(case: tuple[str, bool]) -> None: + """Only plain classes with at most one method are pointless.""" + source, should_flag = case + assert flags(source, "no base, decorator, or behavior") is should_flag + + +@strategies.composite +def comprehension_source(draw: strategies.DrawFn) -> tuple[str, bool]: + """Draw a comprehension and whether its generators make it too complex.""" + generator_count = draw(strategies.integers(min_value=1, max_value=3)) + if_on = draw(strategies.integers(min_value=-1, max_value=generator_count - 1)) + + clauses: list[str] = [] + for index in range(generator_count): + clause = f"for v{index} in xs{index}" + if index == if_on: + clause += f" if v{index}" + clauses.append(clause) + source = f"[v0 {' '.join(clauses)}]\n" + return source, generator_count > 1 and if_on != -1 + + +@given(case=comprehension_source()) +def test_complex_comprehension_flagged_iff_multi_generator_with_filter(case: tuple[str, bool]) -> None: + """Comprehensions are complex only with multiple generators and a filter.""" + source, should_flag = case + assert flags(source, "Overly complex comprehension") is should_flag + + +@strategies.composite +def nested_continue_source(draw: strategies.DrawFn) -> str: + """Draw a continue nested beneath an outer loop and two to four more blocks.""" + depth = draw(strategies.integers(min_value=2, max_value=4)) + blocks = draw( + strategies.lists(strategies.sampled_from(["if cond", "for i in xs"]), min_size=depth, max_size=depth) + ) + + lines = ["for outer in items:"] + indent = " " + for block in blocks: + lines.append(f"{indent}{block}:") + indent += " " + lines.append(f"{indent}continue") + return "\n".join(lines) + "\n" + + +@given(source=nested_continue_source()) +def test_continue_nested_under_stacked_blocks_is_flagged(source: str) -> None: + """Mixed nested if/for blocks always trigger the continue nesting rule.""" + assert flags(source, "Overly-nested 'continue'") + + +def test_single_if_guard_in_a_loop_is_not_flagged() -> None: + """A single if guard directly inside a loop remains readable.""" + assert not flags("for i in items:\n if skip:\n continue\n", "Overly-nested 'continue'") + + +def test_shallow_continue_in_single_loop_is_not_flagged() -> None: + """A continue directly inside one for loop is allowed.""" + assert not flags("for x in items:\n continue\n", "Overly-nested 'continue'") + + +def test_continue_in_while_loop_is_flagged() -> None: + """A continue inside a while loop is banned as a freeze risk.""" + assert flags("while cond:\n continue\n", "while loop banned") diff --git a/tests/preferences/test_preferences_properties.py b/tests/preferences/test_preferences_properties.py deleted file mode 100644 index 3ca4705..0000000 --- a/tests/preferences/test_preferences_properties.py +++ /dev/null @@ -1,247 +0,0 @@ -"""Property-based tests for preferences.preferences.py using Hypothesis. - -"With Hypothesis, you write tests which should pass for all inputs in whatever range you describe, and let -Hypothesis randomly choose which of those inputs to check, including edge cases you might not have thought -about." TESTS THE CODE WITH A RANGE OF INPUTS. -Hypothesis docs: https://hypothesis.readthedocs.io/ - -Tests that preferences.py checks for names, classes, comprehensions, and continue - -Hypothesis persistence: Do not set database=None by default. Local runs use Hypothesis's example -database under .hypothesis/examples, so past failures are replayed first and users can debug them -quickly. CI automatically uses Hypothesis's built-in `ci` profile, which is stateless and deterministic. -If a generated input is important, save it as @example(...) or a normal regression test instead of -relying on the local database. The generated .hypothesis/ directory is gitignored. - -Test hygiene: keep strategies at module scope. Set max_examples only when a test needs a runtime cap. Do -not use function-scoped fixtures with @given; patch per-example state inside helper functions instead. -""" - -from __future__ import annotations - -import importlib -import keyword -import string -import sys -from collections.abc import Callable -from unittest import mock - -from hypothesis import assume, given, strategies - -# preferences.py are optional AST-style checks. Repo-owner can keep or delete. -preferences_violations: Callable[[str, str], str] | None -try: - from preferences.preferences import preferences_violations -except ImportError: - preferences_violations = None - -# Patterns in a deterministic order so hypothesis shrinks toward the first entry predictably. -IDENTIFIER_START = string.ascii_letters + "_" -IDENTIFIER_REST = IDENTIFIER_START + string.digits - - -@strategies.composite -def identifiers(draw: strategies.DrawFn) -> str: - """Draw a valid ASCII Python identifier that is not a keyword.""" - first = draw(strategies.sampled_from(IDENTIFIER_START)) - rest = draw(strategies.text(alphabet=IDENTIFIER_REST, max_size=24)) - name = first + rest - assume(not keyword.iskeyword(name)) - return name - - -IDENTIFIERS = identifiers() - - -def flags(source: str, needle: str) -> bool: - """Whether preferences_violations reports a message containing needle for source.""" - return needle in preferences_violations("m.py", source) if preferences_violations else False - - -# ------------------------------------------------------- preferences.py-absent fallback (optional module) - - -def test_module_tolerates_absent_preferences_on_import() -> None: - """When preferences.preferences.py can't be imported (e.g. human deleted), this module still loads and its - preferences_violations is None, so its gate tests here keep running. mock.patch.dict maps the module - name to None (the standard way to make `import` raise ImportError) and auto-restores it; reloading - this module under that patch exercises the ImportError fallback, then a final reload restores it. - """ - module = sys.modules[__name__] - try: - with mock.patch.dict(sys.modules, {"preferences.preferences": None}): - reloaded = importlib.reload(module) - assert reloaded.preferences_violations is None - finally: - importlib.reload(module) # restore the real preferences_violations for the rest of the suite - - -def test_flags_never_calls_preferences_when_it_is_absent() -> None: - """Deleting preferences.py must not crash flags(). It reports no violation instead. - - A variable named with a leading underscore breaks the underscore rule. The first assert confirms - the rule fires on it. The second feeds in that same variable with the module gone and gets no - violation back, which is only possible if preferences_violations was never called. - """ - source, rule = "_bad = 1\n", "starts with underscore" - assert flags(source, rule) is True - with mock.patch.object(sys.modules[__name__], "preferences_violations", None): - assert flags(source, rule) is False - - -# --------------------------------------------------------------------- underscore-lead identifier rule - - -@given(name=IDENTIFIERS) -def test_underscore_lead_flagged_iff_leading_underscore_not_dunder(name: str) -> None: - """An assignment target trips the underscore rule IFF it starts with '_' and does not end with '__'. - Covers the whole identifier domain, including dunders and lone '_', in one property. - """ - expected = name.startswith("_") and not name.endswith("__") - assert flags(f"{name} = 1\n", "starts with underscore") is expected - - -@given(name=IDENTIFIERS) -def test_underscore_rule_holds_for_function_and_argument_names(name: str) -> None: - """The same underscore rule applies to function names and argument names, not just assignments.""" - assume(not name.endswith("__")) # keep dunder methods/args (__init__ etc.) out of this slice - expected = name.startswith("_") - assert flags(f"def {name}():\n return 1\n", "starts with underscore") is expected - assert flags(f"def f({name}):\n return {name}\n", "starts with underscore") is expected - - -# ------------------------------------------------------------------------------- pointless-class rule - - -@strategies.composite -def class_source(draw: strategies.DrawFn) -> tuple[str, bool]: - """Draw a class definition varying base/decorator/keyword presence and method count. - - Args: - draw: Hypothesis draw callable. - - Returns: - (source, should_flag) where should_flag is the documented intent: trip IFF the class has no - base, no decorator, no keyword, and at most one method. - """ - has_base = draw(strategies.booleans()) - has_decorator = draw(strategies.booleans()) - has_keyword = draw(strategies.booleans()) - method_count = draw(strategies.integers(min_value=0, max_value=3)) - - decorator = "@deco\n" if has_decorator else "" - header_bits = (["Base"] if has_base else []) + (["metaclass=type"] if has_keyword else []) - header = f"({', '.join(header_bits)})" if header_bits else "" - body = "".join(f" def m{i}(self):\n return {i}\n" for i in range(method_count)) or " x = 1\n" - source = f"{decorator}class C{header}:\n{body}" - - should_flag = not has_base and not has_decorator and not has_keyword and method_count <= 1 - return source, should_flag - - -@given(case=class_source()) -def test_pointless_class_flagged_iff_plain_and_at_most_one_method(case: tuple[str, bool]) -> None: - """A class trips the pointless-class rule IFF it is plain (no base/decorator/keyword) with <= 1 - method. Any base, decorator, keyword, or a second method exempts it. - """ - source, should_flag = case - assert flags(source, "no base, decorator, or behavior") is should_flag - - -# ------------------------------------------------------------------------ complex-comprehension rule - - -@strategies.composite -def comprehension_source(draw: strategies.DrawFn) -> tuple[str, bool]: - """Draw a list comprehension with a chosen generator count and which generator (if any) filters. - - Args: - draw: Hypothesis draw callable. - - Returns: - (source, should_flag) where should_flag is the documented intent: trip IFF there is more than - one generator AND at least one generator carries an `if`. Crucially the filtered generator may - be a LATER one, exercising the check's early-return-on-first-match loop. - """ - generator_count = draw(strategies.integers(min_value=1, max_value=3)) - # -1 means "no if on any generator"; otherwise the index of the single generator that filters. - if_on = draw(strategies.integers(min_value=-1, max_value=generator_count - 1)) - - clauses: list[str] = [] - for index in range(generator_count): - clause = f"for v{index} in xs{index}" - if index == if_on: - clause += f" if v{index}" - clauses.append(clause) - source = f"[v0 {' '.join(clauses)}]\n" - - should_flag = generator_count > 1 and if_on != -1 - return source, should_flag - - -@given(case=comprehension_source()) -def test_complex_comprehension_flagged_iff_multi_generator_with_filter(case: tuple[str, bool]) -> None: - """A comprehension trips IFF it has multiple generators AND at least one has an `if` -- regardless of - WHICH generator carries the `if`. The later-generator case guards the check's early return, which - scans generators in order and returns on the first one that filters. - """ - source, should_flag = case - assert flags(source, "Overly complex comprehension") is should_flag - - -# --------------------------------------------------------------------------- chaotic-continue rule - - -@strategies.composite -def nested_continue_source(draw: strategies.DrawFn) -> str: - """Draw a `continue` wrapped in an outer `for` plus TWO-to-four more if/for blocks. - - The rule allows one `if` guard directly inside a loop (`for: if: continue`), so to always be - over-nested we stack at least two blocks below the outer loop. - - Args: - draw: Hypothesis draw callable. - - Returns: - Source whose `continue` sits at least two if/for blocks below its enclosing loop, so the - over-nesting rule always flags it. - """ - depth = draw(strategies.integers(min_value=2, max_value=4)) - blocks = draw( - strategies.lists(strategies.sampled_from(["if cond", "for i in xs"]), min_size=depth, max_size=depth) - ) - - lines = ["for outer in items:"] # an outer loop the continue always belongs to - indent = " " - for block in blocks: - lines.append(f"{indent}{block}:") - indent += " " - lines.append(f"{indent}continue") - return "\n".join(lines) + "\n" - - -@given(source=nested_continue_source()) -def test_continue_nested_under_stacked_blocks_is_flagged(source: str) -> None: - """A `continue` stacked two or more if/for blocks below its enclosing loop is flagged as overly - nested, whatever mix of if/for those blocks are. - """ - assert flags(source, "Overly-nested 'continue'") - - -def test_single_if_guard_in_a_loop_is_not_flagged() -> None: - """The common, readable `for ...: if ...: continue` (one if guard in one loop) is NOT over-nested.""" - assert not flags("for i in items:\n if skip:\n continue\n", "Overly-nested 'continue'") - - -def test_shallow_continue_in_single_loop_is_not_flagged() -> None: - """Control (example, not property): a `continue` directly in one `for` -- parent For, grandparent - module -- is not overly nested, so it is not flagged. - """ - assert not flags("for x in items:\n continue\n", "Overly-nested 'continue'") - - -def test_continue_in_while_loop_is_flagged() -> None: - """Control (example): a `continue` anywhere inside a while loop is flagged (freeze risk), a separate - branch from the nested-if detection. - """ - assert flags("while cond:\n continue\n", "while loop banned") From 71dfbe0eb8afdc054b60f9079c76c14989eeb00b Mon Sep 17 00:00:00 2001 From: Roxana del Toro Date: Mon, 3 Aug 2026 02:00:14 -0700 Subject: [PATCH 5/5] Update _resolve path to executable --- .githooks/_resolve | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.githooks/_resolve b/.githooks/_resolve index f258b88..171414d 100644 --- a/.githooks/_resolve +++ b/.githooks/_resolve @@ -21,3 +21,5 @@ if [ ! -x "$HARNESS" ]; then echo "loopgate: recorded harness '$HARNESS' is gone. Re-run 'harness install'." >&2 exit 1 fi + +export PATH="$(dirname "$HARNESS")${PATH:+:$PATH}"