diff --git a/.githooks/_resolve b/.githooks/_resolve
index db6272d..171414d 100644
--- a/.githooks/_resolve
+++ b/.githooks/_resolve
@@ -10,7 +10,7 @@
#
# Then, git hook does not require `uv` or `pip` or any specific environment layout.
-recorded="${GIT_DIR:-$(git rev-parse --absolute-git-dir)}/harness-path"
+recorded="$(git rev-parse --path-format=absolute --git-common-dir)/harness-path"
if [ ! -r "$recorded" ]; then
echo "loopgate: hooks are not installed. Run 'harness install' in this repo." >&2
exit 1
@@ -21,3 +21,5 @@ if [ ! -x "$HARNESS" ]; then
echo "loopgate: recorded harness '$HARNESS' is gone. Re-run 'harness install'." >&2
exit 1
fi
+
+export PATH="$(dirname "$HARNESS")${PATH:+:$PATH}"
diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml
new file mode 100644
index 0000000..a393641
--- /dev/null
+++ b/.github/workflows/mutation.yml
@@ -0,0 +1,50 @@
+name: Mutation Testing
+
+on:
+ schedule:
+ - cron: "0 0 * * 0" # Weekly at midnight Sunday
+ workflow_dispatch: # adds "Run workflow" button
+
+env:
+ FORCE_COLOR: "1"
+ CLICOLOR_FORCE: "1"
+ TERM: "xterm-256color"
+
+permissions:
+ contents: read
+
+concurrency:
+ group: mutation-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ mutate:
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+
+ steps:
+ - name: Checkout Code
+ uses: actions/checkout@v6
+
+ - name: Install uv
+ uses: astral-sh/setup-uv@v6
+ with:
+ enable-cache: true
+
+ - name: uv Sync Dependencies
+ run: uv sync
+
+ - name: Harness Checks
+ run: uv run --no-sync harness gate # need tests for Mutmut to target
+
+ - name: Run Mutmut
+ run: mutmut run || true
+
+ - name: Export CI/CD Stats
+ run: mutmut export-cicd-stats
+
+ - name: Upload Mutmut JSON Report
+ uses: actions/upload-artifact@v4
+ with:
+ name: mutmut-json-report
+ path: mutants/mutmut-cicd-stats.json
diff --git a/.gitignore b/.gitignore
index 4065523..16e3312 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,5 +27,8 @@ scratchpad/*
scratchpad/runs/*
!scratchpad/runs/.gitkeep
+# mutmut's mutated copies of the source. Ignoring them is also what keeps ruff and semgrep out.
+mutants/
+
# kept at the last line for easy deletion (real projects need lockfile)
uv.lock
diff --git a/README.md b/README.md
index 8a3a63e..01100af 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,8 @@
L∞pGate
-A coding-agent loop harness for Claude, Codex, Copilot, or any CLI agent. A dumb Ralph loop runner tells an agent to "Go!" and hands it a PROMPT. Agents can edit. Gates decide what lands. You set the plan in motion. The loops eat the prompt, and each agent iteration must update specs and commit through guardrails.
+Run coding agents strictly andonly accept changes that pass your quality gates.
+A coding-agent loop harness for Claude, Codex, Copilot, or any CLI agent. A dumb Ralph loop runner tells an agent to "Go!" and hands it a PROMPT. Agents can edit. Gates decide what lands. You set the plan in motion. The loops eat the prompt, and each agent iteration must update specs and commit through guardrails.
@@ -21,13 +22,12 @@
---
-## TL;DR: Getting Started.
+## TL;DR
-1. `gh repo create my-app-name --template /loopgate_harness --private --clone` **or** ['Use This Template'](https://github.com/new?template_name=loopgate_harness&template_owner=rxdt)
-2. Install dependancies e.g. `uv sync && source .venv/bin/activate && harness install [max_iterations] [max_minutes]`
-5. Not what you wanted? Refine [`docs/plan.md`](docs/plan.md) / [`docs/PROMPT.md`](docs/PROMPT.md) and re-run
+1. `gh repo create / --template rxdt/loopgate_harness --private --clone && cd && uv run harness install && source .venv/bin/activate`
+2. `harness run codex`
+
+**Requirements**: `pip`, `uv`, or `poetry`. Python 3.11.
---
@@ -37,10 +37,10 @@
- **Worker-agnostic**: Claude, Codex, Copilot, Agy, or any prompt-reading CLI
- **No lazy**: Agents work, _only if they pass the quality gates you set_ ✅
- **Repo-as-memory workflow**: specs/status/prompt are durable but code is king, leaving you free 😎
-- **Built-in stack**: Ruff, Pyright, Pylint, Semgrep, Complexipy, Hypothesis, 100% coverage ☑☑☑
+- **Built-in stack**: Ruff, Pyright, Pylint, Semgrep, Complexipy, Hypothesis, Mutmut, 100% coverage ☑☑☑
- **Progressive**: Preflight vs full gate split 🆗
- **Forbidden-path containment**: Don't touch that!-configurable 🛑
-- **Installable project template**: `harness install loopgate` gets the repo ready ▶️
+- **Installable project template**: `harness install ` gets the repo ready ▶️
- **No-rot**: Fresh-context agent iterations to reduce context rot 🔄
- **Simple**: One command setup gets you git hooks and everything else
- **No-waste**: Timeouts and time-limits for all loops ⏸
@@ -56,20 +56,27 @@
`docs/PROMPT.md` tells each agent to pick a `spec` and build. `docs/specs/` say _what_ to build. The agent decides _what next_. You keep `docs/plan.md` current, and specs get rewritten from it (agent is told in `docs/PROMPT.md` to update the specs). Each iteration the agent updates its spec and `PROJECT_STATUS`. Ideas from [ghuntley](https://github.com/ghuntley), How to Ralph Wiggum.
> [!TIP]
-> If you don't like _ANYTHING_ in this framework, remove it.
-
-## Start a project
-
-1. From the root, run `harness install ` to name the project, install dependencies, set up the three git hooks, and delete excess files.
-2. Write your grand vision into `docs/plan.md`.
-3. Optionally add the first spec in `docs/specs/`, or have an agent draft the first specs.
-4. Put product code under `src/` and list new source directories in `pyproject.toml [tool.coverage.run]`.
-5. Strict Ruff rules, type checking, pyright, complexipy, and pytest coverage are set in `pyproject.toml`.
-6. Your coding quirks go in [`preferences/preferences.py`](preferences/preferences.py).
-7. Run a loop:
+> If you don't like _ANYTHING_ in this framework, update it.
+
+### Start a project
+
+1. `gh repo create my-app-name --template /loopgate_harness --private --clone` **or**
+ ['Use This Template'](https://github.com/new?template_name=loopgate_harness&template_owner=rxdt)
+2. Source your environment (if applicable)
+3. From the root, run `harness install ` to name the project, install dependencies, set up the git hooks, and delete excess files. Install dependencies e.g. `uv sync && source .venv/bin/activate && harness install [max_iterations] [max_minutes]`
+9. Not what you wanted? Refine [`docs/plan.md`](docs/plan.md) / [`docs/PROMPT.md`](docs/PROMPT.md) and re-run
+10. Strict Ruff rules, type-checking Pyright, Complexipy, and Pytest coverage are set in [`pyproject.toml`](pyproject.toml).
+11. Your coding quirks go in [`preferences/preferences.py`](preferences/preferences.py).
+12. Loop!:
```sh
-harness run [max_iterations] [max_minutes] # agent: claude/codex/agy/copilot. ralph loop runner adds prompt
+harness run [max_iterations] [max_minutes] # agent: claude/codex/agy/copilot. ralph loop runner injects prompt
```
### Works with `uv`, `poetry`, or `pip`
@@ -79,12 +86,12 @@ uv sync
source .venv/bin/activate
harness install
harness gate
-harness run >
+harness run
poetry install
poetry run harness install
poetry run harness gate
-poetry run harness
+poetry run harness run
python -m venv .venv
source .venv/bin/activate
@@ -96,7 +103,7 @@ harness run
-## A L∞PS Loop
+## A L∞Pgate Loop
The repo is the only memory. Each iteration is a fresh-context agent.
@@ -112,25 +119,6 @@ The repo is the only memory. Each iteration is a fresh-context agent.
-## Safety
-
-`harness run` launches an autonomous LLM worker with the configured permissions, e.g.
-`--permission-mode acceptEdits` or `--sandbox danger-full-access`.
-
-The gate bounds what any **commit** may touch, but the worker itself is **not** sandboxed to this repo unless you set that config. Consider the balance: without access it cannot do much. With machine access it can wreak havoc. Under a permissive mode it can run arbitrary shell. You are authorizing real changes. Choose the worker and permission mode deliberately.
-
-#### The Gate: Tiered Checks
-
-⚡ `harness preflight` (pre-commit) → fast checks.
-Ruff lint + check format for everyone, _plus_ **containment** for the agents. Self-heals by un-staging forbidden files.
-
-✅ `harness gate` (CI/PR pre-push). Local checks mirror CI → ruff lint + format report-only, pyright, pylint, semgrep, complexipy, hypothesis, pytest @ 100% cov.
-
-Only humans can bypass triggered gates and commit by adding flag `--no-verify`.
-
-
-
-
## Directory Layout
@@ -173,22 +161,6 @@ patterns = ["# noqa"] # banned in agent-authored diffs
pytest = "uv sync pytest" # one check command, run by the local gate AND CI
```
-## Read this before a first run.
-
-1. **This harness does not sandbox agents.** It _tries_ to harness bad code in loops via gates. Sandboxing agents will, e.g. prevent them from maintaining git, running Playwright, being seen as trustworthy by semgrep leading to cyclical failures, etc.
-
-2. **The gate is a guardrail, not a jail.** Agents are crafty, like people. They will find a way to complete a task at all costs. **Trust nothing and no one.**
-
-3. **Mind your usage limits.** `harness run` works agents to the cap set. You can easily burn through your tokens, context windows, and provider usage limits. **Workers continue running as long as there is work to do.**
-
-4. **`docs/PROMPT.md` tells the worker to push or not**.
-
-5. Protect `main` and run the loop on its own branch.
-
-6. **100% coverage does not mean good tests.** That is quantity, not quality. (Upcoming feature: mutation testing)
-
-7. **Note**: `semgrep --config auto` needs network for semgrep registry rules.
-
## Commands
Tool commands are defined once, in `[tool.harness.gate.checks]` in [pyproject.toml](pyproject.toml). The local gate and CI both derive them from there.
@@ -215,19 +187,20 @@ To run LoopGate with Claude :
harness run claude 2 20
```
-Note: The worker must be installed and authenticated separately.
+Note: The worker must be installed and authenticated separately. **Claude Code exports env vars into every shell it spawns. `RALPH_LOOP=1` can be set globally.**
## Expanding your harness
-- Edit rules at [pyproject.toml](pyproject.toml) for [ruff](https://docs.astral.sh/ruff/), [pylint](https://pypi.org/project/pylint/), [pydoclint](https://pypi.org/project/pydoclint/0.9.1/), [pyright](https://github.com/microsoft/pyright), [pytest](https://docs.pytest.org/en/stable/), [hypothesis](https://hypothesis.readthedocs.io/), [complexipy](https://github.com/rohaquinlop/complexipy)
+- Edit rules at [pyproject.toml](pyproject.toml) for [ruff](https://docs.astral.sh/ruff/), [pylint](https://pypi.org/project/pylint/), [pydoclint](https://pypi.org/project/pydoclint/0.9.1/), [pyright](https://github.com/microsoft/pyright), [pytest](https://docs.pytest.org/en/stable/), [hypothesis](https://hypothesis.readthedocs.io/), [complexipy](https://github.com/rohaquinlop/complexipy), [mutmut](https://mutmut.readthedocs.io/)
- Add forbidden files, directories, or patterns in `[tool.harness.gate]` at [pyproject.toml](pyproject.toml)
- Add [Hypothesis](https://hypothesis.readthedocs.io/) tests in any test directory, examples at [test_properties.py](tests/preferences/test_properties.py).
-- [semgrep](https://docs.semgrep.dev/semgrep-ci/sample-ci-configs) has no repo config here. It uses registry configs plus Semgrep's built-in defaults which ignore tests.
-- Edit `[tool.harness.gate.checks]` in [pyproject.toml](pyproject.toml). [ci.yml](.github/workflows/ci.yml) runs the same `harness gate`.
-- Remove existing preferences or add your own at [preferences.py](preferences/preferences.py). Current preferences:
+- Run [mutmut](https://mutmut.readthedocs.io/) by hand with `uv run mutmut run`, then `uv run mutmut browse`. A surviving mutant is a covered line no assertion checks. It is not a gate check: `mutmut run` exits 0 even with survivors. Its verdicts are cached against source hashes and ignore test edits, so re-run a single mutant by name once you have written a test for it. Examples at [test_with_mutations.py](tests/preferences/test_with_mutations.py).
+- [semgrep](https://docs.semgrep.dev/semgrep-ci/sample-ci-configs) has no repo config here. It uses registry configs / Semgrep's built-in defaults which ignore tests.
+- Update `[tool.harness.gate.checks]` in [pyproject.toml](pyproject.toml). [ci.yml](.github/workflows/ci.yml) runs those **same exact** `harness gate` checks.
+- Add or remove coding preferences [preferences.py](preferences/preferences.py) that only agents in loops **must** respect. Current preferences:
```py
function_argument_assignment_has_star # agents use non-specific `def fun(*)`
@@ -237,7 +210,7 @@ dynamic_star_call # Calls to def fun(*items) breaks when you can't tell how man
pointless_class # ensure classes are added for good reasons (carry state, values, methods)
lazy_assert # enforce real assertions, stronger tests
objects_injected_into_runtime_memory # finds calls that manipulate global state (dangerous, tricky)
-lambda_found # abolish lambdas for agents to keep their code simpler
+lambda_found # abolish lambdas, make agents keep their code simple
lazy_any_type_hints # abolish type `Any` used to bypass strict type-checking
chaotic_continue_statements # abolish unecessary nested continue statements, clean code
complex_comprehension # no needlessly dense list/set/dict comprehensions, prefer linear code
@@ -266,6 +239,10 @@ You don’t have to. The loop runner, Ralph, and the CLI take a prompt, launch a
The included [`harness/js-scaffold`](harness/js-scaffold/package.json) is a simple JavaScript **example** to expand on. Go to [pyproject.toml line 75](pyproject.toml#L75). Update checks. Put `js` into list `[tool.harness].languages`. Remove `py` if unused.
+- **Why not just a shell loop?**
+
+A shell loop only reruns an agent. LoopGate ensures fresh context, durable repo state, time and iteration limits, protected paths, and quality gates that stop bad changes _before_ they land.
+
```
npm run --prefix harness/js-scaffold gate
npm run --prefix harness/js-scaffold preflight
@@ -322,3 +299,38 @@ npm run --prefix harness/js-scaffold preflight
+
+## Read this before a first run.
+
+1. **This harness does not sandbox agents.** It _tries_ to harness bad code in loops via gates. Sandboxing agents will, e.g. prevent them from maintaining git, running Playwright, being seen as trustworthy by semgrep leading to cyclical failures, etc.
+
+2. **The gate is a guardrail, not a jail.** Agents are crafty, like people. They will find a way to complete a task at all costs. **Trust nothing and no one.**
+
+3. **Mind your usage limits.** `harness run` works agents to the cap set. You can easily burn through your tokens, context windows, and provider usage limits. **Workers continue running as long as there is work to do.**
+
+4. **`docs/PROMPT.md` tells the worker to push or not**.
+
+5. Protect `main` and run the loop on its own branch.
+
+6. **100% coverage does not mean good tests.** That is quantity, not quality. Run `uv run mutmut run` to find covered lines that no assertion actually checks.
+
+7. **Note**: `semgrep --config auto` needs network for semgrep registry rules.
+
+## Safety
+
+`harness run` launches an autonomous LLM worker with the configured permissions, e.g.
+`--permission-mode acceptEdits` or `--sandbox danger-full-access`.
+
+The gate bounds what any **commit** may touch, but the worker itself is **not** sandboxed to this repo unless you set that config. Consider the balance: without access it cannot do much. With machine access it can wreak havoc. Under a permissive mode it can run arbitrary shell. You are authorizing real changes. Choose the worker and permission mode deliberately.
+
+#### The Gate: Tiered Checks
+
+⚡ `harness preflight` (pre-commit) → fast checks.
+Ruff lint + check format for everyone, _plus_ **containment** for the agents. Self-heals by un-staging forbidden files.
+
+✅ `harness gate` (CI/PR pre-push). Local checks mirror CI → ruff lint + format report-only, pyright, pylint, semgrep, complexipy, hypothesis, pytest @ 100% cov.
+
+Only humans can bypass triggered gates and commit by adding flag `--no-verify`.
+
+
+
diff --git a/harness/cli.py b/harness/cli.py
index 79b104c..abdcf18 100644
--- a/harness/cli.py
+++ b/harness/cli.py
@@ -61,7 +61,9 @@ def setup_git_hooks(env_bin: Path, is_windows: bool) -> Path:
rprint("\n[cyan2]Setting git hooks[/cyan2] with `git config core.hooksPath .githooks`:")
subprocess.run(("git", "config", "core.hooksPath", ".githooks"), cwd=REPO_ROOT_STR, check=True)
binary = env_bin / ("harness.exe" if is_windows else "harness")
- recorded = Path(run_git(["rev-parse", "--absolute-git-dir"]).strip()).resolve() / "harness-path"
+ recorded = (
+ Path(run_git(["rev-parse", "--path-format=absolute", "--git-common-dir"]).strip()) / "harness-path"
+ )
recorded.write_text(f"{binary.as_posix()}\n", encoding="utf-8", newline="\n")
typer.echo(
subprocess.run(
@@ -228,7 +230,7 @@ def cleanup(cwd: Path, name: str | None) -> bool:
tool.setdefault("pyright", tomlkit.table()).update({"include": ["src", "preferences"]})
tool.setdefault("pytest", tomlkit.table()).setdefault("ini_options", tomlkit.table()).update({
"testpaths": ["tests"],
- "pythonpath": ["src"],
+ "pythonpath": [".", "src"],
})
coverage = tool.setdefault("coverage", tomlkit.table())
coverage.setdefault("run", tomlkit.table()).update({"source": ["src", "preferences"]})
diff --git a/harness/gate.py b/harness/gate.py
index ff7abda..bdcaef4 100644
--- a/harness/gate.py
+++ b/harness/gate.py
@@ -5,7 +5,7 @@
All containment lists and check commands come from [tool.harness] in pyproject.toml, read once at
import into the constants below. A check is a (name, argv) pair; its `preflight`/`blocking` flags sort
-it into the maps and sets this module runs on. Nothing is hardcoded here.
+it into the maps and sets this module runs on.
"""
from __future__ import annotations
@@ -25,6 +25,8 @@
except ImportError: # humans do what they want with preferences.py
prefs = None
+EMPTY_TREE = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" # universal empty tree hash
+
def run_git(args: list[str], repo: Path | None = None, check: bool = True) -> str:
"""Run a git command in the repo and return its stdout.
@@ -199,13 +201,12 @@ def prepare_commit_msg(argv: list[str]) -> int:
Returns:
Status code integer 0 or 1 (git blocks commit on code 1)
"""
- if os.environ.get("RALPH_LOOP") != "1":
+ if not os.environ.get("RALPH_LOOP"):
return 0
commit_msg_file: str = argv[1] if len(argv) > 1 else ""
command = argv[2] if len(argv) > 2 else ""
msg = ""
- empty_tree = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" # universal empty tree hash
- ref = "HEAD" if run_git(["rev-parse", "--verify", "HEAD"], check=False).strip() else empty_tree
+ ref = "HEAD" if run_git(["rev-parse", "--verify", "HEAD"], check=False).strip() else EMPTY_TREE
if command in {"merge", "squash", "rebase", "reset", "clean", "filter-branch"}:
msg = f"You cannot use that git command `{command}`.\n"
if not run_git(["diff-index", "--cached", "--name-only", f"{ref}"]):
diff --git a/harness/tests/test_cli.py b/harness/tests/test_cli.py
index ad4d753..9d6f7fd 100644
--- a/harness/tests/test_cli.py
+++ b/harness/tests/test_cli.py
@@ -320,7 +320,7 @@ def test_installing_the_template_cleans_the_repo_sets_hooks_and_reruns_cleanly(
assert document["tool"]["pytest"]["ini_options"] == {
"addopts": ["-ra"],
"testpaths": ["tests"],
- "pythonpath": ["src"],
+ "pythonpath": [".", "src"],
}
assert document["tool"]["coverage"] == {
"run": {"source": ["src", "preferences"]},
diff --git a/harness/tests/test_gate.py b/harness/tests/test_gate.py
index 06eeb4f..5672220 100644
--- a/harness/tests/test_gate.py
+++ b/harness/tests/test_gate.py
@@ -164,9 +164,7 @@ def test_pre_push_hook_dispatches_gate_and_blocks_push(real_hook_repo: Path) ->
@pytest.mark.parametrize("real_hook_repo", [("prepare-commit-msg",)], indirect=True)
-def test_prepare_commit_msg_hook_rejects_empty_agent_then_accepts_staged_work(
- real_hook_repo: Path,
-) -> None:
+def test_prepare_commit_msg_hook_rejects_empty_agent_then_accepts_staged_work(real_hook_repo: Path) -> None:
"""The hook rejects an empty agent commit, then accepts the agent's staged work."""
before = gate.run_git(["rev-parse", "HEAD"], real_hook_repo).strip()
agent_empty = git_process(
@@ -194,14 +192,7 @@ def test_prepare_commit_msg_hook_rejects_empty_agent_then_accepts_staged_work(
def test_prepare_commit_msg_hook_allows_human_empty_commit(real_hook_repo: Path) -> None:
"""The hook does not apply agent containment to a human's empty commit."""
human_empty = git_process(
- real_hook_repo,
- "commit",
- "--allow-empty",
- "--no-verify",
- "-q",
- "-m",
- "human empty",
- loop=False,
+ real_hook_repo, "commit", "--allow-empty", "--no-verify", "-q", "-m", "human empty", loop=False
)
assert human_empty.returncode == 0
@@ -377,14 +368,16 @@ def test_human_running_the_same_commands_is_not_policed(
*(
pytest.param(f"{directory}blocked.txt", id=f"dir-{directory}")
for directory in gate.FORBIDDEN_DIRS
+ if directory != ".git/"
),
*(pytest.param(path, id=f"file-{path}") for path in gate.FORBIDDEN_FILES),
],
)
-def test_every_configured_forbidden_path_is_ejected(
+def test_every_configured_forbidden_path_is_ejected_except_dot_git(
forbidden_path: str, monkeypatch: pytest.MonkeyPatch, git_repo: Path
) -> None:
- """Every forbidden directory and exact file configured in pyproject is removed from the index."""
+ """Every forbidden directory and exact file configured in pyproject is removed from the index, except for
+ `.git` which never stages files (but is forbidden to be explicit to agents.)"""
monkeypatch.setenv("RALPH_LOOP", "1")
stage(git_repo, forbidden_path, "blocked\n")
@@ -392,6 +385,8 @@ def test_every_configured_forbidden_path_is_ejected(
assert gate.run_non_human_checks() == []
assert gate.run_git(["diff", "--cached", "--name-only"]).splitlines() == []
+ assert ".git/" in gate.FORBIDDEN_DIRS
+
def test_every_configured_check_can_block_the_gate(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None:
"""Each configured check takes its turn failing; all of them run and the failing one blocks."""
diff --git a/mutation/check_mutmut.py b/mutation/check_mutmut.py
new file mode 100644
index 0000000..534d1ff
--- /dev/null
+++ b/mutation/check_mutmut.py
@@ -0,0 +1,105 @@
+"""FAIL CI WHEN MUTMUT'S AGGREGATE EXPORT CONTAINS TEST GAPS.
+
+Mutation testing mutates the source code, e.g. a boolean flips, a constant shifts, a comparison widens.
+A mutant that dies proves a test assertion was working on that line. A mutant that SURVIVES is a variant of
+source code that shows tests execute (i.e. test coverage is met there) BUT the test is flimsy:
+has weak assertions, overly-mock, happy path only, missing edge cases, no real assert in test, weak logic.
+
+i.e. Mutmut tests the tests. Mutmut breaks source then runs test suite. If test fails, the break was noticed,
+a mutated source code variant is "killed." If every test passes after mutation the break went unnoticed,
+mutant code variant "survived".
+
+Mutmut docs: https://mutmut.readthedocs.io/
+ mutmut run # generate mutants and run the test suite against all source code + its mutants
+ mutmut browse # TUI over the survivors
+ mutmut results # plain-text summary
+
+Config in pyproject.toml [tool.mutmut]
+
+Each mutant variant has a name, e.g. tests.test_file.x_lazy_assert__mutmut_2. Mutmut records mutant as killed
+or survived before you inspect, and re-uses variant until source code changes.
+
+CASE:
+- best: 100% test coverage, run mutmut -> many mutations, none survive, nothing for you to kill
+(tests are strongly sensitive to change)
+- good: <100% coverage, run mutmut -> many mutations, none survive, you have to hunt some
+(the tests that exist are good but you miss coverage)
+- worst: 100% coverage, run mutmut -> many mutations, ALL survive + no easy kills
+(there are many low quality tests)
+- realistic: run mutmut, some mutants created, some survive, you find some to kill
+
+PROCESS:
+
+1. Run mutmut: mutations appear / are killed
+2. Inspect: You look at surviving mutations
+3. Test Update: You add or improve test assertions
+4. Re-run: You ensure the mutant is killed
+
+5. Updating Source Code
+- Dead Code
+Sometimes a mutant survives because source code is redundant, e.g. If changing a line doesn't break a test,
+ask if that code is needed. Maybe delete the useless code instead of writing tests.
+- Surviving Mutants
+You do not need to reach zero mutants, e.g. if changing source effects performance negatively, message output
+would change, equivalent code swap e.g. i < 10 => i != 10
+
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import typer
+from rich import print as rprint
+from rich.console import Console
+from rich.table import Table
+
+console = Console(force_terminal=True)
+
+JsonDocument = dict[str, object] | list[object] | str | int | float | bool | None
+
+
+def analyze_mutmut_report(file_path: str = "mutants/mutmut-cicd-stats.json") -> float:
+ """Read the mutmut CI JSON results created each Sunday night.
+
+ Arguments:
+ file_path (str): Default filepath to read mutmut run stats from.
+
+ Returns:
+ mutation_score: float value, killed + timeout mutants as a% of all
+
+ Raises:
+ JSONDecodeError: If the report does not contain valid JSON.
+ Exit: If the report file does not exist.
+ """
+ if not Path(file_path).exists():
+ rprint(rf"[red]Error: Mutmut JSON report not found at [\]'{file_path}'")
+ raise typer.Exit(code=1)
+ data: dict[str, int] = {}
+ with Path(file_path).open("r", encoding="utf-8") as fp:
+ try:
+ data = json.load(fp)
+ except json.JSONDecodeError:
+ rprint(rf"[red]JSONDecodeError [\]'{file_path}'")
+ raise
+
+ mutation_score: float = 0.0
+ total_mutants = data.get("total", 0)
+ skipped = data.get("skipped", 0)
+ tested_mutants = total_mutants - skipped
+ if tested_mutants > 0:
+ killed = data.get("killed", 0)
+ timeout = data.get("timeout", 0)
+ mutation_score = ((killed + timeout) / tested_mutants) * 100
+
+ table = Table(title="\n[cyan2]MUTMUT MUTATION RESULTS[/]\n", box=None, padding=(0, 2))
+ for stat, result in data.items():
+ table.add_row(f"[turquoise] {stat}[/]", f"[blue] {result}[/]")
+ table.add_row(f"[bold italic turquoise] MUTATION SCORE: [/][bold italic blue]{mutation_score}[/]")
+ console.print(table, justify="center")
+ return mutation_score
+
+
+if __name__ == "__main__":
+ analyze_mutmut_report()
diff --git a/pyproject.toml b/pyproject.toml
index c0ffe1f..27d863e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -11,13 +11,15 @@ dependencies = [
dev = [
"complexipy",
"hypothesis",
+ "mutmut",
+ "pip-audit",
"pylint",
"pyright",
"pytest",
"pytest-cov",
"pytest-xdist",
"ruff",
- "semgrep",
+ "semgrep"
]
[project.scripts]
harness = "harness.cli:main" # creates the executable
@@ -114,7 +116,7 @@ pytest = [
]
[tool.harness.FORBIDDEN]
-DIRS = ["harness/", ".githooks/", ".github/", "preferences/", "tests/preferences/"]
+DIRS = ["harness/", ".githooks/", ".github/", "preferences/", "tests/preferences/", ".git/"]
FILES = [
"agents.md",
"pyproject.toml",
@@ -166,7 +168,8 @@ PATTERNS = [
"fmt: off",
"fmt: skip",
"yapf: disable",
- "complexipy: ignore"
+ "complexipy: ignore",
+ "pragma: no mutate"
]
# ==============================================================================
@@ -178,7 +181,7 @@ reportAssertAlwaysTrue = "error" # Catch asserts that are always true.
reportUnusedImport = "none" # Handled by Ruff F401
reportUnusedVariable = "none" # Handled by Ruff F841
verboseOutput = true # JSON report output when Pyright is run
-include = ["src", "preferences", "harness"] # Type-check shipped code plus repository preferences.
+include = ["src", "preferences", "mutation", "harness"] # Type-check shipped code plus repository preferences.
# ==============================================================================
# Pytest Configuration
@@ -187,23 +190,35 @@ include = ["src", "preferences", "harness"] # Type-check shipped code plus repos
minversion = "9.0"
addopts = ["-ra"] # Print useful short reasons for outcomes
testpaths = ["tests", "harness"]
-pythonpath = ["src", "harness"] # Lets tests import Python code located in the named folders
+pythonpath = ["."] # Repo root: tests import root packages by full dotted path
# ==============================================================================
# Test Coverage Configuration
# ==============================================================================
[tool.coverage]
-run.source = ["src", "preferences", "harness"] # ADD EVERY DIRECTORY THAT SHIPS CODE TO ENSURE 100% COVERAGE
+run.source = ["src", "preferences", "mutation", "harness"] # ADD EVERY DIRECTORY THAT SHIPS CODE TO ENSURE 100% COVERAGE
report.show_missing = true # Show exact missed lines when coverage fails
report.skip_covered = false # Keep fully covered files visible in reports
report.fail_under = 100 # Block merges unless all shipped code is covered
skip_covered = false # Same behavior for older coverage config readers
+# ==============================================================================
+# Mutmut (Mutation Testing) Configuration
+# ==============================================================================
+[tool.mutmut]
+max-children = 2
+source_paths = ["preferences/preferences.py", "harness/gate.py"] # what to mutate
+also_copy = ["harness", ".githooks"] # not mutated paths, copied for imports
+do_not_mutate = ["harness/tests/*"] # mutmut can't tell what's a test file
+pytest_add_cli_args_test_selection = [
+ "harness/tests/test_properties.py", "harness/tests/test_gate.py",
+]
+
# ==============================================================================
# Complexipy Configuration
# ==============================================================================
[tool.complexipy]
-paths = ["src", "preferences", "harness"]
+paths = ["src", "preferences", "mutation", "harness"]
exclude = ["**/tests/**"] # Test code can be more branching and example-heavy than production code
max-complexity-allowed = 10 # Cognitive complexity cap catches deeply nested, hard-to-read flow
no-ignore = true # Disallow `# complexipy: ignore`, fix complexity instead of suppressing it
@@ -228,7 +243,9 @@ main.ignore = [
"tests",
"**/tests/**",
"harness/tests",
- "worktrees"
+ ".worktrees",
+ "worktrees",
+ "mutants"
]
format.max-line-length = 110 # Keep Pylint line accounting aligned with Ruff E501
format.max-module-lines = 500 # Enable Pylint C0302 because Ruff has no equivalent
diff --git a/tests/mutation/mutmut-cicd-stats.json b/tests/mutation/mutmut-cicd-stats.json
new file mode 100644
index 0000000..7f7aad2
--- /dev/null
+++ b/tests/mutation/mutmut-cicd-stats.json
@@ -0,0 +1,11 @@
+{
+ "killed": 132,
+ "survived": 0,
+ "total": 133,
+ "no_tests": 0,
+ "skipped": 0,
+ "suspicious": 0,
+ "timeout": 1,
+ "check_was_interrupted_by_user": 0,
+ "segfault": 0
+}
diff --git a/tests/mutation/test_check_mutmut.py b/tests/mutation/test_check_mutmut.py
new file mode 100644
index 0000000..9b1b79a
--- /dev/null
+++ b/tests/mutation/test_check_mutmut.py
@@ -0,0 +1,61 @@
+"""Tests for the mutmut CI report checker."""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import runpy
+from pathlib import Path
+
+import pytest
+import typer
+from click import unstyle
+
+from mutation.check_mutmut import analyze_mutmut_report
+
+
+def test_report_with_timeout_passes_and_renders(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
+ """A timeout counts as detected and remains visible in the report."""
+ report = tmp_path / "mutants" / "mutmut-cicd-stats.json"
+ report.parent.mkdir()
+ report.write_text(
+ Path(__file__).with_name("mutmut-cicd-stats.json").read_text(encoding="utf-8"), encoding="utf-8"
+ )
+
+ checker = Path(__file__).parents[2] / "mutation" / "check_mutmut.py"
+ with contextlib.chdir(tmp_path):
+ runpy.run_path(str(checker), run_name="__main__")
+
+ output = " ".join(unstyle(capsys.readouterr().out).split())
+ assert "MUTMUT MUTATION RESULTS" in output
+ assert "timeout 1" in output
+ assert "MUTATION SCORE: 100.0" in output
+
+
+def test_actionable_result_fails(tmp_path: Path) -> None:
+ """A survived mutant is included in the report analysis."""
+ data = json.loads(Path(__file__).with_name("mutmut-cicd-stats.json").read_text(encoding="utf-8"))
+ data["survived"] = 1
+ data["total"] += 1
+ report = tmp_path / "mutmut-cicd-stats.json"
+ report.write_text(json.dumps(data), encoding="utf-8")
+ mutation_score = analyze_mutmut_report(report)
+ assert mutation_score >= 60.0
+
+
+def test_missing_report_fails(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
+ """A missing export errors CI."""
+ with pytest.raises(typer.Exit) as exc_info:
+ analyze_mutmut_report(tmp_path / "missing.json")
+
+ assert exc_info.value.exit_code == 1
+ assert "Error: Mutmut JSON report not found" in capsys.readouterr().out
+
+
+def test_malformed_report_fails(tmp_path: Path) -> None:
+ """Malformed JSON is reported as malformed JSON."""
+ report = tmp_path / "mutmut-cicd-stats.json"
+ report.write_text("{", encoding="utf-8")
+
+ with pytest.raises(json.JSONDecodeError):
+ analyze_mutmut_report(report)
diff --git a/tests/preferences/test_preferences.py b/tests/preferences/test_preferences.py
index 4b4fabd..1121f07 100644
--- a/tests/preferences/test_preferences.py
+++ b/tests/preferences/test_preferences.py
@@ -10,8 +10,12 @@
import ast
import inspect
+import keyword
+import string
+from unittest.mock import Mock
import pytest
+from hypothesis import assume, given, strategies
preferences = pytest.importorskip("preferences.preferences")
preferences_violations = preferences.preferences_violations
@@ -275,6 +279,22 @@ def test_preferences_violations_returns_grouped_str() -> None:
assert not violations
+def test_locationless_node_violation_reports_unknown_line(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A `preferences` check reports '?' for missing lineno when its AST node has no parser-provided line.
+
+ Args:
+ monkeypatch: Sets/restores the AST parser for the test.
+ """
+ source = "value = lambda: None\n"
+ tree = ast.parse(source)
+ lambda_node = next(node for node in ast.walk(tree) if isinstance(node, ast.Lambda))
+ del lambda_node.lineno
+ monkeypatch.setattr(ast, "parse", Mock(return_value=tree))
+
+ expected = "m.py:?: Lambda found hurting readability and adding complexity."
+ assert preferences_violations("m.py", source) == expected
+
+
def test_clean_file_has_no_complaints() -> None:
"""A compliant module produces no complaints (the empty string)."""
source = (
@@ -338,3 +358,127 @@ def test_syntax_error_raises() -> None:
"""Unparseable source raises SyntaxError; preferences does not swallow it."""
with pytest.raises(SyntaxError):
preferences_violations("m.py", "def broken(:\n")
+
+
+# --------------------------------------------------------------------- generated behavior
+
+
+IDENTIFIER_START = string.ascii_letters + "_"
+IDENTIFIER_REST = IDENTIFIER_START + string.digits
+
+
+@strategies.composite
+def identifiers(draw: strategies.DrawFn) -> str:
+ """Draw a valid ASCII Python identifier that is not a keyword."""
+ first = draw(strategies.sampled_from(IDENTIFIER_START))
+ rest = draw(strategies.text(alphabet=IDENTIFIER_REST, max_size=24))
+ name = first + rest
+ assume(not keyword.iskeyword(name))
+ return name
+
+
+def flags(source: str, needle: str) -> bool:
+ """Return whether the preferences output contains the expected text."""
+ checker = preferences_violations
+ return needle in checker("m.py", source) if checker else False
+
+
+@given(name=identifiers())
+def test_underscore_lead_flagged_iff_leading_underscore_not_dunder(name: str) -> None:
+ """Assignment targets are flagged exactly when they use a non-dunder leading underscore."""
+ expected = name.startswith("_") and not name.endswith("__")
+ assert flags(f"{name} = 1\n", "starts with underscore") is expected
+
+
+@given(name=identifiers())
+def test_underscore_rule_holds_for_function_and_argument_names(name: str) -> None:
+ """The underscore rule applies equally to function and argument names."""
+ assume(not name.endswith("__"))
+ expected = name.startswith("_")
+ assert flags(f"def {name}():\n return 1\n", "starts with underscore") is expected
+ assert flags(f"def f({name}):\n return {name}\n", "starts with underscore") is expected
+
+
+@strategies.composite
+def class_source(draw: strategies.DrawFn) -> tuple[str, bool]:
+ """Draw a class and whether the pointless-class rule should flag it."""
+ has_base = draw(strategies.booleans())
+ has_decorator = draw(strategies.booleans())
+ has_keyword = draw(strategies.booleans())
+ method_count = draw(strategies.integers(min_value=0, max_value=3))
+
+ decorator = "@deco\n" if has_decorator else ""
+ header_bits = (["Base"] if has_base else []) + (["metaclass=type"] if has_keyword else [])
+ header = f"({', '.join(header_bits)})" if header_bits else ""
+ body = "".join(f" def m{i}(self):\n return {i}\n" for i in range(method_count)) or " x = 1\n"
+ source = f"{decorator}class C{header}:\n{body}"
+ should_flag = not has_base and not has_decorator and not has_keyword and method_count <= 1
+ return source, should_flag
+
+
+@given(case=class_source())
+def test_pointless_class_flagged_iff_plain_and_at_most_one_method(case: tuple[str, bool]) -> None:
+ """Only plain classes with at most one method are pointless."""
+ source, should_flag = case
+ assert flags(source, "no base, decorator, or behavior") is should_flag
+
+
+@strategies.composite
+def comprehension_source(draw: strategies.DrawFn) -> tuple[str, bool]:
+ """Draw a comprehension and whether its generators make it too complex."""
+ generator_count = draw(strategies.integers(min_value=1, max_value=3))
+ if_on = draw(strategies.integers(min_value=-1, max_value=generator_count - 1))
+
+ clauses: list[str] = []
+ for index in range(generator_count):
+ clause = f"for v{index} in xs{index}"
+ if index == if_on:
+ clause += f" if v{index}"
+ clauses.append(clause)
+ source = f"[v0 {' '.join(clauses)}]\n"
+ return source, generator_count > 1 and if_on != -1
+
+
+@given(case=comprehension_source())
+def test_complex_comprehension_flagged_iff_multi_generator_with_filter(case: tuple[str, bool]) -> None:
+ """Comprehensions are complex only with multiple generators and a filter."""
+ source, should_flag = case
+ assert flags(source, "Overly complex comprehension") is should_flag
+
+
+@strategies.composite
+def nested_continue_source(draw: strategies.DrawFn) -> str:
+ """Draw a continue nested beneath an outer loop and two to four more blocks."""
+ depth = draw(strategies.integers(min_value=2, max_value=4))
+ blocks = draw(
+ strategies.lists(strategies.sampled_from(["if cond", "for i in xs"]), min_size=depth, max_size=depth)
+ )
+
+ lines = ["for outer in items:"]
+ indent = " "
+ for block in blocks:
+ lines.append(f"{indent}{block}:")
+ indent += " "
+ lines.append(f"{indent}continue")
+ return "\n".join(lines) + "\n"
+
+
+@given(source=nested_continue_source())
+def test_continue_nested_under_stacked_blocks_is_flagged(source: str) -> None:
+ """Mixed nested if/for blocks always trigger the continue nesting rule."""
+ assert flags(source, "Overly-nested 'continue'")
+
+
+def test_single_if_guard_in_a_loop_is_not_flagged() -> None:
+ """A single if guard directly inside a loop remains readable."""
+ assert not flags("for i in items:\n if skip:\n continue\n", "Overly-nested 'continue'")
+
+
+def test_shallow_continue_in_single_loop_is_not_flagged() -> None:
+ """A continue directly inside one for loop is allowed."""
+ assert not flags("for x in items:\n continue\n", "Overly-nested 'continue'")
+
+
+def test_continue_in_while_loop_is_flagged() -> None:
+ """A continue inside a while loop is banned as a freeze risk."""
+ assert flags("while cond:\n continue\n", "while loop banned")