From 90b8a68d754d00a7d3a2c70680588a4c52f81919 Mon Sep 17 00:00:00 2001 From: arpan Date: Mon, 14 Sep 2026 22:29:50 +0530 Subject: [PATCH 1/2] Enforcement coverage, from what is already written MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SPEC-v0.11 §7 and rule 4. A list with a reason, never a score: it does not move the exit code, because a number that ranked a deployment would be verify grading an operator's document in a new costume. Signed-off-by: arpan --- CHANGELOG.md | 16 ++ src/ctrlrun/cli/main.py | 48 +++++- src/ctrlrun/coverage.py | 214 +++++++++++++++++++++++++ tests/test_coverage.py | 337 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 614 insertions(+), 1 deletion(-) create mode 100644 src/ctrlrun/coverage.py create mode 100644 tests/test_coverage.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c7f66139..d6225d12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,22 @@ any change to one appears here. trusted. Found by review; the tests that missed it all tampered with a row's *content*, and `{}` and a float among the controls are both valid JSON. +- **Enforcement coverage: what this deployment has never exercised** (`SPEC-v0.11.md` §7). + `ctrlrun scan --coverage` reads a store and reports the policy entries, gateway tools and + `@protect` actions that no receipt in it names. + + **From what is already written**: no new event type and no new column. The action name lives on + the receipt rather than on the event, and every action that reached a decision leaves one, a + **denial included** — so an action that is always denied counts as exercised, because the deny + rule firing is the action being enforced rather than ignored. + + **It is a list and not a score.** No percentage, no ratio, no badge, and it does not move the + exit code: a number that ranked a deployment would be `verify` grading an operator's document + in a new costume, which `SPEC-v0.4.md` §3.9 forbids. Every entry carries a reason that states + what was not found, and the report says in every rendering, empty or not, that **a policy entry + nothing exercised may be correctly unused** — a quarterly job, a deny rule that exists so the + action is refused rather than unknown, a tool nobody has needed yet. + - **Retention: a prune that leaves the chain verifiable across the gap, a checkpoint, and a hold** (`SPEC-v0.11.md` §4 and rule 2). There has been no retention policy until now, and `../ctrlrun-docs/docs/postgres.md` said so in the same breath as the reason one is hard: diff --git a/src/ctrlrun/cli/main.py b/src/ctrlrun/cli/main.py index 5a990d4f..e5e459df 100644 --- a/src/ctrlrun/cli/main.py +++ b/src/ctrlrun/cli/main.py @@ -2281,12 +2281,21 @@ def _effect_line(record: EffectRecord) -> str: "the list in force and exit.", ) @click.option("--json", "as_json", is_flag=True, help="Emit one ctrlrun.scan/v1 document.") +@click.option( + "--coverage", + "coverage_flag", + is_flag=True, + help="Also report what this store has never exercised (SPEC-v0.11 §7). Opens the store.", +) +@STORE_URL_OPTION def scan( root: Path | None, policy_path: Path | None, excludes: tuple[str, ...], vocabulary_path: str | None, as_json: bool, + coverage_flag: bool, + store_url: str | None, ) -> None: """Report the consequential call sites and policy entries nothing is covering. @@ -2297,6 +2306,12 @@ def scan( It is a finder and not a proof. Every run prints what it could not look at, and a clean scan means nothing was found where it looked. + With --coverage it also reads a store and reports what this deployment declared and has + never exercised. **That half is a list and not a score**: no percentage, no ratio, no badge. + A policy entry nothing exercised may be correctly unused, and it says so. It does not move + the exit code, for the same reason: a number that ranked a deployment would be a verdict on + the operator's document, which this tool does not give. + Exit codes: 0 nothing was found; 1 something was, including a suppressed finding or a call whose name could not be resolved; 2 the scan could not run. """ @@ -2330,12 +2345,43 @@ def scan( click.echo(f"ctrlrun scan: {refused}", err=True) raise SystemExit(2) from refused + measured = None + if coverage_flag: + from ..coverage import coverage as run_coverage + from ..coverage import coverage_lines + + store = _store(store_url) + try: + loaded = _loaded_policy() if policy_path is None else Policy.from_file(policy_path) + measured = run_coverage( + store, + policy_actions=sorted(loaded.actions), + protected_actions=sorted( + {finding.name for finding in report.findings if finding.name is not None} + ), + policy_path=str(policy_path) if policy_path else None, + ) + except CTRLRunError as exc: + raise _fail(exc) from exc + if as_json: - click.echo(json_module.dumps(report_document(report), indent=2)) + document = report_document(report) + if measured is not None: + # A **key**, not a merged document: `ctrlrun.scan/v1` answers a question about + # source and `ctrlrun.coverage/v1` answers one about a store, and folding them + # would make a consumer parse two shapes under one name. + document["coverage"] = measured.to_dict() + click.echo(json_module.dumps(document, indent=2)) else: for line in report_lines(report): click.echo(line) + if measured is not None: + for line in coverage_lines(measured): + click.echo(line) + # **`--coverage` does not move the exit code** (rule 4). An unexercised policy entry is a + # fact about the record, not a finding about the operator, and an exit code that moved with + # it would be the score this item is forbidden to produce, wearing a shell's clothes. if report.exit_code: raise SystemExit(report.exit_code) diff --git a/src/ctrlrun/coverage.py b/src/ctrlrun/coverage.py new file mode 100644 index 00000000..119dea04 --- /dev/null +++ b/src/ctrlrun/coverage.py @@ -0,0 +1,214 @@ +# SPDX-FileCopyrightText: 2026 The CTRLRun contributors +# SPDX-License-Identifier: Apache-2.0 +"""Enforcement coverage: what this deployment has never exercised. SPEC-v0.11 §7. + +The runtime half of `ctrlrun scan`, whose static half landed in v0.10. `scan` reads the source +and asks *is this call protected?*. This reads what the deployment has already recorded and asks +a different question: **which of the things you declared has nothing ever gone through?** + +**From what is already written.** No new event type, no new column. The action name lives on the +**receipt** rather than on the event, which is a fact worth stating because it decides the whole +design: `ACTION_PROPOSED` carries an `action_hash` and nothing that maps it back to a name, and +every action that reached a decision leaves a receipt, a denial included. So the question is +answerable, and if it had needed a new event the answer would have been to say so and stop. + +**Rule 4 (§1.1): a clean result is not a verdict.** No score, no percentage, no ratio, no badge, +and no sentence a reader could quote as one. `SPEC-v0.4.md` §3.9 is the precedent and it is worth +stating in full: `verify` never grades an operator's document, and a coverage number that ranked +their deployment would be the same claim in a new costume. + +**A policy entry nothing exercised may be correctly unused.** An action declared for a quarterly +job, a deny rule that exists so the action is refused rather than unknown, a tool nobody has +needed yet: each is a reasonable thing to find here, and none of them is a defect. This module +reports a **list with a reason**, and the reason is a statement about the record rather than +about the operator. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from enum import StrEnum +from typing import Any, Final, Protocol + +#: `ctrlrun.coverage/v1`. Its own schema because it is its own document: `ctrlrun scan`'s report +#: answers a question about source and this answers one about a store, and folding them into one +#: name would make a consumer parse two shapes under it. +COVERAGE_SCHEMA: Final = "ctrlrun.coverage/v1" + + +class Unexercised(StrEnum): + """What kind of thing nothing has gone through.""" + + POLICY_ACTION = "policy_action" + GATEWAY_TOOL = "gateway_tool" + PROTECTED_ACTION = "protected_action" + + +#: The reason each kind carries. **A statement about the record, never about the operator**: it +#: says what was not found, and says in the same breath that not finding it may be correct. +_REASON: Final = { + Unexercised.POLICY_ACTION: ( + "the policy declares this action and no receipt in this store names it" + ), + Unexercised.GATEWAY_TOOL: ( + "the gateway exposes this tool and no receipt in this store names the action it routes to" + ), + Unexercised.PROTECTED_ACTION: ( + "@protect declares this action in the source and no receipt in this store names it" + ), +} + + +@dataclass(frozen=True) +class Unused: + """One declared thing nothing has exercised, and why it is on the list.""" + + kind: Unexercised + name: str + reason: str + + def to_dict(self) -> dict[str, Any]: + return {"kind": str(self.kind), "name": self.name, "reason": self.reason} + + +@dataclass(frozen=True) +class CoverageReport: + """The list, and what it was computed from (§7). + + **There is no `score`, no `percentage`, no `ratio` and no `exit_code` that grades.** The + counts here are inputs an operator needs to read the list at all -- *nothing exercised, over + a store holding no receipts* and *nothing exercised, over a store holding forty thousand* are + different findings -- and neither is a verdict. `T560` greps this module's own output for the + vocabulary rule 4 forbids. + """ + + #: Everything declared that nothing has exercised, in codepoint order within each kind. + unused: tuple[Unused, ...] + #: How many receipts the answer was computed from. Context, not a denominator. + receipts_read: int + #: How many distinct action names those receipts carry. + actions_seen: tuple[str, ...] + #: Where the declarations came from, so a reader can disagree with the list. + policy_path: str | None = None + + def of(self, kind: Unexercised) -> tuple[Unused, ...]: + return tuple(item for item in self.unused if item.kind is kind) + + def to_dict(self) -> dict[str, Any]: + return { + "schema": COVERAGE_SCHEMA, + "policy": self.policy_path, + "receipts_read": self.receipts_read, + "actions_seen": list(self.actions_seen), + "unused": [item.to_dict() for item in self.unused], + } + + +class _CoverageStore(Protocol): + def receipts(self) -> tuple[Any, ...]: ... + + +def _names_seen(store: _CoverageStore) -> tuple[str, ...]: + """Every action name this store's receipts carry, in codepoint order. + + **Receipts, not events**, and the reason is not a preference: `ACTION_PROPOSED` carries an + `action_hash` and nothing that maps it back to a name, so the events alone cannot answer the + question. Every action that reached a decision leaves a receipt -- a denial included, which + matters here, because an action that is always denied **has** been exercised and belongs + nowhere on this list. + + A row this binary cannot read back (`SPEC-v0.11.md` §5.2) carries no action name and is + skipped. It is already reported as `content_altered` by the chain, and a coverage list is not + the place to report a tamper a second time under a different name. + """ + seen = { + name + for receipt in store.receipts() + for name in (getattr(receipt, "action", None),) + if isinstance(name, str) and name + } + return tuple(sorted(seen)) + + +def coverage( + store: _CoverageStore, + *, + policy_actions: Sequence[str] = (), + gateway_tools: Sequence[tuple[str, str]] = (), + protected_actions: Sequence[str] = (), + policy_path: str | None = None, +) -> CoverageReport: + """What this deployment declared and has never exercised (§7). + + Everything is **supplied** rather than discovered: the policy's actions come from the loaded + policy, the gateway's tools from its configuration, and `@protect`'s actions from `scan`'s + static pass. This module reads a store and matches, in the shape `SPEC-v0.9.md` §5.4 settled + for a scope provider, and for the same reason: a module that resolved an operator's document + would be reading the policy from the wrong layer. + + `gateway_tools` is `(tool, action)` because a tool's own name is what an operator recognises + and the action is what a receipt would carry. + """ + seen = set(_names_seen(store)) + found: list[Unused] = [] + + for action in sorted(set(policy_actions)): + if action not in seen: + found.append( + Unused(Unexercised.POLICY_ACTION, action, _REASON[Unexercised.POLICY_ACTION]) + ) + for tool, action in sorted(set(gateway_tools)): + if action not in seen: + found.append(Unused(Unexercised.GATEWAY_TOOL, tool, _REASON[Unexercised.GATEWAY_TOOL])) + for action in sorted(set(protected_actions)): + if action not in seen: + found.append( + Unused(Unexercised.PROTECTED_ACTION, action, _REASON[Unexercised.PROTECTED_ACTION]) + ) + + return CoverageReport( + unused=tuple(found), + receipts_read=len(store.receipts()), + actions_seen=tuple(sorted(seen)), + policy_path=policy_path, + ) + + +#: The sentence this report ends with, always, whether the list is empty or not. +#: +#: **Rule 4's whole content, in the place a reader cannot miss.** An empty list is the one most +#: likely to be quoted as a verdict, so the sentence is not conditional on there being findings. +#: `SPEC-v0.4.md` §3.9's precedent: `verify` never grades an operator's document. +NOT_A_VERDICT: Final = ( + "This is a list of what has not been exercised, not a score. A policy entry nothing " + "exercised may be correctly unused: a quarterly job, a deny rule that exists so the action " + "is refused rather than unknown, a tool nobody has needed yet." +) + + +def coverage_lines(report: CoverageReport) -> list[str]: + """The human rendering, in the shape `ctrlrun scan` already uses (§7). + + A list with a reason per entry, and the sentence above. **No totals line, no ratio, and no + "N of M"**: the counts that appear are the inputs, labelled as such. + """ + lines = ["ctrlrun scan --coverage", ""] + lines.append( + f"read {report.receipts_read} receipt(s), naming {len(report.actions_seen)} action(s)" + ) + lines.append("") + for kind in Unexercised: + entries = report.of(kind) + if not entries: + continue + lines.append(f"never exercised: {kind} ({len(entries)})") + for entry in entries: + lines.append(f" {entry.name}") + lines.append(f" {entry.reason}") + lines.append("") + if not report.unused: + lines.append("everything declared has been exercised at least once") + lines.append("") + lines.append(NOT_A_VERDICT) + return lines diff --git a/tests/test_coverage.py b/tests/test_coverage.py new file mode 100644 index 00000000..63fcaa26 --- /dev/null +++ b/tests/test_coverage.py @@ -0,0 +1,337 @@ +# SPDX-FileCopyrightText: 2026 The CTRLRun contributors +# SPDX-License-Identifier: Apache-2.0 +"""Enforcement coverage, from what is already written. SPEC-v0.11 §7; T560-T566. + +The runtime half of `ctrlrun scan`. `scan` reads source and asks *is this call protected?*; this +reads what the deployment has recorded and asks *which of the things you declared has nothing +ever gone through?* + +**Rule 4 is what this item breaks if it breaks anything**: a clean result is **not a verdict**. +No score, no percentage, no ratio, no badge, and no sentence a reader could quote as one. +`SPEC-v0.4.md` §3.9 is the precedent, and `T560` greps this item's own output for the vocabulary +it forbids, in the shape `CLAIMS.md` uses, because "there is no score" is a claim about the +environment until something checks it. + +**No new event type and no new column** (`T561`). The action name lives on the **receipt**, not +on the event: `ACTION_PROPOSED` carries an `action_hash` and nothing that maps it back. Measured +before any of this was written:: + + Events written: + 1 ACTION_PROPOSED data={'action_hash': 'sha256:1b3e0dab...'} + 2 POLICY_EVALUATED data={'decision': 'allow', 'reason': 'decision'} + Receipts written: + seq=1 action='stripe.refund' result=committed + seq=2 action='k8s.delete_namespace' result=denied + +Every action that reached a decision leaves a receipt, a **denial included**, which is what makes +the question answerable without adding anything. +""" + +from __future__ import annotations + +import contextlib +import json +import re +from datetime import UTC, datetime, timedelta + +import pytest + +from ctrlrun import Control, Policy, SQLiteStateStore +from ctrlrun.action import Action, Principal +from ctrlrun.coverage import ( + COVERAGE_SCHEMA, + NOT_A_VERDICT, + Unexercised, + coverage, + coverage_lines, +) + +T0 = datetime(2026, 1, 1, 12, 0, tzinfo=UTC) +LEASE = timedelta(minutes=5) +POLICY = """schema: ctrlrun.policy/v2 +actions: + stripe.refund: + effect: "refund:{payment_id}" + decision: allow + k8s.delete_namespace: + decision: deny + github.force_push: + decision: allow + quarterly.reconcile: + decision: allow +""" + +#: `SPEC-v0.4.md` §5.3's list, plus the vocabulary a coverage report is uniquely tempted by. A +#: percentage sign is in here because the shape rule 4 forbids is *87%*, and no prose in this +#: report needs one. +FORBIDDEN = ( + "score", + "coverage:", + "percent", + "%", + "ratio", + "grade", + "badge", + "rating", + "secure", + "compliant", + "certified", + "audited", + "fully covered", + "100", +) + + +@pytest.fixture +def store(tmp_path) -> SQLiteStateStore: + """A store where two of four declared actions have been exercised, one of them denied.""" + from ctrlrun.errors import ActionDenied + + opened = SQLiteStateStore(tmp_path / "state.db", clock=lambda: T0) + control = Control(Policy.from_yaml(POLICY), opened, clock=lambda: T0) + control.execute( + Action( + name="stripe.refund", + arguments={"payment_id": "p1"}, + principal=Principal(agent="a"), + ), + lambda: {"ok": True}, + "refund:p1", + lease=LEASE, + ) + with contextlib.suppress(ActionDenied): + control.execute( + Action( + name="k8s.delete_namespace", + arguments={"ns": "x"}, + principal=Principal(agent="a"), + ), + lambda: {"ok": True}, + None, + ) + return opened + + +# --- T560: rule 4 ------------------------------------------------------------------------------- + + +def test_T560_the_report_is_a_list_and_never_a_score(store) -> None: + """SPEC-v0.11 §1.1 rule 4, and `SPEC-v0.4.md` §3.9 on a new surface. + + **This is the assertion the item exists to keep.** A coverage number that ranked a deployment + would be `verify` grading an operator's document in a new costume, and the shape it would + take is *coverage: 87%*. So the output is greped for the vocabulary, over **both** renderings + and over the empty case as well as the full one: an empty list is the one most likely to be + quoted as a verdict. + """ + report = coverage( + store, + policy_actions=sorted(Policy.from_yaml(POLICY).actions), + gateway_tools=(("refund_tool", "stripe.refund"), ("push_tool", "github.force_push")), + protected_actions=("stripe.refund", "internal.sweep"), + ) + whole = "\n".join(coverage_lines(report)) + assert NOT_A_VERDICT in whole, "the sentence that says this is not a verdict is missing" + + # **`NOT_A_VERDICT` is excluded from the scan, and that is an allow-list of one.** It + # contains the word *score*, because it is the sentence saying there is not one, and a plain + # scan would flag it. `tests/test_docs_production.py` solves the identical problem the + # identical way and says why: the next person would remove the scan as a false positive and + # take the check with it. + rendered = whole.replace(NOT_A_VERDICT, "").lower() + as_json = json.dumps(report.to_dict()).lower() + + for word in FORBIDDEN: + assert word not in rendered, f"the coverage report says {word!r}:\n{rendered}" + assert word not in as_json, f"the coverage document says {word!r}: {as_json}" + + # And no "N of M", which is a ratio with the sign filed off. + assert not re.search(r"\b\d+\s*(of|/)\s*\d+\b", rendered), rendered + + # The document carries no field that could be read as one. + document = report.to_dict() + for field in ("score", "percentage", "ratio", "grade", "covered", "exit_code"): + assert field not in document, f"the coverage document has a {field!r} field" + + # The sentence is present whether or not anything was found. + assert NOT_A_VERDICT in "\n".join(coverage_lines(report)) + + +def test_T560b_an_EMPTY_list_still_says_it_is_not_a_verdict(store) -> None: + """The case most likely to be quoted as one: nothing unexercised at all.""" + clean = coverage(store, policy_actions=("stripe.refund", "k8s.delete_namespace")) + assert clean.unused == () + rendered = "\n".join(coverage_lines(clean)) + assert NOT_A_VERDICT in rendered + assert "everything declared has been exercised at least once" in rendered + scanned = rendered.replace(NOT_A_VERDICT, "").lower() + for word in FORBIDDEN: + assert word not in scanned, f"a clean report says {word!r}:\n{rendered}" + + +def test_T560c_the_reason_is_about_the_record_and_never_about_the_operator(store) -> None: + """§7. A policy entry nothing exercised **may be correctly unused**, and the report says so + rather than implying a defect. + + The reason on each entry states what was not found. It does not say *missing*, *should*, + *incomplete*, or anything else that reads as an instruction. + """ + report = coverage(store, policy_actions=sorted(Policy.from_yaml(POLICY).actions)) + assert report.unused, "this store was supposed to have unexercised entries" + for entry in report.unused: + assert "no receipt" in entry.reason, entry.reason + for judging in ("missing", "should", "incomplete", "gap", "must", "fail"): + assert judging not in entry.reason.lower(), ( + f"the reason for {entry.name!r} reads as a judgement: {entry.reason!r}" + ) + assert "may be correctly unused" in NOT_A_VERDICT + + +# --- T561: from what is already written --------------------------------------------------------- + + +def test_T561_the_answer_comes_from_receipts_and_needs_no_new_event(store) -> None: + """§7. **No new event type, no new column.** + + The action name is on the receipt and not on the event: `ACTION_PROPOSED` carries an + `action_hash` and nothing that maps it back to a name. If the question had needed a new + event, §7 says the question is wrong and the item stops and says so. It did not. + """ + from ctrlrun.receipt import EventType + + events = store.events() + assert events, "this store wrote no events" + for event in events: + assert "action" not in event.data, ( + "an event now carries an action name; §7's design reasoning should be revisited, " + f"because it rests on it not doing so: {event.type} {event.data}" + ) + assert EventType.ACTION_PROPOSED in {event.type for event in events} + + # Every decided action left a receipt, a denial included, which is what makes this + # answerable from what is already there. + named = {receipt.action for receipt in store.receipts()} + assert named == {"stripe.refund", "k8s.delete_namespace"}, named + + report = coverage(store, policy_actions=sorted(Policy.from_yaml(POLICY).actions)) + assert set(report.actions_seen) == named + + +def test_T561b_an_action_that_is_always_DENIED_counts_as_exercised(store) -> None: + """A deny rule that fires **has** been exercised, and belongs nowhere on this list. + + This is the half a design reading only `EXECUTION_COMMITTED` would get wrong: the whole point + of a deny rule is that the action is refused rather than unknown, and reporting it as + unexercised would tell an operator to remove the rule that is working. + """ + report = coverage(store, policy_actions=sorted(Policy.from_yaml(POLICY).actions)) + unexercised = {entry.name for entry in report.of(Unexercised.POLICY_ACTION)} + assert "k8s.delete_namespace" not in unexercised, ( + "an action that was denied was reported as never exercised; the deny rule fired, which " + "is the action being enforced rather than ignored" + ) + assert unexercised == {"github.force_push", "quarterly.reconcile"} + + +def test_T561c_a_row_that_cannot_be_read_back_is_not_reported_here(tmp_path, store) -> None: + """SPEC-v0.11 §5.2. A refused row carries no action name, and the chain already reports it as + `content_altered`. A coverage list is not the place to report a tamper a second time under a + different name.""" + from ctrlrun.coverage import _names_seen + from ctrlrun.receipt import UnreadableReceipt + + class _OneBad: + def __init__(self, real): + self._real = real + + def receipts(self): + rows = list(self._real.receipts()) + rows.append(UnreadableReceipt(seq=99, receipt_id=None, refusal="InvalidArgument")) + return tuple(rows) + + assert _names_seen(_OneBad(store)) == ("k8s.delete_namespace", "stripe.refund") + + +# --- T562: the three kinds ---------------------------------------------------------------------- + + +def test_T562_every_declared_kind_is_reported_with_its_own_reason(store) -> None: + report = coverage( + store, + policy_actions=sorted(Policy.from_yaml(POLICY).actions), + gateway_tools=(("refund_tool", "stripe.refund"), ("push_tool", "github.force_push")), + protected_actions=("stripe.refund", "internal.sweep"), + ) + assert {entry.name for entry in report.of(Unexercised.POLICY_ACTION)} == { + "github.force_push", + "quarterly.reconcile", + } + # The tool is named, not the action it routes to: a tool's own name is what an operator + # recognises in their gateway configuration. + assert {entry.name for entry in report.of(Unexercised.GATEWAY_TOOL)} == {"push_tool"} + assert {entry.name for entry in report.of(Unexercised.PROTECTED_ACTION)} == {"internal.sweep"} + assert report.to_dict()["schema"] == COVERAGE_SCHEMA + + +def test_T562b_a_store_with_no_receipts_reports_everything_and_says_what_it_read(tmp_path) -> None: + """*Nothing exercised, over a store holding no receipts* and *nothing exercised, over a store + holding forty thousand* are different findings, and the report distinguishes them. + + The count is context for reading the list, not a denominator: nothing divides by it. + """ + empty = SQLiteStateStore(tmp_path / "empty.db", clock=lambda: T0) + report = coverage(empty, policy_actions=("a.b", "c.d")) + empty.close() + assert report.receipts_read == 0 + assert len(report.unused) == 2 + rendered = "\n".join(coverage_lines(report)) + assert "read 0 receipt(s)" in rendered + assert NOT_A_VERDICT in rendered + + +# --- T563: the CLI ------------------------------------------------------------------------------ + + +def test_T563_the_coverage_flag_does_not_move_the_exit_code(tmp_path, store) -> None: + """Rule 4 again, in the shape a shell can read. + + An exit code that moved with an unexercised policy entry would be the score this item is + forbidden to produce, wearing a shell's clothes: a CI job would then fail because somebody + declared an action for a quarterly run. + """ + import os + + from click.testing import CliRunner + + from ctrlrun.cli.main import main + + workspace = tmp_path / "work" + workspace.mkdir() + (workspace / "ctrlrun.yaml").write_text(POLICY, encoding="utf-8") + # A tree with nothing in it to find, so `scan`'s own half is clean and only `--coverage` + # could move the code. + (workspace / "app.py").write_text("x = 1\n", encoding="utf-8") + database = tmp_path / "state.db" + store.close() + + cwd = os.getcwd() + os.chdir(workspace) + try: + plain = CliRunner().invoke(main, ["scan"]) + with_coverage = CliRunner().invoke( + main, ["scan", "--coverage", "--store-url", f"sqlite:///{database}"] + ) + finally: + os.chdir(cwd) + + # **The comparison is the claim.** Whatever `scan`'s own half decides, `--coverage` must not + # change it: an exit code that moved with an unexercised policy entry would fail a CI job + # because somebody declared an action for a quarterly run. + assert with_coverage.exit_code == plain.exit_code, ( + f"--coverage moved the exit code from {plain.exit_code} to {with_coverage.exit_code}" + ) + assert "never exercised" in with_coverage.output, with_coverage.output + assert NOT_A_VERDICT in with_coverage.output + scanned = with_coverage.output.replace(NOT_A_VERDICT, "").lower() + for word in FORBIDDEN: + assert word not in scanned, f"ctrlrun scan --coverage says {word!r}" From 7288ac91939ce53076350ae454e2f1c7be931c4c Mon Sep 17 00:00:00 2001 From: arpan Date: Mon, 14 Sep 2026 22:32:59 +0530 Subject: [PATCH 2/2] Two surviving mutations asked for two stronger tests The unreadable-row filter was equivalent to 'is not None' for the only input the test used, and the exit-code comparison was masked because scan found something in that tree on its own and exited 1 either way. Signed-off-by: arpan --- tests/test_coverage.py | 44 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/tests/test_coverage.py b/tests/test_coverage.py index 63fcaa26..6b57d6dd 100644 --- a/tests/test_coverage.py +++ b/tests/test_coverage.py @@ -251,6 +251,20 @@ def receipts(self): assert _names_seen(_OneBad(store)) == ("k8s.delete_namespace", "stripe.refund") + # **And the filter is on the type and on emptiness, not merely on `None`.** A mutation + # relaxing it to `name is not None` survived the version of this test above, because an + # `UnreadableReceipt` has no `action` attribute at all and `getattr` already returns `None`. + # A row whose `action` is an empty string, or is not a string, is the input that tells the + # two apart, and an empty name in a coverage list is an entry an operator cannot act on. + class _Odd: + def __init__(self, *actions): + self._actions = actions + + def receipts(self): + return tuple(type("R", (), {"action": value})() for value in self._actions) + + assert _names_seen(_Odd("", "a.b", None, 7, "c.d")) == ("a.b", "c.d") + # --- T562: the three kinds ---------------------------------------------------------------------- @@ -307,9 +321,23 @@ def test_T563_the_coverage_flag_does_not_move_the_exit_code(tmp_path, store) -> workspace = tmp_path / "work" workspace.mkdir() - (workspace / "ctrlrun.yaml").write_text(POLICY, encoding="utf-8") - # A tree with nothing in it to find, so `scan`'s own half is clean and only `--coverage` - # could move the code. + # **Every action declares an `effect:`**, so `scan`'s own half finds nothing: it reports + # `action_without_effect` otherwise, and that would make the comparison below meaningless. + (workspace / "ctrlrun.yaml").write_text( + "schema: ctrlrun.policy/v2\n" + "actions:\n" + " stripe.refund:\n" + ' effect: "refund:{payment_id}"\n' + " decision: allow\n" + " quarterly.reconcile:\n" + ' effect: "reconcile:{period}"\n' + " decision: allow\n", + encoding="utf-8", + ) + # **A tree `scan`'s own half finds nothing in**, which is what makes the comparison below + # mean anything: with findings of its own, `scan` exits 1 either way and a `--coverage` that + # moved the code would be invisible. A mutation making `--coverage` exit 1 on any unexercised + # entry survived the first version of this test for exactly that reason. (workspace / "app.py").write_text("x = 1\n", encoding="utf-8") database = tmp_path / "state.db" store.close() @@ -327,8 +355,14 @@ def test_T563_the_coverage_flag_does_not_move_the_exit_code(tmp_path, store) -> # **The comparison is the claim.** Whatever `scan`'s own half decides, `--coverage` must not # change it: an exit code that moved with an unexercised policy entry would fail a CI job # because somebody declared an action for a quarterly run. - assert with_coverage.exit_code == plain.exit_code, ( - f"--coverage moved the exit code from {plain.exit_code} to {with_coverage.exit_code}" + assert plain.exit_code == 0, ( + "`ctrlrun scan` found something in this tree on its own, so it exits non-zero either way " + f"and this test cannot see whether --coverage moved it:\n{plain.output}" + ) + assert with_coverage.exit_code == 0, ( + "--coverage moved the exit code. An unexercised policy entry is a fact about the record, " + "not a finding about the operator, and a CI job must not fail because somebody declared " + f"an action for a quarterly run:\n{with_coverage.output}" ) assert "never exercised" in with_coverage.output, with_coverage.output assert NOT_A_VERDICT in with_coverage.output