diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d733b04b..e2720ded 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -120,7 +120,7 @@ jobs: set -eu echo "authority: $AUTHORITY ($AUTHORITY_NA not applicable)" echo "templates: $TEMPLATES ($TEMPLATES_NA not applicable)" - test "$AUTHORITY" = "verified 21/21" + test "$AUTHORITY" = "verified 22/22" # G13 is N/A on SQLite, the action's default store: SQLite has no clock of its own # to diverge from; G15 is N/A because neither document declares `max_attempts`. # G16 is graded on both: verify brings its own precondition provider (SPEC-v0.7 §8.9). @@ -136,7 +136,7 @@ jobs: # binds one, so its count is unchanged and its passing total moved 19 to 20 instead. test "$AUTHORITY_NA" = "2" test "$TEMPLATES" = "verified 11/11" - test "$TEMPLATES_NA" = "12" + test "$TEMPLATES_NA" = "13" test -s verify-badge.json test -s verify-report.json test -s verify-report.xml diff --git a/CHANGELOG.md b/CHANGELOG.md index db0bc7db..08d96f37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,41 @@ any change to one appears here. ## [Unreleased] +## [0.9.0] - Envelope + +*Undated until the tag.* + +Every guarantee before this one answers **whether**. A grant says `amount_lte: 5000`, and is silent +about the thousand actions that each pass it: the authority model bounds one action and has never +bounded an aggregate, so an agent acting entirely within its permissions can still empty an account +one permitted refund at a time. v0.9 answers the other half: **how much, over which records, for +which task?** + +Three dimensions, one rule each. + +**Consequence budgets.** A grant may carry `budgets:`, a metric with a limit over a rolling window, +consumed **on reserve, inside the reservation's own transaction**, because a check on one line and a +consumption on another is a race two processes win together. **Ambiguity is not a refund**: an +`AMBIGUOUS` effect holds its consumption until a human or a hook resolves it, because otherwise an +agent that can manufacture ambiguity can manufacture authority. A budget names a metric, not a +consequence: nothing here ranks, scores or classifies an operator's actions. + +**Scope providers.** `scope=` answers "is this record this principal's?", strictly before the +reservation, which is the bite on an identifier an attacker chose. A grant permits `records.read` on +`customer:*`, and until now nothing had an opinion about whose record `customer:90210` is. + +**Task-bound authority.** `tasks:` narrows a grant to a unit of work, by the same `child ⊆ parent` +rule as every other dimension. It limits blast radius; it does not detect a hijack. + +**What a budget is not.** It **cannot recall an action already in flight**: a rolling window changes +what the next reserve may do and nothing about what is already reserved, so a reservation taken a +second before the window rolls commits regardless. It counts a metric an operator named, an argument +on the action, and is not a consequence model: nothing ranks, scores or classifies what an action +means. It is per store, so two deployments sharing a provider account and not a store each enforce +their own. And it is fail-closed against its own principal: an agent able to manufacture ambiguity +can pin a budget it cannot spend, which is a denial of service against the operator's own agents and +is the deliberate side of the trade against an agent that manufactures authority. + ### Added - **Task-bound authority** (SPEC-v0.9 §6). A grant may carry `tasks:`, a unit-of-work dimension @@ -29,6 +64,22 @@ any change to one appears here. The task reaches the authority decision and the receipt, and **never the action hash**: a field on `Action` would move every hash in existence and invalidate every stored approval. +- **Consequence budgets, enforced** (SPEC-v0.9 §4). **G22.** A budgeted grant charges every + ancestor on reserve, inside the reservation's own transaction, and the ledger is released + exactly when the effect reaches `FAILED`. + + **Ambiguity is not a refund.** An `AMBIGUOUS` effect keeps its consumption until a human or a + `reconcile` hook resolves it, because otherwise an agent that can generate ambiguity can + generate authority, and generating ambiguity is free for any flaky integration. This is the + correctness hole that kept budgets out of four milestones. + + The refusal is `ActionDenied(reason="budget_exhausted")`, naming the grant, the metric and the + window, and **never the remaining balance**: refused actions cost nothing, so a refusal that + reported the balance is an oracle an attacker binary-searches. + + `ctrlrun verify` reports **22/22** on the shipped examples, with G22, G23 and G24 all graded + against positive controls. + - **The budget ledger, and one amendment to a frozen protocol** (SPEC-v0.9 §3). `StateStore` has been frozen since v0.6 and gains exactly two things: `charges=` on `reserve_effect` and `consume_approval_and_reserve`, and `consumptions()` to read the ledger back. Migration @@ -82,17 +133,47 @@ any change to one appears here. precondition hook rather than add a second one. `SPEC-v0.9.md` §5.2.1 records the amendment and the three mechanical differences that justify it. +- **The operator surfaces for a budget** (SPEC-v0.9 §7). **No new command.** `ctrlrun inspect` + gains `--grant GRANT_ID`, which reports each of that grant's budgets as three numbers: + **consumed**, the un-released sum over the rolling window, which is the number that decides; + **held**, the part of it whose effects have not committed; and **why**, the effect holding each + part and the state it is in. + + The third is the deliverable. A budget that refuses while it looks nowhere near its limit is + almost always one unresolved effect, and without the third column an operator cannot get from + the refusal to `ctrlrun resolve`. The view prints that command with the effect key already in + it, because an operator retyping the key from the line above is one transcription away from + resolving a different effect. + + `ctrlrun effects` says what each effect is holding, so `--state ambiguous` answers "what is + pinning this grant". It says **spent** for a committed effect and **holds** for every other, + because §7.2 defines held as the part that has not committed and one word for two numbers would + make the two commands disagree. + + `ctrlrun stats` reports the ledger's row count, so growth is observable before it is a problem. + The ledger only grows: the kernel deletes no row, ships no retention command and has no policy + key that expires evidence. What §7.3 owes instead is the invariant that makes somebody else's + archiving safe, and it states it: rows older than the longest window on any budget of a grant + cannot affect any future decision. + + `ctrlrun.budget/v1` is its own document rather than a key inside `ctrlrun.inspection/v2`, + because that one answers about an action and this answers about a grant: a reader handed one + would have to know which of two shapes it got. Every existing `--json` shape is unchanged, and + T436 asserts that rather than assuming it. + ### Changed - `ctrlrun.receipt/v6` carries `scope_hash` beside `task`, and `ctrlrun.guarantees/v5` carries **G23** beside G24. -- `ctrlrun.policy/v7`, `ctrlrun.receipt/v6` and `ctrlrun.guarantees/v5`. `tasks:` on a grant is - refused in a `v6` document rather than ignored, because an older reader would grant the action - on every task. `DIMENSIONS` grows from six entries to seven, and it is exported and iterated by - `verify`'s G9, so a `--json` consumer counting dimensions sees seven. +- `ctrlrun.policy/v7`, `ctrlrun.receipt/v6` and `ctrlrun.guarantees/v5`. `tasks:` and `budgets:` + on a grant are refused in a `v6` document rather than ignored, because an older reader would + grant the action on every task and against no limit. **`DIMENSIONS` grows from six entries to + eight**, `tasks` and `budgets`, and it is exported and iterated by `verify`'s G9, so a `--json` + consumer counting dimensions sees eight. - The shipped `examples/authority/payments.yaml` binds its `head-of-support` grant to - `refund-run:*`, so the milestone's own guarantee is not `N/A` on what this repository ships. - The authority badge moves from `verified 19/19` to `verified 20/20`. + `refund-run:*` and gives it a daily budget, so the milestone's own guarantees are not `N/A` on + what this repository ships. The authority badge moves from `verified 19/19` to + `verified 22/22`, G22, G23 and G24. - `docs/SPEC-v0.9.md`, the v0.9 "Envelope" contract: consequence budgets, scope providers and task-bound authority, as a delta over v0.1 to v0.8. Documentation only. It specifies the @@ -111,6 +192,63 @@ any change to one appears here. The specification amends one frozen surface: `StateStore`, frozen since `SPEC-v0.6.md` §9.2, gains `charges=` on the two methods that reserve. §3.3 argues it against that section's stated bar. +### Stricter than 0.8.0, with what 0.8.0 did + +- **A 0.8.0 binary refuses a store 0.9.0 has opened.** Migration `0007_budget_ledger` adds the + ledger table, and an older binary opening the migrated database refuses at open, naming the + migration it does not know. Before: there was no `0007`. This is `SPEC-v0.6.md` §3.5's rule and + it makes the upgrade one-way per store: a rollback to 0.8.0 needs the database it had, because a + migration that only runs forwards turns a rollback into silent corruption. + +- **A third-party `StateStore` must implement three more things.** `charges=` on `reserve_effect` + and `consume_approval_and_reserve`, and a `consumptions()` read. Before: `StateStore` was frozen + at `SPEC-v0.6.md` §9.2 and a backend implementing every declared method was complete. A backend + that implements `charges=` and not the read satisfies the protocol and breaks `ctrlrun inspect` + and `ctrlrun verify`, which is why `SPEC-v0.9.md` §3.3 argues the read as part of the amendment + rather than leaving it implicit. + +- **`DIMENSIONS` changed value, from six entries to eight.** It is exported and `verify`'s G9 + iterates it and prints its length, so a `--json` consumer counting dimensions sees eight. Before: + six. `tasks` and `budgets` are the two. + +- **`tasks:` and `budgets:` are refused in a `ctrlrun.policy/v6` document**, rather than ignored as + an unknown key would be. Before: neither key existed. An older reader that ignored them would + grant the action on every task and against no limit, which is the fail-open this refusal closes. + +- **A grant carrying a budget refuses an action that resolves no effect key.** Before: an action + with no `effect:` template was permitted, and it still is on any grant without a budget. With one, + it is refused: there is nothing to charge against, so an agent proposing such actions would spend + nothing against every budget on the chain for ever. `SPEC-v0.9.md` §2.4.1 records the two probes + that moved this out of the loader. + +- **A metric value that is negative, missing, or not an integer is refused**, with `ACTION_DENIED` + and a `denied` receipt. Before: no metric existed. A negative amount would reduce the rolling sum + and refill the budget, which is the compensation `SPEC-v0.9.md` §12 forbids; a missing one + counted as zero would turn the absence of a field into unlimited authority. + +- **`ctrlrun verify` sizes its own action vector to a grant's budgets.** Before: it synthesized a + vector to land in a rule and reported a budget refusing that action as an internal error, exit 3, + on guarantees with nothing to do with budgets. Where no value fits a band, the guarantee is now + `N/A` with a reason that names the action and the grant. + +### Fixed + +- **`resolve_effect` released no budget hold.** It does not go through `_transition`, so a human + resolving an `AMBIGUOUS` effect `FAILED` held its charge for ever: the one act meant to free a + budget was the one path that did not. Fixed in all three backends, inside the same transaction as + the record's own write. + +- **A refused receipt claimed a charge it never made.** `budget_charges` was stamped where the + charges were computed, so a refusal raised later in the same loop reached the receipt with them + set, and a `denied` receipt asserted the action charged the very grant it was refused from + spending against. A receipt asserting a spend that never happened is the one thing an evidence + trail may not do. + +- **An oversized stored window crashed every evaluation in the deployment.** `Authority.evaluate` + reads every delegation row on every evaluation, and an unreadable window raised `OverflowError` + out of it, so one corrupt row denied nothing and crashed everything, for every principal and + every action, with no event and no receipt to find it by. It is `authority_unreadable` now. + ## [0.8.0] - 2026-09-12 - Oversight Every guarantee shipped before this one verifies the principal that **acts**. G7 refuses an action diff --git a/CITATION.cff b/CITATION.cff index 56370197..a5059c37 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -11,7 +11,7 @@ authors: - family-names: Ghoshal given-names: Arpan email: contact@arpanghoshal.com -version: 0.8.0 +version: 0.9.0 repository-code: https://github.com/CTRLRun/ctrlrun url: https://github.com/CTRLRun/ctrlrun license: Apache-2.0 diff --git a/README.md b/README.md index 09264958..c288dd4a 100644 --- a/README.md +++ b/README.md @@ -274,7 +274,7 @@ grades the transport classifier. | **One effect, once** — One logical effect happens at most once, across threads, processes and hosts. | yes | yes | yes | | **Unknown is not failed** — An unknown outcome is AMBIGUOUS, never FAILED, and blocks a blind retry. | yes | yes | yes | | **Fail closed** — An unknown action, a missing policy or a missing principal is denied. | yes | yes | yes | -| **Authority and delegation** — With authority on, every principal needs a grant, and delegation cannot widen one. | yes | yes | yes | +| **Authority and delegation** — Every principal needs a grant, delegation cannot widen one, and a grant bounds the total. | yes | yes | yes | | **Receipts** — Every executed action leaves a portable JSON receipt of who, what and outcome. | yes | yes | yes | diff --git a/adapters/langgraph/README.md b/adapters/langgraph/README.md index 9c15f11e..85e11ce2 100644 --- a/adapters/langgraph/README.md +++ b/adapters/langgraph/README.md @@ -3,7 +3,7 @@ Route a CTRLRun `APPROVE` through **LangGraph's own `interrupt()`**, so the human answers where your LangGraph users already answer. -- **Supported kernel range:** `ctrlrun>=0.5,<0.9` +- **Supported kernel range:** `ctrlrun>=0.5,<0.10` - **Supported framework range:** `langgraph>=1.0,<2.0` - **Primitive reused:** [`interrupt()` and `Command(resume=...)`](https://langchain-ai.github.io/langgraph/how-tos/human_in_the_loop/add-human-in-the-loop/), with a checkpointer. Read 2026-09-05. - **Framework shape:** resumed in place (SPEC-v0.5 §3.5). diff --git a/adapters/langgraph/pyproject.toml b/adapters/langgraph/pyproject.toml index 6a18c080..13c49a91 100644 --- a/adapters/langgraph/pyproject.toml +++ b/adapters/langgraph/pyproject.toml @@ -26,7 +26,7 @@ classifiers = [ # surface that has not been written. The README states the same two, and T137 asserts that what # it states is what CI installed. dependencies = [ - "ctrlrun>=0.5,<0.9", + "ctrlrun>=0.5,<0.10", "langgraph>=1.0,<2.0", ] diff --git a/adapters/langgraph/src/ctrlrun_langgraph/__init__.py b/adapters/langgraph/src/ctrlrun_langgraph/__init__.py index 8c74554e..d56d6fe0 100644 --- a/adapters/langgraph/src/ctrlrun_langgraph/__init__.py +++ b/adapters/langgraph/src/ctrlrun_langgraph/__init__.py @@ -15,7 +15,7 @@ buys one thing over it: the interrupt. If your graph has nowhere for a human to answer, or you are happy for `ApprovalRequired` to reach your own code, use `@protect` and stop here. -Supported kernel range: `ctrlrun>=0.5,<0.9`. Supported framework range: `langgraph>=1.0,<2.0`. +Supported kernel range: `ctrlrun>=0.5,<0.10`. Supported framework range: `langgraph>=1.0,<2.0`. `README.md` states both, and what this adapter's binding check is and is not. """ diff --git a/adapters/openai-agents/README.md b/adapters/openai-agents/README.md index 0fef2d23..7a1a5737 100644 --- a/adapters/openai-agents/README.md +++ b/adapters/openai-agents/README.md @@ -3,7 +3,7 @@ Route a CTRLRun `APPROVE` through the **OpenAI Agents SDK's own tool-approval interruption**, so the human answers where this SDK's users already answer. -- **Supported kernel range:** `ctrlrun>=0.5,<0.9` +- **Supported kernel range:** `ctrlrun>=0.5,<0.10` - **Supported framework range:** `openai-agents>=0.20,<1.0` - **Primitive reused:** [`needs_approval`, `RunResult.interruptions`, `RunState.approve` / `reject`](https://openai.github.io/openai-agents-python/tools/). Read 2026-09-05. - **Framework shape:** decided before invocation (SPEC-v0.5 §3.5). diff --git a/adapters/openai-agents/pyproject.toml b/adapters/openai-agents/pyproject.toml index 17281ac2..78887ddc 100644 --- a/adapters/openai-agents/pyproject.toml +++ b/adapters/openai-agents/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ # not before, so `>=0.5` would claim compatibility with a surface not yet written. T137 asserts # the README states these and that CI ran inside them. dependencies = [ - "ctrlrun>=0.5,<0.9", + "ctrlrun>=0.5,<0.10", "openai-agents>=0.20,<1.0", ] diff --git a/adapters/openai-agents/src/ctrlrun_openai_agents/__init__.py b/adapters/openai-agents/src/ctrlrun_openai_agents/__init__.py index b9885762..6bf6368f 100644 --- a/adapters/openai-agents/src/ctrlrun_openai_agents/__init__.py +++ b/adapters/openai-agents/src/ctrlrun_openai_agents/__init__.py @@ -15,7 +15,7 @@ **You probably do not need this.** `@protect` covers anything in this process with no adapter and no framework support. This buys the interrupt and nothing else. -Supported kernel range: `ctrlrun>=0.5,<0.9`. +Supported kernel range: `ctrlrun>=0.5,<0.10`. Supported framework range: `openai-agents>=0.20,<1.0`. `README.md` states both, and states why this adapter's binding is **attribution** where LangGraph's is prevention. diff --git a/docs/SPEC-v0.9.md b/docs/SPEC-v0.9.md index dc65d5a5..328ea89a 100644 --- a/docs/SPEC-v0.9.md +++ b/docs/SPEC-v0.9.md @@ -637,6 +637,7 @@ def consumptions( grant_id: str | None = None, metric: str | None = None, since: datetime | None = None, + effect_key: str | None = None, ) -> tuple[Consumption, ...]: ... ``` @@ -645,6 +646,14 @@ observable, and a required `grant_id` would make that require enumerating every existed, runtime delegations and revoked grants included, one call each. `Consumption`'s fields are frozen in §10. +**`effect_key` was added by item 5 and is recorded here rather than slipped in**, on §10's rule that +anything not in that section is a spec amendment before it is code. §8.3 makes the resumed leg's +receipt the only receipt an MCP multi round-trip or ACS action ever gets, so it has to report what +that action spent; a gateway that restarted mid-round has nothing in memory to report it from, and +the ledger is the record. Without the filter that read is a scan of the whole ledger per +resumption. The other three filters could not serve: the question is "what did *this effect* spend", +and a resumption knows its effect key and not which grants a chain of ancestors charged. + **And the "why" of §7.2 is a join, not a column.** `get_effect(effect_key)` is already declared (`state.py:563`), so an un-released row's holding state is read by joining its `effect_key` through it. Said explicitly because the alternative an implementer reaches for is putting the effect's state @@ -824,8 +833,32 @@ this project documents is observe-then-enforce: an operator would hit a limit du entire purpose is to hit nothing. **What it does instead**: the observe report says the action *would have been* refused on a budget, -naming the grant and the metric, exactly as it reports what a policy would have decided. That is -worth more than a charge, because it is how an operator sizes a budget before turning it on. +naming the grant and the metric, exactly as it reports what a policy would have decided. The +predicate is `check_charges`, the same function all three stores enforce with, so the report and the +enforcement cannot drift: a pilot that says "this would have been fine" about an action enforce mode +refuses is worse than no pilot. + +§2.3's and §2.4.1's refusals are reported the same way, under their own reasons. Enforce mode +refuses those actions, so saying so is what observe mode is for, and an operator needs to know +whether the budget is too small or the action cannot be measured at all. Observe mode writes no +`denied` receipt for them: it records what enforce mode would have done and refuses nothing. + +### 4.2.1a What observe mode cannot tell you about a budget + +An earlier draft of §4.2.1 ended "that is how an operator sizes a budget before turning it on." +**That is not true, and the limit is worth stating rather than discovering.** + +Observe mode charges nothing, so the ledger it evaluates against is only ever filled by enforce-mode +runs. A deployment observing *every* action has an empty ledger, every predicate passes, and the +report says no budget would have refused anything, no matter how much the agent proposed to spend. +The report is informative in a **mixed** deployment, where a new action is piloted in observe mode +against a grant other actions are already enforcing, and that is the shape §4.2.1's test drives. + +Sizing a budget from an observed run needs the counterfactual spend, which observe mode does write: +every `observed` receipt carries `budget_charges`, what the action *would have* been charged. Adding +those up over a window is the sizing question, and it is a question for a reporting surface over +receipts rather than for the kernel's hot path. The kernel's job here is the honest report of what +enforcement would have done against the state that exists. ### 4.3 One effect key holds at most one charge at a time @@ -1618,7 +1651,7 @@ third-party backend already imports `StateStore` from that module. | `task=` on `@protect` and `Control.execute` | nothing carries a unit of work today, and it cannot go on `Action` without moving every action hash in existence (§6.3.1) | | `task=` on `Authority.evaluate` | **amends a signature frozen in `SPEC-v0.3.md` §11**, and §10.3 records the amendment rather than slipping it in. `Authority.evaluate(action, *, now, store)` is frozen there; the task has to reach the decision, and `v0.7 §6.2`'s context variables are request-time stamps in `approval.py`, not inputs to an authority decision | | `task=` on `Control.evaluate` | **also amends a frozen signature** (`SPEC-v0.3.md` §11: "its signature and `Evaluation`'s two fields are unchanged"). Required by §6.3.2: without it `Control.evaluate` and `Control.execute` disagree about a task-bound grant, and `ctrlrun.adapter.needs_approval` routes through `evaluate` | -| `consumptions()` on `StateStore` | §3.3.3: the surfaces and G22 need a read, and `charges=` is write-only. Rows come back **in insertion order**, which is deterministic and identical across the three backends and is *not* a time ordering: host clock skew, which `v0.7 §3` models, inverts `consumed_at` against the id | +| `consumptions()` on `StateStore` | §3.3.3: the surfaces and G22 need a read, and `charges=` is write-only. Rows come back **in insertion order**, which is deterministic and identical across the three backends and is *not* a time ordering: host clock skew, which `v0.7 §3` models, inverts `consumed_at` against the id. Its `effect_key` filter was added by item 5 for §8.3's resumed receipt, and §3.3.3 records why the other three could not serve | | `check_charges` | §3.3.1's predicate in one place, so three backends cannot drift on the arithmetic. Public for the same reason `plan_reservation` is: a third-party store decides with it rather than reimplementing it | | `Consumption` | what `consumptions()` returns: `grant_id`, `metric`, `amount`, `effect_key`, `attempt`, `consumed_at`, `released_at`. Frozen here because §3.3.3 returns it and §7.2 renders it, and a return type specified nowhere is a spec amendment waiting to happen | @@ -1636,13 +1669,33 @@ shape is frozen here before any of them starts: |---|---|---| | `task` | item 1 | the task the action was bound to, or absent | | `scope_hash` | item 2 | `sha256:…` over the returned scope, never its content (§5.5) | -| `budget_charges` | item 5 | which grants were charged, which metrics, how much | +| `budget_charges` | item 5 | which grants were charged, which metrics, how much. On an `observed` receipt it is the counterfactual charge and not a spend (§4.2.1a), which `result` tells apart | Item 7 asserts every one of them is written by something before the release PR opens. This is `SPEC-v0.7.md` §12's D27 rule, which v0.8 ran for three items without incident. **`ctrlrun.guarantees/v5`** is G1 to G24, moved once by item 1 with G24 (§8). +**`ctrlrun.budget/v1`**, added by **item 6** for §7.2, and recorded here rather than slipped in. +Its own document rather than a key inside `ctrlrun.inspection/v2`, because that one answers about an +**action** and this answers about a **grant**: a reader handed one would have to know which of two +shapes it got before it could read either. §7.1's "no new command" is the surface question and a +separate one, and `ctrlrun inspect --grant` keeps it. + +| Key | Holds | +|---|---| +| `grant_id` | the grant asked about, document grant or runtime delegation alike | +| `at` | the instant the three numbers were read, because every one of them is a rolling-window answer and stale without it | +| `budgets[].metric`, `.limit`, `.window_seconds` | the budget as declared. Seconds, for `_canonical_grant`'s reason in §10 | +| `budgets[].consumed` | the un-released sum over the window: **the number that decides** (§3.3.1) | +| `budgets[].held` | the part of it whose effects have not reached `COMMITTED` (§7.2) | +| `budgets[].holding[]` | `effect_key`, `state`, `amount`, `attempt`, `consumed_at`: §7.2's "why", one entry per held charge. `state` is `null` where the effect record is gone, which §7.3's archiving paragraph permits | + +`ctrlrun.stats/v1` gains `ledger_rows` **additively** and does not move, per §7.3. It is omitted +rather than reported as `0` on a store with no ledger: "no rows" and "this store predates budgets" +are different facts, and a diagnostic that conflates them sends an operator looking for spend that +was never possible. + ### 10.3 Two frozen signatures amended, recorded as `v0.3 §11` requires `SPEC-v0.3.md` §11 freezes `Authority.evaluate(action, *, now, store)` and says of `Control.evaluate` @@ -1744,3 +1797,168 @@ The four open questions this document hands the items are O1 to O4 in the build answered here in the section that carries it: O1 in §3.3, O2 in §4.5, O3 in §5.2 and §5.7, O4 in §6.5. An item that finds one of those answers wrong stops and reports rather than working around it, because all four are load-bearing for a section rather than local to a function. + +### 13.0 What the milestone settled about itself + +**Every design claim this document made about its own code had to be probed, and the ones that were +not were wrong about a third of the time.** Three adversarial review rounds ran against the drafted +spec. Across them, *every citation* they made was accurate, and roughly three *design claims* per +round were false. The same ratio held for the items' own reasoning and for the reviews of the +finished code: the accurate half is always "this line says X", and the unreliable half is always +"therefore Y happens at runtime". + +The rule the milestone adopted after the second round, and which §1's standing instructions now open +with: **probe before you assert.** A ten-line script against the real objects settles in a minute +what cross-module reasoning gets wrong one time in three. Almost every entry below was found by one. + +The second-order version of the same lesson, which cost more: a *test* asserting a runtime claim is +itself a claim, and a green test is not evidence that the claim is load-bearing. Items 2, 4 and 5 +each shipped guards that nothing exercised, found only by mutating the source and watching the suite +stay green. §9's mutation table is the milestone's answer, and it earned its place. + +### 13.1 Item 1: task-bound authority + +**A template that cannot resolve must be refused *inside* the recording path.** `@protect(task= +"{run_id}")` with no such argument raised `EffectKeyError` outside every recorder: no +`ACTION_PROPOSED`, no `ACTION_DENIED`, no `denied` receipt, and a caller holding a template error +about an action nothing recorded. Probed rather than reasoned about, which is the entry: 0 events +and 0 receipts before, `ACTION_PROPOSED` + `ACTION_DENIED` and one `denied` receipt after. + +`_resolve_effect`'s body became `_resolve_template` over either template, so the effect template and +the task template cannot drift into recording different things for the same class of mistake. The +same shape had been found once before on a store refusal escaping `_secure`'s except clauses, and +`control.py`'s own comment records it; §6.3 carries the rule. + +### 13.2 Item 2: scope providers + +**Two refusals need two reasons, and observe mode is where that stops being pedantry.** +`_ObservedRefusedError` carried no reason, so `_observe_secure` blocked with a hardcoded +`out_of_scope` for both cases, and a deployment whose scope *source* was down read a counterfactual +saying the record was not the principal's. Observe mode exists to tell an operator what enforce mode +would do; reporting the wrong category is the one way it can be worse than useless. §5.6 carries the +two reasons, and the same argument recurred one dimension over in item 5 (§13.5). + +**Three guards were green against a mutated kernel**, all of `CONTRIBUTING.md`'s first shape. The +canonicalizer was removable because the shape guard refused the malformed input first and the hash +never had to; the mapping guard was removable because a list reaches `dict()` inside the hash and +raises there anyway; and observe mode had no scope test at all, so collapsing its two refusal paths +into one was invisible. A subsumed branch may be kept for its message, on the condition that a test +asserts which message it got, and those tests now do. + +### 13.3 Item 3: the budget in the document + +**§2.6's window axis was inverted in the drafted spec, and the review's exhaustive check is what +established the right one.** 12,871 transitivity triples with zero non-transitive, and 2,515 +contained pairs against 300 simulated spend timelines each, with zero soundness violations: +`child.window >= parent.window` is correct, because a *shorter* window is a higher rate and therefore +more authority. The draft's `<=` would have accepted a child at 24× its parent's authority. + +**One corrupt delegation row could take down an entire deployment.** An oversized stored window +raised `OverflowError` out of `Authority.evaluate`, and `_candidates` reads *every* delegation row on +*every* evaluation, so one bad row denied nothing and crashed everything, for every principal and +every action, with no event and no receipt to find it by. `OverflowError` was in neither except +tuple. §2.4 carries the bound and the refusal; an unrelated principal's unrelated action now gets +`authority_unreadable`. + +**`timedelta(seconds=True)` is a one-second window.** `_is_int` exists in this item precisely +because `isinstance(True, int)`, and it guarded `limit` while `window` went through a cast one field +away. The failure *grants* authority, and the loader refuses the same input, so it was a direct +violation of §2.2's "the model refuses exactly what the loader refuses". A bool trap is not an edge +case in a codebase that already wrote the guard once and stopped one field short. + +### 13.4 Item 4: the ledger + +**The charge was dropped on both lost-`COMMIT` re-issue branches, which falsified §3.3's own +stronger bar for touching a frozen protocol.** `_resolve_lost_insert` and `_resolve_lost_renewal` +call `_authorize_and_reserve` again after an ambiguous `COMMIT` and did not forward `charges` -- +neither resolver even took them. The retried transaction re-inserted the reservation and nothing +else. Probed: `reserved=True charged=0`, and against a budget permitting one spend, driven ten +times, 10 reservations and 0 ledger rows with a real spend of 1000 against a limit of 100. + +That is `reserved=1, charged=0`: the exact state §3.3.0's throwaway spike named as *disqualifying* +the alternative design, reproduced inside the chosen one. §3.3's second bar is that one re-read +resolves the reservation and the charge together, and it was not met until this was fixed. + +**A concurrency test without a barrier proves nothing, and the milestone learned it twice.** The +item's own race test held 4/4 against a deliberately unlocked implementation: interpreter startup, +importing the package, the connection and the migration check all happen before the contended work +and vary by more than it does, so the processes ran one after another. With a `multiprocessing +.Barrier` the unlocked version spends 2400 against a limit of 1000. A v0.7 test, T247, then flaked +twice on CI in one session with its own guard reporting "nothing was contended" -- the same defect, +in a test written a milestone earlier, fixed the same way. + +### 13.5 Item 5: consumption, holds and releases + +**§4.2's rule is keyed on the state *reached*, never on the call that tried to reach it**, and the +mutation that proves it matters survived the entire suite before item 5's second review. Moving the +release above the state check makes a *refused* `fail_effect` release the hold on an `AMBIGUOUS` +record, which is a manufacturable refund and the thing the item exists to stop. It is also **not** +an equivalent mutant only in the in-memory store: both SQL backends run the transition in one +transaction and roll it back when the check raises, so there the order is redundant with the +rollback, while the in-memory store mutates a dict under a lock and the order *is* the atomicity. +The code says so now, because a future maintainer reading two of the three backends would conclude +the ordering is arbitrary. + +**A refusal nobody can see is not a refusal.** §2.3's and §2.4.1's guards escaped as a bare +`InvalidArgument` with no `ACTION_DENIED` and no receipt, leaving the one record an operator has of +a refused action empty. They also ran *after* the approval gate, so a human could be asked to +approve a refund the kernel had already decided to refuse, and a granted approval was left behind +for an action nothing could execute. Both halves are §2.3's now; the exception type is unchanged, +because neither is a budget running out. + +**A duplicate-charge guard killed §2.2's own motivating shape.** "Two budgets on one metric over two +windows is the first thing an operator asks for", and it arrives as two charges differing only in +`limit` and `window`. A guard refusing *any* duplicate `(grant_id, metric)` pair meant the loader +accepted the document, observe mode reported it clean, `ctrlrun verify` could not grade it, and +enforce mode died with no receipt at all. What the guard is actually for is two charges on one +metric carrying **different amounts**, which §3.4's key would silently collapse. §3.3.1 carries both +sentences. + +**The resumed leg was reporting a number from a context variable.** §8.3 makes that receipt the only +one an MCP multi round-trip or ACS action ever gets, and `resume` never reset the variable: a +restarted gateway reported no charges for an action that spent, and a gateway that had run another +action since reported *that* action's spend. The ledger is the record, and it is read by effect key +and attempt (§3.3.3). The test that caught the first half had to be written to run in a fresh +`contextvars.Context`, because a test reusing the caller's context is testing the case that works. + +**§4.2.1 overclaimed, and the fix was to make the claim true rather than to soften it.** The draft +said observe mode is "how an operator sizes a budget before turning it on". It is not: observe mode +charges nothing, so a deployment observing every action has an empty ledger and the report says no +budget would refuse anything, however much the agent proposes. §4.2.1a states that limit, and +observed receipts now carry the counterfactual charge -- which a probe found to be an empty tuple, +making the sizing path the section described impossible. + +### 13.6 Item 6: the operator surfaces + +**One word for two numbers makes two commands disagree.** §7.2 defines *held* as the part of the +consumed sum whose effects have not committed, so `ctrlrun effects` says **spent** for a committed +effect and **holds** for every other. A committed charge is never released, because a committed +spend is a spend, and calling that a hold would have `effects` and `inspect --grant` reporting +different things under the same word. + +**One document has one producer, and §7.3's row count proved it again.** Adding `ledger_rows` to +`ctrlrun stats` in the CLI alone left the operator MCP server returning a different shape under the +same `ctrlrun.stats/v1` name. `SPEC-mcp-operator §9.1` exists for this and its test caught it +immediately; the count moved into `ctrlrun.reporting` beside every other shared shape. + +**A ledger row whose effect record is gone is reported held, not skipped.** §7.3 permits an operator +to archive rows the window can no longer reach, so a store whose effects were pruned but whose +ledger was not is reachable. Skipping such a row would **under**-report `held`, which is the one +direction this view may not err in, because it is the direction that hides a hold from the person +looking for it. + +### 13.7 What `ctrlrun verify` settled + +**A budget is a configuration fact, and never a defect in verify.** A grant whose budget is smaller +than the vector `_synthesize` picked refuses the action before the guarantee is reached, and verify +reported that as an internal error, exit 3, on guarantees with nothing to do with budgets. This is +the third time the same shape has been found: a shipped example declaring `approvals_required: 2` +did it, then one declaring an approver role did it, and both were fixed by having verify supply what +the document needs. A budget gets the same answer, and verify now sizes its own action vector. + +Where no value fits, the guarantee is `N/A` -- and that needed its own reason rather than falling +through to the grant miss, which told an operator that no grant's `resources:` matched, about a +document whose patterns matched perfectly. A budget smaller than any single action in a band makes +that band unreachable, which is worth saying in those words. §7.4's shipped example keeps a budget +large enough that its own approve band is reachable, because a daily budget smaller than one +permitted action is legal and almost always a mistake. diff --git a/examples/authority/payments.yaml b/examples/authority/payments.yaml index 73c44ef9..e54bc0f2 100644 --- a/examples/authority/payments.yaml +++ b/examples/authority/payments.yaml @@ -41,13 +41,18 @@ authority: # the refund runs it was granted for. `ctrlrun verify` grades G24 against this grant. tasks: ["refund-run:*"] # SPEC-v0.9 §2. How much, over which metric, in how long. `amount` names an action - # argument and is summed; the limit is in minor units like every band above, so this is - # €1,000.00 of refunds in any rolling 24 hours, across this grant and every delegation - # beneath it (§2.7). A grant that omits `budgets:` budgets nothing, which is what the two - # grants below do and why they are unchanged. Nothing sums a metric yet: the ledger and - # the spending are later items, and `ctrlrun verify` grades G22 against this once they land. + # argument and is summed; the limit is in minor units like every band above. + # + # **A budget bounds the aggregate; the constraint above bounds one action.** They are + # different questions and the numbers should say so: `amount_lte` lets a single refund + # reach €100,000, and this lets the day's refunds reach €500,000 across this grant and + # every delegation beneath it (§2.7). A daily budget *smaller* than one permitted action + # is legal and almost always a mistake, because the first refund of the day exhausts it. + # + # A grant that omits `budgets:` budgets nothing, which is what the two grants below do and + # why they are unchanged. `ctrlrun verify` grades G22 against this one. budgets: - - { metric: amount, limit: 100000, window: PT24H } # €1,000.00 a day + - { metric: amount, limit: 50000000, window: PT24H } # €500,000.00 a day # 2. A service account that reconciles, and may never move money. It holds no `delegable` # key at all, which is the default and means it cannot create a delegation. diff --git a/pyproject.toml b/pyproject.toml index 523d32eb..0f3f3914 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "ctrlrun" -version = "0.8.0" +version = "0.9.0" description = "The execution safety layer for AI agents." # Mirrors the repository's GitHub topics, so PyPI search and GitHub search agree. keywords = [ diff --git a/src/ctrlrun/acs.py b/src/ctrlrun/acs.py index c45c0d2d..a0df685d 100644 --- a/src/ctrlrun/acs.py +++ b/src/ctrlrun/acs.py @@ -27,7 +27,7 @@ from typing import Any, Final from .action import Action, Principal -from .control import Control, with_approval +from .control import Control, _UnmeasurableError, with_approval from .effect import resolve_effect_key, resolve_resource from .errors import ( ActionDenied, @@ -149,6 +149,19 @@ def handle( return self._on_request(rpc_id, request_id, params, headers or {}) if method == TOOL_CALL_RESULT: return self._on_result(rpc_id, request_id, params) + except _UnmeasurableError as refused: + # SPEC-v0.9 §2.3, §2.4.1. **Before the clause below, and answered as a decision**, + # because for this refusal there *is* an action: `Control` has already written + # `ACTION_DENIED` and a `denied` receipt for it. An independent review found the two + # disagreeing -- an error envelope says the Guardian could not answer, which a + # platform is free to act on however it likes, while the evidence said the kernel + # refused. That is precisely what the `IdentityError` clause below forbids. + # + # The code follows the refusal rather than being fixed here, for the same reason + # that clause gives: `budget_unmeasurable` and `budget_unkeyed` are different things + # to fix, and the receipt already names which. + _LOG.warning("refused an ACS envelope: %s", refused) + return _final(rpc_id, request_id, DENY, reasoning=str(refused), codes=[refused.reason]) except InvalidArgument as refused: # A malformed payload is not a decision about an action: there is no action. return _error(rpc_id, MALFORMED_ENVELOPE, str(refused)) diff --git a/src/ctrlrun/authority.py b/src/ctrlrun/authority.py index d37ea55f..e16c421b 100644 --- a/src/ctrlrun/authority.py +++ b/src/ctrlrun/authority.py @@ -46,7 +46,7 @@ strict_load, ) from .policy import _equal as _type_strict_equal -from .state import DelegationRecord, StateStore +from .state import Charge, DelegationRecord, StateStore #: SPEC-v0.3 §4.4 — an action name is dotted (`v0.1 §2.1`) and a resource is `type:id`. ACTION_SEPARATOR: Final = "." @@ -849,6 +849,43 @@ def _delegation_from_record(record: DelegationRecord) -> Delegation: ) +#: SPEC-v0.9 §2.3 — the one metric the kernel supplies, and the only one whose meaning does not +#: depend on the document. An action argument literally named `count` does not win it: an operator +#: writing `metric: count` means "how many", and a document that could retarget it would make the +#: one metric independent of the document depend on it. +COUNT_METRIC: Final = "count" + + +def _metric_value(action: Action, metric: str, grant_id: str) -> int: + """What this action spends on this metric (SPEC-v0.9 §2.3). + + `count` is one per action. Every other metric names an **action argument**, by name, and its + value is summed. **An action that does not carry the argument is refused, not treated as + zero**: treating a missing field as zero turns the absence of a value into unlimited + authority, which is the sentence `v0.3 §5.4` exists to refuse on the constraint side. + + The kernel does not know what any metric means. There is no branch here on a metric name + beyond `count`'s own source, no ranking of two metrics, and no default limit for a name the + kernel thinks it recognises (§12). + """ + if metric == COUNT_METRIC: + return 1 + value = action.canonical_arguments.get(metric) + if value is None: + raise InvalidArgument( + f"{action.name}: grant {grant_id!r} budgets {metric!r} and the action carries no " + f"{metric!r} argument. A missing value is refused, never counted as zero " + "(SPEC-v0.9 §2.3)" + ) + if not _is_int(value) or value < 0: + raise InvalidArgument( + f"{action.name}: grant {grant_id!r} budgets {metric!r} and the action's value is " + f"{value!r}; a metric value is a non-negative integer, so money is budgeted in minor " + "units (SPEC-v0.9 §2.3)" + ) + return int(value) + + def contained_dimension(parent: Grant, child: Grant) -> str | None: """The first §5.4 row `child` violates, or `None` where it is contained on every one. @@ -1245,6 +1282,50 @@ def evaluate( return min(failed[reason], key=_by_grant_id) return AuthorityResult(False, NO_AUTHORITY) + def _charges_for( + self, action: Action, result: AuthorityResult, *, store: StateStore + ) -> tuple[Charge, ...]: + """Every budget this action spends against, one `Charge` per ancestor (SPEC-v0.9 §2.7). + + **Package-internal**: §10 freezes `Charge` and `charges=`, not a way to obtain them, and + a public method here would be a surface nothing asked for. + + §2.7 is the rule that makes the feature mean anything. Without charging every ancestor, a + holder of a 100,000-a-day grant delegates ten correctly-contained children and spends + 1,000,000: every link individually valid, the total ten times what anybody granted. + + Returns `()` where the deciding grant and its chain budget nothing, which is every grant + written before v0.9 and why they all upgrade untouched (R5). + """ + if not result.passed or result.grant_id is None: + return () + charged: list[tuple[str, Grant]] = [] + delegation = None + if result.delegation_id is not None: + record = store.get_delegation(result.delegation_id) + delegation = None if record is None else _delegation_from_record(record) + if delegation is None: + grant = self._grants.get(result.grant_id) + if grant is not None: + charged.append((result.grant_id, grant)) + else: + walk = self._walk(delegation, store=store) + charged.append((delegation.delegation_id, delegation.grant)) + charged.extend(zip(walk.ancestor_ids, walk.ancestors, strict=True)) + made: list[Charge] = [] + for grant_id, grant in charged: + for budget in grant.budgets or (): + made.append( + Charge( + grant_id=grant_id, + metric=budget.metric, + amount=_metric_value(action, budget.metric, grant_id), + limit=budget.limit, + window=budget.window, + ) + ) + return tuple(made) + # --- delegation (SPEC-v0.3 §5) ----------------------------------------------------- def plan_break_glass( diff --git a/src/ctrlrun/cli/main.py b/src/ctrlrun/cli/main.py index e43918cc..74406aaf 100644 --- a/src/ctrlrun/cli/main.py +++ b/src/ctrlrun/cli/main.py @@ -22,7 +22,7 @@ from ..action import Principal from ..approval import ApprovalRecord, LocalApprovalProvider -from ..authority import Delegation, grant_from_yaml +from ..authority import Budget, Delegation, grant_from_json, grant_from_yaml from ..control import DEFAULT_STATE_DIR, Control, state_path from ..effect import RESOLVED_BY_HUMAN, EffectRecord, EffectState from ..errors import ( @@ -41,7 +41,14 @@ iso_timestamp, verify_chain, ) -from ..reporting import inspection_for, since_boundary, stats_document +from ..reporting import ( + budget_document, + budget_lines, + inspection_for, + ledger_rows, + since_boundary, + stats_document, +) from ..state import RESOLUTIONS, DelegationRecord, SQLiteStateStore, StateStore from .demo import run_demo @@ -520,16 +527,56 @@ def _report_chain(store: StateStore, *, as_json: bool) -> None: ) @STORE_URL_OPTION def effects(state: str | None, store_url: str | None) -> None: - """Show the logical effects this store knows about.""" + """Show the logical effects this store knows about. + + An effect that still **holds** part of a budget says so (SPEC-v0.9 §7.2): `--state ambiguous` + is how an operator finds what is pinning a grant, and the hold is the reason it matters. + """ try: - found = _store(store_url).list_effects(None if state is None else EffectState(state)) + store = _store(store_url) + found = store.list_effects(None if state is None else EffectState(state)) + holds = _holds_by_effect(store) except CTRLRunError as exc: raise _fail(exc) from exc if not found: click.echo("no effects yet" if state is None else f"no effects are {state}") return for record in found: - click.echo(_effect_line(record)) + line = _effect_line(record) + charges = holds.get(record.effect_key) + if charges: + # **"spent" for a committed effect, "holds" for every other.** §7.2 defines `held` + # as the part of the sum whose effects have not committed, and a committed spend is + # a spend (§4.2): calling it a hold here would have `ctrlrun effects` and + # `ctrlrun inspect --grant` use one word for two different numbers. + verb = "spent" if record.state is EffectState.COMMITTED else "holds" + line += f" {verb} " + ", ".join(charges) + click.echo(line) + + +def _holds_by_effect(store: StateStore) -> dict[str, list[str]]: + """Un-released charges per effect key, whatever state the effect is in (SPEC-v0.9 §7.2). + + "Un-released" is not "held": a committed effect's charge is never released, because a + committed spend is a spend. The caller picks the word from the effect's own state. + + Read once and indexed rather than queried per effect: `ctrlrun effects` lists every effect + in the store, and a lookup inside that loop is one query per row. + + A store with no ledger returns nothing, so this stays a diagnostic that works against a + 0.8.0 database rather than one that refuses it. + """ + try: + rows = store.consumptions() + except CTRLRunError: + return {} + held: dict[str, list[str]] = {} + for row in rows: + if row.released_at is None: + held.setdefault(row.effect_key, []).append( + f"{row.amount} {row.metric} on {row.grant_id}" + ) + return held @main.command() @@ -568,11 +615,36 @@ def resolve(effect_key: str, committed: bool, failed: bool, store_url: str | Non @main.command() -@click.argument("action_id") +@click.argument("action_id", required=False) +@click.option( + "--grant", + "grant_id", + help="Show this grant's budgets instead: consumed, held, and what holds it.", +) @click.option("--json", "as_json", is_flag=True, help="Emit one JSON object instead.") @STORE_URL_OPTION -def inspect(action_id: str, as_json: bool, store_url: str | None) -> None: - """Show one action's whole history: proposal, decision, approval, effect, receipt.""" +def inspect( + action_id: str | None, grant_id: str | None, as_json: bool, store_url: str | None +) -> None: + """Show one action's whole history: proposal, decision, approval, effect, receipt. + + With `--grant`, show that grant's budgets instead: how much of each is consumed over its + rolling window, how much of that is **held** by effects that have not committed, and which + effect holds each part (SPEC-v0.9 §7.2). + + The third number is the one that matters at 3am. A budget that refuses while it looks + nowhere near its limit is almost always one unresolved effect: `ctrlrun resolve` clears it. + """ + if (action_id is None) == (grant_id is None): + # §7.1 keeps both behind one command, which makes "which of the two did you mean" this + # command's own question. Neither names a subject; both name two. + raise click.UsageError( + "give an ACTION_ID, or --grant GRANT_ID, and not both: they inspect different things" + ) + if grant_id is not None: + _inspect_grant(grant_id, as_json, store_url) + return + assert action_id is not None store = _store(store_url) try: # SPEC-mcp-operator §9.1 — one producer for `ctrlrun.inspection/v2`, choosing included, @@ -600,6 +672,51 @@ def inspect(action_id: str, as_json: bool, store_url: str | None) -> None: click.echo(line) +def _inspect_grant(grant_id: str, as_json: bool, store_url: str | None) -> None: + """SPEC-v0.9 §7.2, behind `ctrlrun inspect --grant`. + + The grant's budgets come from the **authority in force**, document grants and runtime + delegations alike, because a delegation carries budgets of its own (§2.6) and an operator + paged about one needs the same three numbers. The ledger is keyed on the grant id either + way, so the read below does not care which kind it found. + """ + try: + control = _control_on(store_url) + budgets = _budgets_of(control, grant_id) + if budgets is None: + # Exits non-zero with nothing on stdout, as `inspect` does for an unknown action, so + # a script cannot mistake "no such grant" for "a grant with no budgets". + raise click.ClickException(f"no grant {grant_id}") + document = budget_document(grant_id, budgets, control._store, control._clock()) + except CTRLRunError as exc: + raise _fail(exc) from exc + if as_json: + click.echo(json.dumps(document, ensure_ascii=False, indent=2)) + return + for line in budget_lines(document): + click.echo(line) + + +def _budgets_of(control: Control, grant_id: str) -> tuple[Budget, ...] | None: + """This grant's budgets, or `None` where no such grant exists. + + `()` and `None` are different answers and the caller treats them differently: a grant that + budgets nothing is a real grant an operator may ask about, and §7.2's view says so. + """ + authority = control._authority + if authority is None: + return None + grant = authority.grants.get(grant_id) + if grant is not None: + return grant.budgets or () + record = control._store.get_delegation(grant_id) + if record is None: + return None + # Stored as JSON, so it is read back through the loader that validates it rather than + # trusted: §2.4's refusals apply to a row a text editor could have written. + return grant_from_json(record.grant_json, delegation_id=grant_id).budgets or () + + def _approvals_for( store: StateStore, receipt: Receipt | None, events: tuple[Event, ...] ) -> tuple[ApprovalRecord, ...]: @@ -773,7 +890,12 @@ def stats(since: str | None, as_json: bool, store_url: str | None) -> None: ] except CTRLRunError as exc: raise _fail(exc) from exc - document = stats_document(counted, mode=policy.mode, boundary=boundary) + document = stats_document( + counted, + mode=policy.mode, + boundary=boundary, + ledger_rows=ledger_rows(_store(store_url)), + ) if as_json: click.echo(json.dumps(document, ensure_ascii=False, indent=2)) return @@ -796,6 +918,9 @@ def _stats_lines(document: Mapping[str, Any]) -> list[str]: lines.append(_stat("denied", document["denied"])) lines += _breakdown(document["denied_by_reason"]) lines.append(_stat("ambiguous outcomes", document["ambiguous_outcomes"])) + if "ledger_rows" in document: + # §7.3: growth is observable before it is a problem. + lines.append(_stat("budget ledger rows", document["ledger_rows"])) lines.append("") if document["mode"] != OBSERVE: # §6.4 — say what is missing rather than print a line the receipts cannot substantiate. diff --git a/src/ctrlrun/control.py b/src/ctrlrun/control.py index ad205417..2691413b 100644 --- a/src/ctrlrun/control.py +++ b/src/ctrlrun/control.py @@ -124,7 +124,14 @@ iso_timestamp, new_receipt_id, ) -from .state import ClockSkew, SQLiteStateStore, StateStore +from .state import ( + BudgetExhaustedError, + Charge, + ClockSkew, + SQLiteStateStore, + StateStore, + check_charges, +) _LOG = logging.getLogger(__name__) @@ -219,6 +226,14 @@ class _Invocation: #: beside it: a refusal whose receipt carried the previous action's scope would be the same #: stale-evidence defect on a new field. _SCOPE_HASH: ContextVar[str | None] = ContextVar("ctrlrun_scope_hash") +#: SPEC-v0.9 §2.7 — the authority decision this action was allowed by, so `_secure` can assemble +#: the charges without re-walking the chain. Set beside `_AUTHORITY_GRANT_ID` and reset with it, +#: for the stale-evidence reason that field's own comment gives. +_AUTHORITY_RESULT: ContextVar[AuthorityResult | None] = ContextVar("ctrlrun_authority_result") +#: SPEC-v0.9 §10.1 — what this action charged, for the receipt. Set where the charges are +#: assembled and reset beside the other two, for the stale-evidence reason `_AUTHORITY_GRANT_ID`'s +#: own comment gives. +_BUDGET_CHARGES: ContextVar[tuple[Mapping[str, Any], ...]] = ContextVar("ctrlrun_budget_charges") #: SPEC-v0.8 §8.2.1, §11.2. How the policy-change flow says that this `ctrlrun.policy.change` #: is one it built. **Package-internal on purpose**, beside `_granting_principal`, and it @@ -534,6 +549,31 @@ def __init__(self, denial: ActionDenied) -> None: self.denial = denial +class _UnmeasurableError(InvalidArgument): + """SPEC-v0.9 §2.3 and §2.4.1's refusal, **carrying its reason to a protocol boundary**. + + An `InvalidArgument` subclass and not a new type, because §2.3 pins that exception and a + caller's `except InvalidArgument` must keep working. What it adds is `reason`, which the + boundaries need and could not get from a message. + + An independent review found why that matters. `gateway/server.py`'s `_through_control` + catches eight exception types and not `InvalidArgument`, so this refusal raised out of the + request handler and the **socket closed with no response** -- the one failure that file's own + comment calls the thing this library exists to prevent. And `acs.py` answered + `-32002 malformed envelope`, whose comment reads "there is no action", for an action it had + just written an `ACTION_DENIED` and a `denied` receipt for; the `IdentityError` clause + directly below it states the rule that breaks, that an answer and the evidence may not + disagree about the same action. + + `_refuse_unmeasurable` has already written the events and the receipt, so this carries only + what a boundary needs to answer with. + """ + + def __init__(self, message: str, *, reason: str) -> None: + super().__init__(message) + self.reason = reason + + class _ObservedRefusalError(Exception): """SPEC-v0.9 §5.2.2 — observe mode's would-have-refused, which escapes `_in_scope` and is swallowed by `_observe_secure`. Package-internal and never public: it is control flow, not a @@ -554,6 +594,18 @@ def __init__(self, reason: str) -> None: #: SPEC-v0.9 §5.6 — the two refusal reasons, distinct because a test asserting only the exception #: type cannot tell which guard fired. `scope_unavailable` is "the provider could not answer"; #: `out_of_scope` is "it answered, and the record is not this principal's". G23 is the first. +#: SPEC-v0.9 §4.5 — its own reason, because an exhausted budget, an out-of-scope record and a +#: failing scope provider all deny the same action with the same exception type. +BUDGET_EXHAUSTED: Final = "budget_exhausted" + +#: SPEC-v0.9 §2.3 and §2.4.1 refuse before anything is charged, and an independent review found +#: both escaping as a bare `InvalidArgument`: no `ACTION_DENIED`, no receipt, nothing in the one +#: record an operator has of a refused action. They keep that exception type, because neither is +#: a budget running out, but they are refusals and they are recorded as refusals. Two reasons +#: rather than one: an operator who declared a budget on an action with no `effect:` template has +#: a different thing to fix than one whose agent proposed a negative amount. +BUDGET_UNMEASURABLE: Final = "budget_unmeasurable" +BUDGET_UNKEYED: Final = "budget_unkeyed" SCOPE_UNAVAILABLE: Final = "scope_unavailable" OUT_OF_SCOPE: Final = "out_of_scope" @@ -912,6 +964,7 @@ def _authority_result( _TASK.set(task) if self._authority is None: _AUTHORITY_GRANT_ID.set(None) + _AUTHORITY_RESULT.set(None) return None result = self._authority.evaluate( action, @@ -924,6 +977,7 @@ def _authority_result( # is the only thing this field is read on. §4.6's `min` already picked which grant of # several decided, so this is that decision and not a guess about it. _AUTHORITY_GRANT_ID.set(result.grant_id if result.passed else None) + _AUTHORITY_RESULT.set(result if result.passed else None) return result def _authority_data(self, result: AuthorityResult) -> dict[str, Any]: @@ -1165,11 +1219,13 @@ def execute( # a copy of the context at creation, so a task started after a break-glass action # carried that id into an unrelated refusal too. _AUTHORITY_GRANT_ID.set(None) + _AUTHORITY_RESULT.set(None) # SPEC-v0.9 §6.3.1 — reset beside it, for the reason the comment above gives about a # stale grant id: a refusal whose receipt carried the previous action's task would be the # same defect on a new field. _TASK.set(None) _SCOPE_HASH.set(None) + _BUDGET_CHARGES.set(()) # SPEC-v0.3 §6.2 — the counterfactual for an observed run, or `None` in enforce mode. # Every branch below reads it to choose between refusing and recording. observation = _Observation() if self._observing else None @@ -1543,10 +1599,20 @@ def _observe_secure( # refusal is recorded and not raised. `v0.3 §6.2`: observe mode records rather than # enforces, and a check that enforced here would refuse during the phase whose entire # purpose is to refuse nothing. + # SPEC-v0.9 §4.2.1 — **above the scope check, because `_secure` computes charges before + # calling `_in_scope`.** An action that is both out of scope and unmeasurable was refused + # `budget_unmeasurable` by enforce mode and reported `out_of_scope` by the pilot. T458. + charges = self._observe_charges(action, effect_key, observation) try: self._in_scope(action, scope, scoped, enforcing=False) except _ObservedRefusalError as would: observation.block(would.reason) + # The charges above were computed before the scope check, where `_secure` computes them: + # §2.3's and §2.4.1's refusals do not depend on anything the approval gate produces and + # are unconditional, so T446 moved them above it in enforce mode; observe mode's copy + # stayed below and told a pilot a human would have been asked about an action enforce + # refuses before anybody is asked. That is T446's own defect on the other side of the + # mode switch, and an independent review found it. T451. approval_id = None if evaluation.decision is Decision.APPROVE: approval_id = _PRESENTED_APPROVAL.get(None) @@ -1627,6 +1693,13 @@ def _observe_secure( effect_key, approval=approval, ) + # SPEC-v0.9 §4.2.1 — **below the take, because the store decides in that order.** + # `plan_reservation` runs before `check_charges` (§3.3), so an effect that is already + # committed raises `DuplicateEffect` and the budget is never consulted. Reporting the + # budget first told an operator to raise a limit when the real answer was that the effect + # had already happened. The clauses above have returned by now on every refusal enforce + # mode would have hit first, so what reaches here is what the budget would decide. T452. + self._observe_spend(action, charges, observation) return approval, reservation def _observe_take( @@ -1724,6 +1797,17 @@ def resume(self, continuation: str, executor: Callable[[], Any]) -> Receipt: held = self._store.take_continuation(continuation) action = held.action started_at, approval, compared = self._resumed_context(action, held.record.created_at) + # SPEC-v0.9 §10.1, read from the **ledger** rather than from the contextvar. §8.3 makes + # this the only receipt an MCP multi round-trip or ACS action ever gets, so it has to + # report what the action spent, and the two ways to get that wrong are both live: a + # gateway that restarted mid-round has an empty contextvar and would report no charges + # for an action that spent, and a gateway that ran another action in this context since + # the suspension would report *that* action's spend. The ledger is the record; the first + # leg wrote it inside the reservation's own transaction. T447, T448. + # In observe mode the ledger is empty by design, so it is recomputed below, after the + # authority result this needs exists. §4.2.1a, T457. + if not self._observing: + self._resumed_charges(held.effect_key, held.record.attempt) # SPEC-v0.3 §2.5 — a continuation is a store-wide token, so a Control in another # environment can reach one. Evaluating a staging action inside a production # deployment is the fail-open §2.5 exists to close. @@ -1756,6 +1840,16 @@ def resume(self, continuation: str, executor: Callable[[], Any]) -> Receipt: EventType.AUTHORITY_DENIED, action, self._authority_data(result), held.effect_key ) evaluation = Evaluation(Decision.DENY, result.reason) + if self._observing: + # SPEC-v0.9 §4.2.1a — **the counterfactual, recomputed.** Observe mode charges + # nothing, so the ledger read above has nothing to find, and §8.3 makes this the only + # receipt an MCP multi round-trip or ACS action ever gets. Without this, §4.2.1a's + # sizing sum silently under-counts exactly the deployments §8.3 is about, which is an + # independent review's finding and a contradiction between two sections of the spec. + # + # Enforce mode keeps the ledger read: there the row is evidence of a spend that + # happened, and a recomputed number would be a claim about it instead. + self._observe_charges(action, held.effect_key, _Observation()) # SPEC-v0.3 §6.3 — a resumption in observe mode gets the same `observed` receipt its # first leg did. It is the *only* receipt an MCP multi round-trip ever gets (§8.3), so # a resumed leg reporting `committed` under a mode that enforces nothing would put the @@ -2207,6 +2301,20 @@ def _secure( may take twice, once more after a `reconcile` hook moves an `AMBIGUOUS` record, and the hook is a network call whose duration would otherwise sit inside the window. """ + # SPEC-v0.9 §2.7 — every ancestor charged, assembled once and passed to both passes so a + # reconcile between them cannot change what this action spends. + # + # **Before the keyless early return below**, because §2.4.1's refusal is exactly about an + # action that reaches it: a budgeted grant whose action resolved no effect key spends + # nothing against every budget on the chain, for ever, and returning early would be the + # kernel declining to notice. + # + # **And before the approval gate**, because §2.3's and §2.4.1's refusals depend on nothing + # the gate produces and are unconditional: the action cannot run whatever a human says. + # Assembling after the gate asks a human to approve a refund the kernel has already + # decided to refuse, and leaves a granted approval behind for an action nothing can + # execute. A probe found `APPROVAL_REQUESTED` written for exactly that shape. T446. + charges = self._charges_for(action, effect_key) approval_id = ( self._presented(action, effect_key, evaluation, started_at, preconditions) if evaluation.decision is Decision.APPROVE @@ -2242,8 +2350,31 @@ def _secure( # inserted here widens it, and a later item adding a check on this path # (the attempt ceiling, §5.5) belongs before this line or after `_take`. self._recheck(action, approval_id, preconditions, compared) - approval, reservation = self._take(action, approval_id, effect_key, lease) + approval, reservation = self._take(action, approval_id, effect_key, lease, charges) + # SPEC-v0.9 §10.1 — **after the store call took**, and an independent review is + # why. Set where the charges were computed, a refusal raised later in this loop + # still reached `_record` with them stamped, so a `denied` receipt claimed the + # action charged the very grant it was refused from spending against. A receipt + # asserting a spend that never happened is the one thing an evidence trail may + # not do. + _BUDGET_CHARGES.set( + tuple( + { + "grant_id": charge.grant_id, + "metric": charge.metric, + "amount": charge.amount, + } + for charge in charges + ) + ) break + except BudgetExhaustedError as exhausted: + # SPEC-v0.9 §3.3.2 — **its own clause, before the `ActionDenied` one**, for the + # reason item 2 met first with the scope refusal: that handler appends + # `APPROVAL_DENIED` unconditionally, which would fabricate an approval denial for + # an action no human ever saw. An exception raised inside an `except` clause + # leaves the whole `try` rather than meeting its siblings. + raise self._refuse_budget(action, exhausted) from None except _ScopeRefusedError as refused: # SPEC-v0.9 §5.6. Its own clause, **before** the `ActionDenied` one: # `_refuse_scope` has already written the events and the receipt, and an @@ -3143,6 +3274,225 @@ def _in_scope( ): raise refuse(action, OUT_OF_SCOPE, f"resource {action.resource!r} is not in this scope") + def _charges_for( + self, + action: Action, + effect_key: str | None, + observation: _Observation | None = None, + ) -> tuple[Charge, ...]: + """What this action spends, one `Charge` per ancestor (SPEC-v0.9 §2.7). + + **And §2.4.1's refusal, here, because this is where the effect key is finally known.** + A budgeted grant that reaches an action whose key resolved to `None` is refused: without + it an agent proposes actions carrying no `effect:` template and spends nothing against + every budget on the chain, for ever, which is the feature's own sharp case answered by + declining to play. §2.4.1 records the two probes that moved this out of the loader: a + loader cannot see a decorator-supplied `effect=`, and cannot run at all on the + standalone-authority path. + """ + if self._authority is None: + return () + result = _AUTHORITY_RESULT.get(None) + if result is None: + return () + try: + charges = self._authority._charges_for(action, result, store=self._store) + except InvalidArgument as unmeasurable: + # §2.3. The kernel cannot measure what this action spends, so it cannot hold the + # grant to its budget, so it declines to run it. Recorded before it is re-raised. + raise self._refuse_unmeasurable( + action, BUDGET_UNMEASURABLE, unmeasurable, observation + ) from None + if charges and effect_key is None: + raise self._refuse_unmeasurable( + action, + BUDGET_UNKEYED, + InvalidArgument( + f"{action.name}: grant {charges[0].grant_id!r} carries a budget and this " + "action resolved no effect key, so nothing could be charged against it. " + "Declare an `effect:` template for the action, or take the budget off the " + "grant (SPEC-v0.9 §2.4.1)" + ), + observation, + ) from None + return charges + + def _observe_charges( + self, action: Action, effect_key: str | None, observation: _Observation + ) -> tuple[Charge, ...]: + """§4.2.1's first half: what this action *would have* been charged, charging nothing. + + §2.3's and §2.4.1's refusals are **reported** here rather than raised: enforce mode + refuses those actions, so saying so is exactly what observe mode is for. They get their + own reasons rather than `budget_exhausted`, because an operator whose pilot says "this + would have been refused" needs to know whether the budget is too small or the action + cannot be measured at all. + + Called above the approval gate, where `_secure` computes the same thing, so the two modes + agree about which refusal comes first (T451). + """ + try: + charges = self._charges_for(action, effect_key, observation) + except InvalidArgument: + # Already reported by `_refuse_unmeasurable`, which blocked rather than denying. + return () + if not charges: + return () + # §4.2.1a — **the counterfactual spend, on the receipt.** The ledger is empty under + # observation, so if the receipt does not carry what this action would have been charged, + # nothing anywhere records it and a budget cannot be sized from an observed run. It + # asserts no spend: the receipt says `observed`, and `v0.3 §6.2` makes every number on an + # observed receipt a counterfactual. T439d. + _BUDGET_CHARGES.set( + tuple( + {"grant_id": charge.grant_id, "metric": charge.metric, "amount": charge.amount} + for charge in charges + ) + ) + return charges + + def _observe_spend( + self, action: Action, charges: tuple[Charge, ...], observation: _Observation + ) -> None: + """§4.2.1's second half: whether the budget would have refused, writing nothing. + + The predicate is `check_charges`, the same function all three stores decide with, so the + report and the enforcement cannot drift: a pilot that said "this would have been fine" + about an action enforce mode refuses is worse than no pilot. + + **The sum is a lock-free read** off the public `consumptions()` rather than a store's + private `_spent`, because this runs outside any reservation and must take no lock and + write nothing. It is therefore stale under concurrency, which is correct for a + counterfactual and would not be for a decision. + + `check_charges` can also raise `InvalidArgument` for two charges on one grant and metric + carrying different amounts (§3.3.1). Nothing reachable produces that shape -- §2.7's + ancestors are distinct grants, and one grant's two budgets on one metric always agree -- + and observe mode is not the place to raise about it if something ever does. + """ + if not charges: + return + now = self._clock() + + def spent(charge: Charge) -> int: + return sum( + row.amount + for row in self._store.consumptions( + grant_id=charge.grant_id, + metric=charge.metric, + since=now - charge.window, + ) + if row.released_at is None + ) + + try: + check_charges(charges, spent) + except BudgetExhaustedError as exhausted: + observation.block(BUDGET_EXHAUSTED) + self._append( + EventType.ACTION_DENIED, + action, + { + "reason": BUDGET_EXHAUSTED, + "grant_id": exhausted.grant_id, + "metric": exhausted.metric, + "window": int(exhausted.window.total_seconds()), + "observed": True, + }, + ) + except InvalidArgument: + return + + def _resumed_charges(self, effect_key: str | None, attempt: int) -> None: + """Stamp the resumed leg's receipt with what its **first** leg charged (§10.1, §8.3). + + One row per grant and metric at this attempt, in the ledger's insertion order, which + §3.3.3 makes identical across the three backends. A released row still counts: it says + what this action spent, and `released_at` is a later fact about the same spend. + """ + if effect_key is None: + _BUDGET_CHARGES.set(()) + return + _BUDGET_CHARGES.set( + tuple( + {"grant_id": row.grant_id, "metric": row.metric, "amount": row.amount} + for row in self._store.consumptions(effect_key=effect_key) + if row.attempt == attempt + ) + ) + + def _refuse_unmeasurable( + self, + action: Action, + reason: str, + error: InvalidArgument, + observation: _Observation | None = None, + ) -> InvalidArgument: + """§2.3 and §2.4.1's refusals, with the events and the receipt they were missing. + + Returns the error for the caller to `raise`, like `_refuse_scope`, so a reader can see + the control flow leaves at the call site. The message is the one the guard already wrote: + it names the grant, the metric and the offending value, and an operator reading the + receipt needs exactly that. + """ + if observation is not None: + # `v0.3 §6.2`: observe mode records what enforce mode would have done and refuses + # nothing. Writing the `denied` receipt below would put a refusal it did not make in + # the store, alongside the `observed` receipt for the run that went ahead: two + # receipts for one action, disagreeing. T439c. + observation.block(reason) + self._append( + EventType.ACTION_DENIED, + action, + {"reason": reason, "error": str(error), "observed": True}, + ) + return _UnmeasurableError(str(error), reason=reason) + self._append(EventType.ACTION_DENIED, action, {"reason": reason, "error": str(error)}) + self._record( + action, + Evaluation(Decision.DENY, reason), + ReceiptResult.DENIED, + self._clock(), + error=str(error), + ) + return _UnmeasurableError(str(error), reason=reason) + + def _refuse_budget(self, action: Action, exhausted: BudgetExhaustedError) -> ActionDenied: + """SPEC-v0.9 §4.5. Names the grant, the metric and the window; **never the balance**. + + A refusal that reported how much was left would be an oracle: refused actions cost + nothing, so an attacker binary-searches the exact limit in a few dozen refusals and then + knows precisely how much authority to use without tripping it. An operator debugging at + 3am gets the number from `inspect`, which needs the store rather than the ability to be + refused. + + The grant named is **the one that refused**, which under §2.7 may be an ancestor rather + than the grant that decided: an operator whose child grant is well within its own budget + needs to be told the parent is not. + """ + error = ( + f"budget {exhausted.metric!r} on grant {exhausted.grant_id!r} over " + f"{exhausted.window} is exhausted" + ) + self._append( + EventType.ACTION_DENIED, + action, + { + "reason": BUDGET_EXHAUSTED, + "grant_id": exhausted.grant_id, + "metric": exhausted.metric, + "window": int(exhausted.window.total_seconds()), + }, + ) + self._record( + action, + Evaluation(Decision.DENY, BUDGET_EXHAUSTED), + ReceiptResult.DENIED, + self._clock(), + error=error, + ) + return ActionDenied(f"{action.name} denied: {error}", reason=BUDGET_EXHAUSTED) + def _refuse_scope(self, action: Action, reason: str, error: str) -> _ScopeRefusedError: """The refusal, with its events and its receipt. Returns it for the caller to raise. @@ -3401,17 +3751,27 @@ def _invalidated( return data def _take( - self, action: Action, approval_id: str | None, effect_key: str | None, lease: timedelta + self, + action: Action, + approval_id: str | None, + effect_key: str | None, + lease: timedelta, + charges: tuple[Charge, ...] = (), ) -> tuple[Approval | None, Reservation | None]: - """Consume the approval, reserve the effect, or both at once (SPEC-v0.1 §4.2 A4).""" + """Consume the approval, reserve the effect, or both at once (SPEC-v0.1 §4.2 A4). + + SPEC-v0.9 §3.3: the charges ride the reservation's own transaction, which is the whole of + why `StateStore` was amended. The branch with no effect key passes none, because there is + no reservation to ride and §2.4.1 has already refused a budgeted grant that reaches it. + """ if approval_id is not None and effect_key is not None: return self._store.consume_approval_and_reserve( - approval_id, action.action_hash, effect_key, action.action_id, lease + approval_id, action.action_hash, effect_key, action.action_id, lease, charges ) if approval_id is not None: return self._store.consume_approval(approval_id, action.action_hash), None if effect_key is not None: - return None, self._store.reserve_effect(effect_key, action.action_id, lease) + return None, self._store.reserve_effect(effect_key, action.action_id, lease, charges) return None, None def _approver_of(self, approval_id: str | None) -> str | None: @@ -3958,6 +4318,7 @@ def _record( authority_grant_id=_AUTHORITY_GRANT_ID.get(None), task=_TASK.get(None), scope_hash=_SCOPE_HASH.get(None), + budget_charges=_BUDGET_CHARGES.get(()), receipt_id=new_receipt_id(), action_id=action.action_id, action=action.name, diff --git a/src/ctrlrun/gateway/operator.py b/src/ctrlrun/gateway/operator.py index 5e129730..4e2c189e 100644 --- a/src/ctrlrun/gateway/operator.py +++ b/src/ctrlrun/gateway/operator.py @@ -51,7 +51,15 @@ IdentityProvider, ) from ..receipt import Event, EventType, iso_timestamp -from ..reporting import effect_document, inspection_for, since_boundary, stats_document +from ..reporting import ( + effect_document, + inspection_for, + since_boundary, + stats_document, +) +from ..reporting import ( + ledger_rows as _ledger_rows, +) from ..state import RESOLUTIONS, StateStore from .mcp import DEFAULT_MAX_BODY_BYTES, ParsedRequest, Refusal, parse_request from .wire import ( @@ -807,8 +815,15 @@ def _stats(self, since: object) -> dict[str, Any]: for receipt in self.store.receipts() if boundary is None or receipt.finished_at >= boundary ] - # §9.1 — one producer for `ctrlrun.stats/v1`. T193 asserts equality with the CLI's. - return stats_document(counted, mode=self._control.policy.mode, boundary=boundary) + # §9.1 — one producer for `ctrlrun.stats/v1`. T193 asserts equality with the CLI's, and + # SPEC-v0.9 §7.3's row count is part of that document: a key the CLI reports and this + # does not is two documents under one schema name, which is what §9.1 exists to stop. + return stats_document( + counted, + mode=self._control.policy.mode, + boundary=boundary, + ledger_rows=_ledger_rows(self.store), + ) # --- the write tools (§4.5) ----------------------------------------------------------- diff --git a/src/ctrlrun/gateway/server.py b/src/ctrlrun/gateway/server.py index d53119fe..8e13a54f 100644 --- a/src/ctrlrun/gateway/server.py +++ b/src/ctrlrun/gateway/server.py @@ -728,7 +728,7 @@ def _through_control( held: dict[str, Any], request_id: JsonRpcId, ) -> _Response: - from ..control import with_approval + from ..control import _UnmeasurableError, with_approval approval = None # SPEC-v0.3 §8.3 — the **combined** decision of §4.6, not the policy axis alone. @@ -783,6 +783,28 @@ def _through_control( action_id=action.action_id, ), ) + except _UnmeasurableError as refused: + # SPEC-v0.9 §2.3, §2.4.1. **Its own clause, because `InvalidArgument` is not an + # `ActionDenied`** -- and without one this raised out of the handler and the socket + # closed with no response, which is the failure `_continue`'s comment below calls + # the one thing this library exists to prevent. An independent review found it. + # + # `-41001` and not a new code: §11 freezes it as "this action is not permitted to + # anyone in this configuration", and an action whose budgeted grant cannot measure + # it is exactly that. The kernel has already written the event and the receipt. + _LOG.warning("refused %s: %s", action.name, refused) + code, token, status = DENIED + return _json( + status, + json_rpc_error( + request_id, + code, + token, + str(refused), + reason=refused.reason, + action_id=action.action_id, + ), + ) except IdentityError as refused: # SPEC-v0.3 §2.3 — `Control.execute` refuses an expired principal with # `IdentityError`, which is deliberately **not** an `ActionDenied` (§11): an agent diff --git a/src/ctrlrun/postgres.py b/src/ctrlrun/postgres.py index ea3f0876..919b2f7d 100644 --- a/src/ctrlrun/postgres.py +++ b/src/ctrlrun/postgres.py @@ -1070,6 +1070,23 @@ def _ambiguate(self, record: EffectRecord, now: datetime) -> None: with contextlib.suppress(Exception): connection.close() + def _release_locked( + self, connection: Any, effect_key: str, state: EffectState, now: datetime + ) -> None: + """SPEC-v0.9 §4.1, §4.4. Released exactly on `FAILED`, by compare-and-set on the flag. + + `WHERE released_at IS NULL` is the compare half, so the re-issue of a lost `UPDATE` + (`v0.6 §4.3.2` Table A2 row 2) is a no-op rather than a second subtraction. A decrement + would not survive that branch, which is why §3.2's column is a nullable timestamp. + """ + if state is not EffectState.FAILED: + return + connection.execute( + f"UPDATE {self._q}.budget_ledger SET released_at = %s " + "WHERE effect_key = %s AND released_at IS NULL", + (now, effect_key), + ) + def _lock_budget_anchors(self, connection: Any, charges: tuple[Charge, ...]) -> None: """`SELECT ... FOR UPDATE` on one row per grant charged, **before** the sum (§3.6). @@ -1142,6 +1159,7 @@ def consumptions( grant_id: str | None = None, metric: str | None = None, since: datetime | None = None, + effect_key: str | None = None, ) -> tuple[Consumption, ...]: clauses: list[str] = [] values: list[Any] = [] @@ -1154,6 +1172,9 @@ def consumptions( if since is not None: clauses.append("consumed_at >= %s") values.append(since) + if effect_key is not None: + clauses.append("effect_key = %s") + values.append(effect_key) where = f" WHERE {' AND '.join(clauses)}" if clauses else "" with self._connection().cursor() as cursor: cursor.execute( @@ -1439,6 +1460,11 @@ def _transition( ), ) updated = cursor.rowcount + if updated == 1: + # SPEC-v0.9 §4.1, inside the same `BEGIN`: the ledger moves with the record or + # neither moves. Only where the compare-and-set actually took, so a transition + # that is about to be refused releases nothing. + self._release_locked(connection, effect_key, state, now) if updated != 1: # The record changed between the read and the write. Re-plan through the same # predicate rather than guessing. @@ -1588,6 +1614,9 @@ def resolve_effect(self, effect_key: str, state: EffectState, resolver: str) -> record = _resolvable(self._read_effect(connection, effect_key), effect_key, state) resolved = _resolved(record, state, resolver, now) self._write_effect(connection, resolved, record) + # SPEC-v0.9 §4.1, §4.2's `resolve_effect(FAILED)` row: this path does not go through + # `_transition`, so the release is here too, inside the same `BEGIN`. + self._release_locked(connection, effect_key, state, now) except BaseException: self._rollback(connection) raise diff --git a/src/ctrlrun/receipt.py b/src/ctrlrun/receipt.py index 4d1adbdc..327f0d71 100644 --- a/src/ctrlrun/receipt.py +++ b/src/ctrlrun/receipt.py @@ -100,7 +100,7 @@ #: guarantee catalogue's own rule: a key listed here is a key `to_dict` projects, so naming one #: before something writes it is a `KeyError` on every receipt, which is the field-level form of #: a stub row. Item 7 asserts all three are present before the release. -_V6_KEYS: Final = (*_V5_KEYS, "task", "scope_hash") +_V6_KEYS: Final = (*_V5_KEYS, "task", "scope_hash", "budget_charges") _KEYS: Final = { _V1: _V1_KEYS, _V2: _V2_KEYS, @@ -482,6 +482,17 @@ class Receipt: #: authorization system's state (`v0.7 §6.10`). Its own domain tag, so it can never equal a #: precondition fingerprint over the same mapping. scope_hash: str | None = None + #: SPEC-v0.9 §10.1: which grants this action charged, which metrics, how much. One entry per + #: ancestor charged (§2.7), so a reader can tell an action that spent a child's budget from + #: one that spent a root's. Empty where the deciding grant budgets nothing, which is every + #: grant written before v0.9. + #: + #: **On an `observed` receipt it is a counterfactual, not a spend** (§4.2.1a). Observe mode + #: charges nothing and its ledger stays empty, so this carries what the action *would have* + #: been charged, which is the number a budget is sized from before it is turned on. `result` + #: is what tells the two apart, and `v0.3 §6.2` makes every number on an observed receipt a + #: counterfactual; a consumer summing these to measure real spend must filter on it. + budget_charges: tuple[Mapping[str, Any], ...] = () #: The schema this receipt is written under (§6.11). A receipt this binary builds is #: `RECEIPT_SCHEMA`; one read from a store keeps the label its document declared, or `""` #: where it declared none, which renders with no `schema` key at all. @@ -575,6 +586,7 @@ def _full_document(self) -> dict[str, Any]: # predates tasks" by the schema label. "task": self.task, "scope_hash": self.scope_hash, + "budget_charges": [dict(charge) for charge in self.budget_charges], } def to_json(self) -> str: @@ -670,6 +682,15 @@ def from_dict(cls, document: Mapping[str, Any]) -> Receipt: if schema == _V6 and isinstance(document.get("scope_hash"), str) else None ), + budget_charges=( + tuple( + entry + for entry in document.get("budget_charges", ()) + if isinstance(entry, Mapping) + ) + if schema == _V6 and isinstance(document.get("budget_charges"), list) + else () + ), schema=schema, ) diff --git a/src/ctrlrun/reporting.py b/src/ctrlrun/reporting.py index 48a790c9..7ffc335b 100644 --- a/src/ctrlrun/reporting.py +++ b/src/ctrlrun/reporting.py @@ -21,8 +21,9 @@ from typing import Any, Final from .approval import ApprovalRecord -from .effect import EffectRecord -from .errors import InvalidArgument +from .authority import Budget +from .effect import EffectRecord, EffectState +from .errors import CTRLRunError, InvalidArgument from .policy import OBSERVE, Decision from .receipt import ( BLOCKED_APPROVAL_REQUIRED, @@ -44,6 +45,12 @@ #: SPEC-v0.3 §6.4 — one `ctrlrun stats --json` document. STATS_SCHEMA: Final = "ctrlrun.stats/v1" +#: SPEC-v0.9 §7.2. Its own document rather than a key inside `ctrlrun.inspection/v2`, because it +#: answers about a **grant** and that one answers about an action: a reader handed one would have +#: to know which of two shapes it got. §7.1 keeps both behind `ctrlrun inspect`, which is the +#: surface question and a separate one. +BUDGET_SCHEMA: Final = "ctrlrun.budget/v1" + #: The three relative units of SPEC-v0.3 §6.4, and the `timedelta` keyword each names. _RELATIVE_UNITS: Final[Mapping[str, str]] = {"m": "minutes", "h": "hours", "d": "days"} @@ -179,7 +186,11 @@ def since_boundary(argument: str | None) -> datetime | None: def stats_document( - counted: Sequence[Receipt], *, mode: str, boundary: datetime | None + counted: Sequence[Receipt], + *, + mode: str, + boundary: datetime | None, + ledger_rows: int | None = None, ) -> dict[str, Any]: """The numbers of SPEC-v0.3 §6.4, from `would_have` in observe mode and `result` in enforce. @@ -197,6 +208,13 @@ def stats_document( "to": iso_timestamp(max(finished)) if finished else None, "actions": len(counted), } + if ledger_rows is not None: + # SPEC-v0.9 §7.3 — "`stats` reports the row count so growth is observable before it is a + # problem." The ledger only grows: the kernel deletes no row, on §12's rule that it does + # not quietly delete evidence. Additive, so a 0.8.0 consumer of this document keeps + # working; omitted entirely on a store with no ledger rather than reported as 0, because + # "no rows" and "this store predates budgets" are different facts. + document["ledger_rows"] = ledger_rows if mode != OBSERVE: refused = [r for r in counted if r.result is ReceiptResult.DENIED] document["denied"] = len(refused) @@ -230,3 +248,128 @@ def tally(reasons: Iterable[str | None]) -> dict[str, int]: for reason in reasons: counts[str(reason)] = counts.get(str(reason), 0) + 1 return dict(sorted(counts.items(), key=lambda pair: (-pair[1], pair[0]))) + + +def budget_document( + grant_id: str, + budgets: Sequence[Budget], + store: StateStore, + now: datetime, +) -> dict[str, Any]: + """One grant's budgets, with the three numbers of SPEC-v0.9 §7.2. + + **consumed** is the un-released sum over the rolling window (§2.5), which is exactly what + §3.3.1's predicate compares against the limit: the number that decides. **held** is the part + of that sum whose effects have not reached `COMMITTED`, and **why** names the effect holding + each one and the state it is in. + + The third is the deliverable. A budget that refuses while an operator can see it is nowhere + near its limit looks like a defect in the kernel, and the true explanation is always the same + shape: some effect is `AMBIGUOUS` and nobody has resolved it (§4.2, R2). Without the `why` an + operator cannot get from the refusal to `ctrlrun resolve`, which is the command that clears + it; with it, the path is one command long. + + A row whose effect record is **missing** is reported with a `null` state rather than skipped. + §7.3 permits an operator to archive rows the window can no longer reach, and a store whose + effects were pruned but whose ledger was not would otherwise under-report `held` silently, + which is the one direction this view must not err in. + """ + reported: list[dict[str, Any]] = [] + for budget in budgets: + rows = [ + row + for row in store.consumptions( + grant_id=grant_id, metric=budget.metric, since=now - budget.window + ) + if row.released_at is None + ] + holding: list[dict[str, Any]] = [] + held = 0 + for row in rows: + effect = store.get_effect(row.effect_key) + state = None if effect is None else str(effect.state) + if state == str(EffectState.COMMITTED): + continue + held += row.amount + holding.append( + { + "effect_key": row.effect_key, + "state": state, + "amount": row.amount, + "attempt": row.attempt, + "consumed_at": iso_timestamp(row.consumed_at), + } + ) + reported.append( + { + "metric": budget.metric, + "limit": budget.limit, + "window_seconds": int(budget.window.total_seconds()), + "consumed": sum(row.amount for row in rows), + "held": held, + "holding": holding, + } + ) + return { + "schema": BUDGET_SCHEMA, + "grant_id": grant_id, + "at": iso_timestamp(now), + "budgets": reported, + } + + +def budget_lines(document: Mapping[str, Any]) -> list[str]: + """§7.2's view for a terminal, from the same document `--json` emits. + + One producer, for `inspection_for`'s reason: two builders that agree today disagree later. + """ + lines = [f"grant {document['grant_id']}"] + budgets = document["budgets"] + if not budgets: + lines.append(" no budgets: this grant bounds no aggregate (SPEC-v0.9 §2.4)") + return lines + for budget in budgets: + window = _window_words(int(budget["window_seconds"])) + lines.append( + f" {budget['metric']}: {budget['consumed']} of {budget['limit']} per {window}" + f", {budget['held']} held" + ) + for holding in budget["holding"]: + state = holding["state"] or "no effect record" + lines.append(f" {holding['amount']} held by {holding['effect_key']} ({state})") + if holding["state"] == str(EffectState.AMBIGUOUS): + # §7.2: the path from the refusal to the command that clears it, one command + # long. **The key, not a placeholder**: an operator who has to retype it from + # the line above is one transcription away from resolving the wrong effect. + lines.append(f" ctrlrun resolve {holding['effect_key']} --committed|--failed") + return lines + + +def _window_words(seconds: int) -> str: + """A rolling window as an operator would say it, falling back to seconds. + + Exact divisors only. "1.2 days" reads as a rounding of something and invites an operator to + wonder which way it went; `104400s` is unambiguous and does not. + """ + for size, unit in ((86400, "day"), (3600, "hour"), (60, "minute")): + if seconds % size == 0 and seconds >= size: + count = seconds // size + return unit if count == 1 else f"{count} {unit}s" + return f"{seconds}s" + + +def ledger_rows(store: StateStore) -> int | None: + """SPEC-v0.9 §7.3's row count, or `None` where this store has no ledger to count. + + Here rather than in either caller, because `ctrlrun stats` and the operator server both + report it and §9.1's rule is that one document has one producer: a key the CLI reports and + the server does not would be two shapes under one schema name, which T193 catches. + + A store written by 0.8.0 has no `budget_ledger` table until it is migrated, and both surfaces + are diagnostics: they report what they can about a store rather than refuse one. `None` omits + the key, so "no rows" and "this store predates budgets" stay distinct facts. + """ + try: + return len(store.consumptions()) + except CTRLRunError: + return None diff --git a/src/ctrlrun/state.py b/src/ctrlrun/state.py index a3af497f..ddb0db0d 100644 --- a/src/ctrlrun/state.py +++ b/src/ctrlrun/state.py @@ -581,24 +581,33 @@ def check_charges( Pure, like `plan_reservation` and for the same reason: every store decides here rather than each deciding for itself, so the arithmetic is one function a test can reach directly. - **Two charges on the same `(grant_id, metric)` in one tuple are refused**, and an independent - review is why. Each charge is evaluated against the *stored* sum, so a sibling in the same - tuple is invisible to the predicate; and the idempotence key of §3.4 carries no window, so - `ON CONFLICT DO NOTHING` then silently drops all but the first row. Together that is a spend - the ledger never records. It happened to be harmless for the one shape §2.2 creates, a grant - with two budgets on one metric over two windows, because those carry equal amounts and want - exactly one row. Refusing the general case makes that a property rather than a coincidence, - and §2.7's per-ancestor charges are distinct grants, so nothing legitimate is refused. + **Every charge is evaluated, including several on one `(grant_id, metric)`.** That is §2.2's + own motivating shape: a grant with two budgets on `amount`, 100,000 a day and 500,000 a month, + is "the first thing an operator asks for", and it arrives here as two charges differing only + in `limit` and `window`. Both predicates run; §3.4's key then writes **one** row, which is + right, because it is one spend measured against two windows. + + **What is refused is two charges on one `(grant_id, metric)` carrying different amounts.** + A charge is invisible to its sibling here (each is compared against the *stored* sum), and + §3.4's key carries no window, so differing amounts would collapse to whichever row landed + first and the ledger would under-record the spend. Nothing legitimate produces that: the + amount comes from the action's own metric value, so two budgets on one metric always agree, + and §2.7's per-ancestor charges are distinct grants. + + An earlier version refused **any** duplicate pair, which made §2.2's shape die at execute + with no receipt: the loader accepted the document, observe mode reported it clean, and + `ctrlrun verify` could not grade it. An independent review found it. """ - seen: set[tuple[str, str]] = set() + amounts: dict[tuple[str, str], int] = {} for charge in charges: key = (charge.grant_id, charge.metric) - if key in seen: + seen = amounts.setdefault(key, charge.amount) + if seen != charge.amount: raise InvalidArgument( - f"two charges on {charge.grant_id!r}/{charge.metric!r} in one reservation: the " - "predicate cannot see a sibling's amount and §3.4's key would drop the second row" + f"two charges on {charge.grant_id!r}/{charge.metric!r} in one reservation carry " + f"different amounts ({seen} and {charge.amount}); §3.4's key would keep one row " + "and the ledger would under-record the spend" ) - seen.add(key) for charge in charges: if spent(charge) + charge.amount > charge.limit: raise BudgetExhaustedError(charge.grant_id, charge.metric, charge.window) @@ -674,6 +683,7 @@ def consumptions( grant_id: str | None = None, metric: str | None = None, since: datetime | None = None, + effect_key: str | None = None, ) -> tuple[Consumption, ...]: """Ledger rows, **in insertion order** (SPEC-v0.9 §3.3.3). @@ -690,6 +700,11 @@ def consumptions( **`grant_id` is optional**, because §7.3 has `stats` report the ledger's row count, and a required one would make that enumerate every grant id that ever existed, runtime delegations and revoked grants included, one call each. + + **`effect_key` is what a resumed leg reads by.** §8.3 makes the resumed receipt the only + receipt an MCP multi round-trip or ACS action ever gets, so it has to report what that + action spent, and a gateway that restarted mid-round has nothing in memory to report it + from. Without this filter that read is a scan of the whole ledger per resumption. """ ... @@ -1281,12 +1296,38 @@ def _charge_locked( ) ) + def _release_locked(self, effect_key: str, state: EffectState, now: datetime) -> None: + """SPEC-v0.9 §4.1: **released exactly when the effect reaches `FAILED`**, held otherwise. + + The ledger has no state machine of its own. `effect.py`'s `plan_reservation` is already + the complete table of exits from a reservation, and this one rule covers every row of + §4.2's nineteen: `COMMITTED` holds permanently, `AMBIGUOUS` holds until a human or a hook + moves it, a lapsed lease holds because no transition has occurred, and only `FAILED` + releases, because that is the one state in which the executor proved nothing happened. + + **Keyed on the state reached, never on the call that reached it** (§4.2's warning): a + `fail_effect` that is *refused* because the record moved on releases nothing, and a + `resolve_effect(FAILED)` by a human releases even though no `fail_effect` ran. + + A compare-and-set on `released_at`, never a decrement (§4.4): `v0.6 §4.3.2` Table A2 row 2 + re-issues a lost `UPDATE` once, and a decrement would subtract twice. + """ + if state is not EffectState.FAILED: + return + self._ledger = [ + replace(row, released_at=now) + if row.effect_key == effect_key and row.released_at is None + else row + for row in self._ledger + ] + def consumptions( self, *, grant_id: str | None = None, metric: str | None = None, since: datetime | None = None, + effect_key: str | None = None, ) -> tuple[Consumption, ...]: with self._lock: return tuple( @@ -1295,6 +1336,7 @@ def consumptions( if (grant_id is None or row.grant_id == grant_id) and (metric is None or row.metric == metric) and (since is None or row.consumed_at >= since) + and (effect_key is None or row.effect_key == effect_key) ) def begin_execution(self, effect_key: str, action_id: str) -> None: @@ -1318,9 +1360,15 @@ def mark_ambiguous(self, effect_key: str, action_id: str, error: str) -> None: def resolve_effect(self, effect_key: str, state: EffectState, resolver: str) -> EffectRecord: resolver = _approver(resolver) with self._lock: + now = self._clock() record = _resolvable(self._effects.get(effect_key), effect_key, state) - resolved = _resolved(record, state, resolver, self._clock()) + resolved = _resolved(record, state, resolver, now) self._effects[effect_key] = resolved + # SPEC-v0.9 §4.1, §4.2's `resolve_effect(FAILED)` row. **This path does not go + # through `_transition`**, so the release has to be here too: a human resolving an + # `AMBIGUOUS` record `FAILED` is exactly the authority R2 says releases a hold, and + # without this the charge would be held for ever by the one act meant to free it. + self._release_locked(effect_key, state, now) return resolved def extend_lease(self, effect_key: str, action_id: str, until: datetime) -> None: @@ -1389,6 +1437,12 @@ def _transition( self._effects[effect_key] = _transitioned( record, state, now, result=result, error=error ) + # SPEC-v0.9 §4.1. **This order is the atomicity**, and unlike the SQL stores there is + # no rollback to fall back on: `_checked` raising is what must leave the ledger + # untouched. Moving the release above it keys it on the *call* rather than the state + # reached, and a refused `fail_effect` then releases the hold on an `AMBIGUOUS` + # record, which is a manufacturable refund. T425 pins it. + self._release_locked(effect_key, state, now) class _HeldConnection: @@ -2046,6 +2100,21 @@ def _authorize_and_reserve( connection.commit() return (approved.as_approval() if approved is not None else None), plan.reservation + def _release_locked( + self, connection: sqlite3.Connection, effect_key: str, state: EffectState, now: datetime + ) -> None: + """SPEC-v0.9 §4.1, §4.4. Released exactly on `FAILED`, by compare-and-set on the flag. + + `WHERE released_at IS NULL` is the compare half, so a re-issued `UPDATE` (`v0.6 §4.3.2` + Table A2 row 2) is a no-op rather than a second subtraction. + """ + if state is not EffectState.FAILED: + return + connection.execute( + "UPDATE budget_ledger SET released_at = ? WHERE effect_key = ? AND released_at IS NULL", + (_iso(now), effect_key), + ) + def _spent(self, connection: sqlite3.Connection, charge: Charge, now: datetime) -> int: """The un-released sum for this charge, over its rolling window (SPEC-v0.9 §2.5).""" row = connection.execute( @@ -2096,6 +2165,7 @@ def consumptions( grant_id: str | None = None, metric: str | None = None, since: datetime | None = None, + effect_key: str | None = None, ) -> tuple[Consumption, ...]: clauses: list[str] = [] values: list[Any] = [] @@ -2108,6 +2178,9 @@ def consumptions( if since is not None: clauses.append("consumed_at >= ?") values.append(_iso(since)) + if effect_key is not None: + clauses.append("effect_key = ?") + values.append(effect_key) where = f" WHERE {' AND '.join(clauses)}" if clauses else "" rows = ( self._connection() @@ -2267,9 +2340,13 @@ def resolve_effect(self, effect_key: str, state: EffectState, resolver: str) -> connection = self._connection() connection.execute("BEGIN IMMEDIATE") try: + now = self._clock() record = _resolvable(self._read_effect(connection, effect_key), effect_key, state) - resolved = _resolved(record, state, resolver, self._clock()) + resolved = _resolved(record, state, resolver, now) self._write_effect(connection, resolved) + # SPEC-v0.9 §4.1, §4.2's `resolve_effect(FAILED)` row: this path does not go through + # `_transition`, so the release is here too, inside the same `BEGIN IMMEDIATE`. + self._release_locked(connection, effect_key, state, now) except BaseException: self._unwind(connection) raise @@ -2313,6 +2390,10 @@ def _transition( self._write_effect( connection, _transitioned(record, state, now, result=result, error=error) ) + # SPEC-v0.9 §4.1, inside the same `BEGIN IMMEDIATE`: the ledger moves with the record + # or neither moves. A release in a second transaction could leave a `FAILED` effect + # holding its charge for ever if the process died between them. + self._release_locked(connection, effect_key, state, now) except BaseException: self._unwind(connection) raise diff --git a/src/ctrlrun/verify/guarantees.py b/src/ctrlrun/verify/guarantees.py index 67165b6f..9fb06f5b 100644 --- a/src/ctrlrun/verify/guarantees.py +++ b/src/ctrlrun/verify/guarantees.py @@ -151,6 +151,15 @@ class Guarantee: "unapproved policy decides no", ("v0.6 §7.1", "v0.8 §10 T354", "v0.8 §10 T355"), ), + Guarantee( + "G22", + # 32 characters exactly, against `report._TITLE_WIDTH`. "held" and not "exhausted": what + # refuses the next reserve is a budget whose consumption is held by an effect nobody has + # resolved, and a title saying "exhausted" would describe the ordinary case and miss the + # one this guarantee is about (SPEC-v0.9 §8). + "held budget refuses next reserve", + ("v0.9 §4.1", "v0.9 §9 T436", "v0.9 §9 T437"), + ), Guarantee( "G23", # 32 characters exactly, against `report._TITLE_WIDTH`. "a failing scope provider" and @@ -257,6 +266,36 @@ class Guarantee: #: SPEC-v0.9 §8, G24. A statement about the operator's **document**, like every reason in this #: module: it says what the document does not declare, not what the kernel is not configured for. NO_TASKS: Final = "no grant names a task" +#: SPEC-v0.9 §8, G22. A statement about the operator's **document**: whether any grant it declares +#: carries a budget at all. +NO_BUDGET: Final = "no grant carries a budget" +#: The action verify can drive carries no value for the metric the budget names, so nothing it +#: could run would spend against it (SPEC-v0.9 §2.3). +NO_BUDGET_METRIC: Final = "no action verify can drive carries the metric the budget names" +#: A document whose one permitted action cannot fit inside its own budget cannot exercise the +#: hold. Legal, and almost always a mistake: the first action of the window exhausts it. +BUDGET_CANNOT_BE_FILLED: Final = "one permitted action does not fit inside the grant's budget" + +#: SPEC-v0.9 §2, for `select`. Distinct from `NO_GRANT_COVERS_SELECTION` because they are +#: unrelated facts and the operator's fix differs: one is a `resources:` pattern, the other is a +#: budget smaller than every action in the band the scenario needs. Reported without it, a policy +#: whose approve band starts above its grant's daily budget was told no grant's `resources:` +#: matched, about a document whose patterns matched perfectly. +NO_ACTION_FITS_THE_BUDGET: Final = ( + "every action reaching this decision exceeds a budget on the grant that covers it" +) + +#: SPEC-v0.9 §2.3, and a **different fix** from the one above: the grant budgets a metric its +#: actions do not carry, so no vector helps and every action it covers is refused for ever. +#: Reported as "exceeds a budget" it sent an operator to raise a limit that was never the problem. +NO_METRIC_TO_MEASURE: Final = ( + "a budget on the grant that covers it names a metric the action does not carry" +) +BUDGET_MISS_NOTE: Final = ( + "a budget smaller than any single action in the band makes that band unreachable: every " + "action needing it would exhaust the whole window. Raise the budget, or narrow the rule " + "that admits actions the budget cannot pay for (SPEC-v0.9 §2)" +) #: SPEC-v0.9 §8.1, G23. A statement about the **document**, which is what §8.1's argument #: actually requires: it forbids an `N/A` about whether a *provider* is configured, because that #: is a fact about an operator's code. Whether any action this configuration admits carries a @@ -351,6 +390,8 @@ class Guarantee: CONTROL_FAILED: Final = "control failed" __all__ = [ + "BUDGET_CANNOT_BE_FILLED", + "BUDGET_MISS_NOTE", "BY_ID", "CANDIDATE_BOUND", "CATALOGUE", @@ -366,15 +407,19 @@ class Guarantee: "GUARANTEES", "NOT_SELECTED", "NO_ACTIONS", + "NO_ACTION_FITS_THE_BUDGET", "NO_APPROVER_ROLE", "NO_APPROVE_RULE", "NO_AUTHORITY_SECTION", + "NO_BUDGET", + "NO_BUDGET_METRIC", "NO_CEILING_DECLARED", "NO_DELEGABLE_GRANT", "NO_EFFECT_TEMPLATE", "NO_EXPIRES_AT", "NO_GRANT_COVERS_SELECTION", "NO_GRANT_MATCHES", + "NO_METRIC_TO_MEASURE", "NO_RESOURCE_TO_SCOPE", "NO_TASKS", "PER_CONNECTION_BACKEND", diff --git a/src/ctrlrun/verify/scenarios.py b/src/ctrlrun/verify/scenarios.py index b9749d3b..176d9222 100644 --- a/src/ctrlrun/verify/scenarios.py +++ b/src/ctrlrun/verify/scenarios.py @@ -61,9 +61,17 @@ Authority, Grant, Subject, + _metric_value, contained_dimension, ) -from ..control import SCOPE_UNAVAILABLE, Control, context, idempotency_token, protect +from ..control import ( + BUDGET_EXHAUSTED, + SCOPE_UNAVAILABLE, + Control, + context, + idempotency_token, + protect, +) from ..effect import ( EffectRecord, EffectState, @@ -610,6 +618,10 @@ class Engine: def __init__(self, loaded: _Loaded, scratch: Path, store_url: str | None = None) -> None: #: Set by `select()` when the miss was on the authority axis (see `unselected`). self._grant_miss: str | None = None + self._budget_miss: str | None = None + self._metric_miss: str | None = None + self._unmeasurable = False + self._declined_on_budget = False #: SPEC-v0.9 §6.3.2 — the active selection's task; see `_control_for`. self._task: str | None = None self._loaded = loaded @@ -782,6 +794,8 @@ def select( needs an action that declares one, and one verify can drive to the top of. """ self._grant_miss = None + self._budget_miss = None + self._metric_miss = None for name in sorted(self.policy.actions): if needs_effect and self.policy.effect_template(name) is None: continue @@ -807,6 +821,7 @@ def select( if synthesized is None: continue arguments, reason = synthesized + self._declined_on_budget = False selection = self._bind(name, arguments, decision, reason, grant_filter) if selection is not None: return selection @@ -814,7 +829,12 @@ def select( # caller's N/A reason can say so: a bare `None` here is indistinguishable from # "no action reaches this decision", and every scenario used to resolve that # ambiguity by asserting its own hardcoded sentence about the policy. - self._grant_miss = self._resource(name, arguments) + # + # **Unless a grant did cover it and its budget is what declined.** Recording a + # resource miss there put a sentence in the report that is false of the document: + # the pattern matched perfectly and the budget was the whole reason. + if not self._declined_on_budget: + self._grant_miss = self._resource(name, arguments) return None def _bind( @@ -860,6 +880,26 @@ def _bind( ) if not grant.matches_shape(action) or not grant.constraints_hold(action): continue + # SPEC-v0.9 §2, and `_identity_the_document_needs`'s precedent exactly: a shipped + # example declaring something the kernel enforces must not make `ctrlrun verify` + # exit 3 on guarantees that have nothing to do with it. A grant whose budget is + # smaller than the vector `_synthesize` picked refuses that action, and the refusal + # reached G1 as an internal error. Verify owns the vector, so verify sizes it. + self._unmeasurable = False + fitted = self._fitted_to_budgets(name, arguments, decision, reason, grant, action) + if fitted is None: + # Recorded, for `unselected`'s reason. A bare `continue` here reported the + # *grant* miss below, so a policy whose approve band starts above its grant's + # daily budget was told no grant's `resources:` matched, about a document whose + # patterns matched perfectly. That is the category error `unselected`'s own + # docstring exists about, one dimension over. + self._declined_on_budget = True + if self._unmeasurable: + self._metric_miss = f"{name} on grant {grant.id!r}" + else: + self._budget_miss = f"{name} on grant {grant.id!r}" + continue + arguments, action = fitted try: effect_key = self._effect_key(action) except CTRLRunError: @@ -878,6 +918,125 @@ def _bind( return selection return None + def _deciding_grant(self, action: Action) -> Grant | None: + """The grant `Authority.evaluate` would resolve for this action, by its own rule. + + **`min(passed, key=_by_grant_id)`**, which is why this exists rather than trusting the + grant `_bind` happens to be iterating. An independent review found the consequence: a + vector resized to fit one grant's budget can fall inside a *different*, lexicographically + earlier grant's `constraints`, and that grant's budget was never checked. Its document + had `aa-narrow` at `amount_lte: 10` with `limit: 0` and `bb-broad` at + `amount_lte: 500000` with `limit: 900`; the resize from 100000 to 1 moved the action from + `bb-broad` to `aa-narrow`, and `ctrlrun verify` exited 3 on a budget it never looked at. + + Document grants only, which is what `_bind` iterates: a scenario's authority comes from + the operator's file, and no delegation exists in a store verify has not created yet. + """ + if self.authority is None: + return None + for grant_id in sorted(self.authority.grants): + grant = self.authority.grants[grant_id] + if grant.matches_shape(action) and grant.constraints_hold(action): + return grant + return None + + #: How many spends of the chosen vector a scenario may take. G4's control leg runs + #: `PROCESSES` children on distinct keys and then contends `PROCESSES` more on one key, so + #: nine of them land; the margin above that is for every other scenario that acts twice. + _BUDGET_HEADROOM: Final = reg.PROCESSES * 2 + 2 + + def _fits_budgets(self, grant: Grant, action: Action, *, room: int = 1) -> bool | None: + """Whether every budget on this grant admits `room` spends of this action. + + `None` is its own answer and not a `False`: a grant budgeting a metric the action does + not carry refuses **every** action it covers, for ever, and that is a fact about the + operator's document rather than a vector verify can size around. Returning `True` there + is what made `ctrlrun verify` exit 3 on §2.3's refusal instead of grading it. + + **`room` is why a vector that fits can still be the wrong one.** A scenario acts more + than once, and a band with a *floor* cannot be shrunk below it: `amount_gte: 200` against + a budget of 900 leaves the synthesized vector untouched, because 200 fits, and then G4 + reports FAIL because nine spends of 200 do not. Verify may say it could not grade a + configuration; it may not report the kernel broken. + """ + for budget in grant.budgets or (): + try: + value = _metric_value(action, budget.metric, grant.id) + except InvalidArgument: + return None + if value * room > budget.limit: + return False + return True + + def _fitted_to_budgets( + self, + name: str, + arguments: dict[str, Any], + decision: Decision, + reason: str, + grant: Grant, + action: Action, + ) -> tuple[dict[str, Any], Action] | None: + """Size verify's own action vector to the budgets that will actually decide it. + + **Verify grades a guarantee, not the operator's budget sizing.** `_synthesize` picks a + vector to land in a rule, and a grant whose budget is smaller than that vector refuses + the action before the guarantee is reached: a EUR 1,000 daily budget under a policy whose + `amount_lte` permits a EUR 100,000 refund made `ctrlrun verify` report an internal error + on G1, which is about approvals. That is `_identity_the_document_needs`'s case in the + budget dimension, and it gets the same answer: verify supplies what the document needs. + + Every candidate is checked against **the grant that would decide it**, not the grant the + caller is holding, because a resize can move the action between grants. The vector is + only changed when a budget would refuse it, so every document without budgets keeps the + vector it had, and a replacement must land in the same rule with the same reason because + `select`'s contract is the decision it was asked for. + + Where nothing fits, the candidate is declined and `select` moves on, so the guarantee + reports `N/A` with a true reason rather than failing a control leg. + """ + deciding = self._deciding_grant(action) or grant + verdict = self._fits_budgets(deciding, action, room=self._BUDGET_HEADROOM) + if verdict is True: + return arguments, action + if verdict is None: + # Unmeasurable: no vector helps, because the metric is absent from the action's whole + # shape rather than too large in this one. The caller needs to tell the two apart, + # because raising a limit fixes one and nothing about the other. + self._unmeasurable = True + return None + limits = [budget.limit for budget in (deciding.budgets or ()) if budget.limit > 0] + smallest = min(limits) if limits else 0 + metric = next(iter(deciding.budgets or ())).metric + # **Room for a scenario that acts more than once**, not for one action. G4's control leg + # alone runs `PROCESSES` children on distinct keys and then contends `PROCESSES` more on + # one, so it needs nine spends to fit; a vector sized to `limit` exactly made its control + # leg pass, its contended leg find zero winners, and the guarantee report **FAIL** -- the + # status that means the kernel is broken -- for a budget that was merely small. Verify + # may say it could not grade a configuration; it may not accuse the kernel of a defect. + headroom = self._BUDGET_HEADROOM + for candidate in (1, smallest // headroom, smallest // 8, smallest // 4, smallest // 2): + if candidate < 1: + continue + tried = {**arguments, metric: candidate} + rebuilt = replace(action, arguments=tried) + evaluation = self.policy.evaluate(rebuilt) + if evaluation.decision is not decision or evaluation.reason != reason: + continue + # **Re-resolved, and it must settle on the same grant.** A resize can move the + # action between grants, and `_bind` is building a selection that names *this* one: + # a vector graded against a different grant's budget would report the wrong grant in + # the result and check a budget nobody will apply. Requiring the same grant subsumes + # re-checking its shape and constraints, because `_deciding_grant` only returns a + # grant that matched both. + settled = self._deciding_grant(rebuilt) + if settled is None or settled.id != grant.id: + continue + if self._fits_budgets(settled, rebuilt, room=headroom) is not True: + continue + return tried, rebuilt + return None + # --- the scratch store, and the Control every scenario drives ----------------------- def control( @@ -1143,6 +1302,8 @@ def unselected_detail(self, note: str | None = None) -> dict[str, Any]: miss when it travelled beside the grant reason, which is the same category error the reason itself had. """ + if self._budget_miss is not None or self._metric_miss is not None: + return {"note": reg.BUDGET_MISS_NOTE} if self._grant_miss is not None: return {"note": reg.GRANT_RESOURCE_NOTE} return {} if note is None else {"note": note} @@ -1156,9 +1317,26 @@ def unselected(self, reason: str) -> str: cases is how `examples/authority/devops.yaml` came to be told "the policy lists no action" about a document listing five, on a run that exited 0. """ - if self._grant_miss is None: + # **Every miss that is true, not the first one found.** An independent review found the + # budget miss taking unconditional precedence while `select` records a grant miss for + # every failed candidate, so a document with one budget-blocked action and one no grant + # covers at all reported only the budget, and the resource miss appeared nowhere. That is + # the category error this docstring is about, one dimension over. + found = [ + f"{reg.NO_METRIC_TO_MEASURE} ({self._metric_miss})" + if self._metric_miss is not None + else None, + f"{reg.NO_ACTION_FITS_THE_BUDGET} ({self._budget_miss})" + if self._budget_miss is not None + else None, + f"{reg.NO_GRANT_COVERS_SELECTION} ({self._grant_miss!r})" + if self._grant_miss is not None + else None, + ] + stated = [line for line in found if line is not None] + if not stated: return reason - return f"{reg.NO_GRANT_COVERS_SELECTION} ({self._grant_miss!r})" + return "; and ".join(stated) def na(self, gid: str, reason: str, **detail: Any) -> GuaranteeResult: """`not_applicable`, with the reason that made it so (§1, §2.1). @@ -3703,6 +3881,136 @@ def body(detail: dict[str, Any]) -> None: finally: store.close() + # --- G22: a budget held by ambiguity refuses the next reserve -------------------------- + + def g22(self) -> GuaranteeResult: + """SPEC-v0.9 §4.1, §8. **R2: ambiguity is not a refund.** + + The half that matters is the hold. An `AMBIGUOUS` effect keeps its consumption until a + human or a hook resolves it, because otherwise an agent that can generate ambiguity can + generate authority, and generating ambiguity is free for any flaky integration. That is + the correctness hole that parked budgets for four milestones. + + Both halves, `v0.4 §1.3`: the budget spends while it has room, and refuses once a held + charge fills it. And the release: `FAILED` gives the room back, which is the other half of + §4.1's single rule and the thing a kernel that simply never released would fail. + """ + if self.authority is None: + return self.na("G22", reg.NO_AUTHORITY_SECTION) + budgeted = [ + grant_id + for grant_id in sorted(self.authority.grants) + if self.authority.grants[grant_id].budgets + ] + if not budgeted: + return self.na("G22", reg.NO_BUDGET) + selection = self.select(needs_effect=True, grant_filter=lambda g: g.id == budgeted[0]) + if selection is None: + return self.na("G22", self.unselected(reg.NO_EFFECT_TEMPLATE)) + grant = self.authority.grants[budgeted[0]] + budget = (grant.budgets or ())[0] + # Decided **before** the scenario is built, so a document that cannot exercise the hold + # reports `N/A` with a reason that is true of it rather than failing a control leg. + try: + per_action = _metric_value(selection.build(), budget.metric, budgeted[0]) + except InvalidArgument: + return self.na("G22", reg.NO_BUDGET_METRIC) + if per_action == 0: + # **The selected action spends nothing, which grades nothing.** `select` picks the + # first rule a document admits, and a band beginning at zero gives an amount of zero, + # so the budget would never move and every assertion below would pass against a + # kernel that does not charge at all. Nudged to the smallest spend the same rule + # admits, which keeps the decision and the resource verify already validated. + selection = replace( + selection, arguments={**dict(selection.arguments), budget.metric: 1} + ) + try: + per_action = _metric_value(selection.build(), budget.metric, budgeted[0]) + except InvalidArgument: + return self.na("G22", reg.NO_BUDGET_METRIC) + if per_action <= 0 or per_action > budget.limit: + return self.na("G22", reg.BUDGET_CANNOT_BE_FILLED) + control, store, recorder, _ = self._control_for("G22", selection) + + def body(detail: dict[str, Any]) -> None: + detail["grant_id"] = budgeted[0] + detail["metric"] = budget.metric + detail["limit"] = budget.limit + detail["per_action"] = per_action + action = selection.build() + charges = control._charges_for(action, selection.effect_key) + + # The control: with room, it runs. + executor = _Executor() + receipt = self.execute( + control, + action, + executor, + selection.effect_key, + self.approve(control, store, action, selection), + ) + _expect_control( + receipt.result is ReceiptResult.COMMITTED and executor.calls == 1, + "with room in the budget the action runs", + f"it ended {receipt.result} after {executor.calls} executor calls", + ) + + # **One held charge for the rest of the budget**, then `AMBIGUOUS`: the state R2 is + # about, where nobody has said whether it happened. One reservation rather than a + # loop, so the scenario costs the same on a budget of 500 and one of 500,000,000; the + # property is the hold, not the arithmetic of filling. + held_key = f"{selection.effect_key}-{reg.SYNTHETIC_PREFIX}-held" + held_action = f"act_{reg.SYNTHETIC_PREFIX}held" + store.reserve_effect( + held_key, + held_action, + _ONE_HOUR, + tuple(replace(charge, amount=charge.limit - per_action) for charge in charges), + ) + store.mark_ambiguous(held_key, held_action, "the outcome is unknown") + detail["held"] = budget.limit - per_action + + later = selection.build() + blocked = _Executor() + refusal = self.refused( + lambda: self.execute(control, later, blocked, f"{selection.effect_key}-next", None), + (ActionDenied,), + "ActionDenied(reason='budget_exhausted') once the budget is held", + "the action ran with the budget held by an unresolved effect", + ) + reason = getattr(refusal, "reason", "") + _expect( + reason == BUDGET_EXHAUSTED, + "ActionDenied(reason='budget_exhausted')", + f"ActionDenied(reason={reason!r})", + ) + _expect( + blocked.calls == 0, + "the executor is not reached once the budget is held", + f"the executor was called {blocked.calls} times", + ) + + # And §4.1's other half: `FAILED` gives the room back. A kernel that never released + # would pass everything above. + # + # Through `resolve_effect`, which is §4.2's own row for this: the record is + # `AMBIGUOUS`, and `v0.1 §5.2` makes a human the only authority that moves it. That + # is also the shape an operator actually meets, `ctrlrun resolve`. + store.resolve_effect(held_key, EffectState.FAILED, "ctrlrun-verify") + after = _Executor() + freed = selection.build() + receipt = self.execute(control, freed, after, f"{selection.effect_key}-freed", None) + _expect( + receipt.result is ReceiptResult.COMMITTED and after.calls == 1, + "a `FAILED` effect releases its charge and the budget spends again", + f"it ended {receipt.result} after {after.calls} executor calls", + ) + + try: + return self.graded("G22", selection, store, recorder, body) + finally: + store.close() + # --- G23: a scope provider that cannot answer refuses --------------------------------- def g23(self) -> GuaranteeResult: diff --git a/tests/test_acs.py b/tests/test_acs.py index 01330b86..50700d00 100644 --- a/tests/test_acs.py +++ b/tests/test_acs.py @@ -764,3 +764,70 @@ def test_a_provider_that_names_nobody_is_still_answered_no_principal(store): assert answer["result"]["reason_codes"] == ["no_principal"] assert store.receipts() == () + + +# --- SPEC-v0.9 §2.3 at the ACS boundary: a denial, never a malformed-envelope error --------- + + +def _budgeted_authority(metric="units"): + from ctrlrun import Authority + + return Authority.from_yaml( + "schema: ctrlrun.policy/v7\n" + "authority:\n" + " grants:\n" + " - id: g\n" + ' subject: { agent: "verified-agent" }\n' + ' actions: ["**"]\n' + " budgets:\n" + f" - {{ metric: {metric}, limit: 100, window: PT24H }}\n", + standalone=True, + ) + + +@pytest.mark.authority +def test_an_unmeasurable_budget_is_denied_and_not_called_a_malformed_envelope(store): + """**An ACS answer and the evidence may not disagree about the same action.** + + §2.3's refusal is an `InvalidArgument`, and `handle`'s clause for those answers + `-32002 malformed envelope` with the comment "a malformed payload is not a decision about an + action: there is no action". For this refusal there *is* an action: the kernel has written + `ACTION_DENIED` and a `denied` receipt for it. An independent review found the two + disagreeing. + + An error envelope tells the platform the Guardian could not answer, which it may act on + however it likes. A `deny` tells it what CTRLRun means, which is that the tool must not run. + The `IdentityError` clause immediately below states this rule for its own case. + """ + from ctrlrun import HeaderIdentityProvider + + control = Control(Policy.from_yaml(POLICY), store, authority=_budgeted_authority()) + hook = AcsControlHook( + control, prefix="acs", identity=HeaderIdentityProvider(agent_header="X-Agent") + ) + + answer = hook.handle(_call(amount=200), headers={"X-Agent": "verified-agent"}) + + assert "error" not in answer, answer + assert answer["result"]["decision"] == "deny", answer + assert answer["result"]["reason_codes"] == ["budget_unmeasurable"], answer + # And the evidence says the same thing about the same action. + assert [str(event.type) for event in store.events()][-1] == "ACTION_DENIED" + assert store.receipts()[-1].decision_reason == "budget_unmeasurable" + + +@pytest.mark.authority +def test_a_genuinely_malformed_envelope_is_still_a_protocol_error(store): + """The control. Narrowing the clause must not turn a real protocol fault into a decision: + there really is no action there, and answering `deny` would claim the kernel decided one.""" + control = Control(Policy.from_yaml(POLICY), store, authority=_budgeted_authority()) + from ctrlrun import HeaderIdentityProvider + + hook = AcsControlHook( + control, prefix="acs", identity=HeaderIdentityProvider(agent_header="X-Agent") + ) + + answer = hook.handle({"jsonrpc": "2.0", "id": 1, "method": "nope"}, headers={}) + + assert "error" in answer, answer + assert store.receipts() == () diff --git a/tests/test_budget_holds.py b/tests/test_budget_holds.py new file mode 100644 index 00000000..bc2d8c02 --- /dev/null +++ b/tests/test_budget_holds.py @@ -0,0 +1,1214 @@ +"""T415 to T443: consumption, reconciliation and release (SPEC-v0.9 §4). + +§4.2's table has nineteen rows and the implementation has one rule: **released exactly when the +effect reaches `FAILED`, held in every other state.** One test per disposition, because v0.8's +item 4 needed three attempts on the analogous lapsed-row case: its spec had ten rows and its code +met an eleventh. +""" + +from __future__ import annotations + +import contextvars +import os +import uuid +from datetime import UTC, datetime, timedelta +from typing import Any + +import pytest + +from ctrlrun.action import Action, Principal +from ctrlrun.authority import Authority +from ctrlrun.control import Control +from ctrlrun.effect import EffectState +from ctrlrun.errors import ( + ActionDenied, + AmbiguousEffect, + DuplicateEffect, + InvalidArgument, + NotExecuted, +) +from ctrlrun.policy import Policy +from ctrlrun.receipt import ReceiptResult +from ctrlrun.state import Charge, InMemoryStateStore, SQLiteStateStore + +pytestmark = pytest.mark.authority + +POSTGRES_URL = os.environ.get("CTRLRUN_TEST_POSTGRES") +NOW = datetime(2026, 9, 13, 12, 0, tzinfo=UTC) +DAY = timedelta(hours=24) +LEASE = timedelta(minutes=5) +AGENT = Principal(agent="payer", user="ada") + +DOC = """ +schema: ctrlrun.policy/v7 +environment: prod +actions: + payments.refund: + effect: "refund:{id}" + decision: allow +authority: + grants: + - id: payer + subject: {agent: "payer"} + actions: ["payments.*"] + budgets: + - {metric: amount, limit: 250, window: PT24H} +""" + + +class _Clock: + def __init__(self, now: datetime = NOW) -> None: + self.now = now + + def __call__(self) -> datetime: + return self.now + + def advance(self, by: timedelta) -> None: + self.now += by + + +@pytest.fixture +def clock() -> _Clock: + return _Clock() + + +@pytest.fixture( + params=[ + "in-memory", + "sqlite", + pytest.param( + "postgres", + marks=pytest.mark.skipif( + POSTGRES_URL is None, reason="CTRLRUN_TEST_POSTGRES is not set" + ), + ), + ] +) +def store(request, tmp_path, clock): + if request.param == "in-memory": + made: Any = InMemoryStateStore(clock=clock) + elif request.param == "sqlite": + made = SQLiteStateStore(tmp_path / "state.db", clock=clock) + else: + from ctrlrun.postgres import PostgresStateStore + + schema = f"holds_{uuid.uuid4().hex[:12]}" + PostgresStateStore.create_schema(POSTGRES_URL, schema) + made = PostgresStateStore(POSTGRES_URL, schema=schema, clock=clock) + yield made + made.close() + if request.param == "postgres": + from ctrlrun.postgres import PostgresStateStore + + PostgresStateStore.drop_schema(POSTGRES_URL, schema) + + +def _control(store, clock) -> Control: + return Control( + policy=Policy.from_yaml(DOC, source=""), + store=store, + clock=clock, + environment="prod", + authority=Authority.from_yaml(DOC, source=""), + ) + + +def _action(identifier: str = "1", amount: int = 100) -> Action: + return Action( + name="payments.refund", + arguments={"amount": amount, "id": identifier}, + principal=AGENT, + environment="prod", + ) + + +def _held(store) -> int: + return sum(row.amount for row in store.consumptions() if row.released_at is None) + + +def test_T415_commit_holds_permanently(store, clock) -> None: + """§4.2 row 1. A committed spend is a spend.""" + control = _control(store, clock) + control.execute(_action(), lambda: {"ok": True}, "refund:1") + assert _held(store) == 100 + + +def test_T416_fail_releases(store, clock) -> None: + """§4.2 row 2. The executor proved nothing happened (`v0.1 §5.5`).""" + control = _control(store, clock) + with pytest.raises(NotExecuted): + control.execute( + _action(), lambda: (_ for _ in ()).throw(NotExecuted("nothing happened")), "refund:1" + ) + assert _held(store) == 0, "a FAILED effect must release its charge" + + +def test_T417_ambiguity_holds(store, clock) -> None: + """§4.2 row 3, and **R2: ambiguity is not a refund.** + + The correctness hole that parked budgets for four milestones. If ambiguity released the hold, + an agent that can generate ambiguity could generate unlimited authority, and generating + ambiguity is free for any flaky integration. + """ + control = _control(store, clock) + with pytest.raises(RuntimeError): + control.execute( + _action(), lambda: (_ for _ in ()).throw(RuntimeError("who knows")), "refund:1" + ) + assert store.get_effect("refund:1").state is EffectState.AMBIGUOUS + assert _held(store) == 100, "an AMBIGUOUS effect must keep its consumption" + + +def test_T418_a_lapsed_lease_holds(store, clock) -> None: + """§4.2 row 4. No transition has occurred, so nothing is released.""" + store.reserve_effect("e1", "a", LEASE, (Charge("payer", "amount", 100, 250, DAY),)) + clock.advance(LEASE * 2) + assert _held(store) == 100 + + +def test_T419_a_human_resolving_FAILED_releases(store, clock) -> None: + """§4.2's `resolve_effect(FAILED)` row, and the defect it caught. + + **`resolve_effect` does not go through `_transition`**, so the release had to be written on + that path as well. Without it the one act meant to free a held charge, a human saying the + effect did not happen, would have held it for ever, which is the exact opposite of R2's + intent. Found when G22's scenario tried to free its own hold. + """ + control = _control(store, clock) + with pytest.raises(RuntimeError): + control.execute( + _action(), lambda: (_ for _ in ()).throw(RuntimeError("who knows")), "refund:1" + ) + assert _held(store) == 100 + store.resolve_effect("refund:1", EffectState.FAILED, "ada@example.com") + assert _held(store) == 0, "a human's FAILED must release, and this path bypasses _transition" + + +def test_T420_a_human_resolving_COMMITTED_holds(store, clock) -> None: + """The other half of the same row: the human said it happened.""" + control = _control(store, clock) + with pytest.raises(RuntimeError): + control.execute( + _action(), lambda: (_ for _ in ()).throw(RuntimeError("who knows")), "refund:1" + ) + store.resolve_effect("refund:1", EffectState.COMMITTED, "ada@example.com") + assert _held(store) == 100 + + +def test_T421_the_release_is_idempotent(store, clock) -> None: + """§4.4. A compare-and-set on the flag, never a decrement: `v0.6 §4.3.2` Table A2 row 2 + re-issues a lost `UPDATE` once, and a decrement would subtract twice.""" + store.reserve_effect("e1", "a", LEASE, (Charge("payer", "amount", 100, 250, DAY),)) + store.begin_execution("e1", "a") + store.fail_effect("e1", "a", "nothing happened") + first = next(row.released_at for row in store.consumptions()) + store.reserve_effect("e1", "a", LEASE, (Charge("payer", "amount", 100, 250, DAY),)) + store.begin_execution("e1", "a") + store.fail_effect("e1", "a", "again") + again = next(row.released_at for row in store.consumptions()) + assert again == first, "a re-issued release must not move a timestamp already set" + + +def test_T426_a_renewal_after_FAILED_charges_again(store, clock) -> None: + """§4.3. `FAILED` is the only state that re-reserves one effect key, and it **should** + charge again: the release already happened and the failure proves the first spend did not.""" + control = _control(store, clock) + with pytest.raises(NotExecuted): + control.execute(_action(), lambda: (_ for _ in ()).throw(NotExecuted("no")), "refund:1") + assert _held(store) == 0 + control.execute(_action(), lambda: {"ok": True}, "refund:1") + rows = store.consumptions() + assert len(rows) == 2, "the renewal writes its own row" + assert {row.attempt for row in rows} == {1, 2}, "distinct by attempt, per §3.4's key" + assert _held(store) == 100 + + +def test_T433_the_refusal_names_the_grant_the_metric_and_the_window(store, clock) -> None: + """§4.5, and **not the remaining amount**, asserted by word. + + A refusal reporting the balance is an oracle: refused actions cost nothing, so an attacker + binary-searches the exact limit in a few dozen refusals. + """ + control = _control(store, clock) + control.execute(_action("1", 250), lambda: {"ok": True}, "refund:1") + with pytest.raises(ActionDenied) as caught: + control.execute(_action("2", 1), lambda: {"ok": True}, "refund:2") + assert caught.value.reason == "budget_exhausted" + message = str(caught.value) + assert "payer" in message and "amount" in message + assert "250" not in message and "249" not in message, ( + f"the refusal discloses the balance, which is an oracle: {message}" + ) + + +def test_T437_a_budget_refusal_writes_no_approval_event(store, clock) -> None: + """§3.3.2's hazard, which item 2 met first with the scope refusal. + + `_secure`'s `ActionDenied` handler appends `APPROVAL_DENIED` unconditionally, so a budget + refusal routed through it fabricates an approval denial for an action no human ever saw. + """ + control = _control(store, clock) + control.execute(_action("1", 250), lambda: {"ok": True}, "refund:1") + with pytest.raises(ActionDenied): + control.execute(_action("2", 1), lambda: {"ok": True}, "refund:2") + kinds = [event.type.value for event in store.events()] + assert "APPROVAL_DENIED" not in kinds + denied = [r for r in store.receipts() if r.result is ReceiptResult.DENIED] + assert len(denied) == 1 and denied[0].decision_reason == "budget_exhausted" + + +def test_T440_a_negative_metric_value_is_refused(store, clock) -> None: + """§2.3. A negative amount would reduce the rolling sum and refill the budget, which is the + compensation §12 forbids. The test that proves it matters alternates `+n` and `-n`.""" + control = _control(store, clock) + control.execute(_action("1", 250), lambda: {"ok": True}, "refund:1") + with pytest.raises(InvalidArgument): + control.execute(_action("2", -250), lambda: {"ok": True}, "refund:2") + assert _held(store) == 250, "a refused negative must not have moved the sum" + + +def test_T441_an_action_with_no_metric_argument_is_refused(store, clock) -> None: + """§2.3: a missing value is refused, never counted as zero. Treating absence as zero turns + the absence of a field into unlimited authority.""" + control = _control(store, clock) + action = Action( + name="payments.refund", arguments={"id": "1"}, principal=AGENT, environment="prod" + ) + with pytest.raises(InvalidArgument) as caught: + control.execute(action, lambda: {"ok": True}, "refund:1") + # **The message, not just the type.** A mutation run found this guard removable: with it + # gone, `None` falls through to the non-integer check and raises anyway, so the test passed + # for a reason that was not this rule. That is CONTRIBUTING.md's first pattern, a subsumed + # guard, and the list allows keeping one for its message on the condition a test asserts it. + assert "carries no 'amount' argument" in str(caught.value), str(caught.value) + + +def test_T442_a_budgeted_grant_refuses_an_action_with_no_effect_key(store, clock) -> None: + """§2.4.1, in the third place this check has lived and the one the probes point at. + + Without it an agent proposes actions carrying no `effect:` template and spends nothing + against every budget on the chain, for ever. + """ + control = _control(store, clock) + with pytest.raises(InvalidArgument) as caught: + control.execute(_action(), lambda: {"ok": True}, None) + assert "effect" in str(caught.value) + assert store.consumptions() == () + + +def test_T412b_every_ancestor_is_charged_through_a_real_chain(store, clock) -> None: + """§2.7, driven end to end rather than at the store. + + **The rule that makes the feature mean anything**, and a mutation run found nothing exercising + it through `Control`: removing the ancestor walk left 84 tests green. Without it a holder of a + 250-a-day grant delegates children, each correctly contained, and every child spends the + parent's budget over again. + """ + from ctrlrun.authority import Grant, Subject + + delegable = DOC.replace( + " - {metric: amount, limit: 250, window: PT24H}", + " - {metric: amount, limit: 250, window: PT24H}\n" + " delegable: true\n" + ' expires_at: "2027-01-01T00:00:00Z"', + ) + control = Control( + policy=Policy.from_yaml(delegable, source=""), + store=store, + clock=clock, + environment="prod", + authority=Authority.from_yaml(delegable, source=""), + ) + child = Grant( + id="", + subject=Subject(agent="payer", user="ada"), + actions=("payments.refund",), + expires_at=datetime(2026, 12, 1, tzinfo=UTC), + budgets=((control.authority.grants["payer"].budgets or ())[0],), + ) + delegation = control.delegate("payer", child, by=AGENT) + + control.execute(_action("1", 100), lambda: {"ok": True}, "refund:1") + charged = {row.grant_id for row in store.consumptions()} + assert charged == {"payer", delegation.delegation_id}, ( + f"every ancestor must be charged, not only the grant that decided: {charged}" + ) + # And the parent's budget is what refuses, even though the child is within its own. + with pytest.raises(ActionDenied) as caught: + control.execute(_action("2", 200), lambda: {"ok": True}, "refund:2") + assert caught.value.reason == "budget_exhausted" + + +def test_T443_a_refused_receipt_records_no_charge(store, clock) -> None: + """SPEC-v0.9 §10.1, and an independent review found the receipt lying. + + `budget_charges` was stamped where the charges were computed, which is before `_take` + attempts the transaction that applies them. Every refusal raised later in `_secure`'s loop + then reached `_record` with them set, so a `denied` receipt claimed the action charged the + very grant it was refused from spending against. + + **A receipt asserting a spend that never happened is the one thing an evidence trail may not + do**, and it is worse than an absent field, because a reader has no way to tell it apart from + a real one. + """ + control = _control(store, clock) + control.execute(_action("1", 250), lambda: {"ok": True}, "refund:1") + with pytest.raises(ActionDenied): + control.execute(_action("2", 100), lambda: {"ok": True}, "refund:2") + + committed = [r for r in store.receipts() if r.result is ReceiptResult.COMMITTED] + denied = [r for r in store.receipts() if r.result is ReceiptResult.DENIED] + assert [dict(c) for c in committed[0].budget_charges] == [ + {"grant_id": "payer", "metric": "amount", "amount": 250} + ] + assert denied[0].budget_charges == (), ( + "a refused action charged nothing; its receipt must not say otherwise" + ) + + +def test_T406a_two_budgets_on_one_metric_is_the_shape_SS2_2_exists_for(store, clock) -> None: + """§2.2's own motivating shape, which an earlier duplicate guard killed at execute. + + "Two budgets on one metric over two windows is the first thing an operator asks for." It + arrives as two charges differing only in `limit` and `window`: both predicates run, §3.4's + key writes **one** row, because it is one spend measured against two windows. + + An independent review found the previous guard refusing any duplicate pair, so the loader + accepted the document, observe mode reported it clean, `ctrlrun verify` could not grade it, + and enforce mode died with no receipt and no event. + """ + text = DOC.replace( + " - {metric: amount, limit: 250, window: PT24H}", + " - {metric: amount, limit: 250, window: PT24H}\n" + " - {metric: amount, limit: 5000, window: P30D}", + ) + control = Control( + policy=Policy.from_yaml(text, source=""), + store=store, + clock=clock, + environment="prod", + authority=Authority.from_yaml(text, source=""), + ) + control.execute(_action("1", 100), lambda: {"ok": True}, "refund:1") + control.execute(_action("2", 100), lambda: {"ok": True}, "refund:2") + assert len(store.consumptions()) == 2, "one row per spend, not one per budget" + # The daily budget binds first, and its window is the one named. + with pytest.raises(ActionDenied) as caught: + control.execute(_action("3", 100), lambda: {"ok": True}, "refund:3") + assert caught.value.reason == "budget_exhausted" + # And the monthly one still holds after the daily window rolls. + clock.advance(DAY + timedelta(seconds=1)) + for index in range(3, 31): + try: + control.execute(_action(str(index), 100), lambda: {"ok": True}, f"refund:{index}") + except ActionDenied: + break + clock.advance(DAY + timedelta(seconds=1)) + held = sum(row.amount for row in store.consumptions() if row.released_at is None) + assert held <= 5000, f"the monthly budget was exceeded: {held}" + + +def test_T406b_two_charges_on_one_metric_with_different_amounts_are_refused(store, clock) -> None: + """The hazard the guard is actually for: §3.4's key carries no window, so differing amounts + would collapse to whichever row landed first and the ledger would under-record the spend.""" + with pytest.raises(InvalidArgument): + store.reserve_effect( + "e1", + "a", + LEASE, + ( + Charge("payer", "amount", 100, 250, DAY), + Charge("payer", "amount", 900, 5000, timedelta(days=30)), + ), + ) + assert store.consumptions() == () + + +# --- §4.2's rows that had no test, and the mutant that survived without them ------------------ + + +def test_T425_the_release_is_keyed_on_the_state_reached_not_the_call(store, clock) -> None: + """**§4.2's warning paragraph, and the mutant that survived the whole suite without it.** + + An independent review moved `_release_locked` above the state check, so the release keyed on + the *call* rather than the state reached, and all 42 tests passed. It is not an equivalent + mutant: a `fail_effect` that is **refused** because the record moved on then releases the hold + on an `AMBIGUOUS` record, which is the manufacturable refund this whole item exists to stop. + + "The release is keyed on the record reaching `FAILED`, never on the call that tried to put it + there." + + The mutant only bites in memory. Both SQL stores run the transition inside one transaction and + roll it back when the check raises, so there the order is equivalent and the rollback carries + §4.1. The in-memory store mutates a dict under a lock and has no rollback, so the order **is** + the atomicity. The test runs on all three anyway: which backend enforces §4.1 by which + mechanism is an implementation detail, and the guarantee is not. + """ + from ctrlrun.errors import AmbiguousEffect + + store.reserve_effect("e1", "a", LEASE, (Charge("payer", "amount", 100, 250, DAY),)) + store.begin_execution("e1", "a") + # The record moves on under the attempt: a human, or another process, declares it ambiguous. + store.mark_ambiguous("e1", "a", "the outcome is unknown") + assert _held(store) == 100 + + with pytest.raises(AmbiguousEffect): + store.fail_effect("e1", "a", "the executor says it did not happen") + + assert _held(store) == 100, ( + "a REFUSED fail_effect released the hold: the release is keyed on the call, not the " + "state reached, and somebody may have committed this effect" + ) + + +def test_T423_a_suspension_holds_its_charge(store, clock) -> None: + """§4.2's suspension row. A continuation extends the lease; no transition, so no release. + + A suspension is the one state that can outlive a whole budget window, so "held in every other + state" is load-bearing here: an elicitation that sits for a day must not let the same grant + spend its daily limit twice. + """ + action = _action() + store.reserve_effect("e1", action.action_id, LEASE, (Charge("payer", "amount", 100, 250, DAY),)) + store.begin_execution("e1", action.action_id) + store.hold_continuation(action, "e1", "cont-1", clock.now + timedelta(hours=1)) + assert _held(store) == 100 + clock.advance(DAY + timedelta(seconds=1)) + assert _held(store) == 100, "a suspension outliving its window must still hold its charge" + + +def test_T424_begin_execution_moves_nothing_in_the_ledger(store, clock) -> None: + """§4.2's `begin_execution` row. Listed because the table claims completeness.""" + store.reserve_effect("e1", "a", LEASE, (Charge("payer", "amount", 100, 250, DAY),)) + before = [(row.effect_key, row.released_at) for row in store.consumptions()] + store.begin_execution("e1", "a") + assert [(row.effect_key, row.released_at) for row in store.consumptions()] == before + + +def test_T422_a_lapsed_lease_another_planner_ambiguates_still_holds(store, clock) -> None: + """§4.2's row 5. The record is `AMBIGUOUS` now, and R2 applies: the charge stays.""" + store.reserve_effect("e1", "a", LEASE, (Charge("payer", "amount", 100, 250, DAY),)) + clock.advance(LEASE * 2) + with pytest.raises(AmbiguousEffect): + store.reserve_effect("e1", "b", LEASE, (Charge("payer", "amount", 100, 250, DAY),)) + assert store.get_effect("e1").state is EffectState.AMBIGUOUS + assert _held(store) == 100 + + +def test_T427_a_refused_commit_releases_nothing(store, clock) -> None: + """§4.2's `commit_effect` refused row: §4.1 over the state actually reached.""" + from ctrlrun.errors import AmbiguousEffect + + store.reserve_effect("e1", "a", LEASE, (Charge("payer", "amount", 100, 250, DAY),)) + store.begin_execution("e1", "a") + store.mark_ambiguous("e1", "a", "unknown") + with pytest.raises(AmbiguousEffect): + store.commit_effect("e1", "a", {"ok": True}) + assert _held(store) == 100 + + +def test_T428_a_human_resolving_FAILED_mid_flight_releases_and_the_call_does_not( + store, clock +) -> None: + """The sub-case the review named: a human resolves `FAILED` while an attempt runs, so the + charge is **already released** and the refused call releases nothing further.""" + from ctrlrun.errors import CTRLRunError + + store.reserve_effect("e1", "a", LEASE, (Charge("payer", "amount", 100, 250, DAY),)) + store.begin_execution("e1", "a") + store.mark_ambiguous("e1", "a", "unknown") + store.resolve_effect("e1", EffectState.FAILED, "ada@example.com") + assert _held(store) == 0 + released = [row.released_at for row in store.consumptions()] + with pytest.raises(CTRLRunError): + store.fail_effect("e1", "a", "the executor says so too") + assert [row.released_at for row in store.consumptions()] == released + + +# --- §4.2's rows that only the Control route can reach --------------------------------------- + +CEILING_DOC = DOC.replace( + " decision: allow", " decision: allow\n max_attempts: 3" +).replace("limit: 250", "limit: 1000") + + +def _ceiling_control(store, clock) -> Control: + return Control( + policy=Policy.from_yaml(CEILING_DOC, source=""), + store=store, + clock=clock, + environment="prod", + authority=Authority.from_yaml(CEILING_DOC, source=""), + ) + + +def _boom() -> Any: + raise NotExecuted("the remote rejected it before doing anything") + + +def test_T429_the_ceiling_refusing_after_the_reservation_was_won_releases(store, clock) -> None: + """§4.2's ceiling row, and three others on the way to it. + + **This is the shape v0.8's item 4 missed.** The kernel wins the reservation, charges for it, + then refuses on its own attempt ceiling and drives `begin_execution` + `fail_effect` itself. + The executor never ran, so by §4.1 the charge is released, and the release is driven by the + kernel rather than by any outcome. + + The route also covers the `reconcile` hook row (the hook moves the record to `FAILED`, which + releases the first charge like a human's `resolve_effect(FAILED)`) and the second-`_take` row + (the renewal takes a **fresh** charge, per §4.3). + """ + control = _ceiling_control(store, clock) + for _ in range(2): + with pytest.raises(NotExecuted): + control.execute(_action(), _boom, "refund:1") + with pytest.raises(TimeoutError): + control.execute(_action(), lambda: (_ for _ in ()).throw(TimeoutError("lost")), "refund:1") + assert store.get_effect("refund:1").state is EffectState.AMBIGUOUS + assert _held(store) == 100, "R2: the ambiguous attempt's charge is held" + + with pytest.raises(ActionDenied) as refused: + control.execute(_action(), _boom, "refund:1", reconcile=lambda key: "not_executed") + assert refused.value.reason == "attempt_ceiling" + assert store.get_effect("refund:1").state is EffectState.FAILED + # The hook released the ambiguous charge; the renewal took a fresh one; the kernel's own + # fail_effect released that one too. Every row is charged, and every row is released. + assert _held(store) == 0 + assert len(store.consumptions()) == 4, "three attempts plus the renewal, each charged once" + + +def test_T430_begin_execution_refused_after_the_reservation_was_won_holds(store, clock) -> None: + """§4.2's `begin_execution`-refused row: the reservation is won and charged, then taken away. + + Mechanically the lapsed-lease row, but a distinct call path: the kernel holds a reservation it + can no longer execute against. The charge is **held**, by the ambiguity rule, because nobody + can say the effect did not happen. + """ + control = _control(store, clock) + taken: list[str] = [] + + def steal() -> Any: # pragma: no cover - never reached + raise AssertionError("the executor must not run") + + original = store.begin_execution + + def refuse(effect_key: str, action_id: str) -> Any: + taken.append(effect_key) + store.mark_ambiguous(effect_key, action_id, "another process got there first") + return original(effect_key, action_id) + + store.begin_execution = refuse # type: ignore[method-assign] + with pytest.raises(AmbiguousEffect): + control.execute(_action(), steal, "refund:1") + assert taken == ["refund:1"] + assert _held(store) == 100, "a reservation taken away is ambiguous, and R2 holds the charge" + + +def test_T431_mark_ambiguous_refused_moves_nothing(store, clock) -> None: + """§4.2's `mark_ambiguous`-refused row. It folds the refusal into the error text rather than + calling `_unrecorded`, so §4.1 applies over the state the record actually reached.""" + store.reserve_effect("e1", "a", LEASE, (Charge("payer", "amount", 100, 250, DAY),)) + store.begin_execution("e1", "a") + store.commit_effect("e1", "a", {"ok": True}) + released = [row.released_at for row in store.consumptions()] + with pytest.raises(DuplicateEffect): + store.mark_ambiguous("e1", "a", "too late") + assert [row.released_at for row in store.consumptions()] == released + assert _held(store) == 100, "the record reached COMMITTED, and a committed spend is a spend" + + +def test_T432_a_refused_retry_charges_nothing(store, clock) -> None: + """§4.2's last row. The refusal happens in `plan_reservation`, **before** any reservation is + won, so there is nothing to charge and nothing to release. + + The retry is refused rather than answered from the record: `DuplicateEffect` is the kernel + telling the caller the effect already happened, which is the point. What matters to §4.2 is + that the second call leaves the ledger exactly as the first left it. + """ + from ctrlrun.errors import DuplicateEffect + + control = _control(store, clock) + control.execute(_action(), lambda: {"ok": True}, "refund:1") + before = [(row.effect_key, row.amount, row.released_at) for row in store.consumptions()] + with pytest.raises(DuplicateEffect): + control.execute(_action(), lambda: {"ok": True}, "refund:1") + after = [(row.effect_key, row.amount, row.released_at) for row in store.consumptions()] + assert after == before, "a refused retry is not a second spend" + assert _held(store) == 100 + + +# --- §2.3 and §2.4.1 refuse, and a refusal is a thing the operator can see -------------------- + +APPROVE_DOC = DOC.replace("decision: allow", "decision: approve") + + +def _approving_control(store, clock): + from ctrlrun.approval import LocalApprovalProvider + + return Control( + policy=Policy.from_yaml(APPROVE_DOC, source=""), + store=store, + approvals=LocalApprovalProvider(store, clock=clock, poll_interval=timedelta(0)), + clock=clock, + environment="prod", + authority=Authority.from_yaml(APPROVE_DOC, source=""), + ) + + +def test_T444_an_unmeasurable_action_is_refused_with_an_event_and_a_receipt(store, clock) -> None: + """§2.3 and §2.4.1 refuse. **A refusal nobody can see is not a refusal.** + + A review found these three escaping as bare `InvalidArgument` with no `ACTION_DENIED` and no + receipt, which leaves the one record an operator has of a refused action empty. The exception + type stays what §2.3 says it is: this is an argument the kernel cannot measure, not a budget + that ran out. + """ + control = _control(store, clock) + for arguments, fragment in ( + ({"amount": -250, "id": "1"}, "-250"), + ({"id": "1"}, "carries no 'amount' argument"), + ): + before = len(store.events()) + with pytest.raises(InvalidArgument) as caught: + control.execute( + Action( + name="payments.refund", + arguments=arguments, + principal=AGENT, + environment="prod", + ), + lambda: {"ok": True}, + "refund:1", + ) + assert fragment in str(caught.value) + written = [str(event.type) for event in store.events()][before:] + assert "ACTION_DENIED" in written, written + receipt = store.receipts()[-1] + assert receipt.result is ReceiptResult.DENIED + assert receipt.decision_reason == "budget_unmeasurable", receipt.decision_reason + assert store.consumptions() == (), "nothing was charged for any of them" + + +def test_T445_a_keyless_budgeted_action_is_refused_with_an_event_and_a_receipt( + store, clock +) -> None: + """§2.4.1's refusal, given the same treatment. Its reason is distinct from §2.3's because an + operator who declared a budget on an action with no `effect:` template has a different thing + to fix than one whose agent proposed a negative amount.""" + control = _control(store, clock) + with pytest.raises(InvalidArgument): + control.execute(_action(), lambda: {"ok": True}, None) + assert "ACTION_DENIED" in [str(event.type) for event in store.events()] + receipt = store.receipts()[-1] + assert receipt.result is ReceiptResult.DENIED + assert receipt.decision_reason == "budget_unkeyed", receipt.decision_reason + + +def test_T446_no_human_is_asked_to_approve_an_action_the_kernel_will_refuse(store, clock) -> None: + """**The ordering half.** §2.3's and §2.4.1's refusals do not depend on anything the approval + gate produces, and they are unconditional: the action can never run, whatever a human says. + + Running them after the gate asks a human to sit and approve a refund the kernel has already + decided to refuse, and leaves a granted approval in the store for an action nothing can + execute. A probe found `APPROVAL_REQUESTED` written for exactly that shape. + """ + control = _approving_control(store, clock) + action = Action( + name="payments.refund", + arguments={"amount": -250, "id": "1"}, + principal=AGENT, + environment="prod", + ) + with pytest.raises(InvalidArgument): + control.execute(action, lambda: {"ok": True}, "refund:1") + written = [str(event.type) for event in store.events()] + assert "APPROVAL_REQUESTED" not in written, written + assert "ACTION_DENIED" in written, written + + +# --- §4.2's resumed leg: the only receipt an MCP or ACS action ever gets ---------------------- + + +def test_T447_a_resumed_leg_reports_the_charges_its_first_leg_took(store, clock) -> None: + """§8.3. **The resumed leg's receipt is the whole evidence for that action**, so it has to + say what the action spent. + + `resume` never touched `_BUDGET_CHARGES`, so the field came from whatever the contextvar + happened to hold. A resumption in a fresh context reported no charges at all for an action + that had spent 100; a resumption after another `execute` in the same context reported *that + action's* spend. Both put a false number on the only receipt there is. + """ + from ctrlrun import Suspended + + control = _control(store, clock) + + def suspends() -> Any: + raise Suspended("round-1") + + with pytest.raises(Suspended): + control.execute(_action("1", 100), suspends, "refund:1") + assert _held(store) == 100 + + # **In a fresh context**, which is the shape a continuation exists for: `hold_continuation` + # carries the whole `Action` "because a resumption is *the same action*, and rehydrating it + # from the store is the only way a gateway that restarted mid-round can still finish one." + # A restarted gateway has no contextvar left, so reading one is reading nothing. + receipt = contextvars.Context().run(control.resume, "round-1", lambda: {"ok": True}) + assert receipt.budget_charges == ({"grant_id": "payer", "metric": "amount", "amount": 100},), ( + receipt.budget_charges + ) + + +def test_T448_a_resumed_leg_never_reports_another_actions_charges(store, clock) -> None: + """The stale half, which is the one that puts a *wrong* number on a receipt rather than a + missing one. One `Control`, one thread, two actions: the second must not inherit the first.""" + from ctrlrun import Suspended + + control = _control(store, clock) + + def suspends() -> Any: + raise Suspended("round-1") + + with pytest.raises(Suspended): + control.execute(_action("1", 100), suspends, "refund:1") + # An unbudgeted action runs to completion in the same context, leaving its own charges set. + control.execute(_action("2", 25), lambda: {"ok": True}, "refund:2") + + receipt = control.resume("round-1", lambda: {"ok": True}) + amounts = [charge["amount"] for charge in receipt.budget_charges] + assert amounts == [100], f"the resumed leg inherited the other action's spend: {amounts}" + + +def test_T449_an_unbudgeted_resumed_leg_reports_no_charges(store, clock) -> None: + """The other direction: a resumption must not manufacture charges either. A store with a + ledger and an action with no budget on its grant reports an empty tuple, not the last thing + the contextvar saw.""" + from ctrlrun import Suspended + + unbudgeted = DOC.replace( + " budgets:\n - {metric: amount, limit: 250, window: PT24H}\n", "" + ) + control = Control( + policy=Policy.from_yaml(unbudgeted, source=""), + store=store, + clock=clock, + environment="prod", + authority=Authority.from_yaml(unbudgeted, source=""), + ) + + def suspends() -> Any: + raise Suspended("round-1") + + with pytest.raises(Suspended): + control.execute(_action("1", 100), suspends, "refund:1") + receipt = control.resume("round-1", lambda: {"ok": True}) + assert receipt.budget_charges == () + + +def test_T450_a_resumed_leg_after_a_renewal_reports_only_its_own_attempts_charges( + store, clock +) -> None: + """§4.3 gives a renewal a **new** charge, so an effect that failed and renewed has two rows + in the ledger. The resumed receipt reports the attempt it is actually on. + + Without the attempt filter the receipt sums a spend that was already released with the one + the action is holding, and claims the action spent twice what it did. + """ + from ctrlrun import Suspended + + control = _control(store, clock) + with pytest.raises(NotExecuted): + control.execute(_action("1", 100), _boom, "refund:1") + assert _held(store) == 0, "§4.2: the failed attempt released its charge" + + def suspends() -> Any: + raise Suspended("round-1") + + with pytest.raises(Suspended): + control.execute(_action("1", 100), suspends, "refund:1") + assert len(store.consumptions()) == 2, "one row per attempt, per §4.3" + + receipt = contextvars.Context().run(control.resume, "round-1", lambda: {"ok": True}) + assert receipt.budget_charges == ({"grant_id": "payer", "metric": "amount", "amount": 100},), ( + receipt.budget_charges + ) + assert receipt.attempt == 2, receipt.attempt + + +# --- §4.2.1: observe mode charges nothing, and says what would have been refused -------------- + +OBSERVE_DOC = DOC.replace("environment: prod", "environment: prod\nmode: observe") + + +def _observing_control(store, clock) -> Control: + return Control( + policy=Policy.from_yaml(OBSERVE_DOC, source=""), + store=store, + clock=clock, + environment="prod", + authority=Authority.from_yaml(OBSERVE_DOC, source=""), + ) + + +def test_T439_observe_mode_charges_nothing(store, clock) -> None: + """§4.2.1, first half. `v0.3 §6.2`'s observe mode enforces nothing and records what it would + have done. A budget consumed there would be the one check in the kernel that enforced under + observation: the run would refuse at the limit while claiming to be observing, and the + counterfactual an operator adopts observe mode to get would be wrong. + """ + control = _observing_control(store, clock) + for index in range(1, 6): + receipt = control.execute(_action(str(index), 100), lambda: {"ok": True}, f"refund:{index}") + assert receipt.result is ReceiptResult.OBSERVED + assert store.consumptions() == (), "observe mode wrote to the ledger" + + +def test_T439a_observe_mode_reports_the_budget_that_would_have_refused(store, clock) -> None: + """§4.2.1, second half, and the half that did not exist. The report says the action *would + have been* refused on a budget, naming the grant and the metric, exactly as it reports what a + policy would have decided. + + The shape that reaches it is a **mixed** deployment: the ledger carries enforced spend, and a + new action is being piloted in observe mode against the same grant. §4.2.1a says why a + deployment observing everything reports nothing here. + """ + enforcing = _control(store, clock) + enforcing.execute(_action("1", 250), lambda: {"ok": True}, "refund:1") + + observing = _observing_control(store, clock) + receipt = observing.execute(_action("2", 100), lambda: {"ok": True}, "refund:2") + + assert receipt.result is ReceiptResult.OBSERVED, "it ran: observe mode refuses nothing" + assert receipt.would_have is not None + assert receipt.would_have.blocked_reason == "budget_exhausted", receipt.would_have + assert store.consumptions() == store.consumptions(grant_id="payer") + assert len(store.consumptions()) == 1, "the observed run charged nothing of its own" + + +def test_T439b_observe_mode_reports_nothing_when_the_budget_has_room(store, clock) -> None: + """The negative. Without it T439a passes for a `block` that fires unconditionally.""" + enforcing = _control(store, clock) + enforcing.execute(_action("1", 100), lambda: {"ok": True}, "refund:1") + + observing = _observing_control(store, clock) + receipt = observing.execute(_action("2", 100), lambda: {"ok": True}, "refund:2") + blocked = receipt.would_have.blocked_reason if receipt.would_have else None + assert blocked != "budget_exhausted", blocked + + +def test_T439c_an_unmeasurable_action_under_observation_reports_and_denies_nothing( + store, clock +) -> None: + """§2.3 and §2.4.1 under `v0.3 §6.2`. Enforce mode refuses these, so observe mode's job is to + say so, and its job is equally to write no `denied` receipt while doing it. + + Routing them through `_refuse_unmeasurable` unguarded would have observe mode record a + refusal it did not make, on top of the `observed` receipt for the run that went ahead: two + receipts for one action, disagreeing. + """ + control = _observing_control(store, clock) + receipt = control.execute( + Action( + name="payments.refund", + arguments={"amount": -250, "id": "1"}, + principal=AGENT, + environment="prod", + ), + lambda: {"ok": True}, + "refund:1", + ) + assert receipt.result is ReceiptResult.OBSERVED, "observe mode refuses nothing" + assert receipt.would_have is not None + assert receipt.would_have.blocked_reason == "budget_unmeasurable", receipt.would_have + assert [r.result for r in store.receipts()] == [ReceiptResult.OBSERVED], store.receipts() + assert store.consumptions() == () + + +def test_T439d_an_observed_receipt_carries_the_charge_it_would_have_taken(store, clock) -> None: + """§4.2.1a. The counterfactual spend, which is the number a budget is sized from. + + A probe found observed receipts carrying an empty tuple, which made §4.2.1a's sizing path + impossible: the ledger is empty under observation, so if the receipts do not carry what the + action would have been charged, nothing anywhere records it. + + It is not a claim that anything was spent. The receipt says `observed`, the ledger is empty, + and `v0.3 §6.2` makes every number on an observed receipt a counterfactual. + """ + control = _observing_control(store, clock) + receipt = control.execute(_action("1", 100), lambda: {"ok": True}, "refund:1") + assert receipt.result is ReceiptResult.OBSERVED + assert receipt.budget_charges == ({"grant_id": "payer", "metric": "amount", "amount": 100},), ( + receipt.budget_charges + ) + assert store.consumptions() == (), "the counterfactual is on the receipt, not in the ledger" + + +def test_T439e_an_observed_receipt_for_an_unbudgeted_grant_carries_none(store, clock) -> None: + """The negative, so T439d cannot pass for a field that is always populated.""" + unbudgeted = OBSERVE_DOC.replace( + " budgets:\n - {metric: amount, limit: 250, window: PT24H}\n", "" + ) + control = Control( + policy=Policy.from_yaml(unbudgeted, source=""), + store=store, + clock=clock, + environment="prod", + authority=Authority.from_yaml(unbudgeted, source=""), + ) + receipt = control.execute(_action("1", 100), lambda: {"ok": True}, "refund:1") + assert receipt.budget_charges == () + + +# --- §4.2.1: the report and the enforcement may not drift on *which* refusal ------------------ + + +def _both_modes(store, clock, document): + """One document, two Controls over separate stores: what enforce did, what observe said.""" + enforcing = Control( + policy=Policy.from_yaml(document, source=""), + store=store, + clock=clock, + environment="prod", + authority=Authority.from_yaml(document, source=""), + ) + observed = document.replace("environment: prod", "environment: prod\nmode: observe") + watcher = InMemoryStateStore(clock=clock) + observing = Control( + policy=Policy.from_yaml(observed, source=""), + store=watcher, + clock=clock, + environment="prod", + authority=Authority.from_yaml(observed, source=""), + ) + return enforcing, observing, watcher + + +def test_T451_observe_reports_the_refusal_enforce_makes_when_a_human_would_be_asked( + store, clock +) -> None: + """§4.2.1: "the report and the enforcement cannot drift." + + T446 moved §2.3's refusal above the approval gate in enforce mode, because the action cannot + run whatever a human says. Observe mode's copy stayed below it, so a pilot was told a human + would have been asked about an action enforce refuses before anybody is asked. That is the + exact defect T446 fixed, surviving on the other side of the mode switch, and an independent + review found it. + """ + document = DOC.replace("decision: allow", "decision: approve") + enforcing, observing, watcher = _both_modes(store, clock, document) + bad = Action( + name="payments.refund", + arguments={"amount": -250, "id": "1"}, + principal=AGENT, + environment="prod", + ) + + with pytest.raises(InvalidArgument): + enforcing.execute(bad, lambda: {"ok": True}, "refund:1") + enforced = store.receipts()[-1].decision_reason + + receipt = observing.execute(bad, lambda: {"ok": True}, "refund:1") + + assert enforced == "budget_unmeasurable" + assert receipt.would_have is not None + assert receipt.would_have.blocked_reason == enforced, ( + f"enforce refused {enforced!r} and the pilot was told {receipt.would_have.blocked_reason!r}" + ) + assert "APPROVAL_REQUESTED" not in [str(e.type) for e in watcher.events()] + + +def test_T452_observe_reports_the_duplicate_enforce_raises_rather_than_the_budget( + store, clock +) -> None: + """The other direction of the same rule, and the one that says where the check belongs. + + The store decides `plan_reservation` **before** `check_charges` (§3.3), so an effect that is + already committed raises `DuplicateEffect` and the budget is never consulted. Observe mode + evaluated the budget first and reported `budget_exhausted` for an action enforce mode refuses + as a duplicate: the operator is told to raise a limit when the real answer is that the effect + already happened. + """ + from ctrlrun.errors import DuplicateEffect + + enforcing, observing, watcher = _both_modes(store, clock, DOC) + # Fill the budget and commit the key, in both stores, so both refusals are live at once. + for control, into in ((enforcing, store), (observing, watcher)): + control.execute(_action("1", 250), lambda: {"ok": True}, "refund:1") + assert into.get_effect("refund:1").state is EffectState.COMMITTED + + with pytest.raises(DuplicateEffect): + enforcing.execute(_action("1", 250), lambda: {"ok": True}, "refund:1") + + receipt = observing.execute(_action("1", 250), lambda: {"ok": True}, "refund:1") + + assert receipt.would_have is not None + assert receipt.would_have.blocked_reason == "duplicate", ( + "the pilot was told the budget refused an action enforce mode refuses as a duplicate: " + f"{receipt.would_have.blocked_reason!r}" + ) + + +def test_T453_the_observed_sum_uses_the_same_window_the_kernel_decides_on(store, clock) -> None: + """§2.5 through the observe report. A mutation run found `since=now - charge.window` removable + with the whole suite green: the report summed the entire ledger and nothing noticed. + + A pilot whose report counts spend the kernel has already forgotten says a budget would refuse + an action the kernel permits, which is the drift §4.2.1 exists to prevent, pointing the other + way. + """ + enforcing = _control(store, clock) + enforcing.execute(_action("1", 250), lambda: {"ok": True}, "refund:1") + clock.advance(DAY + timedelta(seconds=1)) + + observing = _observing_control(store, clock) + receipt = observing.execute(_action("2", 250), lambda: {"ok": True}, "refund:2") + + blocked = receipt.would_have.blocked_reason if receipt.would_have else None + assert blocked != "budget_exhausted", ( + "the report counted a spend that has rolled out of the window, which the kernel would " + "not have counted" + ) + + +def test_T454_the_observed_sum_ignores_released_rows_as_the_stores_do(store, clock) -> None: + """§4.4 through the observe report, and the second mutation that survived: dropping + `released_at is None` left every test green. + + All three stores' `_spent` excludes released rows, so a report that counts them diverges from + the decision by exactly the amount a human has already cleared. An operator who resolves an + effect `FAILED` and watches the pilot still claim the budget is exhausted has been told the + resolution did nothing. + """ + enforcing = _control(store, clock) + enforcing.execute(_action("1", 100), lambda: {"ok": True}, "refund:1") + with pytest.raises(NotExecuted): + enforcing.execute(_action("2", 100), _boom, "refund:2") + assert _held(store) == 100, "the failed attempt released its charge" + assert len(store.consumptions()) == 2, "and the released row is still in the ledger" + + observing = _observing_control(store, clock) + receipt = observing.execute(_action("3", 100), lambda: {"ok": True}, "refund:3") + + # Counting the released row makes the sum 300 against a limit of 250, and the report would + # block. Excluding it, as every store's `_spent` does, makes it 200 and it does not. + blocked = receipt.would_have.blocked_reason if receipt.would_have else None + assert blocked != "budget_exhausted", "the report counted a released charge" + + +def test_T455_the_observed_sum_is_per_grant(store, clock) -> None: + """The third: `grant_id=charge.grant_id` was removable because every test had one grant.""" + from ctrlrun.state import Charge as _Charge + + store.reserve_effect( + "other:1", "act_other", LEASE, (_Charge("somebody-else", "amount", 250, 250, DAY),) + ) + + observing = _observing_control(store, clock) + receipt = observing.execute(_action("1", 250), lambda: {"ok": True}, "refund:1") + + blocked = receipt.would_have.blocked_reason if receipt.would_have else None + assert blocked != "budget_exhausted", "another grant's spend was counted against this one" + + +def test_T456_the_observed_sum_does_report_a_budget_this_grant_really_exhausted( + store, clock +) -> None: + """The positive control for the three above. Without it each of them passes against a report + that never blocks at all, which is CONTRIBUTING.md's third pattern.""" + enforcing = _control(store, clock) + enforcing.execute(_action("1", 250), lambda: {"ok": True}, "refund:1") + + observing = _observing_control(store, clock) + receipt = observing.execute(_action("2", 250), lambda: {"ok": True}, "refund:2") + + assert receipt.would_have is not None + assert receipt.would_have.blocked_reason == "budget_exhausted", receipt.would_have + + +def test_T457_an_observed_resumed_leg_carries_the_counterfactual_too(store, clock) -> None: + """§4.2.1a and §8.3 together, and they contradicted each other. + + §4.2.1a says every `observed` receipt carries `budget_charges`, and makes summing them the + way an operator sizes a budget. §8.3 says the resumed receipt is the **only** receipt an MCP + multi round-trip or ACS action ever gets. But `_resumed_charges` reads the ledger, and under + observation the ledger is empty by design, so exactly the deployments §8.3 is about + contributed nothing to the sum. An independent review found the two texts disagreeing. + + Enforce mode still reads the ledger, because there the ledger is the record of a real spend. + """ + from ctrlrun import Suspended + + observing = _observing_control(store, clock) + + def suspends() -> Any: + raise Suspended("round-1") + + with pytest.raises(Suspended): + observing.execute(_action("1", 100), suspends, "refund:1") + assert store.consumptions() == (), "observe mode charged nothing, as it must not" + + receipt = contextvars.Context().run(observing.resume, "round-1", lambda: {"ok": True}) + + assert receipt.result is ReceiptResult.OBSERVED + assert receipt.budget_charges == ({"grant_id": "payer", "metric": "amount", "amount": 100},), ( + receipt.budget_charges + ) + assert store.consumptions() == (), "and still charged nothing" + + +def test_T457a_an_enforced_resumed_leg_still_reads_the_ledger(store, clock) -> None: + """The control. Observe mode computing its counterfactual must not make enforce mode compute + one too: there the ledger is the record of a spend that really happened, and a recomputed + number would be a claim rather than evidence.""" + from ctrlrun import Suspended + + control = _control(store, clock) + + def suspends() -> Any: + raise Suspended("round-1") + + with pytest.raises(Suspended): + control.execute(_action("1", 100), suspends, "refund:1") + + receipt = contextvars.Context().run(control.resume, "round-1", lambda: {"ok": True}) + + assert receipt.budget_charges == ({"grant_id": "payer", "metric": "amount", "amount": 100},) + assert len(store.consumptions()) == 1, "and the row it read is the one the first leg wrote" + + +def test_T458_observe_and_enforce_agree_when_an_action_is_both_out_of_scope_and_unmeasurable( + store, clock +) -> None: + """The third ordering drift, and the one a review suspected without demonstrating. + + `_secure` runs `_charges_for` **before** `_in_scope`; `_observe_secure` ran the scope check + first. So an action that is both out of scope and unmeasurable was refused + `budget_unmeasurable` by enforce mode and reported `out_of_scope` by the pilot. Whichever + order is right, one of them has to follow the other, and the enforcing one is the one that + decides. + """ + document = DOC.replace( + ' effect: "refund:{id}"', + ' effect: "refund:{id}"\n resource: "payment:{id}"', + ) + enforcing, observing, _ = _both_modes(store, clock, document) + bad = Action( + name="payments.refund", + arguments={"id": "1"}, # no `amount`, so §2.3 cannot measure it + principal=AGENT, + resource="payment:1", + environment="prod", + ) + somebody_else = lambda _action: {"resources": ["payment:999"]} # noqa: E731 + + with pytest.raises(InvalidArgument): + enforcing.execute(bad, lambda: {"ok": True}, "refund:1", scope=somebody_else) + enforced = store.receipts()[-1].decision_reason + + receipt = observing.execute(bad, lambda: {"ok": True}, "refund:1", scope=somebody_else) + + assert enforced == "budget_unmeasurable" + assert receipt.would_have is not None + assert receipt.would_have.blocked_reason == enforced, ( + f"enforce refused {enforced!r} and the pilot was told {receipt.would_have.blocked_reason!r}" + ) diff --git a/tests/test_clock_skew.py b/tests/test_clock_skew.py index 31889a37..4e82159d 100644 --- a/tests/test_clock_skew.py +++ b/tests/test_clock_skew.py @@ -490,7 +490,7 @@ class _MeasuresOnRefusal(_ExposesSkew): reservation meets an expired lease. Only `Control`'s wiring is under test here; the store-side re-measurement is T215's, against a real server.""" - def reserve_effect(self, effect_key, action_id, lease=DEFAULT_LEASE): + def reserve_effect(self, effect_key, action_id, lease=DEFAULT_LEASE, charges=()): try: return super().reserve_effect(effect_key, action_id, lease) except AmbiguousEffect: diff --git a/tests/test_demo.py b/tests/test_demo.py index a3f0f45c..16cb43f1 100644 --- a/tests/test_demo.py +++ b/tests/test_demo.py @@ -109,6 +109,8 @@ "task", # SPEC-v0.9 §5.5, a `ctrlrun.receipt/v6` field. "scope_hash", + # SPEC-v0.9 §10.1, the third `ctrlrun.receipt/v6` field. + "budget_charges", ) diff --git a/tests/test_effect.py b/tests/test_effect.py index b446c3e1..0b60d4b2 100644 --- a/tests/test_effect.py +++ b/tests/test_effect.py @@ -76,7 +76,9 @@ def __init__(self) -> None: super().__init__() self.reservations: list[tuple[str, str]] = [] - def reserve_effect(self, effect_key: str, action_id: str, lease: Any = None) -> Any: + def reserve_effect( + self, effect_key: str, action_id: str, lease: Any = None, charges: Any = () + ) -> Any: self.reservations.append((effect_key, action_id)) raise NotImplementedError("effect reservation is build-list item 6") diff --git a/tests/test_gateway_server.py b/tests/test_gateway_server.py index 760f4e66..a4e9fd7f 100644 --- a/tests/test_gateway_server.py +++ b/tests/test_gateway_server.py @@ -2064,3 +2064,83 @@ def test_a_refused_path_does_not_desynchronise_the_connection(gateway, path): finally: server.shutdown() server.server_close() + + +BUDGETED_AUTHORITY = """ +schema: ctrlrun.policy/v7 +authority: + grants: + - id: refunder + subject: { agent: "refund-agent" } + actions: ["mcp.acme.create_refund"] + resources: ["payment:*"] + constraints: { amount_lte: 20000 } + budgets: + - { metric: units, limit: 100, window: PT24H } +""" + + +@pytest.fixture +def budgeted_client(upstream, store): + """A grant budgeting a metric the tool's arguments do not carry (SPEC-v0.9 §2.3).""" + from ctrlrun import Authority + + policy = Policy.from_yaml(POLICY.replace("ctrlrun.policy/v2", "ctrlrun.policy/v7")) + control = Control( + policy, store, authority=Authority.from_yaml(BUDGETED_AUTHORITY, standalone=True) + ) + config = GatewayConfig(upstream=upstream.url, alias="acme", principal_header="X-Agent", port=0) + forwarder = httpx_forwarder(config) + gateway = Gateway(config, control, forwarder) + server = build_server(gateway) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + with httpx.Client(base_url=f"http://127.0.0.1:{port}", timeout=10) as opened: + yield opened + server.shutdown() + server.server_close() + forwarder.close() + + +@pytest.mark.authority +def test_an_unmeasurable_budget_is_answered_and_never_dropped(budgeted_client, upstream, store): + """**A dropped connection is the one thing this gateway may not do** (SPEC-v0.9 §2.3). + + §2.3's refusal is an `InvalidArgument`, and `_through_control` catches eight exception types + and not that one, so the handler raised out of the request and the socket closed with no + response. An independent review found it: the client sees + `RemoteProtocolError: Server disconnected without sending a response`, and a client that + retries blindly gets nothing while the store accumulates one `ACTION_DENIED` and one `denied` + receipt per attempt. + + The refusal is a decision about an action, so it is answered like one: `-41001`, the code + §11 already freezes for "not permitted to anyone in this configuration", which is exactly + what an action the kernel cannot measure is. + """ + response = _post( + budgeted_client, + _call(arguments={"amount": 10000, "payment_id": "pi_1"}), + headers={"X-Agent": "refund-agent"}, + ) + + assert upstream.calls == [], "fail closed: the upstream must not run" + body = response.json() + assert body["error"]["code"] == -41001, body + assert body["error"]["data"]["reason"] == "budget_unmeasurable", body + assert [str(event.type) for event in store.events()][-1] == "ACTION_DENIED" + assert store.receipts()[-1].decision_reason == "budget_unmeasurable" + + +@pytest.mark.authority +def test_a_retry_of_an_unmeasurable_call_is_answered_every_time(budgeted_client, upstream, store): + """The half that makes the drop expensive rather than merely wrong: a client retrying a + dropped socket gets an answer each time instead of nothing.""" + for _ in range(3): + response = _post( + budgeted_client, + _call(arguments={"amount": 10000, "payment_id": "pi_1"}), + headers={"X-Agent": "refund-agent"}, + ) + assert response.json()["error"]["code"] == -41001 + assert upstream.calls == [] diff --git a/tests/test_operator_surfaces.py b/tests/test_operator_surfaces.py new file mode 100644 index 00000000..afadac83 --- /dev/null +++ b/tests/test_operator_surfaces.py @@ -0,0 +1,329 @@ +"""T434 to T438: what v0.9 built, visible to the person who gets paged (SPEC-v0.9 §7). + +§7.1 adds **no new command**. `inspect`, `effects` and `stats` are extended, because the question +an operator asks here is not a new one: it is *what is the state of this thing*, and a budget is +one more thing those answer about. +""" + +from __future__ import annotations + +import contextlib +import json +import os +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +import pytest +from click.testing import CliRunner + +from ctrlrun.action import Action, Principal +from ctrlrun.authority import Authority +from ctrlrun.cli import main as cli +from ctrlrun.control import Control +from ctrlrun.effect import EffectState +from ctrlrun.policy import Policy +from ctrlrun.reporting import BUDGET_SCHEMA, budget_document, budget_lines +from ctrlrun.state import SQLiteStateStore + +pytestmark = pytest.mark.authority + +NOW = datetime(2026, 9, 13, 12, 0, tzinfo=UTC) +AGENT = Principal(agent="payer", user="ada") + +DOC = """ +schema: ctrlrun.policy/v7 +environment: prod +actions: + payments.refund: + effect: "refund:{id}" + decision: allow +authority: + grants: + - id: payer + subject: {agent: "payer"} + actions: ["payments.*"] + budgets: + - {metric: amount, limit: 1000, window: PT24H} + - id: unbudgeted + subject: {agent: "other"} + actions: ["payments.*"] +""" + + +class _Clock: + def __init__(self) -> None: + self.now = NOW + + def __call__(self) -> datetime: + return self.now + + def advance(self, by: timedelta) -> None: + self.now += by + + +@pytest.fixture +def clock() -> _Clock: + return _Clock() + + +@pytest.fixture +def store(tmp_path, clock): + made = SQLiteStateStore(tmp_path / "state.db", clock=clock) + yield made + made.close() + + +@pytest.fixture +def control(store, clock) -> Control: + return Control( + policy=Policy.from_yaml(DOC, source=""), + store=store, + clock=clock, + environment="prod", + authority=Authority.from_yaml(DOC, source=""), + ) + + +def _action(identifier: str, amount: int) -> Action: + return Action( + name="payments.refund", + arguments={"amount": amount, "id": identifier}, + principal=AGENT, + environment="prod", + ) + + +def _boom() -> Any: + raise TimeoutError("the response was lost") + + +def _budgets(control: Control) -> Any: + return control._authority.grants["payer"].budgets + + +# --- T434: §7.2's three numbers, and the third one is the deliverable ------------------------- + + +def test_T434_consumed_held_and_why(control, store, clock) -> None: + """§7.2. **The third column is the deliverable, not a nicety.** + + A budget that refuses while an operator can see it is nowhere near its limit looks like a + defect in the kernel, and the true explanation is always the same shape: some effect is + `AMBIGUOUS` and nobody has resolved it (§4.2, R2). + """ + control.execute(_action("1", 200), lambda: {"ok": True}, "refund:1") + with pytest.raises(TimeoutError): + control.execute(_action("2", 300), _boom, "refund:2") + assert store.get_effect("refund:2").state is EffectState.AMBIGUOUS + + document = budget_document("payer", _budgets(control), store, clock.now) + budget = document["budgets"][0] + + assert budget["consumed"] == 500, "both spends count against the limit" + assert budget["held"] == 300, "only the one that has not committed is held" + assert [(h["effect_key"], h["state"]) for h in budget["holding"]] == [ + ("refund:2", str(EffectState.AMBIGUOUS)) + ], budget["holding"] + + +def test_T434a_the_view_names_the_command_that_clears_the_hold(control, store, clock) -> None: + """§7.2: "the path is one command long". The **key**, not a placeholder: an operator who has + to retype it from the line above is one transcription away from resolving a different + effect.""" + with pytest.raises(TimeoutError): + control.execute(_action("2", 300), _boom, "refund:2") + + lines = budget_lines(budget_document("payer", _budgets(control), store, clock.now)) + + assert any("ctrlrun resolve refund:2" in line for line in lines), lines + + +def test_T434b_resolving_the_hold_is_what_the_view_says_it_is(control, store, clock) -> None: + """The claim the line makes, driven through. Without this the hint is documentation. + + `--failed` releases and `consumed` drops; §4.2 is what makes that true, and §7.2's view is + only worth printing if it agrees with it. + """ + control.execute(_action("1", 200), lambda: {"ok": True}, "refund:1") + with pytest.raises(TimeoutError): + control.execute(_action("2", 300), _boom, "refund:2") + + store.resolve_effect("refund:2", EffectState.FAILED, "ada@example.com") + + budget = budget_document("payer", _budgets(control), store, clock.now)["budgets"][0] + assert budget["consumed"] == 200, "a released charge no longer counts against the limit" + assert budget["held"] == 0 + assert budget["holding"] == [] + + +def test_T434c_a_committed_effect_is_consumed_and_never_held(control, store, clock) -> None: + """§7.2's definition, and the distinction the whole view rests on. A committed spend is a + spend: it counts, permanently, and it is not something an operator can clear.""" + control.execute(_action("1", 200), lambda: {"ok": True}, "refund:1") + + budget = budget_document("payer", _budgets(control), store, clock.now)["budgets"][0] + assert (budget["consumed"], budget["held"], budget["holding"]) == (200, 0, []) + + +def test_T434d_the_window_rolls_out_of_the_view_as_it_rolls_out_of_the_decision( + control, store, clock +) -> None: + """§2.5. The view reports the number that decides, so it forgets exactly when the kernel + does. A view over a different window would tell an operator the budget is full while the + kernel permits the action, which is worse than no view.""" + control.execute(_action("1", 900), lambda: {"ok": True}, "refund:1") + assert ( + budget_document("payer", _budgets(control), store, clock.now)["budgets"][0]["consumed"] + == 900 + ) + + clock.advance(timedelta(hours=24, seconds=1)) + + assert ( + budget_document("payer", _budgets(control), store, clock.now)["budgets"][0]["consumed"] == 0 + ) + control.execute(_action("2", 900), lambda: {"ok": True}, "refund:2") + + +def test_T434e_a_grant_with_no_budgets_says_so(control, store, clock) -> None: + """`()` and "no such grant" are different answers. A grant that budgets nothing is a real + grant an operator may ask about, and §2.4 permits it.""" + document = budget_document("unbudgeted", (), store, clock.now) + + assert document["budgets"] == [] + assert "no budgets" in "\n".join(budget_lines(document)) + + +def test_T434f_a_ledger_row_whose_effect_is_gone_is_held_not_dropped(control, store, clock) -> None: + """§7.3 permits an operator to archive rows the window can no longer reach. A store whose + effects were pruned but whose ledger was not must not silently **under**-report `held`: that + is the one direction this view may not err in, because it is the direction that hides a hold. + """ + with pytest.raises(TimeoutError): + control.execute(_action("2", 300), _boom, "refund:2") + store._connection().execute("DELETE FROM effects WHERE effect_key='refund:2'") + store._connection().commit() + + budget = budget_document("payer", _budgets(control), store, clock.now)["budgets"][0] + + assert budget["held"] == 300 + assert budget["holding"][0]["state"] is None + lines = budget_lines(budget_document("payer", _budgets(control), store, clock.now)) + assert any("no effect record" in line for line in lines), lines + + +# --- T435: the CLI surfaces, through the commands an operator actually types ------------------- + + +def _project(tmp_path: Path) -> Path: + here = tmp_path / "project" + here.mkdir(exist_ok=True) + (here / "ctrlrun.yaml").write_text(DOC, encoding="utf-8") + return here + + +def _run(args, cwd: Path): + previous = os.getcwd() + os.chdir(cwd) + try: + return CliRunner().invoke(cli.main, args, catch_exceptions=True) + finally: + os.chdir(previous) + + +def _spend(here: Path) -> None: + control = Control.from_file(here / "ctrlrun.yaml") + control.execute(_action("1", 200), lambda: {"ok": True}, "refund:1") + with contextlib.suppress(TimeoutError): + control.execute(_action("2", 300), _boom, "refund:2") + + +def test_T435_inspect_grant_shows_the_budget(tmp_path) -> None: + """§7.1: no new command. `ctrlrun inspect` answers about an approval, an effect and a + delegation already, and a budget is one more thing it answers about.""" + here = _project(tmp_path) + _spend(here) + + result = _run(["inspect", "--grant", "payer"], here) + + assert result.exit_code == 0, result.output + assert "500 of 1000 per day" in result.output, result.output + assert "300 held" in result.output + assert "refund:2" in result.output and "ambiguous" in result.output + + +def test_T435a_inspect_refuses_both_a_grant_and_an_action(tmp_path) -> None: + """One command, two subjects, so "which did you mean" is this command's own question. + Neither names a subject; both name two.""" + here = _project(tmp_path) + + for args in (["inspect"], ["inspect", "act_1", "--grant", "payer"]): + result = _run(args, here) + assert result.exit_code == 2, f"{args}: {result.output}" + + +def test_T435b_an_unknown_grant_exits_non_zero_with_nothing_on_stdout(tmp_path) -> None: + """As `inspect` does for an unknown action, so a script cannot mistake "no such grant" for + "a grant with no budgets".""" + here = _project(tmp_path) + _spend(here) + + result = _run(["inspect", "--grant", "nobody"], here) + + assert result.exit_code != 0 + assert "no grant nobody" in result.output + + +def test_T436_the_json_shapes_are_additive(tmp_path) -> None: + """§7.2: "a 0.8.0 consumer of the same command keeps working, which T436 asserts rather than + assumes." + + The budget view is its **own** document rather than a key inside `ctrlrun.inspection/v2`, + because that one answers about an action: a reader handed one would have to know which of + two shapes it got. So the assertion is that `inspect ACTION_ID --json` and `stats --json` + still carry every key 0.8.0 read. + """ + here = _project(tmp_path) + _spend(here) + + stats = json.loads(_run(["stats", "--json"], here).output) + assert stats["schema"] == "ctrlrun.stats/v1", "a new key must not move the schema" + for key in ("mode", "since", "from", "to", "actions", "denied", "denied_by_reason"): + assert key in stats, key + assert stats["ledger_rows"] == 2, stats + + budget = json.loads(_run(["inspect", "--grant", "payer", "--json"], here).output) + assert budget["schema"] == BUDGET_SCHEMA + + +def test_T437a_stats_reports_the_ledger_row_count(tmp_path) -> None: + """§7.3: "`stats` reports the row count so growth is observable before it is a problem." + The ledger only grows; the kernel deletes no row, on §12's rule about evidence.""" + here = _project(tmp_path) + _spend(here) + + result = _run(["stats"], here) + + assert "budget ledger rows" in result.output, result.output + assert "2" in result.output + + +def test_T438_effects_says_what_an_effect_is_holding(tmp_path) -> None: + """§7.2 through the other command an operator reaches for. `--state ambiguous` is how they + find what is pinning a grant, and the hold is the reason it matters. + + **"spent" for a committed effect, "holds" for every other.** §7.2 defines `held` as the part + whose effects have not committed, so one word for both numbers would make the two commands + disagree about what they are showing. + """ + here = _project(tmp_path) + _spend(here) + + everything = _run(["effects"], here).output + assert "spent 200 amount on payer" in everything, everything + assert "holds 300 amount on payer" in everything, everything + + ambiguous = _run(["effects", "--state", "ambiguous"], here).output + assert "holds 300 amount on payer" in ambiguous + assert "refund:1" not in ambiguous diff --git a/tests/test_preconditions.py b/tests/test_preconditions.py index fb0c88ab..e32beab3 100644 --- a/tests/test_preconditions.py +++ b/tests/test_preconditions.py @@ -2345,7 +2345,7 @@ def test_every_schema_renders_under_its_own_label_and_key_set(): "ctrlrun.receipt/v3": 26, "ctrlrun.receipt/v4": 28, "ctrlrun.receipt/v5": 30, - "ctrlrun.receipt/v6": 32, + "ctrlrun.receipt/v6": 33, "ctrlrun.receipt/v9": 26, "": 25, } diff --git a/tests/test_protect.py b/tests/test_protect.py index 9a484424..18735fa6 100644 --- a/tests/test_protect.py +++ b/tests/test_protect.py @@ -698,6 +698,8 @@ def read(customer_id: str) -> None: ... "task", # SPEC-v0.9 §5.5, a `ctrlrun.receipt/v6` field. "scope_hash", + # SPEC-v0.9 §10.1, the third `ctrlrun.receipt/v6` field. + "budget_charges", } assert document["schema"] == RECEIPT_SCHEMA assert document["receipt_id"].startswith("ctr_") diff --git a/tests/test_schema_completeness.py b/tests/test_schema_completeness.py new file mode 100644 index 00000000..7ec57ff8 --- /dev/null +++ b/tests/test_schema_completeness.py @@ -0,0 +1,138 @@ +"""The release pass's D27 rule: a schema is complete when something actually writes every field. + +`SPEC-v0.7.md` §12 D27, run by v0.8 for three items without incident and by SPEC-v0.9 §10.1 here: +"item 7 asserts every one of them is written by something before the release PR opens." + +**The key existing is not the assertion.** `tests/test_demo.py` and `tests/test_protect.py` already +pin the full field list of `ctrlrun.receipt/v6`, which catches a field that was never added. What +they cannot catch is a field that is present on every receipt and populated by nothing: a schema +bumped for a feature whose write path was never wired, which reads as shipped and is not. +""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from typing import Any + +import pytest + +from ctrlrun.action import Action, Principal +from ctrlrun.authority import Authority +from ctrlrun.control import Control +from ctrlrun.policy import Policy +from ctrlrun.receipt import RECEIPT_SCHEMA +from ctrlrun.state import InMemoryStateStore +from ctrlrun.verify.guarantees import CATALOGUE, GUARANTEES + +pytestmark = pytest.mark.authority + +DOC = """ +schema: ctrlrun.policy/v7 +environment: prod +actions: + payments.refund: + effect: "refund:{id}" + resource: "payment:{id}" + decision: allow +authority: + grants: + - id: payer + subject: {agent: "payer"} + actions: ["payments.*"] + resources: ["payment:*"] + tasks: ["refund-run:*"] + budgets: + - {metric: amount, limit: 1000, window: PT24H} +""" + + +class _Clock: + now = datetime(2026, 9, 13, 12, 0, tzinfo=UTC) + + def __call__(self) -> datetime: + return self.now + + +def test_every_field_v6_froze_is_written_by_something() -> None: + """One action carrying all three, because a field written only by a test double is a field + nothing writes. §10.1 froze `task`, `scope_hash` and `budget_charges` before any item started; + this is the assertion that they arrived.""" + clock = _Clock() + store = InMemoryStateStore(clock=clock) + control = Control( + policy=Policy.from_yaml(DOC, source=""), + store=store, + clock=clock, + environment="prod", + authority=Authority.from_yaml(DOC, source=""), + ) + action = Action( + name="payments.refund", + arguments={"amount": 100, "id": "1"}, + principal=Principal(agent="payer", user="ada"), + resource="payment:1", + environment="prod", + ) + + receipt = control.execute( + action, + lambda: {"ok": True}, + "refund:1", + task="refund-run:march", + scope=lambda _action: {"resources": ["payment:1"]}, + ) + + assert receipt.schema == RECEIPT_SCHEMA + # SPEC-v0.9 §6: the task the action was bound to. + assert receipt.task == "refund-run:march", receipt.task + # §5.5: the hash of what the provider returned, never its content. + assert receipt.scope_hash and receipt.scope_hash.startswith("sha256:"), receipt.scope_hash + # §10.1: which grants were charged, which metrics, how much. + assert receipt.budget_charges == ({"grant_id": "payer", "metric": "amount", "amount": 100},), ( + receipt.budget_charges + ) + + # And the same three survive the round trip an evidence consumer actually makes. + document = receipt.to_dict() + for field in ("task", "scope_hash", "budget_charges"): + assert document[field], f"{field} is empty in the serialized receipt" + + +def test_the_guarantee_catalogue_is_G1_to_G24() -> None: + """`ctrlrun.guarantees/v5`. A catalogue with a stub row reads as a shipped guarantee, which is + why §10.1 has item 1 move the schema once rather than three branches racing it.""" + assert CATALOGUE == "ctrlrun.guarantees/v5", CATALOGUE + + ids = [entry.id for entry in GUARANTEES] + assert ids == [f"G{number}" for number in range(1, 25)], ids + for entry in GUARANTEES: + # A row whose title is a placeholder reads as a shipped guarantee in every report that + # prints the catalogue, which is the failure this assertion is actually for. + assert entry.title.strip(), entry.id + assert not entry.title.upper().startswith("TODO"), entry.id + + +def test_the_policy_schema_accepts_v7_keys_and_an_older_reader_refuses_them() -> None: + """`ctrlrun.policy/v7`. `tasks:` and `budgets:` are **refused** in a `v6` document rather than + ignored, because an older reader would grant the action on every task and against no limit.""" + from ctrlrun.errors import PolicyError + + assert Authority.from_yaml(DOC, source="").grants["payer"].tasks == ("refund-run:*",) + assert Authority.from_yaml(DOC, source="").grants["payer"].budgets[0].limit == 1000 + + older = DOC.replace("ctrlrun.policy/v7", "ctrlrun.policy/v6") + with pytest.raises(PolicyError): + Authority.from_yaml(older, source="") + + +def test_the_window_is_seconds_everywhere_it_is_stored() -> None: + """§10's `Budget` row: a `timedelta` in Python, ISO-8601 in the document, integer seconds in + `_canonical_grant`, because a `timedelta` is not a `PlainValue` and cannot render through + `canonical_bytes`.""" + from ctrlrun.authority import grant_to_json + + grant = Authority.from_yaml(DOC, source="").grants["payer"] + assert grant.budgets[0].window == timedelta(hours=24) + + stored: Any = grant_to_json(grant) + assert '"window": 86400' in stored or '"window":86400' in stored, stored diff --git a/tests/test_verify.py b/tests/test_verify.py index 0163fffd..251798b1 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -177,13 +177,13 @@ def test_T101_a_policy_with_no_approve_rule_makes_G1_and_G2_not_applicable(tmp_p # because this document names no `max_attempts` (SPEC-v0.7 §8.9). The rest are applicable, # G14 among them, and the count is over those. assert report.applicable == 11 - assert report.not_applicable == 12 + assert report.not_applicable == 13 text = report.to_text() # The fraction is passes over applicable and never the catalogue size: with eight N/As a # seventeen-guarantee catalogue must not report seventeen over seventeen. assert f"{len(reg.GUARANTEES)}/{len(reg.GUARANTEES)}" not in text assert f"{report.passed}/{report.applicable} declared guarantees pass." in text - assert "12 not applicable: G1, G2, G8, G9, G13, G15, G16, G17, G18, G19, G23, G24." in text + assert "13 not applicable: G1, G2, G8, G9, G13, G15, G16, G17, G18, G19, G22, G23, G24." in text def test_T101b_zero_applicable_guarantees_is_not_a_pass(tmp_path): @@ -344,7 +344,7 @@ def test_T103_CTRLRUN_STATE_is_not_read_and_not_created(tmp_path, monkeypatch): from ctrlrun.state import _iso -def _always_reserves(self, effect_key, action_id, lease=DEFAULT_LEASE): +def _always_reserves(self, effect_key, action_id, lease=DEFAULT_LEASE, charges=()): now = self._clock() connection = self._connection() connection.execute("BEGIN IMMEDIATE") @@ -864,14 +864,14 @@ def test_the_v1_payments_template_reports_eleven_over_eleven(): report = run(V1_PAYMENTS) assert report.exit_code == 0 - assert (report.passed, report.applicable, report.not_applicable) == (11, 11, 12) + assert (report.passed, report.applicable, report.not_applicable) == (11, 11, 13) text = report.to_text() assert "11/11 declared guarantees pass." in text # G13 is N/A on SQLite, which has no clock of its own; G14 and G15 join G3, G4 and G5 where # the effect template lives in the @protect decorator verify does not read, and where the # document names no `max_attempts`. G16 and G18 are graded: verify brings its own provider # for the first and its own approver identity for the second (§8.9, §11.7). - assert "12 not applicable: G3, G4, G5, G8, G9, G13, G14, G15, G17, G19, G23, G24." in text + assert "13 not applicable: G3, G4, G5, G8, G9, G13, G14, G15, G17, G19, G22, G23, G24." in text # §2.1's rule, and the reason this line exists: **the not-applicable ten are not in the # denominator.** Ten pass and ten are N/A, so a run that folded them in would report 20/20. # It used to read `"10/10" not in text`, which said the same thing while the pass count was diff --git a/tests/test_verify_action.py b/tests/test_verify_action.py index 80a9a77d..0d8ceb00 100644 --- a/tests/test_verify_action.py +++ b/tests/test_verify_action.py @@ -137,10 +137,10 @@ def test_T118_ci_asserts_the_two_shapes_the_specification_names(): steps = _workflow()["jobs"]["verify"]["steps"] script = "\n".join(step.get("run", "") for step in steps) - assert 'test "$AUTHORITY" = "verified 21/21"' in script + assert 'test "$AUTHORITY" = "verified 22/22"' in script assert 'test "$TEMPLATES" = "verified 11/11"' in script assert 'test "$AUTHORITY_NA" = "2"' in script - assert 'test "$TEMPLATES_NA" = "12"' in script + assert 'test "$TEMPLATES_NA" = "13"' in script @pytest.mark.authority @@ -152,13 +152,13 @@ def test_T118_the_two_configurations_really_do_report_those_shapes(): templates = run(V1_PAYMENTS) assert authority.badge is not None - assert authority.badge["message"] == "verified 21/21" + assert authority.badge["message"] == "verified 22/22" # G13 and G15: SQLite has no clock of its own to diverge from, and the document declares # no `max_attempts` (SPEC-v0.7 §8.9). assert authority.not_applicable == 2 assert templates.badge is not None assert templates.badge["message"] == "verified 11/11" - assert templates.not_applicable == 12 + assert templates.not_applicable == 13 def test_T118_the_action_uploads_the_report_and_writes_a_job_summary(): @@ -211,7 +211,7 @@ def test_T119_the_colour_is_about_failures_and_has_no_amber_for_not_applicable( from ctrlrun.verify import scenarios passing = run(V1_PAYMENTS) - assert passing.not_applicable == 12 + assert passing.not_applicable == 13 assert passing.badge is not None assert passing.badge["color"] == BADGE_PASS_COLOR diff --git a/tests/test_verify_authority.py b/tests/test_verify_authority.py index 694b02ab..8bd949ca 100644 --- a/tests/test_verify_authority.py +++ b/tests/test_verify_authority.py @@ -384,3 +384,290 @@ def test_the_principal_is_derived_from_the_grants_subject(tmp_path): # becomes `ctrlrun-verify` (§3.4). assert receipt_agents == set() assert result.grant_id == "wildcards" + + +# --- T413: a budget must not make verify report an internal error (SPEC-v0.9 §2) ------------- + +TIGHT_BUDGET = """ +authority: + grants: + - id: head-of-support + subject: { agent: "head-of-support", user: "dana@example.com" } + actions: ["acme.refund", "acme.read"] + resources: ["payment:*"] + constraints: { amount_gte: 0, amount_lte: 500000 } + environments: ["production"] + expires_at: "2027-01-01T00:00:00Z" + budgets: + - {metric: amount, limit: 900, window: PT24H} +""" + + +def test_T413_a_budget_smaller_than_the_synthesized_vector_does_not_crash_verify(tmp_path): + """**A budget is a configuration fact, never a defect in verify.** + + `_synthesize` picks a vector to land in a rule, and a grant whose budget is smaller than + that vector refuses the action before the guarantee is reached. Verify reported that as an + internal error, exit 3, on guarantees with nothing to do with budgets: a shipped example + carrying a €1,000 daily budget under a policy admitting a €100,000 refund made `ctrlrun + verify` fail on G1, which is about approvals. It is `_identity_the_document_needs`'s case in + the budget dimension, and it gets the same answer: verify sizes its own vector. + + **And the vector is actually resized**, which an independent review found this test was not + checking. Against `ACTIONS` the allow band is `amount_gte: 0, amount_lte: 1000` and + `_synthesize` picks `amount: 0`, which is under any budget, so the original form of this test + passed against a kernel that resized nothing: it was green on the commit before the feature. + The rule below is upper-bound only, so the synthesized vector is the band maximum and a + budget under it has to move it. + """ + upper_bound_only = """ +actions: + acme.refund: + effect: "refund:{payment_id}" + resource: "payment:{payment_id}" + rules: + - when: { amount_lte: 100000 } + decision: allow + - decision: deny +""" + path = _write(tmp_path, V7 + TIGHT_BUDGET + upper_bound_only) + + result = _by_id(run(path, only=("G3",)))["G3"] + + assert result.status is Status.PASS, f"{result.status}: {result.reason}" + graded = dict(result.arguments or {}) + assert graded["amount"] != 100000, "the band maximum was used unchanged" + assert 0 < graded["amount"] <= 900, graded + + +def test_T413a_a_band_no_action_can_pay_for_is_N_A_with_a_reason_about_the_budget(tmp_path): + """The case where no vector fits, which is a real and reportable configuration. + + A budget smaller than any single action in the approve band makes that band unreachable: + every action needing a human would exhaust the whole window. That is worth telling an + operator, and telling them the truth about it. Falling through to the grant miss reported + "no grant's `resources:` matches a resource verify can build" about a document whose + patterns matched perfectly, which is the category error `unselected`'s docstring exists + about, one dimension over. + """ + path = _write(tmp_path, V7 + TIGHT_BUDGET + ACTIONS) + + result = _by_id(run(path, only=("G1",)))["G1"] + + assert result.status is Status.NOT_APPLICABLE + assert result.reason.startswith(reg.NO_ACTION_FITS_THE_BUDGET), result.reason + assert "acme.refund" in result.reason and "head-of-support" in result.reason + # The reason names the action and the grant, because "a budget is in the way" without + # saying which one sends an operator reading a twelve-grant document by hand. + + +def test_T413b_a_document_with_no_budget_selects_exactly_what_it_did_before(tmp_path): + """The vector is only resized when a budget would refuse it, so every document without one + keeps the selection it had. Without this, the fix is a change to all twenty-four scenarios + rather than to the documents that need it.""" + unbudgeted = FULL_AUTHORITY.replace( + " budgets:\n - {metric: amount, limit: 500000, window: PT24H}\n", "" + ) + both = [] + for index, authority in enumerate((unbudgeted, FULL_AUTHORITY)): + directory = tmp_path / str(index) + directory.mkdir() + path = _write(directory, V7 + authority + ACTIONS) + result = _by_id(run(path, only=("G3",)))["G3"] + both.append((result.status, result.action, result.arguments, result.grant_id)) + assert both[0] == both[1], both + + +UNMEASURABLE_BUDGET = TIGHT_BUDGET.replace("metric: amount", "metric: items") + +TWO_ACTIONS = ( + ACTIONS + + """ zzz.wire: + effect: "wire:{payment_id}" + resource: "ledger:{payment_id}" + rules: + - when: { amount_gte: 0, amount_lte: 100000 } + decision: approve + - decision: deny +""" +) + + +def test_T413c_a_budget_naming_a_metric_no_action_carries_is_graded_not_crashed(tmp_path): + """§2.3 through verify. A grant budgeting `items` where every action carries `amount` + refuses **every** action it covers, for ever, and `ctrlrun verify` reported that as an + internal error, exit 3, on guarantees with nothing to do with budgets. + + It is also a **different fix** from a budget that is merely small, so it gets its own reason: + told "exceeds a budget", an operator raises a limit that was never the problem. + """ + path = _write(tmp_path, V7 + UNMEASURABLE_BUDGET + ACTIONS) + + result = _by_id(run(path, only=("G3",)))["G3"] + + assert result.status is Status.NOT_APPLICABLE, f"{result.status}: {result.reason}" + assert result.reason.startswith(reg.NO_METRIC_TO_MEASURE), result.reason + assert "acme.refund" in result.reason + + +def test_T413d_a_budget_miss_does_not_hide_a_grant_miss(tmp_path): + """`select` records a grant miss for **every** failed candidate, so an unconditional + precedence for the budget meant a document with one budget-blocked action and one action no + grant covers at all reported only the budget. The resource miss appeared nowhere. + + That is the category error `unselected`'s own docstring exists about, one dimension over, and + an independent review found it. Both facts are true of the document, so both are stated. + """ + path = _write(tmp_path, V7 + TIGHT_BUDGET + TWO_ACTIONS) + + result = _by_id(run(path, only=("G1",)))["G1"] + + assert result.status is Status.NOT_APPLICABLE + assert reg.NO_ACTION_FITS_THE_BUDGET in result.reason, result.reason + assert reg.NO_GRANT_COVERS_SELECTION in result.reason, result.reason + + +def test_T413e_a_resize_never_moves_the_action_onto_an_unchecked_grant(tmp_path): + """**The resize can change which grant decides**, and that grant's budget was never looked + at. `Authority.evaluate` resolves `min(passed, key=grant_id)`, so a vector shrunk to fit one + grant's budget can fall inside a lexicographically earlier grant's `constraints`. + + An independent review demonstrated it: shrinking 100000 to 1 moved the action from + `bb-broad` to `aa-narrow`, whose budget is zero, and verify exited 3 on a budget it had + never checked. + """ + document = ( + V7 + + """ +authority: + grants: + - id: aa-narrow + subject: { agent: "head-of-support" } + actions: ["acme.refund"] + resources: ["payment:*"] + constraints: { amount_lte: 10 } + budgets: + - {metric: amount, limit: 0, window: PT24H} + - id: bb-broad + subject: { agent: "head-of-support" } + actions: ["acme.refund"] + resources: ["payment:*"] + constraints: { amount_lte: 500000 } + budgets: + - {metric: amount, limit: 90000, window: PT24H} +""" + + """ +actions: + acme.refund: + effect: "refund:{payment_id}" + resource: "payment:{payment_id}" + rules: + - when: { amount_lte: 100000 } + decision: allow + - decision: deny +""" + ) + path = _write(tmp_path, document) + + graded = _by_id(run(path, only=("G3", "G4"))) + + for gid in ("G3", "G4"): + assert graded[gid].status is not Status.FAIL, ( + f"{gid} reported the kernel broken for a configuration reason: {graded[gid].reason}" + ) + + +def test_T413f_a_resized_vector_leaves_room_for_a_scenario_that_acts_more_than_once(tmp_path): + """G4's control leg runs eight children on distinct keys and then contends eight more on + one, so it needs nine spends to fit. A vector sized to the limit exactly made its control + leg pass, its contended leg find zero winners, and the guarantee report **FAIL**. + + **Verify may say it could not grade a configuration; it may not accuse the kernel of a + defect.** The candidates are required to leave that much room, and where none does the + guarantee is `N/A`. + """ + upper_bound_only = """ +actions: + acme.refund: + effect: "refund:{payment_id}" + resource: "payment:{payment_id}" + rules: + - when: { amount_lte: 100000 } + decision: allow + - decision: deny +""" + path = _write(tmp_path, V7 + TIGHT_BUDGET + upper_bound_only) + + result = _by_id(run(path, only=("G4",)))["G4"] + + assert result.status is not Status.FAIL, result.reason + if result.status is Status.PASS: + graded = dict(result.arguments or {}) + # Nine spends have to fit inside 900, so a vector of 900, or of 450, would not do. + assert graded["amount"] * 18 <= 900, graded + + +FLOORED_RULE = """ +actions: + acme.refund: + effect: "refund:{payment_id}" + resource: "payment:{payment_id}" + rules: + - when: { amount_gte: 100, amount_lte: 100000 } + decision: allow + - decision: deny +""" + +TWO_ALLOW_BANDS = """ +actions: + acme.refund: + effect: "refund:{payment_id}" + resource: "payment:{payment_id}" + rules: + - when: { amount_gte: 1000, amount_lte: 100000 } + decision: allow + - when: { amount_lte: 999 } + decision: allow + - decision: deny +""" + + +def test_T413g_a_band_whose_floor_leaves_no_headroom_is_N_A_and_never_FAIL(tmp_path): + """The headroom rule, on the document where it is load-bearing. + + Where the rule admits `1`, the first candidate is tiny and headroom never binds. It binds + when the band has a **floor**: the smallest in-rule value here is 100, nine of which is 900, + and G4 needs nine. A ladder without the headroom requirement picks 112 and the contended leg + finds zero winners, which the guarantee reports as FAIL. + + **Verify may say it could not grade a configuration. It may not report the kernel broken.** + """ + path = _write(tmp_path, V7 + TIGHT_BUDGET + FLOORED_RULE) + + result = _by_id(run(path, only=("G4",)))["G4"] + + assert result.status is not Status.FAIL, result.reason + if result.status is Status.PASS: + assert dict(result.arguments or {})["amount"] * 18 <= 900, result.arguments + + +def test_T413h_a_resize_never_silently_grades_a_different_rule(tmp_path): + """`select`'s contract is the decision **and the rule** it was asked for. + + Two bands reach `allow` here. `_synthesize` picks the upper one, and every value small + enough for the budget lands in the lower one, which is a different rule with a different + reason. Checking only the decision would let verify grade a rule nobody selected and report + it under the first band's name. + + So the honest answer is that no vector fits, and the guarantee is `N/A`. + """ + path = _write(tmp_path, V7 + TIGHT_BUDGET + TWO_ALLOW_BANDS) + + result = _by_id(run(path, only=("G3",)))["G3"] + + if result.status is Status.PASS: + assert dict(result.arguments or {})["amount"] >= 1000, ( + f"verify graded a band it was not asked for: {result.arguments}" + ) + else: + assert result.status is Status.NOT_APPLICABLE, result.reason diff --git a/tests/test_verify_report.py b/tests/test_verify_report.py index e38f4548..d11d5974 100644 --- a/tests/test_verify_report.py +++ b/tests/test_verify_report.py @@ -109,7 +109,7 @@ def test_T113_the_summary_is_the_last_line_and_names_the_not_applicable_ids(tmp_ # G13 is N/A on every SQLite run: SQLite has no clock of its own. G14 needs the effect # template this document keeps in the @protect decorator, and G15 a `max_attempts` it does # not declare (SPEC-v0.7 §8.9). - assert "12 not applicable: G3, G4, G5, G8, G9, G13, G14, G15, G17, G19, G23, G24." in last + assert "13 not applicable: G3, G4, G5, G8, G9, G13, G14, G15, G17, G19, G22, G23, G24." in last # The fraction is passes over applicable. A report with eight N/As does not say 17/17. assert "18/18" not in text @@ -137,9 +137,9 @@ def test_T113_a_failing_report_names_the_subject_and_prints_the_counterexample( # SPEC-v0.8 item 2: G18 joins the catalogue. It is graded wherever the document sends # an action to approval, which the first two of these do, and `N/A` for G1's reason # where nothing does. So the first two gain a pass and the third gains an N/A. - (ALL_APPLICABLE, "16/16 declared guarantees pass. 7 not applicable"), - (WITH_NOT_APPLICABLE, "11/11 declared guarantees pass. 12 not applicable"), - (EMPTY, "0/0 declared guarantees pass. 23 not applicable"), + (ALL_APPLICABLE, "16/16 declared guarantees pass. 8 not applicable"), + (WITH_NOT_APPLICABLE, "11/11 declared guarantees pass. 13 not applicable"), + (EMPTY, "0/0 declared guarantees pass. 24 not applicable"), ], ids=["passing", "some-na", "all-na"], ) @@ -454,7 +454,7 @@ def test_T116_a_run_with_several_not_applicable_still_exits_0(tmp_path, monkeypa result = _cli(tmp_path, WITH_NOT_APPLICABLE) assert result.exit_code == 0 - assert "12 not applicable" in result.stdout + assert "13 not applicable" in result.stdout def test_T116_json_and_junit_can_be_combined(tmp_path, monkeypatch):