diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 282f7ef6..d733b04b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -120,7 +120,7 @@ jobs: set -eu echo "authority: $AUTHORITY ($AUTHORITY_NA not applicable)" echo "templates: $TEMPLATES ($TEMPLATES_NA not applicable)" - test "$AUTHORITY" = "verified 20/20" + test "$AUTHORITY" = "verified 21/21" # G13 is N/A on SQLite, the action's default store: SQLite has no clock of its own # to diverge from; G15 is N/A because neither document declares `max_attempts`. # G16 is graded on both: verify brings its own precondition provider (SPEC-v0.7 §8.9). @@ -136,7 +136,7 @@ jobs: # binds one, so its count is unchanged and its passing total moved 19 to 20 instead. test "$AUTHORITY_NA" = "2" test "$TEMPLATES" = "verified 11/11" - test "$TEMPLATES_NA" = "11" + test "$TEMPLATES_NA" = "12" test -s verify-badge.json test -s verify-report.json test -s verify-report.xml diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b2cb7a2..b9b93a9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,8 +29,32 @@ any change to one appears here. The task reaches the authority decision and the receipt, and **never the action hash**: a field on `Action` would move every hash in existence and invalidate every stored approval. +- **Scope providers** (SPEC-v0.9 §5). `Control.execute(scope=...)` and `@protect(scope=...)` take + a callable that answers what the calling principal's assigned scope is; **the kernel matches** + this action's resource into it, with the relation a grant's `resources:` already uses. It runs + **strictly before the reservation** and before the precondition recheck, so a provider that + hangs leaves nothing reserved and nothing executed. **G23** grades it. + + This is the bite on an identifier an attacker chose: a grant permits `records.read` on + `customer:*`, and until now nothing had an opinion about *whose* record `customer:90210` is. + + Two distinct refusals, never one: `scope_unavailable` when the provider raises, answers with the + wrong shape, or answers something the canonicalizer refuses; `out_of_scope` when it answered and + the resource is not covered. A non-callable `scope=` is `InvalidArgument`, at decoration time + under `@protect`. + + Only the **hash** of what the provider returned reaches the receipt, under its own domain tag so + it can never equal a precondition fingerprint over the same mapping. A scope is a list of what a + principal may touch, and an evidence store is not the place to keep a second copy of it. + + It **amends `SPEC-v0.7.md` §6.9**, which said v0.9's scope providers would configure the + precondition hook rather than add a second one. `SPEC-v0.9.md` §5.2.1 records the amendment and + the three mechanical differences that justify it. + ### Changed +- `ctrlrun.receipt/v6` carries `scope_hash` beside `task`, and `ctrlrun.guarantees/v5` carries + **G23** beside G24. - `ctrlrun.policy/v7`, `ctrlrun.receipt/v6` and `ctrlrun.guarantees/v5`. `tasks:` on a grant is refused in a `v6` document rather than ignored, because an older reader would grant the action on every task. `DIMENSIONS` grows from six entries to seven, and it is exported and iterated by diff --git a/src/ctrlrun/control.py b/src/ctrlrun/control.py index dbb7d91e..ad205417 100644 --- a/src/ctrlrun/control.py +++ b/src/ctrlrun/control.py @@ -8,6 +8,7 @@ from __future__ import annotations import functools +import hashlib import inspect import logging import os @@ -21,7 +22,7 @@ from pathlib import Path from typing import Any, Final, NoReturn, ParamSpec, TypeVar, cast -from .action import Action, Principal +from .action import Action, Principal, canonical_bytes from .approval import ( APPROVAL_UNRECORDED, APPROVALS_UNVERIFIABLE, @@ -49,6 +50,7 @@ unsatisfied, ) from .authority import ( + RESOURCE_SEPARATOR, Authority, AuthorityResult, BreakGlassEnvelope, @@ -56,6 +58,7 @@ Delegation, Grant, _optional_from_yaml, + matches, ) from .effect import ( _EXECUTOR_RUN, @@ -212,6 +215,10 @@ class _Invocation: #: the same reason: set at the one place that knows it rather than at each receipt site, because #: a site that forgot would stamp the **previous** action's task onto this one's evidence. _TASK: ContextVar[str | None] = ContextVar("ctrlrun_task") +#: SPEC-v0.9 §5.5 — the scope hash reaches the receipt the way the task does, and is reset +#: beside it: a refusal whose receipt carried the previous action's scope would be the same +#: stale-evidence defect on a new field. +_SCOPE_HASH: ContextVar[str | None] = ContextVar("ctrlrun_scope_hash") #: SPEC-v0.8 §8.2.1, §11.2. How the policy-change flow says that this `ctrlrun.policy.change` #: is one it built. **Package-internal on purpose**, beside `_granting_principal`, and it @@ -509,6 +516,77 @@ def _hash_or_none(value: object) -> str | None: return value if isinstance(value, str) else None +class _ScopeRefusedError(Exception): + """SPEC-v0.9 §5.6's refusal, carried out of `_secure`'s loop without meeting its handlers. + + **Not an `ActionDenied` subclass, and that is the whole point.** `_secure`'s `except + ActionDenied` appends `APPROVAL_DENIED` unconditionally, so a scope refusal raised as one + fabricates an approval denial for an action no human ever saw, and records `ACTION_DENIED` + twice. `SPEC-v0.9 §3.3.2` names this hazard for the budget refusal a later item adds; it is + the same handler and the same defect, found here first by reading the events a refusal wrote. + + `_refuse_scope` has already written the events and the receipt, so this carries only the + public error the caller should see. + """ + + def __init__(self, denial: ActionDenied) -> None: + super().__init__(str(denial)) + self.denial = denial + + +class _ObservedRefusalError(Exception): + """SPEC-v0.9 §5.2.2 — observe mode's would-have-refused, which escapes `_in_scope` and is + swallowed by `_observe_secure`. Package-internal and never public: it is control flow, not a + refusal, and a caller that could catch it could mistake an observed run for an enforced one. + + **It carries the reason**, and an independent review is why it does. Without it + `_observe_secure` had one hardcoded `out_of_scope` for both refusals, so a deployment whose + scope *source was down* read a counterfactual saying the record was not theirs. Observe mode + exists to tell an operator what enforce mode would do; reporting the wrong category is the one + way it can be worse than useless. + """ + + def __init__(self, reason: str) -> None: + super().__init__(reason) + self.reason = reason + + +#: SPEC-v0.9 §5.6 — the two refusal reasons, distinct because a test asserting only the exception +#: type cannot tell which guard fired. `scope_unavailable` is "the provider could not answer"; +#: `out_of_scope` is "it answered, and the record is not this principal's". G23 is the first. +SCOPE_UNAVAILABLE: Final = "scope_unavailable" +OUT_OF_SCOPE: Final = "out_of_scope" + +#: SPEC-v0.9 §5.5 — its own domain tag, so a scope hash can never equal a precondition +#: fingerprint over the same mapping. That matters precisely because §5.7 permits both. +_SCOPE_SCHEMA: Final = "ctrlrun.scope/v1" + +#: §5.4 — the key the provider answers under. A scope is a set of resource patterns, matched +#: with the relation `authority.py` already uses for a grant's `resources:`. +_SCOPE_RESOURCES: Final = "resources" + + +def _checked_scope(scope: object, where: str) -> _Preconditions | None: + """SPEC-v0.9 §5.6's third row, and `v0.7 §6.2`'s rule for a non-callable `preconditions=`.""" + if scope is not None and not callable(scope): + raise InvalidArgument( + f"{where}: scope must be a callable taking the Action and returning a mapping " + f"with {_SCOPE_RESOURCES!r}, not {type(scope).__name__}" + ) + return cast("_Preconditions | None", scope) + + +def _scope_hash(scope: Mapping[str, Any]) -> str: + """`"sha256:" + hex(SHA-256(canonical_bytes({schema, scope})))` (SPEC-v0.9 §5.5). + + Through `canonical_bytes` and nothing else, so `v0.1 §2.3`'s float rejection and the + non-string-key refusal are inherited rather than re-argued: a scope hashed over a float + would drift. Whatever it raises is the caller's to turn into `scope_unavailable`. + """ + document = {"schema": _SCOPE_SCHEMA, "scope": dict(scope)} + return "sha256:" + hashlib.sha256(canonical_bytes(document)).hexdigest() + + def _checked_preconditions(preconditions: object, where: str) -> _Preconditions | None: """`None`, or a callable (SPEC-v0.7 §6.2). Anything else is a wiring bug, refused at the door: at decoration time for `@protect`, before any evidence for `execute`.""" @@ -987,6 +1065,7 @@ def execute( reconcile_eagerly: bool = False, preconditions: Callable[[Action], Mapping[str, Any]] | None = None, task: str | None = None, + scope: Callable[[Action], Mapping[str, Any]] | None = None, ) -> Receipt: """Decide, run and record one action. Returns the receipt for its terminal state. @@ -1010,6 +1089,15 @@ def execute( decision and **never the action hash**: §6.3.1 argues that at length, and the short form is that a field on `Action` would move every action hash in existence. + `scope` answers whether this action's resource is in the calling principal's assigned + scope (SPEC-v0.9 §5). It is called with the `Action` and returns a mapping carrying a + `resources` list; **the kernel matches**, with the relation a grant's `resources:` uses. + It runs **strictly before the reservation** and before the precondition recheck, so a + provider that hangs can only fail closed. A provider that raises, answers with the wrong + shape, or answers something the canonicalizer refuses denies the action + `scope_unavailable`; a resource the scope does not cover denies it `out_of_scope`. Only + the hash of what it returned reaches the receipt, never the scope itself. + `preconditions` reads the state an approval depends on (SPEC-v0.7 §6). It is called with the `Action` and returns a mapping, which is hashed through `canonical_bytes` and kept only as a fingerprint. Under `APPROVE` it is called when the approval is requested, @@ -1051,6 +1139,8 @@ def execute( f"This call asked for {effect_key!r}" ) provider = _checked_preconditions(preconditions, "execute(preconditions=...)") + scoper = _checked_scope(scope, "execute(scope=...)") + scoped: list[str | None] = [] self._check_environment(action) held = self._lease if lease is None else _checked_lease(lease, "execute(lease=...)") reconciler = _reconciler(reconcile, reconcile_eagerly, "execute") @@ -1079,6 +1169,7 @@ def execute( # stale grant id: a refusal whose receipt carried the previous action's task would be the # same defect on a new field. _TASK.set(None) + _SCOPE_HASH.set(None) # SPEC-v0.3 §6.2 — the counterfactual for an observed run, or `None` in enforce mode. # Every branch below reads it to choose between refusing and recording. observation = _Observation() if self._observing else None @@ -1108,6 +1199,8 @@ def execute( reconciler, held, provider, + scoper, + scoped, ) self._append(EventType.ACTION_DENIED, action, {"reason": PRINCIPAL_EXPIRED}, effect_key) self._record( @@ -1156,6 +1249,8 @@ def execute( reconciler, held, provider, + scoper, + scoped, ) self._refuse_authority(action, result, started_at, effect_key) self._append( @@ -1192,6 +1287,8 @@ def execute( reconciler, held, provider, + scoper, + scoped, ) # SPEC-v0.6 §7.2.1's third bullet: *"the refusal is recorded against the approval # so the history shows a grant that met a denial."* It was not. An independent @@ -1265,10 +1362,21 @@ def execute( reconciler, held, provider, + scoper, + scoped, ) compared = _Compared() approval, reservation = self._secure( - action, evaluation, started_at, effect_key, held, reconciler, provider, compared + action, + evaluation, + started_at, + effect_key, + held, + reconciler, + provider, + compared, + scoper, + scoped, ) attempt = 1 if reservation is None else reservation.attempt # SPEC-v0.7 §5.5 — the check, on the attempt number the store **assigned**, after the @@ -1331,6 +1439,8 @@ def _observed( reconciler: _Reconciler, lease: timedelta, preconditions: _Preconditions | None = None, + scope: _Preconditions | None = None, + scoped: list[str | None] | None = None, ) -> Receipt: """Run an action observe mode has finished deciding about (SPEC-v0.3 §6.2). @@ -1342,7 +1452,15 @@ def _observed( """ compared = _Compared() approval, reservation = self._observe_secure( - action, evaluation, effect_key, lease, observation, preconditions, compared + action, + evaluation, + effect_key, + lease, + observation, + preconditions, + compared, + scope, + scoped, ) held_key = None if reservation is None else effect_key attempt = 1 if reservation is None else reservation.attempt @@ -1392,6 +1510,8 @@ def _observe_secure( observation: _Observation, preconditions: _Preconditions | None, compared: _Compared, + scope: _Preconditions | None = None, + scoped: list[str | None] | None = None, ) -> tuple[Approval | None, Reservation | None]: """Attempt what `_secure` takes, record every refusal, and hold nothing it lost. @@ -1418,6 +1538,15 @@ def _observe_secure( reason, _ = self._policy_approval_state() if reason is not None: observation.block(reason) + # SPEC-v0.9 §5.2.2's observe row. The provider **runs**, so its hash reaches the receipt + # and an operator sizing a scope before turning it on sees what would have happened; the + # refusal is recorded and not raised. `v0.3 §6.2`: observe mode records rather than + # enforces, and a check that enforced here would refuse during the phase whose entire + # purpose is to refuse nothing. + try: + self._in_scope(action, scope, scoped, enforcing=False) + except _ObservedRefusalError as would: + observation.block(would.reason) approval_id = None if evaluation.decision is Decision.APPROVE: approval_id = _PRESENTED_APPROVAL.get(None) @@ -1508,6 +1637,8 @@ def _observe_take( lease: timedelta, preconditions: _Preconditions | None, compared: _Compared, + scope: _Preconditions | None = None, + scoped: list[str | None] | None = None, ) -> tuple[Approval | None, Reservation | None]: """Observe mode's `_take`: **check the grant, never spend it** (SPEC-v0.6 §7.2.3). @@ -2061,6 +2192,8 @@ def _secure( reconciler: _Reconciler, preconditions: _Preconditions | None, compared: _Compared, + scope: _Preconditions | None = None, + scoped: list[str | None] | None = None, ) -> tuple[Approval | None, Reservation | None]: """Take everything this action needs before it may run: the grant, and the key. @@ -2097,6 +2230,12 @@ def _secure( # and whatever the second attempt meets is final. for reconciled in (False, True): try: + # SPEC-v0.9 §5.3, §5.7 — **before the recheck and before every `_take`**. It is + # in the loop and not above it because the `reconcile` hook between the two + # passes is a network call, and a scope fetched before it would be compared + # against a world that moved while it ran. §5.8 states what this costs the + # precondition's own window, which is that it now contains this call. + self._in_scope(action, scope, scoped) if approval_id is not None: # **Immediately before `_take`, and nothing between them.** The window this # narrows is the time from the provider's fetch to the store call; anything @@ -2105,6 +2244,14 @@ def _secure( self._recheck(action, approval_id, preconditions, compared) approval, reservation = self._take(action, approval_id, effect_key, lease) break + except _ScopeRefusedError as refused: + # SPEC-v0.9 §5.6. Its own clause, **before** the `ActionDenied` one: + # `_refuse_scope` has already written the events and the receipt, and an + # exception raised inside an `except` clause leaves the whole `try` rather than + # meeting its siblings. Routed through `except ActionDenied` instead, this would + # append `APPROVAL_DENIED` for an action no human saw and a second + # `ACTION_DENIED` (§3.3.2's hazard, the same handler). + raise refused.denial from None except AmbiguousEffect as refused: # SPEC-v0.7 §3.6, before anything else: a store with its own clock re-measures # when an expired lease is declared AMBIGUOUS, and the report belongs beside this @@ -2931,6 +3078,102 @@ def _read_back(self, request: ApprovalRequest) -> ApprovalRecord | None: ) return None + def _in_scope( + self, + action: Action, + scope: _Preconditions | None, + seen: list[str | None] | None, + *, + enforcing: bool = True, + ) -> None: + """SPEC-v0.9 §5: fetch the principal's scope and match this action's resource into it. + + **Strictly before the reservation, before every `_take`** (§5.3), and before `_recheck` + (§5.7): `out_of_scope` says the principal never had the right to the record and + `precondition_changed` says the record moved, and an operator handed the second when the + first is true goes looking for a race that is not there. + + The ordering is the safety argument and not a preference. After the reservation, a + provider that hangs leaves a lease to lapse and an `AMBIGUOUS` record nobody can resolve: + a *scope check* would have manufactured the state it exists to prevent (`v0.7 §6.2`). + + Called twice where `_secure` takes twice, for `v0.7 §6.2`'s reason: the `reconcile` hook + between them is a network call whose duration would otherwise sit inside the window. + Idempotent from the caller's side, and `seen` keeps the hash the first call computed so + the receipt records one answer rather than the last. + """ + if scope is None: + return + # SPEC-v0.9 §5.2.2's observe row: the provider **runs**, so its hash reaches the receipt + # and an operator sizing a scope before turning it on sees what would have happened, and + # it refuses nothing. `v0.3 §6.2` is the rule: observe mode records rather than enforces, + # and a check that enforced under observation would refuse during the phase whose whole + # purpose is to refuse nothing. + refuse = self._refuse_scope if enforcing else self._would_refuse_scope + try: + answered = scope(action) + if not isinstance(answered, Mapping): + raise TypeError( + f"a scope provider returns a mapping, not {type(answered).__name__}" + ) + digest = _scope_hash(answered) + except Exception as exc: + # **Every failure the provider can produce, not two of them** (§5.6): it raised, it + # answered with the wrong shape, or it answered something the canonicalizer refuses. + # All three are "the scope could not be read", which is fail-closed, and none of them + # is `out_of_scope`, which is a statement that it *was* read. + raise refuse(action, SCOPE_UNAVAILABLE, str(exc)) from exc + if seen is not None: + seen.append(digest) + _SCOPE_HASH.set(digest) + patterns = answered.get(_SCOPE_RESOURCES) + if not isinstance(patterns, (list, tuple)) or not all( + isinstance(one, str) for one in patterns + ): + raise refuse( + action, + SCOPE_UNAVAILABLE, + f"the scope carries no usable {_SCOPE_RESOURCES!r} list", + ) + # §4.4's rule for a grant that declares `resources:`, applied here for the same reason: + # an action carrying no resource does not match a scope that names them, and treating it + # as in-scope would make the check optional for any caller who omitted the field. + if action.resource is None or not any( + matches(pattern, action.resource, separator=RESOURCE_SEPARATOR) for pattern in patterns + ): + raise refuse(action, OUT_OF_SCOPE, f"resource {action.resource!r} is not in this scope") + + def _refuse_scope(self, action: Action, reason: str, error: str) -> _ScopeRefusedError: + """The refusal, with its events and its receipt. Returns it for the caller to raise. + + Returned rather than raised so the call site reads `raise self._refuse_scope(...)` and a + reader can see the control flow leaves there: an exception raised inside a helper is a + `return` a linter cannot see. + """ + self._append(EventType.ACTION_DENIED, action, {"reason": reason, "error": error}) + self._record( + action, + Evaluation(Decision.DENY, reason), + ReceiptResult.DENIED, + self._clock(), + error=error, + ) + return _ScopeRefusedError(ActionDenied(f"{action.name} denied: {reason}", reason=reason)) + + def _would_refuse_scope(self, action: Action, reason: str, error: str) -> _ObservedRefusalError: + """Observe mode's counterpart: record what would have happened, and refuse nothing. + + Returns a sentinel the caller raises, which `_observe_secure` catches. A `None` return + would make `_in_scope`'s `raise` a type error and a separate code path in `_in_scope` + would be the flag through it that `_observe_secure`'s own docstring argues against. + """ + self._append( + EventType.ACTION_DENIED, + action, + {"reason": reason, "error": error, "observed": True}, + ) + return _ObservedRefusalError(reason) + def _recheck( self, action: Action, @@ -3714,6 +3957,7 @@ def _record( # path, and so one nobody would notice breaking. authority_grant_id=_AUTHORITY_GRANT_ID.get(None), task=_TASK.get(None), + scope_hash=_SCOPE_HASH.get(None), receipt_id=new_receipt_id(), action_id=action.action_id, action=action.name, @@ -3917,6 +4161,7 @@ def protect( control: Control | None = None, preconditions: Callable[[Action], Mapping[str, Any]] | None = None, task: str | None = None, + scope: Callable[[Action], Mapping[str, Any]] | None = None, ) -> Callable[[Callable[P, R]], Callable[P, R]]: """Bind a function to an action name: every call becomes a decided, recorded Action. @@ -3951,6 +4196,10 @@ def protect( # that changes per call. The operator declares the template; nothing here infers a task # from an argument it was not pointed at, which is the line §6.3 draws. _check_template(name, "task", task) + # SPEC-v0.9 §5.6's third row: **at decoration time** for `@protect`, which is `v0.7 §6.2`'s + # rule for a non-callable `preconditions=`. A misconfiguration an operator hears about at + # import is one they fix before an agent runs, not during. + _checked_scope(scope, f"protect({name!r}, scope=...)") held = None if lease is None else _checked_lease(lease, f"protect({name!r}, lease=...)") _reconciler(reconcile, reconcile_eagerly, f"protect({name!r}") @@ -4032,6 +4281,7 @@ def executor() -> R: reconcile_eagerly=reconcile_eagerly, preconditions=provider, task=bound_task, + scope=scope, ) except ApprovalRequired as pending: if not wait: diff --git a/src/ctrlrun/receipt.py b/src/ctrlrun/receipt.py index 1b8ba444..4d1adbdc 100644 --- a/src/ctrlrun/receipt.py +++ b/src/ctrlrun/receipt.py @@ -100,7 +100,7 @@ #: guarantee catalogue's own rule: a key listed here is a key `to_dict` projects, so naming one #: before something writes it is a `KeyError` on every receipt, which is the field-level form of #: a stub row. Item 7 asserts all three are present before the release. -_V6_KEYS: Final = (*_V5_KEYS, "task") +_V6_KEYS: Final = (*_V5_KEYS, "task", "scope_hash") _KEYS: Final = { _V1: _V1_KEYS, _V2: _V2_KEYS, @@ -476,6 +476,12 @@ class Receipt: #: which is every 0.8.0 call. **Not part of the action hash** (§6.3.1): a field on `Action` #: would move every hash in existence and invalidate every stored approval. task: str | None = None + #: SPEC-v0.9 §5.5: `"sha256:…"` over what the scope provider returned, or `None` where none + #: was configured. **The hash and never the scope**: a scope is a list of what a principal + #: may touch, and an evidence store is not the place to accumulate a second copy of an + #: authorization system's state (`v0.7 §6.10`). Its own domain tag, so it can never equal a + #: precondition fingerprint over the same mapping. + scope_hash: str | None = None #: The schema this receipt is written under (§6.11). A receipt this binary builds is #: `RECEIPT_SCHEMA`; one read from a store keeps the label its document declared, or `""` #: where it declared none, which renders with no `schema` key at all. @@ -568,6 +574,7 @@ def _full_document(self) -> dict[str, Any]: # conditional key is a `KeyError`, and a reader tells "no task" from "this binary # predates tasks" by the schema label. "task": self.task, + "scope_hash": self.scope_hash, } def to_json(self) -> str: @@ -658,6 +665,11 @@ def from_dict(cls, document: Mapping[str, Any]) -> Receipt: if schema == _V6 and isinstance(document.get("task"), str) else None ), + scope_hash=( + document.get("scope_hash") + if schema == _V6 and isinstance(document.get("scope_hash"), str) + else None + ), schema=schema, ) diff --git a/src/ctrlrun/verify/guarantees.py b/src/ctrlrun/verify/guarantees.py index 2b4d2bc9..67165b6f 100644 --- a/src/ctrlrun/verify/guarantees.py +++ b/src/ctrlrun/verify/guarantees.py @@ -151,6 +151,15 @@ class Guarantee: "unapproved policy decides no", ("v0.6 §7.1", "v0.8 §10 T354", "v0.8 §10 T355"), ), + Guarantee( + "G23", + # 32 characters exactly, against `report._TITLE_WIDTH`. "a failing scope provider" and + # not "an out-of-scope record": what G23 grades is the **unavailable** half, because + # that is the one where a kernel could plausibly fail open by treating an unreadable + # scope as an empty constraint (SPEC-v0.9 §5.6, §8). + "a failing scope provider refuses", + ("v0.9 §5.6", "v0.9 §9 T388", "v0.9 §9 T390"), + ), Guarantee( "G24", # 26 characters against `report._TITLE_WIDTH`'s 32. "grant refused" and not "action @@ -248,6 +257,21 @@ class Guarantee: #: SPEC-v0.9 §8, G24. A statement about the operator's **document**, like every reason in this #: module: it says what the document does not declare, not what the kernel is not configured for. NO_TASKS: Final = "no grant names a task" +#: SPEC-v0.9 §8.1, G23. A statement about the **document**, which is what §8.1's argument +#: actually requires: it forbids an `N/A` about whether a *provider* is configured, because that +#: is a fact about an operator's code. Whether any action this configuration admits carries a +#: resource is a fact about their document, and a scope names resources, so an action with none +#: can never be in one. +NO_RESOURCE_TO_SCOPE: Final = "no action this configuration admits carries a resource" + +#: SPEC-v0.9 §8.1 — G23's note, printed beneath its row the way `REVOCATION_NOTE` is. Whether a +#: deployment configures a scope provider is a fact about its own code, which verify cannot read, +#: so verify supplies one and says so rather than reporting `N/A` about something it never saw. +SCOPE_PROVIDER_NOTE: Final = ( + "G23 is graded against a scope provider verify supplies: whether this deployment configures " + "one is a fact about its own code, which verify cannot read. The gateway and the ACS hook " + "cannot name a provider at all (SPEC-v0.9 §5.2.2)" +) #: SPEC-v0.7 §8.9, G16's note, printed once beneath the table as `EFFECT_TEMPLATE_NOTE` is. G16 #: is graded against verify's own stand-in for the operator's provider, because a provider is @@ -351,8 +375,11 @@ class Guarantee: "NO_EXPIRES_AT", "NO_GRANT_COVERS_SELECTION", "NO_GRANT_MATCHES", + "NO_RESOURCE_TO_SCOPE", + "NO_TASKS", "PER_CONNECTION_BACKEND", "PROCESSES", + "SCOPE_PROVIDER_NOTE", "STORE_READS_APPLICATION_CLOCK", "SYNTHETIC_PREFIX", "Guarantee", diff --git a/src/ctrlrun/verify/scenarios.py b/src/ctrlrun/verify/scenarios.py index a07ae2da..0e1e186f 100644 --- a/src/ctrlrun/verify/scenarios.py +++ b/src/ctrlrun/verify/scenarios.py @@ -63,7 +63,7 @@ Subject, contained_dimension, ) -from ..control import Control, context, idempotency_token, protect +from ..control import SCOPE_UNAVAILABLE, Control, context, idempotency_token, protect from ..effect import ( EffectRecord, EffectState, @@ -1087,6 +1087,7 @@ def execute( approval_id: str | None, preconditions: Callable[[Action], Mapping[str, Any]] | None = None, task: str | None = None, + scope: Callable[[Action], Mapping[str, Any]] | None = None, ) -> Receipt: # SPEC-v0.9 §6.3.2 — the active selection's task unless a scenario named one, so a # document that binds its grant to a task does not turn every other guarantee red. G24 @@ -1096,13 +1097,13 @@ def execute( task = self._task if approval_id is None: return control.execute( - action, executor, effect_key, preconditions=preconditions, task=task + action, executor, effect_key, preconditions=preconditions, task=task, scope=scope ) from ..control import with_approval with with_approval(approval_id): return control.execute( - action, executor, effect_key, preconditions=preconditions, task=task + action, executor, effect_key, preconditions=preconditions, task=task, scope=scope ) def refused( @@ -3702,6 +3703,93 @@ def body(detail: dict[str, Any]) -> None: finally: store.close() + # --- G23: a scope provider that cannot answer refuses --------------------------------- + + def g23(self) -> GuaranteeResult: + """SPEC-v0.9 §5.6, §8, and §8.1 on why this one is never `N/A`. + + A scope provider is a Python callable an operator passes at decoration or call time, so + no document states whether one is configured and `verify` reads a document. Rather than + report `N/A` for a fact it cannot observe, verify **constructs the scenario**: it wires a + provider that raises and grades what the kernel does, the way `v0.4 §3` has it construct + every other scenario. + + Both halves, `v0.4 §1.3`. The positive control is a provider that answers and admits the + resource, without which a kernel refusing every scoped action would grade `PASS`. + """ + selection = self.select() + if selection is None: + return self.na("G23", self.unselected(reg.EVERY_ACTION_DENIED)) + # A scope names resources, so an action carrying none can never be in one (§5.6, and + # `v0.3 §4.4`'s rule for a grant that declares `resources:`). Where nothing this document + # admits has a resource, there is no scope question to grade, and saying so is a + # statement about the document rather than about the operator's code. + if selection.resource is None: + scoped = self.select(needs_effect=False, grant_filter=None) + if scoped is None or scoped.resource is None: + return self.na("G23", reg.NO_RESOURCE_TO_SCOPE) + selection = scoped + control, store, recorder, _ = self._control_for("G23", selection) + resource = selection.resource + + def body(detail: dict[str, Any]) -> None: + detail["resource"] = resource + detail["note"] = reg.SCOPE_PROVIDER_NOTE + # The control: a provider that answers, admitting exactly this resource. + answering = _Executor() + action = selection.build() + receipt = self.execute( + control, + action, + answering, + selection.effect_key, + self.approve(control, store, action, selection), + scope=lambda _action: {"resources": [str(resource)]}, + ) + _expect_control( + receipt.result is ReceiptResult.COMMITTED and answering.calls == 1, + "a provider that answers, admitting this resource, lets the action run", + f"it ended {receipt.result} after {answering.calls} executor calls", + ) + + def unavailable(_action: Action) -> Mapping[str, Any]: + raise RuntimeError(f"{reg.SYNTHETIC_PREFIX}: the scope source is unreachable") + + later = selection.build() + blocked = _Executor() + refusal = self.refused( + lambda: self.execute( + control, later, blocked, selection.effect_key, None, scope=unavailable + ), + (ActionDenied,), + "ActionDenied(reason='scope_unavailable') when the provider raises", + "the action ran with no scope answer", + ) + reason = getattr(refusal, "reason", "") + _expect( + reason == SCOPE_UNAVAILABLE, + "ActionDenied(reason='scope_unavailable')", + f"ActionDenied(reason={reason!r})", + ) + _expect( + blocked.calls == 0, + "the executor is not reached when the scope cannot be read", + f"the executor was called {blocked.calls} times", + ) + # **Nothing reserved**, which is the half of G23 that the ordering exists for: a + # provider called after the reservation would leave a lease to lapse. + record = store.get_effect(str(selection.effect_key)) + _expect( + record is None or record.state is not EffectState.RESERVED, + "nothing is left reserved when the scope provider fails", + f"the effect record is {None if record is None else record.state}", + ) + + try: + return self.graded("G23", selection, store, recorder, body) + finally: + store.close() + # --- G24: a task-bound grant is refused off its task ---------------------------------- def g24(self) -> GuaranteeResult: diff --git a/tests/test_demo.py b/tests/test_demo.py index 84ac3203..a3f0f45c 100644 --- a/tests/test_demo.py +++ b/tests/test_demo.py @@ -107,6 +107,8 @@ "authority_grant_id", # SPEC-v0.9 §6, a `ctrlrun.receipt/v6` field. "task", + # SPEC-v0.9 §5.5, a `ctrlrun.receipt/v6` field. + "scope_hash", ) diff --git a/tests/test_preconditions.py b/tests/test_preconditions.py index 9a4e233a..fb0c88ab 100644 --- a/tests/test_preconditions.py +++ b/tests/test_preconditions.py @@ -2001,6 +2001,16 @@ def _guarantee_titles() -> str: #: provider to recheck with, a provider that produced nothing to compare, and a 0.6 process the #: new one has no way to see. ANOTHER_SUBJECT: dict[str, tuple[str, ...]] = { + "docstrings": ( + # SPEC-v0.9 §5. The subject is the **scope provider**, not the precondition recheck, and + # the claim is true of it in a way it is not true of the recheck: the scope fetch happens + # strictly before the reservation, so a provider that hangs leaves nothing reserved and + # nothing executed. That is a closed hole and not a narrowed window, which is precisely + # why this guard exists to keep the two apart. §5.8 states the window the scope provider + # *does* widen, and says so there rather than claiming otherwise here. + "It runs **strictly before the reservation** and before the precondition recheck, so a " + "provider that hangs can only fail closed.", + ), "CHANGELOG.md": ( '"a moved fingerprint is refused" before the reservation, under `ctrlrun.guarantees/v3`.', # "fail-closed" beside the approver string `ctrlrun:precondition-not-recorded`. @@ -2335,7 +2345,7 @@ def test_every_schema_renders_under_its_own_label_and_key_set(): "ctrlrun.receipt/v3": 26, "ctrlrun.receipt/v4": 28, "ctrlrun.receipt/v5": 30, - "ctrlrun.receipt/v6": 31, + "ctrlrun.receipt/v6": 32, "ctrlrun.receipt/v9": 26, "": 25, } diff --git a/tests/test_protect.py b/tests/test_protect.py index 70457136..9a484424 100644 --- a/tests/test_protect.py +++ b/tests/test_protect.py @@ -696,6 +696,8 @@ def read(customer_id: str) -> None: ... "authority_grant_id", # SPEC-v0.9 §6, a `ctrlrun.receipt/v6` field. "task", + # SPEC-v0.9 §5.5, a `ctrlrun.receipt/v6` field. + "scope_hash", } assert document["schema"] == RECEIPT_SCHEMA assert document["receipt_id"].startswith("ctr_") diff --git a/tests/test_scope_provider.py b/tests/test_scope_provider.py new file mode 100644 index 00000000..d0a084a0 --- /dev/null +++ b/tests/test_scope_provider.py @@ -0,0 +1,364 @@ +"""T388 to T398: scope providers (SPEC-v0.9 §5). + +The bite on an identifier an attacker chose. A grant permits `records.read` on `customer:*`; a +scope provider answers whether *this* customer belongs to *this* principal, strictly before the +reservation, fail-closed when it cannot answer. + +Every refusal test asserts the **reason**, never the type alone: three guards deny the same +action with the same exception, and a test that cannot tell which fired is the first of +CONTRIBUTING.md's four shapes of a false green. +""" + +from __future__ import annotations + +import os +import uuid +from datetime import UTC, datetime +from typing import Any + +import pytest + +from ctrlrun.action import Action, Principal +from ctrlrun.control import Control +from ctrlrun.errors import ActionDenied, InvalidArgument +from ctrlrun.policy import Policy +from ctrlrun.receipt import ReceiptResult +from ctrlrun.state import InMemoryStateStore, SQLiteStateStore + +POSTGRES_URL = os.environ.get("CTRLRUN_TEST_POSTGRES") +NOW = datetime(2026, 9, 13, 12, 0, tzinfo=UTC) +AGENT = Principal(agent="reader", user="ada") + +POLICY = """ +schema: ctrlrun.policy/v7 +environment: prod +actions: + records.read: + decision: allow +""" + + +class _Clock: + def __init__(self, now: datetime = NOW) -> None: + self.now = now + + def __call__(self) -> datetime: + return self.now + + +@pytest.fixture +def clock() -> _Clock: + return _Clock() + + +@pytest.fixture( + params=[ + "in-memory", + "sqlite", + pytest.param( + "postgres", + marks=pytest.mark.skipif( + POSTGRES_URL is None, reason="CTRLRUN_TEST_POSTGRES is not set" + ), + ), + ] +) +def store(request, tmp_path, clock): + if request.param == "in-memory": + made: Any = InMemoryStateStore(clock=clock) + elif request.param == "sqlite": + made = SQLiteStateStore(tmp_path / "state.db", clock=clock) + else: + from ctrlrun.postgres import PostgresStateStore + + schema = f"scope_{uuid.uuid4().hex[:12]}" + PostgresStateStore.create_schema(POSTGRES_URL, schema) + made = PostgresStateStore(POSTGRES_URL, schema=schema, clock=clock) + yield made + made.close() + if request.param == "postgres": + from ctrlrun.postgres import PostgresStateStore + + PostgresStateStore.drop_schema(POSTGRES_URL, schema) + + +def _control(store, clock) -> Control: + return Control( + policy=Policy.from_yaml(POLICY, source=""), + store=store, + clock=clock, + environment="prod", + ) + + +def _action(resource: str | None = "customer:1") -> Action: + return Action( + name="records.read", + arguments={"id": "1"}, + principal=AGENT, + resource=resource, + environment="prod", + ) + + +def _scope(*patterns: str): + """A provider that answers, and records that it was asked.""" + + calls: list[Action] = [] + + def provider(action: Action) -> dict[str, Any]: + calls.append(action) + return {"resources": list(patterns)} + + provider.calls = calls # type: ignore[attr-defined] + return provider + + +def test_T388_a_scope_containing_the_resource_permits(store, clock) -> None: + """G23's positive control. Without it a kernel refusing everything grades PASS.""" + control = _control(store, clock) + receipt = control.execute(_action(), lambda: {"ok": True}, "read:1", scope=_scope("customer:1")) + assert receipt.result is ReceiptResult.COMMITTED + + +def test_T389_a_scope_not_containing_the_resource_refuses(store, clock) -> None: + control = _control(store, clock) + with pytest.raises(ActionDenied) as caught: + control.execute( + _action("customer:90210"), lambda: {"ok": True}, "read:x", scope=_scope("customer:1") + ) + assert caught.value.reason == "out_of_scope" + assert store.get_effect("read:x") is None, "nothing may be reserved for an out-of-scope action" + + +def test_T390_a_provider_that_raises_reserves_nothing_and_executes_nothing(store, clock) -> None: + """G23. The provider fails, and the kernel must not have reserved on the way to finding out.""" + calls: list[int] = [] + + def boom(action: Action) -> dict[str, Any]: + raise RuntimeError("the scope service is down") + + control = _control(store, clock) + with pytest.raises(ActionDenied) as caught: + control.execute(_action(), lambda: calls.append(1) or {"ok": True}, "read:1", scope=boom) + assert caught.value.reason == "scope_unavailable" + assert calls == [], "the executor must not run when the scope cannot be read" + assert store.get_effect("read:1") is None, "nothing reserved" + + +def test_T391_a_provider_returning_a_non_mapping_refuses(store, clock) -> None: + """And the **message** names what was wrong, which is what keeps the guard load-bearing. + + A mutation run found the shape guard removable with every test still green: a list reaches + `dict()` inside the hash and raises there anyway, so the refusal happened for a reason that + was not this check. CONTRIBUTING.md's first pattern allows keeping a subsumed branch for its + message, on the condition that a test asserts which message it got. + """ + control = _control(store, clock) + with pytest.raises(ActionDenied) as caught: + control.execute(_action(), lambda: {"ok": True}, "read:1", scope=lambda action: ["nope"]) + assert caught.value.reason == "scope_unavailable" + denied = [e for e in store.events() if e.type.value == "ACTION_DENIED"][-1] + assert "mapping" in str(denied.data.get("error")), ( + "the refusal must name the shape that was wrong, not merely fail somewhere downstream" + ) + + +def test_T394_observe_mode_runs_the_provider_and_refuses_nothing(store, clock) -> None: + """SPEC-v0.9 §5.2.2's observe row, and `v0.3 §6.2`'s rule. + + Observe mode records what it would have done. A scope check that enforced under observation + would refuse during the phase whose entire purpose is to refuse nothing, and + observe-then-enforce is the documented adoption path. Found by a mutation run: collapsing the + two refusal paths into one left every test green. + """ + observing = Control( + policy=Policy.from_yaml( + POLICY.replace("environment: prod", "environment: prod\nmode: observe"), source="" + ), + store=store, + clock=clock, + environment="prod", + ) + calls: list[int] = [] + receipt = observing.execute( + _action("customer:90210"), + lambda: calls.append(1) or {"ok": True}, + "read:x", + scope=_scope("customer:1"), + ) + # It ran: observe mode executes, and records the counterfactual. + assert calls == [1] + assert receipt.result is ReceiptResult.OBSERVED + denied = [ + event + for event in store.events() + if event.type.value == "ACTION_DENIED" and event.data.get("reason") == "out_of_scope" + ] + assert denied, "observe mode must record the scope refusal it did not enforce" + assert denied[0].data.get("observed") is True + + +def test_T392_a_provider_returning_what_the_canonicalizer_refuses(store, clock) -> None: + """`v0.1 §2.3`'s float rejection, inherited: a scope hashed over a float would drift. + + **The float is deliberately NOT in `resources`.** A mutation run found the first version of + this test green against a kernel with the canonicalizer bypassed entirely: `{"resources": + [1.5]}` is refused by the *shape* guard, which wants a list of strings, so the hash never had + to reject anything. That is CONTRIBUTING.md's first mutation pattern, a subsumed guard, and a + test that cannot tell which one fired proves nothing about either. + + Here `resources` is well-formed and the float sits beside it, so the shape guard passes and + only `canonical_bytes` can refuse. + """ + control = _control(store, clock) + with pytest.raises(ActionDenied) as caught: + control.execute( + _action(), + lambda: {"ok": True}, + "read:1", + scope=lambda a: {"resources": ["customer:1"], "quota": 1.5}, + ) + assert caught.value.reason == "scope_unavailable" + + +def test_T392a_a_non_string_key_in_the_scope_is_refused(store, clock) -> None: + """The canonicalizer's other inherited refusal, for the same reason and by the same route.""" + control = _control(store, clock) + with pytest.raises(ActionDenied) as caught: + control.execute( + _action(), + lambda: {"ok": True}, + "read:1", + scope=lambda a: {"resources": ["customer:1"], 7: "not-a-string-key"}, + ) + assert caught.value.reason == "scope_unavailable" + + +def test_T393_the_provider_is_called_before_the_store_call(store, clock) -> None: + """§5.3's ordering IS the safety argument, so it is asserted rather than assumed.""" + seen: list[Any] = [] + + def provider(action: Action) -> dict[str, Any]: + seen.append(store.get_effect("read:1")) + return {"resources": ["customer:1"]} + + control = _control(store, clock) + control.execute(_action(), lambda: {"ok": True}, "read:1", scope=provider) + assert seen == [None], "the provider ran after the reservation; §5.3 forbids it" + + +def test_T395_a_provider_that_hangs_leaves_nothing_reserved(store, clock) -> None: + """A provider that never answers cannot strand a reservation, because it runs first. + + The real hazard is a *slow* provider; the test drives the same window with one that fails + after an arbitrary delay's worth of work, because a test that really slept would be a + timeout rather than an assertion. + """ + + def slow_then_fail(action: Action) -> dict[str, Any]: + for _ in range(1000): + pass + raise TimeoutError("no answer") + + control = _control(store, clock) + with pytest.raises(ActionDenied): + control.execute(_action(), lambda: {"ok": True}, "read:1", scope=slow_then_fail) + assert store.get_effect("read:1") is None + + +def test_T396_no_provider_is_0_8_0_exactly(store, clock) -> None: + """R5. Absent means absent, proven by driving the whole path and reading the receipt.""" + control = _control(store, clock) + receipt = control.execute(_action(), lambda: {"ok": True}, "read:1") + assert receipt.result is ReceiptResult.COMMITTED + assert receipt.to_dict()["scope_hash"] is None + + +def test_T397_the_hash_reaches_the_receipt_and_the_scope_never_does(store, clock) -> None: + """`v0.7 §6.10`'s rule: evidence verifiable without being a copy of the operator's data.""" + # The other patterns this principal may touch. They are NOT the action's resource, which + # legitimately appears on the receipt: a test whose "secret" was the resource would pass + # against a kernel that wrote the whole scope out. + secret = "customer:the-rest-of-adas-book-of-business" + control = _control(store, clock) + receipt = control.execute( + _action(), lambda: {"ok": True}, "read:1", scope=_scope("customer:1", secret) + ) + document = receipt.to_dict() + assert document["scope_hash"] is not None + assert document["scope_hash"].startswith("sha256:") + import json + + assert secret not in json.dumps(document), "the scope's contents reached the evidence" + + +def test_T398a_a_scope_that_is_not_callable_is_refused(store, clock) -> None: + """§5.6's third row, which had no test until the spec's third review round.""" + control = _control(store, clock) + with pytest.raises(InvalidArgument) as caught: + control.execute(_action(), lambda: {"ok": True}, "read:1", scope="not-callable") + assert "scope" in str(caught.value) + + +def test_T398c_an_action_with_no_resource_is_out_of_scope(store, clock) -> None: + """Fail closed, on `matches_shape`'s rule: an action carrying no resource does not match a + scope that names resources. Treating it as in-scope would make the check optional for any + caller who omitted the field.""" + control = _control(store, clock) + with pytest.raises(ActionDenied) as caught: + control.execute( + _action(resource=None), lambda: {"ok": True}, "read:1", scope=_scope("customer:1") + ) + assert caught.value.reason == "out_of_scope" + + +def test_T398d_a_scope_refusal_writes_no_approval_event_and_one_denial(store, clock) -> None: + """SPEC-v0.9 §3.3.2's hazard, met by §5's refusal first. + + `_secure`'s `except ActionDenied` appends `APPROVAL_DENIED` unconditionally, so a scope + refusal raised as an `ActionDenied` fabricates an approval denial for an action no human ever + saw, and records `ACTION_DENIED` twice. Found by reading the events a refusal actually wrote, + which is the only way it shows: the exception, the reason and the receipt were all correct. + """ + control = _control(store, clock) + with pytest.raises(ActionDenied): + control.execute( + _action("customer:90210"), lambda: {"ok": True}, "read:x", scope=_scope("customer:1") + ) + kinds = [event.type.value for event in store.events()] + assert kinds.count("ACTION_DENIED") == 1, kinds + assert "APPROVAL_DENIED" not in kinds, ( + "a scope refusal must not fabricate an approval denial; no human was asked" + ) + assert len(store.receipts()) == 1 + + +def test_T394a_observe_mode_reports_which_refusal_it_would_have_made(store, clock) -> None: + """An independent review found one hardcoded reason for both refusals. + + A deployment whose scope *source was down* read a counterfactual saying the record was not + theirs. Observe mode exists to tell an operator what enforce mode would do, and reporting the + wrong category is the one way it can be worse than useless. + """ + observing = Control( + policy=Policy.from_yaml( + POLICY.replace("environment: prod", "environment: prod\nmode: observe"), + source="", + ), + store=store, + clock=clock, + environment="prod", + ) + + def down(action: Action) -> dict[str, Any]: + raise RuntimeError("the scope service is down") + + observing.execute(_action(), lambda: {"ok": True}, "read:1", scope=down) + denied = [event for event in store.events() if event.type.value == "ACTION_DENIED"] + assert denied and denied[-1].data["reason"] == "scope_unavailable", ( + "a provider that failed must not be reported as an out-of-scope record" + ) + receipt = store.receipts()[-1] + assert receipt.would_have is not None + assert receipt.would_have.blocked_reason == "scope_unavailable" diff --git a/tests/test_verify.py b/tests/test_verify.py index f0510ea8..0163fffd 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -177,13 +177,13 @@ def test_T101_a_policy_with_no_approve_rule_makes_G1_and_G2_not_applicable(tmp_p # because this document names no `max_attempts` (SPEC-v0.7 §8.9). The rest are applicable, # G14 among them, and the count is over those. assert report.applicable == 11 - assert report.not_applicable == 11 + assert report.not_applicable == 12 text = report.to_text() # The fraction is passes over applicable and never the catalogue size: with eight N/As a # seventeen-guarantee catalogue must not report seventeen over seventeen. assert f"{len(reg.GUARANTEES)}/{len(reg.GUARANTEES)}" not in text assert f"{report.passed}/{report.applicable} declared guarantees pass." in text - assert "11 not applicable: G1, G2, G8, G9, G13, G15, G16, G17, G18, G19, G24." in text + assert "12 not applicable: G1, G2, G8, G9, G13, G15, G16, G17, G18, G19, G23, G24." in text def test_T101b_zero_applicable_guarantees_is_not_a_pass(tmp_path): @@ -864,14 +864,14 @@ def test_the_v1_payments_template_reports_eleven_over_eleven(): report = run(V1_PAYMENTS) assert report.exit_code == 0 - assert (report.passed, report.applicable, report.not_applicable) == (11, 11, 11) + assert (report.passed, report.applicable, report.not_applicable) == (11, 11, 12) text = report.to_text() assert "11/11 declared guarantees pass." in text # G13 is N/A on SQLite, which has no clock of its own; G14 and G15 join G3, G4 and G5 where # the effect template lives in the @protect decorator verify does not read, and where the # document names no `max_attempts`. G16 and G18 are graded: verify brings its own provider # for the first and its own approver identity for the second (§8.9, §11.7). - assert "11 not applicable: G3, G4, G5, G8, G9, G13, G14, G15, G17, G19, G24." in text + assert "12 not applicable: G3, G4, G5, G8, G9, G13, G14, G15, G17, G19, G23, G24." in text # §2.1's rule, and the reason this line exists: **the not-applicable ten are not in the # denominator.** Ten pass and ten are N/A, so a run that folded them in would report 20/20. # It used to read `"10/10" not in text`, which said the same thing while the pass count was diff --git a/tests/test_verify_action.py b/tests/test_verify_action.py index d1f108d6..80a9a77d 100644 --- a/tests/test_verify_action.py +++ b/tests/test_verify_action.py @@ -137,10 +137,10 @@ def test_T118_ci_asserts_the_two_shapes_the_specification_names(): steps = _workflow()["jobs"]["verify"]["steps"] script = "\n".join(step.get("run", "") for step in steps) - assert 'test "$AUTHORITY" = "verified 20/20"' in script + assert 'test "$AUTHORITY" = "verified 21/21"' in script assert 'test "$TEMPLATES" = "verified 11/11"' in script assert 'test "$AUTHORITY_NA" = "2"' in script - assert 'test "$TEMPLATES_NA" = "11"' in script + assert 'test "$TEMPLATES_NA" = "12"' in script @pytest.mark.authority @@ -152,13 +152,13 @@ def test_T118_the_two_configurations_really_do_report_those_shapes(): templates = run(V1_PAYMENTS) assert authority.badge is not None - assert authority.badge["message"] == "verified 20/20" + assert authority.badge["message"] == "verified 21/21" # G13 and G15: SQLite has no clock of its own to diverge from, and the document declares # no `max_attempts` (SPEC-v0.7 §8.9). assert authority.not_applicable == 2 assert templates.badge is not None assert templates.badge["message"] == "verified 11/11" - assert templates.not_applicable == 11 + assert templates.not_applicable == 12 def test_T118_the_action_uploads_the_report_and_writes_a_job_summary(): @@ -211,7 +211,7 @@ def test_T119_the_colour_is_about_failures_and_has_no_amber_for_not_applicable( from ctrlrun.verify import scenarios passing = run(V1_PAYMENTS) - assert passing.not_applicable == 11 + assert passing.not_applicable == 12 assert passing.badge is not None assert passing.badge["color"] == BADGE_PASS_COLOR diff --git a/tests/test_verify_report.py b/tests/test_verify_report.py index 5cc975bb..e38f4548 100644 --- a/tests/test_verify_report.py +++ b/tests/test_verify_report.py @@ -109,7 +109,7 @@ def test_T113_the_summary_is_the_last_line_and_names_the_not_applicable_ids(tmp_ # G13 is N/A on every SQLite run: SQLite has no clock of its own. G14 needs the effect # template this document keeps in the @protect decorator, and G15 a `max_attempts` it does # not declare (SPEC-v0.7 §8.9). - assert "11 not applicable: G3, G4, G5, G8, G9, G13, G14, G15, G17, G19, G24." in last + assert "12 not applicable: G3, G4, G5, G8, G9, G13, G14, G15, G17, G19, G23, G24." in last # The fraction is passes over applicable. A report with eight N/As does not say 17/17. assert "18/18" not in text @@ -137,9 +137,9 @@ def test_T113_a_failing_report_names_the_subject_and_prints_the_counterexample( # SPEC-v0.8 item 2: G18 joins the catalogue. It is graded wherever the document sends # an action to approval, which the first two of these do, and `N/A` for G1's reason # where nothing does. So the first two gain a pass and the third gains an N/A. - (ALL_APPLICABLE, "15/15 declared guarantees pass. 7 not applicable"), - (WITH_NOT_APPLICABLE, "11/11 declared guarantees pass. 11 not applicable"), - (EMPTY, "0/0 declared guarantees pass. 22 not applicable"), + (ALL_APPLICABLE, "16/16 declared guarantees pass. 7 not applicable"), + (WITH_NOT_APPLICABLE, "11/11 declared guarantees pass. 12 not applicable"), + (EMPTY, "0/0 declared guarantees pass. 23 not applicable"), ], ids=["passing", "some-na", "all-na"], ) @@ -454,7 +454,7 @@ def test_T116_a_run_with_several_not_applicable_still_exits_0(tmp_path, monkeypa result = _cli(tmp_path, WITH_NOT_APPLICABLE) assert result.exit_code == 0 - assert "11 not applicable" in result.stdout + assert "12 not applicable" in result.stdout def test_T116_json_and_junit_can_be_combined(tmp_path, monkeypatch):