diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c6e07484b..d5171173de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1005,6 +1005,7 @@ jobs: tests/integration/test_sql_surface_live_ch.py tests/integration/test_pg_clickhouse_live_ch.py tests/integration/test_cell_tenancy_pg.py + tests/integration/test_cell_each_native.py tests/integration/test_router_directory_pg.py tests/integration/test_pg_online_index.py tests/integration/test_pg_owned_cleanup.py diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index ff46557fb8..513a87bbe8 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -112,12 +112,63 @@ What spans orgs becomes per cell: and overrides, policy rules, rate-limit and budget overrides, signing-key rotation, a subject's revocation. `loom cell each -- ` runs an org-free admin command in every cell with a short-lived credential per cell, records each cell's outcome, - resumes the cells that failed, and fails until all succeed; `loom doctor` compares the + resumes confirmed no-action failures, and stops on an uncertain mutation; + `loom doctor` compares the cells' settings and configuration fingerprints and names what differs; - the code and tracker orgs (`LOOM_CODE_ORG`, `LOOM_TRACKER_ORG`) are each cell's own: each cell indexes its own repositories, at the ingest cost of each; - an operator's platform view and the dashboard are per cell, at the cell's host. +## Administration across cells + +`loom cell each` is a serial HTTP coordinator with a closed command list. It does not +start a command in a cell or select a store. It supports live settings, signing keys, +an exact subject's revocation, namespace-name rate limits, operation and consumer +latency budgets, and global blacklist entries. Tenant policy and org-limit sweeps are +outside this command. See [cell administration](../operations/cell-administration.md). + +The operator authenticates a bounded directory census once before the run, reads each +target again before its attempt, and compares a final whole census. Every registered +cell must be serving, enforced and supported; closing placements does not exclude a +cell. Identity includes the deployment, public origin, control generation, complete +configuration digests and separate automation and recovery public identities. A changed +target refuses continuation. Fingerprint digests are SHA-256 over the complete stored +Postgres JSONB text, rather than the shortened fingerprint printed for operators. + +The coordinator reads an explicitly named private key for each cell and proves that +its public thumbprint matches the automation identity before signing an OAuth +assertion. Each exchange requests a sixty-second access token. Credentials travel +only to their exact HTTPS origin, with ordinary certificate verification and without +redirects, proxy discovery or automatic retries. The recovery key is never a fallback. + +The cell applies its ordinary request lifecycle and authentication gates, then requires +a current registered automation platform-admin bot. The bot has one approved public +key and no alternate workload trust. A fresh read authenticated with that cell's own +router credential supplies the authoritative public pair; request text cannot select +an automation identity. Empty or malformed directory metadata refuses. This is a fresh +admission check, not an atomic transaction with later directory changes. + +`loom cell automation` provisions the public pair after proving both explicit keys +against that cell's current bot registrations. It uses the existing native cell +control owner, registration comparison and directory generation claim. The directory +stores only the two purposes, names and public thumbprints in `break_glass`, with an +operator audit in the same transaction. Registration reruns preserve this pair. + +Each action awaits a strict started audit, its owning business helper, and a strict +completed audit. Correlation fields do not provide idempotency. Before a mutating +request can leave the coordinator, its private durable state records `dispatching`. +An ambiguous response becomes `UNSETTLED` and stops later cells. Resume refuses this +uncertainty before reading credentials or opening a network connection. Completed +outcomes remain immutable; an explicit correlated refusal before business work may be +retried. There is no receipt lookup or automatic replay of an uncertain mutation. + +The census is limited to 4,096 cells and four MiB, with one KiB per public target. +Individual action bodies are capped at 64 KiB before JSON parsing, including when the +global request limit is disabled. A run has a one-hour deadline and each server action +has thirty seconds. Local state is capped at 32 MiB, protected by one lifetime file +lock, and replaced with file and directory synchronization. These bounds refuse excess +work rather than silently truncating a fleet, result or security read. + ## The router The router is a stateless HTTP proxy, run as several replicas behind the deployment's @@ -1491,6 +1542,7 @@ entry points and the named exceptions: | `cell_prepare._write_rows` | Fresh claim, observed local rows, exact lock backend | | `cell_attach_pg._mode_transaction` | Attach's claim, supervised mode try, recording/first/fingerprint comparison, exact lock backend | | `cell_control.backout` | Fresh claim, fenced close and zero-placement read, observed local rows, exact lock backend | +| `cell_automation.provision`, `directory.set_administration` | Fresh claim, paired authority, two live bot proofs, fenced public metadata and operator audit | | `cell_update_pg.write.change` | Fresh directory claim; owner's bounded transaction on the exact ATTACH backend, paired registration and exact observed local identity/pair before tenancy repair and acceptance write | | `directory.attach_begin`, `attach_place`, `attach_enforced`, `attach_serving`, `remove` | Claimed generation, full registration and exact pair, including retries/no-ops | | `directory.update_addresses` | Claimed generation, full registration and exact pair under directory/row locks; commit-time address check, audit and synchronous WAL acknowledgement in one bounded transaction | diff --git a/docs/architecture/system-overview.md b/docs/architecture/system-overview.md index 1b962fd956..8e93adfff3 100644 --- a/docs/architecture/system-overview.md +++ b/docs/architecture/system-overview.md @@ -108,9 +108,10 @@ HTTP answers with a status and a `detail` body, MCP with a result that carries ` ### HTTP `build_router()` in `server/transport/http.py` adds one `POST /v1/` route per op. -`BodyLimitMiddleware` first refuses a body over `LOOM_MAX_REQUEST_BYTES` (32 MiB) with 413; -0 disables the cap. Each request then passes these phases in order (`_make_endpoint`). A -refusal ends the request. +`BodyLimitMiddleware` refuses oversized bodies with 413. `LOOM_MAX_REQUEST_BYTES` +(32 MiB) sets the global cap; 0 disables only that cap. Fixed cell administration +caps remain. Each request then passes these phases in order (`_make_endpoint`). +A refusal ends the request. | Phase | Code | Refusal | | -------------------- | -------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | diff --git a/docs/operations/cell-administration.md b/docs/operations/cell-administration.md new file mode 100644 index 0000000000..b8002387c7 --- /dev/null +++ b/docs/operations/cell-administration.md @@ -0,0 +1,107 @@ +# Administer every cell + +`loom cell each` applies one supported administration command to every registered cell +and records confirmed outcomes in a private local file. Run it from the operator's +machine with access to the router and each cell's public HTTPS origin. It needs no +local Postgres or ClickHouse connection, server profile, or root signing secret. + +Every cell must be serving and enforced. A cell closed to placements remains a target. +An attaching cell, unsupported server, changed deployment, missing key or incomplete +automation registration refuses the run. + +## Prepare credentials + +Each cell has two distinct key-proof platform-admin bots: one for automation and one +for recovery. Their names and public key thumbprints are registered in the directory. +Keep their private keys separate. The automation bot accepts exactly its approved +public key and has no alternate workload trust. This restriction lets the cell match a +short-lived access token's bot to the one approved key without inventing an assertion +key claim in the token. + +Register both purposes only after each bot has its own approved key and platform-admin +grant. On the configured cell operator profile, run `loom cell automation --cell CELL +--automation-bot NAME --automation-thumbprint THUMBPRINT --automation-key PRIVATE_FILE +--recovery-bot NAME --recovery-thumbprint THUMBPRINT --recovery-key PRIVATE_FILE`. +The command holds the cell's control lock, claims its current directory registration, +and proves both keys through the cell's public authentication and bot-registration +endpoints before writing the public pair. Neither private key is sent to the router. + +Create a private credential manifest with these fields: + +| Field | Value | +| ------------------- | ----------------------------------------------------------------------------- | +| `version` | `1` | +| `router_token_file` | Absolute path to the private router operator-token file | +| `keys` | One entry per target, with `cell`, `deployment_id` and an absolute `key_file` | +| `ca_file` | Optional absolute path to an explicit CA certificate bundle | + +The manifest, token and key files must be regular files owned by the current user, +with private permissions and private containing directories. Symlinks and unsafe +writable ancestors refuse. The command does not change permissions or use ambient +server credentials. It reads each automation key at use, derives its public thumbprint, +and refuses a mismatch before signing. It never tries a recovery key. + +Use the router's exact HTTPS origin for `--router`. Redirects and ambient HTTP proxies +are disabled. The operator token goes only to the router; assertions and cell tokens +go only to the cell's exact public origin. Normal CA and hostname verification applies. +For private PKI, configure the cell server's operating-system CA trust for its router +connection. The server uses the normal TLS trust store, including `SSL_CERT_FILE` +when the operator supplies it, while ignoring ambient HTTP proxies. The manifest's +`ca_file` controls the command's router and cell connections; it does not change the +server's trust store. Neither side disables certificate or hostname verification. + +## Run a supported command + +Set `ROUTER`, `CREDENTIALS` and `STATE` to the trusted router origin, private manifest +path and a fresh private outcome-file path, then run: + +```bash +loom cell each --router "$ROUTER" --credentials "$CREDENTIALS" --state "$STATE" -- settings list +``` + +The supported command families are closed: + +| Family | Actions | +| -------------- | ------------------------------------------------------- | +| `settings` | `list`, `set`, `reset`, `history` for live settings | +| `signing-key` | `list`, `rotate`, `retire` | +| `revoke-token` | One exact global subject | +| `ratelimit` | Namespace-name limit, shared across orgs with that name | +| `budget` | Operation budget or consumer multiplier | +| `blacklist` | Global `add`, `list` | + +Tenant policy, org-limit sweeps, arbitrary commands and native store operations are +not supported. Use `loom cell each --help` for coordinator options. A server lacking +the capability contract or requested action refuses before business work; the command +does not fall back to an older API. +Global blacklist removal is unavailable. A parsed removal command receives no +advertised capability. + +The command runs one cell at a time. It reads the directory before the run, again for +each target, and once after all attempts. Exit zero requires every target to have a +confirmed completion and the final identities to match. Directory drift discovered +at the end fails the run while retaining the completed history. + +## Resume confirmed failures + +Use the existing state file without supplying another command: + +```bash +loom cell each --router "$ROUTER" --credentials "$CREDENTIALS" --resume "$STATE" +``` + +Completed cells are not repeated. Confirmed pre-business refusals can be attempted +again with fresh credentials. The original command and router binding cannot change. +New cells require complete key coverage and are appended only while existing target +identities remain unchanged. Removed or replaced targets refuse continuation. + +`dispatching` and `UNSETTLED` indicate that a mutation may have reached a cell without +a confirmed response. Resume stops before credential or network access. Preserve the +state and inspect that cell's audit and business state using the separate recovery +procedure; do not edit away uncertainty or automatically replay the command. Action +correlation is not idempotency, and an HTTP error alone does not prove no action ran. + +Keep outcome files private. They contain public target identities, command arguments, +completion hashes and result digests, with no access token or private key. Command +arguments can still contain sensitive setting values or blacklist text. Preserve the +file and its sibling lock location for the lifetime of the run. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 2bd191143d..16d14789cd 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -330,6 +330,8 @@ python scripts/codegen/docs_reference.py | `loom cell` | Operate a deployment's cells through the router's directory. | | | `loom cell scim-token` | Generate a cell's SCIM token offline. | | | `loom cell scim-token new` | Print a random token for the named cell. | `--cell` | +| `loom cell each` | Run a supported admin command in every cell | `--router`, `--credentials`, `--state`, `--resume`, `` | +| `loom cell automation` | Prove and register distinct automation and recovery bot keys. | `--cell`, `--router`, `--automation-bot`, `--automation-thumbprint`, `--automation-key`, `--recovery-bot`, `--recovery-thumbprint`, `--recovery-key`, `--ca-file` | | `loom cell credential` | Make a router credential, or rotate a cell's. | ``, `--cell`, `--next`, `--finish`, `--revert`, `--operator`, `--router` | | `loom cell limit` | Close, reopen or bound a cell to new orgs. | `--cell`, `--closed`, `--open`, `--placements-max`, `--router` | | `loom cell which` | Print the lookup's cell and host for an org (the first cell if not placed). | `--org`, `--router` | diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 1e6e899d1c..a1a133a8a6 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -129,7 +129,7 @@ Authentication, tokens and identity providers. | `LOOM_RESTRICTED_SERVE` | `bool` | `True` | | Kill switch for restricted namespaces. True: served to a principal whose token names the namespace AND who holds a live direct grant. False: every read is refused while ingest continues. | | `LOOM_RESTRICTED_AUDIT_INLINE` | `bool` | `True` | | Wait for the `namespace.restricted_access` audit row of each admitted restricted read before answering; false writes it beside the answer, lost if the process dies first. | | `LOOM_CORS_ORIGINS` | `str` | `''` | | Comma-separated origin list. "*" allows all (acceptable when auth_required=True). Empty disables CORS entirely. | -| `LOOM_MAX_REQUEST_BYTES` | `int` | `33554432` | | Request-body cap, bytes, enforced by ASGI middleware before any parser (413 PAYLOAD_TOO_LARGE); uvicorn has no limit of its own. 0 disables. | +| `LOOM_MAX_REQUEST_BYTES` | `int` | `33554432` | | Global request-body cap, bytes, enforced by ASGI middleware before parsing (413 PAYLOAD_TOO_LARGE). 0 disables the global cap; fixed cell administration caps remain. | | `LOOM_HSTS_MAX_AGE_SECONDS` | `int` | `31536000` | | Strict-Transport-Security max-age, seconds; browsers ignore it over plain HTTP. 0 sends no header. | | `LOOM_POLICY_MODE` | `'off' \| 'shadow' \| 'enforce'` | `'off'` | | History policy over the agent trace: "off" reads/writes nothing; "shadow" records verdicts in loom.policy_decisions without blocking; "enforce" denies (403 POLICY_FORBIDDEN) and records them too. In enforce, writes fail closed on evaluation errors; reads never do. | | `LOOM_POLICY_MAX_WINDOW_SECONDS` | `int` | `86400` | | Cap on any rule's look-back window, in seconds. | diff --git a/docs/reference/errors.md b/docs/reference/errors.md index 003aff192d..705b71b7ff 100644 --- a/docs/reference/errors.md +++ b/docs/reference/errors.md @@ -752,6 +752,10 @@ HTTP 503, raised as `server.core.pg.PostgresUnavailableError`. 503 — Postgres, HTTP 409, raised as `server.core.errors.PreconditionFailedError`. 409 — `if_match_sha256` mismatch, an `approved_sha256` that no longer matches the definition's text, or an erasure that landed while a `namespace.restore` ran. Category `input`, retryable: no. What to do: [common errors](#common-errors-and-what-to-do). +#### `PREFLIGHT_REFUSED` + +Raised in `server/cli/cell_each_http.py`. + #### `PROBE_BUSY` Raised in `server/router/app.py`. Category `capacity`, retryable: yes. @@ -780,6 +784,10 @@ HTTP 429, raised as `server.core.errors.RateLimitedError`. 429 — the caller is HTTP 409, raised as `server.core.errors.ReadsLlmFreeError`. 409 — `memory.ask` or `code.ask` asked for a model's answer while `LOOM_READS_LLM_FREE` is on. Category `input`, retryable: no. What to do: [common errors](#common-errors-and-what-to-do). +#### `READ_INCOMPLETE` + +Raised in `server/cli/cell_each_state.py`. + #### `RECALL_REQUIRES_FILTER` Raised in `server/memory/app/recall.py`. @@ -928,6 +936,10 @@ Raised in `server/knowledge/app/writes.py`. Raised in `server/knowledge/app/sources.py`, `server/tracker/app/sources.py`. +#### `SUBJECT_REVOCATION_INCOMPLETE` + +Raised in `server/insights/app/cell_admin_security_actions.py`. + #### `SUBSTRING_NOT_FOUND` Raised in `server/memory/app/profile.py`, `server/memory/app/working_memory.py`. diff --git a/docs/reference/http-api.md b/docs/reference/http-api.md index 7230e8a512..76fa05f3e6 100644 --- a/docs/reference/http-api.md +++ b/docs/reference/http-api.md @@ -100,10 +100,22 @@ the same routes), and the curated `agent.deployment` methods over them. | `POST /v1/admin/audit.summary` | `platform_admin` | Aggregate audit_log by event_type in a window. | | `POST /v1/admin/bots.create` | `platform_admin` | Register a bot. | | `POST /v1/admin/bots.disable` | `platform_admin` | Disable a bot: its proofs are refused and the tokens it holds revoked. | -| `POST /v1/admin/bots.list` | `platform_admin` | Every registered bot: its namespaces and roles, key or issuer, and limits. | +| `POST /v1/admin/bots.list` | `platform_admin` | Every registered bot, or with self_only the caller's fresh bounded public identity. | | `POST /v1/admin/bots.provision_agent` | `platform_admin` | Make the namespace of a trust's agent before its first exchange, as that exchange would, so grants can name the agent first. | | `POST /v1/admin/budget.set` | `platform_admin` | Install or clear a per-operation latency-budget override. | | `POST /v1/admin/budget.violations` | `platform_admin` | Aggregate recent `budget.violated` events (loom.events) for ops alerting. | +| `POST /v1/admin/cell.blacklist_global_add` | `platform_admin automation bot` | Add one exact global blacklist pattern with the verified bot as its author. | +| `POST /v1/admin/cell.blacklist_global_list` | `platform_admin automation bot` | List bounded exact-global blacklist rows and report whether more rows exist. | +| `POST /v1/admin/cell.budget_set` | `platform_admin automation bot` | Change one latency budget through this verified cell's ordinary budget owner. | +| `POST /v1/admin/cell.capabilities` | `platform_admin automation bot` | Report this verified cell's supported administration actions and fixed limits. | +| `POST /v1/admin/cell.ratelimit_namespace_set` | `platform_admin automation bot` | Change a namespace-label rate override through the cell's ordinary limiter. | +| `POST /v1/admin/cell.settings_history` | `platform_admin automation bot` | Read the bounded change history of one live setting in this verified cell. | +| `POST /v1/admin/cell.settings_list` | `platform_admin automation bot` | List non-secret settings after fresh cell and automation authorization. | +| `POST /v1/admin/cell.settings_set` | `platform_admin automation bot` | Change one live setting and confirm its completed cell administration audits. | +| `POST /v1/admin/cell.signing_keys_list` | `platform_admin automation bot` | List public signing-key metadata after fresh cell and automation authorization. | +| `POST /v1/admin/cell.signing_keys_retire` | `platform_admin automation bot` | Retire one signing key through this verified cell's ordinary retirement checks. | +| `POST /v1/admin/cell.signing_keys_rotate` | `platform_admin automation bot` | Rotate a signing key through the ordinary cell owner and return public metadata. | +| `POST /v1/admin/cell.subject_revoke` | `platform_admin automation bot` | Revoke one exact global subject through the bounded ordinary security owner. | | `POST /v1/admin/claim_release` | `org_admin` | Release a self-signup claim: revoke its grants, clear the owner, deactivate it; the name stays reserved. | | `POST /v1/admin/consumers` | `platform_admin` | Per-consumer calls, error rate, p50, spend and outcome feedback over each requested window — read-only aggregates over agent_events and code_search_outcome, joined by consumer name. | | `POST /v1/admin/events.tail` | `platform_admin` | Recent rows from `loom.agent_events` for live debugging. | diff --git a/llms.txt b/llms.txt index f6a1d81cf2..b05926e26f 100644 --- a/llms.txt +++ b/llms.txt @@ -107,6 +107,7 @@ Start with [the docs map](docs/index.md). To install Loom, a coding agent reads - [Exporting usage to a data warehouse](docs/operations/analytics-export.md): `loom export-analytics` writes one UTC day of usage and a snapshot of the namespaces to an S3 bucket as Parquet files, one file per entity per day, for a data warehouse to load. - [Backup and recovery](docs/operations/backup-and-recovery.md): A Loom backup is a ClickHouse backup, plus the document bucket while any catalog row names an object in it, and under the Postgres authority a backup of Postgres too. +- [Administer every cell](docs/operations/cell-administration.md): `loom cell each` applies one supported administration command to every registered cell and records confirmed outcomes in a private local file. - [Attaching the first cell](docs/operations/cell-attach.md): `loom cell attach` turns a prepared first cell (`loom cell prepare`, mode `recording`, directory row `attaching`) into a `serving` one: it places every org the cell holds in the router's directory, raises the cell's mode to `enforced` and… - [Cell control upgrades and recovery](docs/operations/cell-control.md): Cell control mutations require generation-aware clients and router handlers, and direct access to the cell's Postgres. - [Update a cell's configuration or addresses](docs/operations/cell-update.md): `loom cell update` reserves a configuration before the cell accepts it. diff --git a/loom.example.yaml b/loom.example.yaml index 2bc42fdfdd..f40f693755 100644 --- a/loom.example.yaml +++ b/loom.example.yaml @@ -905,8 +905,9 @@ settings: # Empty disables CORS entirely. [restart] # cors_origins: "" - # Request-body cap, bytes, enforced by ASGI middleware before any parser (413 - # PAYLOAD_TOO_LARGE); uvicorn has no limit of its own. 0 disables. [restart] + # Global request-body cap, bytes, enforced by ASGI middleware before parsing (413 + # PAYLOAD_TOO_LARGE). 0 disables the global cap; fixed cell administration caps + # remain. [restart] # max_request_bytes: 33554432 # Strict-Transport-Security max-age, seconds; browsers ignore it over plain HTTP. 0 diff --git a/schemas/loom-settings.schema.json b/schemas/loom-settings.schema.json index 75c2c45c1f..653f939d6c 100644 --- a/schemas/loom-settings.schema.json +++ b/schemas/loom-settings.schema.json @@ -3763,7 +3763,7 @@ "max_request_bytes": { "default": 33554432, "type": "integer", - "description": "Request-body cap, bytes, enforced by ASGI middleware before any parser (413 PAYLOAD_TOO_LARGE); uvicorn has no limit of its own. 0 disables." + "description": "Global request-body cap, bytes, enforced by ASGI middleware before parsing (413 PAYLOAD_TOO_LARGE). 0 disables the global cap; fixed cell administration caps remain." }, "mcp_always_load_tools": { "default": "", diff --git a/scripts/codegen/sdk_py_admin_gen.py b/scripts/codegen/sdk_py_admin_gen.py index 27de06110c..5b9ab55a19 100644 --- a/scripts/codegen/sdk_py_admin_gen.py +++ b/scripts/codegen/sdk_py_admin_gen.py @@ -44,10 +44,12 @@ def _routers() -> list[tuple[Any, str]]: from server.identity.app import claim_release from server.identity.app.admin_account_routes import router as accounts from server.insights.app.admin_routes import admin_router, org_admin_router + from server.insights.app.cell_admin_routes import cell_admin_router from server.insights.app.provider_routes import provider_router return [ (admin_router, "platform_admin"), + (cell_admin_router, "platform_admin automation bot"), (org_admin_router, "org_admin"), (provider_router, "platform_admin"), (accounts, "platform_admin"), diff --git a/sdk/python/loom/admin_routes.py b/sdk/python/loom/admin_routes.py index b61a4ca769..18f5a012e7 100644 --- a/sdk/python/loom/admin_routes.py +++ b/sdk/python/loom/admin_routes.py @@ -44,7 +44,7 @@ async def bots_disable(self, body: dict[str, Any] | None = None) -> Any: ) async def bots_list(self, body: dict[str, Any] | None = None) -> Any: - """`POST /v1/admin/bots.list` (platform_admin). Every registered bot: its namespaces and roles, key or issuer, and limits.""" + """`POST /v1/admin/bots.list` (platform_admin). Every registered bot, or with self_only the caller's fresh bounded public identity.""" return await self._agent.transport.request_json( "POST", "/v1/admin/bots.list", json=body or {} ) @@ -67,6 +67,78 @@ async def budget_violations(self, body: dict[str, Any] | None = None) -> Any: "POST", "/v1/admin/budget.violations", json=body or {} ) + async def cell_blacklist_global_add(self, body: dict[str, Any] | None = None) -> Any: + """`POST /v1/admin/cell.blacklist_global_add` (platform_admin automation bot). Add one exact global blacklist pattern with the verified bot as its author.""" + return await self._agent.transport.request_json( + "POST", "/v1/admin/cell.blacklist_global_add", json=body or {} + ) + + async def cell_blacklist_global_list(self, body: dict[str, Any] | None = None) -> Any: + """`POST /v1/admin/cell.blacklist_global_list` (platform_admin automation bot). List bounded exact-global blacklist rows and report whether more rows exist.""" + return await self._agent.transport.request_json( + "POST", "/v1/admin/cell.blacklist_global_list", json=body or {} + ) + + async def cell_budget_set(self, body: dict[str, Any] | None = None) -> Any: + """`POST /v1/admin/cell.budget_set` (platform_admin automation bot). Change one latency budget through this verified cell's ordinary budget owner.""" + return await self._agent.transport.request_json( + "POST", "/v1/admin/cell.budget_set", json=body or {} + ) + + async def cell_capabilities(self, body: dict[str, Any] | None = None) -> Any: + """`POST /v1/admin/cell.capabilities` (platform_admin automation bot). Report this verified cell's supported administration actions and fixed limits.""" + return await self._agent.transport.request_json( + "POST", "/v1/admin/cell.capabilities", json=body or {} + ) + + async def cell_ratelimit_namespace_set(self, body: dict[str, Any] | None = None) -> Any: + """`POST /v1/admin/cell.ratelimit_namespace_set` (platform_admin automation bot). Change a namespace-label rate override through the cell's ordinary limiter.""" + return await self._agent.transport.request_json( + "POST", "/v1/admin/cell.ratelimit_namespace_set", json=body or {} + ) + + async def cell_settings_history(self, body: dict[str, Any] | None = None) -> Any: + """`POST /v1/admin/cell.settings_history` (platform_admin automation bot). Read the bounded change history of one live setting in this verified cell.""" + return await self._agent.transport.request_json( + "POST", "/v1/admin/cell.settings_history", json=body or {} + ) + + async def cell_settings_list(self, body: dict[str, Any] | None = None) -> Any: + """`POST /v1/admin/cell.settings_list` (platform_admin automation bot). List non-secret settings after fresh cell and automation authorization.""" + return await self._agent.transport.request_json( + "POST", "/v1/admin/cell.settings_list", json=body or {} + ) + + async def cell_settings_set(self, body: dict[str, Any] | None = None) -> Any: + """`POST /v1/admin/cell.settings_set` (platform_admin automation bot). Change one live setting and confirm its completed cell administration audits.""" + return await self._agent.transport.request_json( + "POST", "/v1/admin/cell.settings_set", json=body or {} + ) + + async def cell_signing_keys_list(self, body: dict[str, Any] | None = None) -> Any: + """`POST /v1/admin/cell.signing_keys_list` (platform_admin automation bot). List public signing-key metadata after fresh cell and automation authorization.""" + return await self._agent.transport.request_json( + "POST", "/v1/admin/cell.signing_keys_list", json=body or {} + ) + + async def cell_signing_keys_retire(self, body: dict[str, Any] | None = None) -> Any: + """`POST /v1/admin/cell.signing_keys_retire` (platform_admin automation bot). Retire one signing key through this verified cell's ordinary retirement checks.""" + return await self._agent.transport.request_json( + "POST", "/v1/admin/cell.signing_keys_retire", json=body or {} + ) + + async def cell_signing_keys_rotate(self, body: dict[str, Any] | None = None) -> Any: + """`POST /v1/admin/cell.signing_keys_rotate` (platform_admin automation bot). Rotate a signing key through the ordinary cell owner and return public metadata.""" + return await self._agent.transport.request_json( + "POST", "/v1/admin/cell.signing_keys_rotate", json=body or {} + ) + + async def cell_subject_revoke(self, body: dict[str, Any] | None = None) -> Any: + """`POST /v1/admin/cell.subject_revoke` (platform_admin automation bot). Revoke one exact global subject through the bounded ordinary security owner.""" + return await self._agent.transport.request_json( + "POST", "/v1/admin/cell.subject_revoke", json=body or {} + ) + async def claim_release(self, body: dict[str, Any] | None = None) -> Any: """`POST /v1/admin/claim_release` (org_admin). Release a self-signup claim: revoke its grants, clear the owner, deactivate it; the name stays reserved.""" return await self._agent.transport.request_json( diff --git a/server/auth/grant_claims.py b/server/auth/grant_claims.py index 7092f532bd..8d6ec208a1 100644 --- a/server/auth/grant_claims.py +++ b/server/auth/grant_claims.py @@ -103,11 +103,40 @@ def collections_change(org: str) -> AbstractAsyncContextManager[None]: return _claimed("collections:" + json.dumps(org), f"the collections of {org}") -def service_account_change(org: str, name: str) -> AbstractAsyncContextManager[None]: +def service_account_change( + org: str, + name: str, + *, + deadline: float | None = None, + ch: object = None, +) -> AbstractAsyncContextManager[None]: """Hold a person's service-account name in `org` from the check that it is free to its row: a second create with the name would replace the first row, and its token, still live, would be listed nowhere.""" - return _claimed("service_account:" + json.dumps([org, name]), f"the token name {name!r}") + key = "service_account:" + json.dumps([org, name]) + if deadline is not None: + return _bounded_service_account(key, deadline=deadline, ch=ch) + return _claimed(key, f"the token name {name!r}") + + +@asynccontextmanager +async def _bounded_service_account(key: str, *, deadline: float, ch: object) -> AsyncIterator[None]: + from server.core.leases import holding + + async with holding( + key, + ttl_s=TTL_S, + wait_s=0, + ch=ch, + deadline=deadline, + confirm_release=True, + ) as held: + if not held: + raise GrantBusyError("another change holds the service-account name") + async with _bounded( + "a service-account name", 2.0, min(CHANGE_S, deadline - time.monotonic()) + ): + yield def bot_change(name: str) -> AbstractAsyncContextManager[None]: diff --git a/server/auth/keyring.py b/server/auth/keyring.py index ff5c413d4b..c52fdc74f8 100644 --- a/server/auth/keyring.py +++ b/server/auth/keyring.py @@ -466,7 +466,7 @@ async def refresh(*, ch: _CHClient | None = None) -> bool: return await _read(ch=ch) is not None -async def _read(*, ch: _CHClient | None = None) -> _View | None: +async def _read(*, ch: _CHClient | None = None, deadline: float | None = None) -> _View | None: """Re-read the keyring and install the view; None when the read failed.""" from server.core.ch import get_client, sequential_reads @@ -475,11 +475,21 @@ async def _read(*, ch: _CHClient | None = None) -> _View | None: # Sequentially consistent: a rotation or retirement another process # acknowledged is in the read, whichever replica answers. with sequential_reads(): - rows = await (ch or get_client()).query_all(READ_SQL) - except Exception as e: # noqa: BLE001 — keep the last keyring on a CH glitch + if deadline is None: + rows = await (ch or get_client()).query_all(READ_SQL) + else: + rows = await _bounded_rows(ch or get_client(), deadline=deadline) + if max(len(_view.keys), len(_view.retired), len(_view.retired_public)) > 128: + raise ValueError("keyring history exceeds its allowance") + rows = [{**r, "retired": r["retired_at"] is not None} for r in rows] + except Exception as e: # Keep the last keyring when the ordinary read fails. + if deadline is not None: + raise ValueError("the complete keyring could not be read") from e log.warning("signing keyring read failed: %s — keeping %d keys", e, len(_view.keys)) return None view = _keep_retired(_parse(rows), _view) + if deadline is not None and len(set(view.keys) | view.retired | view.unsealable) > 128: + raise ValueError("keyring history exceeds its allowance") view = dataclasses.replace(view, read_at=time.monotonic(), loaded=True) for kid in sorted(view.unsealable - _warned): _warned.add(kid) @@ -578,25 +588,30 @@ def _is_this_root(recorded: str) -> bool: return hmac.compare_digest(recorded, _root_record()) -async def record_root(*, ch: _CHClient, force: bool = False) -> None: +async def record_root( + *, + ch: _CHClient, + force: bool = False, + deadline: float | None = None, +) -> None: """Write the `default` row for this root secret, when it has none (or always, with `force`: the operator replaced the root on purpose). A retired `default` stays retired: the row is written one version above the one read, so a retire written meanwhile stays the newer row.""" - row = await ch.query_one( - f"SELECT {', '.join(_COLUMNS)}, version FROM loom.signing_keys FINAL " - "WHERE kid = {kid:String}", - params={"kid": ROOT_KID}, - ) + if deadline is None: + row = await ch.query_one( + f"SELECT {', '.join(_COLUMNS)}, version FROM loom.signing_keys FINAL " + "WHERE kid = {kid:String}", + params={"kid": ROOT_KID}, + ) + else: + rows = await _bounded_rows(ch, deadline=deadline, kid=ROOT_KID) + row = rows[0] if rows else None if row is not None and row["sealed_key"] and not force: return version = int(row["version"]) + 1 if row else 1 row = row or _root_row_values() - await ch.insert( - "loom.signing_keys", - [{**row, "sealed_key": _root_record(), "version": version}], - settings={"async_insert": 0}, - ) + await _insert_key(ch, {**row, "sealed_key": _root_record(), "version": version}, deadline) def _root_row_values() -> dict[str, Any]: @@ -611,19 +626,24 @@ def _root_row_values() -> dict[str, Any]: } -async def _same_root(ch: _CHClient) -> _View: +async def _same_root(ch: _CHClient, *, deadline: float | None = None) -> _View: """Refuse a keyring change unless this process read the keyring and holds the root secret it was recorded under: a key sealed with another root does not open on the servers, and which keys sign here is not what they see. Returns the view read: the change judges it, not `_view`, which the refresh loop may replace meanwhile.""" - view = await _read(ch=ch) + view = await _read(ch=ch) if deadline is None else await _read(ch=ch, deadline=deadline) if view is None: raise ValueError("the keyring could not be read; nothing was changed") if view.root_row == "other": raise ValueError(_OTHER_ROOT) if view.root_row == "absent" and not settings.jwt_secret_is_dev_default(): - await record_root(ch=ch) + if deadline is None: + await record_root(ch=ch) + else: + if len(set(view.keys) | view.retired | view.unsealable) >= 128: + raise ValueError("keyring exceeds its allowance") + await record_root(ch=ch, deadline=deadline) return view @@ -742,15 +762,26 @@ def _new_key(algorithm: str) -> tuple[str, str]: return pem, public -async def rotate(algorithm: str, *, activate_in_s: float, ch: _CHClient) -> SigningKey: +async def rotate( + algorithm: str, + *, + activate_in_s: float, + ch: _CHClient, + deadline: float | None = None, +) -> SigningKey: """Write a new key that signs from `activate_in_s` from now on the ClickHouse server's clock: it must exceed two of the servers' keyring refresh intervals, so every process verifies the key before it signs.""" if algorithm not in ALGORITHMS: raise ValueError(f"algorithm must be one of {ALGORITHMS}") - await _same_root(ch) + if deadline is None: + await _same_root(ch) + else: + view = await _same_root(ch, deadline=deadline) + if len(set(view.keys) | view.retired | view.unsealable | {ROOT_KID}) >= 128: + raise ValueError("keyring exceeds its allowance") secret, public = _new_key(algorithm) - now = await _ch_now(ch) + now = await _ch_now(ch) if deadline is None else await _bounded_now(ch, deadline) key = SigningKey( f"k{now:%Y%m%d%H%M%S}-{secrets.token_hex(3)}", algorithm, @@ -758,11 +789,7 @@ async def rotate(algorithm: str, *, activate_in_s: float, ch: _CHClient) -> Sign public, now + timedelta(seconds=activate_in_s), ) - await ch.insert( - "loom.signing_keys", - [_row(key, created_at=now, retired_at=None)], - settings={"async_insert": 0}, - ) + await _insert_key(ch, _row(key, created_at=now, retired_at=None), deadline) return key @@ -795,15 +822,25 @@ async def retire(kid: str, *, ch: _CHClient) -> bool: raise ValueError(_LOST) from None -async def _retire(kid: str, *, ch: _CHClient, held: Callable[[], bool]) -> bool: - view = await _same_root(ch) - row = await ch.query_one( - f"SELECT {', '.join(_COLUMNS)} FROM loom.signing_keys FINAL WHERE kid = {{kid:String}}", - params={"kid": kid}, - ) +async def _retire( + kid: str, + *, + ch: _CHClient, + held: Callable[[], bool], + deadline: float | None = None, +) -> bool: + view = await _same_root(ch) if deadline is None else await _same_root(ch, deadline=deadline) + if deadline is None: + row = await ch.query_one( + f"SELECT {', '.join(_COLUMNS)} FROM loom.signing_keys FINAL WHERE kid = {{kid:String}}", + params={"kid": kid}, + ) + else: + rows = await _bounded_rows(ch, deadline=deadline, kid=kid) + row = rows[0] if rows else None if row is None and kid != ROOT_KID: return False - now = await _ch_now(ch) + now = await _ch_now(ch) if deadline is None else await _bounded_now(ch, deadline) signers = {k.kid for k in _keys(view).values() if k.activate_at <= now} if _root_signs(view.retired): signers.add(ROOT_KID) @@ -820,7 +857,10 @@ async def _retire(kid: str, *, ch: _CHClient, held: Callable[[], bool]) -> bool: if not held(): raise ValueError(_LAPSED) row = row or {**_root_row_values(), "sealed_key": _root_record()} - await ch.insert("loom.signing_keys", [{**row, "retired_at": now}], settings={"async_insert": 0}) + if deadline is not None: + # As in the ordinary path, the store supplies this retirement's new version. + row = {key: row[key] for key in _COLUMNS} + await _insert_key(ch, {**row, "retired_at": now}, deadline) return True @@ -831,6 +871,178 @@ async def listing(*, ch: _CHClient) -> list[dict[str, Any]]: ) +async def _bounded_rows( + ch: Any, + *, + deadline: float, + kid: str | None = None, +) -> list[dict[str, Any]]: + from server.core.ch_bounded import BoundedReadError, read_rows + + maximum = 128 if kid is None else 1 + where = "" if kid is None else " WHERE kid = {kid:String}" + rows = await read_rows( + "SELECT kid, algorithm, sealed_key, public_pem, version, " + "toUnixTimestamp64Milli(created_at) AS created_at, " + "toUnixTimestamp64Milli(activate_at) AS activate_at, " + "toUnixTimestamp64Milli(retired_at) AS retired_at, " + "toUnixTimestamp64Milli(now64(3)) AS server_now " + "FROM loom.signing_keys FINAL" + where + " LIMIT {limit:UInt32} FORMAT JSONEachRow", + {"kid": kid, "limit": maximum + 1} if kid is not None else {"limit": maximum + 1}, + ch=ch, + deadline=deadline, + row_bytes=16384, + total_bytes=1024**2, + max_rows=maximum, + tag="cell.each.keyring_read", + ) + seen: set[str] = set() + for row in rows: + if ( + set(row) != {*_COLUMNS, "version", "server_now"} + or any( + type(row[key]) is not str + for key in ("kid", "algorithm", "sealed_key", "public_pem") + ) + or row["kid"] in seen + or not row["kid"] + or len(row["kid"].encode()) > 256 + or row["algorithm"] not in ALGORITHMS + or type(row["version"]) is not int + or row["version"] < 0 + or kid is not None + and row["kid"] != kid + ): + raise BoundedReadError("Malformed keyring row") + seen.add(row["kid"]) + for key in ("created_at", "activate_at", "retired_at", "server_now"): + if row[key] is None and key == "retired_at": + continue + if type(row[key]) is not int: + raise BoundedReadError("Malformed keyring timestamp") + try: + row[key] = datetime(1970, 1, 1) + timedelta(milliseconds=row[key]) + except (ValueError, OverflowError) as error: + raise BoundedReadError("Malformed keyring timestamp") from error + return rows + + +async def _bounded_now(ch: Any, deadline: float) -> datetime: + from server.core.ch_bounded import BoundedReadError, read_rows + + rows = await read_rows( + "SELECT toUnixTimestamp64Milli(now64(3)) AS now LIMIT 2 FORMAT JSONEachRow", + {}, + ch=ch, + deadline=deadline, + row_bytes=1024, + total_bytes=1024, + max_rows=1, + tag="cell.each.keyring_clock", + ) + if len(rows) != 1 or set(rows[0]) != {"now"} or type(rows[0]["now"]) is not int: + raise BoundedReadError("Malformed keyring clock") + return datetime(1970, 1, 1) + timedelta(milliseconds=rows[0]["now"]) + + +async def _insert_key(ch: Any, row: dict[str, Any], deadline: float | None) -> None: + if deadline is None: + await ch.insert("loom.signing_keys", [row], settings={"async_insert": 0}) + return + import asyncio + + from server.core.ch import set_query_tag, statement_timeout + from server.core.ch_bounded import BoundedReadError + + remaining = min(2.0, deadline - time.monotonic()) + if remaining <= 0: + raise BoundedReadError("Keyring write allowance exhausted") + stored = {key: value for key, value in row.items() if key in {*_COLUMNS, "version"}} + set_query_tag("cell.each.keyring_write") + async with asyncio.timeout(remaining): + with statement_timeout(remaining): + await ch.insert( + "loom.signing_keys", + [stored], + settings={ + "async_insert": 0, + "max_memory_usage": 128 * 1024**2, + }, + ) + + +async def listing_bounded(*, ch: Any, deadline: float) -> list[dict[str, Any]]: + from server.core.ch import sequential_reads + + with sequential_reads(): + rows = await _bounded_rows(ch, deadline=deadline) + return [ + {key: row[key] for key in ("kid", "algorithm", "created_at", "activate_at", "retired_at")} + for row in sorted(rows, key=lambda row: (row["activate_at"], row["kid"])) + ] + + +async def rotate_bounded( + algorithm: str, + *, + activate_in_s: float, + ch: Any, + deadline: float, +) -> SigningKey: + import math + + from server.core.ch_bounded import BoundedReadError, read_rows + + if settings.jwt_secret_is_dev_default() or root_is_weak(): + raise BoundedReadError("A strong deployment root is required") + rows = await read_rows( + f"SELECT max(refresh_seconds) AS s FROM {READERS_TABLE} " + "WHERE seen_at > now64(3) - toIntervalSecond({age:UInt32}) " + "LIMIT 2 FORMAT JSONEachRow", + {"age": int(2 * _ANNOUNCE_S)}, + ch=ch, + deadline=deadline, + row_bytes=1024, + total_bytes=1024, + max_rows=1, + tag="cell.each.keyring_refresh_interval", + ) + if ( + len(rows) != 1 + or set(rows[0]) != {"s"} + or type(rows[0]["s"]) not in (int, float) + or not math.isfinite(rows[0]["s"]) + or rows[0]["s"] < 0 + ): + raise BoundedReadError("The serving refresh interval could not be verified") + refresh = rows[0]["s"] or settings.jwt_keyring_refresh_seconds + if not math.isfinite(activate_in_s) or activate_in_s < 2 * refresh: + raise BoundedReadError("Activation precedes the serving refresh allowance") + return await rotate(algorithm, activate_in_s=activate_in_s, ch=ch, deadline=deadline) + + +async def retire_bounded(kid: str, *, ch: Any, deadline: float) -> bool: + from server.core import leases + from server.core.ch import sequential_reads, statement_timeout + from server.core.ch_bounded import BoundedReadError + + async with leases.holding( + RETIRE_CLAIM, + ttl_s=60, + wait_s=0, + ch=ch, + deadline=deadline, + confirm_release=True, + ) as held: + if not held: + raise BoundedReadError("Another key retirement holds the claim") + with ( + sequential_reads(), + statement_timeout(min(2.0, max(0.001, deadline - time.monotonic()))), + ): + return await _retire(kid, ch=ch, held=lambda: True, deadline=deadline) + + def _row(key: SigningKey, *, created_at: datetime, retired_at: datetime | None) -> dict[str, Any]: return { "kid": key.kid, diff --git a/server/auth/oauth_store.py b/server/auth/oauth_store.py index 3eef77b12d..2be3450444 100644 --- a/server/auth/oauth_store.py +++ b/server/auth/oauth_store.py @@ -220,12 +220,45 @@ def _refresh_row(rid: str, g: Grant, version: int) -> dict[str, Any]: } -async def session_families(subject: str) -> dict[str, int]: +async def session_families( + subject: str, + *, + deadline: float | None = None, + ch: Any = None, +) -> dict[str, int]: """The sign-in sessions of `subject`, in any org, that hold a refresh token not yet expired, ended ones included: each family, and when its newest refresh token was written (unix seconds), which a refresh is judged by.""" from server.core.ch import get_client + if deadline is not None: + from server.core.ch_bounded import BoundedReadError, read_rows + + rows = await read_rows( + f"SELECT family, toUnixTimestamp(max(created_at)) AS at FROM {_T} FINAL " + "WHERE record_type = {t:String} AND subject = {s:String} AND family != '' " + "AND expires_at > toUInt32(now()) GROUP BY family LIMIT 1025 FORMAT JSONEachRow", + {"t": _REFRESH_TYPE, "s": subject}, + ch=ch or get_client(), + deadline=deadline, + row_bytes=512, + total_bytes=256 * 1024, + max_rows=1024, + tag="cell.each.subject_sessions", + ) + found: dict[str, int] = {} + for row in rows: + if ( + set(row) != {"family", "at"} + or type(row["family"]) is not str + or not row["family"] + or row["family"] in found + or type(row["at"]) is not int + or not 0 <= row["at"] <= 2**32 - 1 + ): + raise BoundedReadError("Malformed session-family view") + found[row["family"]] = row["at"] + return found rows = await get_client().query_all( f"SELECT family, toUnixTimestamp(max(created_at)) AS at FROM {_T} FINAL " "WHERE record_type = {t:String} AND subject = {s:String} AND family != '' " diff --git a/server/auth/revocation.py b/server/auth/revocation.py index 94468805b9..860d21a4d4 100644 --- a/server/auth/revocation.py +++ b/server/auth/revocation.py @@ -279,6 +279,9 @@ async def revoke( reason: str = "", revoked_by: str = "", issued_before: datetime | None = None, + deadline: float | None = None, + ch: Any = None, + progress: dict[str, Any] | None = None, ) -> dict[str, Any]: """Revoke every token whose `jti` OR `sub` equals `token_id` and update this process's snapshot so `is_revoked` answers True at once. @@ -306,6 +309,7 @@ async def revoke( revoked_by=revoked_by, issued_before=issued_before, reread=False, + **({} if deadline is None else {"deadline": deadline, "ch": ch, "progress": progress}), ) @@ -338,9 +342,14 @@ async def _write( revoked_by: str, issued_before: datetime | None, reread: bool, + deadline: float | None = None, + ch: Any = None, + progress: dict[str, Any] | None = None, ) -> dict[str, Any]: from server.core.ch import get_client + if deadline is not None: + check_bounded_bookkeeping(key) now = datetime.now(UTC).replace(microsecond=0, tzinfo=None) expires_at = revocation_expiry(expires_at) if issued_before is not None and issued_before.tzinfo is not None: @@ -352,28 +361,177 @@ async def _write( "reason": reason or "", "revoked_by": revoked_by or "", "issued_before": issued_before, - "version": await _next_version(key), + "version": ( + await _next_version(key) + if deadline is None + else await _bounded_version(key, ch=ch or get_client(), deadline=deadline) + ), } - await get_client().insert("loom.revoked_tokens", [row]) - remember_row(row) + if deadline is None: + await get_client().insert("loom.revoked_tokens", [row]) + else: + await _bounded_insert(row, ch=ch or get_client(), deadline=deadline) + if progress is not None: + progress["recorded"] = True + if deadline is None: + remember_row(row) + else: + remember_row(row, bounded=True) if reread: # a caller writing several re-reads once, after them await refresh() return row @asynccontextmanager -async def subject_change(subject: str) -> AsyncIterator[None]: +async def subject_change( + subject: str, + *, + deadline: float | None = None, + ch: Any = None, +) -> AsyncIterator[None]: """Serialize subject revocations across every writer and replica.""" from server.core import leases from server.core.errors import GrantBusyError name = f"subject-revocation:{uuid.uuid5(_LEASE_NAMESPACE, subject)}" - async with leases.holding(name, ttl_s=60.0, wait_s=10.0) as held: + bounded: dict[str, Any] = ( + {} + if deadline is None + else { + "deadline": deadline, + "confirm_release": True, + "ch": ch, + } + ) + async with leases.holding( + name, ttl_s=60.0, wait_s=10.0 if deadline is None else 0, **bounded + ) as held: if not held: raise GrantBusyError("another revocation of this subject is running; retry shortly") yield +async def _bounded_version(key: str, *, ch: Any, deadline: float) -> int: + from server.core.versions import next_version + + return await next_version(ch, "loom.revoked_tokens", {"jti": key}, deadline=deadline) + + +async def _bounded_insert(row: dict[str, Any], *, ch: Any, deadline: float) -> None: + from server.core.ch import set_query_tag, statement_timeout + from server.core.ch_bounded import BoundedReadError + + remaining = min(2.0, deadline - time.monotonic()) + if remaining <= 0: + raise BoundedReadError("Revocation write allowance exhausted") + check_bounded_bookkeeping(row["jti"]) + set_query_tag("cell.each.subject_write") + async with asyncio.timeout(remaining): + with statement_timeout(remaining): + await ch.insert( + "loom.revoked_tokens", + [row], + settings={ + "async_insert": 0, + "wait_for_async_insert": 1, + "max_memory_usage": 128 * 1024**2, + }, + ) + + +async def stored_bounded(key: str, *, deadline: float, ch: Any) -> dict[str, Any] | None: + from server.core.ch_bounded import BoundedReadError, read_rows + + rows = await read_rows( + "SELECT jti, reason, revoked_by, version, " + "toUnixTimestamp64Milli(revoked_at) AS revoked_at, " + "toUnixTimestamp64Milli(expires_at) AS expires_at, " + "toUnixTimestamp64Milli(issued_before) AS issued_before FROM (" + "SELECT jti, reason, revoked_by, version, revoked_at, expires_at, issued_before " + "FROM loom.revoked_tokens FINAL WHERE jti = {key:String} AND expires_at > now() " + "LIMIT 2) FORMAT JSONEachRow", + {"key": key}, + ch=ch, + deadline=deadline, + row_bytes=1024, + total_bytes=1024, + max_rows=1, + tag="cell.each.subject_record", + ) + if not rows: + return None + row = rows[0] + if ( + set(row) + != {"jti", "reason", "revoked_by", "version", "revoked_at", "expires_at", "issued_before"} + or row["jti"] != key + or type(row["reason"]) is not str + or type(row["revoked_by"]) is not str + or type(row["version"]) is not int + or not 0 <= row["version"] <= 2**64 - 1 + ): + raise BoundedReadError("Malformed subject record") + _bounded_times(row, ("revoked_at", "expires_at", "issued_before")) + return row + + +def _bounded_times(row: dict[str, Any], fields: tuple[str, ...]) -> None: + from server.core.ch_bounded import BoundedReadError + + for field in fields: + if row[field] is None and field == "issued_before": + continue + if type(row[field]) is not int: + raise BoundedReadError("Malformed revocation timestamp") + try: + row[field] = datetime(1970, 1, 1) + timedelta(milliseconds=row[field]) + except (ValueError, OverflowError) as error: + raise BoundedReadError("Malformed revocation timestamp") from error + + +async def cutoffs_bounded( + subjects: list[str], + *, + deadline: float, + ch: Any, + permanent: bool = False, +) -> dict[str, int | None]: + from server.core.ch_bounded import BoundedReadError, read_rows + + if len(subjects) > 100 or any(len(subject.encode()) > 512 for subject in subjects): + raise BoundedReadError("Revocation key set exceeds its allowance") + found: dict[str, int | None] = {} + for start in range(0, len(subjects), 5): + keys = _keys_of(subjects[start : start + 5]) + rows = await read_rows( + "SELECT jti, toUnixTimestamp64Milli(issued_before) AS issued_before, " + "toUnixTimestamp64Milli(expires_at) AS expires_at FROM (" + "SELECT jti, issued_before, expires_at FROM loom.revoked_tokens FINAL " + "WHERE jti IN {keys:Array(String)} AND expires_at > now() " + "LIMIT {limit:UInt32}) FORMAT JSONEachRow", + {"keys": keys, "limit": len(keys) + 1}, + ch=ch, + deadline=deadline, + row_bytes=1024, + total_bytes=65536, + max_rows=len(keys), + tag="cell.each.subject_cutoffs", + ) + seen: set[str] = set() + for row in rows: + if ( + set(row) != {"jti", "issued_before", "expires_at"} + or type(row["jti"]) is not str + or row["jti"] not in keys + or row["jti"] in seen + ): + raise BoundedReadError("Malformed revocation view") + seen.add(row["jti"]) + _bounded_times(row, ("issued_before", "expires_at")) + found.update(_cutoff_rows(rows, permanent=permanent)) + return found + + async def cut_off( subject: str, *, issued_before: datetime, reason: str, revoked_by: str ) -> dict[str, Any]: @@ -452,11 +610,13 @@ def revoked_here(jti: str) -> None: _apply_locally(jti, None) -def remember_row(row: dict[str, Any]) -> None: +def remember_row(row: dict[str, Any], *, bounded: bool = False) -> None: """Apply a committed or retained row without advancing the full snapshot's age.""" global _cut_off key = str(row["jti"]) + if bounded: + check_bounded_bookkeeping(key) cutoff = _cutoff_seconds(row["issued_before"]) version = int(row["version"]) if key.startswith(CUTOFF_PREFIX): @@ -467,6 +627,33 @@ def remember_row(row: dict[str, Any]) -> None: _apply_locally(key, cutoff, version) +def check_bounded_bookkeeping(key: str) -> None: + """Each bounds the existing copy-on-write cache work, not the durable table. + + Ordinary revocations retain their normal behavior for larger snapshots. + A failed recheck after the insert leaves the action unsettled. + """ + from server.core.ch_bounded import BoundedReadError + + if ( + any( + len(collection) > 4096 + for collection in ( + _revoked_jtis, + _revoked_folded, + _revoked_before, + _cut_off, + ) + ) + or len(_recent) >= 4096 + or len(_revoked_jtis) == 4096 + and key not in _revoked_jtis + or len(_revoked_folded) == 4096 + and key.lower() not in _revoked_folded + ): + raise BoundedReadError("Local revocation bookkeeping exceeds its allowance") + + def _apply_locally(key: str, cutoff: int | None, version: int | None = None) -> None: """Fold a revocation this process just wrote into its snapshot. The periodic re-read can fail, or land on a replica the insert has not @@ -519,10 +706,14 @@ async def cutoffs(subjects: list[str], *, permanent: bool = False) -> dict[str, """Each of `subjects` that is revoked, read from ClickHouse as `refresh` reads it: None when outright, else the cutoff (unix seconds) a token must be issued at or after to live. `permanent` excludes temporary suspensions.""" - found: dict[str, int | None] = {} if not subjects: - return found - for r in await _read_revocations(_keys_of(subjects)): + return {} + return _cutoff_rows(await _read_revocations(_keys_of(subjects)), permanent=permanent) + + +def _cutoff_rows(rows: list[dict[str, Any]], *, permanent: bool) -> dict[str, int | None]: + found: dict[str, int | None] = {} + for r in rows: if permanent and not never_lapses(r): continue key, cutoff = str(r["jti"]), _cutoff_seconds(r.get("issued_before")) diff --git a/server/auth/service_accounts.py b/server/auth/service_accounts.py index 0d975a878a..816401ada0 100644 --- a/server/auth/service_accounts.py +++ b/server/auth/service_accounts.py @@ -15,8 +15,8 @@ import asyncio import logging import time -from dataclasses import dataclass -from datetime import datetime +from dataclasses import dataclass, field +from datetime import datetime, timedelta from typing import Any from server.auth.fail_closed import note_unreadable @@ -260,7 +260,12 @@ async def _add_lineage( _MADE_BY = "(created_by IN {subs:Array(String)} OR hasAny(lineage, {subs:Array(String)}))" -async def revoke_created_by(subjects: list[str], *, revoked_by: str) -> set[str]: +async def revoke_created_by( + subjects: list[str], + *, + revoked_by: str, + work: _RevocationWork | None = None, +) -> set[str]: """Mark revoked each token any of `subjects` made, directly or through their tokens, whose token the revocations kill, so its name comes free; one made after a cutoff stays live. A retry finds only what an earlier @@ -270,7 +275,7 @@ async def revoke_created_by(subjects: list[str], *, revoked_by: str) -> set[str] from server.auth.delegation import MAX_CHAIN_DEPTH from server.core.ch import get_client, own_write_reads - ch = get_client() + ch = get_client() if work is None else work.ch seen = set(subjects) creators = sorted(seen) dead: set[str] = set() # the `sa:` subjects of the dead tokens found so far @@ -281,21 +286,31 @@ async def revoke_created_by(subjects: list[str], *, revoked_by: str) -> set[str] made: set[str] = set() for i in range(0, len(creators), _CREATORS_AT_ONCE): chunk = creators[i : i + _CREATORS_AT_ONCE] - with own_write_reads(): - rows = await ch.query_all( - "SELECT org, name, sa_subject, created_by, created_at, lineage, " - "revoked_at IS NULL AS live FROM loom.service_accounts FINAL " - f"WHERE {_MADE_BY} AND {_UNEXPIRED}", - params={"subs": chunk}, - ) + if work is None: + with own_write_reads(): + rows = await ch.query_all( + "SELECT org, name, sa_subject, created_by, created_at, lineage, " + "revoked_at IS NULL AS live FROM loom.service_accounts FINAL " + f"WHERE {_MADE_BY} AND {_UNEXPIRED}", + params={"subs": chunk}, + ) + else: + rows = await work.read(f"{_MADE_BY} AND {_UNEXPIRED}", {"subs": chunk}) # Read after the list, so it tells whether each token listed is dead. - revoked = await revocation.cutoffs(chunk, permanent=True) + revoked = ( + await revocation.cutoffs(chunk, permanent=True) + if work is None + else await work.cutoffs(chunk) + ) # Only those still dead: a row listed revoked is judged again, as the gate would. dead |= { str(r["sa_subject"]) for r in rows if not r["live"] and _is_dead(r, revoked, dead) } live = [r for r in rows if r["live"]] - dead |= await _revoke_all(live, chunk, revoked, dead, revoked_by) + if work is None: + dead |= await _revoke_all(live, chunk, revoked, dead, revoked_by) + else: + dead |= await _revoke_all(live, chunk, revoked, dead, revoked_by, work=work) # Revoked ones too: a retry still reaches what their tokens made. made |= {str(r["sa_subject"]) for r in rows} made -= seen @@ -334,6 +349,8 @@ async def _revoke_all( revoked: dict[str, int | None], dead: set[str], revoked_by: str, + *, + work: _RevocationWork | None = None, ) -> set[str]: """Revoke those of the rows `found` whose token is dead, each name claimed as a create claims it and read again under it: a revoked copy of a row @@ -341,41 +358,30 @@ async def _revoke_all( claim at a time, so two passes over the same names never wait on each other in a cycle, and each row that lands stays revoked for a retry. Returns the `sa:` subjects revoked.""" - from server.auth import revocation from server.auth.grant_claims import service_account_change - from server.core.ch import get_client, own_write_reads - from server.core.datetimes import naive_utc_now_seconds + from server.core.ch import get_client - ch = get_client() - gate = asyncio.Semaphore(_CLAIMS_AT_ONCE) + ch = get_client() if work is None else work.ch + gate = asyncio.Semaphore(_CLAIMS_AT_ONCE if work is None else 4) listed = {str(r["sa_subject"]) for r in found} async def revoke(org: str, name: str) -> str | None: - from server.auth.authz import identity_write - - async with gate, service_account_change(org, name): - with own_write_reads(): - row = await ch.query_one( - "SELECT * FROM loom.service_accounts FINAL " - "WHERE org = {org:String} AND name = {name:String} " - f"AND {_MADE_BY} AND revoked_at IS NULL AND {_UNEXPIRED}", - params={"org": org, "name": name, "subs": creators}, + async with gate: + bounded: dict[str, Any] = {} if work is None else {"deadline": work.deadline, "ch": ch} + if work is not None: + work.statements(6) # The existing finite claim's worst path, including release. + async with service_account_change(org, name, **bounded): + return await _revoke_named( + org, + name, + creators, + revoked, + dead, + revoked_by, + listed, + ch=ch, + work=work, ) - if row is None: - return None - row = dict(row) - # A token made since the list may postdate `revoked` (a reactivation's). - now_revoked = revoked - if row["sa_subject"] not in listed: - now_revoked = await revocation.cutoffs(sorted(_above(row)), permanent=True) - if not _is_dead(row, now_revoked, dead): - return None - revoked_copy = revoked_row( - row, revoked_at=naive_utc_now_seconds(), revoked_by=revoked_by - ) - with identity_write(): - await ch.insert("loom.service_accounts", [revoked_copy]) - return str(row["sa_subject"]) try: async with asyncio.TaskGroup() as tasks: @@ -385,6 +391,240 @@ async def revoke(org: str, name: str) -> str | None: return {subject for task in done if (subject := task.result()) is not None} +async def _revoke_named( + org: str, + name: str, + creators: list[str], + revoked: dict[str, int | None], + dead: set[str], + revoked_by: str, + listed: set[str], + *, + ch: Any, + work: _RevocationWork | None, +) -> str | None: + from server.auth import revocation + from server.auth.authz import identity_write + from server.core.ch import own_write_reads + from server.core.datetimes import naive_utc_now_seconds + + if work is None: + with own_write_reads(): + row = await ch.query_one( + "SELECT * FROM loom.service_accounts FINAL " + "WHERE org = {org:String} AND name = {name:String} " + f"AND {_MADE_BY} AND revoked_at IS NULL AND {_UNEXPIRED}", + params={"org": org, "name": name, "subs": creators}, + ) + else: + rows = await work.read( + "org = {org:String} AND name = {name:String} " + f"AND {_MADE_BY} AND revoked_at IS NULL AND {_UNEXPIRED}", + {"org": org, "name": name, "subs": creators}, + point=True, + ) + row = rows[0] if rows else None + if row is None: + return None + row = dict(row) + # A replacement under the name may postdate the discovery's revocation view. + now_revoked = revoked + if row["sa_subject"] not in listed: + now_revoked = ( + await revocation.cutoffs(sorted(_above(row)), permanent=True) + if work is None + else await work.cutoffs(sorted(_above(row))) + ) + if not _is_dead(row, now_revoked, dead): + return None + revoked_copy = revoked_row(row, revoked_at=naive_utc_now_seconds(), revoked_by=revoked_by) + with identity_write(): + if work is None: + await ch.insert("loom.service_accounts", [revoked_copy]) + else: + await work.insert(revoked_copy) + work.confirmed_dead.add(str(row["sa_subject"])) + return str(row["sa_subject"]) + + +_SA_COLUMNS = ( + "org", + "name", + "sa_subject", + "namespaces", + "roles", + "ttl_days", + "created_by", + "created_at", + "expires_at", + "revoked_at", + "revoked_by", + "version", + "kind", + "client", + "inherit_creator_grants", + "ns_roles", + "lineage", + "saves_to", +) + + +@dataclass +class _RevocationWork: + """One finite descendant pass; counters are neither a cache nor a replay record.""" + + ch: Any + deadline: float + roots: set[str] + fetched: int = 0 + used_statements: int = 0 + mutations: int = 0 + subjects: set[str] = field(default_factory=set) + confirmed_dead: set[str] = field(default_factory=set) + + def statements(self, count: int) -> None: + from server.core.ch_bounded import BoundedReadError + + if self.used_statements + count > 1024: + raise BoundedReadError("Descendant statements exceed their allowance") + self.used_statements += count + + async def read( + self, + predicate: str, + params: dict[str, Any], + *, + point: bool = False, + ) -> list[dict[str, Any]]: + from server.core.ch_bounded import BoundedReadError, read_rows + + maximum = min(1 if point else 256, 512 - self.fetched) + if maximum <= 0: + raise BoundedReadError("Descendant rows exceed their allowance") + self.statements(1) + dates = {"created_at", "expires_at", "revoked_at"} + projection = ", ".join( + f"toUnixTimestamp64Milli(toDateTime64({key}, 3)) AS {key}" if key in dates else key + for key in _SA_COLUMNS + ) + # Reserve before awaiting: four point readers share this same finite pass. + self.fetched += maximum + rows = await read_rows( + "SELECT " + + projection + + " FROM (SELECT " + + ", ".join(_SA_COLUMNS) + + " FROM loom.service_accounts FINAL WHERE " + + predicate + + " LIMIT {limit:UInt32}) FORMAT JSONEachRow", + {**params, "limit": maximum + 1}, + ch=self.ch, + deadline=self.deadline, + row_bytes=8191, + total_bytes=maximum * 8192, + max_rows=maximum, + tag="cell.each.subject_descendants", + ) + # Charging the full row allowance covers whitespace and newlines across every read. + self.fetched -= maximum - len(rows) + for row in rows: + self._row(row) + subject = row["sa_subject"] + if subject not in self.roots and subject not in self.subjects: + if len(self.subjects) == 256: + raise BoundedReadError("Descendant subjects exceed their allowance") + self.subjects.add(subject) + return rows + + def _row(self, row: dict[str, Any]) -> None: + from server.core.ch_bounded import BoundedReadError + + if set(row) != set(_SA_COLUMNS): + raise BoundedReadError("Malformed descendant row") + for key in ( + "org", + "name", + "sa_subject", + "created_by", + "revoked_by", + "kind", + "client", + "ns_roles", + "saves_to", + ): + if type(row[key]) is not str: + raise BoundedReadError("Malformed descendant identity") + for key in ("namespaces", "roles", "lineage"): + if type(row[key]) is not list or any(type(value) is not str for value in row[key]): + raise BoundedReadError("Malformed descendant collection") + if ( + type(row["version"]) is not int + or not 0 <= row["version"] < 2**64 - 1 + or type(row["ttl_days"]) is not int + or type(row["inherit_creator_grants"]) not in (int, bool) + or row["inherit_creator_grants"] not in (0, 1) + ): + raise BoundedReadError("Malformed descendant row") + for key in ("created_at", "expires_at", "revoked_at"): + if row[key] is None and key != "created_at": + continue + if type(row[key]) is not int: + raise BoundedReadError("Malformed descendant timestamp") + try: + row[key] = datetime(1970, 1, 1) + timedelta(milliseconds=row[key]) + except (ValueError, OverflowError) as error: + raise BoundedReadError("Malformed descendant timestamp") from error + row["live"] = row["revoked_at"] is None + + async def cutoffs(self, subjects: list[str]) -> dict[str, int | None]: + from server.auth import revocation + + self.statements((len(subjects) + 4) // 5) + return await revocation.cutoffs_bounded( + subjects, + permanent=True, + ch=self.ch, + deadline=self.deadline, + ) + + async def insert(self, row: dict[str, Any]) -> None: + from server.core.ch import set_query_tag, statement_timeout + from server.core.ch_bounded import BoundedReadError + + remaining = min(2.0, self.deadline - time.monotonic()) + if self.mutations >= 256 or remaining <= 0: + raise BoundedReadError("Descendant mutation allowance exhausted") + self.statements(1) + self.mutations += 1 + set_query_tag("cell.each.subject_descendant_write") + async with asyncio.timeout(remaining): + with statement_timeout(remaining): + await self.ch.insert( + "loom.service_accounts", + [row], + settings={ + "async_insert": 0, + "wait_for_async_insert": 1, + "max_memory_usage": 128 * 1024**2, + }, + ) + + +async def revoke_created_by_bounded( + subject: str, + *, + revoked_by: str, + ch: Any, + deadline: float, + progress: dict[str, int], +) -> set[str]: + work = _RevocationWork(ch=ch, deadline=deadline, roots={subject}) + try: + return await revoke_created_by([subject], revoked_by=revoked_by, work=work) + finally: + progress["tokens"] = len(work.confirmed_dead) + + __all__ = [ "SACreator", "hand_down_lineage", diff --git a/server/auth/subject_revocation.py b/server/auth/subject_revocation.py index db52f4d0c1..217664752e 100644 --- a/server/auth/subject_revocation.py +++ b/server/auth/subject_revocation.py @@ -13,12 +13,14 @@ (`service_accounts.revoke_created_by`). `record` writes the revocation (or finds one that already covers it) and -`finish` counts and marks what it ends and audits it: a failure in `record` -wrote nothing, one in `finish` left the revocation in force. +`finish` counts and marks what it ends and audits it. A bounded caller treats a +lost write or claim-release acknowledgement as unsettled, even during `record`. """ from __future__ import annotations +import asyncio +import time from dataclasses import dataclass from datetime import UTC, datetime from typing import Any @@ -64,11 +66,33 @@ async def audit_revocation( by: str, org: str, scim: bool = False, + deadline: float | None = None, **extra: Any, ) -> None: """The durable record of a revocation: identifiers only, never token material. A subject's revocation spans every org; `org` is where it is recorded ('-' for the deployment, an end user's own org for theirs).""" + if deadline is not None: + remaining = min(2.0, deadline - time.monotonic()) + if remaining <= 0: + return # This ordinary best-effort audit is not the feature's completion proof. + from server.core.ch import statement_timeout + + try: + async with asyncio.timeout(remaining): + with statement_timeout(remaining): + await audit_revocation( + row, + token_id=token_id, + kind=kind, + by=by, + org=org, + scim=scim, + **extra, + ) + except TimeoutError: + return + return await audit.emit( "token.revoked", org=org, @@ -104,33 +128,72 @@ class Revoked: async def record( - subject: str, *, by: str, reason: str = "", at: datetime | None = None + subject: str, + *, + by: str, + reason: str = "", + at: datetime | None = None, + deadline: float | None = None, + ch: Any = None, + progress: dict[str, Any] | None = None, ) -> tuple[dict[str, Any], bool]: """Revoke `subject` for good: outright (`at` None), on its own key, unless that key already revokes it outright for good; or up to the cutoff `at` (from `cutoff`), on the cutoff key beside it, unless that key already holds a later cutoff. The subject's own row is never touched by a cutoff, so an outright revocation stays, a time-boxed one until it lapses. Returns the row - in force for the request and whether it was there already. Raises, having - written nothing that lands, when the revocation cannot be written.""" + in force for the request and whether it was there already. A lost write or + release acknowledgement may leave the row in force.""" subject = subject.strip() # as `revocation.revoke` keys it + if deadline is not None and at is not None: + raise ValueError("A bounded subject action is outright only") if at is not None: return await revocation.record_cutoff( subject, issued_before=at, reason=reason, revoked_by=by ) - async with revocation.subject_change(subject): - current = await revocation.stored(subject) + bounded: dict[str, Any] = {} if deadline is None else {"deadline": deadline, "ch": ch} + async with revocation.subject_change(subject, **bounded): + current = ( + await revocation.stored(subject) + if deadline is None + else await revocation.stored_bounded(subject, **bounded) + ) if ( current is not None and current["issued_before"] is None and revocation.never_lapses(current) ): - revocation.remember_row(current) + if progress is not None: + progress["recorded"] = True + revocation.remember_row(current, **({} if deadline is None else {"bounded": True})) return current, True - return await revocation.revoke(subject, reason=reason, revoked_by=by), False + if deadline is None: + row = await revocation.revoke(subject, reason=reason, revoked_by=by) + else: + row = await revocation.revoke( + subject, + reason=reason, + revoked_by=by, + deadline=deadline, + ch=ch, + progress=progress, + ) + if progress is not None: + progress["recorded"] = True + return row, False -async def finish(subject: str, row: dict[str, Any], kept: bool, *, by: str, org: str) -> Revoked: +async def finish( + subject: str, + row: dict[str, Any], + kept: bool, + *, + by: str, + org: str, + deadline: float | None = None, + ch: Any = None, + progress: dict[str, int] | None = None, +) -> Revoked: """After `record`: count the sign-in sessions the subject's revocations end (a logout or a refusal ended some already; one refreshed since a cutoff lives on), mark revoked the tokens they end, and audit it. Raises, the @@ -142,30 +205,64 @@ async def finish(subject: str, row: dict[str, Any], kept: bool, *, by: str, org: subject = subject.strip() outright, in_force, sessions, tokens = False, None, 0, 0 try: - revoked = await revocation.cutoffs([subject]) # its own row, the cutoff's, SCIM's + revoked = ( + await revocation.cutoffs([subject]) + if deadline is None + else await revocation.cutoffs_bounded([subject], deadline=deadline, ch=ch) + ) await stop_connections([subject], respect_cutoff=True) if subject in revoked: at = revoked[subject] outright = at is None in_force = None if at is None else datetime.fromtimestamp(at, UTC) - families = await oauth_store.session_families(subject) + families = ( + await oauth_store.session_families(subject) + if deadline is None + else await oauth_store.session_families(subject, deadline=deadline, ch=ch) + ) sessions = sum( revocation.kills(at, newest) and not revocation.is_revoked(family) for family, newest in families.items() ) - tokens = len(await service_accounts.revoke_created_by([subject], revoked_by=by)) + if progress is not None: + progress["sessions"] = sessions + if deadline is None: + tokens = len(await service_accounts.revoke_created_by([subject], revoked_by=by)) + else: + tokens = len( + await service_accounts.revoke_created_by_bounded( + subject, + revoked_by=by, + deadline=deadline, + ch=ch, + progress=progress if progress is not None else {}, + ) + ) finally: # a revocation that landed is recorded, even if the marking did not finish end_user_org = org_of_end_user_revocation(subject) - await audit_revocation( - row, - token_id=subject, - kind="subject", - by=by, - org=end_user_org or org, - sessions=sessions, - tokens=tokens, - kept=kept, - ) + if deadline is None: + await audit_revocation( + row, + token_id=subject, + kind="subject", + by=by, + org=end_user_org or org, + sessions=sessions, + tokens=tokens, + kept=kept, + ) + else: + await audit_revocation( + row, + token_id=subject, + kind="subject", + by=by, + org=end_user_org or org, + sessions=sessions, + tokens=tokens, + kept=kept, + deadline=deadline, + ) return Revoked( row=row, kept=kept, outright=outright, cutoff=in_force, sessions=sessions, tokens=tokens ) diff --git a/server/cli/__init__.py b/server/cli/__init__.py index c031b61446..ed3e765386 100644 --- a/server/cli/__init__.py +++ b/server/cli/__init__.py @@ -50,6 +50,6 @@ imports `app`. """ -from server.cli.app import main +from server.cli._entrypoint import main __all__ = ["main"] diff --git a/server/cli/__main__.py b/server/cli/__main__.py index e518ab3bef..47360a59ae 100644 --- a/server/cli/__main__.py +++ b/server/cli/__main__.py @@ -2,6 +2,6 @@ import sys -from server.cli.app import main +from server.cli import main sys.exit(main()) diff --git a/server/cli/_entrypoint.py b/server/cli/_entrypoint.py new file mode 100644 index 0000000000..6bd49c1c36 --- /dev/null +++ b/server/cli/_entrypoint.py @@ -0,0 +1,16 @@ +"""Load server command modules only after selecting the client-only path.""" + +from __future__ import annotations + +import sys + + +def main(argv: list[str] | None = None) -> int: + raw = sys.argv[1:] if argv is None else argv + if raw[:2] == ["cell", "each"]: + from server.cli.cell_each_run import main as each_main + + return each_main(raw[2:]) + from server.cli.app import main as server_main + + return server_main(raw) diff --git a/server/cli/app.py b/server/cli/app.py index e82bab470f..a12742c384 100644 --- a/server/cli/app.py +++ b/server/cli/app.py @@ -373,6 +373,10 @@ def _add_sso_parsers(sub: Any) -> None: def main(argv: list[str] | None = None) -> int: raw = sys.argv[1:] if argv is None else argv + if raw[:2] == ["cell", "each"]: + from server.cli.cell_each_run import main as each_main + + return each_main(raw[2:]) if raw and raw[0] in SDK_COMMANDS: # Before the checkout's .env, which describes the server and would # otherwise override the user's own session (LOOM_API_KEY, LOOM_BASE_URL). diff --git a/server/cli/cell.py b/server/cli/cell.py index b586e12cad..185eb597a5 100644 --- a/server/cli/cell.py +++ b/server/cli/cell.py @@ -438,6 +438,12 @@ def _add_cell_parser(sub: Any) -> None: scim_new.add_argument("--cell", required=True, help="The cell's exact LOOM_CELL name") scim_new.set_defaults(func=_cmd_scim_token) + from server.cli.cell_automation import add_automation_parser + from server.cli.cell_each_run import add_each_parser + + add_each_parser(csub) + add_automation_parser(csub) + cred = csub.add_parser("credential", help="Make a router credential, or rotate a cell's.") cred.add_argument("action", nargs="?", choices=["new"], help="new: make one offline") cred.add_argument("--cell", help="The cell whose credential to rotate") diff --git a/server/cli/cell_automation.py b/server/cli/cell_automation.py new file mode 100644 index 0000000000..727722e6eb --- /dev/null +++ b/server/cli/cell_automation.py @@ -0,0 +1,101 @@ +"""Provision the two public bot purposes through a fenced operator mutation.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import sys +from pathlib import Path +from typing import Any + +import httpx + +from server.cli import cell_bot_proof, cell_control +from server.router import cell_admin + + +async def provision( + owner: cell_control.Owner, router: Any, args: argparse.Namespace, transport: Any = None +) -> dict[str, Any]: + """Prove both live keys while one native control owner holds the cell lock.""" + + async def call(path: str, body: dict[str, Any]) -> dict[str, Any]: + return await cell_control.request(router, args.cell, transport, path, body) + + row = await cell_control.request(router, args.cell, transport, "", method="GET") + if row.get("state") != "serving" or row.get("mode") != "enforced": + raise cell_control._refused("automation provisioning requires a serving, enforced cell") + row = await cell_control.claim(owner.conn, call, row) + automation = cell_bot_proof.BotKey( + args.automation_bot, args.automation_thumbprint, args.automation_key + ) + recovery = cell_bot_proof.BotKey(args.recovery_bot, args.recovery_thumbprint, args.recovery_key) + bots = await owner.call( + lambda: cell_bot_proof.prove_pair( + cell=row["cell"], + deployment_id=row["deployment_id"], + origin="https://" + row["host"], + automation=automation, + recovery=recovery, + ca_file=args.ca_file, + ), + total_s=35.0, + ) + public = cell_admin.public_pair(json.dumps(bots, ensure_ascii=True, separators=(",", ":"))) + answer = await cell_control.mutate(owner.conn, call, row, "/administration", {"bots": bots}) + expected = {"version": 1, "cell": row["cell"], "deployment_id": row["deployment_id"], **public} + if answer != expected: + raise cell_control._refused( + "automation metadata reply unknown: stop and read the directory" + ) + return answer + + +def command(args: argparse.Namespace) -> int: + from server.cli import cell + + router = cell._cell_router(args) + if isinstance(router, int): + return router + try: + answer = asyncio.run( + cell_control.run("automation", lambda owner: provision(owner, router, args)) + ) + except (cell_control.Refused, ValueError, OSError, TimeoutError, httpx.HTTPError) as error: + # Exception messages from key files and transports can contain private paths. + print( + f"loom cell automation: refused ({type(error).__name__}); " + "read the directory before retrying", + file=sys.stderr, + ) + return 1 + print(json.dumps(answer, indent=2, sort_keys=True)) + return 0 + + +def add_automation_parser(sub: Any) -> None: + parser = sub.add_parser( + "automation", help="Prove and register distinct automation and recovery bot keys." + ) + parser.add_argument("--cell", required=True, help="The serving cell to provision") + parser.add_argument("--router", help="The router's URL (default: LOOM_CELL_ROUTER_URL)") + for purpose in ("automation", "recovery"): + parser.add_argument( + f"--{purpose}-bot", required=True, help=f"The {purpose} bot's exact registered name" + ) + parser.add_argument( + f"--{purpose}-thumbprint", + required=True, + help=f"The {purpose} bot's approved public key thumbprint", + ) + parser.add_argument( + f"--{purpose}-key", + required=True, + type=Path, + help=f"The {purpose} bot's private key file", + ) + parser.add_argument( + "--ca-file", type=Path, help="Private file containing an explicit CA certificate bundle" + ) + parser.set_defaults(func=command) diff --git a/server/cli/cell_bot_proof.py b/server/cli/cell_bot_proof.py new file mode 100644 index 0000000000..299096cee5 --- /dev/null +++ b/server/cli/cell_bot_proof.py @@ -0,0 +1,201 @@ +"""Prove two explicit bot keys before an operator provisions their public purposes.""" + +from __future__ import annotations + +import asyncio +import re +import time +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from server.cli import cell_each_http as transport +from server.cli import cell_each_state as private_files + +if TYPE_CHECKING: + import httpx + + +@dataclass(frozen=True) +class BotKey: + name: str + thumbprint: str + key_file: Path + + +def _refuse() -> ValueError: + return ValueError("Bot provisioning proof refused") + + +def _text(value: str, maximum: int) -> bool: + if type(value) is not str: + return False + try: + size = len(value.encode("utf-8")) + except UnicodeError: + return False + return ( + bool(value) + and size <= maximum + and not any(ord(char) < 32 or ord(char) == 127 for char in value) + ) + + +def _inputs(cell: str, deployment_id: str, automation: BotKey, recovery: BotKey) -> None: + if not _text(cell, 32) or re.fullmatch(r"[a-z0-9-]{1,32}", cell) is None: + raise _refuse() + if not _text(deployment_id, 256): + raise _refuse() + for bot in (automation, recovery): + if ( + not _text(bot.name, 256) + or type(bot.thumbprint) is not str + or re.fullmatch(r"[A-Za-z0-9_-]{43}", bot.thumbprint) is None + ): + raise _refuse() + if automation.name == recovery.name or automation.thumbprint == recovery.thumbprint: + raise _refuse() + + +def _whoami(answer: dict[str, Any], name: str) -> str: + if answer.keys() != { + "authenticated", + "subject", + "org", + "namespaces", + "roles", + "is_service_account", + "kid", + "exp", + "iat", + }: + raise _refuse() + expires, issued = answer["exp"], answer["iat"] + if ( + answer["authenticated"] is not True + or answer["is_service_account"] is not True + or answer["subject"] != "sa:" + name + or answer["namespaces"] != ["*"] + or answer["roles"] != ["PLATFORM_ADMIN"] + or not _text(answer["org"], 256) + or not _text(answer["kid"], 256) + or type(expires) is not int + or type(issued) is not int + or not 0 < expires - issued <= 60 + or not 35 <= expires - time.time() <= 60 + ): + raise _refuse() + org: str = answer["org"] + return org + + +def _registration( + answer: dict[str, Any], cell: str, deployment_id: str, bot: BotKey, org: str +) -> None: + if answer.keys() != { + "version", + "cell", + "deployment_id", + "name", + "org", + "thumbprint", + "active", + "platform_admin", + } or not transport._version(answer["version"]): + raise _refuse() + expected = (cell, deployment_id, bot.name, org, bot.thumbprint) + actual = tuple(answer[key] for key in ("cell", "deployment_id", "name", "org", "thumbprint")) + if actual != expected or answer["active"] is not True or answer["platform_admin"] is not True: + raise _refuse() + + +async def _one( + client: httpx.AsyncClient, + origin: str, + cell: str, + deployment_id: str, + bot: BotKey, + deadline: float, +) -> None: + endpoint = origin + "/oauth/token" + pem = private_files.read_private(bot.key_file, 65536) + assertion = transport.bot_assertion( + pem, name=bot.name, thumbprint=bot.thumbprint, endpoint=endpoint + ) + issued = time.monotonic() + status, answer = await transport._request( + client, + "POST", + endpoint, + limit=16384, + deadline=deadline, + form={ + "grant_type": "client_credentials", + "client_id": bot.name, + "client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer", + "client_assertion": assertion, + "expires_in": "60", + }, + ) + if status != 200: + raise _refuse() + token, _expires = transport.bot_access(answer, bot.name, issued) + status, answer = await transport._request( + client, "GET", origin + "/v1/whoami", bearer=token, limit=2048, deadline=deadline + ) + if status != 200: + raise _refuse() + org = _whoami(answer, bot.name) + status, answer = await transport._request( + client, + "POST", + origin + "/v1/admin/bots.list?self_only=true", + bearer=token, + limit=2048, + deadline=deadline, + ) + if status != 200: + raise _refuse() + _registration(answer, cell, deployment_id, bot, org) + + +async def prove_pair( + *, + cell: str, + deployment_id: str, + origin: str, + automation: BotKey, + recovery: BotKey, + ca_file: Path | None = None, +) -> list[dict[str, str]]: + """Return public records after two live proofs; perform no metadata write or action.""" + import httpx + + _inputs(cell, deployment_id, automation, recovery) + origin = transport._origin(origin) + tls = transport._tls({"ca_file": ca_file} if ca_file is not None else {}) + deadline = time.monotonic() + 30.0 + async with asyncio.timeout_at(deadline): + async with httpx.AsyncClient( + verify=tls, follow_redirects=False, trust_env=False, timeout=15.0 + ) as client: + status, metadata = await transport._request( + client, + "GET", + origin + "/.well-known/oauth-authorization-server", + limit=16384, + deadline=deadline, + ) + if ( + status != 200 + or metadata.get("token_endpoint") != origin + "/oauth/token" + or (metadata.get("issuer", origin) != origin) + ): + raise _refuse() + # Both purposes require their own key proof, never a fallback for an action. + await _one(client, origin, cell, deployment_id, automation, deadline) + await _one(client, origin, cell, deployment_id, recovery, deadline) + return [ + {"purpose": purpose, "name": bot.name, "thumbprint": bot.thumbprint} + for purpose, bot in (("automation", automation), ("recovery", recovery)) + ] diff --git a/server/cli/cell_each.py b/server/cli/cell_each.py new file mode 100644 index 0000000000..03df8d2451 --- /dev/null +++ b/server/cli/cell_each.py @@ -0,0 +1,306 @@ +"""Closed typed intent for cell administration.""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +import uuid +from dataclasses import dataclass +from decimal import Decimal +from typing import Any + +MAX_MULTIPLIER_BYTES = 32 # not a setting: canonical multiplier width is part of the wire grammar + + +def _refuse() -> ValueError: + return ValueError("Unsupported cell administration command") + + +def _text(value: str, maximum: int, *, empty: bool = False, allow_controls: bool = False) -> str: + try: + size = len(value.encode("utf-8")) + except UnicodeError: + raise _refuse() from None + if ( + (not empty and not size) + or size > maximum + or "\0" in value + or (not allow_controls and any(ord(c) < 32 or ord(c) == 127 for c in value)) + ): + raise _refuse() + return value + + +def _options(tokens: list[str], allowed: set[str]) -> dict[str, str]: + if len(tokens) % 2: + raise _refuse() + result: dict[str, str] = {} + for index in range(0, len(tokens), 2): + key = tokens[index] + if key not in allowed or key in result: + raise _refuse() + result[key] = tokens[index + 1] + return result + + +def _integer(value: str, maximum: int, minimum: int = 0) -> int: + if not value.isascii() or not value.isdecimal() or len(value) > 16: + raise _refuse() + number = int(value) + if not minimum <= number <= maximum: + raise _refuse() + return number + + +def _number(value: str, *, positive: bool = False) -> float: + if not re.fullmatch(r"[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?", value): + raise _refuse() + number = float(value) + if not math.isfinite(number) or number < 0 or (positive and number == 0): + raise _refuse() + return number + + +def _multiplier(value: str) -> str: + if len(value) > MAX_MULTIPLIER_BYTES or not re.fullmatch( + r"[+]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]{1,3})?", value + ): + raise _refuse() + number = Decimal(value) + if ( + not number.is_finite() + or not 0 <= number <= 1000 + or (number and number.adjusted() < 3 - MAX_MULTIPLIER_BYTES) + ): + raise _refuse() + # For values below one, even one significant digit needs 3-adjusted bytes with x. + canonical = format(number, "f") if number else "0" + if "." in canonical: + canonical = canonical.rstrip("0").rstrip(".") + canonical = (canonical or "0") + "x" + if len(canonical) > MAX_MULTIPLIER_BYTES: + raise _refuse() + return canonical + + +@dataclass(frozen=True) +class Command: + """An immutable intent; callers receive a separate decoded arguments value.""" + + action: str + mutating: bool + _arguments_json: str + digest: str + + @property + def arguments(self) -> dict[str, Any]: + arguments: dict[str, Any] = json.loads(self._arguments_json) + return arguments + + @property + def path(self) -> str: + return "/v1/admin/cell." + self.action + + +def _command(action: str, arguments: dict[str, Any], *, mutating: bool = True) -> Command: + encoded = json.dumps(arguments, sort_keys=True, separators=(",", ":"), allow_nan=False) + if len(encoded.encode()) > 65536: + raise _refuse() + intent = json.dumps( + {"action": action, "arguments": arguments}, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) + return Command(action, mutating, encoded, hashlib.sha256(intent.encode()).hexdigest()) + + +def _settings(tokens: list[str]) -> Command: + if not tokens: + raise _refuse() + verb, rest = tokens[0], tokens[1:] + if verb == "list": + options = _options(rest, {"--prefix"}) + return _command( + "settings_list", + {"prefix": _text(options.get("--prefix", ""), 256, empty=True)}, + mutating=False, + ) + if verb == "history" and rest: + options = _options(rest[1:], {"--limit"}) + return _command( + "settings_history", + {"name": _text(rest[0], 256), "limit": _integer(options.get("--limit", "50"), 50, 1)}, + mutating=False, + ) + if verb in {"set", "reset"} and len(rest) >= (2 if verb == "set" else 1): + count = 2 if verb == "set" else 1 + options = _options(rest[count:], {"--expect"}) + return _command( + "settings_set", + { + "name": _text(rest[0], 256), + "value": _text(rest[1], 16384, empty=True, allow_controls=True) + if verb == "set" + else None, + "clear": verb == "reset", + "expected": _text(options["--expect"], 16384, empty=True, allow_controls=True) + if "--expect" in options + else None, + }, + ) + raise _refuse() + + +def _signing(tokens: list[str]) -> Command: + if tokens == ["list"]: + return _command("signing_keys_list", {}, mutating=False) + if len(tokens) == 2 and tokens[0] == "retire": + return _command("signing_keys_retire", {"kid": _text(tokens[1], 256)}) + if tokens and tokens[0] == "rotate": + options = _options(tokens[1:], {"--algorithm", "--activate-in"}) + algorithm = options.get("--algorithm", "HS256") + if algorithm not in {"HS256", "RS256"}: + raise _refuse() + return _command( + "signing_keys_rotate", + {"algorithm": algorithm, "activate_in": _number(options.get("--activate-in", "300"))}, + ) + raise _refuse() + + +def _revoke(tokens: list[str]) -> Command: + options = _options(tokens, {"--subject", "--reason"}) + if "--subject" not in options: + raise _refuse() + subject = _text(options["--subject"], 512) + if "/user:" in subject: + raise _refuse() + return _command( + "subject_revoke", + { + "subject": subject, + "reason": _text(options.get("--reason", ""), 1024, empty=True, allow_controls=True), + }, + ) + + +def _rate(tokens: list[str]) -> Command: + if not tokens or tokens[0] not in {"set", "reset"}: + raise _refuse() + clear = tokens[0] == "reset" + required = {"--namespace", "--op-class"} + if not clear: + required |= {"--capacity", "--rate"} + options = _options(tokens[1:], required) + if options.keys() != required: + raise _refuse() + namespace = _text(options["--namespace"], 256) + op_class = _text(options["--op-class"], 64) + if namespace == "*": + raise _refuse() + return _command( + "ratelimit_namespace_set", + { + "namespace": namespace, + "op_class": op_class, + "clear": clear, + "capacity": None if clear else _number(options["--capacity"], positive=True), + "rate": None if clear else _number(options["--rate"]), + }, + ) + + +def _budget(tokens: list[str]) -> Command: + if len(tokens) < 2 or tokens[0] not in {"set", "reset"}: + raise _refuse() + clear = tokens[0] == "reset" + allowed = {"--consumer"} if clear else {"--consumer", "--budget-ms", "--multiplier"} + options = _options(tokens[2:], allowed) + consumer = _text(options["--consumer"], 64) if "--consumer" in options else None + operation = _text(tokens[1], 128) + if operation == "*" and consumer is None: + raise _refuse() + if not clear and (("--budget-ms" in options) == ("--multiplier" in options)): + raise _refuse() + if "--multiplier" in options and consumer is None: + raise _refuse() + return _command( + "budget_set", + { + "op": operation, + "consumer": consumer, + "clear": clear, + "budget_ms": _integer(options["--budget-ms"], 86400000) + if "--budget-ms" in options + else None, + "budget": _multiplier(options["--multiplier"]) if "--multiplier" in options else None, + }, + ) + + +def _blacklist(tokens: list[str]) -> Command: + if len(tokens) < 2 or tokens[0] != "global": + raise _refuse() + verb, rest = tokens[1], tokens[2:] + if verb == "list": + options = _options(rest, {"--limit"}) + return _command( + "blacklist_global_list", + {"limit": _integer(options.get("--limit", "50"), 50, 1)}, + mutating=False, + ) + if verb == "remove" and len(rest) == 1: + try: + pattern_id = str(uuid.UUID(rest[0])) + except ValueError: + raise _refuse() from None + return _command("blacklist_global_remove", {"pattern_id": pattern_id}) + if verb == "add" and rest: + options = _options(rest[1:], {"--type", "--reason"}) + kind = options.get("--type", "substring") + pattern = _text(rest[0], 4096, allow_controls=True) + if kind not in {"substring", "regex"} or (kind == "regex" and len(pattern) > 256): + raise _refuse() + reason = _text(options.get("--reason", ""), 1024, empty=True, allow_controls=True) + return _command( + "blacklist_global_add", + { + "pattern_id": str(uuid.uuid4()), + "pattern": pattern, + "pattern_type": kind, + "reason": reason, + }, + ) + raise _refuse() + + +def parse_command(argv: list[str]) -> Command: + """Parse intent only; server capabilities and owning validation still authorize work.""" + if ( + not isinstance(argv, list) + or not argv + or len(argv) > 256 + or any(type(x) is not str for x in argv) + ): + raise _refuse() + try: + size = sum(len(token.encode("utf-8")) + 1 for token in argv) + except UnicodeError: + raise _refuse() from None + if size > 16384 or any("\0" in token for token in argv): + raise _refuse() + parsers = { + "settings": _settings, + "signing-key": _signing, + "revoke-token": _revoke, + "ratelimit": _rate, + "budget": _budget, + "blacklist": _blacklist, + } + parser = parsers.get(argv[0]) + if parser is None: + raise _refuse() + return parser(argv[1:]) diff --git a/server/cli/cell_each_http.py b/server/cli/cell_each_http.py new file mode 100644 index 0000000000..74c8daab4f --- /dev/null +++ b/server/cli/cell_each_http.py @@ -0,0 +1,960 @@ +"""Serial same-origin cell administration without ambient server credentials.""" + +from __future__ import annotations + +import asyncio +import base64 +import hashlib +import ipaddress +import json +import re +import ssl +import sys +import time +import uuid +from datetime import datetime +from pathlib import Path +from typing import TYPE_CHECKING, Any +from urllib.parse import quote, urlsplit + +from server.cli import cell_each_state as history + +if TYPE_CHECKING: + import httpx + + from server.cli.cell_each import Command + +_ACTION_LIMIT = 2 * 1024 * 1024 # not a setting: the server contract fixes the maximum result size +_LIMITS = {"action_timeout_s": 30, "max_body_bytes": 65536, "max_response_bytes": _ACTION_LIMIT} +_THUMBPRINT_MEMBERS = { + "EC": ("crv", "kty", "x", "y"), + "RSA": ("e", "kty", "n"), + "OKP": ("crv", "kty", "x"), +} +_REFUSAL_CODES = { + "automation_required", + "security_unavailable", + "short_token_required", + "cell_unavailable", + "cell_identity_changed", + "automation_unprovisioned", + "automation_identity_changed", + "automation_key_changed", + "automation_revoked", + "invalid_arguments", + "action_unavailable", +} + + +class Refused(ValueError): + """A locally diagnosed input, identity or preflight failure.""" + + +def _refuse() -> Refused: + return Refused("Cell administration refused") + + +def _encoded(value: Any) -> bytes: + try: + return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() + except (ValueError, TypeError, UnicodeError, RecursionError): + raise _refuse() from None + + +def _object(raw: bytes) -> dict[str, Any]: + try: + value = json.loads( + raw.decode("utf-8"), + object_pairs_hook=history._pairs, + parse_constant=history._constant, + parse_float=history._float, + ) + except (ValueError, UnicodeError, RecursionError): + raise _refuse() from None + if type(value) is not dict: + raise _refuse() + pending: list[Any] = [value] + while pending: + item = pending.pop() + if type(item) is str: + try: + item.encode("utf-8") + except UnicodeError: + raise _refuse() from None + elif type(item) is list: + pending.extend(item) + elif type(item) is dict: + pending.extend(item.keys()) + pending.extend(item.values()) + return value + + +def _version(value: Any) -> bool: + return type(value) is int and value == 1 + + +def _origin(value: str) -> str: + if type(value) is not str or len(value) > 512 or not value.isascii(): + raise _refuse() + try: + parsed = urlsplit(value) + port = parsed.port + except ValueError: + raise _refuse() from None + if ( + parsed.scheme != "https" + or not parsed.hostname + or parsed.username is not None + or parsed.password is not None + or parsed.path + or parsed.query + or parsed.fragment + or value != f"https://{parsed.netloc}" + or parsed.netloc != parsed.netloc.lower() + or parsed.netloc.endswith(":") + or port == 443 + ): + raise _refuse() + host = parsed.hostname + try: + ipaddress.ip_address(host) + except ValueError: + if len(host) > 253 or any( + re.fullmatch(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?", part) is None + for part in host.split(".") + ): + raise _refuse() from None + return value + + +def _private_path(value: Any) -> Path: + if type(value) is not str or not value or len(value.encode()) > 4096 or "\0" in value: + raise _refuse() + path = Path(value) + if not path.is_absolute() or ".." in path.parts: + raise _refuse() + return path + + +def _manifest(path: Path) -> dict[str, Any]: + value = _object(history.read_private(path, 1024 * 1024)) + if value.keys() not in ( + {"version", "router_token_file", "keys"}, + {"version", "router_token_file", "keys", "ca_file"}, + ): + raise _refuse() + if ( + not _version(value["version"]) + or type(value["keys"]) is not list + or not 1 <= len(value["keys"]) <= 4096 + ): + raise _refuse() + value["router_token_file"] = _private_path(value["router_token_file"]) + if "ca_file" in value: + value["ca_file"] = _private_path(value["ca_file"]) + keys: dict[tuple[str, str], Path] = {} + for row in value["keys"]: + if type(row) is not dict or row.keys() != {"cell", "deployment_id", "key_file"}: + raise _refuse() + if not history._text(row["cell"], 32) or not history._text(row["deployment_id"], 256): + raise _refuse() + identity = (row["cell"], row["deployment_id"]) + if identity in keys: + raise _refuse() + keys[identity] = _private_path(row["key_file"]) + value["keys"] = keys + return value + + +def _router_token(manifest: dict[str, Any]) -> str: + try: + token = history.read_private(manifest["router_token_file"], 4096).decode("ascii").strip() + except UnicodeError: + raise _refuse() from None + if re.fullmatch(r"lco_[0-9a-f]{128}", token) is None: + raise _refuse() + return token + + +def _tls(manifest: dict[str, Any]) -> ssl.SSLContext: + if "ca_file" not in manifest: + return ssl.create_default_context() + try: + return ssl.create_default_context( + cadata=history.read_private(manifest["ca_file"], 1024 * 1024).decode("ascii") + ) + except (ValueError, UnicodeError, ssl.SSLError): + raise _refuse() from None + + +async def _request( + client: httpx.AsyncClient, + method: str, + url: str, + *, + limit: int, + deadline: float, + bearer: str = "", + body: dict[str, Any] | None = None, + form: dict[str, str] | None = None, + action: bool = False, +) -> tuple[int, dict[str, Any]]: + remaining = min(35.0 if action else 15.0, deadline - time.monotonic()) + if remaining <= 0: + raise _refuse() + headers = {"Accept": "application/json", "Accept-Encoding": "identity"} + # This serial client never carries one origin's cookies to another request. + client.cookies.clear() + if bearer: + headers["Authorization"] = "Bearer " + bearer + arguments: dict[str, Any] = {"headers": headers, "timeout": remaining} + if body is not None: + encoded = _encoded(body) + if len(encoded) > 65536: + raise _refuse() + arguments["content"] = encoded + headers["Content-Type"] = "application/json" + if form is not None: + arguments["data"] = form + async with asyncio.timeout(remaining): + # A local deadline stops admission, not a remote handler already given work. + async with client.stream(method, url, **arguments) as response: + if response.headers.get("content-encoding", "identity").lower() != "identity": + raise _refuse() + length = response.headers.get("content-length") + if length is not None and ( + not length.isascii() + or not length.isdecimal() + or len(length) > 12 + or int(length) > limit + ): + raise _refuse() + raw = bytearray() + async for chunk in response.aiter_raw(chunk_size=16384): + if len(raw) + len(chunk) > limit: + raise _refuse() + raw.extend(chunk) + return response.status_code, _object(bytes(raw)) + + +def _target(value: dict[str, Any]) -> dict[str, Any]: + history._target(value) + if re.fullmatch(r"[a-z0-9-]{1,32}", value["cell"]) is None: + raise _refuse() + _origin(value["origin"]) + return value + + +async def _census( + client: httpx.AsyncClient, router: str, token: str, deadline: float +) -> list[dict[str, Any]]: + status, answer = await _request( + client, + "GET", + router + "/cells/v1/operator/cells", + bearer=token, + limit=4 * 1024 * 1024, + deadline=deadline, + ) + if status != 200 or answer.keys() != {"version", "cells"} or not _version(answer["version"]): + raise _refuse() + rows = answer["cells"] + if type(rows) is not list or not 1 <= len(rows) <= 4096: + raise _refuse() + cells: set[str] = set() + deployments: set[str] = set() + origins: set[str] = set() + for row in rows: + _target(row) + if row["cell"] in cells or row["deployment_id"] in deployments or row["origin"] in origins: + raise _refuse() + cells.add(row["cell"]) + deployments.add(row["deployment_id"]) + origins.add(row["origin"]) + return sorted(rows, key=lambda row: row["cell"]) + + +async def _point( + client: httpx.AsyncClient, router: str, token: str, target: dict[str, Any], deadline: float +) -> None: + endpoint = ( + router + "/cells/v1/operator/cells/" + quote(target["cell"], safe="") + "/administration" + ) + status, answer = await _request( + client, "GET", endpoint, bearer=token, limit=256 * 1024, deadline=deadline + ) + if ( + status != 200 + or answer.keys() != {"version", "cell"} + or not _version(answer["version"]) + or _target(answer["cell"]) != target + ): + raise _refuse() + + +def _assertion(pem: bytes, target: dict[str, Any], endpoint: str) -> str: + automation, recovery = target["automation"], target["recovery"] + if automation["thumbprint"] == recovery["thumbprint"] or automation["name"] == recovery["name"]: + raise _refuse() + return bot_assertion( + pem, name=automation["name"], thumbprint=automation["thumbprint"], endpoint=endpoint + ) + + +def bot_assertion(pem: bytes, *, name: str, thumbprint: str, endpoint: str) -> str: + """Sign for one explicitly selected bot after checking its public key identity.""" + import jwt + from cryptography.exceptions import UnsupportedAlgorithm + from cryptography.hazmat.primitives.asymmetric import ec, ed25519, rsa + from cryptography.hazmat.primitives.serialization import load_pem_private_key + from jwt.algorithms import ECAlgorithm, OKPAlgorithm, RSAAlgorithm + + if ( + not history._text(name, 256) + or type(thumbprint) is not str + or re.fullmatch(r"[A-Za-z0-9_-]{43}", thumbprint) is None + or type(endpoint) is not str + or not endpoint.endswith("/oauth/token") + ): + raise _refuse() + _origin(endpoint[: -len("/oauth/token")]) + try: + key = load_pem_private_key(pem, password=None) + except (ValueError, TypeError, UnsupportedAlgorithm): + raise _refuse() from None + if isinstance(key, ec.EllipticCurvePrivateKey) and key.curve.name == "secp256r1": + algorithm, jwk = "ES256", ECAlgorithm.to_jwk(key.public_key(), as_dict=True) + elif isinstance(key, ed25519.Ed25519PrivateKey): + algorithm, jwk = "EdDSA", OKPAlgorithm.to_jwk(key.public_key(), as_dict=True) + elif isinstance(key, rsa.RSAPrivateKey) and key.key_size >= 2048: + algorithm, jwk = "RS256", RSAAlgorithm.to_jwk(key.public_key(), as_dict=True) + else: + raise _refuse() + members = {name: jwk[name] for name in _THUMBPRINT_MEMBERS[jwk["kty"]]} + actual_thumbprint = ( + base64.urlsafe_b64encode(hashlib.sha256(_encoded(members)).digest()).rstrip(b"=").decode() + ) + if actual_thumbprint != thumbprint: + raise _refuse() + now = int(time.time()) + claims = { + "iss": name, + "sub": name, + "aud": endpoint, + "iat": now, + "exp": now + 120, + "jti": uuid.uuid4().hex, + } + return jwt.encode(claims, key, algorithm=algorithm, headers={"kid": thumbprint}) + + +def _access(answer: dict[str, Any], target: dict[str, Any], issued: float) -> tuple[str, float]: + return bot_access(answer, target["automation"]["name"], issued) + + +def bot_access(answer: dict[str, Any], name: str, issued: float) -> tuple[str, float]: + """Bound the normal token response for an explicitly selected bot.""" + import jwt + + if ( + not history._text(name, 256) + or not {"access_token", "token_type", "expires_in"} <= answer.keys() + ): + raise _refuse() + token, lifetime = answer["access_token"], answer["expires_in"] + if ( + type(token) is not str + or not 1 <= len(token) <= 16384 + or any(ord(char) < 33 or ord(char) > 126 for char in token) + or answer["token_type"] != "Bearer" + or type(lifetime) is not int + or not 30 <= lifetime <= 60 + ): + raise _refuse() + try: + claims = jwt.decode(token, options={"verify_signature": False, "verify_exp": False}) + except jwt.PyJWTError: + raise _refuse() from None + # TLS and the server's normal verification supply authority, never this unverified view. + expires, starts = claims.get("exp"), claims.get("iat") + if ( + type(expires) is not int + or type(starts) is not int + or not 0 < expires - starts <= 60 + or not 30 <= expires - time.time() <= 60 + or claims.get("sub") != "sa:" + name + ): + raise _refuse() + end = min(issued + lifetime, time.monotonic() + expires - time.time()) + if end - time.monotonic() < 30: + raise _refuse() + return token, end + + +async def _token( + client: httpx.AsyncClient, target: dict[str, Any], key_file: Path, deadline: float +) -> tuple[str, float]: + origin = target["origin"] + status, metadata = await _request( + client, + "GET", + origin + "/.well-known/oauth-authorization-server", + limit=16384, + deadline=deadline, + ) + endpoint = origin + "/oauth/token" + if ( + status != 200 + or metadata.get("token_endpoint") != endpoint + or metadata.get("issuer", origin) != origin + ): + raise _refuse() + pem = history.read_private(key_file, 65536) + assertion = _assertion(pem, target, endpoint) + issued = time.monotonic() + status, answer = await _request( + client, + "POST", + endpoint, + limit=16384, + deadline=deadline, + form={ + "grant_type": "client_credentials", + "client_id": target["automation"]["name"], + "client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer", + "client_assertion": assertion, + "expires_in": "60", + }, + ) + if status != 200: + raise _refuse() + return _access(answer, target, issued) + + +def _context(target: dict[str, Any], state: dict[str, Any] | None = None) -> dict[str, Any]: + value = { + "version": 1, + "expected_cell": target["cell"], + "expected_deployment_id": target["deployment_id"], + "expected_bot": target["automation"]["name"], + } + if state is not None: + value.update( + run_id=state["run_id"], + attempt=state["outcomes"][target["cell"]]["attempt"], + command_digest=state["command"]["digest"], + ) + return value + + +def _same_context(actual: Any, expected: dict[str, Any]) -> bool: + return ( + type(actual) is dict + and actual.keys() == expected.keys() + and all( + type(actual[key]) is type(value) and actual[key] == value + for key, value in expected.items() + ) + ) + + +async def _capability( + client: httpx.AsyncClient, target: dict[str, Any], token: str, action: str, deadline: float +) -> None: + context = _context(target) + status, answer = await _request( + client, + "POST", + target["origin"] + "/v1/admin/cell.capabilities", + bearer=token, + body={"context": context}, + limit=8192, + deadline=deadline, + ) + if ( + status != 200 + or answer.keys() != {"version", "context", "actions", "limits"} + or not _version(answer["version"]) + or not _same_context(answer["context"], context) + or answer["limits"] != _LIMITS + ): + raise _refuse() + actions = answer["actions"] + if ( + type(actions) is not list + or len(actions) > 12 + or any(type(item) is not str or item not in history._ACTIONS for item in actions) + or len(set(actions)) != len(actions) + or action not in actions + ): + raise _refuse() + + +def _completion( + answer: dict[str, Any], target: dict[str, Any], state: dict[str, Any] +) -> dict[str, Any]: + required = { + "version", + "status", + "context", + "action", + "started_audit", + "completed_audit", + "result", + } + if ( + answer.keys() != required + or not _version(answer["version"]) + or answer["status"] != "completed" + or not _same_context(answer["context"], _context(target, state)) + or answer["action"] != state["command"]["action"] + ): + raise _refuse() + for key in ("started_audit", "completed_audit"): + if type(answer[key]) is not str or re.fullmatch(r"[0-9a-f]{64}", answer[key]) is None: + raise _refuse() + _result(answer["action"], answer["result"], state["command"]["arguments"]) + value = { + key: answer[key] + for key in ("version", "context", "action", "started_audit", "completed_audit") + } + value["result_digest"] = hashlib.sha256(_encoded(answer["result"])).hexdigest() + if len(_encoded(value)) > 2048: + raise _refuse() + return value + + +def _setting(row: Any) -> None: + keys = {"name", "env", "value", "default", "source", "live", "description"} + if type(row) is not dict or row.keys() != keys or len(_encoded(row)) > 16384: + raise _refuse() + if ( + not history._text(row["name"], 256) + or not history._text(row["env"], 512) + or type(row["description"]) is not str + or row["live"] is not True + or type(row["source"]) is not str + or row["source"] + not in {"override", "dashboard", "env", "env_file", "settings_file", "inferred", "default"} + ): + raise _refuse() + + +def _settings_result(action: str, value: dict[str, Any], arguments: dict[str, Any]) -> None: + if action == "settings_set": + if value.keys() != {"ok", "setting"} or value["ok"] is not True: + raise _refuse() + _setting(value["setting"]) + if value["setting"]["name"] != arguments["name"]: + raise _refuse() + return + field = "settings" if action == "settings_list" else "changes" + maximum = 4096 if field == "settings" else arguments["limit"] + if value.keys() != {field} or type(value[field]) is not list or len(value[field]) > maximum: + raise _refuse() + if field == "changes" and len(_encoded(value)) > 512 * 1024: + raise _refuse() + names: set[str] = set() + for row in value[field]: + if field == "settings": + _setting(row) + if not row["name"].startswith(arguments["prefix"]) or row["name"] in names: + raise _refuse() + names.add(row["name"]) + else: + keys = {"ts", "actor", "name", "value", "previous", "previous_source", "cleared"} + if ( + type(row) is not dict + or row.keys() != keys + or len(_encoded(row)) > 16384 + or row["name"] != arguments["name"] + or not history._text(row["actor"], 512) + or not history._text(row["ts"], 64) + or type(row["cleared"]) is not bool + or ( + row["previous_source"] is not None + and not history._text(row["previous_source"], 32) + ) + ): + raise _refuse() + + +def _rate_result(value: dict[str, Any], arguments: dict[str, Any]) -> None: + keys = {"ok", "scope", "key", "op_class", "capacity", "rate", "cleared"} + if ( + value.keys() != keys + or value["ok"] is not True + or value["scope"] != "namespace" + or value["key"] != arguments["namespace"] + or value["op_class"] != arguments["op_class"] + ): + raise _refuse() + if arguments["clear"]: + if ( + value["cleared"] is not True + or value["capacity"] is not None + or value["rate"] is not None + ): + raise _refuse() + elif value["cleared"] is not None or any( + type(value[key]) not in {int, float} or value[key] != arguments[key] + for key in ("capacity", "rate") + ): + raise _refuse() + + +def _budget_result(value: dict[str, Any], arguments: dict[str, Any]) -> None: + keys = {"ok", "op", "budget_ms", "cleared", "consumer", "budget"} + if ( + value.keys() != keys + or value["ok"] is not True + or value["op"] != arguments["op"] + or value["consumer"] != arguments["consumer"] + ): + raise _refuse() + if arguments["clear"]: + if ( + value["cleared"] is not True + or value["budget_ms"] is not None + or value["budget"] is not None + ): + raise _refuse() + return + expected = arguments["budget"] if arguments["budget"] is not None else arguments["budget_ms"] + if ( + value["cleared"] is not None + or value["budget_ms"] != arguments["budget_ms"] + or type(value["budget_ms"]) is not type(arguments["budget_ms"]) + or value["budget"] != (expected if arguments["consumer"] is not None else None) + or type(value["budget"]) + is not type(expected if arguments["consumer"] is not None else None) + ): + raise _refuse() + + +def _timestamp(value: Any) -> None: + if not history._text(value, 64) or "T" not in value: + raise _refuse() + try: + datetime.fromisoformat(value) + except ValueError: + raise _refuse() from None + + +def _signing_result(action: str, value: dict[str, Any], arguments: dict[str, Any]) -> None: + if action == "signing_keys_retire": + if ( + value.keys() != {"kid", "retired"} + or value["kid"] != arguments["kid"] + or type(value["retired"]) is not bool + ): + raise _refuse() + return + if action == "signing_keys_rotate": + if ( + value.keys() != {"kid", "algorithm", "activate_at"} + or not history._text(value["kid"], 256) + or value["algorithm"] != arguments["algorithm"] + ): + raise _refuse() + _timestamp(value["activate_at"]) + return + if value.keys() != {"keys"} or type(value["keys"]) is not list or len(value["keys"]) > 128: + raise _refuse() + kids: set[str] = set() + for row in value["keys"]: + if ( + type(row) is not dict + or row.keys() != {"kid", "algorithm", "created_at", "activate_at", "retired_at"} + or not history._text(row["kid"], 256) + or row["kid"] in kids + or type(row["algorithm"]) is not str + or row["algorithm"] not in {"HS256", "RS256"} + or len(_encoded(row)) > 16384 + ): + raise _refuse() + kids.add(row["kid"]) + _timestamp(row["created_at"]) + _timestamp(row["activate_at"]) + if row["retired_at"] is not None: + _timestamp(row["retired_at"]) + + +def _policy_text(value: Any, maximum: int, *, empty: bool = False) -> bool: + return type(value) is str and (empty or bool(value)) and len(value.encode("utf-8")) <= maximum + + +def _blacklist_result(action: str, value: dict[str, Any], arguments: dict[str, Any]) -> None: + if action == "blacklist_global_add": + if ( + value.keys() != {"pattern_id", "created_at"} + or value["pattern_id"] != arguments["pattern_id"] + ): + raise _refuse() + _timestamp(value["created_at"]) + return + if ( + value.keys() != {"rows", "truncated"} + or type(value["rows"]) is not list + or len(value["rows"]) > arguments["limit"] + or type(value["truncated"]) is not bool + ): + raise _refuse() + identities: set[str] = set() + for row in value["rows"]: + if ( + type(row) is not dict + or row.keys() + != { + "pattern_id", + "pattern", + "pattern_type", + "reason", + "created_by", + "hit_count", + "last_hit_at", + "created_at", + } + or not history._uuid(row["pattern_id"]) + or row["pattern_id"] in identities + or not _policy_text(row["pattern"], 32768) + or type(row["pattern_type"]) is not str + or row["pattern_type"] not in {"substring", "regex"} + or not _policy_text(row["reason"], 32768, empty=True) + or not _policy_text(row["created_by"], 32768, empty=True) + or type(row["hit_count"]) is not int + or not 0 <= row["hit_count"] <= 2**64 - 1 + ): + raise _refuse() + identities.add(row["pattern_id"]) + _timestamp(row["created_at"]) + if row["last_hit_at"] is not None: + _timestamp(row["last_hit_at"]) + + +def _subject_result(value: dict[str, Any], arguments: dict[str, Any]) -> None: + if ( + value.keys() + != {"subject", "recorded", "already_revoked", "outright", "cutoff", "sessions", "tokens"} + or value["subject"] != arguments["subject"].strip() + or value["recorded"] is not True + or type(value["already_revoked"]) is not bool + or value["outright"] is not True + or value["cutoff"] is not None + or type(value["sessions"]) is not int + or value["sessions"] < 0 + or type(value["tokens"]) is not int + or value["tokens"] < 0 + ): + raise _refuse() + + +def _result(action: str, value: Any, arguments: dict[str, Any]) -> None: + if type(value) is not dict or len(_encoded(value)) > _ACTION_LIMIT: + raise _refuse() + if action in {"settings_list", "settings_set", "settings_history"}: + _settings_result(action, value, arguments) + elif action == "ratelimit_namespace_set": + _rate_result(value, arguments) + elif action == "budget_set": + _budget_result(value, arguments) + elif action in {"signing_keys_list", "signing_keys_rotate", "signing_keys_retire"}: + _signing_result(action, value, arguments) + elif action in {"blacklist_global_add", "blacklist_global_list"}: + _blacklist_result(action, value, arguments) + elif action == "subject_revoke": + _subject_result(value, arguments) + else: + # An unimplemented adapter cannot become completed through an arbitrary result. + raise _refuse() + + +def _no_action(answer: dict[str, Any], target: dict[str, Any], state: dict[str, Any]) -> str | None: + required = {"version", "status", "phase", "context", "action", "code"} + if ( + answer.keys() == required + and _version(answer["version"]) + and answer["status"] == "refused" + and _same_context(answer["context"], _context(target, state)) + and answer["action"] == state["command"]["action"] + and answer["phase"] == "before_business" + and type(answer["code"]) is str + and answer["code"] in _REFUSAL_CODES + ): + return answer["code"].upper() + return None + + +def _adopt( + state: dict[str, Any] | None, + command: Command | None, + targets: list[dict[str, Any]], + router: str, + path: Path, +) -> dict[str, Any]: + if state is None: + if command is None: + raise _refuse() + state = history.new_state(command, targets=targets, run_id=str(uuid.uuid4())) + state["router"] = router + history.write_state(path, state, exclusive=True) + return state + if history.has_uncertainty(state) or state["router"] != router or command is not None: + raise _refuse() + current = {target["cell"]: target for target in targets} + if any(current.get(target["cell"]) != target for target in state["targets"]): + raise _refuse() + old = {target["cell"] for target in state["targets"]} + state["targets"] = targets + for target in targets: + if target["cell"] not in old: + state["outcomes"][target["cell"]] = { + "status": "pending", + "attempt": 0, + "code": "", + "completion": None, + } + history.write_state(path, state) + return state + + +def _mark( + state: dict[str, Any], + target: dict[str, Any], + status: str, + *, + code: str = "", + completion: dict[str, Any] | None = None, +) -> None: + outcome = state["outcomes"][target["cell"]] + if outcome["status"] in {"pending", "known-no-action"}: + outcome["attempt"] += 1 + outcome.update(status=status, code=code, completion=completion) + + +async def _cell( + client: httpx.AsyncClient, + target: dict[str, Any], + state: dict[str, Any], + manifest: dict[str, Any], + router: str, + router_token: str, + path: Path, + deadline: float, +) -> bool: + try: + await _point(client, router, router_token, target, deadline) + token, token_end = await _token( + client, target, manifest["keys"][(target["cell"], target["deployment_id"])], deadline + ) + await _capability(client, target, token, state["command"]["action"], deadline) + if min(token_end, deadline) - time.monotonic() < 35: + raise _refuse() + preview = _context(target, state) + preview["attempt"] += 1 + if len(_encoded({"context": preview, **state["command"]["arguments"]})) > 65536: + raise _refuse() + except Exception: # noqa: BLE001 — preflight has invoked no administration business helper + _mark(state, target, "known-no-action", code="PREFLIGHT_REFUSED") + history.write_state(path, state) + print(target["cell"] + " known-no-action PREFLIGHT_REFUSED", file=sys.stderr) + return False + _mark(state, target, "dispatching") + history.write_state(path, state) + body = {"context": _context(target, state), **state["command"]["arguments"]} + try: + status, answer = await _request( + client, + "POST", + target["origin"] + "/v1/admin/cell." + state["command"]["action"], + bearer=token, + body=body, + limit=_ACTION_LIMIT, + deadline=min(deadline, time.monotonic() + 35), + action=True, + ) + code = _no_action(answer, target, state) + if code is None and status != 200: + raise _refuse() + completion = _completion(answer, target, state) if code is None else None + except Exception: # noqa: BLE001 — every post-send failure must retain uncertain completion + outcome_status = "UNSETTLED" if state["command"]["mutating"] else "known-no-action" + code = "COMPLETION_UNKNOWN" if state["command"]["mutating"] else "READ_INCOMPLETE" + _mark(state, target, outcome_status, code=code) + history.write_state(path, state) + print(target["cell"] + " " + outcome_status + " " + code, file=sys.stderr) + return False + if code is not None: + _mark(state, target, "known-no-action", code=code) + history.write_state(path, state) + print(target["cell"] + " known-no-action " + code, file=sys.stderr) + return False + _mark(state, target, "completed", completion=completion) + history.write_state(path, state) + print(target["cell"] + " completed") + if not state["command"]["mutating"]: + print(json.dumps(answer["result"], sort_keys=True, ensure_ascii=True, allow_nan=False)) + return True + + +async def _execute( + router: str, + credentials: Path, + state_path: Path, + command: Command | None, + state: dict[str, Any] | None, +) -> int: + import httpx + + if state is not None: + history.resume_read(state_path, state) + if state is not None and history.has_uncertainty(state): + print("UNSETTLED: an earlier action has no confirmed completion", file=sys.stderr) + return 1 + deadline = time.monotonic() + 3600 + router = _origin(router) + manifest = _manifest(credentials) + token = _router_token(manifest) + transport = httpx.AsyncHTTPTransport(verify=_tls(manifest), retries=0, trust_env=False) + async with httpx.AsyncClient( + transport=transport, trust_env=False, follow_redirects=False, timeout=15.0 + ) as client: + targets = await _census(client, router, token, deadline) + if any((row["cell"], row["deployment_id"]) not in manifest["keys"] for row in targets): + raise _refuse() + state = _adopt(state, command, targets, router, state_path) + for target in targets: + if state["outcomes"][target["cell"]]["status"] == "completed": + continue + if time.monotonic() >= deadline: + return 1 + if not await _cell( + client, target, state, manifest, router, token, state_path, deadline + ): + return 1 + final = await _census(client, router, token, deadline) + if final != targets or any( + row["status"] != "completed" for row in state["outcomes"].values() + ): + raise _refuse() + return 0 + + +def execute( + *, + router: str, + credentials: Path, + state_path: Path, + command: Command | None, + state: dict[str, Any] | None, +) -> int: + """Run once under the caller's lock, without replaying a sent action. + + DNS, regular-file I/O and cancellation retain the normal runtime's scheduling + limits. A local timeout supplies no remote write or process-quiescence proof. + """ + try: + return asyncio.run(_execute(router, credentials, state_path, command, state)) + except Exception: # noqa: BLE001 — custody and transport failures must never print private details + print("Cell administration stopped without complete confirmation", file=sys.stderr) + return 1 diff --git a/server/cli/cell_each_run.py b/server/cli/cell_each_run.py new file mode 100644 index 0000000000..3811dab821 --- /dev/null +++ b/server/cli/cell_each_run.py @@ -0,0 +1,103 @@ +"""Client-only command entry and conservative local resume admission.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path +from typing import NoReturn + +from server.cli.cell_each_state import has_uncertainty, read_state, resume_read, run_lock + + +class _Parser(argparse.ArgumentParser): + def error(self, _message: str) -> NoReturn: + self.exit(2, "Invalid cell administration options\n") + + +def _parser(*, add_help: bool = True) -> argparse.ArgumentParser: + parser = _Parser( + prog="loom cell each", + description="Run a supported admin command in every cell.", + add_help=add_help, + ) + parser.add_argument("--router", required=True, help="Trusted public router origin") + parser.add_argument( + "--credentials", type=Path, required=True, help="Private automation key manifest" + ) + state = parser.add_mutually_exclusive_group(required=True) + state.add_argument("--state", type=Path, help="Create an exclusive private outcome file") + state.add_argument("--resume", type=Path, help="Resume an existing private outcome file") + parser.add_argument("command", nargs=argparse.REMAINDER, help="Supported command after --") + return parser + + +def add_each_parser(sub: argparse._SubParsersAction[argparse.ArgumentParser]) -> None: + """Share the exact client flags with normal CLI help and its generated tree.""" + parser = sub.add_parser( + "each", + help="Run a supported admin command in every cell", + parents=[_parser(add_help=False)], + ) + parser.set_defaults(func=_locked) + + +def _run(args: argparse.Namespace) -> int: + from server.cli.cell_each import parse_command + + command = None + saved = None + try: + if args.resume is not None: + if args.command: + raise ValueError("A resumed run supplies its original command") + saved = read_state(args.resume) + resume_read(args.resume, saved) + if has_uncertainty(saved): + print("UNSETTLED: an earlier action has no confirmed completion", file=sys.stderr) + return 1 + else: + tokens = args.command[1:] if args.command[:1] == ["--"] else args.command + command = parse_command(tokens) + except (OSError, ValueError): + print("Invalid private cell administration input", file=sys.stderr) + return 2 + from server.cli.cell_each_http import execute + + return execute( + router=args.router, + credentials=args.credentials, + state_path=args.resume if args.resume is not None else args.state, + command=command, + state=saved, + ) + + +def _locked(args: argparse.Namespace) -> int: + try: + with run_lock(args.resume if args.resume is not None else args.state): + return _run(args) + except BlockingIOError: + print("Cell administration run is already active", file=sys.stderr) + return 1 + except (OSError, ValueError): + print("Invalid private cell administration input", file=sys.stderr) + return 2 + + +def main(argv: list[str] | None = None) -> int: + """Lock and inspect local history before any credential or HTTP facilities.""" + raw = sys.argv[1:] if argv is None else argv + try: + if ( + type(raw) is not list + or len(raw) + 3 > 256 + or any(type(value) is not str or "\0" in value for value in raw) + or sum(len(value.encode("utf-8")) + 1 for value in raw) + len(b"loom\0cell\0each\0") + > 16384 + ): + raise ValueError + except (ValueError, UnicodeError): + print("Invalid cell administration options", file=sys.stderr) + return 2 + return _locked(_parser().parse_args(raw)) diff --git a/server/cli/cell_each_state.py b/server/cli/cell_each_state.py new file mode 100644 index 0000000000..26972f9b90 --- /dev/null +++ b/server/cli/cell_each_state.py @@ -0,0 +1,522 @@ +"""Private local history for a serial cell administration run.""" + +from __future__ import annotations + +import fcntl +import hashlib +import json +import math +import os +import re +import stat +import uuid +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import TYPE_CHECKING, Any +from urllib.parse import urlsplit + +if TYPE_CHECKING: + from server.cli.cell_each import Command + +MAX_STATE_BYTES = 32 * 1024 * 1024 # not a setting: private state has one fixed custody bound +MAX_TARGETS = 4096 # not a setting: persisted target lists share the directory's protocol ceiling +_STATES = {"pending", "known-no-action", "dispatching", "completed", "UNSETTLED"} +_READS = {"settings_list", "settings_history", "signing_keys_list", "blacklist_global_list"} +_ACTIONS = _READS | { + "settings_set", + "signing_keys_rotate", + "signing_keys_retire", + "subject_revoke", + "ratelimit_namespace_set", + "budget_set", + "blacklist_global_add", + "blacklist_global_remove", +} +_TARGET_KEYS = { + "cell", + "deployment_id", + "origin", + "state", + "mode", + "mutation_generation", + "configuration_digests", + "automation", + "recovery", +} +_TOP_KEYS = {"version", "run_id", "router", "command", "targets", "outcomes"} +_ARGUMENTS = { + "settings_list": {"prefix"}, + "settings_set": {"name", "value", "clear", "expected"}, + "settings_history": {"name", "limit"}, + "signing_keys_list": set(), + "signing_keys_rotate": {"algorithm", "activate_in"}, + "signing_keys_retire": {"kid"}, + "subject_revoke": {"subject", "reason"}, + "ratelimit_namespace_set": {"namespace", "op_class", "clear", "capacity", "rate"}, + "budget_set": {"op", "consumer", "clear", "budget_ms", "budget"}, + "blacklist_global_add": {"pattern_id", "pattern", "pattern_type", "reason"}, + "blacklist_global_list": {"limit"}, + "blacklist_global_remove": {"pattern_id"}, +} +_ARGUMENT_TYPES = { + "clear": (bool,), + "limit": (int,), + "value": (str, type(None)), + "expected": (str, type(None)), + "capacity": (float, type(None)), + "rate": (float, type(None)), + "budget_ms": (int, type(None)), + "budget": (str, type(None)), + "consumer": (str, type(None)), + "activate_in": (float,), +} + + +def _bad() -> ValueError: + return ValueError("Invalid private cell administration state") + + +def _json(value: Any) -> bytes: + try: + return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() + except (TypeError, ValueError, UnicodeError, RecursionError): + raise _bad() from None + + +def _pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise _bad() + result[key] = value + return result + + +def _constant(_value: str) -> Any: + raise _bad() + + +def _float(value: str) -> float: + number = float(value) + if not math.isfinite(number): + raise _bad() + return number + + +def _text(value: Any, maximum: int, *, empty: bool = False) -> bool: + return ( + type(value) is str + and (empty or bool(value)) + and len(value.encode("utf-8")) <= maximum + and not any(ord(char) < 32 or ord(char) == 127 for char in value) + ) + + +def _uuid(value: Any) -> bool: + try: + return type(value) is str and str(uuid.UUID(value)) == value + except ValueError: + return False + + +def _bot(value: Any) -> bool: + return ( + type(value) is dict + and value.keys() == {"name", "thumbprint"} + and _text(value["name"], 256) + and type(value["thumbprint"]) is str + and re.fullmatch(r"[A-Za-z0-9_-]{43}", value["thumbprint"]) is not None + ) + + +def _target(value: Any) -> None: + if type(value) is not dict or value.keys() != _TARGET_KEYS: + raise _bad() + if not _text(value["cell"], 128) or not _text(value["deployment_id"], 256): + raise _bad() + origin = value["origin"] + if not _text(origin, 512): + raise _bad() + parsed = urlsplit(origin) + if ( + parsed.scheme != "https" + or not parsed.hostname + or parsed.username is not None + or parsed.password is not None + or parsed.path + or parsed.query + or parsed.fragment + or origin != f"https://{parsed.netloc}" + or parsed.netloc != parsed.netloc.lower() + ): + raise _bad() + if value["state"] != "serving" or value["mode"] != "enforced": + raise _bad() + if not _uuid(value["mutation_generation"]): + raise _bad() + digests = value["configuration_digests"] + if ( + type(digests) is not list + or not 1 <= len(digests) <= 8 + or any( + type(item) is not str or re.fullmatch(r"[0-9a-f]{64}", item) is None for item in digests + ) + or len(set(digests)) != len(digests) + ): + raise _bad() + if not _bot(value["automation"]) or not _bot(value["recovery"]): + raise _bad() + if any(value["automation"][key] == value["recovery"][key] for key in ("name", "thumbprint")): + raise _bad() + if len(_json(value)) > 1024: + raise _bad() + + +def _command(value: Any) -> None: + if type(value) is not dict or value.keys() != {"action", "arguments", "digest", "mutating"}: + raise _bad() + if type(value["action"]) is not str or value["action"] not in _ACTIONS: + raise _bad() + if type(value["arguments"]) is not dict or len(_json(value["arguments"])) > 65536: + raise _bad() + if value["arguments"].keys() != _ARGUMENTS[value["action"]]: + raise _bad() + if any( + type(item) not in {str, bool, int, float, type(None)} + for item in value["arguments"].values() + ): + raise _bad() + if any( + type(item) not in _ARGUMENT_TYPES.get(key, (str,)) + for key, item in value["arguments"].items() + ): + raise _bad() + if type(value["mutating"]) is not bool or value["mutating"] != (value["action"] not in _READS): + raise _bad() + intent = {"action": value["action"], "arguments": value["arguments"]} + if value["digest"] != hashlib.sha256(_json(intent)).hexdigest(): + raise _bad() + + +def _outcome(value: Any) -> None: + if type(value) is not dict or value.keys() != {"status", "attempt", "code", "completion"}: + raise _bad() + if type(value["status"]) is not str or value["status"] not in _STATES: + raise _bad() + if type(value["attempt"]) is not int or not 0 <= value["attempt"] <= 1000000: + raise _bad() + if type(value["code"]) is not str or re.fullmatch(r"[A-Z0-9_]{0,64}", value["code"]) is None: + raise _bad() + if value["status"] == "pending" and value["attempt"] != 0: + raise _bad() + if value["status"] != "pending" and value["attempt"] == 0: + raise _bad() + completion = value["completion"] + if value["status"] == "completed": + if type(completion) is not dict or not completion or len(_json(completion)) > 2048: + raise _bad() + elif completion is not None: + raise _bad() + + +def _completion(state: dict[str, Any], target: dict[str, Any]) -> None: + outcome = state["outcomes"][target["cell"]] + if outcome["status"] != "completed": + return + value = outcome["completion"] + if value.keys() != { + "version", + "context", + "action", + "started_audit", + "completed_audit", + "result_digest", + }: + raise _bad() + context = { + "version": 1, + "expected_cell": target["cell"], + "expected_deployment_id": target["deployment_id"], + "expected_bot": target["automation"]["name"], + "run_id": state["run_id"], + "attempt": outcome["attempt"], + "command_digest": state["command"]["digest"], + } + saved_context = value["context"] + if ( + type(value["version"]) is not int + or value["version"] != 1 + or type(saved_context) is not dict + or saved_context.keys() != context.keys() + or any( + type(saved_context[key]) is not type(item) or saved_context[key] != item + for key, item in context.items() + ) + ): + raise _bad() + if value["action"] != state["command"]["action"]: + raise _bad() + for key in ("started_audit", "completed_audit", "result_digest"): + if type(value[key]) is not str or re.fullmatch(r"[0-9a-f]{64}", value[key]) is None: + raise _bad() + + +def _validate(state: Any) -> None: + if type(state) is not dict or state.keys() != _TOP_KEYS: + raise _bad() + if type(state["version"]) is not int or state["version"] != 1 or not _uuid(state["run_id"]): + raise _bad() + if state["router"] is not None and not _text(state["router"], 512): + raise _bad() + _command(state["command"]) + targets, outcomes = state["targets"], state["outcomes"] + if ( + type(targets) is not list + or not 1 <= len(targets) <= MAX_TARGETS + or type(outcomes) is not dict + ): + raise _bad() + cells: set[str] = set() + for target in targets: + _target(target) + if target["cell"] in cells: + raise _bad() + cells.add(target["cell"]) + if outcomes.keys() != cells: + raise _bad() + for outcome in outcomes.values(): + _outcome(outcome) + for target in targets: + _completion(state, target) + if sum(outcome["status"] in {"dispatching", "UNSETTLED"} for outcome in outcomes.values()) > 1: + raise _bad() + + +def new_state(command: Command, *, targets: list[dict[str, Any]], run_id: str) -> dict[str, Any]: + """Freeze accepted public targets and intent without credential references.""" + if type(targets) is not list or not 1 <= len(targets) <= MAX_TARGETS: + raise _bad() + for target in targets: + _target(target) + state = { + "version": 1, + "run_id": run_id, + "router": None, + "command": { + "action": command.action, + "arguments": command.arguments, + "digest": command.digest, + "mutating": command.mutating, + }, + "targets": targets, + "outcomes": { + target["cell"]: {"status": "pending", "attempt": 0, "code": "", "completion": None} + for target in targets + }, + } + _validate(state) + # Own the accepted snapshot instead of retaining a caller's mutable target list. + owned: dict[str, Any] = json.loads(_json(state)) + return owned + + +def has_uncertainty(state: dict[str, Any]) -> bool: + """An outstanding dispatch cannot be settled by local history or a retry.""" + return any( + value["status"] in {"dispatching", "UNSETTLED"} for value in state["outcomes"].values() + ) + + +def resume_read(path: Path, state: dict[str, Any]) -> None: + """A lost fixed read may repeat; it supplies no unknown business mutator.""" + if state["command"]["mutating"]: + return + for outcome in state["outcomes"].values(): + if outcome["status"] == "dispatching": + outcome.update(status="known-no-action", code="READ_INCOMPLETE") + write_state(path, state) + + +@contextmanager +def private_parent(path: Path) -> Iterator[tuple[int, str]]: + """Pin every directory without following links; the leaf remains owner-only.""" + parts = path.absolute().parts + if not parts[-1] or any(part in {".", ".."} for part in parts[1:]): + raise _bad() + descriptor = os.open("/", os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC) + try: + for part in parts[1:-1]: + child = os.open( + part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=descriptor + ) + os.close(descriptor) + descriptor = child + info = os.fstat(descriptor) + if info.st_uid not in {0, os.geteuid()}: + raise _bad() + writable = info.st_mode & 0o022 + sticky_root = info.st_uid == 0 and info.st_mode & stat.S_ISVTX + if writable and not sticky_root: + raise _bad() + info = os.fstat(descriptor) + if info.st_uid != os.geteuid() or info.st_mode & 0o077: + raise _bad() + yield descriptor, parts[-1] + finally: + os.close(descriptor) + + +def _read_bytes_at(parent: int, name: str, limit: int) -> tuple[bytes, os.stat_result]: + descriptor = os.open( + name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC, dir_fd=parent + ) + with os.fdopen(descriptor, "rb") as stream: + info = os.fstat(stream.fileno()) + if ( + not stat.S_ISREG(info.st_mode) + or info.st_uid != os.geteuid() + or info.st_mode & 0o077 + or not info.st_mode & stat.S_IRUSR + or info.st_nlink != 1 + or info.st_size > limit + ): + raise _bad() + raw = stream.read(limit + 1) + after = os.fstat(stream.fileno()) + stable = ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns") + if len(raw) > limit or any(getattr(after, key) != getattr(info, key) for key in stable): + raise _bad() + return raw, info + + +def read_private(path: Path, limit: int) -> bytes: + """Read a bounded private input through the same pinned directory boundary.""" + if type(limit) is not int or not 0 < limit <= MAX_STATE_BYTES: + raise _bad() + with private_parent(Path(path)) as (parent, name): + raw, _info = _read_bytes_at(parent, name, limit) + return raw + + +@contextmanager +def run_lock(path: Path) -> Iterator[None]: + """Admit one local coordinator; lock release never settles a remote action.""" + with private_parent(Path(path)) as (parent, name): + descriptor = os.open( + f".{name}.lock", + os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC, + 0o600, + dir_fd=parent, + ) + try: + info = os.fstat(descriptor) + if ( + not stat.S_ISREG(info.st_mode) + or info.st_uid != os.geteuid() + or stat.S_IMODE(info.st_mode) != 0o600 + or info.st_nlink != 1 + ): + raise _bad() + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + yield + finally: + os.close(descriptor) + + +def _read_at(parent: int, name: str) -> tuple[dict[str, Any], os.stat_result]: + raw, info = _read_bytes_at(parent, name, MAX_STATE_BYTES) + try: + state = json.loads( + raw.decode("utf-8"), + object_pairs_hook=_pairs, + parse_constant=_constant, + parse_float=_float, + ) + _validate(state) + except (ValueError, TypeError, UnicodeError, RecursionError): + raise _bad() from None + return state, info + + +def read_state(path: Path) -> dict[str, Any]: + """Read one bounded complete state; malformed history never becomes a new run.""" + with private_parent(Path(path)) as (parent, name): + state, _info = _read_at(parent, name) + return state + + +def _transition(previous: dict[str, Any], current: dict[str, Any]) -> None: + for key in ("version", "run_id", "router", "command"): + if previous[key] != current[key]: + raise _bad() + targets = {row["cell"]: row for row in current["targets"]} + if any(targets.get(row["cell"]) != row for row in previous["targets"]): + raise _bad() + allowed = { + "pending": {"pending", "dispatching", "known-no-action"}, + "known-no-action": {"known-no-action", "dispatching"}, + "dispatching": {"dispatching", "completed", "known-no-action", "UNSETTLED"}, + "completed": {"completed"}, + "UNSETTLED": {"UNSETTLED"}, + } + for cell, old in previous["outcomes"].items(): + new = current["outcomes"][cell] + if new["status"] not in allowed[old["status"]]: + raise _bad() + if old["status"] in {"completed", "UNSETTLED"} and new != old: + raise _bad() + increment = old["status"] in {"pending", "known-no-action"} and new != old + if new["attempt"] != old["attempt"] + int(increment): + raise _bad() + changes = sum(current["outcomes"][cell] != old for cell, old in previous["outcomes"].items()) + if changes > 1: + raise _bad() + if has_uncertainty(previous): + if current["targets"] != previous["targets"]: + raise _bad() + for cell, old in previous["outcomes"].items(): + if old["status"] != "dispatching" and current["outcomes"][cell] != old: + raise _bad() + for cell in current["outcomes"].keys() - previous["outcomes"].keys(): + if current["outcomes"][cell]["status"] != "pending": + raise _bad() + + +def _publish(parent: int, name: str, raw: bytes, *, exclusive: bool) -> None: + temporary = f".{name}.{uuid.uuid4().hex}.tmp" + descriptor = os.open( + temporary, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, + 0o600, + dir_fd=parent, + ) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(raw) + stream.flush() + os.fsync(stream.fileno()) + if exclusive: + os.link(temporary, name, src_dir_fd=parent, dst_dir_fd=parent, follow_symlinks=False) + os.unlink(temporary, dir_fd=parent) + else: + os.replace(temporary, name, src_dir_fd=parent, dst_dir_fd=parent) + os.fsync(parent) + finally: + try: + os.unlink(temporary, dir_fd=parent) + except FileNotFoundError: + pass + + +def write_state(path: Path, state: dict[str, Any], *, exclusive: bool = False) -> None: + """Publish acknowledged local history before admitting any later action.""" + _validate(state) + raw = _json(state) + if len(raw) > MAX_STATE_BYTES: + raise _bad() + with private_parent(Path(path)) as (parent, name): + if not exclusive: + previous, _info = _read_at(parent, name) + _transition(previous, state) + _publish(parent, name, raw, exclusive=exclusive) diff --git a/server/core/ch_bounded.py b/server/core/ch_bounded.py new file mode 100644 index 0000000000..4799b99b19 --- /dev/null +++ b/server/core/ch_bounded.py @@ -0,0 +1,112 @@ +"""Finite JSONEachRow reads with limits enforced before decoding rows.""" + +from __future__ import annotations + +import asyncio +import json +import math +import time +from typing import Any + +from server.core.ch import own_write_reads, set_query_tag, statement_timeout + + +class BoundedReadError(ValueError): + """The complete answer cannot be established within its read allowance.""" + + +def _object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise BoundedReadError("Duplicate result field") + result[key] = value + return result + + +def _constant(value: str) -> Any: + raise BoundedReadError("Nonfinite result value") + + +def _float(value: str) -> float: + result = float(value) + if not math.isfinite(result): + raise BoundedReadError("Nonfinite result value") + return result + + +def decode_json(text: str) -> Any: + """Decode already byte-bounded JSON without duplicate keys or nonfinite numbers.""" + try: + return json.loads( + text, + object_pairs_hook=_object, + parse_constant=_constant, + parse_float=_float, + ) + except (UnicodeError, ValueError, RecursionError) as error: + raise BoundedReadError("Malformed JSON value") from error + + +def _row(line: bytes) -> dict[str, Any]: + try: + row = decode_json(line.decode("utf-8")) + except UnicodeError as error: + raise BoundedReadError("Result row is not UTF-8") from error + if not isinstance(row, dict): + raise BoundedReadError("Result row is not an object") + return row + + +async def read_rows( + sql: str, + params: dict[str, Any], + *, + ch: Any, + deadline: float, + row_bytes: int, + total_bytes: int, + max_rows: int, + tag: str, + max_read_rows: int = 1_000_000, + max_read_bytes: int = 128 * 1024**2, +) -> list[dict[str, Any]]: + """Read a caller's fixed SELECT with its explicit LIMIT and JSONEachRow format.""" + remaining = min(2.0, deadline - time.monotonic()) + if remaining <= 0 or min(row_bytes, total_bytes, max_rows, max_read_rows, max_read_bytes) <= 0: + raise BoundedReadError("Read allowance exhausted") + limits = { + "max_execution_time": remaining, + "max_memory_usage": 128 * 1024**2, + "max_rows_to_read": min(max_read_rows, 1_000_000), + "max_bytes_to_read": min(max_read_bytes, 128 * 1024**2), + "max_result_rows": max_rows + 1, + "max_result_bytes": total_bytes + 1, + "read_overflow_mode": "throw", + "result_overflow_mode": "throw", + "timeout_overflow_mode": "throw", + "output_format_json_quote_64bit_integers": 0, + } + rows: list[dict[str, Any]] = [] + pending = bytearray() + consumed = 0 + set_query_tag(tag) + async with asyncio.timeout(remaining): + with own_write_reads(), statement_timeout(remaining): + async with ch.raw_stream(sql, params=params, settings=limits) as chunks: + async for chunk in chunks: + consumed += len(chunk) + if consumed > total_bytes: + raise BoundedReadError("Result exceeds its byte bound") + pending.extend(chunk) + while (end := pending.find(b"\n")) >= 0: + if end > row_bytes or len(rows) >= max_rows: + raise BoundedReadError("Result exceeds its row bound") + line = bytes(pending[:end]) + del pending[: end + 1] + rows.append(_row(line)) + if len(pending) > row_bytes: + raise BoundedReadError("Row exceeds its byte bound") + if pending: + raise BoundedReadError("Incomplete result row") + return rows diff --git a/server/core/leases.py b/server/core/leases.py index 30a8e1572f..46c7e37e1c 100644 --- a/server/core/leases.py +++ b/server/core/leases.py @@ -122,6 +122,7 @@ def __init__( holder: str | None = None, may_take_over: Callable[[str], Awaitable[bool]] | None = None, expect_free: bool = False, + deadline: float | None = None, ) -> None: self.name = name self.ttl_s = ttl_s @@ -129,6 +130,7 @@ def __init__( self.holder = holder or default_holder() self.may_take_over = may_take_over self.expect_free = expect_free + self.deadline = deadline self.lost = False # When the last attempt or renewal that landed was sent (monotonic). self._renewed = 0.0 @@ -170,7 +172,7 @@ async def release(self) -> None: if not self._sent: return try: - await self.ch.execute(LOCK_RELEASE, params=self._params()) + await self._execute(LOCK_RELEASE, self._params()) self._sent = False except Exception as e: # noqa: BLE001 — the lease expires by its TTL log.warning("lease %s: release failed (%s); it expires by its TTL", self.name, e) @@ -237,15 +239,57 @@ def _params(self, **extra: Any) -> dict[str, Any]: } async def _holder_row(self) -> tuple[str, bool] | None: + if self.deadline is not None: + from server.core.ch_bounded import BoundedReadError, read_rows + + rows = await read_rows( + f"SELECT holder, (renewed_at + toIntervalSecond(ttl_s)) < now64(3) AS stale " + f"FROM {LOCK_TABLE} WHERE name = {{name:String}} LIMIT 2 FORMAT JSONEachRow", + {"name": self.name}, + ch=self.ch, + deadline=self.deadline, + row_bytes=4096, + total_bytes=4096, + max_rows=1, + tag="cell.each.claim_read", + ) + if not rows: + return None + row = rows[0] + if ( + set(row) != {"holder", "stale"} + or type(row["holder"]) is not str + or type(row["stale"]) is not int + or row["stale"] not in (0, 1) + ): + raise BoundedReadError("Malformed claim holder") + return row["holder"], bool(row["stale"]) row = await self.ch.query_one(LOCK_HOLDER, {"name": self.name}) return None if row is None else (str(row["holder"]), bool(row["stale"])) + async def _execute(self, sql: str, params: dict[str, Any]) -> None: + if self.deadline is None: + await self.ch.execute(sql, params=params) + return + from server.core.ch import set_query_tag, statement_timeout + from server.core.ch_bounded import BoundedReadError + + remaining = min(2.0, self.deadline - time.monotonic()) + if remaining <= 0: + raise BoundedReadError("Claim allowance exhausted") + set_query_tag("cell.each.claim_write") + async with asyncio.timeout(remaining): + with statement_timeout(remaining): + await self.ch.execute( + sql, params=params, settings={"max_memory_usage": 128 * 1024**2} + ) + async def _keeper_attempt(self) -> bool: row = None if self.expect_free else await self._holder_row() if row is None: self._sent = True try: - await self.ch.execute(LOCK_INSERT, params=self._params()) + await self._execute(LOCK_INSERT, self._params()) return True except Exception as e: if not key_exists(e): @@ -261,7 +305,7 @@ async def _keeper_attempt(self) -> bool: return False self._sent = True try: - await self.ch.execute(LOCK_TAKEOVER, params=self._params(old=holder)) + await self._execute(LOCK_TAKEOVER, self._params(old=holder)) except Exception as e: if not lost_race(e): raise @@ -272,7 +316,7 @@ async def _keeper_attempt(self) -> bool: return won async def _keeper_renew(self) -> bool: - await self.ch.execute(LOCK_RENEW, params=self._params()) + await self._execute(LOCK_RENEW, self._params()) return await self._holder_row() == (self.holder, False) @@ -288,6 +332,7 @@ async def take( holder: str | None = None, may_take_over: Callable[[str], Awaitable[bool]] | None = None, expect_free: bool = False, + deadline: float | None = None, ) -> Lease | None: """The lease on `name`, or None while another holder keeps it past `wait_s`. `renew` keeps it renewed until `release`. Raises LeaseUnavailable when the store @@ -302,6 +347,7 @@ async def take( holder=holder, may_take_over=may_take_over, expect_free=expect_free, + **({} if deadline is None else {"deadline": deadline}), ) # No attempt starts after the wait, however late we woke; 0 is one attempt. won = await lease.attempt() if wait_s <= 0 or time.monotonic() < until else False @@ -397,7 +443,13 @@ class ClaimExpired(LeaseUnavailable): @asynccontextmanager async def holding( - name: str, *, ttl_s: float, wait_s: float = 0.0, ch: Any = None + name: str, + *, + ttl_s: float, + wait_s: float = 0.0, + ch: Any = None, + deadline: float | None = None, + confirm_release: bool = False, ) -> AsyncIterator[bool]: """Claim `name` for the block, waiting up to `wait_s`: yields True when this caller has it, False when another holder keeps it. LeaseUnavailable when the @@ -405,7 +457,8 @@ async def holding( the TTL.""" from server.core.ch import statement_timeout - lease = await take(name, ttl_s=ttl_s, wait_s=wait_s, ch=ch) + bounded: dict[str, Any] = {} if deadline is None else {"deadline": deadline} + lease = await take(name, ttl_s=ttl_s, wait_s=wait_s, ch=ch, **bounded) if lease is None: yield False return @@ -424,6 +477,8 @@ async def holding( ) from e finally: await release_shielded(lease) + if confirm_release and lease._sent: + raise LeaseUnavailable("claim release was not confirmed", operation=name) # KeeperMap keys their writer deletes, unless it died first: per table, the delete of diff --git a/server/core/pg.py b/server/core/pg.py index 725200e6f4..eeaf69530c 100644 --- a/server/core/pg.py +++ b/server/core/pg.py @@ -990,7 +990,7 @@ async def fence_lock(conn: Any, org: str) -> None: POLICY_LOCK_KEY = "hashtextextended('loom.policy/' || k.org || '/' || k.scope, 0)" -async def bump_policy(org: str, scope: str = "*") -> None: +async def bump_policy(org: str, scope: str = "*", *, deadline: float | None = None) -> None: """Advance a policy generation (org '' for the deployment's): an admission that reached its verdict under the old policy starts over (`pg_memory.commit_versions`). Fails closed: the change answers 503 until the generation moves, and its retry @@ -998,6 +998,8 @@ async def bump_policy(org: str, scope: str = "*") -> None: if not enabled(): return async with transaction() as conn: + if deadline is not None: + await _bounded_policy_statement(conn, deadline) # After every commit that holds the policy shared (`pg_memory.commit_versions`). await conn.execute( f"SELECT pg_advisory_xact_lock({POLICY_LOCK_KEY}) " @@ -1005,6 +1007,8 @@ async def bump_policy(org: str, scope: str = "*") -> None: org, scope, ) + if deadline is not None: + await _bounded_policy_statement(conn, deadline) await conn.execute( "INSERT INTO loom.scope_generations (org, scope, generation) VALUES ($1, $2, 1) " "ON CONFLICT (org, scope) " @@ -1012,3 +1016,14 @@ async def bump_policy(org: str, scope: str = "*") -> None: org, scope, ) + + +async def _bounded_policy_statement(conn: Any, deadline: float) -> None: + remaining_ms = int(min(2.0, deadline - time.monotonic()) * 1000) + if remaining_ms <= 0: + raise PostgresUnavailableError("policy change allowance exhausted") + await conn.execute( + "SELECT set_config('statement_timeout', $1, true)", + str(remaining_ms), + timeout=remaining_ms / 1000, + ) diff --git a/server/core/ratelimit.py b/server/core/ratelimit.py index 8951361253..2e1ef6ae2d 100644 --- a/server/core/ratelimit.py +++ b/server/core/ratelimit.py @@ -715,6 +715,42 @@ def _check_shared( reset_s=(lim.capacity - remaining) / max(lim.rate, 1e-9), ) + def check_bounded_namespace_change(self, namespace: str, op_class: str, *, clear: bool) -> None: + """Bound adoption without opening or reconfiguring the shared bucket table.""" + from server.core.ch_bounded import BoundedReadError + + if len(DEFAULT_LIMITS) > 64 or op_class not in DEFAULT_LIMITS: + raise BoundedReadError("Unsupported rate-limit class") + if any( + len(buckets) > MAX_BUCKETS + for buckets in ( + self._ns_buckets, + self._org_buckets, + self._actor_buckets, + ) + ): + raise BoundedReadError("Rate-limit buckets exceed their allowance") + if len(self._ns_overrides) > 1024 or ( + not clear + and (namespace, op_class) not in self._ns_overrides + and len(self._ns_overrides) == 1024 + ): + raise BoundedReadError("Namespace rate overrides exceed their allowance") + mode, path, slots = ( + str(settings.ratelimit_shared), + str(settings.ratelimit_shared_path), + int(settings.ratelimit_shared_slots), + ) + workers = max(1, int(settings.web_workers)) + expected = (mode, path, slots, workers) + if ( + _wants_table(mode, workers) + and self._shared is None + or self._shared is not None + and self._shared_key != expected + ): + raise BoundedReadError("Rate-limit shared backend is not initialized") + def set_namespace_limit(self, namespace: str, op_class: str, limit: Limit) -> None: """Install a per-namespace override for one op_class.""" self._ns_overrides[(namespace, op_class)] = limit diff --git a/server/core/runtime_overrides.py b/server/core/runtime_overrides.py index 8f7c4a741d..0c973c3569 100644 --- a/server/core/runtime_overrides.py +++ b/server/core/runtime_overrides.py @@ -14,6 +14,7 @@ import time from typing import Any +from server.core.ch_bounded import BoundedReadError, decode_json, read_rows from server.core.ch_errors import error_code from server.core.errors import BackendUnavailableError, LoomServerError from server.core.repo_base import _CHClient, _default_client @@ -65,31 +66,89 @@ class OverrideNotStoredError(LoomServerError): status_code = 409 -async def put(kind: str, key: tuple[str, ...], value: Any, *, ch: _CHClient | None = None) -> None: - version = await _write(kind, key, json.dumps(value), deleted=0, ch=ch) +async def put( + kind: str, + key: tuple[str, ...], + value: Any, + *, + ch: _CHClient | None = None, + deadline: float | None = None, +) -> None: + version = await _write(kind, key, json.dumps(value), deleted=0, ch=ch, deadline=deadline) _own[(kind, key)] = (value, version, time.monotonic_ns()) -async def clear(kind: str, key: tuple[str, ...], *, ch: _CHClient | None = None) -> None: - version = await _write(kind, key, "null", deleted=1, ch=ch) +async def clear( + kind: str, + key: tuple[str, ...], + *, + ch: _CHClient | None = None, + deadline: float | None = None, +) -> None: + version = await _write(kind, key, "null", deleted=1, ch=ch, deadline=deadline) _own[(kind, key)] = (None, version, time.monotonic_ns()) async def _write( - kind: str, key: tuple[str, ...], value: str, *, deleted: int, ch: _CHClient | None + kind: str, + key: tuple[str, ...], + value: str, + *, + deleted: int, + ch: _CHClient | None, + deadline: float | None = None, ) -> int: - """Store one row; returns its version. A refused write raises `OverrideNotStoredError`, - one a retry may store `BackendUnavailableError`.""" + """Store one row; return its version. Bounded callers never infer no write + from a lost acknowledgement.""" global _last_version client = ch or _default_client() row = {"kind": kind, "key": json.dumps(list(key)), "value": value, "deleted": deleted} try: - stamp = await client.query_one(_VERSION_SQL) + if deadline is None: + stamp = await client.query_one(_VERSION_SQL) + else: + check_bounded_write(kind, key, value) + stamps = await read_rows( + _VERSION_SQL + " LIMIT 2 FORMAT JSONEachRow", + {}, + ch=client, + deadline=deadline, + row_bytes=1024, + total_bytes=1024, + max_rows=1, + tag="cell.each.override_clock", + ) + if len(stamps) != 1 or type(stamps[0].get("v")) is not int: + raise BoundedReadError("Malformed override clock") + stamp = stamps[0] version = max(int((stamp or {})["v"]), _last_version + 1) - await client.insert(TABLE, [{**row, "version": version}], settings={"async_insert": 0}) + if deadline is None: + await client.insert(TABLE, [{**row, "version": version}], settings={"async_insert": 0}) + else: + import asyncio + + from server.core.ch import set_query_tag, statement_timeout + + remaining = min(2.0, deadline - time.monotonic()) + if remaining <= 0: + raise BoundedReadError("Override write allowance exhausted") + check_bounded_write(kind, key, value) + set_query_tag("cell.each.override_write") + async with asyncio.timeout(remaining): + with statement_timeout(remaining): + await client.insert( + TABLE, + [{**row, "version": version}], + settings={"async_insert": 0, "max_memory_usage": 128 * 1024**2}, + ) except LoomServerError: raise # already typed: unreachable or timed out except Exception as e: # a refusal is the caller's 409, anything else a 503 + if deadline is not None: + raise BackendUnavailableError( + "the bounded override change was not confirmed", + operation="runtime_overrides.write", + ) from e logger.warning("runtime override %s %s not stored: %s", kind, key, e) if error_code(e) in _REFUSED_CODES: raise OverrideNotStoredError( @@ -104,6 +163,95 @@ async def _write( return version +def check_bounded_write(kind: str, key: tuple[str, ...], value: str) -> None: + """Refuse unsupported local bookkeeping before a bounded point write.""" + if len(_own) > 4096 or ((kind, key) not in _own and len(_own) == 4096): + raise BoundedReadError("Override bookkeeping exceeds its allowance") + maximum = 16384 if kind == SETTING else 1024 + payload = { + "kind": kind, + "key": json.dumps(list(key)), + "value": value, + "deleted": 0, + "version": 2**64 - 1, + } + if len(json.dumps(payload).encode("utf-8")) > maximum: + raise BoundedReadError("Override key and value exceed their allowance") + + +async def load_settings( + *, + deadline: float, + name: str | None = None, + ch: _CHClient | None = None, +) -> Snapshot: + """Read setting-kind rows, or one exact setting, without a full override scan.""" + from server.settings import Settings + + if len(Settings.model_fields) > 4096: + raise BoundedReadError("Settings catalog exceeds its allowance") + maximum = 1024 if name is None else 1 + params: dict[str, Any] = {"kind": SETTING, "limit": maximum + 1} + predicate = "" + if name is not None: + predicate = " AND key = {key:String}" + params["key"] = json.dumps([name]) + rows = await read_rows( + "SELECT key, value, deleted, version FROM loom.runtime_overrides FINAL " + "WHERE kind = {kind:String}" + predicate + " LIMIT {limit:UInt32} FORMAT JSONEachRow", + params, + ch=ch or _default_client(), + deadline=deadline, + row_bytes=16384, + total_bytes=2 * 1024**2, + max_rows=maximum, + tag="cell.each.setting_overrides", + ) + out: dict[tuple[str, ...], Any] = {} + seen: dict[tuple[str, ...], int] = {} + for row in rows: + if set(row) != {"key", "value", "deleted", "version"}: + raise BoundedReadError("Malformed setting override") + if type(row["key"]) is not str or type(row["value"]) is not str: + raise BoundedReadError("Malformed setting override") + decoded = decode_json(row["key"]) + if not isinstance(decoded, list) or len(decoded) != 1 or type(decoded[0]) is not str: + raise BoundedReadError("Malformed setting override key") + key = (decoded[0],) + version, deleted = row["version"], row["deleted"] + if ( + key in seen + or type(version) is not int + or version < 0 + or type(deleted) is not int + or deleted not in (0, 1) + or name is not None + and key != (name,) + ): + raise BoundedReadError("Ambiguous setting override") + seen[key] = version + if not deleted: + out[key] = decode_json(row["value"]) + # Point lookups retain own-write visibility without traversing other override kinds. + names = Settings.model_fields if name is None else (name,) + now = time.monotonic_ns() + for field in names: + key = (field,) + owned = _own.get((SETTING, key)) + if owned is None: + continue + value, version, written = owned + if seen.get(key, -1) >= version or now - written > _OWN_HOLD_NS: + del _own[(SETTING, key)] + elif value is None: + out.pop(key, None) + else: + out[key] = value + if len(out) > maximum: + raise BoundedReadError("Setting overrides exceed their allowance") + return {SETTING: out} + + async def load(*, ch: _CHClient | None = None) -> Snapshot: """kind → key → value, cleared keys left out. A row that does not parse is skipped, the rest adopted.""" diff --git a/server/core/settings_overrides.py b/server/core/settings_overrides.py index 6c78b13ee2..4c4b91b83d 100644 --- a/server/core/settings_overrides.py +++ b/server/core/settings_overrides.py @@ -13,13 +13,15 @@ from __future__ import annotations +import json import logging -from collections.abc import Callable, Iterable +from collections.abc import Callable, Iterable, Iterator from typing import Any from pydantic import ValidationError as PydanticValidationError from server.core import runtime_overrides as ro +from server.core.ch_bounded import BoundedReadError from server.core.errors import LoomServerError, ValidationError from server.core.repo_base import _CHClient from server.settings import Settings, settings @@ -91,6 +93,19 @@ async def change( overwritten unseen; two changes in the same instant can still both pass (the table has no conditional write).""" snapshot = await refresh(ch=ch) + return await _change(name, value, snapshot, clear=clear, expected=expected, ch=ch) + + +async def _change( + name: str, + value: Any, + snapshot: ro.Snapshot, + *, + clear: bool, + expected: Any, + ch: _CHClient | None, + deadline: float | None = None, +) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: known = name in Settings.model_fields before = row(name) if known else None previous = before["value"] if before else None @@ -107,15 +122,90 @@ async def change( if clear: if (name,) not in snapshot.get(KIND, {}): raise ValidationError(f"{name} has no override to reset") - await ro.clear(KIND, (name,), ch=ch) + bounded: dict[str, Any] = {} if deadline is None else {"deadline": deadline} + await ro.clear(KIND, (name,), ch=ch, **bounded) _restore(name) else: checked = check(name, value) - await ro.put(KIND, (name,), to_json(Settings, name, checked), ch=ch) + if deadline is not None: + _finite(name, checked) + bounded = {} if deadline is None else {"deadline": deadline} + await ro.put(KIND, (name,), to_json(Settings, name, checked), ch=ch, **bounded) _apply(name, checked) return before, row(name) if known else None +def check_selected(name: str) -> None: + """Only known, non-secret LIVE fields belong to cell administration.""" + field = Settings.model_fields.get(name) + if ( + len(Settings.model_fields) > 4096 + or field is None + or not is_live(field) + or is_secret_field(name) + ): + raise BoundedReadError("Cell administration requires a non-secret LIVE setting") + + +def _finite(name: str, value: Any) -> None: + try: + json.dumps(to_json(Settings, name, value), allow_nan=False) + except (ValueError, TypeError, OverflowError): + raise BoundedReadError("Setting value is not finite JSON") from None + + +async def change_bounded( + name: str, + value: Any = None, + *, + clear: bool = False, + expected: Any = NOT_GIVEN, + deadline: float, + ch: _CHClient | None = None, +) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: + """Refresh only this setting before the ordinary precondition and stored-first change.""" + check_selected(name) + snapshot = await ro.load_settings(name=name, deadline=deadline, ch=ch) + if (name,) in snapshot[KIND]: + checked = check(name, snapshot[KIND][(name,)]) + _finite(name, checked) + _apply(name, checked) + else: + _restore(name) + return await _change( + name, + value, + snapshot, + clear=clear, + expected=expected, + ch=ch, + deadline=deadline, + ) + + +async def refresh_bounded(*, deadline: float, ch: _CHClient | None = None) -> None: + """Adopt a complete bounded setting snapshot only after all selected values validate.""" + if len(Settings.model_fields) > 4096: + raise BoundedReadError("Settings catalog exceeds its allowance") + eligible = { + name + for name, field in Settings.model_fields.items() + if is_live(field) and not is_secret_field(name) + } + snapshot = await ro.load_settings(deadline=deadline, ch=ch) + wanted = {} + for key, raw in snapshot[KIND].items(): + name = key[0] + if name in eligible: + value = check(name, raw) + _finite(name, value) + wanted[name] = value + for name, value in wanted.items(): + _apply(name, value) + for name in eligible - wanted.keys(): + _restore(name) + + def adopted() -> bool: """Whether this process took the deployment's setting overrides once: before that, a LIVE setting reads the value the process started with, even where an operator @@ -204,7 +294,12 @@ def _shown(name: str, value: Any) -> Any: return settings._mask(name, value) if is_secret_field(name) else to_json(Settings, name, value) -def rows(*, prefix: str = "", changed: bool = False) -> list[dict[str, Any]]: +def _rows( + *, + prefix: str, + changed: bool, + selected: bool = False, +) -> Iterator[dict[str, Any]]: """Every setting whose name starts with `prefix`: its value in force, its default, where the value came from and whether it takes an override. `changed` keeps the ones off their default or overridden.""" @@ -212,25 +307,46 @@ def rows(*, prefix: str = "", changed: bool = False) -> list[dict[str, Any]]: about = descriptions(Settings) sources = _sources() - out = [] for name, field in Settings.model_fields.items(): if not name.startswith(prefix): continue + if selected and (not is_live(field) or is_secret_field(name)): + continue value = getattr(settings, name) default = field.get_default(call_default_factory=True) if changed and value == default and sources[name] != "override": continue - out.append( - { - "name": name, - "env": f"LOOM_{name.upper()}", - "value": _shown(name, value), - "default": _shown(name, default), - "source": sources[name], - "live": is_live(field), - "description": about.get(name, ""), - } - ) + yield { + "name": name, + "env": f"LOOM_{name.upper()}", + "value": _shown(name, value), + "default": _shown(name, default), + "source": sources[name], + "live": is_live(field), + "description": about.get(name, ""), + } + + +def rows(*, prefix: str = "", changed: bool = False) -> list[dict[str, Any]]: + """The current value, source and description of every matching setting.""" + return list(_rows(prefix=prefix, changed=changed)) + + +def selected_rows(*, prefix: str = "") -> list[dict[str, Any]]: + """Bound the non-secret LIVE listing before accumulating its output rows.""" + if len(Settings.model_fields) > 4096: + raise BoundedReadError("Settings catalog exceeds its allowance") + out = [] + consumed = 0 + for item in _rows(prefix=prefix, changed=False, selected=True): + try: + encoded = json.dumps(item, ensure_ascii=False, allow_nan=False, separators=(",", ":")) + except (ValueError, TypeError, OverflowError): + raise BoundedReadError("Setting result is not finite JSON") from None + consumed += len(encoded.encode("utf-8")) + if len(encoded.encode("utf-8")) > 16384 or consumed > 2 * 1024**2: + raise BoundedReadError("Setting result exceeds its allowance") + out.append(item) return out diff --git a/server/core/versions.py b/server/core/versions.py index 1f28c17f1e..f264e58b86 100644 --- a/server/core/versions.py +++ b/server/core/versions.py @@ -14,12 +14,44 @@ from server.core.ch import own_write_reads -async def next_version(ch: Any, table: str, key: dict[str, str]) -> int: +async def next_version( + ch: Any, + table: str, + key: dict[str, str], + *, + deadline: float | None = None, +) -> int: """A version past the newest row of `key` (its columns' values) in `table`, on ClickHouse's clock. Under FINAL the highest version wins, not the latest write, so it must pass whatever any host wrote; one clock for every host keeps their writes in order.""" where = " AND ".join(f"{column} = {{k{i}:String}}" for i, column in enumerate(key)) + if deadline is not None: + from server.core.ch_bounded import BoundedReadError, read_rows + + rows = await read_rows( + "SELECT toUInt64(toUnixTimestamp64Nano(now64(9))) AS clock, max(version) AS previous " + f"FROM {table} WHERE {where} LIMIT 2 FORMAT JSONEachRow", + {f"k{i}": value for i, value in enumerate(key.values())}, + ch=ch, + deadline=deadline, + row_bytes=1024, + total_bytes=1024, + max_rows=1, + tag="cell.each.row_version", + ) + if ( + len(rows) != 1 + or set(rows[0]) != {"clock", "previous"} + or any( + type(rows[0][name]) is not int or not 0 <= rows[0][name] < 2**64 - 1 + for name in ("clock", "previous") + ) + ): + raise BoundedReadError("A successor version could not be established") + clock: int = rows[0]["clock"] + previous: int = rows[0]["previous"] + return max(clock, previous + 1) with own_write_reads(): # the newest row may have landed a moment ago elsewhere row = await ch.query_one( "SELECT greatest(toUInt64(toUnixTimestamp64Nano(now64(9))), max(version) + 1) AS v " diff --git a/server/identity/app/admin_account_routes.py b/server/identity/app/admin_account_routes.py index 1928df69f3..4a7fd31284 100644 --- a/server/identity/app/admin_account_routes.py +++ b/server/identity/app/admin_account_routes.py @@ -220,8 +220,12 @@ async def bots_create(input: BotInput, info: Admin) -> dict[str, Any]: @router.post("/v1/admin/bots.list") -async def bots_list() -> dict[str, Any]: - """Every registered bot: its namespaces and roles, key or issuer, and limits.""" +async def bots_list(info: Admin = None, self_only: bool = False) -> dict[str, Any]: + """Every registered bot, or with self_only the caller's fresh bounded public identity.""" + if self_only: + from server.insights.app.cell_admin_auth import registered_bot_identity + + return await registered_bot_identity(info) rows = await machines.listed() return { "bots": [ diff --git a/server/insights/api/models_cell_admin.py b/server/insights/api/models_cell_admin.py new file mode 100644 index 0000000000..a0b77a7829 --- /dev/null +++ b/server/insights/api/models_cell_admin.py @@ -0,0 +1,249 @@ +"""Closed inputs for administration of one verified cell.""" + +from __future__ import annotations + +import json +import math +import re +from decimal import Decimal +from typing import Annotated, Any, Literal +from uuid import UUID + +from pydantic import AfterValidator, BaseModel, ConfigDict, Field, field_validator, model_validator + + +def _text(value: str, maximum: int, *, empty: bool = False, controls: bool = False) -> str: + try: + size = len(value.encode("utf-8")) + except UnicodeError: + raise ValueError("Text must be UTF-8") from None + if (not empty and not size) or size > maximum or "\0" in value: + raise ValueError("Text is empty, oversized or contains NUL") + if not controls and any(ord(char) < 32 or ord(char) == 127 for char in value): + raise ValueError("Identifier contains control characters") + return value + + +def _uuid(value: str) -> str: + try: + parsed = UUID(value) + except ValueError: + raise ValueError("Expected a canonical UUID") from None + if str(parsed) != value: + raise ValueError("Expected a canonical UUID") + return value + + +def _finite_json(value: Any) -> Any: + pending = [value] + while pending: + item = pending.pop() + if item is None or type(item) in (bool, int): + continue + if type(item) is float: + if not math.isfinite(item): + raise ValueError("Setting input must contain only finite JSON values") + elif type(item) is str: + try: + item.encode("utf-8") + except UnicodeError: + raise ValueError("Setting input must be UTF-8") from None + elif type(item) is list: + pending.extend(item) + elif type(item) is dict and all(type(key) is str for key in item): + pending.extend(item.keys()) + pending.extend(item.values()) + else: + raise ValueError("Setting input must be JSON") + return value + + +Name = Annotated[str, AfterValidator(lambda value: _text(value, 256))] +ShortName = Annotated[str, AfterValidator(lambda value: _text(value, 64))] +Prefix = Annotated[str, AfterValidator(lambda value: _text(value, 256, empty=True))] +PatternID = Annotated[str, AfterValidator(_uuid)] +Reason = Annotated[str, AfterValidator(lambda value: _text(value, 1024, empty=True, controls=True))] +Nonnegative = Annotated[float, Field(ge=0, allow_inf_nan=False)] + + +class CellAdminModel(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True, hide_input_in_errors=True) + + +class CellTargetContext(CellAdminModel): + version: Literal[1] + expected_cell: Name + expected_deployment_id: Name + expected_bot: Name + + @field_validator("version", mode="before") + @classmethod + def _version(cls, value: Any) -> int: + if type(value) is not int or value != 1: + raise ValueError("Unsupported cell administration version") + return value + + +class CellActionContext(CellTargetContext): + run_id: PatternID + attempt: int = Field(ge=1, le=2**31 - 1) + command_digest: str = Field(min_length=64, max_length=64, pattern=r"^[0-9a-f]{64}$") + + @field_validator("run_id") + @classmethod + def _run_id(cls, value: str) -> str: + if UUID(value).version != 4: + raise ValueError("Run identity must be a UUIDv4") + return value + + +class CellCapabilitiesInput(CellAdminModel): + context: CellTargetContext + + +class CellActionInput(CellAdminModel): + context: CellActionContext + + @model_validator(mode="after") + def _argument_size(self) -> CellActionInput: + arguments = self.model_dump(exclude={"context"}) + try: + encoded = json.dumps(arguments, sort_keys=True, separators=(",", ":"), allow_nan=False) + except (ValueError, TypeError, OverflowError): + raise ValueError("Arguments must be finite JSON") from None + if len(encoded.encode("utf-8")) > 65536: + raise ValueError("Arguments exceed the cell administration limit") + return self + + +class CellSettingsListInput(CellActionInput): + prefix: Prefix = "" + + +class CellSettingsSetInput(CellActionInput): + name: Name + # The owning setting coerces its raw value; model validation must not change it. + value: Any = Field(default=None, repr=False) + clear: bool = False + expected: Any = Field(default=None, repr=False) + + @field_validator("value", "expected", mode="before") + @classmethod + def _json(cls, value: Any) -> Any: + return _finite_json(value) + + @model_validator(mode="after") + def _value_or_clear(self) -> CellSettingsSetInput: + if self.clear and self.value is not None: + raise ValueError("A cleared setting cannot carry a value") + if not self.clear and "value" not in self.model_fields_set: + raise ValueError("A setting change requires a value") + return self + + +class CellSettingsHistoryInput(CellActionInput): + name: Name + limit: int = Field(default=50, ge=1, le=50) + + +class CellSigningKeysListInput(CellActionInput): + pass + + +class CellSigningKeysRotateInput(CellActionInput): + algorithm: Literal["HS256", "RS256"] = "HS256" + activate_in: Nonnegative = 300.0 + + +class CellSigningKeysRetireInput(CellActionInput): + kid: Name + + +class CellSubjectRevokeInput(CellActionInput): + subject: Annotated[str, AfterValidator(lambda value: _text(value, 512))] + reason: Reason = Field(default="", repr=False) + + @field_validator("subject") + @classmethod + def _global_subject(cls, value: str) -> str: + if "/user:" in value: + raise ValueError("An org-qualified end user is not a global subject") + return value + + +class CellRatelimitNamespaceSetInput(CellActionInput): + namespace: Name + op_class: ShortName + capacity: Annotated[float, Field(gt=0, allow_inf_nan=False)] | None = None + rate: Nonnegative | None = None + clear: bool = False + + @field_validator("namespace") + @classmethod + def _namespace_label(cls, value: str) -> str: + if value == "*": + raise ValueError("A namespace-name rate override requires an exact label") + return value + + @model_validator(mode="after") + def _values_or_clear(self) -> CellRatelimitNamespaceSetInput: + if self.clear: + if self.capacity is not None or self.rate is not None: + raise ValueError("A cleared rate override cannot carry values") + elif self.capacity is None or self.rate is None: + raise ValueError("A rate override requires capacity and rate") + return self + + +class CellBudgetSetInput(CellActionInput): + op: Annotated[str, AfterValidator(lambda value: _text(value, 128))] + consumer: ShortName | None = None + budget_ms: int | None = Field(default=None, ge=0, le=86400000) + budget: str | None = Field(default=None, max_length=32) + clear: bool = False + + @field_validator("budget") + @classmethod + def _multiplier(cls, value: str | None) -> str | None: + if value is None: + return None + # The command sends a bounded canonical decimal, never an exponent or float. + if not re.fullmatch(r"(?:0|[1-9][0-9]*)(?:\.[0-9]*[1-9])?x", value): + raise ValueError("Expected a canonical budget multiplier") + if not 0 <= Decimal(value[:-1]) <= 1000: + raise ValueError("Budget multiplier is outside its permitted range") + return value + + @model_validator(mode="after") + def _value_or_clear(self) -> CellBudgetSetInput: + if self.op == "*" and self.consumer is None: + raise ValueError("An all-operation budget requires a consumer") + if self.clear: + if self.budget_ms is not None or self.budget is not None: + raise ValueError("A cleared budget cannot carry a value") + elif (self.budget_ms is None) == (self.budget is None): + raise ValueError("A budget requires exactly one value") + if self.budget is not None and self.consumer is None: + raise ValueError("A budget multiplier requires a consumer") + return self + + +class CellBlacklistGlobalAddInput(CellActionInput): + pattern_id: PatternID + pattern: Annotated[str, AfterValidator(lambda value: _text(value, 4096, controls=True))] = ( + Field(repr=False) + ) + pattern_type: Literal["substring", "regex"] = "substring" + reason: Reason = Field(default="", repr=False) + + @model_validator(mode="after") + def _new_pattern(self) -> CellBlacklistGlobalAddInput: + if UUID(self.pattern_id).version != 4: + raise ValueError("A new pattern identity must be a UUIDv4") + if self.pattern_type == "regex" and len(self.pattern) > 256: + raise ValueError("Regex pattern exceeds its character limit") + return self + + +class CellBlacklistGlobalListInput(CellActionInput): + limit: int = Field(default=50, ge=1, le=50) diff --git a/server/insights/app/admin_routes.py b/server/insights/app/admin_routes.py index 4a13c1e313..e67f2f27f0 100644 --- a/server/insights/app/admin_routes.py +++ b/server/insights/app/admin_routes.py @@ -465,66 +465,17 @@ async def admin_budget_set(input: BudgetSetInput) -> BudgetSetOutput: With `consumer`: {consumer, op | "*", budget_ms | budget: "3x"} installs for that consumer only; {consumer, op, clear: true} removes it. """ - from server.core import runtime_overrides as ro - from server.core.numbers import parse_budget_value - from server.observability.budgets import get_overrides, stored_ms + from server.insights.app.limit_changes import change_budget set_query_tag("admin.budget_set") - overrides = get_overrides() - if input.consumer: - if input.clear: - await ro.clear(ro.BUDGET_CONSUMER, (input.consumer, input.op or "*")) - overrides.clear_for_consumer(input.consumer, input.op) - return BudgetSetOutput(ok=True, cleared=True, op=input.op, consumer=input.consumer) - value: int | str | None = input.budget if input.budget is not None else input.budget_ms - if value is None: - raise HTTPException( - status_code=422, - detail={ - "code": "VALIDATION_FAILED", - "message": "budget_ms (ms) or budget ('x') required when clear=false", - }, - ) - if parse_budget_value(value) is None: - raise HTTPException( - status_code=422, - detail={ - "code": "VALIDATION_FAILED", - "message": "budget must be a non-negative integer (ms) or 'x'", - }, - ) - # Stored first: a failed write changes no process. - await ro.put(ro.BUDGET_CONSUMER, (input.consumer, input.op or "*"), value) - overrides.set_for_consumer(input.consumer, input.op, value) - return BudgetSetOutput( - ok=True, - op=input.op, + return BudgetSetOutput.model_validate( + await change_budget( + input.op, consumer=input.consumer, - budget=value, - budget_ms=value if isinstance(value, int) else None, + budget_ms=input.budget_ms, + budget=input.budget, + clear=input.clear, ) - if input.clear: - await ro.clear(ro.BUDGET_OP, (input.op,)) - overrides.clear(input.op) - return BudgetSetOutput.model_validate({"ok": True, "cleared": True, "op": input.op}) - if input.budget_ms is None: - raise HTTPException( - status_code=422, - detail={ - "code": "VALIDATION_FAILED", - "message": "budget_ms required when clear=false", - }, - ) - try: - stored_ms(input.budget_ms) # refused before it is stored - except ValueError as e: - raise HTTPException( - status_code=422, detail={"code": "VALIDATION_FAILED", "message": str(e)} - ) from e - await ro.put(ro.BUDGET_OP, (input.op,), input.budget_ms) - overrides.set(input.op, input.budget_ms) - return BudgetSetOutput.model_validate( - {"ok": True, "op": input.op, "budget_ms": input.budget_ms} ) @@ -557,20 +508,20 @@ async def admin_ratelimit_set(input: RatelimitSetInput) -> RatelimitSetOutput: from server.auth import tenancy await tenancy.ensure_held(org, None) + if ns: + from server.insights.app.limit_changes import change_namespace_rate + + return RatelimitSetOutput.model_validate( + await change_namespace_rate( + ns, + input.op_class, + capacity=input.capacity, + rate=input.rate, + clear=input.clear, + ) + ) limiter = get_limiter() if input.clear: - if ns: - await ro.clear(ro.RATELIMIT_NAMESPACE, (ns, input.op_class)) - limiter.clear_namespace_limit(ns, input.op_class) - return RatelimitSetOutput.model_validate( - { - "ok": True, - "cleared": True, - "scope": "namespace", - "key": ns, - "op_class": input.op_class, - } - ) await ro.clear(ro.RATELIMIT_ORG, (org, input.op_class)) limiter.clear_org_limit(org, input.op_class) return RatelimitSetOutput.model_validate( @@ -591,23 +542,6 @@ async def admin_ratelimit_set(input: RatelimitSetInput) -> RatelimitSetOutput: }, ) value = {"capacity": input.capacity, "rate": input.rate} - if ns: - await ro.put(ro.RATELIMIT_NAMESPACE, (ns, input.op_class), value) - limiter.set_namespace_limit( - ns, - input.op_class, - Limit(capacity=input.capacity, rate=input.rate), - ) - return RatelimitSetOutput.model_validate( - { - "ok": True, - "scope": "namespace", - "key": ns, - "op_class": input.op_class, - "capacity": input.capacity, - "rate": input.rate, - } - ) await ro.put(ro.RATELIMIT_ORG, (org, input.op_class), value) limiter.set_org_limit( org, diff --git a/server/insights/app/cell_admin_actions.py b/server/insights/app/cell_admin_actions.py new file mode 100644 index 0000000000..3fe5c41b8a --- /dev/null +++ b/server/insights/app/cell_admin_actions.py @@ -0,0 +1,142 @@ +"""Bounded cell adapters for the ordinary settings, rate and budget owners.""" + +from __future__ import annotations + +import asyncio +import json +import time +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from typing import Any + +from pydantic import BaseModel + +from server.core import settings_overrides +from server.core.ch import statement_timeout +from server.core.ch_bounded import BoundedReadError +from server.insights.api.models_cell_admin import ( + CellBudgetSetInput, + CellRatelimitNamespaceSetInput, + CellSettingsHistoryInput, + CellSettingsListInput, + CellSettingsSetInput, +) +from server.insights.app import limit_changes, settings_changes +from server.insights.app.admin_models import ( + BudgetSetOutput, + RatelimitSetOutput, + SettingChange, + SettingRow, + SettingSetOutput, + SettingsHistoryOutput, + SettingsListOutput, +) + + +@asynccontextmanager +async def _allowance(deadline: float) -> AsyncIterator[float]: + # The caller's earlier auth/audit deadline remains authoritative. + deadline = min(deadline, time.monotonic() + 30.0) + remaining = deadline - time.monotonic() + if remaining <= 0: + raise BoundedReadError("Cell action allowance exhausted") + async with asyncio.timeout_at(deadline): + with statement_timeout(min(2.0, remaining)): + yield deadline + + +def _result(model: BaseModel) -> dict[str, Any]: + result = model.model_dump(mode="json") + encoded = json.dumps( + result, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False + ) + if len(encoded.encode("utf-8")) > 2 * 1024**2: + raise BoundedReadError("Cell action result exceeds its allowance") + return result + + +async def settings_list( + input: CellSettingsListInput, + *, + deadline: float, + ch: Any = None, +) -> dict[str, Any]: + async with _allowance(deadline) as deadline: + await settings_overrides.refresh_bounded(deadline=deadline, ch=ch) + rows = settings_overrides.selected_rows(prefix=input.prefix) + return _result(SettingsListOutput(settings=[SettingRow(**row) for row in rows])) + + +async def settings_set( + input: CellSettingsSetInput, + *, + actor: str, + deadline: float, + ch: Any = None, +) -> dict[str, Any]: + async with _allowance(deadline) as deadline: + row = await settings_changes.change( + input.name, + input.value, + clear=input.clear, + expected=( + input.expected if input.expected is not None else settings_overrides.NOT_GIVEN + ), + actor=actor, + ch=ch, + deadline=deadline, + ) + return _result(SettingSetOutput(ok=True, setting=SettingRow(**row) if row else None)) + + +async def settings_history( + input: CellSettingsHistoryInput, + *, + deadline: float, + ch: Any = None, +) -> dict[str, Any]: + async with _allowance(deadline) as deadline: + rows = await settings_changes.history(input.name, input.limit, ch=ch, deadline=deadline) + return _result(SettingsHistoryOutput(changes=[SettingChange(**row) for row in rows])) + + +async def ratelimit_namespace_set( + input: CellRatelimitNamespaceSetInput, + *, + deadline: float, + ch: Any = None, +) -> dict[str, Any]: + async with _allowance(deadline) as deadline: + result = await limit_changes.change_namespace_rate( + input.namespace, + input.op_class, + capacity=input.capacity, + rate=input.rate, + clear=input.clear, + deadline=deadline, + ch=ch, + ) + return _result(RatelimitSetOutput.model_validate(result)) + + +async def budget_set( + input: CellBudgetSetInput, + *, + deadline: float, + ch: Any = None, +) -> dict[str, Any]: + from server.operations.registry import OPERATIONS_BY_NAME + + if len(OPERATIONS_BY_NAME) > 4096 or input.op != "*" and input.op not in OPERATIONS_BY_NAME: + raise BoundedReadError("Unsupported operation budget") + async with _allowance(deadline) as deadline: + result = await limit_changes.change_budget( + input.op, + consumer=input.consumer, + budget_ms=input.budget_ms, + budget=input.budget, + clear=input.clear, + deadline=deadline, + ch=ch, + ) + return _result(BudgetSetOutput.model_validate(result)) diff --git a/server/insights/app/cell_admin_auth.py b/server/insights/app/cell_admin_auth.py new file mode 100644 index 0000000000..36b4266d7d --- /dev/null +++ b/server/insights/app/cell_admin_auth.py @@ -0,0 +1,355 @@ +"""Fresh positive authority for a cell's registered automation administrator.""" + +from __future__ import annotations + +import asyncio +import json +import math +import re +import ssl +import time +from dataclasses import dataclass +from typing import Annotated, Any, cast +from urllib.parse import quote, urlsplit + +import httpx +import jwt +from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat +from fastapi import Depends, HTTPException, Request + +from server.auth import keyring, machines, revocation +from server.auth.jwt import AuthInfo +from server.core import cell +from server.core.ch import set_query_tag +from server.core.ch_bounded import BoundedReadError, decode_json +from server.insights.api.models_cell_admin import CellTargetContext +from server.insights.app.admin_routes import require_admin +from server.insights.store import cell_admin_auth as repository +from server.operations.base import Role +from server.settings import settings + + +@dataclass(frozen=True) +class AutomationBinding: + cell: str + deployment_id: str + name: str + thumbprint: str + + +@dataclass(frozen=True) +class VerifiedCellAdmin: + auth: AuthInfo + binding: AutomationBinding + + +@dataclass(frozen=True) +class RegisteredBot: + auth: AuthInfo + cell: str + deployment_id: str + name: str + thumbprint: str + + +def _refuse(code: str, status: int = 403) -> HTTPException: + return HTTPException(status_code=status, detail={"code": code}) + + +def _principal(auth: AuthInfo | None) -> AuthInfo: + if ( + not settings.auth_actually_required() + or not isinstance(auth, AuthInfo) + or auth.is_service_account is not True + or not auth.has_role(Role.PLATFORM_ADMIN) + or auth.namespaces not in ((), ("*",)) + or any( + ( + auth.ns_roles, + auth.write_namespaces, + auth.impersonated_by, + auth.read_scope, + auth.read_user_id, + auth.parents, + auth.parent_subjects, + auth.parent_iats, + auth.creator, + auth.session_id, + auth.first_party, + auth.agent_principal, + auth.saves_to, + ) + ) + ): + raise _refuse("automation_required") + if not auth.subject.startswith("sa:") or not auth.subject[3:] or not auth.token_id: + raise _refuse("automation_required") + return auth + + +async def require_cell_admin( + request: Request, + auth: Annotated[AuthInfo | None, Depends(require_admin)], +) -> AuthInfo: + """Require a bot and strict JSON before a cell action can do business work.""" + principal = _principal(auth) + # The front-door middleware caps bytes; retain raw fields for duplicate detection. + try: + decode_json((await request.body()).decode("utf-8")) + except (BoundedReadError, UnicodeError): + raise _refuse("invalid_arguments", 400) from None + return principal + + +def _security_fresh(auth: AuthInfo) -> None: + ages = (keyring.snapshot_age_seconds(), revocation.snapshot_age_seconds()) + if ( + not keyring.loaded() + or not revocation.loaded() + or any( + not math.isfinite(age) or age < 0 or age > settings.jwt_revocation_max_stale_seconds + for age in ages + ) + ): + raise _refuse("security_unavailable", 503) + now = time.time() + if ( + type(auth.issued_at) is not int + or type(auth.expires_at) is not int + or not 0 < auth.expires_at - auth.issued_at <= 60 + or auth.issued_at > now + 30 + or not 35 <= auth.expires_at - now <= 60 + ): + raise _refuse("short_token_required") + + +def _live_cell() -> tuple[str, str]: + if not settings.cell or not cell.rows_read() or cell.mode() != "enforced" or cell.gate_reason(): + raise _refuse("cell_unavailable", 503) + return settings.cell, cell.deployment_id() + + +def _accepted(context: CellTargetContext) -> None: + if (context.expected_cell, context.expected_deployment_id) != _live_cell(): + raise _refuse("cell_identity_changed") + + +def _router_origin() -> str: + raw = settings.cell_router_url + try: + parsed = urlsplit(raw) + valid = ( + parsed.scheme == "https" + and parsed.hostname + and parsed.port != 0 + and parsed.username is None + and parsed.password is None + and not parsed.query + and not parsed.fragment + and parsed.path in ("", "/") + and raw == raw.strip() + and "\\" not in raw + and not any(ord(char) < 33 or ord(char) == 127 for char in raw) + ) + except ValueError: + valid = False + if not valid or not settings.cell_router_token: + raise _refuse("automation_unprovisioned", 503) + return f"https://{parsed.netloc}" + + +def _pair(value: Any) -> tuple[str, str]: + if not isinstance(value, dict) or set(value) != {"name", "thumbprint"}: + raise BoundedReadError("Invalid automation metadata") + name, thumbprint = value["name"], value["thumbprint"] + if ( + not isinstance(name, str) + or not name + or len(name.encode("utf-8")) > 256 + or any(ord(char) < 32 or ord(char) == 127 for char in name) + or not isinstance(thumbprint, str) + or re.fullmatch(r"[A-Za-z0-9_-]{43}", thumbprint) is None + ): + raise BoundedReadError("Invalid automation metadata") + return name, thumbprint + + +def _binding(payload: bytes, context: CellTargetContext) -> AutomationBinding: + value = decode_json(payload.decode("utf-8")) + if ( + not isinstance(value, dict) + or set(value) != {"version", "cell", "deployment_id", "automation", "recovery"} + or type(value["version"]) is not int + or value["version"] != 1 + ): + raise BoundedReadError("Invalid automation metadata") + automation, recovery = _pair(value["automation"]), _pair(value["recovery"]) + if automation[0] == recovery[0] or automation[1] == recovery[1]: + raise BoundedReadError("Automation and recovery must be distinct") + if (value["cell"], value["deployment_id"], automation[0]) != ( + context.expected_cell, + context.expected_deployment_id, + context.expected_bot, + ): + raise _refuse("automation_identity_changed") + return AutomationBinding(value["cell"], value["deployment_id"], *automation) + + +async def _authority(context: CellTargetContext, deadline: float) -> AutomationBinding: + origin = _router_origin() + remaining = min(2.0, deadline - time.monotonic()) + if remaining <= 0: + raise _refuse("security_unavailable", 503) + headers = { + "Authorization": f"Bearer {settings.cell_router_token}", + "Accept-Encoding": "identity", + } + url = f"{origin}/cells/v1/cells/{quote(settings.cell, safe='')}/administration" + async with asyncio.timeout(remaining): + async with httpx.AsyncClient( + verify=ssl.create_default_context(), + trust_env=False, + follow_redirects=False, + timeout=remaining, + ) as client: + async with client.stream("GET", url, headers=headers) as response: + if ( + response.status_code != 200 + or response.headers.get("content-encoding", "identity").lower() != "identity" + ): + raise _refuse("automation_unprovisioned", 503) + data = bytearray() + async for chunk in response.aiter_raw(): + if len(data) + len(chunk) > 8192: + raise BoundedReadError("Automation metadata is oversized") + data.extend(chunk) + return _binding(bytes(data), context) + + +def _public_thumbprint(raw: Any) -> str: + if not isinstance(raw, str): + raise _refuse("automation_key_changed") + parsed = decode_json(raw) + if not isinstance(parsed, dict) or set(parsed) != {"keys"}: + raise _refuse("automation_key_changed") + keys = parsed["keys"] + if not isinstance(keys, list) or len(keys) != 1 or not isinstance(keys[0], dict): + raise _refuse("automation_key_changed") + if set(keys[0]) - {"kty", "crv", "x", "y", "n", "e", "alg", "use", "kid"}: + raise _refuse("automation_key_changed") + key = jwt.PyJWKSet.from_json(raw).keys[0] + public = key.key.public_bytes(Encoding.PEM, PublicFormat.SubjectPublicKeyInfo) + canonical = decode_json(machines.public_jwks(public))["keys"][0] + if keys[0] != canonical: + raise _refuse("automation_key_changed") + return str(canonical["kid"]) + + +def _registered(row: dict[str, Any] | None, name: str, auth: AuthInfo) -> str: + if row is None or ( + row.get("name") != name + or row.get("org") != auth.org + or type(row.get("active")) is not int + or row["active"] != 1 + or type(row.get("platform_admin")) is not int + or row["platform_admin"] != 1 + or row.get("claims") != {} + or any(row.get(field) != "" for field in ("issuer", "audience", "agent_claim", "jwks_uri")) + ): + raise _refuse("automation_required") + maximum = row.get("max_access_token_s") + if row.get("ns_roles") != [] or type(maximum) is not int or not 60 <= maximum <= 86400: + raise _refuse("automation_required") + return _public_thumbprint(row.get("public_keys")) + + +def _revoked(rows: list[dict[str, Any]], issued_at: int) -> bool: + for row in rows: + outright, cutoff = row.get("outright"), row.get("cutoff") + if type(outright) is not int or outright not in (0, 1) or type(cutoff) is not int: + raise BoundedReadError("Malformed revocation") + if outright or revocation.kills(cutoff, issued_at): + return True + return False + + +async def verify_registered_bot( + auth: AuthInfo | None, *, deadline: float, ch: Any = None +) -> RegisteredBot: + """Prove only the caller's live sole-key registration, without assigning a purpose.""" + auth = _principal(auth) + set_query_tag("cell.admin.security") + _security_fresh(auth) + identity = _live_cell() + name = auth.subject[3:] + try: + row = await repository.machine(name, deadline=deadline, ch=ch) + thumbprint = _registered(row, name, auth) + keys = sorted( + { + key + for subject in (auth.subject, auth.token_id) + for key in ( + subject, + revocation.CUTOFF_PREFIX + subject, + revocation.SCIM_PREFIX + subject, + ) + } + ) + rows = await repository.revocations(keys, deadline=deadline, ch=ch) + if _revoked(rows, cast(int, auth.issued_at)): + raise _refuse("automation_revoked") + except ( + BoundedReadError, + UnicodeError, + ValueError, + TypeError, + AttributeError, + jwt.PyJWTError, + httpx.HTTPError, + TimeoutError, + ): + raise _refuse("security_unavailable", 503) from None + _security_fresh(auth) + if _live_cell() != identity: + raise _refuse("cell_identity_changed") + return RegisteredBot(auth, *identity, name, thumbprint) + + +async def registered_bot_identity(auth: AuthInfo | None) -> dict[str, Any]: + """The caller's bounded public proof; it grants no automation or recovery purpose.""" + # not a setting: bot proof shares the fixed thirty-second cell action envelope + async with asyncio.timeout(30): + bot = await verify_registered_bot(auth, deadline=time.monotonic() + 30) + answer = { + "version": 1, + "cell": bot.cell, + "deployment_id": bot.deployment_id, + "name": bot.name, + "org": bot.auth.org, + "thumbprint": bot.thumbprint, + "active": True, + "platform_admin": True, + } + if len(json.dumps(answer, allow_nan=False).encode()) > 2048: + raise _refuse("security_unavailable", 503) + return answer + + +async def verify_automation( + auth: AuthInfo, context: CellTargetContext, *, deadline: float, ch: Any = None +) -> VerifiedCellAdmin: + """Freshly bind this request to accepted cell metadata and its sole bot key.""" + set_query_tag("cell.admin.security") + auth = _principal(auth) + _security_fresh(auth) + _accepted(context) + try: + binding = await _authority(context, deadline) + except (BoundedReadError, UnicodeError, ValueError, httpx.HTTPError, TimeoutError): + raise _refuse("security_unavailable", 503) from None + bot = await verify_registered_bot(auth, deadline=deadline, ch=ch) + if bot.name != binding.name or bot.thumbprint != binding.thumbprint: + raise _refuse("automation_identity_changed") + _accepted(context) + return VerifiedCellAdmin(auth, binding) diff --git a/server/insights/app/cell_admin_routes.py b/server/insights/app/cell_admin_routes.py new file mode 100644 index 0000000000..6c4ce66d46 --- /dev/null +++ b/server/insights/app/cell_admin_routes.py @@ -0,0 +1,321 @@ +"""The authenticated capabilities of this cell's administration surface.""" + +from __future__ import annotations + +import asyncio +import json +import time +from collections.abc import Awaitable, Callable +from typing import Annotated, Any + +from fastapi import APIRouter, Depends, HTTPException, Request, Response + +from server.auth.context import current_auth_var +from server.auth.jwt import AuthInfo +from server.core.ch import set_query_tag, statement_timeout +from server.core.ch_bounded import BoundedReadError +from server.core.hashing import sha256_value +from server.insights.api.models_cell_admin import ( + CellActionInput, + CellBlacklistGlobalAddInput, + CellBlacklistGlobalListInput, + CellBudgetSetInput, + CellCapabilitiesInput, + CellRatelimitNamespaceSetInput, + CellSettingsHistoryInput, + CellSettingsListInput, + CellSettingsSetInput, + CellSigningKeysListInput, + CellSigningKeysRetireInput, + CellSigningKeysRotateInput, + CellSubjectRevokeInput, +) +from server.insights.app import cell_admin_actions as actions +from server.insights.app import cell_admin_security_actions as security_actions +from server.insights.app.admin_routes import refuse_browser_reach +from server.insights.app.cell_admin_auth import require_cell_admin, verify_automation +from server.observability import audit + +cell_admin_router = APIRouter( + prefix="/v1/admin", tags=["admin"], dependencies=[Depends(require_cell_admin)] +) + +AVAILABLE_ACTIONS = ( + "settings_list", + "settings_set", + "settings_history", + "ratelimit_namespace_set", + "budget_set", + "signing_keys_list", + "signing_keys_rotate", + "signing_keys_retire", + "subject_revoke", + "blacklist_global_add", + "blacklist_global_list", +) +_REFUSALS = frozenset( + { + "automation_required", + "security_unavailable", + "short_token_required", + "cell_unavailable", + "cell_identity_changed", + "automation_unprovisioned", + "automation_identity_changed", + "automation_key_changed", + "automation_revoked", + "invalid_arguments", + "action_unavailable", + } +) + + +def _encode(value: Any, maximum: int) -> bytes: + raw = json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False).encode() + if len(raw) > maximum: + raise BoundedReadError("Cell administration response exceeds its allowance") + return raw + + +def _audit_subject(subject: dict[str, Any]) -> None: + # audit_row uses this ASCII JSON representation, including its spaces. + if len(json.dumps(subject, sort_keys=True, default=str).encode()) > 2048: + raise BoundedReadError("Cell administration audit exceeds its allowance") + + +def _intent(body: CellActionInput, action: str) -> None: + arguments = body.model_dump(mode="json", exclude={"context"}) + digest = sha256_value({"action": action, "arguments": arguments}) + if digest != body.context.command_digest: + raise HTTPException(422, detail={"code": "invalid_arguments"}) + + +async def _audit(event: str, actor: str, subject: dict[str, Any], deadline: float) -> str: + _audit_subject(subject) + remaining = min(2.0, deadline - time.monotonic()) + if remaining <= 0: + raise BoundedReadError("Cell administration allowance exhausted") + with statement_timeout(remaining): + row_hash = await audit.emit_strict( + event, org="-", namespace="-", actor=actor, subject=subject + ) + if not row_hash: + raise BoundedReadError("Cell administration audit was not confirmed") + return row_hash + + +def _refusal(body: CellActionInput, action: str, error: Exception) -> Response: + code = ( + error.detail.get("code") + if isinstance(error, HTTPException) and isinstance(error.detail, dict) + else "security_unavailable" + ) + if not isinstance(code, str) or code not in _REFUSALS: + code = "invalid_arguments" + value = { + "version": 1, + "status": "refused", + "phase": "before_business", + "context": body.context.model_dump(), + "action": action, + "code": code, + } + status = error.status_code if isinstance(error, HTTPException) else 503 + return Response(_encode(value, 2048), status_code=status, media_type="application/json") + + +async def _perform( + request: Request, + body: CellActionInput, + auth: AuthInfo, + action: str, + handler: Callable[..., Awaitable[dict[str, Any]]], +) -> Response: + deadline = time.monotonic() + 30.0 + subject = {"context": body.context.model_dump(), "action": action} + token = current_auth_var.set(auth) + try: + async with asyncio.timeout_at(deadline): + try: + refuse_browser_reach(request, "cell administration") + _intent(body, action) + verified = await verify_automation(auth, body.context, deadline=deadline) + set_query_tag("cell.each." + action) + _audit_subject({**subject, "started_audit": "0" * 64, "result_digest": "0" * 64}) + started = await _audit( + "cell.each_started", verified.auth.subject, subject, deadline + ) + except Exception as error: # noqa: BLE001 — only this pre-business block may certify refusal + return _refusal(body, action, error) + options: dict[str, Any] = {"deadline": deadline} + if action == "settings_set": + options["actor"] = verified.auth.subject + # Every failure from this invocation onward remains unsettled to the client. + result = await handler(body, **options) + _encode(result, 2 * 1024**2) + digest = sha256_value(result) + completed = await _audit( + "cell.each_completed", + verified.auth.subject, + {**subject, "started_audit": started, "result_digest": digest}, + deadline, + ) + metadata = { + "version": 1, + "status": "completed", + "context": body.context.model_dump(), + "action": action, + "started_audit": started, + "completed_audit": completed, + } + _encode(metadata, 2048) + return Response( + _encode({**metadata, "result": result}, 2 * 1024**2), + media_type="application/json", + ) + finally: + current_auth_var.reset(token) + + +@cell_admin_router.post("/cell.capabilities") +async def cell_capabilities( + body: CellCapabilitiesInput, + auth: Annotated[AuthInfo, Depends(require_cell_admin)], +) -> Response: + """Report this verified cell's supported administration actions and fixed limits.""" + deadline = time.monotonic() + 30.0 + async with asyncio.timeout_at(deadline): + await verify_automation(auth, body.context, deadline=deadline) + value = { + "version": 1, + "context": body.context.model_dump(), + "actions": list(AVAILABLE_ACTIONS), + "limits": { + "action_timeout_s": 30, + "max_body_bytes": 65536, + "max_response_bytes": 2097152, + }, + } + return Response(_encode(value, 8192), media_type="application/json") + + +@cell_admin_router.post("/cell.settings_list") +async def cell_settings_list( + request: Request, + body: CellSettingsListInput, + auth: Annotated[AuthInfo, Depends(require_cell_admin)], +) -> Response: + """List non-secret settings after fresh cell and automation authorization.""" + return await _perform(request, body, auth, "settings_list", actions.settings_list) + + +@cell_admin_router.post("/cell.settings_set") +async def cell_settings_set( + request: Request, + body: CellSettingsSetInput, + auth: Annotated[AuthInfo, Depends(require_cell_admin)], +) -> Response: + """Change one live setting and confirm its completed cell administration audits.""" + return await _perform(request, body, auth, "settings_set", actions.settings_set) + + +@cell_admin_router.post("/cell.settings_history") +async def cell_settings_history( + request: Request, + body: CellSettingsHistoryInput, + auth: Annotated[AuthInfo, Depends(require_cell_admin)], +) -> Response: + """Read the bounded change history of one live setting in this verified cell.""" + return await _perform(request, body, auth, "settings_history", actions.settings_history) + + +@cell_admin_router.post("/cell.ratelimit_namespace_set") +async def cell_ratelimit_namespace_set( + request: Request, + body: CellRatelimitNamespaceSetInput, + auth: Annotated[AuthInfo, Depends(require_cell_admin)], +) -> Response: + """Change a namespace-label rate override through the cell's ordinary limiter.""" + return await _perform( + request, body, auth, "ratelimit_namespace_set", actions.ratelimit_namespace_set + ) + + +@cell_admin_router.post("/cell.budget_set") +async def cell_budget_set( + request: Request, + body: CellBudgetSetInput, + auth: Annotated[AuthInfo, Depends(require_cell_admin)], +) -> Response: + """Change one latency budget through this verified cell's ordinary budget owner.""" + return await _perform(request, body, auth, "budget_set", actions.budget_set) + + +@cell_admin_router.post("/cell.signing_keys_list") +async def cell_signing_keys_list( + request: Request, + body: CellSigningKeysListInput, + auth: Annotated[AuthInfo, Depends(require_cell_admin)], +) -> Response: + """List public signing-key metadata after fresh cell and automation authorization.""" + return await _perform( + request, body, auth, "signing_keys_list", security_actions.signing_keys_list + ) + + +@cell_admin_router.post("/cell.signing_keys_rotate") +async def cell_signing_keys_rotate( + request: Request, + body: CellSigningKeysRotateInput, + auth: Annotated[AuthInfo, Depends(require_cell_admin)], +) -> Response: + """Rotate a signing key through the ordinary cell owner and return public metadata.""" + return await _perform( + request, body, auth, "signing_keys_rotate", security_actions.signing_keys_rotate + ) + + +@cell_admin_router.post("/cell.signing_keys_retire") +async def cell_signing_keys_retire( + request: Request, + body: CellSigningKeysRetireInput, + auth: Annotated[AuthInfo, Depends(require_cell_admin)], +) -> Response: + """Retire one signing key through this verified cell's ordinary retirement checks.""" + return await _perform( + request, body, auth, "signing_keys_retire", security_actions.signing_keys_retire + ) + + +@cell_admin_router.post("/cell.blacklist_global_add") +async def cell_blacklist_global_add( + request: Request, + body: CellBlacklistGlobalAddInput, + auth: Annotated[AuthInfo, Depends(require_cell_admin)], +) -> Response: + """Add one exact global blacklist pattern with the verified bot as its author.""" + return await _perform( + request, body, auth, "blacklist_global_add", security_actions.blacklist_global_add + ) + + +@cell_admin_router.post("/cell.subject_revoke") +async def cell_subject_revoke( + request: Request, + body: CellSubjectRevokeInput, + auth: Annotated[AuthInfo, Depends(require_cell_admin)], +) -> Response: + """Revoke one exact global subject through the bounded ordinary security owner.""" + return await _perform(request, body, auth, "subject_revoke", security_actions.subject_revoke) + + +@cell_admin_router.post("/cell.blacklist_global_list") +async def cell_blacklist_global_list( + request: Request, + body: CellBlacklistGlobalListInput, + auth: Annotated[AuthInfo, Depends(require_cell_admin)], +) -> Response: + """List bounded exact-global blacklist rows and report whether more rows exist.""" + return await _perform( + request, body, auth, "blacklist_global_list", security_actions.blacklist_global_list + ) diff --git a/server/insights/app/cell_admin_security_actions.py b/server/insights/app/cell_admin_security_actions.py new file mode 100644 index 0000000000..2933a8fc7e --- /dev/null +++ b/server/insights/app/cell_admin_security_actions.py @@ -0,0 +1,202 @@ +"""Cell adapters for signing, revocation and global refusal policy owners.""" + +from __future__ import annotations + +import asyncio +import time +from typing import Any + +from fastapi import HTTPException + +from server.auth import keyring, subject_revocation +from server.auth.context import current_auth +from server.core.ch import statement_timeout +from server.core.ch_bounded import BoundedReadError +from server.core.repo_base import _default_client +from server.insights.api.models_cell_admin import ( + CellBlacklistGlobalAddInput, + CellBlacklistGlobalListInput, + CellSigningKeysListInput, + CellSigningKeysRetireInput, + CellSigningKeysRotateInput, + CellSubjectRevokeInput, +) +from server.insights.app.cell_admin_actions import _allowance +from server.memory.app import blacklist +from server.observability import audit + + +async def signing_keys_list( + input: CellSigningKeysListInput, + *, + deadline: float, + ch: Any = None, +) -> dict[str, Any]: + async with _allowance(deadline) as deadline: + rows = await keyring.listing_bounded(ch=ch or _default_client(), deadline=deadline) + return { + "keys": [ + { + "kid": row["kid"], + "algorithm": row["algorithm"], + "created_at": row["created_at"].isoformat(), + "activate_at": row["activate_at"].isoformat(), + "retired_at": row["retired_at"].isoformat() + if row["retired_at"] is not None + else None, + } + for row in rows + ] + } + + +async def signing_keys_rotate( + input: CellSigningKeysRotateInput, + *, + deadline: float, + ch: Any = None, +) -> dict[str, Any]: + async with _allowance(deadline) as deadline: + key = await keyring.rotate_bounded( + input.algorithm, + activate_in_s=input.activate_in, + ch=ch or _default_client(), + deadline=deadline, + ) + return { + "kid": key.kid, + "algorithm": key.algorithm, + "activate_at": key.activate_at.isoformat(), + } + + +async def signing_keys_retire( + input: CellSigningKeysRetireInput, + *, + deadline: float, + ch: Any = None, +) -> dict[str, Any]: + async with _allowance(deadline) as deadline: + retired = await keyring.retire_bounded( + input.kid, + ch=ch or _default_client(), + deadline=deadline, + ) + return {"kid": input.kid, "retired": retired} + + +async def blacklist_global_list( + input: CellBlacklistGlobalListInput, + *, + deadline: float, + ch: Any = None, +) -> dict[str, Any]: + async with _allowance(deadline) as deadline: + return await blacklist.list_global_bounded( + input.limit, + ch=ch or _default_client(), + deadline=deadline, + ) + + +async def blacklist_global_add( + input: CellBlacklistGlobalAddInput, + *, + deadline: float, + ch: Any = None, +) -> dict[str, Any]: + async with _allowance(deadline) as deadline: + result = await blacklist.add_global_bounded( + input.pattern_id, + input.pattern, + input.pattern_type, + input.reason, + ch=ch or _default_client(), + deadline=deadline, + ) + await _blacklist_audit( + "blacklist.add", + input.pattern_id, + input.context.command_digest, + deadline, + ) + return {"pattern_id": result.pattern_id, "created_at": result.created_at.isoformat()} + + +async def _blacklist_audit(event: str, pattern_id: str, digest: str, deadline: float) -> None: + auth = current_auth() + remaining = min(2.0, deadline - time.monotonic()) + if auth is None or remaining <= 0: + raise BoundedReadError("Blacklist audit allowance exhausted") + async with asyncio.timeout(remaining): + with statement_timeout(remaining): + confirmed = await audit.emit_strict( + event, + org="-", + namespace="-", + actor=auth.subject, + subject={ + "pattern_id": pattern_id, + "scope": "global", + "command_digest": digest, + "confirmed": True, + }, + ) + if not confirmed: + raise BoundedReadError("Blacklist domain audit was not confirmed") + + +async def subject_revoke( + input: CellSubjectRevokeInput, + *, + deadline: float, + ch: Any = None, +) -> dict[str, Any]: + auth = current_auth() + if auth is None or subject_revocation.refusal(input.subject) is not None: + raise BoundedReadError("An authenticated outright subject action is required") + progress: dict[str, Any] = {"recorded": False, "sessions": 0, "tokens": 0} + async with _allowance(deadline) as deadline: + client = ch or _default_client() + try: + row, kept = await subject_revocation.record( + input.subject, + by=auth.subject, + reason=input.reason, + deadline=deadline, + ch=client, + progress=progress, + ) + result = await subject_revocation.finish( + input.subject, + row, + kept, + by=auth.subject, + org="-", + deadline=deadline, + ch=client, + progress=progress, + ) + if not result.outright or result.cutoff is not None: + raise BoundedReadError("The outright subject revocation was not confirmed") + except Exception as error: # Confirmed partial facts must never become success. + if not progress["recorded"]: + raise + raise HTTPException( + status_code=503, + detail={ + "code": "SUBJECT_REVOCATION_INCOMPLETE", + "recorded": True, + "sessions_at_least": progress["sessions"], + "tokens_at_least": progress["tokens"], + }, + ) from error + return { + "subject": input.subject.strip(), + "recorded": True, + "already_revoked": result.kept, + "outright": result.outright, + "cutoff": result.cutoff.isoformat() if result.cutoff else None, + "sessions": result.sessions, + "tokens": result.tokens, + } diff --git a/server/insights/app/limit_changes.py b/server/insights/app/limit_changes.py new file mode 100644 index 0000000000..82775e5d50 --- /dev/null +++ b/server/insights/app/limit_changes.py @@ -0,0 +1,135 @@ +"""Stored-first changes to namespace rates and operation latency budgets.""" + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import HTTPException + +from server.core import runtime_overrides as ro +from server.core.numbers import parse_budget_value +from server.core.ratelimit import Limit, get_limiter +from server.observability.budgets import get_overrides, stored_ms + +log = logging.getLogger(__name__) + + +async def change_namespace_rate( + namespace: str, + op_class: str, + *, + capacity: float | None, + rate: float | None, + clear: bool, + deadline: float | None = None, + ch: Any = None, +) -> dict[str, Any]: + limiter = get_limiter() + bounded: dict[str, Any] = {} if deadline is None else {"deadline": deadline, "ch": ch} + if deadline is not None: + limiter.check_bounded_namespace_change(namespace, op_class, clear=clear) + result: dict[str, Any] = { + "ok": True, + "scope": "namespace", + "key": namespace, + "op_class": op_class, + } + if clear: + await ro.clear(ro.RATELIMIT_NAMESPACE, (namespace, op_class), **bounded) + if deadline is not None: + limiter.check_bounded_namespace_change(namespace, op_class, clear=True) + limiter.clear_namespace_limit(namespace, op_class) + return {**result, "cleared": True} + if capacity is None or rate is None: + raise HTTPException( + status_code=422, + detail={ + "code": "VALIDATION_FAILED", + "message": "capacity and rate required when clear=false", + }, + ) + await ro.put( + ro.RATELIMIT_NAMESPACE, + (namespace, op_class), + {"capacity": capacity, "rate": rate}, + **bounded, + ) + if deadline is not None: + limiter.check_bounded_namespace_change(namespace, op_class, clear=False) + limiter.set_namespace_limit(namespace, op_class, Limit(capacity=capacity, rate=rate)) + return {**result, "capacity": capacity, "rate": rate} + + +async def change_budget( + op: str, + *, + consumer: str | None, + budget_ms: int | None, + budget: int | str | None, + clear: bool, + deadline: float | None = None, + ch: Any = None, +) -> dict[str, Any]: + overrides = get_overrides() + bounded: dict[str, Any] = {} if deadline is None else {"deadline": deadline, "ch": ch} + adoption: dict[str, Any] = {} if deadline is None else {"bounded": True} + if deadline is not None: + overrides.check_bounded_change(op, consumer=consumer, clear=clear) + if consumer: + if clear: + await ro.clear(ro.BUDGET_CONSUMER, (consumer, op or "*"), **bounded) + overrides.clear_for_consumer(consumer, op, **adoption) + return {"ok": True, "cleared": True, "op": op, "consumer": consumer} + value = budget if budget is not None else budget_ms + if value is None: + raise HTTPException( + status_code=422, + detail={ + "code": "VALIDATION_FAILED", + "message": "budget_ms (ms) or budget ('x') required when clear=false", + }, + ) + if parse_budget_value(value) is None: + raise HTTPException( + status_code=422, + detail={ + "code": "VALIDATION_FAILED", + "message": "budget must be a non-negative integer (ms) or 'x'", + }, + ) + await ro.put(ro.BUDGET_CONSUMER, (consumer, op or "*"), value, **bounded) + overrides.set_for_consumer(consumer, op, value, **adoption) + return { + "ok": True, + "op": op, + "consumer": consumer, + "budget": value, + "budget_ms": value if isinstance(value, int) else None, + } + if clear: + await ro.clear(ro.BUDGET_OP, (op,), **bounded) + overrides.clear(op, **adoption) + return {"ok": True, "cleared": True, "op": op} + if budget_ms is None: + raise HTTPException( + status_code=422, + detail={ + "code": "VALIDATION_FAILED", + "message": "budget_ms required when clear=false", + }, + ) + try: + stored_ms(budget_ms) + except ValueError as error: + log.info("budget change validation refused: %s", error) + raise HTTPException( + status_code=422, + detail={ + "code": "VALIDATION_FAILED", + "message": "budget_ms must be a whole number of ms >= 0", + }, + ) from error + await ro.put(ro.BUDGET_OP, (op,), budget_ms, **bounded) + overrides.set(op, budget_ms, **adoption) + return {"ok": True, "op": op, "budget_ms": budget_ms} diff --git a/server/insights/app/settings_changes.py b/server/insights/app/settings_changes.py index 6280ff770e..aeb9b2c7ce 100644 --- a/server/insights/app/settings_changes.py +++ b/server/insights/app/settings_changes.py @@ -5,10 +5,14 @@ from __future__ import annotations +import asyncio import json +import time from typing import Any from server.core import settings_overrides +from server.core.ch import statement_timeout +from server.core.ch_bounded import BoundedReadError, decode_json from server.core.repo_base import _CHClient from server.insights.store import admin as admin_repo from server.observability import audit @@ -24,37 +28,68 @@ async def change( expected: Any = settings_overrides.NOT_GIVEN, actor: str, ch: _CHClient | None = None, + deadline: float | None = None, ) -> dict[str, Any] | None: """`settings_overrides.change`, audited; returns the setting's row after.""" - before, row = await settings_overrides.change( - name, value, clear=clear, expected=expected, ch=ch - ) - await audit.emit( - EVENT, - org="-", - namespace="-", - actor=actor, - subject={ - "name": name, - "value": row["value"] if row else None, - "previous": before["value"] if before else None, - # Where `previous` came from: an override to set back, or the value of the - # process that made the change (an env or a default), which a reset restores. - # None for a setting this build lacks (the reset of a retired one's override). - "previous_source": before["source"] if before else None, - "cleared": clear, - }, - ) + if deadline is None: + before, row = await settings_overrides.change( + name, value, clear=clear, expected=expected, ch=ch + ) + else: + before, row = await settings_overrides.change_bounded( + name, + value, + clear=clear, + expected=expected, + ch=ch, + deadline=deadline, + ) + subject = { + "name": name, + "value": row["value"] if row else None, + "previous": before["value"] if before else None, + "previous_source": before["source"] if before else None, + "cleared": clear, + } + if deadline is None: + await audit.emit(EVENT, org="-", namespace="-", actor=actor, subject=subject) + else: + remaining = min(2.0, deadline - time.monotonic()) + if remaining <= 0: + raise BoundedReadError("Setting audit allowance exhausted") + async with asyncio.timeout(remaining): + with statement_timeout(remaining): + await audit.emit(EVENT, org="-", namespace="-", actor=actor, subject=subject) return row async def history( - name: str = "", limit: int = 50, *, ch: _CHClient | None = None + name: str = "", + limit: int = 50, + *, + ch: _CHClient | None = None, + deadline: float | None = None, ) -> list[dict[str, Any]]: """The recorded changes of setting `name` ("" = every setting), newest first.""" + if deadline is None: + rows = await admin_repo.setting_changes(EVENT, name, limit, ch=ch) + else: + settings_overrides.check_selected(name) + rows = await admin_repo.setting_changes_bounded( + EVENT, name, limit, ch=ch, deadline=deadline + ) out = [] - for r in await admin_repo.setting_changes(EVENT, name, limit, ch=ch): - subject = json.loads(r["subject"]) + for r in rows: + subject = json.loads(r["subject"]) if deadline is None else decode_json(r["subject"]) + if deadline is not None and ( + not isinstance(subject, dict) + or set(subject) != {"name", "value", "previous", "previous_source", "cleared"} + or subject["name"] != name + or type(subject["cleared"]) is not bool + or subject["previous_source"] is not None + and type(subject["previous_source"]) is not str + ): + raise BoundedReadError("Malformed setting-history subject") out.append( { "ts": r["ts"], diff --git a/server/insights/store/admin.py b/server/insights/store/admin.py index ff7145cc57..9e1359b0a0 100644 --- a/server/insights/store/admin.py +++ b/server/insights/store/admin.py @@ -6,8 +6,10 @@ from __future__ import annotations +from datetime import UTC, datetime, timedelta from typing import Any +from server.core.ch_bounded import BoundedReadError, read_rows from server.core.ch_names import SCHEMA, database, logical, telemetry_database from server.core.event_source import ALL_EVENTS from server.core.naming import not_mirror_sql @@ -94,6 +96,45 @@ async def setting_changes( ) +async def setting_changes_bounded( + event_type: str, + name: str, + limit: int, + *, + deadline: float, + ch: _CHClient | None = None, +) -> list[dict[str, Any]]: + """The requested newest setting changes, bounded before their subject JSON decodes.""" + if not name or not 1 <= limit <= 50: + raise BoundedReadError("Unsupported setting-history selection") + rows = await read_rows( + "SELECT toUnixTimestamp64Milli(ts) AS ts_ms, actor, subject FROM loom.audit_log " + "WHERE org = '-' AND namespace = '-' AND event_type = {event:String} " + "AND JSONExtractString(subject, 'name') = {name:String} " + "ORDER BY ts DESC LIMIT {limit:UInt32} FORMAT JSONEachRow", + {"event": event_type, "name": name, "limit": limit}, + ch=ch or _default_client(), + deadline=deadline, + row_bytes=16384, + total_bytes=512 * 1024, + max_rows=limit, + tag="cell.each.setting_history", + ) + for row in rows: + if ( + set(row) != {"ts_ms", "actor", "subject"} + or type(row["ts_ms"]) is not int + or type(row["actor"]) is not str + or type(row["subject"]) is not str + ): + raise BoundedReadError("Malformed setting-history row") + try: + row["ts"] = datetime(1970, 1, 1, tzinfo=UTC) + timedelta(milliseconds=row.pop("ts_ms")) + except OverflowError: + raise BoundedReadError("Invalid setting-history timestamp") from None + return rows + + async def events_tail( since_seconds: int, limit: int, diff --git a/server/insights/store/cell_admin_auth.py b/server/insights/store/cell_admin_auth.py new file mode 100644 index 0000000000..b92c459829 --- /dev/null +++ b/server/insights/store/cell_admin_auth.py @@ -0,0 +1,67 @@ +"""Bounded point reads for one cell administrator's security state.""" + +from __future__ import annotations + +from collections import Counter +from typing import Any + +from server.core.ch import get_client +from server.core.ch_bounded import BoundedReadError, read_rows + + +async def machine(name: str, *, deadline: float, ch: Any = None) -> dict[str, Any] | None: + """Read only the named machine, including disabled state; never enumerate bots.""" + if not name or len(name.encode("utf-8")) > 256 or "\0" in name: + raise BoundedReadError("Invalid machine name") + rows = await read_rows( + "SELECT name, org, ns_roles, public_keys, issuer, audience, claims, agent_claim, " + "jwks_uri, max_access_token_s, platform_admin, active " + "FROM loom.machine_identities FINAL WHERE name = {name:String} " + "LIMIT 2 FORMAT JSONEachRow", + {"name": name}, + deadline=deadline, + row_bytes=64 * 1024, + total_bytes=128 * 1024, + max_rows=1, + tag="cell.admin.machine", + ch=ch or get_client(), + ) + return rows[0] if rows else None + + +async def revocations(keys: list[str], *, deadline: float, ch: Any = None) -> list[dict[str, Any]]: + """Read the complete finite token/subject keys, including owed revocations.""" + if ( + not keys + or len(keys) != len(set(keys)) + or len(keys) > 16 + or any(not key or len(key.encode("utf-8")) > 1024 for key in keys) + ): + raise BoundedReadError("Invalid revocation key set") + rows = await read_rows( + "SELECT jti, outright, cutoff FROM (" + "SELECT jti, isNull(issued_before) AS outright, " + "toInt64(ifNull(toUnixTimestamp(issued_before), 0)) AS cutoff " + "FROM loom.revoked_tokens FINAL " + "WHERE jti IN {keys:Array(String)} AND expires_at > now() " + "UNION ALL SELECT id AS jti, toUInt8(1) AS outright, toInt64(0) AS cutoff " + "FROM loom.oauth_state FINAL WHERE record_type = 'revocation_owed' " + "AND id IN {keys:Array(String)} AND expires_at > toUInt32(now())" + ") LIMIT 65 FORMAT JSONEachRow", + {"keys": keys}, + deadline=deadline, + row_bytes=1024, + total_bytes=64 * 1024, + max_rows=64, + tag="cell.admin.revocations", + ch=ch or get_client(), + ) + counts: Counter[str] = Counter() + for row in rows: + key = row.get("jti") + if not isinstance(key, str) or key not in keys: + raise BoundedReadError("Unexpected revocation key") + counts[key] += 1 + if counts[key] > 4: + raise BoundedReadError("Ambiguous revocation multiplicity") + return rows diff --git a/server/main.py b/server/main.py index 777b0b78ac..e2867a2b61 100644 --- a/server/main.py +++ b/server/main.py @@ -386,9 +386,9 @@ async def _with_repo_umask(target: FastAPI) -> AsyncIterator[None]: _otel.init_otel(app) -# Request-body cap — the one bounded input at the front door (413 before any -# parser runs; LOOM_MAX_REQUEST_BYTES, 0 disables). Added first, so it is the -# innermost middleware: the 413 still passes out through the header middleware. +# Request-body caps reject oversized input before parsing. LOOM_MAX_REQUEST_BYTES +# is the global cap; 0 disables only that cap. Fixed cell administration caps remain. +# This middleware is innermost so 413 responses pass through header middleware. from server.transport.body_limit import BodyLimitMiddleware as _BodyLimit # noqa: E402 # An upload's route bounds its own body (LOOM_KNOWLEDGE_UPLOAD_MAX_BYTES). @@ -575,10 +575,14 @@ def attach_browser_middleware(target: FastAPI) -> None: from server.insights.app.admin_routes import admin_router as _admin_router # noqa: E402 from server.insights.app.admin_routes import org_admin_router as _org_admin_router # noqa: E402 +from server.insights.app.cell_admin_routes import ( # noqa: E402 + cell_admin_router as _cell_admin_router, +) from server.insights.app.provider_routes import provider_router as _provider_router # noqa: E402 app.include_router(_admin_router) app.include_router(_org_admin_router) +app.include_router(_cell_admin_router) app.include_router(_provider_router) from server.identity.app.admin_account_routes import router as _account_admin_router # noqa: E402 diff --git a/server/memory/app/blacklist.py b/server/memory/app/blacklist.py index cfc93226be..075719102f 100644 --- a/server/memory/app/blacklist.py +++ b/server/memory/app/blacklist.py @@ -3,8 +3,10 @@ from __future__ import annotations import logging +import time import uuid -from typing import TYPE_CHECKING +from datetime import datetime +from typing import TYPE_CHECKING, Any from server.core.datetimes import naive_utc_now_seconds from server.core.error_summary import error_summary @@ -17,6 +19,7 @@ BlacklistRemoveOutput, BlacklistSweepInput, BlacklistSweepOutput, + PatternType, _BlacklistRow, ) from server.memory.app.blacklist_guard import ( @@ -135,9 +138,23 @@ async def handle_blacklist_add(input: BlacklistAddInput) -> BlacklistAddOutput: await _reject_bad_regex(input.pattern, input.scope, *stored) pattern_id = str(uuid.uuid4()) + now = await _store_pattern(input, pattern_id) + if not input.sweep: + return BlacklistAddOutput(pattern_id=pattern_id, created_at=now) + return await _sweep_added(input, pattern_id, now) + + +async def _store_pattern( + input: BlacklistAddInput, + pattern_id: str, + *, + deadline: float | None = None, + ch: Any = None, +) -> datetime: now = naive_utc_now_seconds() - stored_org, stored_ns = stored + stored_org, stored_ns = _scope_args(input.scope, input.org, input.namespace) created_by = _stamped_author(input.created_by) + bounded: dict[str, Any] = {} if deadline is None else {"deadline": deadline, "ch": ch} await blacklist_repo.insert_row( { "pattern_id": pattern_id, @@ -151,12 +168,17 @@ async def handle_blacklist_add(input: BlacklistAddInput) -> BlacklistAddOutput: "hit_count": 0, "created_at": now, "tombstoned_at": None, - } + }, + **bounded, ) invalidate_cache() - await _bump_version(updated_by=f"add:{created_by or 'unknown'}", org=stored_org) - if not input.sweep: - return BlacklistAddOutput(pattern_id=pattern_id, created_at=now) + await _bump_version(updated_by=f"add:{created_by or 'unknown'}", org=stored_org, **bounded) + return now + + +async def _sweep_added( + input: BlacklistAddInput, pattern_id: str, now: datetime +) -> BlacklistAddOutput: # The pattern reaches the rows written before it. from server.memory.app.blacklist_sweep import SweepResult, sweep_error_text, sweep_pattern @@ -182,6 +204,113 @@ async def handle_blacklist_add(input: BlacklistAddInput) -> BlacklistAddOutput: ) +def require_global_authority(verb: str) -> None: + from server.auth.context import current_auth + from server.core.errors import AuthInsufficientError + + if current_auth() is None: + raise AuthInsufficientError("Authenticated global authority is required") + _require_scope_authority("global", verb, row_org="*") + + +async def add_global_bounded( + pattern_id: str, + pattern: str, + pattern_type: PatternType, + reason: str, + *, + ch: Any, + deadline: float, +) -> BlacklistAddOutput: + from server.core import leases + from server.core.ch import sequential_reads, statement_timeout + from server.core.ch_bounded import BoundedReadError + + require_global_authority("add") + parsed = uuid.UUID(pattern_id) + if str(parsed) != pattern_id or parsed.version != 4: + raise BoundedReadError("A fresh canonical pattern identity is required") + input = BlacklistAddInput( + org="*", + namespace="*", + scope="global", + sweep=False, + pattern=pattern, + pattern_type=pattern_type, + reason=reason, + ) + await blacklist_repo.check_bounded_storage(ch=ch, deadline=deadline) + if pattern_type == "regex": + await _check_global_regex(pattern, ch=ch, deadline=deadline) + async with leases.holding( + "blacklist-pattern:" + pattern_id, + ttl_s=60, + wait_s=0, + ch=ch, + deadline=deadline, + confirm_release=True, + ) as held: + if not held: + raise BoundedReadError("The pattern identity is unavailable") + with ( + sequential_reads(), + statement_timeout(min(2.0, max(0.001, deadline - time.monotonic()))), + ): + if await blacklist_repo.collision_bounded(pattern_id, ch=ch, deadline=deadline): + raise BoundedReadError("The pattern identity is unavailable") + now = await _store_pattern(input, pattern_id, ch=ch, deadline=deadline) + return BlacklistAddOutput(pattern_id=pattern_id, created_at=now) + + +async def _check_global_regex(pattern: str, *, ch: Any, deadline: float) -> None: + from server.core.ch import sequential_reads + from server.core.ch_bounded import BoundedReadError + from server.core.errors import ValidationError + + try: + total = compile_regex(pattern).programsize + except ValidationError as error: + raise BoundedReadError("The global regex is unsupported") from error + with sequential_reads(): + patterns = await blacklist_repo.global_regexes_bounded(ch=ch, deadline=deadline) + for stored in patterns: + try: + total += compile_regex(stored).programsize + except ValidationError: + continue # The ordinary cache likewise excludes invalid stored programmes. + if total > REGEX_PROGRAM_PER_SCOPE: + raise BoundedReadError("The global regex programme exceeds its allowance") + if total > REGEX_PROGRAM_PER_SCOPE: + raise BoundedReadError("The global regex programme exceeds its allowance") + + +async def list_global_bounded(limit: int, *, ch: Any, deadline: float) -> dict[str, Any]: + from server.core.ch import sequential_reads + + require_global_authority("list") + await blacklist_repo.check_bounded_storage(ch=ch, deadline=deadline) + with sequential_reads(): + rows = await blacklist_repo.global_list_bounded(limit, ch=ch, deadline=deadline) + return { + "rows": [ + { + "pattern_id": row["pattern_id"], + "pattern": row["pattern"], + "pattern_type": row["pattern_type"], + "reason": row["reason"], + "created_by": row["created_by"], + "hit_count": row["hit_count"], + "last_hit_at": row["last_hit_at"].isoformat() + if row["last_hit_at"] is not None + else None, + "created_at": row["created_at"].isoformat(), + } + for row in rows[:limit] + ], + "truncated": len(rows) > limit, + } + + async def handle_blacklist_sweep(input: BlacklistSweepInput) -> BlacklistSweepOutput: """Tombstone the live rows an existing pattern matches. @@ -298,7 +427,11 @@ async def handle_blacklist_remove( # An org-scoped pattern of another org is not addressable here. return BlacklistRemoveOutput(removed=False) _require_scope_authority(scope, "remove", row_org=row_org) - await blacklist_repo.tombstone_pattern(input.pattern_id, now) + removed = await blacklist_repo.tombstone_pattern( + input.pattern_id, now, selected=(row_org, row_ns, scope) + ) + if not removed: + return BlacklistRemoveOutput(removed=False) invalidate_cache() await _bump_version(updated_by=f"remove:{input.removed_by or 'unknown'}", org=row_org) return BlacklistRemoveOutput(removed=True, removed_at=now) diff --git a/server/memory/app/blacklist_guard.py b/server/memory/app/blacklist_guard.py index 42eb767ff2..79f7589a96 100644 --- a/server/memory/app/blacklist_guard.py +++ b/server/memory/app/blacklist_guard.py @@ -160,16 +160,33 @@ async def _current_version() -> tuple[int, int]: return await _repo.version_watermark(ch=get_client()) -async def _bump_version(updated_by: str = "", *, org: str = "*") -> None: +async def _bump_version( + updated_by: str = "", + *, + org: str = "*", + deadline: float | None = None, + ch: Any = None, +) -> None: """Move the watermark for every replica: one insert, no read, so two concurrent add/remove calls cannot write the same version. Under the Postgres authority the pattern's policy generation moves too (its org's, or the deployment's for a global one, `org` '*'), so a write admitted against the old patterns starts over and is checked against the new ones.""" - await _repo.insert_version(time.time_ns(), updated_by=updated_by, ch=get_client()) + if deadline is None: + await _repo.insert_version(time.time_ns(), updated_by=updated_by, ch=get_client()) + else: + await _repo.insert_version( + time.time_ns(), + updated_by=updated_by, + ch=ch or get_client(), + deadline=deadline, + ) from server.core import pg - await pg.bump_policy("" if org == "*" else org) + if deadline is None: + await pg.bump_policy("" if org == "*" else org) + else: + await pg.bump_policy("" if org == "*" else org, deadline=deadline) def invalidate_cache() -> None: diff --git a/server/memory/store/blacklist.py b/server/memory/store/blacklist.py index 4ab65aafbc..b95bb60661 100644 --- a/server/memory/store/blacklist.py +++ b/server/memory/store/blacklist.py @@ -5,16 +5,23 @@ to encode all three lookup shapes; centralizing here keeps the predicates in one place. -Soft-delete via tombstoned_at; an ALTER UPDATE flips it. The in-process -cache and when it refreshes live in server.memory.app.blacklist_guard; the -statements it runs (the active patterns, the version watermark) are here. +Soft-delete inserts a complete row with a newer created_at version and tombstoned_at. +The in-process cache and its refresh live in server.memory.app.blacklist_guard; +the statements it runs (active patterns and the version watermark) are here. """ from __future__ import annotations -from datetime import datetime +import asyncio +import time +from datetime import datetime, timedelta from typing import Any +from uuid import UUID +from server.core import ch_names +from server.core.ch import set_query_tag, statement_timeout +from server.core.ch_bounded import BoundedReadError, read_rows +from server.core.datetimes import fits_datetime_column, to_naive_utc_second from server.core.repo_base import _CHClient, _default_client @@ -22,15 +29,17 @@ async def insert_row( row: dict[str, Any], *, ch: _CHClient | None = None, + deadline: float | None = None, ) -> None: """Insert one blacklist pattern. async_insert=0 so the next list sees the new row.""" ch = ch or _default_client() - await ch.insert( - "loom.blacklist", - [row], - settings={"async_insert": 0, "wait_for_async_insert": 1}, - ) + if deadline is None: + await ch.insert( + "loom.blacklist", [row], settings={"async_insert": 0, "wait_for_async_insert": 1} + ) + else: + await _bounded_insert("loom.blacklist", row, ch=ch, deadline=deadline) async def list_for_scope( @@ -107,19 +116,74 @@ async def tombstone_pattern( when: datetime, *, ch: _CHClient | None = None, -) -> None: - """Soft-delete via ALTER UPDATE. mutations_sync=2 so the next - list call doesn't see the row.""" - ch = ch or _default_client() - await ch.execute( - """ - ALTER TABLE loom.blacklist - UPDATE tombstoned_at = {ts:DateTime} - WHERE pattern_id = {pid:UUID} - SETTINGS mutations_sync=2 - """, - params={"pid": pattern_id, "ts": when}, + selected: tuple[str, str, str] | None = None, +) -> bool: + """Insert a complete tombstone past the selected row's DateTime version.""" + client = ch or _default_client() + deadline = time.monotonic() + 30.0 + await check_bounded_storage(ch=client, deadline=deadline) + predicate = "pattern_id = {pid:UUID}" + params: dict[str, Any] = {"pid": pattern_id, "limit": 3} + if selected is not None: + predicate += ( + " AND org = {org:String} AND namespace = {ns:String} AND scope = {scope:String}" + ) + params.update(org=selected[0], ns=selected[1], scope=selected[2], limit=2) + rows = await read_rows( + "SELECT " + + _ROW_PROJECTION + + " FROM loom.blacklist FINAL WHERE " + + predicate + + " LIMIT {limit:UInt32} FORMAT JSONEachRow", + params, + ch=client, + deadline=deadline, + row_bytes=32768, + total_bytes=65536, + max_rows=2 if selected is None else 1, + tag="blacklist.remove.selected", + ) + if len(rows) > 1: + raise BoundedReadError("The refusal identity is ambiguous") + if not rows: + return False + row = _decode_bounded_row(rows[0]) + if row["tombstoned_at"] is not None: + return False + epoch = datetime(1970, 1, 1) + created_at: datetime = row["created_at"] + stored = (created_at - epoch).days * 86400 + (created_at - epoch).seconds + if stored == 2**32 - 1: + raise BoundedReadError("A refusal successor version cannot be represented") + if not fits_datetime_column(when): + raise BoundedReadError("Invalid refusal removal time") + clocks = await read_rows( + "SELECT partition_key, toUnixTimestamp(now()) AS now FROM system.tables " + "WHERE database = {database:String} AND name = 'blacklist' LIMIT 2 FORMAT JSONEachRow", + {"database": ch_names.database()}, + ch=client, + deadline=deadline, + row_bytes=1024, + total_bytes=1024, + max_rows=1, + tag="blacklist.remove.successor", ) + if ( + len(clocks) != 1 + or set(clocks[0]) != {"partition_key", "now"} + or clocks[0]["partition_key"] != "" + or type(clocks[0]["now"]) is not int + or not 0 <= clocks[0]["now"] <= 2**32 - 1 + ): + raise BoundedReadError("The refusal replacement storage could not be established") + clock: int = clocks[0]["now"] + successor = epoch + timedelta(seconds=max(clock, stored + 1)) + await insert_row( + {**row, "created_at": successor, "tombstoned_at": to_naive_utc_second(when)}, + ch=client, + deadline=deadline, + ) + return True async def active_patterns(*, ch: _CHClient | None = None) -> list[dict[str, Any]]: @@ -164,11 +228,207 @@ async def version_watermark(*, ch: _CHClient | None = None) -> tuple[int, int]: return (int(row.get("n") or 0), int(row.get("s") or 0)) -async def insert_version(version: int, *, updated_by: str, ch: _CHClient | None = None) -> None: +async def insert_version( + version: int, + *, + updated_by: str, + ch: _CHClient | None = None, + deadline: float | None = None, +) -> None: """Append one watermark row. No read first: two concurrent bumps write two distinct rows.""" ch = ch or _default_client() - await ch.insert( - "loom.blacklist_version", - [{"singleton": 0, "version": int(version), "updated_by": updated_by}], + row = {"singleton": 0, "version": int(version), "updated_by": updated_by} + if deadline is None: + await ch.insert("loom.blacklist_version", [row]) + else: + await _bounded_insert("loom.blacklist_version", row, ch=ch, deadline=deadline) + + +async def _bounded_insert(table: str, row: dict[str, Any], *, ch: Any, deadline: float) -> None: + remaining = min(2.0, deadline - time.monotonic()) + if remaining <= 0: + raise BoundedReadError("Blacklist write allowance exhausted") + set_query_tag("cell.each.blacklist_write") + async with asyncio.timeout(remaining): + with statement_timeout(remaining): + await ch.insert( + table, + [row], + settings={ + "async_insert": 0, + "wait_for_async_insert": 1, + "max_memory_usage": 128 * 1024**2, + }, + ) + + +_COLUMN_TYPES = { + "pattern_id": "UUID", + "org": "LowCardinality(String)", + "namespace": "LowCardinality(String)", + "pattern": "String", + "pattern_type": "Enum8('substring'=1,'regex'=2)", + "scope": "Enum8('namespace'=1,'org'=2,'global'=3)", + "reason": "String", + "created_by": "String", + "hit_count": "UInt64", + "created_at": "DateTime", + "tombstoned_at": "Nullable(DateTime)", + "last_hit_at": "Nullable(DateTime)", +} +_ROW_PROJECTION = ( + "pattern_id, org, namespace, pattern, pattern_type, scope, " + "reason, created_by, hit_count, toUnixTimestamp(created_at) AS created_at, " + "toUnixTimestamp(tombstoned_at) AS tombstoned_at, " + "toUnixTimestamp(last_hit_at) AS last_hit_at" +) + + +async def check_bounded_storage(*, ch: Any, deadline: float) -> None: + import re + + tables = await read_rows( + "SELECT engine, engine_full, sorting_key FROM system.tables " + "WHERE database = {database:String} AND name = 'blacklist' LIMIT 2 FORMAT JSONEachRow", + {"database": ch_names.database()}, + ch=ch, + deadline=deadline, + row_bytes=16384, + total_bytes=16384, + max_rows=1, + tag="cell.each.blacklist_shape", + ) + if len(tables) != 1 or set(tables[0]) != {"engine", "engine_full", "sorting_key"}: + raise BoundedReadError("Unsupported blacklist storage") + table = tables[0] + # Engine arguments end before ORDER BY/SETTINGS; quoted Keeper paths can hold parentheses. + definition = re.match( + rf"{re.escape(str(table['engine']))}\(((?:[^'()]|'(?:[^'\\]|\\.|'')*')*)\)(?=\s|$)", + table["engine_full"] if type(table["engine_full"]) is str else "", + ) + if ( + table["engine"] + not in {"ReplacingMergeTree", "ReplicatedReplacingMergeTree", "SharedReplacingMergeTree"} + or type(table["engine_full"]) is not str + or type(table["sorting_key"]) is not str + or not table["engine_full"].startswith(table["engine"] + "(") + or definition is None + or re.search(r"(?:^|,)\s*created_at\s*$", definition[1]) is None + or table["sorting_key"].replace(" ", "").strip("()") != "org,namespace,pattern_id" + ): + raise BoundedReadError("Unsupported blacklist storage") + columns = await read_rows( + "SELECT name, type FROM system.columns " + "WHERE database = {database:String} AND table = 'blacklist' " + "ORDER BY position LIMIT 13 FORMAT JSONEachRow", + {"database": ch_names.database()}, + ch=ch, + deadline=deadline, + row_bytes=1024, + total_bytes=16384, + max_rows=12, + tag="cell.each.blacklist_columns", + ) + found: dict[str, str] = {} + for column in columns: + if ( + set(column) != {"name", "type"} + or type(column["name"]) is not str + or type(column["type"]) is not str + or column["name"] in found + ): + raise BoundedReadError("Unsupported blacklist columns") + found[column["name"]] = column["type"].replace(" ", "") + if found != _COLUMN_TYPES: + raise BoundedReadError("Unsupported blacklist columns") + + +def _decode_bounded_row(row: dict[str, Any]) -> dict[str, Any]: + if set(row) != set(_COLUMN_TYPES): + raise BoundedReadError("Malformed blacklist row") + for key in ( + "pattern_id", + "org", + "namespace", + "pattern", + "pattern_type", + "scope", + "reason", + "created_by", + ): + if type(row[key]) is not str: + raise BoundedReadError("Malformed blacklist field") + try: + if str(UUID(row["pattern_id"])) != row["pattern_id"]: + raise ValueError + except ValueError as error: + raise BoundedReadError("Malformed blacklist identity") from error + if ( + row["scope"] not in {"global", "org", "namespace"} + or row["pattern_type"] not in {"substring", "regex"} + or type(row["hit_count"]) is not int + or not 0 <= row["hit_count"] <= 2**64 - 1 + ): + raise BoundedReadError("Malformed blacklist row") + for key in ("created_at", "last_hit_at", "tombstoned_at"): + value = row[key] + if value is None and key != "created_at": + continue + if type(value) is not int or not 0 <= value <= 2**32 - 1: + raise BoundedReadError("Malformed blacklist timestamp") + row[key] = datetime(1970, 1, 1) + timedelta(seconds=value) + return row + + +async def collision_bounded(pattern_id: str, *, ch: Any, deadline: float) -> bool: + rows = await read_rows( + "SELECT " + _ROW_PROJECTION + " FROM loom.blacklist FINAL " + "WHERE pattern_id = {pid:UUID} LIMIT 3 FORMAT JSONEachRow", + {"pid": pattern_id}, + ch=ch, + deadline=deadline, + row_bytes=32768, + total_bytes=65536, + max_rows=2, + tag="cell.each.blacklist_collision", + ) + for row in rows: + _decode_bounded_row(row) + return bool(rows) + + +async def global_list_bounded(limit: int, *, ch: Any, deadline: float) -> list[dict[str, Any]]: + if not 1 <= limit <= 50: + raise BoundedReadError("Unsupported blacklist display limit") + rows = await read_rows( + "SELECT " + _ROW_PROJECTION + " FROM loom.blacklist FINAL " + "WHERE org = '*' AND namespace = '*' AND scope = 'global' AND tombstoned_at IS NULL " + "ORDER BY created_at DESC, pattern_id LIMIT {limit:UInt32} FORMAT JSONEachRow", + {"limit": limit + 1}, + ch=ch, + deadline=deadline, + row_bytes=32768, + total_bytes=1792 * 1024, + max_rows=limit + 1, + tag="cell.each.blacklist_list", + ) + return [_decode_bounded_row(row) for row in rows] + + +async def global_regexes_bounded(*, ch: Any, deadline: float) -> list[str]: + rows = await read_rows( + "SELECT pattern FROM loom.blacklist FINAL WHERE org = '*' AND namespace = '*' " + "AND scope = 'global' AND pattern_type = 'regex' AND tombstoned_at IS NULL " + "LIMIT 1025 FORMAT JSONEachRow", + {}, + ch=ch, + deadline=deadline, + row_bytes=2048, + total_bytes=1024**2, + max_rows=1024, + tag="cell.each.blacklist_regexes", ) + if any(set(row) != {"pattern"} or type(row["pattern"]) is not str for row in rows): + raise BoundedReadError("Malformed global regex programme") + return [row["pattern"] for row in rows] diff --git a/server/observability/budgets.py b/server/observability/budgets.py index fec3ee4003..41323ec740 100644 --- a/server/observability/budgets.py +++ b/server/observability/budgets.py @@ -78,17 +78,40 @@ def __init__(self) -> None: # -- per op (unchanged API) ------------------------------------------- - def set(self, op_name: str, budget_ms: int) -> None: + def check_bounded_change(self, op_name: str, *, consumer: str | None, clear: bool) -> None: + """Inspect only map cardinalities before a bounded point change.""" + with self._lock: + self._check_bounded_change(op_name, consumer=consumer, clear=clear) + + def _check_bounded_change(self, op_name: str, *, consumer: str | None, clear: bool) -> None: + from server.core.ch_bounded import BoundedReadError + + if len(self._overrides) > 4096 or len(self._by_consumer) > 1024: + raise BoundedReadError("Budget overrides exceed their allowance") + if not clear: + if consumer: + if (consumer, op_name or ANY_OP) not in self._by_consumer and len( + self._by_consumer + ) == 1024: + raise BoundedReadError("Consumer budgets exceed their allowance") + elif op_name not in self._overrides and len(self._overrides) == 4096: + raise BoundedReadError("Operation budgets exceed their allowance") + + def set(self, op_name: str, budget_ms: int, *, bounded: bool = False) -> None: stored = stored_ms(budget_ms) with self._lock: + if bounded: + self._check_bounded_change(op_name, consumer=None, clear=False) self._overrides[op_name] = stored def get(self, op_name: str) -> int | None: with self._lock: return self._overrides.get(op_name) - def clear(self, op_name: str) -> None: + def clear(self, op_name: str, *, bounded: bool = False) -> None: with self._lock: + if bounded: + self._check_bounded_change(op_name, consumer=None, clear=True) self._overrides.pop(op_name, None) def snapshot(self) -> dict[str, int]: @@ -97,7 +120,14 @@ def snapshot(self) -> dict[str, int]: # -- per consumer -------------------------------------- - def set_for_consumer(self, consumer: str, op_name: str, value: int | str) -> None: + def set_for_consumer( + self, + consumer: str, + op_name: str, + value: int | str, + *, + bounded: bool = False, + ) -> None: """`value` is milliseconds or a "x" multiplier; `op_name` "*" covers every op for the consumer.""" if not consumer: @@ -106,6 +136,8 @@ def set_for_consumer(self, consumer: str, op_name: str, value: int | str) -> Non if parsed is None: raise ValueError("budget must be a non-negative integer (ms) or 'x'") with self._lock: + if bounded: + self._check_bounded_change(op_name, consumer=consumer, clear=False) self._by_consumer[(consumer, op_name or ANY_OP)] = parsed def get_for_consumer(self, consumer: str, op_name: str) -> int | float | None: @@ -115,8 +147,16 @@ def get_for_consumer(self, consumer: str, op_name: str) -> int | float | None: hit = self._by_consumer.get((consumer, ANY_OP)) return hit - def clear_for_consumer(self, consumer: str, op_name: str = ANY_OP) -> None: + def clear_for_consumer( + self, + consumer: str, + op_name: str = ANY_OP, + *, + bounded: bool = False, + ) -> None: with self._lock: + if bounded: + self._check_bounded_change(op_name, consumer=consumer, clear=True) self._by_consumer.pop((consumer, op_name or ANY_OP), None) def snapshot_by_consumer(self) -> dict[str, dict[str, int | float]]: diff --git a/server/router/app.py b/server/router/app.py index 87cc8d775e..bd3725660e 100644 --- a/server/router/app.py +++ b/server/router/app.py @@ -67,7 +67,7 @@ from server.core import cell as cell_state from server.core.cell_probe import PATH as PROBE_PATH -from server.router import directory, fingerprints, routing +from server.router import cell_admin, directory, fingerprints, routing from server.router.limits import ( ATTACH_BATCH, ATTACH_BEGIN_MAX_BYTES, @@ -204,6 +204,9 @@ def __init__( self._probe_transport = probe_transport self.placements = Limiter(settings.router_placements_per_minute, clock=clock) self.lookups = Limiter(settings.router_lookups_per_minute, clock=clock) + # Fresh administration reads share lookup admission's configured per-address + # budget, with separate counters so they do not consume ordinary lookups. + self._administration = Limiter(settings.router_lookups_per_minute, clock=clock) # The cells' credential hashes, read from the directory at most once per # LOOM_ROUTER_DIRECTORY_CACHE_S: a call with a wrong credential reads nothing. # A failed read stands for the window too, so an outage is not read once per call. @@ -701,6 +704,51 @@ async def _operator_body( return _refused(400, "this call's body is a JSON object") return body + def _administration_limit(self, request: Request) -> JSONResponse | None: + key = request.client.host if request.client else "unknown" + if not self._administration.admit(key): + return _refused( + 429, + "administration read limit reached", + {"Retry-After": str(self._administration.wait_s(key))}, + ) + return None + + async def administration_identity(self, request: Request) -> JSONResponse: + """Return only this freshly authenticated cell's two public bot identities.""" + if refusal := _malformed_cell_refusal(request): + return refusal + token = _bearer(request) + if not token.startswith("lcr_") or len(token) > 256: + return _refused(401, "the cell credential is not this cell's") + if refusal := self._administration_limit(request): + return refusal + try: + answer = await directory.administration_identity( + request.path_params["cell"], hashlib.sha256(token.encode()).hexdigest() + ) + except (directory.DirectoryUnavailable, cell_admin.InvalidAdministration): + return _refused(503, "cell administration identity is unavailable") + if answer is None: + return _refused(401, "the cell credential is not this cell's") + return JSONResponse(answer) + + async def administration_cells(self, request: Request) -> JSONResponse: + """The operator's closed, bounded fleet census or fresh exact point read.""" + if refusal := self._operator(request): + return refusal + if "cell" in request.path_params: + if refusal := _malformed_cell_refusal(request): + return refusal + if refusal := self._administration_limit(request): + return refusal + try: + return JSONResponse( + await directory.administration_cells(request.path_params.get("cell")) + ) + except (directory.DirectoryUnavailable, cell_admin.InvalidAdministration): + return _refused(503, "cell administration census is unavailable") + async def operator_cell(self, request: Request) -> JSONResponse: """`GET /cells/v1/operator/cells/`: the cell's row as attach reads it (its refusals, its done check and its resume).""" @@ -713,6 +761,31 @@ async def operator_cell(self, request: Request) -> JSONResponse: lambda found: {**_attach_answer(found), "other": found.other}, ) + async def set_administration(self, request: Request) -> JSONResponse: + """Record the operator's two proved public identities with a current claim.""" + body = await self._operator_body(request) + if isinstance(body, JSONResponse): + return body + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected + if set(body) != {"bots"} or not isinstance(body["bots"], list): + return _refused(400, "administration contains only the two public bots") + try: + cell_admin.public_pair( + json.dumps(body["bots"], ensure_ascii=True, separators=(",", ":")) + ) + except cell_admin.InvalidAdministration: + return _refused( + 400, "administration requires distinct automation and recovery public identities" + ) + return await _answered( + lambda: directory.set_administration( + request.path_params["cell"], body["bots"], expected=expected + ), + lambda answer: answer, + ) + async def claim(self, request: Request) -> JSONResponse: body = await self._operator_body(request, ATTACH_BEGIN_MAX_BYTES) if isinstance(body, JSONResponse): @@ -1631,7 +1704,23 @@ async def _lifespan(_app: Starlette) -> AsyncIterator[None]: Route("/cells/v1/placements", router.place, methods=["POST"]), Route("/cells/v1/operator/cells/{cell}/place", router.operator_place, methods=["POST"]), Route("/cells/v1/cells/{cell}", router.cell_state, methods=["GET"]), + Route( + "/cells/v1/cells/{cell}/administration", + router.administration_identity, + methods=["GET"], + ), + Route("/cells/v1/operator/cells", router.administration_cells, methods=["GET"]), + Route( + "/cells/v1/operator/cells/{cell}/administration", + router.administration_cells, + methods=["GET"], + ), Route("/cells/v1/operator/cells/{cell}/limit", router.set_limit, methods=["POST"]), + Route( + "/cells/v1/operator/cells/{cell}/administration", + router.set_administration, + methods=["POST"], + ), Route("/cells/v1/operator/cells/{cell}/register", router.register, methods=["POST"]), Route( "/cells/v1/operator/cells/{cell}/credential", diff --git a/server/router/cell_admin.py b/server/router/cell_admin.py new file mode 100644 index 0000000000..7d3dd74376 --- /dev/null +++ b/server/router/cell_admin.py @@ -0,0 +1,156 @@ +"""Closed public identities used by serial cell administration.""" + +from __future__ import annotations + +import json +import re +from typing import Any +from uuid import UUID + +from server.core.cell import normalise_host + +MAX_CELLS = 4096 # not a setting: coordinator and directory share one finite fleet protocol +MAX_CENSUS_BYTES = 4 * 1024 * 1024 # not a setting: the client refuses larger census documents + + +class InvalidAdministration(ValueError): + """The directory does not hold a complete supported administration identity.""" + + +def _text(value: Any, maximum: int) -> str: + try: + size = len(value.encode("utf-8")) if type(value) is str else 0 + except UnicodeError as error: + raise InvalidAdministration("Invalid administration identity") from error + if ( + type(value) is not str + or not value + or size > maximum + or any(ord(char) < 32 or ord(char) == 127 for char in value) + ): + raise InvalidAdministration("Invalid administration identity") + return value + + +def _unique(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + answer: dict[str, Any] = {} + for key, value in pairs: + if key in answer: + raise InvalidAdministration("Repeated administration field") + answer[key] = value + return answer + + +def public_pair(raw: str) -> dict[str, dict[str, str]]: + """Require two distinct public identities; empty legacy metadata refuses.""" + _text(raw, 1024) + try: + entries = json.loads(raw, object_pairs_hook=_unique) + except (ValueError, RecursionError) as error: + raise InvalidAdministration("Invalid administration metadata") from error + if type(entries) is not list or len(entries) != 2: + raise InvalidAdministration("Two administration identities are required") + answer: dict[str, dict[str, str]] = {} + for entry in entries: + if type(entry) is not dict or entry.keys() != {"purpose", "name", "thumbprint"}: + raise InvalidAdministration("Invalid administration metadata") + purpose = entry["purpose"] + if ( + type(purpose) is not str + or purpose not in {"automation", "recovery"} + or purpose in answer + ): + raise InvalidAdministration("Distinct administration purposes are required") + name = _text(entry["name"], 256) + thumbprint = _text(entry["thumbprint"], 43) + if re.fullmatch(r"[A-Za-z0-9_-]{43}", thumbprint) is None: + raise InvalidAdministration("Invalid public key thumbprint") + answer[purpose] = {"name": name, "thumbprint": thumbprint} + if any(answer["automation"][key] == answer["recovery"][key] for key in ("name", "thumbprint")): + raise InvalidAdministration("Distinct administration identities are required") + return answer + + +def identity(row: Any) -> dict[str, Any]: + cell = _text(row["cell"], 32) + if re.fullmatch(r"[a-z0-9-]{1,32}", cell) is None: + raise InvalidAdministration("Invalid cell") + return { + "version": 1, + "cell": cell, + "deployment_id": _text(row["deployment_id"], 256), + **public_pair(row["administration"]), + } + + +def selected_record(raw: Any) -> dict[str, Any]: + """The directory projects at most one KiB before the driver receives a row.""" + if type(raw) is not str or len(raw.encode("utf-8")) > 1024: + raise InvalidAdministration("Administration projection is oversized") + try: + row = json.loads(raw, object_pairs_hook=_unique) + except (ValueError, RecursionError) as error: + raise InvalidAdministration("Invalid administration projection") from error + if type(row) is not dict or row.keys() != { + "cell", + "deployment_id", + "administration", + "host", + "state", + "mode", + "mutation_generation", + "current_digest", + "next_digest", + }: + raise InvalidAdministration("Invalid administration projection") + return row + + +def target(row: Any) -> dict[str, Any]: + public = identity(row) + host = _text(row["host"], 512) + if normalise_host(host) != host or row["state"] != "serving" or row["mode"] != "enforced": + raise InvalidAdministration("Cell is not ready for administration") + generation = str(row["mutation_generation"]) + try: + if str(UUID(generation)) != generation: + raise ValueError + except ValueError as error: + raise InvalidAdministration("Invalid cell generation") from error + digests = [row["current_digest"]] + if row["next_digest"] is not None: + digests.append(row["next_digest"]) + if any( + type(item) is not str or re.fullmatch(r"[0-9a-f]{64}", item) is None for item in digests + ): + raise InvalidAdministration("Missing configuration identity") + answer = {key: value for key, value in public.items() if key != "version"} + answer.update( + origin="https://" + host, + state="serving", + mode="enforced", + mutation_generation=generation, + configuration_digests=sorted(set(digests)), + ) + if len(json.dumps(answer, ensure_ascii=True, separators=(",", ":")).encode()) > 1024: + raise InvalidAdministration("Administration identity is too large") + return answer + + +def census(rows: list[Any]) -> dict[str, Any]: + return census_targets([target(row) for row in rows]) + + +def census_targets(cells: list[dict[str, Any]]) -> dict[str, Any]: + if not 1 <= len(cells) <= MAX_CELLS: + raise InvalidAdministration("Administration census is empty or too large") + for key in ("cell", "deployment_id", "origin"): + if len({row[key] for row in cells}) != len(cells): + raise InvalidAdministration("Administration identities must be distinct") + answer = {"version": 1, "cells": cells} + if ( + len(json.dumps(answer, ensure_ascii=True, separators=(",", ":")).encode()) + > MAX_CENSUS_BYTES + ): + raise InvalidAdministration("Administration census is too large") + return answer diff --git a/server/router/directory.py b/server/router/directory.py index aa7ac03328..b1af83935e 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -33,7 +33,7 @@ from server.core import cell as cell_state from server.core import pg from server.core.paths import ROUTER_MIGRATIONS_DIR -from server.router import fingerprints +from server.router import cell_admin, fingerprints from server.settings import settings # The directory's own migration lock: never the memory authority's. @@ -292,6 +292,108 @@ async def migrate() -> list[str]: _CELL_REGISTERED = f"SELECT {_CELL_COLUMNS}, {', '.join(_REGISTERED)} FROM loom_router.cells" _ROUTING = "SELECT cell, host, state, is_first, issuers, upstream_url FROM loom_router.cells" +# Public administration identities are projected before driver materialization. The +# generation is the update foundation's existing UUID, never a locally invented counter. +_ADMINISTRATION = ( + "cell, left(deployment_id, 257) AS deployment_id, " + "left(break_glass::text, 1025) AS administration" +) +_ADMINISTRATION_READ_S = 2.0 # not a setting: fresh identity reads fit the fixed action envelope +_ADMIN_TARGET = ( + f"{_ADMINISTRATION}, left(host, 513) AS host, state, mode, mutation_generation, " + "CASE WHEN octet_length(current_fp::text) <= 65536 " + "THEN encode(sha256(convert_to(current_fp::text, 'UTF8')), 'hex') END AS current_digest, " + "CASE WHEN next_fp IS NOT NULL AND octet_length(next_fp::text) <= 65536 " + "THEN encode(sha256(convert_to(next_fp::text, 'UTF8')), 'hex') " + "WHEN next_fp IS NOT NULL THEN 'oversized' END AS next_digest" +) + + +def _administration_projection(selection: str) -> str: + # A NULL payload signals excess rather than filtering a cell out of the census. + return ( + "SELECT CASE WHEN octet_length(row_to_json(selected)::text) <= 1024 " + "THEN row_to_json(selected)::text END AS payload FROM (" + f"SELECT {_ADMIN_TARGET} FROM loom_router.cells {selection}) AS selected " + "ORDER BY selected.cell" + ) + + +@asynccontextmanager +async def _administration_read() -> AsyncIterator[Any]: + """One fresh bounded read, with no fleet credential or successful-state cache.""" + async with _reading(), asyncio.timeout(_ADMINISTRATION_READ_S), _connection() as conn: + async with conn.transaction(readonly=True): + await conn.execute("SET LOCAL statement_timeout = '2s'") + yield conn + + +async def administration_identity(name: str, presented: str) -> dict[str, Any] | None: + """Authenticate this exact row before exposing its two public bot identities.""" + async with _administration_read() as conn: + row = await conn.fetchrow( + f"SELECT {_ADMINISTRATION}, credential_sha256, credential_next_sha256 " + "FROM loom_router.cells WHERE cell = $1 LIMIT 1", + name, + timeout=_ADMINISTRATION_READ_S, + ) + if row is None or not holds( + presented, + [value for value in (row["credential_sha256"], row["credential_next_sha256"]) if value], + ): + return None + return cell_admin.identity(row) + + +async def administration_cells(name: str | None = None) -> dict[str, Any]: + """Require every registered cell ready, including cells closed to new placements.""" + async with _administration_read() as conn: + if name is not None: + row = await conn.fetchrow( + _administration_projection("WHERE cell = $1 LIMIT 1"), + name, + timeout=_ADMINISTRATION_READ_S, + ) + if row is None: + raise cell_admin.InvalidAdministration("No such administration cell") + return { + "version": 1, + "cell": cell_admin.target(cell_admin.selected_record(row["payload"])), + } + cells: list[dict[str, Any]] = [] + size = len(b'{"version":1,"cells":[]}') + query = _administration_projection("ORDER BY cell LIMIT 4097") + async for row in conn.cursor(query, prefetch=1, timeout=_ADMINISTRATION_READ_S): + target = cell_admin.target(cell_admin.selected_record(row["payload"])) + size += len( + json.dumps(target, ensure_ascii=True, separators=(",", ":")).encode() + ) + bool(cells) + if len(cells) >= cell_admin.MAX_CELLS or size > cell_admin.MAX_CENSUS_BYTES: + raise cell_admin.InvalidAdministration("Administration census is too large") + cells.append(target) + return cell_admin.census_targets(cells) + + +async def set_administration( + cell_name: str, bots: list[dict[str, str]], *, expected: Control +) -> dict[str, Any]: + """Record only the public pair under the existing directory claim and audit.""" + encoded = json.dumps(bots, ensure_ascii=True, separators=(",", ":")) + public = cell_admin.public_pair(encoded) + + async def change_locked(conn: Any, found: Attaching) -> dict[str, Any]: + if found.row.state != "serving" or found.row.mode != "enforced": + raise OperatorRefused("automation provisioning requires a serving, enforced cell") + await conn.execute( + "UPDATE loom_router.cells SET break_glass = $2, updated_at = now() WHERE cell = $1", + cell_name, + bots, + ) + await _audit(conn, "administration", {"cell": cell_name, "bots": bots}) + return {"version": 1, "cell": cell_name, "deployment_id": found.row.deployment_id, **public} + + return await _attach_change(cell_name, expected, change_locked) + _WINNER = ( "SELECT p.cell, c.deployment_id FROM loom_router.placements AS p " diff --git a/server/settings/catalog.py b/server/settings/catalog.py index c2a2f43d87..3d486658cf 100644 --- a/server/settings/catalog.py +++ b/server/settings/catalog.py @@ -611,8 +611,8 @@ def jwt_public_key_map(self) -> dict[str, str]: # Comma-separated origin list. "*" allows all (acceptable when # auth_required=True). Empty disables CORS entirely. cors_origins: str = "" - # Request-body cap, bytes, enforced by ASGI middleware before any parser - # (413 PAYLOAD_TOO_LARGE); uvicorn has no limit of its own. 0 disables. + # Global request-body cap, bytes, enforced by ASGI middleware before parsing + # (413 PAYLOAD_TOO_LARGE). 0 disables the global cap; fixed cell administration caps remain. max_request_bytes: int = 32 * 1024 * 1024 # Strict-Transport-Security max-age, seconds; browsers ignore it over plain HTTP. # 0 sends no header. diff --git a/server/transport/body_limit.py b/server/transport/body_limit.py index 89a439aeaa..bd7fd48092 100644 --- a/server/transport/body_limit.py +++ b/server/transport/body_limit.py @@ -12,15 +12,17 @@ `413 PAYLOAD_TOO_LARGE` in the same `{"detail": {code, message}}` shape every other loom error uses — the body is never read; * a chunked / undeclared body is counted as it streams and cut off at the - cap with the same 413 (the handler's `await request.json()` then fails - closed as a 400, since the body was never delivered whole). + cap with the same 413; an overflow aborts the downstream body read + before a complete JSON prefix can reach a handler. The cap is `LOOM_MAX_REQUEST_BYTES` (32 MiB by default — a 400-message `memory.set_from_messages` is a few hundred KB, the largest `memory.import` about 10 MB of printable ASCII text, more when its text is non-ASCII or control characters). Websocket and lifespan scopes pass through untouched, and so do the paths in `own_limit`, whose route streams the body against a cap of its -own (`POST /v1/knowledge.upload`, `LOOM_KNOWLEDGE_UPLOAD_MAX_BYTES`). +own (`POST /v1/knowledge.upload`, `LOOM_KNOWLEDGE_UPLOAD_MAX_BYTES`). Fixed cell +administration caps still apply when the global cap is disabled; a positive +global cap can only make those limits stricter. """ from __future__ import annotations @@ -32,9 +34,25 @@ CODE = "PAYLOAD_TOO_LARGE" +_CELL_BODY_LIMITS = { + "/v1/admin/cell.capabilities": 1024, + "/v1/admin/cell.settings_list": 65536, + "/v1/admin/cell.settings_set": 65536, + "/v1/admin/cell.settings_history": 65536, + "/v1/admin/cell.signing_keys_list": 65536, + "/v1/admin/cell.signing_keys_rotate": 65536, + "/v1/admin/cell.signing_keys_retire": 65536, + "/v1/admin/cell.subject_revoke": 65536, + "/v1/admin/cell.ratelimit_namespace_set": 65536, + "/v1/admin/cell.budget_set": 65536, + "/v1/admin/cell.blacklist_global_add": 65536, + "/v1/admin/cell.blacklist_global_list": 65536, + "/v1/admin/cell.blacklist_global_remove": 65536, +} + def _too_large_response(limit: int, declared: int | None) -> tuple[dict[str, Any], bytes]: - msg = f"request body exceeds LOOM_MAX_REQUEST_BYTES ({limit} bytes)" + ( + msg = f"request body exceeds limit ({limit} bytes)" + ( f"; declared {declared}" if declared is not None else "" ) detail = classified({"code": CODE, "message": msg, "limit_bytes": limit}, 413) @@ -46,8 +64,12 @@ def _too_large_response(limit: int, declared: int | None) -> tuple[dict[str, Any return {"type": "http.response.start", "status": 413, "headers": headers}, body +class _BodyTooLarge(Exception): + """Stop body parsing without exposing a valid truncated prefix.""" + + class BodyLimitMiddleware: - """Refuse request bodies larger than `max_bytes`.""" + """Refuse bodies above the global and fixed administration caps.""" def __init__(self, app: Any, max_bytes: int, own_limit: tuple[str, ...] = ()) -> None: self.app = app @@ -55,10 +77,14 @@ def __init__(self, app: Any, max_bytes: int, own_limit: tuple[str, ...] = ()) -> self.own_limit = frozenset(own_limit) async def __call__(self, scope: Any, receive: Any, send: Any) -> None: + limit = self.max_bytes + fixed_limit = _CELL_BODY_LIMITS.get(scope.get("path")) + if fixed_limit is not None: + limit = min(limit, fixed_limit) if limit > 0 else fixed_limit if ( scope.get("type") != "http" - or self.max_bytes <= 0 - or scope.get("path") in self.own_limit + or limit <= 0 + or (fixed_limit is None and scope.get("path") in self.own_limit) ): await self.app(scope, receive, send) return @@ -71,8 +97,8 @@ async def __call__(self, scope: Any, receive: Any, send: Any) -> None: except ValueError: declared = None break - if declared is not None and declared > self.max_bytes: - start, body = _too_large_response(self.max_bytes, declared) + if declared is not None and declared > limit: + start, body = _too_large_response(limit, declared) await send(start) await send({"type": "http.response.body", "body": body}) return @@ -85,11 +111,9 @@ async def counting_receive() -> Any: message = await receive() if message.get("type") == "http.request": seen += len(message.get("body") or b"") - if seen > self.max_bytes: + if seen > limit: tripped = True - # Starve the downstream reader: a truncated body with - # more_body=False makes `request.json()` fail closed. - return {"type": "http.request", "body": b"", "more_body": False} + raise _BodyTooLarge return message sent_start = False @@ -97,7 +121,7 @@ async def counting_receive() -> Any: async def guarded_send(message: Any) -> None: nonlocal sent_start if tripped and not sent_start and message.get("type") == "http.response.start": - start, body = _too_large_response(self.max_bytes, None) + start, body = _too_large_response(limit, None) sent_start = True await send(start) await send({"type": "http.response.body", "body": body}) @@ -108,7 +132,13 @@ async def guarded_send(message: Any) -> None: sent_start = True await send(message) - await self.app(scope, counting_receive, guarded_send) + try: + await self.app(scope, counting_receive, guarded_send) + except _BodyTooLarge: + if not sent_start: + start, body = _too_large_response(limit, None) + await send(start) + await send({"type": "http.response.body", "body": body}) __all__ = ["CODE", "BodyLimitMiddleware"] diff --git a/tests/analyzer/cases/planes.py b/tests/analyzer/cases/planes.py index d1b555dd73..ce17fc308f 100644 --- a/tests/analyzer/cases/planes.py +++ b/tests/analyzer/cases/planes.py @@ -17,6 +17,7 @@ import types import typing from typing import Any +from unittest.mock import patch from server.auth.context import current_auth from server.core.naming import not_mirror_sql @@ -655,7 +656,44 @@ async def _ledgers(ch: RecordingCH, c: dict[str, Any]) -> None: await C.tombstone_ids(ORG, [_UUID], "note", _DT, ch=ch, namespace=NS) await B.find_active_pattern(_UUID, org=ORG if f else None, ch=ch) await B.list_for_scope(ORG, NS, scope="namespace" if f else None, ch=ch) - await B.tombstone_pattern(_UUID, _DT, ch=ch) + + # Record the real bounded SELECT builders at their database adapter. + async def capture(sql: str, params: dict[str, Any], **kwargs) -> list[dict[str, Any]]: + await ch.query_all(sql, params) + if "SELECT engine, engine_full, sorting_key" in sql: + return [ + { + "engine": "ReplacingMergeTree", + "engine_full": "ReplacingMergeTree(created_at)", + "sorting_key": "org, namespace, pattern_id", + } + ] + if "FROM system.columns" in sql: + return [{"name": name, "type": kind} for name, kind in B._COLUMN_TYPES.items()] + if "SELECT partition_key" in sql: + return [{"partition_key": "", "now": 1_800_000_000}] + if "FROM loom.blacklist FINAL WHERE" in sql: + return [ + { + "pattern_id": _UUID, + "org": ORG, + "namespace": NS, + "scope": "namespace", + "pattern": "restricted", + "pattern_type": "substring", + "reason": "policy", + "created_by": AGENT, + "hit_count": 0, + "created_at": 1_700_000_000, + "last_hit_at": None, + "tombstoned_at": None, + } + ] + raise AssertionError("Unexpected blacklist tombstone statement") + + with patch.object(B, "read_rows", capture): + await B.tombstone_pattern(_UUID, _DT, ch=ch) + await B.tombstone_pattern(_UUID, _DT, ch=ch, selected=(ORG, NS, "namespace")) await W.blacklist_updated_at(ch=ch) await W.last_writes(ORG, [NS, "n2"], ch=ch) await L.delete_procedure_runs_by_key(ORG, NS, "k1", ch=ch) diff --git a/tests/analyzer/shapes.txt b/tests/analyzer/shapes.txt index 955a96a71a..4dbeb82fcf 100644 --- a/tests/analyzer/shapes.txt +++ b/tests/analyzer/shapes.txt @@ -94,6 +94,7 @@ 09a255c7f251 SELECT DISTINCT toString(memory_id) AS id FROM loom.entity_edge_rows WHERE org = {org:String} AND user_id = {uid:String} AND memory_id NOT IN (SELECT memory_id FROM loom.memory FINAL WHERE org = {org: 09d051a238d8 DELETE FROM loom.embedding_cache WHERE (org = {org:String} AND ?) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS lightweight_deletes_sync = ? 09d112610873 SELECT count() AS n FROM loom.memory_categories FINAL WHERE org = {org:String} AND namespace = {ns:String} +09e81ac17b26 SELECT engine, engine_full, sorting_key FROM system.tables WHERE database = {database:String} AND name = ? LIMIT ? FORMAT JSONEachRow 09fabae3c761 WITH m AS ( SELECT org, namespace, agent, user_id, session_id, memory_key, argMax(status, version) AS ms FROM loom.memory WHERE org = {org:String} AND created_at > now() - INTERVAL {days:UInt32} DAY G 0a1376fd93a3 ALTER TABLE loom.memory_webhook_outbox DELETE WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync 0a1c979e7c7c SELECT pr_number, groupUniqArray(author) AS reviewers FROM loom.code_pr_reviews FINAL WHERE org = {org:String} AND namespace = {ns:String} AND pr_number IN {nums:Array(UInt32)} AND is_bot = false AND @@ -936,6 +937,7 @@ 6f5e97facae7 SELECT count() AS n FROM loom.session_summaries FINAL WHERE org = {org:String} AND user_id = {uid:String} 6f6581cc7134 SELECT f AS path, argMin(pr_number, merged_at) AS first_pr, min(merged_at) AS first_merged_at FROM loom.code_pr_history FINAL ARRAY JOIN files AS f WHERE org = {org:String} AND namespace = {ns:String} 6f7989d65b42 SELECT count() AS n FROM loom.task_events WHERE org = {org:String} AND user_id = {uid:String} AND ? +6fc21c322007 SELECT pattern_id, org, namespace, pattern, pattern_type, scope, reason, created_by, hit_count, toUnixTimestamp(created_at) AS created_at, toUnixTimestamp(tombstoned_at) AS tombstoned_at, toUnixTimest 6fd3e3093469 SELECT DISTINCT path FROM loom.code_chunks FINAL WHERE org = {org:String} AND namespace = {ns:String} AND embedding_model = {model:String} AND startsWith(path, {prefix:String}) AND tombstoned_at IS NU 701267d73992 SELECT org, namespace, agent, user_id, session_id, argMaxIf(actor, version, actor != ? AND NOT has({pipeline:Array(String)}, actor) AND NOT startsWith(actor, {cli:String})) AS writer FROM loom.memory 701c7e908795 ALTER TABLE loom.memory_record_conflicts DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? @@ -1070,7 +1072,6 @@ 7cfce8998b1b SELECT memory_key, toString(memory_id) AS id, version, memory_type, toString(status) AS status, source_class, agent, user_id, session_id, actor, updated_at, sensitivity, org, namespace, redacted_at FR 7d0f95e88408 SELECT count() AS n FROM loom.agent_runs WHERE org = {org:String} AND user_id = {uid:String} AND ? 7d1aef7d3300 SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user -7d43341a2607 ALTER TABLE loom.blacklist UPDATE tombstoned_at = {ts:DateTime} WHERE pattern_id = {pid:UUID} SETTINGS mutations_sync=? 7d82a831fe45 SELECT count() AS n FROM loom.scim_users FINAL WHERE deleted = ? AND has(work_addresses, {a:String}) 7dafb5876695 WITH reversals AS ( SELECT erasure_id, count() AS n FROM loom.erasure_reversals FINAL WHERE org = {org:String} GROUP BY erasure_id ) SELECT e.id AS id, e.scope_type AS scope_type, e.scope_value AS sco 7dd6dd6493db SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description FROM loom.code_chunks FINAL WHERE org = {org:String} AND namespace = {ns:String} AND embedding_model @@ -1544,6 +1545,7 @@ b2b8650dca93 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e. b2d5a25cd043 SELECT count() AS n FROM loom.agent_runs WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND ? b2eda427e5b8 SELECT toString(evidence_id) AS eid, session_id, role, actor, toString(observed_at) AS observed, content, matched FROM ( SELECT evidence_id, session_id, role, actor, observed_at, content, tombstoned_a b3008a1630c8 SELECT n.org AS org, n.namespace AS namespace, n.owner_email AS owner_email, live.live_rows AS live_rows, greatest(ifNull(w.last_write, toDateTime64(?, ?)), toDateTime64(live.last_created, ?)) AS last +b30d0b818b67 SELECT partition_key, toUnixTimestamp(now()) AS now FROM system.tables WHERE database = {database:String} AND name = ? LIMIT ? FORMAT JSONEachRow b30fdbb7b57a SELECT src, dst, relation, direction, pair_co_occur, valid_from, valid_to, degree FROM ( SELECT src, dst, relation, direction, pair_co_occur, valid_from, valid_to, row_number() OVER (PARTITION BY src b32239abb324 SELECT org, namespace, agent, thread_id, msg_id, role, content, reasoning, user_id, run_id, metadata, tokens, ts FROM loom.conversations WHERE org = {org:String} AND namespace = {ns:String} AND thread b33bc129de89 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g @@ -1770,6 +1772,7 @@ ca58f3affe58 ALTER TABLE loom.conversation_threads UPDATE tombstoned_at = {now:D ca924a95865f SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g ca9ec9e5facb SELECT count() AS c FROM ( SELECT evidence_id, argMax(tombstoned_at, created_at) AS tomb_latest FROM loom.memory_belief_sources WHERE org = {o:String} AND namespace = {n:String} AND memory_key = {k:St caa73a929cde SELECT toUnixTimestamp64Milli(now64(?)) AS ms +cabcea8d5cfd SELECT name, type FROM system.columns WHERE database = {database:String} AND table = ? ORDER BY position LIMIT ? FORMAT JSONEachRow cabd0fc4905d ALTER TABLE loom.consolidation_runs DELETE WHERE (org = {org:String} AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? cacd77ea0602 SELECT count() AS n FROM loom.code_search_trace WHERE org = {org:String} AND user_id = {uid:String} cad920c3af77 SELECT org, code_namespace, branch_namespace, pr_number, branch, verdict, epistemic, basis, reason, cherry_pick_pr, cherry_pick_sha, backport_pr, label_wanted, wanted_branch, wanted_source, settled, p @@ -2037,6 +2040,7 @@ e583636bd338 ALTER TABLE loom.entity_edge_memories DELETE WHERE (org = {org:Stri e5977d678a52 ALTER TABLE loom.code_search_trace DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? e5a6b0c13d38 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description FROM loom.code_chunks FINAL WHERE (org, namespace, embedding_model, path, chunk_id) IN {keys:Array(Tu e5bac6e4a74e SELECT memory_id, version, agent, user_id, session_id, memory_key, created_at FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND memory_key IN {keys:Array(String)} AND tom +e5bf0d7aa5f5 SELECT pattern_id, org, namespace, pattern, pattern_type, scope, reason, created_by, hit_count, toUnixTimestamp(created_at) AS created_at, toUnixTimestamp(tombstoned_at) AS tombstoned_at, toUnixTimest e5ca09ca8882 SELECT ? FROM loom.revoked_tokens FINAL WHERE jti IN {keys:Array(String)} AND expires_at > now() AND issued_before IS NULL UNION ALL SELECT ? FROM loom.oauth_state FINAL WHERE record_type = ? AND id = e5d3479fff14 SELECT agent, user_id, session_id, memory_key, cosineDistance(embedding, {emb:Array(Float32)}) AS dist FROM loom.memory_embeddings FINAL WHERE org = {org:String} AND namespace = {ns:String} AND user_i e5df581175ec SELECT count() AS n FROM loom.knowledge_gaps WHERE org = {org:String} AND namespace IN {ns:Array(String)} diff --git a/tests/integration/_cell_each_fleet.py b/tests/integration/_cell_each_fleet.py new file mode 100644 index 0000000000..76b98c6e3b --- /dev/null +++ b/tests/integration/_cell_each_fleet.py @@ -0,0 +1,407 @@ +"""Two independently configured HTTPS cells and a real private router directory.""" + +from __future__ import annotations + +import asyncio +import hashlib +import ipaddress +import json +import os +import re +import secrets +import ssl +import subprocess +import sys +import uuid +from collections.abc import Iterator +from contextlib import ExitStack, contextmanager +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +import httpx +import jwt +import pytest +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.x509.oid import NameOID + +from server.auth import machines +from server.cli import cell_bot_proof +from server.cli import cell_each_http as wire +from tests.integration._pg_database import create_database, drop_database +from tests.integration._ports import free_port +from tests.integration._resources import ch_client_from_env, ch_target, unique_org +from tests.integration._spawn import ROOT, loom_server + + +def private_file(path: Path, value: str | bytes) -> Path: + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) + with os.fdopen(fd, "wb") as stream: + stream.write(value.encode() if isinstance(value, str) else value) + return path + + +def tls_pair(home: Path, name: str) -> tuple[Path, Path]: + key = ec.generate_private_key(ec.SECP256R1()) + subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, name)]) + now = datetime.now(UTC) + certificate = ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(subject) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - timedelta(minutes=5)) + .not_valid_after(now + timedelta(days=1)) + .add_extension(x509.BasicConstraints(ca=True, path_length=0), critical=True) + .add_extension( + x509.SubjectAlternativeName([x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]), + critical=False, + ) + .sign(key, hashes.SHA256()) + ) + return ( + private_file(home / f"{name}.crt", certificate.public_bytes(serialization.Encoding.PEM)), + private_file( + home / f"{name}.pem", + key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ), + ), + ) + + +def child(command: list[str], env: dict[str, str]) -> subprocess.CompletedProcess[str]: + # stderr can include a DSN on a bootstrap failure, so never put it in an assertion. + result = subprocess.run( + [sys.executable, "-m", *command], + cwd=ROOT, + env={**os.environ, **env}, + text=True, + capture_output=True, + timeout=240, + ) + if result.returncode: + # Allow only migration names, error classes and numeric codes into CI output. + # The streams may hold connection credentials, beyond generic log redaction. + migrations = {path.name for path in (ROOT / "schemas" / "migrations").glob("*.sql")} + stages = re.findall(r"(?m)^\s*[→✓✗] (\d{3}_[a-z0-9_]{1,160}\.sql)", result.stdout) + kinds = re.findall( + r"\b(DatabaseError|OperationalError|PreflightError|ValidationError|" + r"ConnectionError|TimeoutError|KeeperRequired)\b", + result.stderr, + ) + codes = re.findall(r"\bCode: (\d{1,5})\b", result.stderr) + detail = { + "exit": result.returncode, + "migration": [name for name in stages if name in migrations][-1:], + "error_kinds": sorted(set(kinds)), + "clickhouse_codes": sorted(set(codes))[:8], + } + raise AssertionError(f"Private fixture command {command[:1]} failed: {detail}") + return result + + +async def _register(directory: str, node: dict[str, Any]) -> None: + import asyncpg + + conn = await asyncpg.connect(directory) + try: + await conn.execute( + "INSERT INTO loom_router.cells " + "(cell, is_first, host, state, mode, credential_sha256, deployment_id, current_fp) " + "VALUES ($1, $2, $3, 'serving', 'enforced', $4, $5, $6::jsonb)", + node["cell"], + node["cell"] == "alpha", + node["origin"].removeprefix("https://"), + hashlib.sha256(node["credential"].encode()).hexdigest(), + node["deployment_id"], + json.dumps(node["fingerprint"]), + ) + finally: + await conn.close() + + +def _cell_env(name: str, dsn: str, database: str, router: str, ca: Path) -> dict[str, str]: + port = free_port() + origin = f"https://127.0.0.1:{port}" + return { + "LOOM_ENV_FILE": "", + "LOOM_CELL": name, + "LOOM_PG_DSN": dsn, + "LOOM_STORAGE_AUTHORITY": "postgres", + "LOOM_CH_DATABASE": database, + "LOOM_CH_TELEMETRY_DATABASE": database + "_telemetry", + "LOOM_CH_READER_USER": "", + "LOOM_CH_CLUSTER": "", + "LOOM_PUBLIC_BASE_URL": origin, + "LOOM_OAUTH_ISSUER": origin, + "LOOM_OAUTH_ENABLED": "true", + "LOOM_AUTH_REQUIRED": "true", + "LOOM_MCP_AUTH_MODE": "required", + "LOOM_JWT_SECRET": secrets.token_hex(32), + "LOOM_JWT_ISSUER": origin, + "LOOM_JWT_AUDIENCE": "loom-platform", + "LOOM_JWT_TRUSTED_ISSUERS": "", + "LOOM_JWT_END_USER_ISSUERS_JSON": "", + "LOOM_OIDC_ISSUER": "", + "LOOM_OIDC_CLIENT_ID": "", + "LOOM_SCIM_TOKEN": "", + "LOOM_SELF_SIGNUP": "off", + "LOOM_CODE_ORG": "", + "LOOM_TRACKER_ORG": "", + "LOOM_DOCUMENT_S3_BUCKET": "", + "LOOM_DOCUMENT_S3_PREFIX": "originals", + "LOOM_BACKGROUND_WORKERS": "off", + "LOOM_EMBEDDING_MODEL": "fake-ngram-1024", + "LOOM_LLM_PROVIDER": "fake", + "LOOM_CELL_ROUTER_URL": router, + "LOOM_CELL_ROUTER_TOKEN": "lcr_" + secrets.token_hex(64), + "LOOM_CELL_PROBE_PORT": str(free_port()), + "LOOM_BOOT_MIGRATE": "true", + "LOOM_TLS_CERT_FILE": "", + "LOOM_TLS_KEY_FILE": "", + "SSL_CERT_FILE": str(ca), + } + + +def _keys(home: Path, name: str) -> dict[str, Any]: + private, public = machines.generate_key() + thumbprint = json.loads(machines.public_jwks(public))["keys"][0]["kid"] + return { + "name": name, + "thumbprint": thumbprint, + "key_file": private_file(home / f"{name}.pem", private), + "public_key": public.decode(), + } + + +def _bots(node: dict[str, Any], home: Path, ca: Path) -> None: + env = node["env"] + now = int(datetime.now(UTC).timestamp()) + token = jwt.encode( + { + "sub": "native-operator", + "org": node["org"], + "namespaces": ["*"], + "roles": ["platform_admin"], + "iss": env["LOOM_JWT_ISSUER"], + "aud": env["LOOM_JWT_AUDIENCE"], + "sa": False, + "iat": now, + "exp": now + 3600, + "jti": str(uuid.uuid4()), + }, + env["LOOM_JWT_SECRET"], + algorithm="HS256", + headers={"kid": "default"}, + ) + node["admin_token"] = token + with httpx.Client(verify=ssl.create_default_context(cafile=str(ca)), trust_env=False) as client: + for purpose in ("automation", "recovery"): + bot = node[purpose] = _keys(home, f"{purpose}-{node['cell']}") + answer = client.post( + node["origin"] + "/v1/admin/bots.create", + headers={"Authorization": "Bearer " + token}, + json={ + "name": bot["name"], + "org": node["org"], + "namespace_roles": [], + "public_key_pem": bot["public_key"], + "platform_admin": True, + }, + timeout=30, + ) + assert answer.status_code == 200, "Normal platform-admin bot registration failed" + + +def bot_key(bot: dict[str, Any]) -> cell_bot_proof.BotKey: + return cell_bot_proof.BotKey(bot["name"], bot["thumbprint"], bot["key_file"]) + + +def _provision(node: dict[str, Any], router: str, operator: str, ca: Path) -> None: + command = ["server.cli", "cell", "automation", "--cell", node["cell"], "--router", router] + for purpose in ("automation", "recovery"): + bot = node[purpose] + command += [ + f"--{purpose}-bot", + bot["name"], + f"--{purpose}-thumbprint", + bot["thumbprint"], + f"--{purpose}-key", + str(bot["key_file"]), + ] + answer = child( + [*command, "--ca-file", str(ca)], + {**node["env"], "LOOM_ROUTER_OPERATOR_TOKEN": operator}, + ) + public = json.loads(answer.stdout) + assert public == { + "version": 1, + "cell": node["cell"], + "deployment_id": node["deployment_id"], + **{p: {k: node[p][k] for k in ("name", "thumbprint")} for p in ("automation", "recovery")}, + } + + +@contextmanager +def fleet(home: Path) -> Iterator[dict[str, Any]]: + admin = os.environ.get("LOOM_PG_DSN") or pytest.skip("LOOM_PG_DSN not set") + ch_target() + home.mkdir(mode=0o700) + certificate = tls_pair(home, "trusted") + operator = "lco_" + secrets.token_hex(64) + router_port = free_port() + router = f"https://127.0.0.1:{router_port}" + with ExitStack() as stack: + router_name, router_dsn = asyncio.run(create_database(admin)) + stack.callback(lambda: asyncio.run(drop_database(admin, router_name))) + router_env = { + "LOOM_ENV_FILE": "", + "LOOM_ROUTER_PG_DSN": router_dsn, + "LOOM_PG_DSN": "", + "LOOM_TLS_CERT_FILE": "", + "LOOM_TLS_KEY_FILE": "", + "LOOM_ROUTER_OPERATOR_SHA256": hashlib.sha256(operator.encode()).hexdigest(), + } + child(["server.cli", "router", "migrate"], router_env) + stack.enter_context( + loom_server( + "each-router", router_env, port=router_port, tls_files=certificate, router=True + ) + ) + ch = ch_client_from_env(database="default") + stack.callback(ch.close) + nodes = [] + for name in ("alpha", "beta"): + database = "each_" + uuid.uuid4().hex + for db in (database, database + "_telemetry"): + ch.command(f"CREATE DATABASE `{db}`") + stack.callback(ch.command, f"DROP DATABASE `{db}` SYNC") + pg_name, dsn = asyncio.run(create_database(admin)) + stack.callback(lambda n=pg_name: asyncio.run(drop_database(admin, n))) + env = _cell_env(name, dsn, database, router, certificate[0]) + child(["server.core.schema.migrate"], env) + result = home / f"{name}.json" + org = unique_org("each-" + name) + child(["tests.integration.fixtures.cell_each_seed", str(result), org], env) + node = { + "cell": name, + "org": org, + "env": env, + "origin": env["LOOM_PUBLIC_BASE_URL"], + "credential": env["LOOM_CELL_ROUTER_TOKEN"], + **json.loads(result.read_text()), + } + asyncio.run(_register(router_dsn, node)) + node["process"] = stack.enter_context(ExitStack()) + node["process"].enter_context( + loom_server( + "each-" + name, + env, + port=int(node["origin"].rsplit(":", 1)[1]), + tls_files=certificate, + ) + ) + _bots(node, home, certificate[0]) + _provision(node, router, operator, certificate[0]) + nodes.append(node) + token_file = private_file(home / "operator.token", operator) + manifest = private_file( + home / "credentials.json", + json.dumps( + { + "version": 1, + "router_token_file": str(token_file), + "ca_file": str(certificate[0]), + "keys": [ + { + "cell": n["cell"], + "deployment_id": n["deployment_id"], + "key_file": str(n["automation"]["key_file"]), + } + for n in nodes + ], + } + ), + ) + yield { + "home": home, + "nodes": nodes, + "router": router, + "operator": operator, + "certificate": certificate, + "manifest": manifest, + "router_dsn": router_dsn, + } + + +def target(node: dict[str, Any]) -> dict[str, Any]: + return { + "cell": node["cell"], + "deployment_id": node["deployment_id"], + "origin": node["origin"], + "automation": {k: node["automation"][k] for k in ("name", "thumbprint")}, + "recovery": {k: node["recovery"][k] for k in ("name", "thumbprint")}, + } + + +def admin_read( + f: dict[str, Any], node: dict[str, Any], method: str, body: dict[str, Any] +) -> dict[str, Any]: + with httpx.Client( + verify=ssl.create_default_context(cafile=str(f["certificate"][0])), trust_env=False + ) as client: + answer = client.post( + node["origin"] + "/v1/admin/" + method, + headers={"Authorization": "Bearer " + node["admin_token"]}, + json=body, + timeout=30, + ) + assert answer.status_code == 200, "Ordinary owner readback failed" + return answer.json() + + +async def access(f: dict[str, Any], node: dict[str, Any]) -> str: + import time + + async with httpx.AsyncClient( + verify=wire._tls({"ca_file": f["certificate"][0]}), trust_env=False + ) as client: + token, _end = await wire._token( + client, target(node), node["automation"]["key_file"], time.monotonic() + 30 + ) + answer = await client.get( + node["origin"] + "/v1/whoami", headers={"Authorization": "Bearer " + token} + ) + assert answer.status_code == 200 + identity = answer.json() + assert identity["subject"] == "sa:" + node["automation"]["name"] + assert identity["is_service_account"] is True + assert 0 < identity["exp"] - identity["iat"] <= 60 + return token + + +@contextmanager +def router_trust(f: dict[str, Any], node: dict[str, Any], fault: str) -> Iterator[None]: + """Restart only this fixture's cell, retaining its real stores and public identity.""" + changes = ( + {"SSL_CERT_FILE": str(tls_pair(f["home"], "untrusted")[0])} + if fault == "untrusted" + else {"LOOM_CELL_ROUTER_URL": f["router"].replace("127.0.0.1", "localhost")} + ) + node["process"].close() + port = int(node["origin"].rsplit(":", 1)[1]) + try: + with loom_server( + "each-refusal", {**node["env"], **changes}, port=port, tls_files=f["certificate"] + ): + yield + finally: + node["process"].enter_context( + loom_server("each-" + node["cell"], node["env"], port=port, tls_files=f["certificate"]) + ) diff --git a/tests/integration/_spawn.py b/tests/integration/_spawn.py index b2a35a5785..ff0146d406 100644 --- a/tests/integration/_spawn.py +++ b/tests/integration/_spawn.py @@ -11,6 +11,7 @@ from __future__ import annotations import os +import ssl import subprocess import sys import tempfile @@ -45,6 +46,8 @@ def loom_server( ready_timeout: float = 45.0, log_level: str = "warning", port: int | None = None, + tls_files: tuple[Path, Path] | None = None, + router: bool = False, ) -> Iterator[str]: """Run `server.main:app` with `env` layered over this process's environment; yield its base URL. `port` is kernel-assigned unless the @@ -53,7 +56,13 @@ def loom_server( itself, past the in-process guard.""" ch_target() port = port or free_port() - base = f"http://127.0.0.1:{port}" + base = f"{'https' if tls_files else 'http'}://127.0.0.1:{port}" + app = "server.router.app:build_app" if router else "server.main:app" + extra = ["--factory"] if router else [] + if tls_files is not None: + certificate, key = tls_files + extra += ["--ssl-certfile", str(certificate), "--ssl-keyfile", str(key)] + verify = ssl.create_default_context(cafile=str(tls_files[0])) if tls_files else True # No drain: a test server that kept serving past SIGTERM would outlast the # stop grace and be killed, ten seconds a test. A caller may still set one. full_env = {**os.environ, "LOOM_SHUTDOWN_READY_DELAY_S": "0", **env} @@ -68,7 +77,7 @@ def loom_server( sys.executable, "-m", "uvicorn", - "server.main:app", + app, "--host", "127.0.0.1", "--port", @@ -77,6 +86,7 @@ def loom_server( "asyncio", "--log-level", log_level, + *extra, ], env=full_env, cwd=str(ROOT), @@ -88,7 +98,11 @@ def loom_server( def _up_or_exited() -> bool: if proc.poll() is not None: return True - return httpx.get(f"{base}/v1/health", timeout=1.0).status_code == 200 + path = "/livez" if router else "/v1/health" + return ( + httpx.get(f"{base}{path}", timeout=1.0, verify=verify, trust_env=False).status_code + == 200 + ) try: wait_until(_up_or_exited, max_wait=ready_timeout, interval=0.2, description=name) diff --git a/tests/integration/fixtures/cell_each_seed.py b/tests/integration/fixtures/cell_each_seed.py new file mode 100644 index 0000000000..71825894e9 --- /dev/null +++ b/tests/integration/fixtures/cell_each_seed.py @@ -0,0 +1,49 @@ +"""Prepare one private cell's real stores before its HTTPS process starts.""" + +from __future__ import annotations + +import asyncio +import json +import sys +from pathlib import Path + +from server.auth import keyring, revocation +from server.core import cell, pg, pg_profile +from server.core.ch import get_client, set_query_tag +from server.core.schema import pg_migrate +from server.settings import settings + + +async def seed(result_file: Path, org: str) -> None: + ch = get_client() + await ch.connect() + try: + set_query_tag("test.cell_each.seed") + await pg_migrate.migrate() + await pg_profile.require_authority(ch) + deployment = await pg_profile.deployment_id() + fingerprint = cell.as_value(cell.fingerprint(deployment)) + # These databases have no serving process yet. The rows are the prepared + # fixture state; actual automation metadata is written through the CLI. + await pg.execute("INSERT INTO loom.tenancy (org) VALUES ($1)", org) + await pg.execute( + "INSERT INTO loom.cell_first (is_first, cell) VALUES ($1, $2)", + settings.cell == "alpha", + settings.cell, + ) + await pg.execute("INSERT INTO loom.tenancy_mode (mode) VALUES ('enforced')") + await pg.execute("INSERT INTO loom.cell_self (current_fp) VALUES ($1)", fingerprint) + await keyring.record_root(ch=ch) + assert await keyring.refresh(ch=ch) + await revocation.refresh() + await asyncio.to_thread( + result_file.write_text, + json.dumps({"deployment_id": deployment, "fingerprint": fingerprint}), + ) + finally: + await pg.close_pool() + await ch.disconnect() + + +if __name__ == "__main__": + asyncio.run(seed(Path(sys.argv[1]), sys.argv[2])) diff --git a/tests/integration/test_blacklist_tombstone_version.py b/tests/integration/test_blacklist_tombstone_version.py new file mode 100644 index 0000000000..827dfe97e6 --- /dev/null +++ b/tests/integration/test_blacklist_tombstone_version.py @@ -0,0 +1,120 @@ +"""A refusal tombstone must dominate the selected row's stored version.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from datetime import UTC, datetime, timedelta +from typing import Any +from uuid import uuid4 + +import pytest + +from server.core.ch import get_client, set_query_tag +from server.memory.store.blacklist import insert_row, tombstone_pattern +from tests.integration._resources import unique_org + + +@pytest.fixture +async def pattern_store() -> AsyncIterator[Any]: + client = get_client() + await client.connect() + try: + yield client + finally: + await client.disconnect() + + +def _row(org: str, pattern_id: str, created_at: datetime) -> dict[str, Any]: + return { + "pattern_id": pattern_id, + "org": org, + "namespace": "version-proof", + "pattern": "refusal-" + pattern_id, + "pattern_type": "substring", + "scope": "namespace", + "reason": "retain the selected fields", + "created_by": "process:version-proof", + "hit_count": 7, + "last_hit_at": created_at - timedelta(seconds=1), + "created_at": created_at, + "tombstoned_at": None, + } + + +async def _stored(client: Any, row: dict[str, Any]) -> dict[str, Any]: + result = await client.query_one( + "SELECT pattern_id, org, namespace, pattern, pattern_type, scope, reason, created_by, " + "hit_count, created_at, tombstoned_at, last_hit_at FROM loom.blacklist FINAL " + "WHERE org = {org:String} AND namespace = {ns:String} AND pattern_id = {pid:UUID}", + params={"org": row["org"], "ns": row["namespace"], "pid": row["pattern_id"]}, + ) + assert result is not None + return dict(result) + + +async def _remove_fixture(client: Any, row: dict[str, Any]) -> None: + await client.execute( + "DELETE FROM loom.blacklist WHERE org = {org:String} AND namespace = {ns:String} " + "AND pattern_id = {pid:UUID} SETTINGS lightweight_deletes_sync = 2", + params={"org": row["org"], "ns": row["namespace"], "pid": row["pattern_id"]}, + ) + + +def _comparable(row: dict[str, Any]) -> dict[str, Any]: + return { + key: value.replace(tzinfo=None) + if isinstance(value, datetime) + else str(value) + if key == "pattern_id" + else value + for key, value in row.items() + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("ahead_seconds", [0, 86400], ids=["current", "future"]) +async def test_tombstone_dominates_version_and_preserves_selected_fields( + pattern_store: Any, + ahead_seconds: int, +) -> None: + set_query_tag("test.blacklist.tombstone_version") + now = datetime.now(UTC).replace(tzinfo=None, microsecond=0) + row = _row( + unique_org("blacklist-version"), str(uuid4()), now + timedelta(seconds=ahead_seconds) + ) + try: + await insert_row(row, ch=pattern_store) + before = await _stored(pattern_store, row) + assert _comparable(before) == row + assert before["tombstoned_at"] is None + await tombstone_pattern(row["pattern_id"], now, ch=pattern_store) + after = await _stored(pattern_store, row) + assert after["created_at"].replace(tzinfo=None) > before["created_at"].replace(tzinfo=None) + assert after["tombstoned_at"].replace(tzinfo=None) == now + assert { + key: value for key, value in after.items() if key not in {"created_at", "tombstoned_at"} + } == { + key: value + for key, value in before.items() + if key not in {"created_at", "tombstoned_at"} + } + finally: + await _remove_fixture(pattern_store, row) + + +@pytest.mark.asyncio +async def test_tombstone_refuses_unrepresentable_successor_without_changing_row( + pattern_store: Any, +) -> None: + set_query_tag("test.blacklist.tombstone_overflow") + maximum = datetime(1970, 1, 1) + timedelta(seconds=2**32 - 1) + row = _row(unique_org("blacklist-version"), str(uuid4()), maximum) + try: + await insert_row(row, ch=pattern_store) + before = await _stored(pattern_store, row) + assert _comparable(before) == row + with pytest.raises(ValueError): + await tombstone_pattern(row["pattern_id"], datetime.now(UTC), ch=pattern_store) + assert await _stored(pattern_store, row) == before + finally: + await _remove_fixture(pattern_store, row) diff --git a/tests/integration/test_cell_each_native.py b/tests/integration/test_cell_each_native.py new file mode 100644 index 0000000000..e029015f12 --- /dev/null +++ b/tests/integration/test_cell_each_native.py @@ -0,0 +1,262 @@ +"""Real cell administration through OAuth, the directory and two HTTPS cells. + +The PostgreSQL CI matrix supplies the disposable services. Each cell here has its +own migrated PostgreSQL and ClickHouse databases, root, bot keys and public origin. +No authentication, registration, business handler or endpoint is replaced. +""" + +from __future__ import annotations + +import asyncio +import json +import os +import subprocess +import sys +import uuid +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import httpx +import pytest + +from server.cli import cell_bot_proof, cell_each_run +from server.cli import cell_each_http as wire +from server.cli.cell_each import parse_command +from server.cli.cell_each_state import read_state +from tests.integration._cell_each_fleet import ( + access, + admin_read, + bot_key, + fleet, + router_trust, + target, +) +from tests.integration._spawn import ROOT + +SETTING = "memory_ask_recency_weight" + + +@pytest.fixture(scope="module") +def cells(tmp_path_factory: pytest.TempPathFactory) -> Iterator[dict[str, Any]]: + with fleet(tmp_path_factory.mktemp("cell-each") / "private") as value: + yield value + + +def _row(cells: dict[str, Any], node: dict[str, Any]) -> dict[str, Any]: + rows = admin_read(cells, node, "settings.list", {"prefix": SETTING})["settings"] + assert len(rows) == 1 + return rows[0] + + +def _history(cells: dict[str, Any], node: dict[str, Any]) -> list[dict[str, Any]]: + return admin_read(cells, node, "settings.history", {"name": SETTING, "limit": 50})["changes"] + + +def _argv(cells: dict[str, Any], state: Path, value: str) -> list[str]: + return [ + "--router", + cells["router"], + "--credentials", + str(cells["manifest"]), + "--state", + str(state), + "--", + "settings", + "set", + SETTING, + value, + ] + + +def test_real_provisioning_short_tokens_and_each_write_readback(cells): + nodes = cells["nodes"] + assert nodes[0]["deployment_id"] != nodes[1]["deployment_id"] + separate_pg = nodes[0]["env"]["LOOM_PG_DSN"] != nodes[1]["env"]["LOOM_PG_DSN"] + assert separate_pg + assert nodes[0]["env"]["LOOM_CH_DATABASE"] != nodes[1]["env"]["LOOM_CH_DATABASE"] + before = [_row(cells, node) for node in nodes] + for node in nodes: + asyncio.run(access(cells, node)) + pair = asyncio.run( + cell_bot_proof.prove_pair( + cell=node["cell"], + deployment_id=node["deployment_id"], + origin=node["origin"], + automation=bot_key(node["automation"]), + recovery=bot_key(node["recovery"]), + ca_file=cells["certificate"][0], + ) + ) + assert pair == [ + {"purpose": p, **{k: node[p][k] for k in ("name", "thumbprint")}} + for p in ("automation", "recovery") + ] + # The actual client receives no native connection or root credential. + env = {key: value for key, value in os.environ.items() if not key.startswith("LOOM_")} + state = cells["home"] / "completed.json" + result = subprocess.run( + [sys.executable, "-m", "server.cli", "cell", "each", *_argv(cells, state, "0.625")], + cwd=ROOT, + env=env, + text=True, + capture_output=True, + timeout=120, + ) + assert result.returncode == 0, result.stderr + saved = read_state(state) + assert {name: row["status"] for name, row in saved["outcomes"].items()} == { + "alpha": "completed", + "beta": "completed", + } + for node, old in zip(nodes, before, strict=True): + assert _row(cells, node) == {**old, "value": 0.625, "source": "override"} + completion = saved["outcomes"][node["cell"]]["completion"] + assert completion["started_audit"] != completion["completed_audit"] + + +def test_wrong_key_and_mismatched_deployment_refuse_without_effect(cells): + alpha, beta = cells["nodes"] + before = _row(cells, alpha) + + async def refused(): + async with httpx.AsyncClient( + verify=wire._tls({"ca_file": cells["certificate"][0]}), trust_env=False + ) as client: + # This is a real invalid signature for alpha's registered client, not + # a local thumbprint mismatch that never contacts the token endpoint. + assertion = wire.bot_assertion( + beta["automation"]["key_file"].read_bytes(), + name=alpha["automation"]["name"], + thumbprint=beta["automation"]["thumbprint"], + endpoint=alpha["origin"] + "/oauth/token", + ) + answer = await client.post( + alpha["origin"] + "/oauth/token", + data={ + "grant_type": "client_credentials", + "client_id": alpha["automation"]["name"], + "client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer", + "client_assertion": assertion, + "expires_in": "60", + }, + ) + assert answer.status_code == 401 + token = await access(cells, alpha) + context = wire._context(target(alpha)) + context["expected_deployment_id"] = beta["deployment_id"] + answer = await client.post( + alpha["origin"] + "/v1/admin/cell.capabilities", + headers={"Authorization": "Bearer " + token}, + json={"context": context}, + ) + assert answer.status_code == 403 + assert answer.json()["detail"] == { + "code": "cell_identity_changed", + "category": "auth", + "retryable": False, + } + + asyncio.run(refused()) + assert _row(cells, alpha) == before + + +def test_lost_real_business_reply_stays_unsettled_and_resume_never_replays(cells, monkeypatch): + nodes = cells["nodes"] + before = [_row(cells, node) for node in nodes] + state = cells["home"] / "uncertain.json" + original = httpx.AsyncHTTPTransport + sent: list[str] = [] + lost: list[tuple[int, str]] = [] + + class LostReply(httpx.AsyncBaseTransport): + def __init__(self, **kwargs): + self.real = original(**kwargs) + + async def handle_async_request(self, request): + response = await self.real.handle_async_request(request) + if request.url.path == "/v1/admin/cell.settings_set": + sent.append(str(request.url)) + if request.url.port == int(nodes[1]["origin"].rsplit(":", 1)[1]): + try: + await response.aread() + lost.append((response.status_code, json.loads(response.content)["status"])) + finally: + await response.aclose() + raise httpx.ReadError( + "Reply lost after the real acknowledged write", request=request + ) + return response + + async def aclose(self): + await self.real.aclose() + + with monkeypatch.context() as patch: + patch.setattr(httpx, "AsyncHTTPTransport", LostReply) + assert cell_each_run.main(_argv(cells, state, "0.375")) == 1 + assert sent == [node["origin"] + "/v1/admin/cell.settings_set" for node in nodes] + assert lost == [(200, "completed")] + saved = read_state(state) + assert saved["outcomes"]["alpha"]["status"] == "completed" + assert saved["outcomes"]["beta"]["status"] == "UNSETTLED" + for node, old in zip(nodes, before, strict=True): + assert _row(cells, node) == {**old, "value": 0.375, "source": "override"} + histories = [_history(cells, node) for node in nodes] + original_state = state.read_bytes() + + def no_transport(**_kwargs): + pytest.fail("UNSETTLED resume attempted network access") + + with monkeypatch.context() as patch: + patch.setattr(httpx, "AsyncHTTPTransport", no_transport) + assert ( + cell_each_run.main( + [ + "--router", + cells["router"], + "--credentials", + str(cells["home"] / "missing.json"), + "--resume", + str(state), + ] + ) + == 1 + ) + assert state.read_bytes() == original_state + assert [_history(cells, node) for node in nodes] == histories + + +@pytest.mark.parametrize("fault", ["untrusted", "wrong-host"]) +def test_router_tls_refusal_precedes_business(cells, fault): + node = cells["nodes"][1] + before = _row(cells, node) + history = _history(cells, node) + command = parse_command(["settings", "set", SETTING, "0.8125"]) + + async def refused(): + token = await access(cells, node) + context = { + **wire._context(target(node)), + "run_id": str(uuid.uuid4()), + "attempt": 1, + "command_digest": command.digest, + } + async with httpx.AsyncClient( + verify=wire._tls({"ca_file": cells["certificate"][0]}), trust_env=False + ) as client: + answer = await client.post( + node["origin"] + "/v1/admin/cell.settings_set", + headers={"Authorization": "Bearer " + token}, + json={"context": context, **command.arguments}, + timeout=35, + ) + assert answer.status_code == 503 + result = answer.json() + assert result["status"] == "refused" + assert result["phase"] == "before_business" + assert result["code"] == "security_unavailable" + + with router_trust(cells, node, fault): + asyncio.run(refused()) + assert _row(cells, node) == before + assert _history(cells, node) == history diff --git a/tests/unit/snapshots/cli_help.json b/tests/unit/snapshots/cli_help.json index 8db6694739..09840ebbab 100644 --- a/tests/unit/snapshots/cli_help.json +++ b/tests/unit/snapshots/cli_help.json @@ -434,8 +434,8 @@ "options: -h, --help show this help message and exit" ], "loom cell": [ - "usage: loom cell [-h] {scim-token,credential,limit,which,remove,place,prepare,attach,update} ...", - "positional arguments: {scim-token,credential,limit,which,remove,place,prepare,attach,update} scim-token Generate a cell's SCIM token offline. credential Make a router credential, or rotate a cell's. limit Close, reopen or bound a cell to new orgs. which Print the lookup's cell and host for an org (the first cell if not placed). remove Remove the directory row of a cell attaching with no placement. place Place an org in this cell and record its tenancy. prepare Register this cell in the router's directory and write its rows. attach Attach the prepared first cell: place its orgs, serve. update Reserve configuration, finish or revert its rollout, or refresh verified addresses.", + "usage: loom cell [-h] {scim-token,each,automation,credential,limit,which,remove,place,prepare,attach,update} ...", + "positional arguments: {scim-token,each,automation,credential,limit,which,remove,place,prepare,attach,update} scim-token Generate a cell's SCIM token offline. each Run a supported admin command in every cell automation Prove and register distinct automation and recovery bot keys. credential Make a router credential, or rotate a cell's. limit Close, reopen or bound a cell to new orgs. which Print the lookup's cell and host for an org (the first cell if not placed). remove Remove the directory row of a cell attaching with no placement. place Place an org in this cell and record its tenancy. prepare Register this cell in the router's directory and write its rows. attach Attach the prepared first cell: place its orgs, serve. update Reserve configuration, finish or revert its rollout, or refresh verified addresses.", "options: -h, --help show this help message and exit" ], "loom cell scim-token": [ @@ -447,6 +447,15 @@ "usage: loom cell scim-token new [-h] --cell CELL", "options: -h, --help show this help message and exit --cell CELL The cell's exact LOOM_CELL name" ], + "loom cell each": [ + "usage: loom cell each [-h] --router ROUTER --credentials CREDENTIALS (--state STATE | --resume RESUME) ...", + "positional arguments: command Supported command after --", + "options: -h, --help show this help message and exit --router ROUTER Trusted public router origin --credentials CREDENTIALS Private automation key manifest --state STATE Create an exclusive private outcome file --resume RESUME Resume an existing private outcome file" + ], + "loom cell automation": [ + "usage: loom cell automation [-h] --cell CELL [--router ROUTER] --automation-bot AUTOMATION_BOT --automation-thumbprint AUTOMATION_THUMBPRINT --automation-key AUTOMATION_KEY --recovery-bot RECOVERY_BOT --recovery-thumbprint RECOVERY_THUMBPRINT --recovery-key RECOVERY_KEY [--ca-file CA_FILE]", + "options: -h, --help show this help message and exit --cell CELL The serving cell to provision --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL) --automation-bot AUTOMATION_BOT The automation bot's exact registered name --automation-thumbprint AUTOMATION_THUMBPRINT The automation bot's approved public key thumbprint --automation-key AUTOMATION_KEY The automation bot's private key file --recovery-bot RECOVERY_BOT The recovery bot's exact registered name --recovery-thumbprint RECOVERY_THUMBPRINT The recovery bot's approved public key thumbprint --recovery-key RECOVERY_KEY The recovery bot's private key file --ca-file CA_FILE Private file containing an explicit CA certificate bundle" + ], "loom cell credential": [ "usage: loom cell credential [-h] [--cell CELL] [--next SHA256] [--finish] [--revert] [--operator] [--router ROUTER] [{new}]", "positional arguments: {new} new: make one offline", diff --git a/tests/unit/snapshots/cli_tree.json b/tests/unit/snapshots/cli_tree.json index 03d434256a..6bd28e8e17 100644 --- a/tests/unit/snapshots/cli_tree.json +++ b/tests/unit/snapshots/cli_tree.json @@ -12506,6 +12506,327 @@ } ] }, + "each": { + "prog": "loom cell each", + "usage": null, + "description": null, + "epilog": null, + "formatter_class": "argparse:HelpFormatter", + "add_help": true, + "allow_abbrev": true, + "defaults": { + "func": "server.cli.cell_each_run:_locked" + }, + "groups": [ + { + "title": "positional arguments", + "description": null, + "dests": [ + "command" + ] + }, + { + "title": "options", + "description": null, + "dests": [ + "help", + "router", + "credentials", + "state", + "resume" + ] + } + ], + "mutually_exclusive": [ + { + "required": true, + "dests": [ + "state", + "resume" + ] + } + ], + "arguments": [ + { + "option_strings": [ + "-h", + "--help" + ], + "dest": "help", + "action": "_HelpAction", + "default": "==SUPPRESS==", + "const": null, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "show this help message and exit", + "metavar": null + }, + { + "option_strings": [ + "--router" + ], + "dest": "router", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": true, + "help": "Trusted public router origin", + "metavar": null + }, + { + "option_strings": [ + "--credentials" + ], + "dest": "credentials", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": "pathlib:Path", + "required": true, + "help": "Private automation key manifest", + "metavar": null + }, + { + "option_strings": [ + "--state" + ], + "dest": "state", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": "pathlib:Path", + "required": false, + "help": "Create an exclusive private outcome file", + "metavar": null + }, + { + "option_strings": [ + "--resume" + ], + "dest": "resume", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": "pathlib:Path", + "required": false, + "help": "Resume an existing private outcome file", + "metavar": null + }, + { + "option_strings": [], + "dest": "command", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": "...", + "type": null, + "required": null, + "help": "Supported command after --", + "metavar": null + } + ] + }, + "automation": { + "prog": "loom cell automation", + "usage": null, + "description": null, + "epilog": null, + "formatter_class": "argparse:HelpFormatter", + "add_help": true, + "allow_abbrev": true, + "defaults": { + "func": "server.cli.cell_automation:command" + }, + "groups": [ + { + "title": "positional arguments", + "description": null, + "dests": [] + }, + { + "title": "options", + "description": null, + "dests": [ + "help", + "cell", + "router", + "automation_bot", + "automation_thumbprint", + "automation_key", + "recovery_bot", + "recovery_thumbprint", + "recovery_key", + "ca_file" + ] + } + ], + "mutually_exclusive": [], + "arguments": [ + { + "option_strings": [ + "-h", + "--help" + ], + "dest": "help", + "action": "_HelpAction", + "default": "==SUPPRESS==", + "const": null, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "show this help message and exit", + "metavar": null + }, + { + "option_strings": [ + "--cell" + ], + "dest": "cell", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": true, + "help": "The serving cell to provision", + "metavar": null + }, + { + "option_strings": [ + "--router" + ], + "dest": "router", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "The router's URL (default: LOOM_CELL_ROUTER_URL)", + "metavar": null + }, + { + "option_strings": [ + "--automation-bot" + ], + "dest": "automation_bot", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": true, + "help": "The automation bot's exact registered name", + "metavar": null + }, + { + "option_strings": [ + "--automation-thumbprint" + ], + "dest": "automation_thumbprint", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": true, + "help": "The automation bot's approved public key thumbprint", + "metavar": null + }, + { + "option_strings": [ + "--automation-key" + ], + "dest": "automation_key", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": "pathlib:Path", + "required": true, + "help": "The automation bot's private key file", + "metavar": null + }, + { + "option_strings": [ + "--recovery-bot" + ], + "dest": "recovery_bot", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": true, + "help": "The recovery bot's exact registered name", + "metavar": null + }, + { + "option_strings": [ + "--recovery-thumbprint" + ], + "dest": "recovery_thumbprint", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": true, + "help": "The recovery bot's approved public key thumbprint", + "metavar": null + }, + { + "option_strings": [ + "--recovery-key" + ], + "dest": "recovery_key", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": "pathlib:Path", + "required": true, + "help": "The recovery bot's private key file", + "metavar": null + }, + { + "option_strings": [ + "--ca-file" + ], + "dest": "ca_file", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": "pathlib:Path", + "required": false, + "help": "Private file containing an explicit CA certificate bundle", + "metavar": null + } + ] + }, "credential": { "prog": "loom cell credential", "usage": null, @@ -13592,6 +13913,8 @@ }, "aliases_of": { "scim-token": "loom cell scim-token", + "each": "loom cell each", + "automation": "loom cell automation", "credential": "loom cell credential", "limit": "loom cell limit", "which": "loom cell which", @@ -13607,6 +13930,16 @@ "help": "Generate a cell's SCIM token offline.", "metavar": "scim-token" }, + { + "dest": "each", + "help": "Run a supported admin command in every cell", + "metavar": "each" + }, + { + "dest": "automation", + "help": "Prove and register distinct automation and recovery bot keys.", + "metavar": "automation" + }, { "dest": "credential", "help": "Make a router credential, or rotate a cell's.", diff --git a/tests/unit/snapshots/sdk_public_api.json b/tests/unit/snapshots/sdk_public_api.json index 48adad0e03..94eab93a2c 100644 --- a/tests/unit/snapshots/sdk_public_api.json +++ b/tests/unit/snapshots/sdk_public_api.json @@ -2441,6 +2441,42 @@ "budget_violations": [ "body" ], + "cell_blacklist_global_add": [ + "body" + ], + "cell_blacklist_global_list": [ + "body" + ], + "cell_budget_set": [ + "body" + ], + "cell_capabilities": [ + "body" + ], + "cell_ratelimit_namespace_set": [ + "body" + ], + "cell_settings_history": [ + "body" + ], + "cell_settings_list": [ + "body" + ], + "cell_settings_set": [ + "body" + ], + "cell_signing_keys_list": [ + "body" + ], + "cell_signing_keys_retire": [ + "body" + ], + "cell_signing_keys_rotate": [ + "body" + ], + "cell_subject_revoke": [ + "body" + ], "claim_release": [ "body" ], diff --git a/tests/unit/test_blacklist_authority.py b/tests/unit/test_blacklist_authority.py index a6445d280c..b37c8e4c80 100644 --- a/tests/unit/test_blacklist_authority.py +++ b/tests/unit/test_blacklist_authority.py @@ -78,8 +78,11 @@ async def find_active_pattern(self, pattern_id, *, org=None, **_): return None return dict(row) - async def tombstone_pattern(self, pattern_id, when, **_): + async def tombstone_pattern(self, pattern_id, when, *, selected, **_): + row = self.rows[pattern_id] + assert selected == (row["org"], row["namespace"], row["scope"]) self.tombstoned.append(pattern_id) + return True @pytest.fixture(autouse=True) @@ -236,6 +239,25 @@ def test_remove_unknown_id_is_a_miss(repo: _Repo) -> None: assert out.removed is False and out.removed_at is None +def test_remove_unconfirmed_row_does_not_publish_success(repo: _Repo, monkeypatch) -> None: + repo.rows["p1"] = _row("p1", "namespace") + set_current_auth(_auth("ADMIN")) + published: list[str] = [] + + async def vanished(*args, **kwargs): + return False + + async def bump(**kwargs): + published.append("version") + + monkeypatch.setattr(h.blacklist_repo, "tombstone_pattern", vanished) + monkeypatch.setattr(h, "_bump_version", bump) + monkeypatch.setattr(h, "invalidate_cache", lambda: published.append("cache")) + out = asyncio.run(h.handle_blacklist_remove(BlacklistRemoveInput(org=ORG, pattern_id="p1"))) + assert out.removed is False and out.removed_at is None + assert published == [] + + # ------------------------------------------------- a token narrowed to one user diff --git a/tests/unit/test_blacklist_repo.py b/tests/unit/test_blacklist_repo.py index 9a8febe999..e01beed5df 100644 --- a/tests/unit/test_blacklist_repo.py +++ b/tests/unit/test_blacklist_repo.py @@ -2,6 +2,8 @@ from __future__ import annotations +import json +from contextlib import asynccontextmanager from datetime import UTC, datetime from typing import Any @@ -91,13 +93,131 @@ async def test_find_active_pattern_falls_back_to_global() -> None: assert out == {"pattern_id": "pid-1"} +class _TombstoneCH(StubCH): + """Supplies database rows through the real bounded JSON reader.""" + + def __init__(self, rows: list[dict[str, Any]], *, partition_key: str = "") -> None: + super().__init__() + self.stream_calls: list[tuple[str, dict[str, Any]]] = [] + self.responses = [ + [ + { + "engine": "ReplacingMergeTree", + "engine_full": "ReplacingMergeTree(created_at)", + "sorting_key": "org, namespace, pattern_id", + } + ], + [ + {"name": name, "type": kind} + for name, kind in { + "pattern_id": "UUID", + "org": "LowCardinality(String)", + "namespace": "LowCardinality(String)", + "pattern": "String", + "pattern_type": "Enum8('substring'=1,'regex'=2)", + "scope": "Enum8('namespace'=1,'org'=2,'global'=3)", + "reason": "String", + "created_by": "String", + "hit_count": "UInt64", + "created_at": "DateTime", + "tombstoned_at": "Nullable(DateTime)", + "last_hit_at": "Nullable(DateTime)", + }.items() + ], + rows, + [{"partition_key": partition_key, "now": 1_800_000_000}], + ] + + @asynccontextmanager + async def raw_stream(self, sql, params=None, settings=None): + self.stream_calls.append((sql, params or {})) + rows = self.responses.pop(0) + + async def chunks(): + for row in rows: + yield (json.dumps(row) + "\n").encode() + + yield chunks() + + +def _tombstone_row(created_at: int = 1_700_000_000) -> dict[str, Any]: + return { + "pattern_id": "a046002d-c888-4b54-852b-167f1cb8a443", + "org": "acme", + "namespace": "clinical", + "pattern": "medical", + "pattern_type": "substring", + "scope": "namespace", + "reason": "subject restriction", + "created_by": "admin", + "hit_count": 19, + "created_at": created_at, + "tombstoned_at": None, + "last_hit_at": 1_650_000_000, + } + + @pytest.mark.asyncio -async def test_tombstone_pattern_uses_mutations_sync() -> None: - """mutations_sync=2 so the next list call doesn't see the row.""" - ch = StubCH() - await tombstone_pattern("pid-1", datetime(2026, 5, 17, tzinfo=UTC), ch=ch) - sql, params = ch.execute_calls[0] - assert "ALTER TABLE loom.blacklist" in sql - assert "tombstoned_at = {ts:DateTime}" in sql - assert "mutations_sync=2" in sql - assert params["pid"] == "pid-1" +@pytest.mark.parametrize("created_at", [1_700_000_000, 1_900_000_000]) +@pytest.mark.parametrize("selected", [None, ("acme", "clinical", "namespace")]) +async def test_tombstone_preserves_selected_row_with_newer_version(created_at, selected) -> None: + row = _tombstone_row(created_at) + ch = _TombstoneCH([row]) + when = datetime(2026, 5, 17, tzinfo=UTC) + assert await tombstone_pattern(row["pattern_id"], when, ch=ch, selected=selected) + table, rows, settings = ch.insert_calls[0] + assert table == "loom.blacklist" and len(rows) == 1 + expected = { + **row, + "created_at": datetime.fromtimestamp(max(1_800_000_000, created_at + 1), UTC).replace( + tzinfo=None + ), + "tombstoned_at": when.replace(tzinfo=None), + "last_hit_at": datetime.fromtimestamp(row["last_hit_at"], UTC).replace(tzinfo=None), + } + assert rows[0] == expected + assert settings["async_insert"] == 0 and settings["wait_for_async_insert"] == 1 + assert ch.execute_calls == [] + sql, params = ch.stream_calls[2] + assert "FINAL" in sql and params["pid"] == row["pattern_id"] + if selected is not None: + assert "org = {org:String}" in sql and params["org"] == selected[0] + assert "namespace = {ns:String}" in sql and params["ns"] == selected[1] + assert "scope = {scope:String}" in sql and params["scope"] == selected[2] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("already_removed", [False, True]) +async def test_tombstone_missing_or_already_removed_does_not_write(already_removed) -> None: + row = _tombstone_row() + row["tombstoned_at"] = 1_750_000_000 + ch = _TombstoneCH([row] if already_removed else []) + assert not await tombstone_pattern(row["pattern_id"], datetime(2026, 5, 17, tzinfo=UTC), ch=ch) + assert ch.insert_calls == [] and ch.execute_calls == [] + + +@pytest.mark.asyncio +async def test_tombstone_unrepresentable_version_does_not_write() -> None: + row = _tombstone_row(2**32 - 1) + ch = _TombstoneCH([row]) + with pytest.raises(ValueError, match="successor version"): + await tombstone_pattern(row["pattern_id"], datetime(2026, 5, 17, tzinfo=UTC), ch=ch) + assert ch.insert_calls == [] and ch.execute_calls == [] + + +@pytest.mark.asyncio +async def test_tombstone_partitioned_storage_does_not_write() -> None: + row = _tombstone_row() + ch = _TombstoneCH([row], partition_key="toYYYYMM(created_at)") + with pytest.raises(ValueError, match="replacement storage"): + await tombstone_pattern(row["pattern_id"], datetime(2026, 5, 17, tzinfo=UTC), ch=ch) + assert ch.insert_calls == [] and ch.execute_calls == [] + + +@pytest.mark.asyncio +async def test_tombstone_ambiguous_uuid_does_not_write() -> None: + row = _tombstone_row() + ch = _TombstoneCH([row, {**row, "org": "another"}]) + with pytest.raises(ValueError, match="ambiguous"): + await tombstone_pattern(row["pattern_id"], datetime(2026, 5, 17, tzinfo=UTC), ch=ch) + assert ch.insert_calls == [] and ch.execute_calls == [] diff --git a/tests/unit/test_cell_admin_json.py b/tests/unit/test_cell_admin_json.py new file mode 100644 index 0000000000..4705370de1 --- /dev/null +++ b/tests/unit/test_cell_admin_json.py @@ -0,0 +1,48 @@ +"""Cell administration rejects ambiguous JSON without changing the byte-cap contract.""" + +from __future__ import annotations + +import json +from unittest.mock import AsyncMock + +import httpx +import pytest +from fastapi import FastAPI + +from server.insights.app import cell_admin_routes as routes +from server.insights.app.admin_routes import require_admin +from server.settings import settings +from tests.unit.principals import principal + + +@pytest.mark.asyncio +@pytest.mark.parametrize("extra", ["", ',"context":null', ',"unused":NaN', ',"unused":1e999']) +async def test_authenticated_raw_json_is_strict_before_security_reads(monkeypatch, extra): + auth = principal("platform_admin", subject="sa:automation", namespaces=(), token_id="token") + monkeypatch.setattr(type(settings), "auth_actually_required", lambda self: True) + verified = AsyncMock() + monkeypatch.setattr(routes, "verify_automation", verified) + context = { + "version": 1, + "expected_cell": "alpha", + "expected_deployment_id": "deployment", + "expected_bot": "automation", + } + app = FastAPI() + app.include_router(routes.cell_admin_router) + + async def authenticated(): + return auth + + app.dependency_overrides[require_admin] = authenticated + raw = '{"context":' + json.dumps(context) + extra + "}" + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="https://cell.example" + ) as client: + response = await client.post( + "/v1/admin/cell.capabilities", + content=raw, + headers={"Content-Type": "application/json"}, + ) + assert response.status_code == (400 if extra else 200) + assert verified.await_count == int(not extra) diff --git a/tests/unit/test_cell_admin_subject_route.py b/tests/unit/test_cell_admin_subject_route.py new file mode 100644 index 0000000000..1ce735fbac --- /dev/null +++ b/tests/unit/test_cell_admin_subject_route.py @@ -0,0 +1,35 @@ +"""The exact subject adapter is exposed through the ordinary positive bot guard.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock + +import httpx +import pytest +from fastapi import FastAPI + +from server.insights.app import cell_admin_routes as routes +from server.settings import settings + + +@pytest.mark.asyncio +async def test_subject_route_refuses_authoff_before_business(monkeypatch): + monkeypatch.setattr(type(settings), "auth_actually_required", lambda self: False) + business = AsyncMock(side_effect=AssertionError("business reached without authority")) + monkeypatch.setattr(routes.security_actions, "subject_revoke", business) + app = FastAPI() + app.include_router(routes.cell_admin_router) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="https://cell.example" + ) as client: + response = await client.post("/v1/admin/cell.subject_revoke", json={}) + assert response.status_code == 403 + business.assert_not_awaited() + + +def test_capabilities_advertise_only_the_implemented_subject_and_blacklist_actions(): + assert "subject_revoke" in routes.AVAILABLE_ACTIONS + assert {action for action in routes.AVAILABLE_ACTIONS if action.startswith("blacklist_")} == { + "blacklist_global_add", + "blacklist_global_list", + } diff --git a/tests/unit/test_cell_admin_trust.py b/tests/unit/test_cell_admin_trust.py new file mode 100644 index 0000000000..6c0c526000 --- /dev/null +++ b/tests/unit/test_cell_admin_trust.py @@ -0,0 +1,53 @@ +"""Operator TLS trust does not enable proxy, redirect or verification bypasses.""" + +from __future__ import annotations + +import json +import ssl +import time + +import httpx +import pytest + +from server.insights.api.models_cell_admin import CellTargetContext +from server.insights.app import cell_admin_auth as authority +from server.settings import settings + + +@pytest.mark.asyncio +async def test_router_authority_uses_explicit_default_trust_without_proxies(monkeypatch): + context = CellTargetContext( + version=1, expected_cell="alpha", expected_deployment_id="deployment", expected_bot="auto" + ) + monkeypatch.setattr(settings, "cell", "alpha") + monkeypatch.setattr(settings, "cell_router_url", "https://router.example") + monkeypatch.setattr(settings, "cell_router_token", "private-cell-token") + trust = ssl.create_default_context() + monkeypatch.setattr(authority.ssl, "create_default_context", lambda: trust) + original = httpx.AsyncClient + observed = [] + + def reply(request): + observed.append(request) + value = { + "version": 1, + "cell": "alpha", + "deployment_id": "deployment", + "automation": {"name": "auto", "thumbprint": "A" * 43}, + "recovery": {"name": "recovery", "thumbprint": "B" * 43}, + } + return httpx.Response(200, stream=httpx.ByteStream(json.dumps(value).encode())) + + def client(**kwargs): + assert kwargs["verify"] is trust + assert kwargs["trust_env"] is False + assert kwargs["follow_redirects"] is False + assert trust.verify_mode == ssl.CERT_REQUIRED and trust.check_hostname + return original(**kwargs, transport=httpx.MockTransport(reply)) + + monkeypatch.setattr(httpx, "AsyncClient", client) + binding = await authority._authority(context, time.monotonic() + 5) + assert binding.name == "auto" and binding.thumbprint == "A" * 43 + assert len(observed) == 1 + assert str(observed[0].url) == "https://router.example/cells/v1/cells/alpha/administration" + assert observed[0].headers["Authorization"] == "Bearer private-cell-token" diff --git a/tests/unit/test_cell_automation.py b/tests/unit/test_cell_automation.py new file mode 100644 index 0000000000..ad9efa1346 --- /dev/null +++ b/tests/unit/test_cell_automation.py @@ -0,0 +1,236 @@ +"""Only a proved public bot pair may reach fenced directory provisioning.""" + +from __future__ import annotations + +import argparse +import contextlib +import copy +import json +from pathlib import Path +from typing import Any +from unittest.mock import AsyncMock +from uuid import UUID + +import httpx +import pytest +from starlette.testclient import TestClient + +from server.cli import cell_bot_proof, cell_control +from server.router import app as router_app +from server.router import directory +from server.settings import settings +from tests.unit.test_router_attach import OPERATOR, _cell_row, _Conn, _Store + + +def pair() -> list[dict[str, str]]: + return [ + {"purpose": purpose, "name": purpose, "thumbprint": thumbprint * 43} + for purpose, thumbprint in (("automation", "A"), ("recovery", "B")) + ] + + +def test_cell_cli_exposes_explicit_distinct_bot_key_inputs(): + from server.cli.cell import _add_cell_parser + + parser = argparse.ArgumentParser() + _add_cell_parser(parser.add_subparsers()) + commands = ( + parser._subparsers._group_actions[0].choices["cell"]._subparsers._group_actions[0].choices + ) + assert "automation" in commands + args = parser.parse_args( + [ + "cell", + "automation", + "--cell", + "eu-1", + "--automation-bot", + "automation", + "--automation-thumbprint", + "A" * 43, + "--automation-key", + "/automation", + "--recovery-bot", + "recovery", + "--recovery-thumbprint", + "B" * 43, + "--recovery-key", + "/recovery", + ] + ) + assert (args.automation_bot, args.recovery_bot) == ("automation", "recovery") + assert (args.automation_key, args.recovery_key) == (Path("/automation"), Path("/recovery")) + + +@pytest.fixture +def store(monkeypatch): + import hashlib + + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(OPERATOR.encode()).hexdigest() + ) + state = _Store(_cell_row(state="serving", mode="enforced", break_glass=[])) + + class Conn(_Conn): + async def execute(self, sql: str, *args: Any) -> str: + if "SET break_glass = $2" in sql: + self._count() + self.s.cells[args[0]]["break_glass"] = copy.deepcopy(args[1]) + return "UPDATE 1" + return await super().execute(sql, *args) + + @contextlib.asynccontextmanager + async def connection(**_kwargs): + yield Conn(state) + + monkeypatch.setattr(directory, "_connection", connection) + return state + + +def post(client, state, entries, *, generation=None, token=OPERATOR): + row = state.cells["eu-1"] + expected = {name: row[name] for name in directory.CONTROL_FIELDS} + expected["mutation_generation"] = str(generation or row["mutation_generation"]) + return client.post( + "/cells/v1/operator/cells/eu-1/administration", + headers={"Authorization": "Bearer " + token, "Content-Type": "application/json"}, + content=json.dumps({"expected": expected, "bots": entries}, ensure_ascii=True), + ) + + +def test_metadata_write_records_only_public_pair_and_audits(store): + before = copy.deepcopy(store.cells["eu-1"]) + client = TestClient(router_app.build_app(router_app.Router())) + answer = post(client, store, pair()) + assert answer.status_code == 200 + assert answer.json() == { + "version": 1, + "cell": "eu-1", + "deployment_id": "dep-1", + **{ + entry["purpose"]: {key: entry[key] for key in ("name", "thumbprint")} + for entry in pair() + }, + } + assert store.cells["eu-1"] == {**before, "break_glass": pair()} + assert store.audit == [("operator", "administration", {"cell": "eu-1", "bots": pair()})] + + +@pytest.mark.parametrize( + "fault", ["credential", "generation", "same-key", "secret", "missing", "surrogate"] +) +def test_invalid_provisioning_never_changes_metadata_or_audit(store, fault): + entries = pair() + kwargs = {} + if fault == "credential": + kwargs["token"] = "wrong" + elif fault == "generation": + kwargs["generation"] = UUID(int=99) + elif fault == "same-key": + entries[1]["thumbprint"] = entries[0]["thumbprint"] + elif fault == "secret": + entries[0]["key_file"] = "/private/key" + elif fault == "surrogate": + entries[0]["name"] = "\ud800" + else: + entries.pop() + before = copy.deepcopy(store.cells) + client = TestClient(router_app.build_app(router_app.Router())) + answer = post(client, store, entries, **kwargs) + assert answer.status_code in {401, 400, 409} + assert store.cells == before and store.audit == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("proof_fails", [False, True]) +async def test_live_pair_proof_precedes_the_only_metadata_mutation(monkeypatch, proof_fails): + from server.cli import cell_automation + + row = _cell_row(state="serving", mode="enforced") + row["mutation_generation"] = str(row["mutation_generation"]) + args = argparse.Namespace( + cell="eu-1", + automation_bot="automation", + automation_thumbprint="A" * 43, + automation_key=Path("/automation"), + recovery_bot="recovery", + recovery_thumbprint="B" * 43, + recovery_key=Path("/recovery"), + ca_file=None, + ) + events = [] + + async def request(_url, _name, _transport, path, body=None, **kwargs): + events.append(path or "read") + if path == "/administration": + assert body["bots"] == pair() + assert body["expected"]["mutation_generation"] == str(UUID(int=2)) + return { + "version": 1, + "cell": "eu-1", + "deployment_id": "dep-1", + **{ + entry["purpose"]: {key: entry[key] for key in ("name", "thumbprint")} + for entry in pair() + }, + } + if path == "/claim": + return {**row, "mutation_generation": str(UUID(int=2))} + return row + + async def prove(**kwargs): + events.append("proof") + assert kwargs["origin"] == "https://loom.example" + assert isinstance(kwargs["automation"], cell_bot_proof.BotKey) + if proof_fails: + raise ValueError("proof refused") + return pair() + + class Owner: + conn = object() + + async def call(self, work, *, total_s): + return await work() + + monkeypatch.setattr(cell_control, "request", request) + monkeypatch.setattr(cell_control, "paired", AsyncMock()) + monkeypatch.setattr(cell_control, "check_lock", AsyncMock()) + monkeypatch.setattr(cell_bot_proof, "prove_pair", prove) + work = cell_automation.provision(Owner(), httpx.URL("https://router.example"), args) + if proof_fails: + with pytest.raises(ValueError, match="proof refused"): + await work + assert events == ["read", "/claim", "proof"] + else: + answer = await work + assert answer["automation"]["name"] == "automation" + assert events == ["read", "/claim", "proof", "/administration"] + + +@pytest.mark.asyncio +async def test_configured_authority_mismatch_prevents_claim_and_key_proof(monkeypatch): + from server.cli import cell_automation + + row = _cell_row(state="serving", mode="enforced") + row["mutation_generation"] = str(row["mutation_generation"]) + request = AsyncMock(return_value=row) + proof = AsyncMock(side_effect=AssertionError("proof before authority pairing")) + monkeypatch.setattr(cell_control, "request", request) + monkeypatch.setattr(cell_control, "check_lock", AsyncMock()) + monkeypatch.setattr(cell_bot_proof, "prove_pair", proof) + + class Owner: + class Connection: + async def fetchval(self, sql): + assert sql == "SELECT deployment_id FROM loom.authority" + return "different-deployment" + + conn = Connection() + + with pytest.raises(cell_control.Refused, match="do not match"): + await cell_automation.provision( + Owner(), httpx.URL("https://router.example"), argparse.Namespace(cell="eu-1") + ) + assert request.await_count == 1 + assert request.call_args.kwargs == {"method": "GET"} + proof.assert_not_awaited() diff --git a/tests/unit/test_cell_blacklist_storage.py b/tests/unit/test_cell_blacklist_storage.py new file mode 100644 index 0000000000..bba05ed7d5 --- /dev/null +++ b/tests/unit/test_cell_blacklist_storage.py @@ -0,0 +1,65 @@ +"""Global blacklist admission accepts the real engine clause and configured catalog.""" + +from __future__ import annotations + +import time + +import pytest + +from server.core.ch_bounded import BoundedReadError +from server.memory.store import blacklist +from server.settings import settings + + +def catalog(engine_full, database="loom"): + async def rows(sql, params, **kwargs): + if database != "loom" and params.get("database") != database: + return [] + if "system.tables" in sql: + return [ + { + "engine": engine_full.split("(", 1)[0], + "engine_full": engine_full, + "sorting_key": "org, namespace, pattern_id", + } + ] + return [{"name": name, "type": kind} for name, kind in blacklist._COLUMN_TYPES.items()] + + return rows + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "engine", + [ + "ReplacingMergeTree(created_at) ORDER BY (org, namespace, pattern_id) SETTINGS index_granularity = 8192", + "ReplicatedReplacingMergeTree('/tables/blacklist', 'replica', created_at) ORDER BY (org, namespace, pattern_id) SETTINGS index_granularity = 8192", + "SharedReplacingMergeTree('/tables/black(list)', 'replica', created_at) ORDER BY (org, namespace, pattern_id)", + ], +) +async def test_real_engine_full_admits_supported_global_actions(monkeypatch, engine): + monkeypatch.setattr(blacklist, "read_rows", catalog(engine)) + await blacklist.check_bounded_storage(ch=object(), deadline=time.monotonic() + 30) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "engine", + [ + "ReplacingMergeTree(hit_count) ORDER BY (org, namespace, pattern_id)", + "ReplicatedReplacingMergeTree('/tables/created_at)', 'replica', hit_count) ORDER BY (org, namespace, pattern_id)", + ], +) +async def test_another_replacement_version_never_admits_global_actions(monkeypatch, engine): + monkeypatch.setattr(blacklist, "read_rows", catalog(engine)) + with pytest.raises(BoundedReadError, match="Unsupported blacklist storage"): + await blacklist.check_bounded_storage(ch=object(), deadline=time.monotonic() + 30) + + +@pytest.mark.asyncio +async def test_storage_admission_reads_the_configured_database(monkeypatch): + monkeypatch.setattr(settings, "ch_database", "private_cell") + monkeypatch.setattr( + blacklist, "read_rows", catalog("ReplacingMergeTree(created_at)", "private_cell") + ) + await blacklist.check_bounded_storage(ch=object(), deadline=time.monotonic() + 30) diff --git a/tests/unit/test_cell_control_census.py b/tests/unit/test_cell_control_census.py index b80384b90e..f45df7da60 100644 --- a/tests/unit/test_cell_control_census.py +++ b/tests/unit/test_cell_control_census.py @@ -116,6 +116,7 @@ def test_cell_control_migrations_are_classified(): }, "server/router/directory.py": { ("update_addresses.change", "UPDATE", "loom_router.cells"): 1, + ("set_administration.change_locked", "UPDATE", "loom_router.cells"): 1, ("set_limit.change", "UPDATE", "loom_router.cells"): 1, ("rotate_credential.change", "UPDATE", "loom_router.cells"): 1, ("claim.change_locked", "UPDATE", "loom_router.cells"): 1, @@ -353,6 +354,19 @@ def function(path: str, name: str) -> ast.AST: "configuration_placements", ["_attach_change(cell_name, expected, read)"], ), + ( + "server/cli/cell_automation.py", + "command", + ['cell_control.run("automation", lambda owner: provision(owner, router, args))'], + ), + ( + "server/cli/cell_automation.py", + "provision", + [ + "cell_control.claim(owner.conn, call, row)", + 'cell_control.mutate(owner.conn, call, row, "/administration", {"bots": bots})', + ], + ), ( "server/cli/cell_prepare.py", "prepare", @@ -498,6 +512,7 @@ def function(path: str, name: str) -> ast.AST: ["_attach_change(cell_name, expected, change_locked)"], ) for name in ( + "set_administration", "attach_begin", "attach_place", "attach_enforced", diff --git a/tests/unit/test_cell_each_admin_contract.py b/tests/unit/test_cell_each_admin_contract.py new file mode 100644 index 0000000000..2febe090de --- /dev/null +++ b/tests/unit/test_cell_each_admin_contract.py @@ -0,0 +1,167 @@ +"""Cell action models and admission refuse authority that cannot name a fresh bot.""" + +from __future__ import annotations + +import importlib +from typing import Any + +import httpx +import pytest + +ACTION_PATHS = [ + "capabilities", + "settings_list", + "settings_set", + "settings_history", + "signing_keys_list", + "signing_keys_rotate", + "signing_keys_retire", + "subject_revoke", +] + + +def _context() -> dict[str, Any]: + return { + "version": 1, + "expected_cell": "alpha", + "expected_deployment_id": "deploy-alpha", + "expected_bot": "automation-alpha", + "run_id": "a046002d-c888-4b54-852b-167f1cb8a443", + "attempt": 1, + "command_digest": "1" * 64, + } + + +def test_typed_inputs_keep_the_raw_setting_and_exact_global_subject() -> None: + models = importlib.import_module("server.insights.api.models_cell_admin") + setting = models.CellSettingsSetInput.model_validate( + { + "context": _context(), + "name": "memory_ask_recency_weight", + "value": "0.25", + "clear": False, + "expected": None, + } + ) + subject = models.CellSubjectRevokeInput.model_validate( + {"context": _context(), "subject": "sa:worker", "reason": "retire"} + ) + assert setting.value == "0.25" and setting.clear is False + assert subject.subject == "sa:worker" and subject.reason == "retire" + + +@pytest.mark.parametrize("action", ACTION_PATHS) +async def test_authoff_cannot_admit_cell_administration( + action: str, monkeypatch: pytest.MonkeyPatch +) -> None: + from fastapi import FastAPI + + from server.settings import settings + + routes = importlib.import_module("server.insights.app.cell_admin_routes") + monkeypatch.setattr(type(settings), "auth_actually_required", lambda self: False) + app = FastAPI() + app.include_router(routes.cell_admin_router) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="https://cell.example.com" + ) as client: + response = await client.post(f"/v1/admin/cell.{action}", json={}) + # Authority is decided before malformed body validation or store/business work. + assert response.status_code == 403 + + +@pytest.mark.parametrize( + "fields,expected_status", + [ + ({"service_account": False}, 403), + ({"subject": "person@example.com"}, 403), + ({"creator": "person@example.com"}, 403), + ({"agent_principal": "agent:worker"}, 403), + ({"ns_roles": (("work", "admin"),)}, 403), + ({"namespaces": ("work",)}, 403), + ({"impersonated_by": "person@example.com"}, 403), + ({}, 422), + ], + ids=[ + "human", + "subject", + "delegated", + "agent", + "scoped-role", + "namespace", + "impersonated", + "healthy", + ], +) +async def test_verified_platform_role_does_not_replace_the_positive_bot_guard( + fields: dict[str, Any], + expected_status: int, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from fastapi import FastAPI + + from server.insights.app.admin_routes import require_admin + from server.settings import settings + from tests.unit.principals import principal + + monkeypatch.setattr(settings, "auth_required", True) + monkeypatch.setattr(settings, "identity_provider", "") + + routes = importlib.import_module("server.insights.app.cell_admin_routes") + options = { + "subject": "sa:automation-alpha", + "namespaces": (), + "token_id": "automation-token", + **fields, + } + auth = principal("platform_admin", **options) + + async def verified_admin() -> Any: + return auth + + app = FastAPI() + app.include_router(routes.cell_admin_router) + app.dependency_overrides[require_admin] = verified_admin + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="https://cell.example.com" + ) as client: + response = await client.post("/v1/admin/cell.settings_set", json={}) + assert response.status_code == expected_status + + +@pytest.mark.parametrize( + "extra", ["org", "namespace", "actor", "function", "url", "issued_before", "expires_at", "scim"] +) +def test_global_subject_input_cannot_be_widened_or_restore_access(extra: str) -> None: + from pydantic import ValidationError + + models = importlib.import_module("server.insights.api.models_cell_admin") + with pytest.raises(ValidationError): + models.CellSubjectRevokeInput.model_validate( + {"context": _context(), "subject": "sa:worker", "reason": "retire", extra: "unexpected"} + ) + + +@pytest.mark.parametrize("subject", ["", "acme/user:member", "sa:bad\x00key", "s" * 513]) +def test_global_subject_input_rejects_ambiguous_or_oversized_subjects(subject: str) -> None: + from pydantic import ValidationError + + models = importlib.import_module("server.insights.api.models_cell_admin") + with pytest.raises(ValidationError): + models.CellSubjectRevokeInput.model_validate({"context": _context(), "subject": subject}) + + +@pytest.mark.parametrize("bad", [float("nan"), float("inf"), -float("inf")]) +def test_nested_nonfinite_setting_input_refuses_before_business_coercion(bad: float) -> None: + from pydantic import ValidationError + + models = importlib.import_module("server.insights.api.models_cell_admin") + with pytest.raises(ValidationError): + models.CellSettingsSetInput.model_validate( + { + "context": _context(), + "name": "memory_ask_recency_weight", + "value": {"nested": [bad]}, + "clear": False, + } + ) diff --git a/tests/unit/test_cell_each_body_limit.py b/tests/unit/test_cell_each_body_limit.py new file mode 100644 index 0000000000..2372e349c9 --- /dev/null +++ b/tests/unit/test_cell_each_body_limit.py @@ -0,0 +1,205 @@ +"""Fixed administration caps precede JSON parsing when the global cap is off.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +PATHS = [ + ("capabilities", 1024), + ("settings_list", 65536), + ("settings_set", 65536), + ("settings_history", 65536), + ("signing_keys_list", 65536), + ("signing_keys_rotate", 65536), + ("signing_keys_retire", 65536), + ("subject_revoke", 65536), +] + + +async def _request(path: str, body: bytes, *, declared: bool) -> tuple[int, list[bytes], int]: + from server.transport.body_limit import BodyLimitMiddleware + + messages: list[dict[str, Any]] = [] + delivered: list[bytes] = [] + reads = 0 + chunks = [body] if declared else [body[:-1], body[-1:]] + + async def receive() -> dict[str, Any]: + nonlocal reads + reads += 1 + chunk = chunks.pop(0) + return {"type": "http.request", "body": chunk, "more_body": bool(chunks)} + + async def send(message: dict[str, Any]) -> None: + messages.append(message) + + async def downstream(scope: Any, receive: Any, send: Any) -> None: + while True: + message = await receive() + delivered.append(message["body"]) + if not message.get("more_body"): + break + await send({"type": "http.response.start", "status": 200, "headers": []}) + await send({"type": "http.response.body", "body": b"{}"}) + + scope = { + "type": "http", + "path": path, + "method": "POST", + "headers": [(b"content-length", str(len(body)).encode())] if declared else [], + } + await BodyLimitMiddleware(downstream, max_bytes=0)(scope, receive, send) + return messages[0]["status"], delivered, reads + + +@pytest.mark.parametrize(("action", "limit"), PATHS) +@pytest.mark.parametrize("declared", [True, False], ids=["declared", "chunked"]) +async def test_feature_body_overflow_is_cut_off_before_materialization( + action: str, limit: int, declared: bool +) -> None: + body = b"x" * (limit + 1) + status, delivered, reads = await _request(f"/v1/admin/cell.{action}", body, declared=declared) + assert status == 413 + assert len(b"".join(delivered)) <= limit + if declared: + assert reads == 0 and delivered == [] + + +@pytest.mark.parametrize(("action", "limit"), PATHS) +async def test_feature_body_at_the_limit_reaches_the_handler(action: str, limit: int) -> None: + body = b"x" * limit + status, delivered, _ = await _request(f"/v1/admin/cell.{action}", body, declared=False) + assert status == 200 and b"".join(delivered) == body + + +async def test_disabling_global_cap_keeps_ordinary_paths_unchanged() -> None: + body = b"x" * 65537 + status, delivered, _ = await _request("/ordinary", body, declared=True) + assert status == 200 and b"".join(delivered) == body + + +@pytest.fixture +def authenticated_settings_router(monkeypatch: pytest.MonkeyPatch) -> tuple[Any, Any, Any]: + from unittest.mock import AsyncMock + + from fastapi import FastAPI + + from server.insights.app import cell_admin_routes as routes + from server.insights.app.admin_routes import require_admin + from server.insights.app.cell_admin_auth import AutomationBinding, VerifiedCellAdmin + from server.settings import settings + from tests.unit.principals import principal + + auth = principal( + "platform_admin", namespaces=(), subject="sa:automation-alpha", token_id="automation-token" + ) + + async def verified_admin() -> Any: + return auth + + monkeypatch.setattr(settings, "auth_required", True) + monkeypatch.setattr(settings, "identity_provider", "") + verified = VerifiedCellAdmin( + auth, AutomationBinding("alpha", "deploy-alpha", "automation-alpha", "A" * 43) + ) + monkeypatch.setattr(routes, "verify_automation", AsyncMock(return_value=verified)) + business = AsyncMock(return_value={"updated": True}) + audit = AsyncMock(side_effect=["1" * 64, "2" * 64]) + monkeypatch.setattr(routes.actions, "settings_set", business) + monkeypatch.setattr(routes.audit, "emit_strict", audit) + app = FastAPI() + app.include_router(routes.cell_admin_router) + app.dependency_overrides[require_admin] = verified_admin + return app, business, audit + + +def _settings_action_prefix() -> bytes: + import json + + from server.core.hashing import sha256_value + + arguments = { + "name": "memory_ask_recency_weight", + "value": "0.25", + "clear": False, + "expected": None, + } + context = { + "version": 1, + "expected_cell": "alpha", + "expected_deployment_id": "deploy-alpha", + "expected_bot": "automation-alpha", + "run_id": "a046002d-c888-4b54-852b-167f1cb8a443", + "attempt": 1, + "command_digest": sha256_value({"action": "settings_set", "arguments": arguments}), + } + return json.dumps({"context": context, **arguments}).encode() + + +async def _settings_chunks(app: Any, prefix: bytes, suffix: bytes) -> list[dict[str, Any]]: + from server.transport.body_limit import BodyLimitMiddleware + + chunks = [prefix, suffix] + messages: list[dict[str, Any]] = [] + + async def receive() -> dict[str, Any]: + chunk = chunks.pop(0) + return {"type": "http.request", "body": chunk, "more_body": bool(chunks)} + + async def send(message: dict[str, Any]) -> None: + messages.append(message) + + scope = { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": "POST", + "scheme": "https", + "path": "/v1/admin/cell.settings_set", + "root_path": "", + "query_string": b"", + "headers": [(b"content-type", b"application/json")], + "client": ("127.0.0.1", 1), + "server": ("cell.example", 443), + } + await BodyLimitMiddleware(app, max_bytes=0)(scope, receive, send) + assert chunks == [] + return messages + + +@pytest.mark.parametrize("suffix_byte", [b" ", b"!"], ids=["whitespace", "invalid-json"]) +async def test_valid_action_prefix_overflow_never_runs_business_or_audit( + suffix_byte: bytes, authenticated_settings_router: tuple[Any, Any, Any] +) -> None: + import json + + app, business, audit = authenticated_settings_router + prefix = _settings_action_prefix() + suffix = suffix_byte * (65537 - len(prefix)) + messages = await _settings_chunks(app, prefix, suffix) + assert messages[0]["status"] == 413 + body = json.loads(b"".join(message.get("body", b"") for message in messages)) + assert body["detail"]["code"] == "PAYLOAD_TOO_LARGE" + assert body["detail"]["limit_bytes"] == 65536 + business.assert_not_awaited() + audit.assert_not_awaited() + + +async def test_valid_action_at_limit_runs_business_and_both_audits( + authenticated_settings_router: tuple[Any, Any, Any], +) -> None: + import json + + app, business, audit = authenticated_settings_router + prefix = _settings_action_prefix() + messages = await _settings_chunks(app, prefix, b" " * (65536 - len(prefix))) + assert messages[0]["status"] == 200 + body = json.loads(b"".join(message.get("body", b"") for message in messages)) + assert body["status"] == "completed" and body["result"] == {"updated": True} + business.assert_awaited_once() + assert [call.args[0] for call in audit.await_args_list] == [ + "cell.each_started", + "cell.each_completed", + ] diff --git a/tests/unit/test_cell_each_command_extensions.py b/tests/unit/test_cell_each_command_extensions.py new file mode 100644 index 0000000000..91a6ae5b7e --- /dev/null +++ b/tests/unit/test_cell_each_command_extensions.py @@ -0,0 +1,193 @@ +"""Closed command normalization preserves scope, identity and immutable intent.""" + +from __future__ import annotations + +import uuid + +import pytest + + +@pytest.mark.parametrize( + ("tokens", "action", "arguments", "mutating"), + [ + ( + [ + "ratelimit", + "set", + "--namespace", + "work", + "--op-class", + "memory", + "--capacity", + "2", + "--rate", + "0", + ], + "ratelimit_namespace_set", + { + "namespace": "work", + "op_class": "memory", + "capacity": 2.0, + "rate": 0.0, + "clear": False, + }, + True, + ), + ( + ["ratelimit", "reset", "--namespace", "work", "--op-class", "memory"], + "ratelimit_namespace_set", + { + "namespace": "work", + "op_class": "memory", + "capacity": None, + "rate": None, + "clear": True, + }, + True, + ), + ( + ["budget", "set", "memory.get", "--budget-ms", "0"], + "budget_set", + {"op": "memory.get", "consumer": None, "budget_ms": 0, "budget": None, "clear": False}, + True, + ), + ( + ["budget", "set", "*", "--consumer", "worker", "--multiplier", "1.2500"], + "budget_set", + {"op": "*", "consumer": "worker", "budget_ms": None, "budget": "1.25x", "clear": False}, + True, + ), + ( + ["budget", "reset", "memory.get", "--consumer", "worker"], + "budget_set", + { + "op": "memory.get", + "consumer": "worker", + "budget_ms": None, + "budget": None, + "clear": True, + }, + True, + ), + (["blacklist", "global", "list"], "blacklist_global_list", {"limit": 50}, False), + ( + ["blacklist", "global", "remove", "7fdc7f78-5946-478e-81dc-b3effcc71f9c"], + "blacklist_global_remove", + {"pattern_id": "7fdc7f78-5946-478e-81dc-b3effcc71f9c"}, + True, + ), + ], +) +def test_fixed_extension_shapes(tokens, action, arguments, mutating): + from server.cli.cell_each import parse_command + + command = parse_command(tokens) + assert command.action == action + assert command.path == "/v1/admin/cell." + action + assert command.arguments == arguments + assert command.mutating is mutating + + +def test_add_allocates_one_common_pattern_identity_after_validation(monkeypatch): + from server.cli import cell_each + + allocated = [] + identifier = uuid.UUID("7fdc7f78-5946-478e-81dc-b3effcc71f9c") + + def allocate(): + allocated.append(identifier) + return identifier + + monkeypatch.setattr(cell_each.uuid, "uuid4", allocate) + command = cell_each.parse_command( + ["blacklist", "global", "add", "denied", "--reason", "policy"] + ) + assert command.action == "blacklist_global_add" + assert command.arguments == { + "pattern_id": str(identifier), + "pattern": "denied", + "pattern_type": "substring", + "reason": "policy", + } + assert allocated == [identifier] + command.arguments["pattern_id"] = "changed" + assert command.arguments["pattern_id"] == str(identifier) + assert allocated == [identifier] + + +@pytest.mark.parametrize( + "tokens", + [ + ["policy", "set", "*", "*"], + ["ratelimit", "set", "--org", "acme", "--capacity", "2", "--rate", "1"], + ["ratelimit", "reset", "--namespace", "*", "--op-class", "memory"], + [ + "ratelimit", + "set", + "--namespace", + "work", + "--op-class", + "memory", + "--capacity", + "0", + "--rate", + "1", + ], + [ + "ratelimit", + "set", + "--namespace", + "work", + "--op-class", + "memory", + "--capacity", + "2", + "--rate", + "nan", + ], + ["budget", "set", "*", "--budget-ms", "1"], + ["budget", "set", "memory.get", "--multiplier", "2"], + [ + "budget", + "set", + "memory.get", + "--budget-ms", + "1", + "--multiplier", + "2", + "--consumer", + "worker", + ], + ["budget", "set", "memory.get", "--budget-ms", "86400001"], + ["budget", "set", "memory.get", "--consumer", "worker", "--multiplier", "1e-999"], + [ + "blacklist", + "global", + "add", + "denied", + "--pattern-id", + "7fdc7f78-5946-478e-81dc-b3effcc71f9c", + ], + ["blacklist", "global", "add", "denied", "--sweep", "true"], + ["blacklist", "global", "list", "--limit", "51"], + ["settings", "list", "--prefix", "first", "--prefix", "second"], + ], +) +def test_unsupported_scope_bounds_and_replay_controls_refuse(tokens, capsys): + from server.cli.cell_each import parse_command + + with pytest.raises(ValueError) as refused: + parse_command(tokens) + assert str(refused.value) == "Unsupported cell administration command" + assert capsys.readouterr() == ("", "") + + +def test_argument_limits_apply_before_normalization(): + from server.cli.cell_each import parse_command + + with pytest.raises(ValueError): + parse_command(["settings", "list"] + ["x"] * 255) + with pytest.raises(ValueError): + parse_command(["settings", "set", "name", "x" * 16384]) + with pytest.raises(ValueError): + parse_command(["revoke-token", "--subject", "sa:bad\0key"]) diff --git a/tests/unit/test_cell_each_command_text_boundaries.py b/tests/unit/test_cell_each_command_text_boundaries.py new file mode 100644 index 0000000000..ca40a80f4a --- /dev/null +++ b/tests/unit/test_cell_each_command_text_boundaries.py @@ -0,0 +1,107 @@ +"""Raw policy intent and canonical numeric boundaries survive closed parsing.""" + +from __future__ import annotations + +import hashlib +import json + +import pytest + + +def _digest(command): + return hashlib.sha256( + json.dumps( + {"action": command.action, "arguments": command.arguments}, + sort_keys=True, + separators=(",", ":"), + ).encode() + ).hexdigest() + + +@pytest.mark.parametrize("payload", ["line1\nline2", "first\tsecond", "text\x7fend"]) +def test_raw_setting_value_and_expected_keep_controls(payload, capsys): + from server.cli.cell_each import parse_command + + command = parse_command( + ["settings", "set", "extraction_defer_agents", payload, "--expect", payload] + ) + assert command.arguments["value"] == payload + assert command.arguments["expected"] == payload + assert command.digest == _digest(command) + assert capsys.readouterr() == ("", "") + + +@pytest.mark.parametrize("payload", ["line1\nline2", "first\tsecond", "text\x7fend"]) +def test_global_policy_payload_keeps_controls_in_private_intent(payload, capsys): + from server.cli.cell_each import parse_command + + command = parse_command(["blacklist", "global", "add", payload, "--reason", payload]) + assert command.arguments["pattern"] == payload + assert command.arguments["reason"] == payload + assert command.digest == _digest(command) + assert capsys.readouterr() == ("", "") + + +def test_exact_subject_reason_preserves_raw_text(): + from server.cli.cell_each import parse_command + + command = parse_command( + ["revoke-token", "--subject", "sa:worker", "--reason", "one\ntwo\tthree"] + ) + assert command.arguments["reason"] == "one\ntwo\tthree" + assert command.digest == _digest(command) + + +@pytest.mark.parametrize( + "tokens", + [ + ["settings", "list", "--prefix", "bad\nname"], + ["settings", "set", "bad\tname", "raw\nvalue"], + ["signing-key", "retire", "bad\x7fkid"], + ["revoke-token", "--subject", "sa:bad\nname"], + ["ratelimit", "reset", "--namespace", "bad\tname", "--op-class", "memory"], + ["ratelimit", "reset", "--namespace", "work", "--op-class", "bad\nclass"], + ["budget", "reset", "bad\nop"], + ["budget", "reset", "memory.get", "--consumer", "bad\tconsumer"], + ["blacklist", "global", "add", "bad\0pattern"], + ["settings", "set", "name", "bad\0value"], + ], +) +def test_identifier_and_nul_refusals_remain(tokens): + from server.cli.cell_each import parse_command + + with pytest.raises(ValueError) as refused: + parse_command(tokens) + assert str(refused.value) == "Unsupported cell administration command" + + +@pytest.mark.parametrize("raw", ["1e-29", "10e-30"]) +def test_multiplier_accepts_exact_32_byte_canonical_boundary(raw): + from server.cli.cell_each import parse_command + + command = parse_command( + ["budget", "set", "memory.get", "--consumer", "worker", "--multiplier", raw] + ) + expected = "0." + "0" * 28 + "1x" + assert len(expected.encode("ascii")) == 32 + assert command.arguments["budget"] == expected + assert command.digest == _digest(command) + + +@pytest.mark.parametrize("raw", ["1e-30", "1.1e-29", "1e-999", "1e-999999999"]) +def test_multiplier_refuses_oversized_canonical_expansion(raw): + from server.cli.cell_each import parse_command + + with pytest.raises(ValueError): + parse_command(["budget", "set", "memory.get", "--consumer", "worker", "--multiplier", raw]) + + +@pytest.mark.parametrize("raw", ["0e-999", "0e999"]) +def test_zero_multiplier_is_canonical_without_exponent_expansion(raw): + from server.cli.cell_each import parse_command + + command = parse_command( + ["budget", "set", "memory.get", "--consumer", "worker", "--multiplier", raw] + ) + assert command.arguments["budget"] == "0x" + assert command.digest == _digest(command) diff --git a/tests/unit/test_cell_each_commands.py b/tests/unit/test_cell_each_commands.py new file mode 100644 index 0000000000..86e594e5e9 --- /dev/null +++ b/tests/unit/test_cell_each_commands.py @@ -0,0 +1,146 @@ +"""Only the closed org-free command shapes become fixed administration requests.""" + +from __future__ import annotations + +import hashlib +import importlib +import json +from typing import Any + +import pytest + + +def _parse(argv: list[str]) -> Any: + return importlib.import_module("server.cli.cell_each").parse_command(argv) + + +@pytest.mark.parametrize( + ("argv", "action", "arguments", "mutating"), + [ + (["settings", "list"], "settings_list", {"prefix": ""}, False), + ( + ["settings", "list", "--prefix", "memory_"], + "settings_list", + {"prefix": "memory_"}, + False, + ), + ( + ["settings", "set", "memory_ask_recency_weight", "0.25"], + "settings_set", + { + "name": "memory_ask_recency_weight", + "value": "0.25", + "clear": False, + "expected": None, + }, + True, + ), + ( + ["settings", "reset", "memory_ask_recency_weight", "--expect", "0.25"], + "settings_set", + {"name": "memory_ask_recency_weight", "value": None, "clear": True, "expected": "0.25"}, + True, + ), + ( + ["settings", "history", "memory_ask_recency_weight"], + "settings_history", + {"name": "memory_ask_recency_weight", "limit": 50}, + False, + ), + (["signing-key", "list"], "signing_keys_list", {}, False), + ( + ["signing-key", "rotate"], + "signing_keys_rotate", + {"algorithm": "HS256", "activate_in": 300.0}, + True, + ), + ( + ["signing-key", "rotate", "--algorithm", "RS256", "--activate-in", "600"], + "signing_keys_rotate", + {"algorithm": "RS256", "activate_in": 600.0}, + True, + ), + (["signing-key", "retire", "key-1"], "signing_keys_retire", {"kid": "key-1"}, True), + ( + ["revoke-token", "--subject", "sa:worker", "--reason", "retire"], + "subject_revoke", + {"subject": "sa:worker", "reason": "retire"}, + True, + ), + ( + ["revoke-token", "--subject", "person@example.com"], + "subject_revoke", + {"subject": "person@example.com", "reason": ""}, + True, + ), + ], +) +def test_closed_selector_preserves_cli_defaults_and_raw_setting_values( + argv: list[str], action: str, arguments: dict[str, Any], mutating: bool +) -> None: + command = _parse(argv) + assert command.action == action + assert command.path == f"/v1/admin/cell.{action}" + assert command.arguments == arguments + assert command.mutating is mutating + expected = hashlib.sha256( + json.dumps( + {"action": action, "arguments": arguments}, sort_keys=True, separators=(",", ":") + ).encode() + ).hexdigest() + assert command.digest == expected + + +@pytest.mark.parametrize( + "argv", + [ + [], + ["call", "settings.set"], + ["/v1/admin/settings.set"], + ["sh", "-c", "true"], + ["cell", "each", "--", "settings", "list"], + ["settings", "history"], + ["settings", "history", "x", "--limit", "51"], + ["settings", "history", "x", "--limit", "0"], + ["settings", "set", "x", "1", "--data", "@payload"], + ["settings", "set", "x", "1", ";", "signing-key", "rotate"], + ["signing-key", "record-root"], + ["signing-key", "rotate", "--algorithm", "ES256"], + ["signing-key", "rotate", "--activate-in", "nan"], + ["signing-key", "rotate", "--activate-in", "inf"], + ["signing-key", "rotate", "--activate-in", "1e9999"], + ["revoke-token", "--subject", "acme/user:member"], + ["revoke-token", "--subject", "sa:worker", "--org", "acme"], + ["revoke-token", "--subject", "sa:worker", "--issued-before", "2030-01-01"], + ["revoke-token", "--subject", "sa:worker", "--expires-at", "2030-01-01"], + ["revoke-token", "--subject", "sa:worker", "--scim"], + ["revoke-token", "--jti", "token-1"], + ["revoke-token", "--list"], + ["bot", "create", "worker"], + ["migrate"], + ], +) +def test_unsupported_command_never_becomes_an_action(argv: list[str]) -> None: + with pytest.raises((ValueError, SystemExit)) as refused: + _parse(argv) + if isinstance(refused.value, SystemExit): + assert refused.value.code != 0 + + +def test_digest_keeps_the_normal_server_ascii_json_semantics() -> None: + from server.core.hashing import sha256_value + + command = _parse(["revoke-token", "--subject", "sa:worker", "--reason", "clé"]) + assert command.digest == sha256_value( + {"action": command.action, "arguments": command.arguments} + ) + + +def test_rejected_command_cannot_echo_untrusted_arguments( + capsys: pytest.CaptureFixture[str], +) -> None: + marker = "untrusted-argument-should-not-print" + with pytest.raises((ValueError, SystemExit)): + _parse(["call", marker]) + captured = capsys.readouterr() + assert marker not in captured.out + captured.err diff --git a/tests/unit/test_cell_each_entrypoint.py b/tests/unit/test_cell_each_entrypoint.py new file mode 100644 index 0000000000..b3ab316fbd --- /dev/null +++ b/tests/unit/test_cell_each_entrypoint.py @@ -0,0 +1,74 @@ +"""Cell administration starts on a client without loading server configuration.""" + +from __future__ import annotations + +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + + +@pytest.mark.parametrize("entrypoint", ["console", "module"]) +def test_each_help_never_imports_the_server_cli(entrypoint: str, tmp_path: Path) -> None: + (tmp_path / ".env").write_text("LOOM_JWT_SECRET=must-not-be-loaded\n", encoding="utf-8") + source_root = Path(__file__).resolve().parents[2] + script = textwrap.dedent( + """ + import importlib.abc + import os + import runpy + import sys + + sys.path.insert(0, sys.argv.pop(1)) + entrypoint = sys.argv.pop(1) + + class ClientOnly(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + forbidden = ( + 'server.cli.app', 'server.settings', 'server.wiring', + 'server.core.ch', 'server.core.pg', 'server.cli._ch', + 'server.providers', 'dotenv', + ) + if any(fullname == p or fullname.startswith(p + '.') for p in forbidden): + raise AssertionError('server import: ' + fullname) + return None + + sys.meta_path.insert(0, ClientOnly()) + sys.argv = ['loom', 'cell', 'each', '--help'] + try: + if entrypoint == 'module': + runpy.run_module('server.cli', run_name='__main__') + else: + from server.cli import main + raise SystemExit(main()) + except SystemExit as exc: + assert exc.code in (None, 0), exc.code + assert os.environ.get('LOOM_JWT_SECRET') != 'must-not-be-loaded' + """ + ) + result = subprocess.run( + [sys.executable, "-P", "-c", script, str(source_root), entrypoint], + cwd=tmp_path, + capture_output=True, + text=True, + timeout=15, + check=False, + ) + assert result.returncode == 0, result.stderr + assert "--credentials" in result.stdout and "--resume" in result.stdout + + +def test_direct_app_call_hands_off_before_dotenv_or_wiring(monkeypatch: pytest.MonkeyPatch) -> None: + from server.cli import app + + def forbidden() -> None: + raise AssertionError("each must hand off before dotenv and native wiring") + + monkeypatch.setattr(app, "_load_dotenv_like_the_container", forbidden) + monkeypatch.setattr(app, "wire", forbidden) + try: + assert app.main(["cell", "each", "--help"]) == 0 + except SystemExit as exited: + assert exited.code == 0 diff --git a/tests/unit/test_cell_each_fleet_diagnostics.py b/tests/unit/test_cell_each_fleet_diagnostics.py new file mode 100644 index 0000000000..cf5345c3ff --- /dev/null +++ b/tests/unit/test_cell_each_fleet_diagnostics.py @@ -0,0 +1,91 @@ +"""Native fixture failures expose useful metadata without child credentials.""" + +from __future__ import annotations + +import ast +import subprocess + +import pytest + +from tests.integration import _cell_each_fleet as fleet + + +def test_failed_child_reports_only_allowed_metadata(monkeypatch): + secrets = [ + "postgres://operator:private-password@private.example/private_database", + "Bearer private-bearer-token", + "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJwcml2YXRlIn0.private-signature", + "password=private-password", + "001_private_secret.sql", + ] + result = subprocess.CompletedProcess( + args=["python", "-m", "server.core.schema.migrate"], + returncode=1, + stdout=( + "x" * 10000 + + "\n → 000_migrations.sql\n ✓ 001_private_secret.sql\n" + + "\n".join(secrets) + ), + stderr="\n".join(secrets) + "\nDatabaseError: Code: 60. refused\nPrivateSecretError", + ) + monkeypatch.setattr(fleet.subprocess, "run", lambda *args, **kwargs: result) + with pytest.raises(AssertionError) as caught: + fleet.child(["server.core.schema.migrate"], {}) + message = str(caught.value) + prefix = "Private fixture command ['server.core.schema.migrate'] failed: " + assert message.startswith(prefix) + assert ast.literal_eval(message.removeprefix(prefix)) == { + "exit": 1, + "migration": ["000_migrations.sql"], + "error_kinds": ["DatabaseError"], + "clickhouse_codes": ["60"], + } + assert all(secret not in message for secret in secrets) + assert "PrivateSecretError" not in message + assert len(message) < 400 + + +def test_successful_child_returns_the_original_result(monkeypatch): + result = subprocess.CompletedProcess(args=["python"], returncode=0, stdout="value", stderr="") + calls = [] + + def run(*args, **kwargs): + calls.append((args, kwargs)) + return result + + monkeypatch.setattr(fleet.subprocess, "run", run) + assert fleet.child(["server.core.schema.migrate"], {}) is result + assert len(calls) == 1 + assert calls[0][1]["timeout"] == 240 + assert calls[0][1]["capture_output"] is True + + +@pytest.mark.parametrize("name", ["alpha", "beta"]) +def test_private_cell_environment_passes_startup_settings(monkeypatch, tmp_path, name): + import os + + from server.settings import Settings + + for key in list(os.environ): + if key.startswith("LOOM_"): + monkeypatch.delenv(key) + ports = iter((18443, 18444)) + monkeypatch.setattr(fleet, "free_port", lambda: next(ports)) + env = fleet._cell_env( + name, + "postgresql://fixture:fixture@127.0.0.1:15432/private_cell", + "private_each", + "https://127.0.0.1:18445", + tmp_path / "ca.pem", + ) + for key, value in env.items(): + monkeypatch.setenv(key, value) + + configured = Settings(_env_file=None) + + assert configured.cell == name + assert configured.storage_authority == "postgres" + assert configured.ch_database == "private_each" + assert configured.ch_telemetry_database == "private_each_telemetry" + assert configured.document_s3_prefix + assert configured.auth_actually_required() diff --git a/tests/unit/test_cell_each_http.py b/tests/unit/test_cell_each_http.py new file mode 100644 index 0000000000..a516a423c1 --- /dev/null +++ b/tests/unit/test_cell_each_http.py @@ -0,0 +1,460 @@ +"""Unknown remote completion stays uncertain and cannot cause another send.""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import time +from pathlib import Path +from typing import Any + +import httpx +import pytest + +from server.cli import cell_each +from server.cli import cell_each_http as client +from server.cli import cell_each_state as history + + +def _target() -> dict[str, Any]: + return { + "cell": "alpha", + "deployment_id": "deploy-alpha", + "origin": "https://alpha.example.com", + "state": "serving", + "mode": "enforced", + "mutation_generation": "c33e76ce-f023-4b45-92ea-2e9bfa008583", + "configuration_digests": ["1" * 64], + "automation": {"name": "automation-alpha", "thumbprint": "A" * 43}, + "recovery": {"name": "recovery-alpha", "thumbprint": "B" * 43}, + } + + +def _state(tmp_path: Path) -> tuple[dict[str, Any], Path]: + directory = tmp_path / "private" + directory.mkdir(mode=0o700) + state = history.new_state( + cell_each.parse_command(["settings", "set", "memory_ask_recency_weight", "0.25"]), + targets=[_target()], + run_id="a046002d-c888-4b54-852b-167f1cb8a443", + ) + state["router"] = "https://router.example.com" + path = directory / "state.json" + history.write_state(path, state, exclusive=True) + return state, path + + +def _preflight(monkeypatch: pytest.MonkeyPatch) -> None: + async def point(*args: Any, **kwargs: Any) -> None: + pass + + async def token(*args: Any, **kwargs: Any) -> tuple[str, float]: + return "unit-bearer", time.monotonic() + 60 + + monkeypatch.setattr(client, "_point", point) + monkeypatch.setattr(client, "_token", token) + monkeypatch.setattr(client, "_capability", point) + + +@pytest.mark.parametrize( + "failure", ["generic_503", "wrong_context", "missing_audit", "untyped_result"] +) +@pytest.mark.asyncio +async def test_ambiguous_action_never_becomes_retryable( + failure: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + state, path = _state(tmp_path) + _preflight(monkeypatch) + sends: list[str] = [] + + async def request(*args: Any, **kwargs: Any) -> tuple[int, dict[str, Any]]: + on_disk = history.read_state(path)["outcomes"]["alpha"] + assert on_disk["status"] == "dispatching" and on_disk["attempt"] == 1 + sends.append("action") + if failure == "generic_503": + return 503, {"detail": {"code": "SECURITY_UNAVAILABLE"}} + answer = { + "version": 1, + "status": "completed", + "context": client._context(_target(), state), + "action": "settings_set", + "started_audit": "a" * 64, + "completed_audit": "b" * 64, + "result": { + "ok": True, + "setting": { + "name": "memory_ask_recency_weight", + "env": "LOOM_MEMORY_ASK_RECENCY_WEIGHT", + "value": 0.25, + "default": 0.0, + "source": "override", + "live": True, + "description": "Weight", + }, + }, + } + if failure == "wrong_context": + answer["context"]["expected_deployment_id"] = "foreign" + elif failure == "missing_audit": + answer["completed_audit"] = "" + else: + answer["result"] = {"ok": True, "credential": "must-not-be-saved"} + return 200, answer + + monkeypatch.setattr(client, "_request", request) + manifest = {"keys": {("alpha", "deploy-alpha"): path.parent / "unused.pem"}} + assert not await client._cell( + None, _target(), state, manifest, state["router"], "unit-root", path, time.monotonic() + 60 + ) + saved = history.read_state(path) + assert saved["outcomes"]["alpha"]["status"] == "UNSETTLED" + assert saved["outcomes"]["alpha"]["attempt"] == 1 + assert saved["outcomes"]["alpha"]["completion"] is None + assert sends == ["action"] + assert "must-not-be-saved" not in path.read_text() + + +@pytest.mark.asyncio +async def test_failed_dispatch_persistence_prevents_action_request( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + state, path = _state(tmp_path) + _preflight(monkeypatch) + + def fail(*args: Any, **kwargs: Any) -> None: + raise OSError("write failed") + + async def forbidden(*args: Any, **kwargs: Any) -> Any: + raise AssertionError("an action cannot precede durable dispatch") + + monkeypatch.setattr(history, "write_state", fail) + monkeypatch.setattr(client, "_request", forbidden) + manifest = {"keys": {("alpha", "deploy-alpha"): path.parent / "unused.pem"}} + with pytest.raises(OSError, match="write failed"): + await client._cell( + None, + _target(), + state, + manifest, + state["router"], + "unit-root", + path, + time.monotonic() + 60, + ) + assert history.read_state(path)["outcomes"]["alpha"]["status"] == "pending" + + +@pytest.mark.parametrize( + "change", ["none", "wrong_phase", "wrong_bot", "boolean_version", "unknown_code"] +) +def test_only_matching_closed_before_business_receipt_is_retryable( + change: str, tmp_path: Path +) -> None: + state, _path = _state(tmp_path) + state["outcomes"]["alpha"].update(status="dispatching", attempt=1) + answer = { + "version": 1, + "status": "refused", + "phase": "before_business", + "context": client._context(_target(), state), + "action": "settings_set", + "code": "cell_unavailable", + } + if change == "wrong_phase": + answer["phase"] = "after_business" + elif change == "wrong_bot": + answer["context"]["expected_bot"] = "recovery-alpha" + elif change == "boolean_version": + answer["context"]["version"] = True + elif change == "unknown_code": + answer["code"] = "arbitrary_gateway_failure" + assert client._no_action(answer, _target(), state) == ( + "CELL_UNAVAILABLE" if change == "none" else None + ) + + +class _Chunks(httpx.AsyncByteStream): + def __init__(self, chunks: list[bytes]) -> None: + self.chunks = chunks + + async def __aiter__(self): + for chunk in self.chunks: + yield chunk + + +@pytest.mark.parametrize( + "body", [b'{"same":1,"same":2}', b'{"number":NaN}', b'{"number":1e9999}', b"x" * 65] +) +@pytest.mark.asyncio +async def test_bounded_response_rejects_duplicate_nonfinite_and_overflow(body: bytes) -> None: + transport = httpx.MockTransport(lambda _request: httpx.Response(200, stream=_Chunks([body]))) + async with httpx.AsyncClient(transport=transport, trust_env=False) as session: + with pytest.raises(ValueError): + await client._request( + session, "GET", "https://alpha.example.com", limit=64, deadline=time.monotonic() + 5 + ) + + +@pytest.mark.parametrize( + "origin", + [ + "http://alpha.example.com", + "https://user@alpha.example.com", + "https://alpha.example.com/path", + "https://alpha.example.com?query", + "https://alpha.example.com#fragment", + "https://alpha.example.com:bad", + "https://alpha.example.com\\foreign", + ], +) +def test_cell_credential_cannot_target_an_unvalidated_origin(origin: str) -> None: + with pytest.raises(ValueError): + client._origin(origin) + + +def test_private_manifest_has_no_ambient_or_recovery_fallback(tmp_path: Path) -> None: + directory = tmp_path / "private" + directory.mkdir(mode=0o700) + path = directory / "manifest.json" + path.write_text( + json.dumps( + { + "version": 1, + "router_token_file": str(directory / "operator"), + "keys": [ + { + "cell": "alpha", + "deployment_id": "deploy-alpha", + "key_file": str(directory / "automation"), + } + ], + "recovery_key_file": str(directory / "recovery"), + } + ) + ) + path.chmod(0o600) + with pytest.raises(ValueError): + client._manifest(path) + + +def test_private_reader_requires_a_private_parent_even_at_the_root() -> None: + with pytest.raises(ValueError): + history.read_private(Path("/cell-each-private-input"), 64) + + +def test_lost_read_can_resume_without_clearing_an_unknown_mutation(tmp_path: Path) -> None: + directory = tmp_path / "private" + directory.mkdir(mode=0o700) + path = directory / "state.json" + state = history.new_state( + cell_each.parse_command(["settings", "list"]), + targets=[_target()], + run_id="a046002d-c888-4b54-852b-167f1cb8a443", + ) + history.write_state(path, state, exclusive=True) + state["outcomes"]["alpha"].update(status="dispatching", attempt=1) + history.write_state(path, state) + history.resume_read(path, state) + assert history.read_state(path)["outcomes"]["alpha"]["status"] == "known-no-action" + assert history.read_state(path)["outcomes"]["alpha"]["code"] == "READ_INCOMPLETE" + + +def test_directory_generation_is_the_existing_uuid_contract() -> None: + target = _target() + target["mutation_generation"] = 1 + with pytest.raises(ValueError): + client._target(target) + assert ( + client._target(_target())["mutation_generation"] == "c33e76ce-f023-4b45-92ea-2e9bfa008583" + ) + + +def test_wrong_automation_key_is_refused_before_signing(monkeypatch: pytest.MonkeyPatch) -> None: + import jwt + from cryptography.hazmat.primitives import serialization + from cryptography.hazmat.primitives.asymmetric import ec + + key = ec.generate_private_key(ec.SECP256R1()) + pem = key.private_bytes( + serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption() + ) + + def forbidden(*args: Any, **kwargs: Any) -> Any: + raise AssertionError("a foreign key must not sign a client assertion") + + monkeypatch.setattr(jwt, "encode", forbidden) + with pytest.raises(ValueError): + client._assertion(pem, _target(), "https://alpha.example.com/oauth/token") + + +def test_explicit_bot_assertion_keeps_the_selected_identity() -> None: + import jwt + from cryptography.hazmat.primitives import serialization + from cryptography.hazmat.primitives.asymmetric import ec + from jwt.algorithms import ECAlgorithm + + key = ec.generate_private_key(ec.SECP256R1()) + pem = key.private_bytes( + serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption() + ) + public = ECAlgorithm.to_jwk(key.public_key(), as_dict=True) + members = {name: public[name] for name in ("crv", "kty", "x", "y")} + thumbprint = ( + base64.urlsafe_b64encode(hashlib.sha256(client._encoded(members)).digest()) + .rstrip(b"=") + .decode() + ) + endpoint = "https://alpha.example.com/oauth/token" + token = client.bot_assertion( + pem, name="selected-recovery", thumbprint=thumbprint, endpoint=endpoint + ) + claims = jwt.decode( + token, + key.public_key(), + algorithms=["ES256"], + audience=endpoint, + options={"verify_exp": False, "verify_iat": False}, + ) + assert claims["iss"] == claims["sub"] == "selected-recovery" + assert claims["exp"] - claims["iat"] == 120 + assert jwt.get_unverified_header(token)["kid"] == thumbprint + + +def test_historical_policy_display_does_not_apply_new_input_size_limits() -> None: + row = { + "pattern_id": "a046002d-c888-4b54-852b-167f1cb8a443", + "pattern": "é" * 4096 + "\n\t\0", + "pattern_type": "substring", + "reason": "旧" * 1024, + "created_by": "historical-writer", + "hit_count": 0, + "last_hit_at": None, + "created_at": "2026-01-01T00:00:00", + } + client._result("blacklist_global_list", {"rows": [row], "truncated": False}, {"limit": 50}) + with pytest.raises(ValueError): + client._result( + "blacklist_global_list", {"rows": [row, row], "truncated": False}, {"limit": 50} + ) + + +@pytest.mark.asyncio +async def test_repeated_preflight_refusals_are_durable_without_dispatch( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + state, path = _state(tmp_path) + + async def refuse(*args: Any, **kwargs: Any) -> None: + raise ValueError("authority unavailable before business") + + monkeypatch.setattr(client, "_point", refuse) + for attempt in (1, 2): + assert not await client._cell( + None, _target(), state, {}, state["router"], "unit-root", path, time.monotonic() + 60 + ) + saved = history.read_state(path)["outcomes"]["alpha"] + assert saved["status"] == "known-no-action" and saved["attempt"] == attempt + + +@pytest.mark.asyncio +async def test_recorded_revocation_without_finished_cleanup_remains_unsettled( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + directory = tmp_path / "private" + directory.mkdir(mode=0o700) + path = directory / "state.json" + state = history.new_state( + cell_each.parse_command(["revoke-token", "--subject", "sa:obsolete"]), + targets=[_target()], + run_id="a046002d-c888-4b54-852b-167f1cb8a443", + ) + state["router"] = "https://router.example.com" + history.write_state(path, state, exclusive=True) + _preflight(monkeypatch) + + async def partial(*args: Any, **kwargs: Any) -> tuple[int, dict[str, Any]]: + return 503, { + "detail": { + "code": "SUBJECT_REVOCATION_INCOMPLETE", + "recorded": True, + "sessions_at_least": 2, + "tokens_at_least": 1, + } + } + + monkeypatch.setattr(client, "_request", partial) + manifest = {"keys": {("alpha", "deploy-alpha"): directory / "unused.pem"}} + assert not await client._cell( + None, _target(), state, manifest, state["router"], "unit-root", path, time.monotonic() + 60 + ) + outcome = history.read_state(path)["outcomes"]["alpha"] + assert outcome["status"] == "UNSETTLED" and outcome["attempt"] == 1 + assert outcome["completion"] is None + + +@pytest.mark.asyncio +async def test_confirmed_typed_action_persists_only_safe_completion( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + state, path = _state(tmp_path) + _preflight(monkeypatch) + + async def request(*args: Any, **kwargs: Any) -> tuple[int, dict[str, Any]]: + assert history.read_state(path)["outcomes"]["alpha"]["status"] == "dispatching" + return 200, { + "version": 1, + "status": "completed", + "context": client._context(_target(), state), + "action": "settings_set", + "started_audit": "a" * 64, + "completed_audit": "b" * 64, + "result": { + "ok": True, + "setting": { + "name": "memory_ask_recency_weight", + "env": "LOOM_MEMORY_ASK_RECENCY_WEIGHT", + "value": 0.25, + "default": 0.0, + "source": "override", + "live": True, + "description": "raw-response-must-not-be-persisted", + }, + }, + } + + monkeypatch.setattr(client, "_request", request) + manifest = {"keys": {("alpha", "deploy-alpha"): path.parent / "unused.pem"}} + assert await client._cell( + None, _target(), state, manifest, state["router"], "unit-root", path, time.monotonic() + 60 + ) + outcome = history.read_state(path)["outcomes"]["alpha"] + assert outcome["status"] == "completed" and outcome["attempt"] == 1 + assert outcome["completion"]["started_audit"] == "a" * 64 + assert outcome["completion"]["completed_audit"] == "b" * 64 + assert "result" not in outcome["completion"] + assert "raw-response-must-not-be-persisted" not in path.read_text() + + +@pytest.mark.asyncio +async def test_same_client_does_not_forward_cookies_between_requests() -> None: + cookies: list[str | None] = [] + + def handler(request: httpx.Request) -> httpx.Response: + cookies.append(request.headers.get("cookie")) + return httpx.Response( + 200, + headers={"Set-Cookie": "token=must-not-travel; Domain=.example.com; Path=/"}, + stream=_Chunks([b"{}"]), + ) + + async with httpx.AsyncClient( + transport=httpx.MockTransport(handler), trust_env=False + ) as session: + await client._request( + session, "GET", "https://alpha.example.com", limit=64, deadline=time.monotonic() + 5 + ) + await client._request( + session, "GET", "https://beta.example.com", limit=64, deadline=time.monotonic() + 5 + ) + assert cookies == [None, None] diff --git a/tests/unit/test_cell_each_outcomes.py b/tests/unit/test_cell_each_outcomes.py new file mode 100644 index 0000000000..dea33df6d0 --- /dev/null +++ b/tests/unit/test_cell_each_outcomes.py @@ -0,0 +1,191 @@ +"""A durable unknown mutation prevents network or key access on resume.""" + +from __future__ import annotations + +import importlib +import json +import os +import stat +from pathlib import Path +from typing import Any + +import pytest + +RUN_ID = "a046002d-c888-4b54-852b-167f1cb8a443" + + +def _module() -> Any: + return importlib.import_module("server.cli.cell_each_state") + + +def _state(module: Any) -> dict[str, Any]: + from server.cli.cell_each import parse_command + + command = parse_command(["settings", "set", "memory_ask_recency_weight", "0.25"]) + return module.new_state( + command, + targets=[ + { + "cell": "alpha", + "deployment_id": "deploy-alpha", + "origin": "https://alpha.example.com", + "state": "serving", + "mode": "enforced", + "mutation_generation": "c33e76ce-f023-4b45-92ea-2e9bfa008583", + "configuration_digests": ["1" * 64], + "automation": {"name": "automation-alpha", "thumbprint": "A" * 43}, + "recovery": {"name": "recovery-alpha", "thumbprint": "B" * 43}, + } + ], + run_id=RUN_ID, + ) + + +@pytest.mark.parametrize("status", ["dispatching", "UNSETTLED"]) +def test_uncertain_resume_stops_before_credentials_or_http( + status: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + import httpx + + from server.cli.cell_each_run import main + + module = _module() + directory = tmp_path / "private" + directory.mkdir(mode=0o700) + path = directory / "state.json" + state = _state(module) + state["outcomes"]["alpha"].update(status=status, attempt=1) + module.write_state(path, state, exclusive=True) + assert module.read_state(path) == state + before = path.read_bytes() + + def no_network(*args: Any, **kwargs: Any) -> Any: + raise AssertionError("an uncertain mutation forbids all network access") + + monkeypatch.setattr(httpx, "Client", no_network) + monkeypatch.setattr(httpx, "AsyncClient", no_network) + # Missing credentials prove resume inspects durable uncertainty before any key/manifest read. + assert ( + main( + [ + "--router", + "https://router.example.com", + "--credentials", + str(directory / "absent"), + "--resume", + str(path), + ] + ) + != 0 + ) + after = module.read_state(path) + assert after["outcomes"]["alpha"]["status"] in ("dispatching", "UNSETTLED") + assert path.read_bytes() == before or after["outcomes"]["alpha"]["status"] == "UNSETTLED" + output = capsys.readouterr() + assert "UNSETTLED" in output.out + output.err + assert "absent" not in output.out + output.err + + +def test_state_creation_is_exclusive_and_owner_only(tmp_path: Path) -> None: + module = _module() + directory = tmp_path / "private" + directory.mkdir(mode=0o700) + path = directory / "state.json" + state = _state(module) + module.write_state(path, state, exclusive=True) + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + assert module.read_state(path) == state + before = path.read_bytes() + with pytest.raises((FileExistsError, ValueError)): + module.write_state(path, state, exclusive=True) + assert path.read_bytes() == before + + +def test_failed_replace_preserves_last_valid_outcome( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = _module() + directory = tmp_path / "private" + directory.mkdir(mode=0o700) + path = directory / "state.json" + state = _state(module) + module.write_state(path, state, exclusive=True) + before = path.read_bytes() + state["outcomes"]["alpha"].update(status="dispatching", attempt=1) + + def disk_failure(*args: Any, **kwargs: Any) -> Any: + raise OSError("simulated replace failure") + + monkeypatch.setattr(os, "replace", disk_failure) + with pytest.raises(OSError): + module.write_state(path, state) + assert path.read_bytes() == before + assert [p.name for p in directory.iterdir()] == ["state.json"] + + +def test_outcome_replacement_syncs_file_then_rename_then_directory( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = _module() + directory = tmp_path / "private" + directory.mkdir(mode=0o700) + path = directory / "state.json" + state = _state(module) + module.write_state(path, state, exclusive=True) + state["outcomes"]["alpha"].update(status="dispatching", attempt=1) + events: list[str] = [] + real_fsync, real_replace = os.fsync, os.replace + + def fsync(fd: int) -> None: + events.append("directory_sync" if stat.S_ISDIR(os.fstat(fd).st_mode) else "file_sync") + real_fsync(fd) + + def replace(*args: Any, **kwargs: Any) -> None: + events.append("rename") + real_replace(*args, **kwargs) + + monkeypatch.setattr(os, "fsync", fsync) + monkeypatch.setattr(os, "replace", replace) + module.write_state(path, state) + rename = events.index("rename") + assert "file_sync" in events[:rename] + assert "directory_sync" in events[rename + 1 :] + assert module.read_state(path)["outcomes"]["alpha"]["status"] == "dispatching" + + +def test_duplicate_field_in_an_otherwise_valid_state_refuses(tmp_path: Path) -> None: + module = _module() + directory = tmp_path / "private" + directory.mkdir(mode=0o700) + path = directory / "state.json" + state = _state(module) + module.write_state(path, state, exclusive=True) + assert module.read_state(path) == state + encoded = path.read_text(encoding="utf-8") + path.write_text( + '{"run_id":' + json.dumps(RUN_ID) + "," + encoded.lstrip()[1:], encoding="utf-8" + ) + with pytest.raises(ValueError): + module.read_state(path) + + +@pytest.mark.parametrize( + "bad", [b'{"version":1,"version":1}', b'{"value":NaN}', b'{"value":1e9999}', b"\xff"] +) +def test_malformed_state_is_never_adopted(bad: bytes, tmp_path: Path) -> None: + module = _module() + directory = tmp_path / "private" + directory.mkdir(mode=0o700) + path = directory / "state.json" + path.write_bytes(bad) + path.chmod(0o600) + with pytest.raises(ValueError): + module.read_state(path) + + +def test_outcome_state_omits_credential_material() -> None: + module = _module() + state = _state(module) + encoded = json.dumps(state) + assert "private_key" not in encoded and "bearer" not in encoded and "credentials" not in encoded + assert "deployment_id" in encoded and RUN_ID in encoded diff --git a/tests/unit/test_cell_each_scim_revocation.py b/tests/unit/test_cell_each_scim_revocation.py new file mode 100644 index 0000000000..67019c26e6 --- /dev/null +++ b/tests/unit/test_cell_each_scim_revocation.py @@ -0,0 +1,151 @@ +"""Descendant revocation discards provisioning decisions that its write overtakes.""" + +from __future__ import annotations + +import asyncio +import time +from datetime import datetime + +import pytest + +from server.auth import login, scim, service_accounts +from server.core import ch_bounded +from server.settings import settings +from tests.unit.test_scim_people import people as people + + +@pytest.mark.parametrize("bounded", [False, True], ids=["ordinary", "bounded"]) +@pytest.mark.parametrize("outcome", ["success", "error", "lost-reply", "cancel"]) +async def test_descendant_write_drops_warm_and_overtaken_provisioning_permits( + people, monkeypatch, bounded, outcome +): + monkeypatch.setattr(settings, "grant_cache_ttl_seconds", 240) + owner, subject = "alice@example.test", "sa:personal:child" + people.provision(owner) + original = { + "org": "acme", + "name": "child", + "sa_subject": subject, + "namespaces": ["private"], + "roles": ["writer"], + "ttl_days": 0, + "created_by": owner, + "created_at": datetime(2020, 1, 1), + "expires_at": None, + "revoked_at": None, + "revoked_by": "", + "version": 1, + "kind": "personal", + "client": "unit-client", + "inherit_creator_grants": False, + "ns_roles": "{}", + "lineage": [], + "saves_to": "private", + } + people.services[subject] = dict(original) + assert await scim.refuses_subject(subject, org="acme") is False + + started, release = asyncio.Event(), asyncio.Event() + pending = None + pause_ancestry = False + stored = [] + query_one = people.query_one + error = { + "error": RuntimeError("identity write rejected"), + "lost-reply": ConnectionError("identity write landed without its reply"), + "cancel": asyncio.CancelledError("identity write cancelled after landing"), + }.get(outcome) + + async def read(sql, params=None): + nonlocal pause_ancestry + if "SELECT * FROM loom.service_accounts" in sql: + return dict(people.services[subject]) + row = await query_one(sql, params=params) + if pause_ancestry and params is not None and params.get("sub") == subject: + pause_ancestry = False + started.set() + await release.wait() + return row + + async def read_rows(sql, params, **kwargs): + assert "FROM loom.service_accounts FINAL" in sql + assert params["org"] == "acme" and params["name"] == "child" + row = dict(people.services[subject]) + row["created_at"] = int( + (original["created_at"] - datetime(1970, 1, 1)).total_seconds() * 1000 + ) + return [row] + + async def insert(table, rows, **kwargs): + nonlocal pending, pause_ancestry + assert table == "loom.service_accounts" and len(rows) == 1 + people.unavailable = True + try: + with pytest.raises(login.SignInUnavailable): + await scim.refuses_subject(subject, org="acme") + finally: + people.unavailable = False + pause_ancestry = True + pending = asyncio.create_task(scim.refuses_subject(subject, org="acme")) + await asyncio.wait_for(started.wait(), 2) + if outcome != "error": + stored.extend(dict(row) for row in rows) + people.services[subject] = dict(rows[0]) + if error is not None: + raise error + + monkeypatch.setattr(people, "query_one", read) + monkeypatch.setattr(people, "insert", insert, raising=False) + monkeypatch.setattr(ch_bounded, "read_rows", read_rows) + work = ( + service_accounts._RevocationWork(ch=people, deadline=time.monotonic() + 30, roots={owner}) + if bounded + else None + ) + try: + call = service_accounts._revoke_named( + "acme", + "child", + [owner], + {owner: None}, + set(), + "operator", + {subject}, + ch=people, + work=work, + ) + if error is None: + assert await call == subject + else: + with pytest.raises(type(error)) as caught: + await call + assert caught.value is error + release.set() + assert pending is not None + assert await pending is False + people.unavailable = True + try: + with pytest.raises(login.SignInUnavailable): + await scim.refuses_subject(subject, org="acme") + finally: + people.unavailable = False + assert await scim.refuses_subject(subject, org="acme") is (outcome != "error") + if outcome == "error": + assert stored == [] and people.services[subject] == original + else: + assert len(stored) == 1 + row = people.services[subject] + assert isinstance(row["revoked_at"], datetime) + assert row["revoked_by"] == "operator" and row["version"] > original["version"] + assert row == { + **original, + "revoked_at": row["revoked_at"], + "revoked_by": "operator", + "version": row["version"], + } + if work is not None: + assert work.confirmed_dead == ({subject} if outcome == "success" else set()) + finally: + release.set() + if pending is not None and not pending.done(): + await pending diff --git a/tests/unit/test_cell_each_subject_semantics.py b/tests/unit/test_cell_each_subject_semantics.py new file mode 100644 index 0000000000..a62dfd806d --- /dev/null +++ b/tests/unit/test_cell_each_subject_semantics.py @@ -0,0 +1,134 @@ +"""The deployment CLI revokes one exact subject across orgs through shared steps.""" + +from __future__ import annotations + +import argparse +import asyncio +from types import SimpleNamespace +from typing import Any + +import pytest + + +@pytest.mark.parametrize("subject", ["sa:worker", "person@example.com"]) +def test_ordinary_cli_global_subject_is_recorded_and_finished_without_org_expansion( + subject: str, monkeypatch: pytest.MonkeyPatch +) -> None: + from server.auth import subject_revocation + from server.cli import access + + calls: list[tuple[str, str, dict[str, Any]]] = [] + governing = {"reason": "retire", "issued_before": None, "expires_at": None} + + async def record(value: str, **kwargs: Any) -> tuple[dict[str, Any], bool]: + calls.append(("record", value, kwargs)) + return governing, False + + async def finish(value: str, row: Any, kept: bool, **kwargs: Any) -> Any: + assert row is governing and kept is False + calls.append(("finish", value, kwargs)) + return SimpleNamespace(row=row, kept=kept, sessions=0, tokens=0) + + def no_org_expansion(*args: Any, **kwargs: Any) -> Any: + raise AssertionError("the global command must not use org alias expansion") + + monkeypatch.setattr(subject_revocation, "record", record) + monkeypatch.setattr(subject_revocation, "finish", finish) + monkeypatch.setattr(subject_revocation, "revoke_in_org", no_org_expansion) + monkeypatch.setattr(access, "run_connected", lambda work, **kwargs: asyncio.run(work(None))) + monkeypatch.setattr(access, "_print_revoked", lambda *args, **kwargs: None) + parser = argparse.ArgumentParser() + access._add_revoke_token_parser(parser.add_subparsers(dest="command", required=True)) + args = parser.parse_args(["revoke-token", "--subject", subject, "--reason", "retire"]) + assert access._cmd_revoke_token(args) == 0 + assert [name for name, _, _ in calls] == ["record", "finish"] + assert all(value == subject for _, value, _ in calls) + assert calls[0][2]["reason"] == "retire" + assert calls[1][2]["org"] == "-" + assert calls[0][2]["by"] == calls[1][2]["by"] + assert calls[0][2]["by"].startswith("cli:") + + +@pytest.mark.parametrize("cleanup_fails", [False, True], ids=["closed", "unavailable"]) +async def test_bounded_subject_completion_requires_connected_account_cleanup( + monkeypatch: pytest.MonkeyPatch, cleanup_fails: bool +) -> None: + import time + + from fastapi import HTTPException + + from server.auth import ( + account_lifecycle, + oauth_store, + revocation, + service_accounts, + subject_revocation, + ) + from server.insights.app import cell_admin_security_actions as actions + + subject = "sa:retired" + events = [] + recorded = [] + audits = [] + row = {"jti": subject, "issued_before": None} + + async def record(value, *, progress, **kwargs): + assert value == subject + recorded.append(dict(row)) + progress["recorded"] = True + events.append("recorded") + return row, False + + async def cutoffs(subjects, **kwargs): + assert subjects == [subject] + assert kwargs["deadline"] > time.monotonic() + return {subject: None} + + async def cleanup(subjects, *, respect_cutoff): + assert recorded == [row] + assert subjects == [subject] and respect_cutoff is True + events.append("cleanup") + if cleanup_fails: + raise ConnectionError("connected-account cleanup unavailable") + events.append("closed") + + async def families(value, **kwargs): + assert value == subject and "closed" in events + events.append("sessions") + return {"family": 0} + + async def revoke_created(value, **kwargs): + assert value == subject and "closed" in events + events.append("tokens") + return [{"name": "retired-child"}] + + async def audit(*args, **kwargs): + audits.append(kwargs) + + monkeypatch.setattr(actions, "current_auth", lambda: SimpleNamespace(subject="operator")) + monkeypatch.setattr(subject_revocation, "record", record) + monkeypatch.setattr(revocation, "cutoffs_bounded", cutoffs) + monkeypatch.setattr(revocation, "is_revoked", lambda _family: False) + monkeypatch.setattr(account_lifecycle, "stop_connections", cleanup) + monkeypatch.setattr(oauth_store, "session_families", families) + monkeypatch.setattr(service_accounts, "revoke_created_by_bounded", revoke_created) + monkeypatch.setattr(subject_revocation, "audit_revocation", audit) + command = SimpleNamespace(subject=subject, reason="retire") + if cleanup_fails: + with pytest.raises(HTTPException) as caught: + await actions.subject_revoke(command, deadline=time.monotonic() + 30, ch=object()) + assert caught.value.status_code == 503 + assert caught.value.detail == { + "code": "SUBJECT_REVOCATION_INCOMPLETE", + "recorded": True, + "sessions_at_least": 0, + "tokens_at_least": 0, + } + assert events == ["recorded", "cleanup"] + else: + result = await actions.subject_revoke(command, deadline=time.monotonic() + 30, ch=object()) + assert result["recorded"] is True and result["outright"] is True + assert result["sessions"] == result["tokens"] == 1 + assert events == ["recorded", "cleanup", "closed", "sessions", "tokens"] + assert recorded == [row] + assert len(audits) == 1 diff --git a/tests/unit/test_cell_tenancy.py b/tests/unit/test_cell_tenancy.py index 41eba75980..bb6b3bcb3f 100644 --- a/tests/unit/test_cell_tenancy.py +++ b/tests/unit/test_cell_tenancy.py @@ -432,6 +432,18 @@ async def _held(org: Any, auth: Any) -> None: "/v1/admin/bots.provision_agent POST": "a bot of this cell, by name", "/v1/admin/budget.set POST": "a per-cell budget setting row, not tenant data", "/v1/admin/budget.violations POST": "a platform admin's read: an org the cell does not hold has no rows", + "/v1/admin/cell.capabilities POST": "the verified cell's administration contract, not tenant data", + "/v1/admin/cell.settings_list POST": "the cell's non-secret runtime settings", + "/v1/admin/cell.settings_set POST": "one live setting of this cell, not tenant data", + "/v1/admin/cell.settings_history POST": "the cell's live-setting history", + "/v1/admin/cell.ratelimit_namespace_set POST": "a per-cell namespace-label limit, not a tenant row", + "/v1/admin/cell.budget_set POST": "a per-cell operation or consumer latency budget", + "/v1/admin/cell.signing_keys_list POST": "the cell's public signing-key metadata", + "/v1/admin/cell.signing_keys_rotate POST": "the cell's own signing-key lifecycle", + "/v1/admin/cell.signing_keys_retire POST": "the cell's own signing-key lifecycle", + "/v1/admin/cell.subject_revoke POST": "one exact global security principal of this cell", + "/v1/admin/cell.blacklist_global_add POST": "one global policy row with fixed wildcard scope", + "/v1/admin/cell.blacklist_global_list POST": "the cell's exact global policy rows", "/v1/admin/claim_release POST": "named", "/v1/admin/consumers POST": "a platform admin's read: an org the cell does not hold has no rows", "/v1/admin/events.tail POST": "a platform admin's read: an org the cell does not hold has no rows", diff --git a/tests/unit/test_clickhouse_mutation_inventory.py b/tests/unit/test_clickhouse_mutation_inventory.py index 7bf18dcb36..5e29a4c680 100644 --- a/tests/unit/test_clickhouse_mutation_inventory.py +++ b/tests/unit/test_clickhouse_mutation_inventory.py @@ -31,7 +31,6 @@ ("server/memory/app/projections.py", "entity_triples", "UPDATE"): 3, # In-place retries cannot recreate a transcript erased before a delayed command lands. ("server/memory/app/extraction_queue.py", "memory_extraction_queue", "UPDATE"): 1, - ("server/memory/store/blacklist.py", "blacklist", "UPDATE"): 1, ("server/memory/store/conflicts.py", "conflict_allowlist", "UPDATE"): 1, ("server/knowledge/store/conversations.py", "conversation_threads", "UPDATE"): 1, # tombstone_thread and conversation.compact's tombstone_messages. diff --git a/tests/unit/test_exception_text_ratchet.py b/tests/unit/test_exception_text_ratchet.py index 161e2fd546..7d11f1486d 100644 --- a/tests/unit/test_exception_text_ratchet.py +++ b/tests/unit/test_exception_text_ratchet.py @@ -87,7 +87,7 @@ "server/identity/app/history_policy.py": 1, "server/identity/app/history_policy_ops.py": 2, "server/identity/app/namespace_classification.py": 1, - "server/insights/app/admin_routes.py": 2, + "server/insights/app/admin_routes.py": 1, "server/insights/app/provider_validation.py": 3, "server/insights/store/deploy_delta.py": 1, "server/knowledge/app/conversations.py": 1, diff --git a/tests/unit/test_op_budgets.py b/tests/unit/test_op_budgets.py index 610ce27ff2..672f328559 100644 --- a/tests/unit/test_op_budgets.py +++ b/tests/unit/test_op_budgets.py @@ -79,3 +79,51 @@ def test_an_op_this_build_lacks_is_logged_and_ignored( with caplog.at_level(logging.WARNING): assert BudgetOverrides().effective("memory.search", 300) == 900 assert "memory.a_later_op" in caplog.text + + +@pytest.mark.parametrize("failure", ["invalid", "private", "none"]) +async def test_budget_change_keeps_validation_text_private_and_adopts_only_after_storage( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, failure: str +) -> None: + from unittest.mock import AsyncMock, Mock + + from fastapi import HTTPException + + from server.insights.app import limit_changes + + overrides = BudgetOverrides() + adopted = Mock(wraps=overrides.set) + put = AsyncMock() + monkeypatch.setattr(limit_changes, "get_overrides", lambda: overrides) + monkeypatch.setattr(overrides, "set", adopted) + monkeypatch.setattr(limit_changes.ro, "put", put) + private = "private-budget-backend-detail" + if failure == "private": + monkeypatch.setattr(limit_changes, "stored_ms", Mock(side_effect=ValueError(private))) + arguments = dict( + consumer=None, budget_ms=-1 if failure == "invalid" else 42, budget=None, clear=False + ) + if failure != "none": + with caplog.at_level(logging.INFO, logger="server.insights.app.limit_changes"): + with pytest.raises(HTTPException) as refused: + await limit_changes.change_budget("memory.search", **arguments) + assert refused.value.status_code == 422 + assert refused.value.detail == { + "code": "VALIDATION_FAILED", + "message": "budget_ms must be a whole number of ms >= 0", + } + assert private not in str(refused.value.detail) + put.assert_not_awaited() + adopted.assert_not_called() + if failure == "private": + assert private in caplog.text + return + + async def stored(*_args, **_kwargs): + adopted.assert_not_called() + + put.side_effect = stored + result = await limit_changes.change_budget("memory.search", **arguments) + assert result == {"ok": True, "op": "memory.search", "budget_ms": 42} + put.assert_awaited_once_with(limit_changes.ro.BUDGET_OP, ("memory.search",), 42) + adopted.assert_called_once_with("memory.search", 42) diff --git a/tests/unit/test_ratelimit_bounded_change.py b/tests/unit/test_ratelimit_bounded_change.py new file mode 100644 index 0000000000..d81a6765f7 --- /dev/null +++ b/tests/unit/test_ratelimit_bounded_change.py @@ -0,0 +1,178 @@ +"""Bounded rate changes refuse before storage and recheck before local adoption.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, Mock + +import pytest + +from server.core import ratelimit +from server.core.ch_bounded import BoundedReadError +from server.core.ratelimit import Limit, Limiter, TokenBucket +from server.insights.app import limit_changes +from server.settings import settings + + +@pytest.fixture +def limiter(monkeypatch: pytest.MonkeyPatch) -> Limiter: + monkeypatch.setattr(settings, "ratelimit_shared", "off") + monkeypatch.setattr(settings, "ratelimit_shared_path", "") + monkeypatch.setattr(settings, "ratelimit_shared_slots", 128) + monkeypatch.setattr(settings, "web_workers", 1) + return Limiter(clock=lambda: 10.0) + + +@pytest.mark.parametrize("failure", ["unknown-class", "class-count"]) +def test_unknown_rate_classes_refuse_without_opening_a_backend( + limiter: Limiter, monkeypatch: pytest.MonkeyPatch, failure: str +) -> None: + if failure == "class-count": + monkeypatch.setattr( + ratelimit, "DEFAULT_LIMITS", {f"class-{i}": Limit(1, 1) for i in range(65)} + ) + backend = Mock(side_effect=AssertionError("preflight must not resolve a backend")) + monkeypatch.setattr(limiter, "_store", backend) + op_class = "class-0" if failure == "class-count" else "unknown" + with pytest.raises(BoundedReadError, match="Unsupported rate-limit class"): + limiter.check_bounded_namespace_change("work", op_class, clear=False) + backend.assert_not_called() + assert limiter._ns_overrides == {} + + +@pytest.mark.parametrize("tier", ["_ns_buckets", "_org_buckets", "_actor_buckets"]) +def test_every_bucket_tier_refuses_overflow_without_evicting_existing_work( + limiter: Limiter, monkeypatch: pytest.MonkeyPatch, tier: str +) -> None: + monkeypatch.setattr(ratelimit, "MAX_BUCKETS", 2) + bucket = TokenBucket(Limit(1, 0), clock=lambda: 10.0) + assert bucket.try_acquire() + entries = {(str(i),): bucket for i in range(3)} + monkeypatch.setattr(limiter, tier, entries) + with pytest.raises(BoundedReadError, match="buckets exceed"): + limiter.check_bounded_namespace_change("work", "memory", clear=False) + assert getattr(limiter, tier) == entries and len(entries) == 3 + assert bucket.tokens == 0 + assert limiter._ns_overrides == {} + + +@pytest.mark.parametrize( + ("count", "existing", "clear", "allowed"), + [ + (1023, False, False, True), + (1024, False, False, False), + (1024, True, False, True), + (1024, False, True, True), + (1025, True, True, False), + ], +) +def test_override_capacity_keeps_replacement_and_removal_available( + limiter: Limiter, count: int, existing: bool, clear: bool, allowed: bool +) -> None: + limiter._ns_overrides = {(f"ns-{i}", "memory"): Limit(1, 0) for i in range(count)} + target = "ns-0" if existing else "new" + before = dict(limiter._ns_overrides) + if allowed: + limiter.check_bounded_namespace_change(target, "memory", clear=clear) + else: + with pytest.raises(BoundedReadError, match="overrides exceed"): + limiter.check_bounded_namespace_change(target, "memory", clear=clear) + assert limiter._ns_overrides == before + + +@pytest.mark.parametrize( + ("mode", "workers", "shared", "changed", "allowed"), + [ + ("off", 1, False, "", True), + ("auto", 1, False, "", True), + ("on", 1, False, "", False), + ("auto", 2, False, "", False), + ("on", 1, True, "", True), + ("auto", 2, True, "", True), + ("on", 1, True, "mode", False), + ("on", 1, True, "path", False), + ("on", 1, True, "slots", False), + ("on", 1, True, "workers", False), + ], +) +def test_preflight_uses_only_the_initialized_backend_and_captured_configuration( + limiter: Limiter, + monkeypatch: pytest.MonkeyPatch, + mode: str, + workers: int, + shared: bool, + changed: str, + allowed: bool, +) -> None: + monkeypatch.setattr(settings, "ratelimit_shared", mode) + monkeypatch.setattr(settings, "web_workers", workers) + store = Mock() + if shared: + monkeypatch.setattr(limiter, "_shared", store) + monkeypatch.setattr(limiter, "_shared_key", (mode, "", 128, workers)) + if changed: + field, value = { + "mode": ("ratelimit_shared", "off"), + "path": ("ratelimit_shared_path", "/unused/rate-buckets"), + "slots": ("ratelimit_shared_slots", 256), + "workers": ("web_workers", 2), + }[changed] + monkeypatch.setattr(settings, field, value) + old_store, old_key = limiter._shared, limiter._shared_key + resolve = Mock(side_effect=AssertionError("preflight must not open or replace a backend")) + monkeypatch.setattr(limiter, "_store", resolve) + if allowed: + limiter.check_bounded_namespace_change("work", "memory", clear=False) + else: + with pytest.raises(BoundedReadError, match="backend is not initialized"): + limiter.check_bounded_namespace_change("work", "memory", clear=False) + resolve.assert_not_called() + assert store.mock_calls == [] + assert limiter._shared is old_store and limiter._shared_key == old_key + + +@pytest.mark.parametrize("clear", [False, True]) +@pytest.mark.parametrize("failure", ["before", "storage", "after", "none"]) +async def test_stored_rate_change_rechecks_before_adoption( + limiter: Limiter, monkeypatch: pytest.MonkeyPatch, clear: bool, failure: str +) -> None: + old = Limit(7, 3) + limiter._ns_overrides[("work", "memory")] = old + monkeypatch.setattr(limit_changes, "get_limiter", lambda: limiter) + connection = object() + stored = [] + + async def persist(*args, **kwargs): + assert limiter._ns_overrides[("work", "memory")] == old + assert kwargs == {"deadline": 123.0, "ch": connection} + if failure == "storage": + raise ConnectionError("storage unavailable") + stored.append(args) + if failure == "after": + monkeypatch.setattr(settings, "ratelimit_shared", "on") + + put, remove = AsyncMock(side_effect=persist), AsyncMock(side_effect=persist) + monkeypatch.setattr(limit_changes.ro, "put", put) + monkeypatch.setattr(limit_changes.ro, "clear", remove) + if failure == "before": + monkeypatch.setattr(settings, "ratelimit_shared", "on") + arguments = dict(capacity=2, rate=1, clear=clear, deadline=123.0, ch=connection) + if failure != "none": + with pytest.raises(ConnectionError if failure == "storage" else BoundedReadError): + await limit_changes.change_namespace_rate("work", "memory", **arguments) + assert limiter._ns_overrides[("work", "memory")] == old + assert len(stored) == (1 if failure == "after" else 0) + else: + result = await limit_changes.change_namespace_rate("work", "memory", **arguments) + assert result["ok"] is True + if clear: + assert ("work", "memory") not in limiter._ns_overrides + else: + assert limiter._ns_overrides[("work", "memory")] == Limit(2, 1) + assert len(stored) == 1 + assert (remove.call_count, put.call_count) == ( + (0, 0) if failure == "before" else ((1, 0) if clear else (0, 1)) + ) + if stored: + assert stored[0][:2] == (limit_changes.ro.RATELIMIT_NAMESPACE, ("work", "memory")) + if not clear: + assert stored[0][2] == {"capacity": 2, "rate": 1} diff --git a/tests/unit/test_router_cell_administration.py b/tests/unit/test_router_cell_administration.py new file mode 100644 index 0000000000..21db61403f --- /dev/null +++ b/tests/unit/test_router_cell_administration.py @@ -0,0 +1,179 @@ +"""Fresh cell credentials and complete public identities gate administration reads.""" + +from __future__ import annotations + +import hashlib +import json +from contextlib import asynccontextmanager +from typing import Any +from uuid import UUID + +import pytest + +from server.router import cell_admin, directory + + +def _row(**changes: Any) -> dict[str, Any]: + row: dict[str, Any] = { + "cell": "later", + "deployment_id": "deployment-later", + "host": "later.example.test", + "state": "serving", + "mode": "enforced", + "closed": True, + "mutation_generation": UUID("c33e76ce-f023-4b45-92ea-2e9bfa008583"), + "current_digest": "a" * 64, + "next_digest": None, + "administration": json.dumps( + [ + {"purpose": "automation", "name": "automation-later", "thumbprint": "A" * 43}, + {"purpose": "recovery", "name": "recovery-later", "thumbprint": "B" * 43}, + ] + ), + "credential_sha256": hashlib.sha256(b"old").hexdigest(), + "credential_next_sha256": None, + } + row.update(changes) + return row + + +def test_census_includes_closed_later_cell_without_exposing_credentials() -> None: + answer = cell_admin.census([_row()]) + target = answer["cells"][0] + assert target["cell"] == "later" + assert target["origin"] == "https://later.example.test" + assert target["mutation_generation"] == "c33e76ce-f023-4b45-92ea-2e9bfa008583" + assert set(target) == { + "cell", + "deployment_id", + "origin", + "state", + "mode", + "mutation_generation", + "configuration_digests", + "automation", + "recovery", + } + + +@pytest.mark.parametrize( + "metadata", + [ + "[]", + "{}", + "[NaN, NaN]", + "[null, null]", + " " * 1025, + '[{"purpose":"automation","purpose":"recovery"},{}]', + json.dumps([{"purpose": "automation", "name": "one", "thumbprint": "A" * 43}] * 2), + ], +) +def test_incomplete_or_ambiguous_public_metadata_refuses(metadata: str) -> None: + with pytest.raises(cell_admin.InvalidAdministration): + cell_admin.identity(_row(administration=metadata)) + + +@pytest.mark.parametrize( + "field,value", + [ + ("state", "attaching"), + ("mode", "recording"), + ("host", "user@later.example.test"), + ("host", "later.example.test/path"), + ("host", "Later.example.test"), + ("current_digest", None), + ("next_digest", "oversized"), + ("mutation_generation", 1), + ], +) +def test_unsafe_or_incomplete_target_is_never_silently_omitted(field: str, value: Any) -> None: + with pytest.raises(cell_admin.InvalidAdministration): + cell_admin.census([_row(**{field: value})]) + + +def test_distinct_cells_cannot_share_an_origin() -> None: + with pytest.raises(cell_admin.InvalidAdministration): + cell_admin.census([_row(), _row(cell="other", deployment_id="other-deployment")]) + + +@pytest.mark.asyncio +async def test_fresh_same_row_credentials_decide_every_read( + monkeypatch: pytest.MonkeyPatch, +) -> None: + row = _row() + reads: list[str] = [] + + class Connection: + async def fetchrow(self, _sql: str, name: str, **_options: Any) -> Any: + reads.append(name) + return dict(row) if name == row["cell"] else None + + @asynccontextmanager + async def connection() -> Any: + yield Connection() + + monkeypatch.setattr(directory, "_administration_read", connection) + old, new = (hashlib.sha256(value).hexdigest() for value in (b"old", b"new")) + answer = await directory.administration_identity("later", old) + assert answer is not None and answer["automation"]["name"] == "automation-later" + row["credential_sha256"] = new + assert await directory.administration_identity("later", old) is None + assert await directory.administration_identity("later", new) == answer + assert await directory.administration_identity("another", new) is None + row["credential_next_sha256"] = old + assert await directory.administration_identity("later", old) == answer + assert reads == ["later", "later", "later", "another", "later"] + + +@pytest.mark.asyncio +async def test_wrong_credential_cannot_probe_invalid_bot_metadata( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class Connection: + async def fetchrow(self, _sql: str, _name: str, **_options: Any) -> Any: + return _row(administration="not public metadata") + + @asynccontextmanager + async def connection() -> Any: + yield Connection() + + monkeypatch.setattr(directory, "_administration_read", connection) + assert await directory.administration_identity("later", "f" * 64) is None + + +@pytest.mark.parametrize("payload", [None, "界" * 400]) +def test_oversize_projection_is_a_refusal_not_a_missing_target(payload: Any) -> None: + with pytest.raises(cell_admin.InvalidAdministration): + cell_admin.selected_record(payload) + + +@pytest.mark.asyncio +async def test_census_stops_reading_at_first_unsupported_cell( + monkeypatch: pytest.MonkeyPatch, +) -> None: + visited: list[str] = [] + + class Connection: + async def cursor(self, _sql: str, **options: Any) -> Any: + assert options["prefetch"] == 1 + for name, state in [ + ("later", "serving"), + ("other", "attaching"), + ("unread", "serving"), + ]: + visited.append(name) + record = _row(cell=name, deployment_id="deployment-" + name, state=state) + record.pop("closed") + record.pop("credential_sha256") + record.pop("credential_next_sha256") + record["mutation_generation"] = str(record["mutation_generation"]) + yield {"payload": json.dumps(record)} + + @asynccontextmanager + async def connection() -> Any: + yield Connection() + + monkeypatch.setattr(directory, "_administration_read", connection) + with pytest.raises(cell_admin.InvalidAdministration): + await directory.administration_cells() + assert visited == ["later", "other"]